Example #1
0
int main(int argc, char **argv) 
{
    if (argc != 6) {
        printf("Usage: %s <list.in> <depth> <branching_factor> "
               "<restarts> <tree.out>\n", argv[0]);
        return 1;
    }

    const char *list_in = argv[1];
    int depth = atoi(argv[2]);
    int bf = atoi(argv[3]);
    int restarts = atoi(argv[4]);
    const char *tree_out = argv[5];
    std::ofstream log("debug_VocabLearn.txt");
    log << "Building tree with depth: " << depth << ", branching factor: " << bf << ", and restarts: " << restarts << std::endl;
    printf("Building tree with depth: %d, branching factor: %d, "
           "and restarts: %d\n", depth, bf, restarts);

    FILE *f = fopen(list_in, "r");
    
    std::vector<std::string> key_files;
    char buf[256];
    while (fgets(buf, 256, f)) {
        /* Remove trailing newline */
        if (buf[strlen(buf) - 1] == '\n')
            buf[strlen(buf) - 1] = 0;

        key_files.push_back(std::string(buf));
    }

    fclose(f);

    int num_files = (int) key_files.size();
    unsigned long total_keys = 0;
    for (int i = 0; i < num_files; i++) {
        int num_keys = GetNumberOfKeys(key_files[i].c_str());
        total_keys += num_keys;
    }

    log << "Total number of keys: " << total_keys << std::endl;
    printf("Total number of keys: %lu\n", total_keys);
    fflush(stdout);

    int dim = 128;
    unsigned long long len = (unsigned long long) total_keys * dim;

    int num_arrays = 
        len / MAX_ARRAY_SIZE + ((len % MAX_ARRAY_SIZE) == 0 ? 0 : 1);

    unsigned char **vs = new unsigned char *[num_arrays];

    log << "Allocating " << len << " bytes in total, in " << num_arrays << " arrays" << std::endl;
    printf("Allocating %llu bytes in total, in %d arrays\n", len, num_arrays);

    unsigned long long total = 0;
    for (int i = 0; i < num_arrays; i++) {
        unsigned long long remainder = len - total;
        unsigned long len_curr = MIN(remainder, MAX_ARRAY_SIZE);

        log << "Allocating array of size " << len_curr << std::endl;
        printf("Allocating array of size %lu\n", len_curr);
        fflush(stdout);
        vs[i] = new unsigned char[len_curr];

        remainder -= len_curr;
    }

    /* Create the array of pointers */
    log << "Allocating pointer array of size " << sizeof(unsigned char) * total_keys << std::endl;
    printf("Allocating pointer array of size %lu\n", 4 * total_keys);
    fflush(stdout);

    unsigned char **vp = new unsigned char *[total_keys];
    
    unsigned long off = 0;
    unsigned long curr_key = 0;
    int curr_array = 0;
    for (int i = 0; i < num_files; i++) {
    	log << "Reading key file " << key_files[i] << std::endl;
        //printf("  Reading keyfile %s\n", key_files[i].c_str());
        fflush(stdout);

        unsigned char *keys;
        int num_keys = 0;

        keypt_t *info = NULL;
        num_keys = ReadKeyFileCXX(key_files[i].c_str(), &keys);
        log << "There are " << num_keys << " keys in file " << key_files[i] << std::endl;
//        num_keys = ReadKeyFile(key_files[i].c_str(), &keys);

#ifdef DEBUG
        std::ofstream debug_log("debug_log_2.txt");
        debug_log << num_keys << " " << dim << std::endl;
        for (int i=0; i<num_keys; i++) {
        	for (int k=0; k<dim; k++) {
        		debug_log << (int) keys[dim*i + k] << " ";
        	}
        	debug_log << std::endl;
        }
        debug_log.close();
#endif

        if (num_keys > 0) {
            for (int j = 0; j < num_keys; j++) {
#ifdef DEBUG
            	log << "Copying key " << j << std::endl;
#endif
                for (int k = 0; k < dim; k++) {
#ifdef DEBUG
                	log << "\tReading ";
#endif
                	unsigned char val = keys[j * dim + k];
#ifdef DEBUG
                	log << (int) val << ". Writing ";
#endif
                    vs[curr_array][off + k] = val;
#ifdef DEBUG
                    log << (int) vs[curr_array][off + k] << std::endl;
#endif
                }
#ifdef DEBUG
                log << "vp[curr_key] = vs[curr_array] + off;" << std::endl;
#endif
                vp[curr_key] = vs[curr_array] + off;
                curr_key++;
                off += dim;
                if (off == MAX_ARRAY_SIZE) {
                    off = 0;
                    curr_array++;
                }
            }

            delete [] keys;

            if (info != NULL)
                delete [] info;
        }
    }

    log << "Building tree..." << std::endl;
    VocabTree tree;
    tree.Build(total_keys, dim, depth, bf, restarts, vp);
    tree.Write(tree_out);

    log.close();

    return 0;
}
Example #2
0
int main(int argc, char **argv) 
{
    if (argc != 6) {
        printf("Usage: %s <list.in> <depth> <branching_factor> "
               "<restarts> <tree.out>\n", argv[0]);
        return 1;
    }

    const char *list_in = argv[1];
    int depth = atoi(argv[2]);
    int bf = atoi(argv[3]);
    int restarts = atoi(argv[4]);
    const char *tree_out = argv[5];

    printf("Building tree with depth: %d, branching factor: %d, "
           "and restarts: %d\n", depth, bf, restarts);

    FILE *f = fopen(list_in, "r");
    if (f == NULL) {
      printf("Could not open file: %s\n", list_in);
      return 1;
    }
    
    std::vector<std::string> key_files;
    char buf[256];
    while (fgets(buf, 256, f)) {
        /* Remove trailing newline */
        if (buf[strlen(buf) - 1] == '\n')
            buf[strlen(buf) - 1] = 0;

        key_files.push_back(std::string(buf));
    }

    fclose(f);
    
    int num_files = (int) key_files.size();
    unsigned long total_keys = 0;
    for (int i = 0; i < num_files; i++) {
        int num_keys = GetNumberOfKeys(key_files[i].c_str());
        total_keys += num_keys;
    }

    printf("Total number of keys: %lu\n", total_keys);

    // Reduce the branching factor if need be if there are not
    // enough keys, to avoid problems later.
    if (bf >= (int)total_keys){
      bf = total_keys - 1;
      printf("Reducing the branching factor to: %d\n", bf);
    }
    
    fflush(stdout);

    int dim = 128;
    unsigned long long len = (unsigned long long) total_keys * dim;

    int num_arrays = 
        len / MAX_ARRAY_SIZE + ((len % MAX_ARRAY_SIZE) == 0 ? 0 : 1);

    unsigned char **vs = new unsigned char *[num_arrays];

    printf("Allocating %llu bytes in total, in %d arrays\n", len, num_arrays);

    unsigned long long total = 0;
    for (int i = 0; i < num_arrays; i++) {
        unsigned long long remainder = len - total;
        unsigned long len_curr = MIN(remainder, MAX_ARRAY_SIZE);

        printf("Allocating array of size %lu\n", len_curr);
        fflush(stdout);
        vs[i] = new unsigned char[len_curr];

        remainder -= len_curr;
    }

    /* Create the array of pointers */
    printf("Allocating pointer array of size %lu\n", 4 * total_keys);
    fflush(stdout);

    unsigned char **vp = new unsigned char *[total_keys];
    
    unsigned long off = 0;
    unsigned long curr_key = 0;
    int curr_array = 0;
    for (int i = 0; i < num_files; i++) {
        printf("  Reading keyfile %s\n", key_files[i].c_str());
        fflush(stdout);

        short int *keys;
        int num_keys = 0;

        keypt_t *info = NULL;
        num_keys = ReadKeyFile(key_files[i].c_str(), &keys);

        if (num_keys > 0) {
            for (int j = 0; j < num_keys; j++) {
                for (int k = 0; k < dim; k++) {
                    vs[curr_array][off + k] = keys[j * dim + k];
                }
                
                vp[curr_key] = vs[curr_array] + off;
                curr_key++;
                off += dim;
                if (off == MAX_ARRAY_SIZE) {
                    off = 0;
                    curr_array++;
                }
            }

            delete [] keys;

            if (info != NULL)
                delete [] info;
        }
    }

    VocabTree tree;
    tree.Build(total_keys, dim, depth, bf, restarts, vp);
    tree.Write(tree_out);

    return 0;
}