shroud_clearing_action(const config& cfg) : route() , view_info(cfg.child_or_empty("unit")) , original_village_owner(cfg["village_owner"].to_int()) , take_village_timebonus(cfg["village_timebonus"].to_bool()) { read_locations(cfg, route); }
move_action(const config & unit_cfg, const config & route_cfg, int sm, int timebonus, int orig, const map_location::DIRECTION dir) : undo_action(unit_cfg), starting_moves(sm), original_village_owner(orig), countdown_time_bonus(timebonus), starting_dir(dir), goto_hex(unit_cfg["goto_x"].to_int(-999) - 1, unit_cfg["goto_y"].to_int(-999) - 1) { read_locations(route_cfg, route); }
sourcespec::sourcespec(const config& cfg) : id_(cfg["id"]), files_(cfg["sounds"]), min_delay_(cfg["delay"].to_int(DEFAULT_DELAY)), chance_(cfg["chance"].to_int(DEFAULT_CHANCE)), loops_(cfg["loop"]), range_(cfg["full_range"].to_int(3)), faderange_(cfg["fade_range"].to_int(14)), check_fogged_(cfg["check_fogged"].to_bool(true)), check_shrouded_(cfg["check_shrouded"].to_bool(true)), locations_() { read_locations(cfg, locations_); }
shroud_clearing_action(const config& cfg) : route() , view_info(cfg.child_or_empty("unit")) { read_locations(cfg, route); }
int main(int argc, char **argv) { char *input_dir, *output_dir; if(argc != 3) { usage(argc, argv); return EXIT_FAILURE; } else { input_dir = argv[1]; output_dir = argv[2]; } if(strcmp(input_dir, output_dir) == 0) { fprintf(stderr,"ERROR: Input and output directories are the same..exiting\n"); return EXIT_FAILURE; } struct timeval tstart, tend; gettimeofday(&tstart, NULL); char locations_filename[MAXLEN], forests_filename[MAXLEN]; int64_t *forests=NULL, *tree_roots=NULL; my_snprintf(locations_filename, MAXLEN, "%s/locations.dat", input_dir); my_snprintf(forests_filename, MAXLEN, "%s/forests.list", input_dir); fprintf(stderr, ANSI_COLOR_MAGENTA"Reading forests...."ANSI_COLOR_RESET"\n"); const int64_t ntrees = read_forests(forests_filename, &forests, &tree_roots); fprintf(stderr, ANSI_COLOR_GREEN"Reading forests......done"ANSI_COLOR_RESET"\n\n"); /* fprintf(stderr, "Number of trees = %"PRId64"\n\n",ntrees); */ struct locations *locations = my_malloc(sizeof(*locations), ntrees); int nfiles = 0, BOX_DIVISIONS=0; fprintf(stderr, ANSI_COLOR_MAGENTA"Reading locations...."ANSI_COLOR_RESET"\n"); const int64_t ntrees_loc = read_locations(locations_filename, ntrees, locations, &nfiles, &BOX_DIVISIONS); fprintf(stderr, ANSI_COLOR_GREEN"Reading locations......done"ANSI_COLOR_RESET"\n\n"); XASSERT(ntrees == ntrees_loc, "ntrees=%"PRId64" should be equal to ntrees_loc=%"PRId64"\n", ntrees, ntrees_loc); /* the following function will sort locations and forests based on tree root id*/ assign_forest_ids(ntrees, locations, forests, tree_roots); /* Forests are now contained inside locations -> free the pointers */ free(forests);free(tree_roots); FILE **tree_outputs = my_malloc(sizeof(FILE *), nfiles); FILE **tree_inputs = my_malloc(sizeof(FILE *), nfiles); int *tree_inputs_fd = my_malloc(sizeof(*tree_inputs_fd), nfiles); int *tree_outputs_fd = my_malloc(sizeof(*tree_outputs_fd), nfiles); XASSERT(sizeof(off_t) == 8, "File offset bits must be 64\n" "Please ensure "ANSI_COLOR_RED"#define _FILE_OFFSET_BITS 64"ANSI_COLOR_RESET" is present\n"); off_t *tree_outputs_fd_offset = my_malloc(sizeof(*tree_outputs_fd_offset), nfiles); int64_t *tree_counts = my_calloc(sizeof(*tree_counts), nfiles); int64_t *inp_file_sizes = my_calloc(sizeof(*inp_file_sizes), nfiles); char buffer[MAXLEN]; for (int i=0; i<BOX_DIVISIONS; i++) { for (int j=0; j<BOX_DIVISIONS; j++) { for(int k=0; k<BOX_DIVISIONS; k++) { my_snprintf(buffer,MAXLEN,"%s/tree_%d_%d_%d.dat", input_dir, i, j, k); int id = id = i*BOX_DIVISIONS*BOX_DIVISIONS + j*BOX_DIVISIONS + k; tree_inputs[id] = my_fopen(buffer, "r"); XASSERT(setvbuf(tree_inputs[id], NULL, _IONBF, 0) == 0, "Could not set unbuffered fgets"); my_fseek(tree_inputs[id],0L, SEEK_END); inp_file_sizes[id] = ftello(tree_inputs[id]); rewind(tree_inputs[id]); tree_inputs_fd[id] = fileno(tree_inputs[id]); my_snprintf(buffer,MAXLEN,"%s/tree_%d_%d_%d.dat", output_dir, i, j, k); unlink(buffer); tree_outputs[id] = my_fopen(buffer, "w"); /* setbuf(tree_outputs[id], _IOFBF); */ tree_outputs_fd[id] = fileno(tree_outputs[id]); } } } /* the following function will sort locations based on 1) filename 2) offsets */ sort_locations_file_offset(ntrees, locations); /* holder to check later that bytes have been assigned */ for(int64_t i=0;i<ntrees;i++) { locations[i].bytes = -1;/* Make sure bytes is a signed type! */ } /* Create a copy of current locations */ struct locations *new_locations = my_malloc(sizeof(*new_locations), ntrees); assert(sizeof(*new_locations) == sizeof(*locations) && "locations struct is varying in size! The sky is falling!!"); memcpy(new_locations, locations, sizeof(*locations) * ntrees); /* figure out the byte size for each tree */ int64_t start = locations[0].offset; int64_t start_fileid = locations[0].fileid; /* tree_roots are 64 bit integers -> max digits in decimal = log10(2^64) < 20. Add 1 char for +-, in case consistent tree changes. and then strlen('#tree ') and the previous \n. I need to read up to previous newline. */ const int64_t guess_max_linesize = 20 + 1 + 6 + 1; fprintf(stderr, ANSI_COLOR_MAGENTA"Calculating the number of bytes for each tree...."ANSI_COLOR_RESET"\n"); /* setup the progressbar */ int interrupted=0; init_my_progressbar(ntrees, &interrupted); for(int64_t i=1;i<=ntrees-1;i++) { my_progressbar(i, &interrupted); const int64_t fileid = locations[i].fileid; /* Are we starting on a new file ?*/ if(start_fileid != fileid) { /* fill out the bytes for the last tree in the previous file */ const int64_t num_bytes = compute_numbytes_with_off(inp_file_sizes[start_fileid], start); locations[i-1].bytes = num_bytes; new_locations[i-1].bytes = num_bytes; /* now we reset the start fields */ start = locations[i].offset; start_fileid = locations[i].fileid; continue; } const int64_t current_offset_guess = locations[i].offset - guess_max_linesize; my_fseek(tree_inputs[fileid], current_offset_guess, SEEK_SET); while(1) { const int a = fgetc(tree_inputs[fileid]); if(a == EOF) { fprintf(stderr,"Encountered EOF while looking for end of current tree\n"); exit(EXIT_FAILURE); } const unsigned char c = (unsigned char) a; if(c == '\n') { const int64_t num_bytes = compute_numbytes(tree_inputs[start_fileid], start); locations[i-1].bytes = num_bytes; new_locations[i-1].bytes = num_bytes; /* fprintf(stderr,"%"PRId64"\n",num_bytes); */ start = locations[i].offset; break; } } } /* fill out the bytes for the last tree */ { start = locations[ntrees-1].offset; const int64_t fileid = locations[ntrees-1].fileid; my_fseek(tree_inputs[fileid], 0L, SEEK_END); const int64_t num_bytes = compute_numbytes(tree_inputs[fileid], start); locations[ntrees-1].bytes = num_bytes; new_locations[ntrees-1].bytes = num_bytes; } finish_myprogressbar(&interrupted); fprintf(stderr, ANSI_COLOR_GREEN"Calculating the number of bytes for each tree.....done"ANSI_COLOR_RESET"\n\n"); for(int64_t i=ntrees-1;i>=0;i--) { XASSERT(locations[i].bytes > 0, "locations[%"PRId64"].bytes = %"PRId64" should be positive\n", i,locations[i].bytes); XASSERT(new_locations[i].bytes == locations[i].bytes, "locations[%"PRId64"].bytes = %"PRId64" should be equal new_locations->bytes = %"PRId64"\n", i,locations[i].bytes,new_locations[i].bytes); XASSERT(strncmp(new_locations[i].filename, locations[i].filename, LOCATIONS_FILENAME_SIZE) == 0, "new_locations[%"PRId64"].filename = %s should equal locations filename = %s\n", i, new_locations[i].filename, locations[i].filename); assert(new_locations[i].forestid == locations[i].forestid); assert(new_locations[i].tree_root == locations[i].tree_root); assert(new_locations[i].fileid == locations[i].fileid); assert(new_locations[i].offset == locations[i].offset); assert(new_locations[i].bytes == locations[i].bytes); /* fprintf(stderr,"locations[%"PRId64"].bytes = %"PRId64"\n", */ /* i,locations[i].bytes); */ } /* Check that the preceeding bytes computation is correct */ { int64_t *total_tree_bytes = my_calloc(sizeof(*total_tree_bytes), nfiles); for(int64_t i=0;i<ntrees;i++) { /* add the number of bytes for tree in each file */ total_tree_bytes[locations[i].fileid] += locations[i].bytes; } for(int i=0;i<nfiles;i++) { XASSERT(total_tree_bytes[i] < inp_file_sizes[i], "Bytes in tree = %"PRId64" must be smaller than file size = %"PRId64"\n", total_tree_bytes[i], inp_file_sizes[i]); } free(total_tree_bytes); } /* Now assign all trees in the same forest to the same file The new fileids goes into new_locations (which is otherwise a copy of locations) */ assign_trees_in_forest_to_same_file(ntrees, locations, new_locations, nfiles, BOX_DIVISIONS); /* Now write out both the old and the new struct locations */ my_snprintf(buffer, MAXLEN, "%s/forests_and_locations_old.list",output_dir); write_forests_and_locations(buffer, ntrees, locations); /* write new the forests file */ my_snprintf(buffer,MAXLEN,"%s/forests.list", output_dir); unlink(buffer); FILE *fp_forests = my_fopen(buffer,"w"); fprintf(fp_forests, "#TreeRootID ForestID\n"); for(int64_t i=0;i<ntrees;i++) { fprintf(fp_forests, "%"PRId64" %"PRId64"\n", locations[i].tree_root, locations[i].forestid); } fclose(fp_forests); /* open the locations file*/ my_snprintf(buffer,MAXLEN,"%s/locations.dat", output_dir); unlink(buffer); FILE *fp_locations = my_fopen(buffer,"w"); fprintf(fp_locations, "#TreeRootID FileID Offset Filename\n"); /* copy the headers between the tree_* files */ /* break when the number of trees is encountered -- should be the first one line that doesn't have a '#' character at front */ int64_t *tree_header_offsets = my_malloc(sizeof(*tree_header_offsets), nfiles); for(int i=0;i<nfiles;i++) { /* All of the file pointers have been moved around to figure out the bytes -> reposition them at the beginning of the tree_*.dat file */ rewind(tree_inputs[i]); while(fgets(buffer, MAXLEN, tree_inputs[i]) != NULL) { if(buffer[0] != '#') { tree_header_offsets[i] = ftello(tree_outputs[i]); /* write a place holder for the number of trees in the file. There are 18 X's in the following line, DO NOT CHANGE. */ fprintf(tree_outputs[i], "XXXXXXXXXXXXXXXXXX\n"); //For the number of trees break; } else { fprintf(tree_outputs[i], "%s", buffer); } } tree_outputs_fd_offset[i] = ftello(tree_outputs[i]); } /* Figure out the offsets and write out a binary file containing the new locations info */ for(int64_t i=0;i<ntrees;i++) { const int tree_bytes_line_size = my_snprintf(buffer, MAXLEN, "#tree %"PRId64"\n", locations[i].tree_root); const int64_t bytes_to_write = locations[i].bytes; const int64_t out_fileid = new_locations[i].fileid; XASSERT(out_fileid < nfiles, "Output fileid = %"PRId64" must be smaller than total number of files = %d\n" , out_fileid, nfiles); /* XASSERT(new_locations[i].bytes == bytes_to_write, */ /* "new locations bytes = %"PRId64"should be identical to old locations bytes = %"PRId64"\n", */ /* new_locations[i].bytes,bytes_to_write); */ new_locations[i].offset = tree_outputs_fd_offset[out_fileid] + tree_bytes_line_size; tree_outputs_fd_offset[out_fileid] += (bytes_to_write + tree_bytes_line_size); } /* Valgrind complains there is use of uninitialized bytes -> so ditching this binary file output for now */ /* /\* Output the binary locations struct so I can skip over recalculating the bytes *\/ */ /* { */ /* my_snprintf(buffer, MAXLEN, "%s/new_locations.binary",output_dir); */ /* FILE *fp = my_fopen(buffer, "w"); */ /* /\* fprintf(stderr,"ntrees = %"PRId64"\n",ntrees); *\/ */ /* my_fwrite(&ntrees, sizeof(int64_t), 1, fp); */ /* const size_t size_of_struct = sizeof(struct locations); */ /* /\* fprintf(stderr,"struct size = %zu\n", size_of_struct); *\/ */ /* my_fwrite(&size_of_struct, sizeof(size_t), 1, fp); */ /* /\* my_fwrite(new_locations, size_of_struct, ntrees, fp); *\/ */ /* fclose(fp); */ /* } */ /* Write out the combined forests and locations file */ my_snprintf(buffer, MAXLEN, "%s/forests_and_locations_new.list",output_dir); write_forests_and_locations(buffer, ntrees, new_locations); fprintf(stderr, ANSI_COLOR_MAGENTA"Writing out trees in contiguous order...."ANSI_COLOR_RESET"\n"); interrupted=0; init_my_progressbar(ntrees, &interrupted); /* Now copy each one of the trees */ for(int64_t i=0;i<ntrees;i++) { my_progressbar(i, &interrupted); const int64_t fileid = locations[i].fileid; XASSERT(locations[i].tree_root == new_locations[i].tree_root, "locations->tree_root = %"PRId64" must equal new_locations->tree_root = %"PRId64"\n", locations[i].tree_root, new_locations[i].tree_root); XASSERT(locations[i].forestid == new_locations[i].forestid, "locations->forestid = %"PRId64" must equal new_locations->forestid = %"PRId64"\n", locations[i].forestid, new_locations[i].forestid); /* Make sure all output is done using new_locations[i].fileid */ const int64_t out_fileid = new_locations[i].fileid; FILE *out_fp = tree_outputs[out_fileid]; /* const int tree_bytes_line_size = fprintf(out_fp, "#tree %"PRId64"\n", locations[i].tree_root); */ fprintf(out_fp, "#tree %"PRId64"\n", locations[i].tree_root); fflush(out_fp); const int64_t offset = locations[i].offset; const int64_t bytes_to_write = locations[i].bytes; if(bytes_to_write == 0) { fprintf(stderr, "Strange! bytes for tree data = %zu should not be 0\n", bytes_to_write); continue; } #ifdef USE_FGETS //USE_FGETS -> stdio.h family #warning using fgets (slowest) FILE *in_fp = tree_inputs[fileid]; my_fseek(in_fp, (long) offset, SEEK_SET); const long actual_offset = ftello(out_fp); XASSERT(actual_offset == new_locations[i].offset, "actual offset = %ld should equal calculated offset = %"PRId64"\n", actual_offset, new_locations[i].offset); /* new_locations[i].offset = ftello(out_fp); */ const int64_t bytes_written = copy_bytes_between_two_files(bytes_to_write, in_fp, out_fp); #else //use pread/write #warning using pread int in_fd = tree_inputs_fd[fileid]; int out_fd = tree_outputs_fd[out_fileid]; off_t in_offset = offset; const int64_t bytes_written = copy_bytes_with_pread(bytes_to_write, in_fd, out_fd, in_offset); /* I have already figured out the offsets */ /* new_locations[i].offset = tree_outputs_fd_offset[out_fileid] + tree_bytes_line_size; */ #endif//USE_FGETS -> stdio.h family XASSERT(bytes_written == bytes_to_write, "bytes_to_write = %zu does not equal bytes_written = %zu\n", bytes_to_write, bytes_written); /* Update the number of trees in that file */ tree_counts[out_fileid]++; /* write the locations info*/ const int ii = out_fileid/(BOX_DIVISIONS*BOX_DIVISIONS); const int jj = (out_fileid%((int64_t)(BOX_DIVISIONS*BOX_DIVISIONS)))/BOX_DIVISIONS; const int kk = out_fileid%((int64_t)BOX_DIVISIONS); fprintf(fp_locations, "%"PRId64" %"PRId64" %"PRId64" tree_%d_%d_%d.dat\n", new_locations[i].tree_root, out_fileid, new_locations[i].offset, ii, jj, kk); /* This line is only required if offsets have not been computed earlier */ /* tree_outputs_fd_offset[out_fileid] += (bytes_written + tree_bytes_line_size) ; */ } /* fill in the number of trees written per file. the number in the format *MUST EXACTLY* match the number of XXX's in the previous place-holder. */ for(int i=0;i<nfiles;i++) { FILE *out_fp = tree_outputs[i]; fseek(out_fp, tree_header_offsets[i], SEEK_SET); fprintf(out_fp, "%-18"PRId64"\n", tree_counts[i]); } finish_myprogressbar(&interrupted); fprintf(stderr, ANSI_COLOR_GREEN "Writing out trees in contiguous order.....done"ANSI_COLOR_RESET"\n\n"); /* close open file pointers + free memory for file pointers */ fclose(fp_locations); for(int i=0;i<nfiles;i++) { fclose(tree_inputs[i]); fclose(tree_outputs[i]); } free(tree_inputs);free(tree_outputs); free(tree_inputs_fd);free(tree_outputs_fd); /* free other heap allocations */ free(tree_header_offsets); free(tree_outputs_fd_offset); free(tree_counts); free(inp_file_sizes); free(locations); free(new_locations); gettimeofday(&tend, NULL); fprintf(stderr,"Wrote out %"PRId64" trees in contiguous order. Time taken = %0.2g seconds\n", ntrees, ADD_DIFF_TIME(tstart, tend)); return EXIT_SUCCESS; }