shroud_clearing_action(const config& cfg)
: route()
, view_info(cfg.child_or_empty("unit"))
, original_village_owner(cfg["village_owner"].to_int())
, take_village_timebonus(cfg["village_timebonus"].to_bool())
{
read_locations(cfg, route);
}
move_action(const config & unit_cfg, const config & route_cfg,
int sm, int timebonus, int orig, const map_location::DIRECTION dir) :
undo_action(unit_cfg),
starting_moves(sm),
original_village_owner(orig),
countdown_time_bonus(timebonus),
starting_dir(dir),
goto_hex(unit_cfg["goto_x"].to_int(-999) - 1,
unit_cfg["goto_y"].to_int(-999) - 1)
{
read_locations(route_cfg, route);
}
sourcespec::sourcespec(const config& cfg) :
id_(cfg["id"]),
files_(cfg["sounds"]),
min_delay_(cfg["delay"].to_int(DEFAULT_DELAY)),
chance_(cfg["chance"].to_int(DEFAULT_CHANCE)),
loops_(cfg["loop"]),
range_(cfg["full_range"].to_int(3)),
faderange_(cfg["fade_range"].to_int(14)),
check_fogged_(cfg["check_fogged"].to_bool(true)),
check_shrouded_(cfg["check_shrouded"].to_bool(true)),
locations_()
{
read_locations(cfg, locations_);
}
shroud_clearing_action(const config& cfg)
: route()
, view_info(cfg.child_or_empty("unit"))
{
read_locations(cfg, route);
}
int main(int argc, char **argv)
{
char *input_dir, *output_dir;
if(argc != 3) {
usage(argc, argv);
return EXIT_FAILURE;
} else {
input_dir = argv[1];
output_dir = argv[2];
}
if(strcmp(input_dir, output_dir) == 0) {
fprintf(stderr,"ERROR: Input and output directories are the same..exiting\n");
return EXIT_FAILURE;
}
struct timeval tstart, tend;
gettimeofday(&tstart, NULL);
char locations_filename[MAXLEN], forests_filename[MAXLEN];
int64_t *forests=NULL, *tree_roots=NULL;
my_snprintf(locations_filename, MAXLEN, "%s/locations.dat", input_dir);
my_snprintf(forests_filename, MAXLEN, "%s/forests.list", input_dir);
fprintf(stderr, ANSI_COLOR_MAGENTA"Reading forests...."ANSI_COLOR_RESET"\n");
const int64_t ntrees = read_forests(forests_filename, &forests, &tree_roots);
fprintf(stderr, ANSI_COLOR_GREEN"Reading forests......done"ANSI_COLOR_RESET"\n\n");
/* fprintf(stderr, "Number of trees = %"PRId64"\n\n",ntrees); */
struct locations *locations = my_malloc(sizeof(*locations), ntrees);
int nfiles = 0, BOX_DIVISIONS=0;
fprintf(stderr, ANSI_COLOR_MAGENTA"Reading locations...."ANSI_COLOR_RESET"\n");
const int64_t ntrees_loc = read_locations(locations_filename, ntrees, locations, &nfiles, &BOX_DIVISIONS);
fprintf(stderr, ANSI_COLOR_GREEN"Reading locations......done"ANSI_COLOR_RESET"\n\n");
XASSERT(ntrees == ntrees_loc,
"ntrees=%"PRId64" should be equal to ntrees_loc=%"PRId64"\n",
ntrees, ntrees_loc);
/* the following function will sort locations and forests based on tree root id*/
assign_forest_ids(ntrees, locations, forests, tree_roots);
/* Forests are now contained inside locations -> free the pointers */
free(forests);free(tree_roots);
FILE **tree_outputs = my_malloc(sizeof(FILE *), nfiles);
FILE **tree_inputs = my_malloc(sizeof(FILE *), nfiles);
int *tree_inputs_fd = my_malloc(sizeof(*tree_inputs_fd), nfiles);
int *tree_outputs_fd = my_malloc(sizeof(*tree_outputs_fd), nfiles);
XASSERT(sizeof(off_t) == 8,
"File offset bits must be 64\n"
"Please ensure "ANSI_COLOR_RED"#define _FILE_OFFSET_BITS 64"ANSI_COLOR_RESET" is present\n");
off_t *tree_outputs_fd_offset = my_malloc(sizeof(*tree_outputs_fd_offset), nfiles);
int64_t *tree_counts = my_calloc(sizeof(*tree_counts), nfiles);
int64_t *inp_file_sizes = my_calloc(sizeof(*inp_file_sizes), nfiles);
char buffer[MAXLEN];
for (int i=0; i<BOX_DIVISIONS; i++) {
for (int j=0; j<BOX_DIVISIONS; j++) {
for(int k=0; k<BOX_DIVISIONS; k++) {
my_snprintf(buffer,MAXLEN,"%s/tree_%d_%d_%d.dat", input_dir, i, j, k);
int id = id = i*BOX_DIVISIONS*BOX_DIVISIONS + j*BOX_DIVISIONS + k;
tree_inputs[id] = my_fopen(buffer, "r");
XASSERT(setvbuf(tree_inputs[id], NULL, _IONBF, 0) == 0,
"Could not set unbuffered fgets");
my_fseek(tree_inputs[id],0L, SEEK_END);
inp_file_sizes[id] = ftello(tree_inputs[id]);
rewind(tree_inputs[id]);
tree_inputs_fd[id] = fileno(tree_inputs[id]);
my_snprintf(buffer,MAXLEN,"%s/tree_%d_%d_%d.dat", output_dir, i, j, k);
unlink(buffer);
tree_outputs[id] = my_fopen(buffer, "w");
/* setbuf(tree_outputs[id], _IOFBF); */
tree_outputs_fd[id] = fileno(tree_outputs[id]);
}
}
}
/* the following function will sort locations based on 1) filename 2) offsets */
sort_locations_file_offset(ntrees, locations);
/* holder to check later that bytes have been assigned */
for(int64_t i=0;i<ntrees;i++) {
locations[i].bytes = -1;/* Make sure bytes is a signed type! */
}
/* Create a copy of current locations */
struct locations *new_locations = my_malloc(sizeof(*new_locations), ntrees);
assert(sizeof(*new_locations) == sizeof(*locations) && "locations struct is varying in size! The sky is falling!!");
memcpy(new_locations, locations, sizeof(*locations) * ntrees);
/* figure out the byte size for each tree */
int64_t start = locations[0].offset;
int64_t start_fileid = locations[0].fileid;
/* tree_roots are 64 bit integers -> max digits in decimal = log10(2^64) < 20.
Add 1 char for +-, in case consistent tree changes. and then strlen('#tree ')
and the previous \n. I need to read up to previous newline.
*/
const int64_t guess_max_linesize = 20 + 1 + 6 + 1;
fprintf(stderr, ANSI_COLOR_MAGENTA"Calculating the number of bytes for each tree...."ANSI_COLOR_RESET"\n");
/* setup the progressbar */
int interrupted=0;
init_my_progressbar(ntrees, &interrupted);
for(int64_t i=1;i<=ntrees-1;i++) {
my_progressbar(i, &interrupted);
const int64_t fileid = locations[i].fileid;
/* Are we starting on a new file ?*/
if(start_fileid != fileid) {
/* fill out the bytes for the last tree in the previous file */
const int64_t num_bytes = compute_numbytes_with_off(inp_file_sizes[start_fileid], start);
locations[i-1].bytes = num_bytes;
new_locations[i-1].bytes = num_bytes;
/* now we reset the start fields */
start = locations[i].offset;
start_fileid = locations[i].fileid;
continue;
}
const int64_t current_offset_guess = locations[i].offset - guess_max_linesize;
my_fseek(tree_inputs[fileid], current_offset_guess, SEEK_SET);
while(1) {
const int a = fgetc(tree_inputs[fileid]);
if(a == EOF) {
fprintf(stderr,"Encountered EOF while looking for end of current tree\n");
exit(EXIT_FAILURE);
}
const unsigned char c = (unsigned char) a;
if(c == '\n') {
const int64_t num_bytes = compute_numbytes(tree_inputs[start_fileid], start);
locations[i-1].bytes = num_bytes;
new_locations[i-1].bytes = num_bytes;
/* fprintf(stderr,"%"PRId64"\n",num_bytes); */
start = locations[i].offset;
break;
}
}
}
/* fill out the bytes for the last tree */
{
start = locations[ntrees-1].offset;
const int64_t fileid = locations[ntrees-1].fileid;
my_fseek(tree_inputs[fileid], 0L, SEEK_END);
const int64_t num_bytes = compute_numbytes(tree_inputs[fileid], start);
locations[ntrees-1].bytes = num_bytes;
new_locations[ntrees-1].bytes = num_bytes;
}
finish_myprogressbar(&interrupted);
fprintf(stderr, ANSI_COLOR_GREEN"Calculating the number of bytes for each tree.....done"ANSI_COLOR_RESET"\n\n");
for(int64_t i=ntrees-1;i>=0;i--) {
XASSERT(locations[i].bytes > 0,
"locations[%"PRId64"].bytes = %"PRId64" should be positive\n",
i,locations[i].bytes);
XASSERT(new_locations[i].bytes == locations[i].bytes,
"locations[%"PRId64"].bytes = %"PRId64" should be equal new_locations->bytes = %"PRId64"\n",
i,locations[i].bytes,new_locations[i].bytes);
XASSERT(strncmp(new_locations[i].filename, locations[i].filename, LOCATIONS_FILENAME_SIZE) == 0,
"new_locations[%"PRId64"].filename = %s should equal locations filename = %s\n",
i, new_locations[i].filename, locations[i].filename);
assert(new_locations[i].forestid == locations[i].forestid);
assert(new_locations[i].tree_root == locations[i].tree_root);
assert(new_locations[i].fileid == locations[i].fileid);
assert(new_locations[i].offset == locations[i].offset);
assert(new_locations[i].bytes == locations[i].bytes);
/* fprintf(stderr,"locations[%"PRId64"].bytes = %"PRId64"\n", */
/* i,locations[i].bytes); */
}
/* Check that the preceeding bytes computation is correct */
{
int64_t *total_tree_bytes = my_calloc(sizeof(*total_tree_bytes), nfiles);
for(int64_t i=0;i<ntrees;i++) {
/* add the number of bytes for tree in each file */
total_tree_bytes[locations[i].fileid] += locations[i].bytes;
}
for(int i=0;i<nfiles;i++) {
XASSERT(total_tree_bytes[i] < inp_file_sizes[i],
"Bytes in tree = %"PRId64" must be smaller than file size = %"PRId64"\n",
total_tree_bytes[i], inp_file_sizes[i]);
}
free(total_tree_bytes);
}
/* Now assign all trees in the same forest to the same file
The new fileids goes into new_locations (which is otherwise a copy of locations)
*/
assign_trees_in_forest_to_same_file(ntrees, locations, new_locations, nfiles, BOX_DIVISIONS);
/* Now write out both the old and the new struct locations */
my_snprintf(buffer, MAXLEN, "%s/forests_and_locations_old.list",output_dir);
write_forests_and_locations(buffer, ntrees, locations);
/* write new the forests file */
my_snprintf(buffer,MAXLEN,"%s/forests.list", output_dir);
unlink(buffer);
FILE *fp_forests = my_fopen(buffer,"w");
fprintf(fp_forests, "#TreeRootID ForestID\n");
for(int64_t i=0;i<ntrees;i++) {
fprintf(fp_forests, "%"PRId64" %"PRId64"\n",
locations[i].tree_root, locations[i].forestid);
}
fclose(fp_forests);
/* open the locations file*/
my_snprintf(buffer,MAXLEN,"%s/locations.dat", output_dir);
unlink(buffer);
FILE *fp_locations = my_fopen(buffer,"w");
fprintf(fp_locations, "#TreeRootID FileID Offset Filename\n");
/* copy the headers between the tree_* files */
/* break when the number of trees is encountered -- should be the first one line that doesn't have a '#' character at front */
int64_t *tree_header_offsets = my_malloc(sizeof(*tree_header_offsets), nfiles);
for(int i=0;i<nfiles;i++) {
/* All of the file pointers have been moved around to figure out the bytes
-> reposition them at the beginning of the tree_*.dat file
*/
rewind(tree_inputs[i]);
while(fgets(buffer, MAXLEN, tree_inputs[i]) != NULL) {
if(buffer[0] != '#') {
tree_header_offsets[i] = ftello(tree_outputs[i]);
/* write a place holder for the number of trees in the file.
There are 18 X's in the following line, DO NOT CHANGE.
*/
fprintf(tree_outputs[i], "XXXXXXXXXXXXXXXXXX\n"); //For the number of trees
break;
} else {
fprintf(tree_outputs[i], "%s", buffer);
}
}
tree_outputs_fd_offset[i] = ftello(tree_outputs[i]);
}
/* Figure out the offsets and write out a binary file containing the new locations info */
for(int64_t i=0;i<ntrees;i++) {
const int tree_bytes_line_size = my_snprintf(buffer, MAXLEN, "#tree %"PRId64"\n", locations[i].tree_root);
const int64_t bytes_to_write = locations[i].bytes;
const int64_t out_fileid = new_locations[i].fileid;
XASSERT(out_fileid < nfiles,
"Output fileid = %"PRId64" must be smaller than total number of files = %d\n" ,
out_fileid, nfiles);
/* XASSERT(new_locations[i].bytes == bytes_to_write, */
/* "new locations bytes = %"PRId64"should be identical to old locations bytes = %"PRId64"\n", */
/* new_locations[i].bytes,bytes_to_write); */
new_locations[i].offset = tree_outputs_fd_offset[out_fileid] + tree_bytes_line_size;
tree_outputs_fd_offset[out_fileid] += (bytes_to_write + tree_bytes_line_size);
}
/* Valgrind complains there is use of uninitialized bytes -> so ditching this binary file output for now */
/* /\* Output the binary locations struct so I can skip over recalculating the bytes *\/ */
/* { */
/* my_snprintf(buffer, MAXLEN, "%s/new_locations.binary",output_dir); */
/* FILE *fp = my_fopen(buffer, "w"); */
/* /\* fprintf(stderr,"ntrees = %"PRId64"\n",ntrees); *\/ */
/* my_fwrite(&ntrees, sizeof(int64_t), 1, fp); */
/* const size_t size_of_struct = sizeof(struct locations); */
/* /\* fprintf(stderr,"struct size = %zu\n", size_of_struct); *\/ */
/* my_fwrite(&size_of_struct, sizeof(size_t), 1, fp); */
/* /\* my_fwrite(new_locations, size_of_struct, ntrees, fp); *\/ */
/* fclose(fp); */
/* } */
/* Write out the combined forests and locations file */
my_snprintf(buffer, MAXLEN, "%s/forests_and_locations_new.list",output_dir);
write_forests_and_locations(buffer, ntrees, new_locations);
fprintf(stderr, ANSI_COLOR_MAGENTA"Writing out trees in contiguous order...."ANSI_COLOR_RESET"\n");
interrupted=0;
init_my_progressbar(ntrees, &interrupted);
/* Now copy each one of the trees */
for(int64_t i=0;i<ntrees;i++) {
my_progressbar(i, &interrupted);
const int64_t fileid = locations[i].fileid;
XASSERT(locations[i].tree_root == new_locations[i].tree_root,
"locations->tree_root = %"PRId64" must equal new_locations->tree_root = %"PRId64"\n",
locations[i].tree_root, new_locations[i].tree_root);
XASSERT(locations[i].forestid == new_locations[i].forestid,
"locations->forestid = %"PRId64" must equal new_locations->forestid = %"PRId64"\n",
locations[i].forestid, new_locations[i].forestid);
/* Make sure all output is done using new_locations[i].fileid */
const int64_t out_fileid = new_locations[i].fileid;
FILE *out_fp = tree_outputs[out_fileid];
/* const int tree_bytes_line_size = fprintf(out_fp, "#tree %"PRId64"\n", locations[i].tree_root); */
fprintf(out_fp, "#tree %"PRId64"\n", locations[i].tree_root);
fflush(out_fp);
const int64_t offset = locations[i].offset;
const int64_t bytes_to_write = locations[i].bytes;
if(bytes_to_write == 0) {
fprintf(stderr, "Strange! bytes for tree data = %zu should not be 0\n", bytes_to_write);
continue;
}
#ifdef USE_FGETS //USE_FGETS -> stdio.h family
#warning using fgets (slowest)
FILE *in_fp = tree_inputs[fileid];
my_fseek(in_fp, (long) offset, SEEK_SET);
const long actual_offset = ftello(out_fp);
XASSERT(actual_offset == new_locations[i].offset,
"actual offset = %ld should equal calculated offset = %"PRId64"\n",
actual_offset, new_locations[i].offset);
/* new_locations[i].offset = ftello(out_fp); */
const int64_t bytes_written = copy_bytes_between_two_files(bytes_to_write, in_fp, out_fp);
#else //use pread/write
#warning using pread
int in_fd = tree_inputs_fd[fileid];
int out_fd = tree_outputs_fd[out_fileid];
off_t in_offset = offset;
const int64_t bytes_written = copy_bytes_with_pread(bytes_to_write, in_fd, out_fd, in_offset);
/* I have already figured out the offsets */
/* new_locations[i].offset = tree_outputs_fd_offset[out_fileid] + tree_bytes_line_size; */
#endif//USE_FGETS -> stdio.h family
XASSERT(bytes_written == bytes_to_write,
"bytes_to_write = %zu does not equal bytes_written = %zu\n",
bytes_to_write, bytes_written);
/* Update the number of trees in that file */
tree_counts[out_fileid]++;
/* write the locations info*/
const int ii = out_fileid/(BOX_DIVISIONS*BOX_DIVISIONS);
const int jj = (out_fileid%((int64_t)(BOX_DIVISIONS*BOX_DIVISIONS)))/BOX_DIVISIONS;
const int kk = out_fileid%((int64_t)BOX_DIVISIONS);
fprintf(fp_locations, "%"PRId64" %"PRId64" %"PRId64" tree_%d_%d_%d.dat\n",
new_locations[i].tree_root, out_fileid, new_locations[i].offset, ii, jj, kk);
/* This line is only required if offsets have not been computed earlier */
/* tree_outputs_fd_offset[out_fileid] += (bytes_written + tree_bytes_line_size) ; */
}
/* fill in the number of trees written per file. the number in the format
*MUST EXACTLY* match the number of XXX's in the previous place-holder.
*/
for(int i=0;i<nfiles;i++) {
FILE *out_fp = tree_outputs[i];
fseek(out_fp, tree_header_offsets[i], SEEK_SET);
fprintf(out_fp, "%-18"PRId64"\n", tree_counts[i]);
}
finish_myprogressbar(&interrupted);
fprintf(stderr, ANSI_COLOR_GREEN "Writing out trees in contiguous order.....done"ANSI_COLOR_RESET"\n\n");
/* close open file pointers + free memory for file pointers */
fclose(fp_locations);
for(int i=0;i<nfiles;i++) {
fclose(tree_inputs[i]);
fclose(tree_outputs[i]);
}
free(tree_inputs);free(tree_outputs);
free(tree_inputs_fd);free(tree_outputs_fd);
/* free other heap allocations */
free(tree_header_offsets);
free(tree_outputs_fd_offset);
free(tree_counts);
free(inp_file_sizes);
free(locations);
free(new_locations);
gettimeofday(&tend, NULL);
fprintf(stderr,"Wrote out %"PRId64" trees in contiguous order. Time taken = %0.2g seconds\n",
ntrees, ADD_DIFF_TIME(tstart, tend));
return EXIT_SUCCESS;
}
