int merge_index_files(const char* quadfn, const char* ckdtfn, const char* skdtfn,
const char* indexfn) {
quadfile* quad = NULL;
codetree* code = NULL;
startree_t* star = NULL;
int rtn;
if (merge_index_open_files(quadfn, ckdtfn, skdtfn, &quad, &code, &star)) {
rtn = -1;
goto cleanup;
}
logmsg("Writing index to %s ...\n", indexfn);
rtn = merge_index(quad, code, star, indexfn);
cleanup:
if (code)
codetree_close(code);
if (star)
startree_close(star);
if (quad)
quadfile_close(quad);
return rtn;
}
void multiindex_unload_starkd(multiindex_t* mi) {
int i;
for (i=0; i<multiindex_n(mi); i++) {
index_t* ind = multiindex_get(mi, i);
ind->starkd = NULL;
}
if (mi->starkd) {
startree_close(mi->starkd);
mi->starkd = NULL;
}
}
int allquads_close(allquads_t* aq) {
startree_close(aq->starkd);
// fix output file headers.
if (quadfile_fix_header(aq->quads) ||
quadfile_close(aq->quads)) {
ERROR("Couldn't write quad output file");
return -1;
}
if (codefile_fix_header(aq->codes) ||
codefile_close(aq->codes)) {
ERROR("Couldn't write code output file");
return -1;
}
return 0;
}
void multiindex_close(multiindex_t* mi) {
if (!mi)
return;
if (mi->starkd) {
startree_close(mi->starkd);
mi->starkd = NULL;
}
if (mi->inds) {
int i;
for (i=0; i<pl_size(mi->inds); i++) {
index_t* ind = pl_get(mi->inds, i);
ind->starkd = NULL;
index_free(ind);
}
pl_free(mi->inds);
mi->inds = NULL;
}
if (mi->fits) {
anqfits_close(mi->fits);
mi->fits = NULL;
}
}
int hpquads_files(const char* skdtfn,
const char* codefn,
const char* quadfn,
int Nside,
double scale_min_arcmin,
double scale_max_arcmin,
int dimquads,
int passes,
int Nreuses,
int Nloosen,
int id,
anbool scanoccupied,
void* sort_data,
int (*sort_func)(const void*, const void*),
int sort_size,
char** args, int argc) {
quadfile_t* quads;
codefile_t* codes;
startree_t* starkd;
int rtn;
logmsg("Reading star kdtree %s ...\n", skdtfn);
starkd = startree_open(skdtfn);
if (!starkd) {
ERROR("Failed to open star kdtree %s\n", skdtfn);
return -1;
}
logmsg("Will write to quad file %s and code file %s\n", quadfn, codefn);
quads = quadfile_open_for_writing(quadfn);
if (!quads) {
ERROR("Couldn't open file %s to write quads.\n", quadfn);
return -1;
}
codes = codefile_open_for_writing(codefn);
if (!codes) {
ERROR("Couldn't open file %s to write codes.\n", codefn);
return -1;
}
rtn = hpquads(starkd, codes, quads, Nside,
scale_min_arcmin, scale_max_arcmin,
dimquads, passes, Nreuses, Nloosen, id,
scanoccupied,
sort_data, sort_func, sort_size,
args, argc);
if (rtn)
return rtn;
if (quadfile_close(quads)) {
ERROR("Couldn't write quad output file");
return -1;
}
if (codefile_close(codes)) {
ERROR("Couldn't write code output file");
return -1;
}
startree_close(starkd);
return rtn;
}
int main(int argc, char** argv) {
int argchar;
allquads_t* aq;
int loglvl = LOG_MSG;
int i, N;
char* catfn = NULL;
startree_t* starkd;
fitstable_t* cat;
char* racol = NULL;
char* deccol = NULL;
int datatype = KDT_DATA_DOUBLE;
int treetype = KDT_TREE_DOUBLE;
int buildopts = 0;
int Nleaf = 0;
char* indexfn = NULL;
//index_params_t* p;
aq = allquads_init();
aq->skdtfn = "allquads.skdt";
aq->codefn = "allquads.code";
aq->quadfn = "allquads.quad";
while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
switch (argchar) {
case 'v':
loglvl++;
break;
case 'd':
aq->dimquads = atoi(optarg);
break;
case 'I':
aq->id = atoi(optarg);
break;
case 'h':
print_help(argv[0]);
exit(0);
case 'i':
catfn = optarg;
break;
case 'o':
indexfn = optarg;
break;
case 'u':
aq->quad_d2_upper = arcmin2distsq(atof(optarg));
aq->use_d2_upper = TRUE;
break;
case 'l':
aq->quad_d2_lower = arcmin2distsq(atof(optarg));
aq->use_d2_lower = TRUE;
break;
default:
return -1;
}
log_init(loglvl);
if (!catfn || !indexfn) {
printf("Specify in & out filenames, bonehead!\n");
print_help(argv[0]);
exit( -1);
}
if (optind != argc) {
print_help(argv[0]);
printf("\nExtra command-line args were given: ");
for (i=optind; i<argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
exit(-1);
}
if (!aq->id)
logmsg("Warning: you should set the unique-id for this index (with -I).\n");
if (aq->dimquads > DQMAX) {
ERROR("Quad dimension %i exceeds compiled-in max %i.\n", aq->dimquads, DQMAX);
exit(-1);
}
aq->dimcodes = dimquad2dimcode(aq->dimquads);
// Read reference catalog, write star kd-tree
logmsg("Building star kdtree: reading %s, writing to %s\n", catfn, aq->skdtfn);
logverb("Reading star catalogue...");
cat = fitstable_open(catfn);
if (!cat) {
ERROR("Couldn't read catalog");
exit(-1);
}
logmsg("Got %i stars\n", fitstable_nrows(cat));
starkd = startree_build(cat, racol, deccol, datatype, treetype,
buildopts, Nleaf, argv, argc);
if (!starkd) {
ERROR("Failed to create star kdtree");
exit(-1);
}
logmsg("Star kd-tree contains %i data points in dimension %i\n",
startree_N(starkd), startree_D(starkd));
N = startree_N(starkd);
for (i=0; i<N; i++) {
double ra,dec;
int ok;
ok = startree_get_radec(starkd, i, &ra, &dec);
logmsg(" data %i: ok %i, RA,Dec %g, %g\n", i, ok, ra, dec);
}
if (startree_write_to_file(starkd, aq->skdtfn)) {
ERROR("Failed to write star kdtree");
exit(-1);
}
startree_close(starkd);
fitstable_close(cat);
logmsg("Wrote star kdtree to %s\n", aq->skdtfn);
logmsg("Running allquads...\n");
if (allquads_open_outputs(aq)) {
exit(-1);
}
if (allquads_create_quads(aq)) {
exit(-1);
}
if (allquads_close(aq)) {
exit(-1);
}
logmsg("allquads: wrote %s, %s\n", aq->quadfn, aq->codefn);
// build-index:
//build_index_defaults(&p);
// codetree
/*
if (step_codetree(p, codes, &codekd,
codefn, &ckdtfn, tempfiles))
return -1;
*/
char* ckdtfn=NULL;
char* tempdir = NULL;
ckdtfn = create_temp_file("ckdt", tempdir);
logmsg("Creating code kdtree, reading %s, writing to %s\n", aq->codefn, ckdtfn);
if (codetree_files(aq->codefn, ckdtfn, 0, 0, 0, 0, argv, argc)) {
ERROR("codetree failed");
return -1;
}
char* skdt2fn=NULL;
char* quad2fn=NULL;
// unpermute-stars
logmsg("Unpermute-stars...\n");
skdt2fn = create_temp_file("skdt2", tempdir);
quad2fn = create_temp_file("quad2", tempdir);
logmsg("Unpermuting stars from %s and %s to %s and %s\n",
aq->skdtfn, aq->quadfn, skdt2fn, quad2fn);
if (unpermute_stars_files(aq->skdtfn, aq->quadfn, skdt2fn, quad2fn,
TRUE, FALSE, argv, argc)) {
ERROR("Failed to unpermute-stars");
return -1;
}
allquads_free(aq);
// unpermute-quads
/*
if (step_unpermute_quads(p, quads2, codekd, &quads3, &codekd2,
quad2fn, ckdtfn, &quad3fn, &ckdt2fn, tempfiles))
return -1;
*/
char* quad3fn=NULL;
char* ckdt2fn=NULL;
ckdt2fn = create_temp_file("ckdt2", tempdir);
quad3fn = create_temp_file("quad3", tempdir);
logmsg("Unpermuting quads from %s and %s to %s and %s\n",
quad2fn, ckdtfn, quad3fn, ckdt2fn);
if (unpermute_quads_files(quad2fn, ckdtfn,
quad3fn, ckdt2fn, argv, argc)) {
ERROR("Failed to unpermute-quads");
return -1;
}
// index
/*
if (step_merge_index(p, codekd2, quads3, starkd2, p_index,
ckdt2fn, quad3fn, skdt2fn, indexfn))
return -1;
*/
quadfile_t* quad;
codetree_t* code;
startree_t* star;
logmsg("Merging %s and %s and %s to %s\n",
quad3fn, ckdt2fn, skdt2fn, indexfn);
if (merge_index_open_files(quad3fn, ckdt2fn, skdt2fn,
&quad, &code, &star)) {
ERROR("Failed to open index files for merging");
return -1;
}
if (merge_index(quad, code, star, indexfn)) {
ERROR("Failed to write merged index");
return -1;
}
codetree_close(code);
startree_close(star);
quadfile_close(quad);
printf("Done.\n");
free(ckdtfn);
free(skdt2fn);
free(quad2fn);
free(ckdt2fn);
free(quad3fn);
return 0;
}
int main(int argc, char** args) {
int argchar;
char* outfn = NULL;
char* fn;
rdlist_t* rdls;
startree_t* skdt = NULL;
int i;
int loglvl = LOG_MSG;
while ((argchar = getopt (argc, args, OPTIONS)) != -1)
switch (argchar) {
case 'v':
loglvl++;
break;
case 'r':
outfn = optarg;
break;
case 'h':
print_help(args[0]);
exit(0);
}
log_init(loglvl);
if (!outfn || (optind == argc)) {
print_help(args[0]);
exit(-1);
}
rdls = rdlist_open_for_writing(outfn);
if (!rdls) {
ERROR("Failed to open RDLS file %s for output", outfn);
exit(-1);
}
if (rdlist_write_primary_header(rdls)) {
ERROR("Failed to write RDLS header");
exit(-1);
}
for (; optind<argc; optind++) {
int Nstars;
fn = args[optind];
logmsg("Opening star kdtree %s...\n", fn);
skdt = startree_open(fn);
if (!skdt) {
ERROR("Failed to read star kdtree %s", fn);
exit(-1);
}
Nstars = startree_N(skdt);
if (rdlist_write_header(rdls)) {
ERROR("Failed to write new RDLS field header");
exit(-1);
}
logmsg("Reading stars...\n");
for (i=0; i<Nstars; i++) {
double xyz[3];
double radec[2];
if (!(i % 200000)) {
printf(".");
fflush(stdout);
}
startree_get(skdt, i, xyz);
xyzarr2radecdegarr(xyz, radec);
if (rdlist_write_one_radec(rdls, radec[0], radec[1])) {
ERROR("Failed to write a RA,Dec entry");
exit(-1);
}
}
printf("\n");
startree_close(skdt);
if (rdlist_fix_header(rdls)) {
ERROR("Failed to fix RDLS field header");
exit(-1);
}
}
if (rdlist_fix_primary_header(rdls) ||
rdlist_close(rdls)) {
ERROR("Failed to close RDLS file");
exit(-1);
}
return 0;
}
startree_t* startree_build(fitstable_t* intable,
const char* racol, const char* deccol,
// keep RA,Dec in the tag-along table?
//anbool keep_radec,
// KDT_DATA_*, KDT_TREE_*
int datatype, int treetype,
// KD_BUILD_*
int buildopts,
int Nleaf,
char** args, int argc) {
double* ra = NULL;
double* dec = NULL;
double* xyz = NULL;
int N;
startree_t* starkd = NULL;
int tt;
int d;
double low[3];
double high[3];
qfits_header* hdr;
qfits_header* inhdr;
int i;
if (!racol)
racol = "RA";
if (!deccol)
deccol = "DEC";
if (!datatype)
datatype = KDT_DATA_U32;
if (!treetype)
treetype = KDT_TREE_U32;
if (!buildopts)
buildopts = KD_BUILD_SPLIT;
if (!Nleaf)
Nleaf = 25;
ra = fitstable_read_column(intable, racol, TFITS_BIN_TYPE_D);
if (!ra) {
ERROR("Failed to read RA from column %s", racol);
goto bailout;
}
dec = fitstable_read_column(intable, deccol, TFITS_BIN_TYPE_D);
if (!dec) {
ERROR("Failed to read RA from column %s", racol);
goto bailout;
}
N = fitstable_nrows(intable);
xyz = malloc(N * 3 * sizeof(double));
if (!xyz) {
SYSERROR("Failed to malloc xyz array to build startree");
goto bailout;
}
radecdeg2xyzarrmany(ra, dec, xyz, N);
free(ra);
ra = NULL;
free(dec);
dec = NULL;
starkd = startree_new();
if (!starkd) {
ERROR("Failed to allocate startree");
goto bailout;
}
tt = kdtree_kdtypes_to_treetype(KDT_EXT_DOUBLE, treetype, datatype);
starkd->tree = kdtree_new(N, 3, Nleaf);
for (d=0; d<3; d++) {
low[d] = -1.0;
high[d] = 1.0;
}
kdtree_set_limits(starkd->tree, low, high);
logverb("Building star kdtree...\n");
starkd->tree = kdtree_build(starkd->tree, xyz, N, 3, Nleaf, tt, buildopts);
if (!starkd->tree) {
ERROR("Failed to build star kdtree");
startree_close(starkd);
starkd = NULL;
goto bailout;
}
starkd->tree->name = strdup(STARTREE_NAME);
inhdr = fitstable_get_primary_header(intable);
hdr = startree_header(starkd);
an_fits_copy_header(inhdr, hdr, "HEALPIX");
an_fits_copy_header(inhdr, hdr, "HPNSIDE");
an_fits_copy_header(inhdr, hdr, "ALLSKY");
an_fits_copy_header(inhdr, hdr, "JITTER");
an_fits_copy_header(inhdr, hdr, "CUTNSIDE");
an_fits_copy_header(inhdr, hdr, "CUTMARG");
an_fits_copy_header(inhdr, hdr, "CUTDEDUP");
an_fits_copy_header(inhdr, hdr, "CUTNSWEP");
//fits_copy_header(inhdr, hdr, "CUTBAND");
//fits_copy_header(inhdr, hdr, "CUTMINMG");
//fits_copy_header(inhdr, hdr, "CUTMAXMG");
BOILERPLATE_ADD_FITS_HEADERS(hdr);
qfits_header_add(hdr, "HISTORY", "This file was created by the command-line:", NULL, NULL);
fits_add_args(hdr, args, argc);
qfits_header_add(hdr, "HISTORY", "(end of command line)", NULL, NULL);
qfits_header_add(hdr, "HISTORY", "** History entries copied from the input file:", NULL, NULL);
fits_copy_all_headers(inhdr, hdr, "HISTORY");
qfits_header_add(hdr, "HISTORY", "** End of history entries.", NULL, NULL);
for (i=1;; i++) {
char key[16];
int n;
sprintf(key, "SWEEP%i", i);
n = qfits_header_getint(inhdr, key, -1);
if (n == -1)
break;
an_fits_copy_header(inhdr, hdr, key);
}
bailout:
if (ra)
free(ra);
if (dec)
free(dec);
if (xyz)
free(xyz);
return starkd;
}
int unpermute_stars_files(const char* skdtinfn, const char* quadinfn,
const char* skdtoutfn, const char* quadoutfn,
anbool dosweeps, anbool check,
char** args, int argc) {
quadfile_t* qfin;
quadfile_t* qfout;
startree_t* treein;
startree_t* treeout;
fitstable_t* tagout = NULL;
fitstable_t* tagin;
int rtn;
logmsg("Reading star tree from %s ...\n", skdtinfn);
treein = startree_open(skdtinfn);
if (!treein) {
ERROR("Failed to read star kdtree from %s.\n", skdtinfn);
return -1;
}
logmsg("Reading quadfile from %s ...\n", quadinfn);
qfin = quadfile_open(quadinfn);
if (!qfin) {
ERROR("Failed to read quadfile from %s.\n", quadinfn);
return -1;
}
logmsg("Writing quadfile to %s ...\n", quadoutfn);
qfout = quadfile_open_for_writing(quadoutfn);
if (!qfout) {
ERROR("Failed to write quadfile to %s.\n", quadoutfn);
return -1;
}
rtn = unpermute_stars(treein, qfin, &treeout, qfout,
dosweeps, check, args, argc);
if (rtn)
return rtn;
if (quadfile_close(qfout)) {
ERROR("Failed to close output quadfile.\n");
return -1;
}
logmsg("Writing star kdtree to %s ...\n", skdtoutfn);
if (startree_write_to_file(treeout, skdtoutfn)) {
ERROR("Failed to write star kdtree.\n");
return -1;
}
if (startree_has_tagalong(treein)) {
logmsg("Permuting tag-along table...\n");
tagin = startree_get_tagalong(treein);
if (tagin) {
tagout = fitstable_open_for_appending(skdtoutfn);
tagout->table = fits_copy_table(tagin->table);
tagout->table->nr = 0;
if (unpermute_stars_tagalong(treein, tagout)) {
ERROR("Failed to permute tag-along table");
return -1;
}
if (fitstable_close(tagout)) {
ERROR("Failed to close tag-along data");
return -1;
}
}
}
quadfile_close(qfin);
startree_close(treein);
free(treeout->sweep);
free(treeout->tree);
treeout->tree = NULL;
startree_close(treeout);
return 0;
}
static startree_t* my_open(const char* fn, anqfits_t* fits) {
struct timeval tv1, tv2;
startree_t* s;
bl* chunks;
int i;
kdtree_fits_t* io;
char* treename = STARTREE_NAME;
const char* thefn = fn;
assert(fn || fits);
if (!thefn)
thefn = fits->filename;
s = startree_alloc();
if (!s)
return NULL;
gettimeofday(&tv1, NULL);
if (fn)
io = kdtree_fits_open(fn);
else
io = kdtree_fits_open_fits(fits);
gettimeofday(&tv2, NULL);
debug("kdtree_fits_open() took %g ms\n", millis_between(&tv1, &tv2));
if (!io) {
ERROR("Failed to open FITS file \"%s\"", thefn);
goto bailout;
}
gettimeofday(&tv1, NULL);
if (!kdtree_fits_contains_tree(io, treename))
treename = NULL;
gettimeofday(&tv2, NULL);
debug("kdtree_fits_contains_tree() took %g ms\n", millis_between(&tv1, &tv2));
gettimeofday(&tv1, NULL);
s->tree = kdtree_fits_read_tree(io, treename, &s->header);
gettimeofday(&tv2, NULL);
debug("kdtree_fits_read_tree() took %g ms\n", millis_between(&tv1, &tv2));
if (!s->tree) {
ERROR("Failed to read kdtree from file \"%s\"", thefn);
goto bailout;
}
// Check the tree dimensionality.
// (because code trees can be confused...)
if (s->tree->ndim != 3) {
logverb("File %s contains a kd-tree with dim %i (not 3), named %s\n",
thefn, s->tree->ndim, treename);
s->tree->io = NULL;
goto bailout;
}
gettimeofday(&tv1, NULL);
chunks = get_chunks(s, NULL);
for (i=0; i<bl_size(chunks); i++) {
fitsbin_chunk_t* chunk = bl_access(chunks, i);
void** dest = chunk->userdata;
kdtree_fits_read_chunk(io, chunk);
*dest = chunk->data;
}
bl_free(chunks);
gettimeofday(&tv2, NULL);
debug("reading chunks took %g ms\n", millis_between(&tv1, &tv2));
// kdtree_fits_t is a typedef of fitsbin_t
fitsbin_close_fd(io);
return s;
bailout:
kdtree_fits_io_close(io);
startree_close(s);
return NULL;
}
