int multiindex_add_index(multiindex_t* mi, const char* fn, int flags) {
anqfits_t* fits;
quadfile_t* quads = NULL;
codetree_t* codes = NULL;
index_t* ind = NULL;
logverb("Reading index file \"%s\"...\n", fn);
fits = anqfits_open(fn);
if (!fits) {
ERROR("Failed to open FITS file \"%s\"", fn);
goto bailout;
}
logverb("Reading quads from file \"%s\"...\n", fn);
quads = quadfile_open_fits(fits);
if (!quads) {
ERROR("Failed to read quads from file \"%s\"", fn);
anqfits_close(fits);
goto bailout;
}
logverb("Reading codes from file \"%s\"...\n", fn);
codes = codetree_open_fits(fits);
if (!codes) {
ERROR("Failed to read quads from file \"%s\"", fn);
quadfile_close(quads);
anqfits_close(fits);
goto bailout;
}
ind = index_build_from(codes, quads, mi->starkd);
ind->fits = fits;
if (!ind->indexname)
ind->indexname = strdup(fn);
// shouldn't be needed, but set anyway
ind->quadfn = strdup(fn);
ind->codefn = strdup(fn);
pl_append(mi->inds, ind);
if (flags & INDEX_ONLY_LOAD_METADATA) {
// don't let it unload the starkd!
ind->starkd = NULL;
index_unload(ind);
ind->starkd = mi->starkd;
}
return 0;
bailout:
if (quads) {
quadfile_close(quads);
}
if (codes) {
codetree_close(codes);
}
if (fits) {
anqfits_close(fits);
}
return -1;
}
int unpermute_quads_files(const char* quadinfn, const char* ckdtinfn,
const char* quadoutfn, const char* ckdtoutfn,
char** args, int argc) {
quadfile* quadin;
quadfile* quadout;
codetree* treein;
codetree* treeout;
logmsg("Reading code tree from %s ...\n", ckdtinfn);
treein = codetree_open(ckdtinfn);
if (!treein) {
ERROR("Failed to read code kdtree from %s", ckdtinfn);
return -1;
}
logmsg("Reading quads from %s ...\n", quadinfn);
quadin = quadfile_open(quadinfn);
if (!quadin) {
ERROR("Failed to read quads from %s", quadinfn);
return -1;
}
logmsg("Writing quads to %s ...\n", quadoutfn);
quadout = quadfile_open_for_writing(quadoutfn);
if (!quadout) {
ERROR("Failed to write quads to %s", quadoutfn);
return -1;
}
if (unpermute_quads(quadin, treein, quadout, &treeout, args, argc)) {
return -1;
}
if (quadfile_close(quadout)) {
ERROR("Failed to close output quadfile");
return -1;
}
quadfile_close(quadin);
logmsg("Writing code kdtree to %s ...\n", ckdtoutfn);
if (codetree_write_to_file(treeout, ckdtoutfn) ||
codetree_close(treeout)) {
ERROR("Failed to write code kdtree");
return -1;
}
free(treein->tree);
treein->tree = NULL;
codetree_close(treein);
return 0;
}
int merge_index_files(const char* quadfn, const char* ckdtfn, const char* skdtfn,
const char* indexfn) {
quadfile* quad = NULL;
codetree* code = NULL;
startree_t* star = NULL;
int rtn;
if (merge_index_open_files(quadfn, ckdtfn, skdtfn, &quad, &code, &star)) {
rtn = -1;
goto cleanup;
}
logmsg("Writing index to %s ...\n", indexfn);
rtn = merge_index(quad, code, star, indexfn);
cleanup:
if (code)
codetree_close(code);
if (star)
startree_close(star);
if (quad)
quadfile_close(quad);
return rtn;
}
int allquads_close(allquads_t* aq) {
startree_close(aq->starkd);
// fix output file headers.
if (quadfile_fix_header(aq->quads) ||
quadfile_close(aq->quads)) {
ERROR("Couldn't write quad output file");
return -1;
}
if (codefile_fix_header(aq->codes) ||
codefile_close(aq->codes)) {
ERROR("Couldn't write code output file");
return -1;
}
return 0;
}
int main(int argc, char *argv[]) {
int argidx, argchar;
char *qidxfname = NULL;
char *quadfname = NULL;
quadfile* quad;
qidxfile* qidx;
int q, s;
int dimquads;
while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
switch (argchar) {
case 'f':
qidxfname = mk_qidxfn(optarg);
quadfname = mk_quadfn(optarg);
break;
case '?':
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
case 'h':
fprintf(stderr, HelpString);
return (HELP_ERR);
default:
return (OPT_ERR);
}
if (optind < argc) {
for (argidx = optind; argidx < argc; argidx++)
fprintf (stderr, "Non-option argument %s\n", argv[argidx]);
fprintf(stderr, HelpString);
return (OPT_ERR);
}
quad = quadfile_open(quadfname, 0);
if (!quad) {
fprintf(stderr, "Couldn't open quads file %s.\n", quadfname);
exit(-1);
}
qidx = qidxfile_open(qidxfname, 0);
if (!qidx) {
fprintf(stderr, "Couldn't open qidx file %s.\n", qidxfname);
exit(-1);
}
if (quad->numquads != qidx->numquads) {
fprintf(stderr, "Number of quads does not agree: %i vs %i\n",
quad->numquads, qidx->numquads);
exit(-1);
}
dimquads = quadfile_dimquads(quad);
printf("Checking stars...\n");
for (s=0; s<qidx->numstars; s++) {
uint32_t* quads;
int nquads;
int j;
qidxfile_get_quads(qidx, s, &quads, &nquads);
for (j=0; j<nquads; j++) {
int star[dimquads];
int k, n;
quadfile_get_stars(quad, quads[j], star);
n = 0;
for (k=0; k<dimquads; k++) {
if (star[k] == s)
n++;
}
if (n != 1) {
fprintf(stderr, "Star %i, quad %i: found %i instances of the quad in the qidx (not 1)\n",
s, quads[j], n);
fprintf(stderr, " found: ");
for (k=0; k<dimquads; k++) {
fprintf(stderr, "%i ", star[k]);
}
fprintf(stderr, "\n");
}
}
}
printf("Checking quads...\n");
for (q=0; q<quad->numquads; q++) {
int star[dimquads];
uint32_t* quads;
int nquads;
int j;
quadfile_get_stars(quad, q, star);
for (j=0; j<dimquads; j++) {
int k;
int n;
qidxfile_get_quads(qidx, star[j], &quads, &nquads);
n = 0;
for (k=0; k<nquads; k++) {
if (quads[k] == q)
n++;
}
if (n != 1) {
fprintf(stderr, "Quad %i, star %i: found %i instances of the quad in the qidx (not 1)\n",
q, star[j], n);
fprintf(stderr, " found: ");
for (k=0; k<nquads; k++) {
fprintf(stderr, "%i ", quads[k]);
}
fprintf(stderr, "\n");
}
}
}
quadfile_close(quad);
qidxfile_close(qidx);
return 0;
}
int hpquads_files(const char* skdtfn,
const char* codefn,
const char* quadfn,
int Nside,
double scale_min_arcmin,
double scale_max_arcmin,
int dimquads,
int passes,
int Nreuses,
int Nloosen,
int id,
anbool scanoccupied,
void* sort_data,
int (*sort_func)(const void*, const void*),
int sort_size,
char** args, int argc) {
quadfile_t* quads;
codefile_t* codes;
startree_t* starkd;
int rtn;
logmsg("Reading star kdtree %s ...\n", skdtfn);
starkd = startree_open(skdtfn);
if (!starkd) {
ERROR("Failed to open star kdtree %s\n", skdtfn);
return -1;
}
logmsg("Will write to quad file %s and code file %s\n", quadfn, codefn);
quads = quadfile_open_for_writing(quadfn);
if (!quads) {
ERROR("Couldn't open file %s to write quads.\n", quadfn);
return -1;
}
codes = codefile_open_for_writing(codefn);
if (!codes) {
ERROR("Couldn't open file %s to write codes.\n", codefn);
return -1;
}
rtn = hpquads(starkd, codes, quads, Nside,
scale_min_arcmin, scale_max_arcmin,
dimquads, passes, Nreuses, Nloosen, id,
scanoccupied,
sort_data, sort_func, sort_size,
args, argc);
if (rtn)
return rtn;
if (quadfile_close(quads)) {
ERROR("Couldn't write quad output file");
return -1;
}
if (codefile_close(codes)) {
ERROR("Couldn't write code output file");
return -1;
}
startree_close(starkd);
return rtn;
}
int main(int argc, char** argv) {
int argchar;
allquads_t* aq;
int loglvl = LOG_MSG;
int i, N;
char* catfn = NULL;
startree_t* starkd;
fitstable_t* cat;
char* racol = NULL;
char* deccol = NULL;
int datatype = KDT_DATA_DOUBLE;
int treetype = KDT_TREE_DOUBLE;
int buildopts = 0;
int Nleaf = 0;
char* indexfn = NULL;
//index_params_t* p;
aq = allquads_init();
aq->skdtfn = "allquads.skdt";
aq->codefn = "allquads.code";
aq->quadfn = "allquads.quad";
while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
switch (argchar) {
case 'v':
loglvl++;
break;
case 'd':
aq->dimquads = atoi(optarg);
break;
case 'I':
aq->id = atoi(optarg);
break;
case 'h':
print_help(argv[0]);
exit(0);
case 'i':
catfn = optarg;
break;
case 'o':
indexfn = optarg;
break;
case 'u':
aq->quad_d2_upper = arcmin2distsq(atof(optarg));
aq->use_d2_upper = TRUE;
break;
case 'l':
aq->quad_d2_lower = arcmin2distsq(atof(optarg));
aq->use_d2_lower = TRUE;
break;
default:
return -1;
}
log_init(loglvl);
if (!catfn || !indexfn) {
printf("Specify in & out filenames, bonehead!\n");
print_help(argv[0]);
exit( -1);
}
if (optind != argc) {
print_help(argv[0]);
printf("\nExtra command-line args were given: ");
for (i=optind; i<argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
exit(-1);
}
if (!aq->id)
logmsg("Warning: you should set the unique-id for this index (with -I).\n");
if (aq->dimquads > DQMAX) {
ERROR("Quad dimension %i exceeds compiled-in max %i.\n", aq->dimquads, DQMAX);
exit(-1);
}
aq->dimcodes = dimquad2dimcode(aq->dimquads);
// Read reference catalog, write star kd-tree
logmsg("Building star kdtree: reading %s, writing to %s\n", catfn, aq->skdtfn);
logverb("Reading star catalogue...");
cat = fitstable_open(catfn);
if (!cat) {
ERROR("Couldn't read catalog");
exit(-1);
}
logmsg("Got %i stars\n", fitstable_nrows(cat));
starkd = startree_build(cat, racol, deccol, datatype, treetype,
buildopts, Nleaf, argv, argc);
if (!starkd) {
ERROR("Failed to create star kdtree");
exit(-1);
}
logmsg("Star kd-tree contains %i data points in dimension %i\n",
startree_N(starkd), startree_D(starkd));
N = startree_N(starkd);
for (i=0; i<N; i++) {
double ra,dec;
int ok;
ok = startree_get_radec(starkd, i, &ra, &dec);
logmsg(" data %i: ok %i, RA,Dec %g, %g\n", i, ok, ra, dec);
}
if (startree_write_to_file(starkd, aq->skdtfn)) {
ERROR("Failed to write star kdtree");
exit(-1);
}
startree_close(starkd);
fitstable_close(cat);
logmsg("Wrote star kdtree to %s\n", aq->skdtfn);
logmsg("Running allquads...\n");
if (allquads_open_outputs(aq)) {
exit(-1);
}
if (allquads_create_quads(aq)) {
exit(-1);
}
if (allquads_close(aq)) {
exit(-1);
}
logmsg("allquads: wrote %s, %s\n", aq->quadfn, aq->codefn);
// build-index:
//build_index_defaults(&p);
// codetree
/*
if (step_codetree(p, codes, &codekd,
codefn, &ckdtfn, tempfiles))
return -1;
*/
char* ckdtfn=NULL;
char* tempdir = NULL;
ckdtfn = create_temp_file("ckdt", tempdir);
logmsg("Creating code kdtree, reading %s, writing to %s\n", aq->codefn, ckdtfn);
if (codetree_files(aq->codefn, ckdtfn, 0, 0, 0, 0, argv, argc)) {
ERROR("codetree failed");
return -1;
}
char* skdt2fn=NULL;
char* quad2fn=NULL;
// unpermute-stars
logmsg("Unpermute-stars...\n");
skdt2fn = create_temp_file("skdt2", tempdir);
quad2fn = create_temp_file("quad2", tempdir);
logmsg("Unpermuting stars from %s and %s to %s and %s\n",
aq->skdtfn, aq->quadfn, skdt2fn, quad2fn);
if (unpermute_stars_files(aq->skdtfn, aq->quadfn, skdt2fn, quad2fn,
TRUE, FALSE, argv, argc)) {
ERROR("Failed to unpermute-stars");
return -1;
}
allquads_free(aq);
// unpermute-quads
/*
if (step_unpermute_quads(p, quads2, codekd, &quads3, &codekd2,
quad2fn, ckdtfn, &quad3fn, &ckdt2fn, tempfiles))
return -1;
*/
char* quad3fn=NULL;
char* ckdt2fn=NULL;
ckdt2fn = create_temp_file("ckdt2", tempdir);
quad3fn = create_temp_file("quad3", tempdir);
logmsg("Unpermuting quads from %s and %s to %s and %s\n",
quad2fn, ckdtfn, quad3fn, ckdt2fn);
if (unpermute_quads_files(quad2fn, ckdtfn,
quad3fn, ckdt2fn, argv, argc)) {
ERROR("Failed to unpermute-quads");
return -1;
}
// index
/*
if (step_merge_index(p, codekd2, quads3, starkd2, p_index,
ckdt2fn, quad3fn, skdt2fn, indexfn))
return -1;
*/
quadfile_t* quad;
codetree_t* code;
startree_t* star;
logmsg("Merging %s and %s and %s to %s\n",
quad3fn, ckdt2fn, skdt2fn, indexfn);
if (merge_index_open_files(quad3fn, ckdt2fn, skdt2fn,
&quad, &code, &star)) {
ERROR("Failed to open index files for merging");
return -1;
}
if (merge_index(quad, code, star, indexfn)) {
ERROR("Failed to write merged index");
return -1;
}
codetree_close(code);
startree_close(star);
quadfile_close(quad);
printf("Done.\n");
free(ckdtfn);
free(skdt2fn);
free(quad2fn);
free(ckdt2fn);
free(quad3fn);
return 0;
}
int unpermute_stars_files(const char* skdtinfn, const char* quadinfn,
const char* skdtoutfn, const char* quadoutfn,
anbool dosweeps, anbool check,
char** args, int argc) {
quadfile_t* qfin;
quadfile_t* qfout;
startree_t* treein;
startree_t* treeout;
fitstable_t* tagout = NULL;
fitstable_t* tagin;
int rtn;
logmsg("Reading star tree from %s ...\n", skdtinfn);
treein = startree_open(skdtinfn);
if (!treein) {
ERROR("Failed to read star kdtree from %s.\n", skdtinfn);
return -1;
}
logmsg("Reading quadfile from %s ...\n", quadinfn);
qfin = quadfile_open(quadinfn);
if (!qfin) {
ERROR("Failed to read quadfile from %s.\n", quadinfn);
return -1;
}
logmsg("Writing quadfile to %s ...\n", quadoutfn);
qfout = quadfile_open_for_writing(quadoutfn);
if (!qfout) {
ERROR("Failed to write quadfile to %s.\n", quadoutfn);
return -1;
}
rtn = unpermute_stars(treein, qfin, &treeout, qfout,
dosweeps, check, args, argc);
if (rtn)
return rtn;
if (quadfile_close(qfout)) {
ERROR("Failed to close output quadfile.\n");
return -1;
}
logmsg("Writing star kdtree to %s ...\n", skdtoutfn);
if (startree_write_to_file(treeout, skdtoutfn)) {
ERROR("Failed to write star kdtree.\n");
return -1;
}
if (startree_has_tagalong(treein)) {
logmsg("Permuting tag-along table...\n");
tagin = startree_get_tagalong(treein);
if (tagin) {
tagout = fitstable_open_for_appending(skdtoutfn);
tagout->table = fits_copy_table(tagin->table);
tagout->table->nr = 0;
if (unpermute_stars_tagalong(treein, tagout)) {
ERROR("Failed to permute tag-along table");
return -1;
}
if (fitstable_close(tagout)) {
ERROR("Failed to close tag-along data");
return -1;
}
}
}
quadfile_close(qfin);
startree_close(treein);
free(treeout->sweep);
free(treeout->tree);
treeout->tree = NULL;
startree_close(treeout);
return 0;
}
int main(int argc, char *argv[]) {
char* progname = argv[0];
int argidx, argchar;
char *idxfname = NULL;
char *quadfname = NULL;
il** quadlist;
quadfile* quads;
qidxfile* qidx;
int q;
int i;
int numused;
qfits_header* quadhdr;
qfits_header* qidxhdr;
int dimquads;
anbool check = FALSE;
int loglvl = LOG_MSG;
if (argc <= 2) {
printHelp(progname);
exit(-1);
}
while ((argchar = getopt (argc, argv, OPTIONS)) != -1)
switch (argchar) {
case 'c':
check = TRUE;
break;
case 'v':
loglvl++;
break;
case 'i':
quadfname = optarg;
break;
case 'o':
idxfname = optarg;
break;
case '?':
fprintf(stderr, "Unknown option `-%c'.\n", optopt);
case 'h':
printHelp(progname);
exit(-1);
default:
return (OPT_ERR);
}
log_init(loglvl);
if (optind < argc) {
for (argidx = optind; argidx < argc; argidx++)
fprintf (stderr, "Non-option argument %s\n", argv[argidx]);
printHelp(progname);
exit(-1);
}
logmsg("quadidx: indexing quads in \"%s\"...\n", quadfname);
logmsg("will write to file \"%s\".\n", idxfname);
quads = quadfile_open(quadfname);
if (!quads) {
ERROR("Couldn't open quads file \"%s\"", quadfname);
exit(-1);
}
logmsg("%u quads, %u stars.\n", quads->numquads, quads->numstars);
if (check) {
logmsg("Running quadfile_check()...\n");
if (quadfile_check(quads)) {
ERROR("quadfile_check() failed");
exit(-1);
}
logmsg("Check passed.\n");
}
quadlist = calloc(quads->numstars, sizeof(il*));
if (!quadlist) {
SYSERROR("Failed to allocate list of quad contents");
exit(-1);
}
dimquads = quadfile_dimquads(quads);
for (q=0; q<quads->numquads; q++) {
unsigned int inds[dimquads];
quadfile_get_stars(quads, q, inds);
// append this quad index to the lists of each of its stars.
for (i=0; i<dimquads; i++) {
il* list;
int starind = inds[i];
list = quadlist[starind];
// create the list if necessary
if (!list) {
list = il_new(10);
quadlist[starind] = list;
}
il_append(list, q);
}
}
// first count numused:
// how many stars are members of quads.
numused = 0;
for (i=0; i<quads->numstars; i++) {
il* list = quadlist[i];
if (!list) continue;
numused++;
}
logmsg("%u stars used\n", numused);
qidx = qidxfile_open_for_writing(idxfname, quads->numstars, quads->numquads);
if (!qidx) {
logmsg("Couldn't open outfile qidx file %s.\n", idxfname);
exit(-1);
}
quadhdr = quadfile_get_header(quads);
qidxhdr = qidxfile_get_header(qidx);
an_fits_copy_header(quadhdr, qidxhdr, "INDEXID");
an_fits_copy_header(quadhdr, qidxhdr, "HEALPIX");
BOILERPLATE_ADD_FITS_HEADERS(qidxhdr);
qfits_header_add(qidxhdr, "HISTORY", "This file was created by the program \"quadidx\".", NULL, NULL);
qfits_header_add(qidxhdr, "HISTORY", "quadidx command line:", NULL, NULL);
fits_add_args(qidxhdr, argv, argc);
qfits_header_add(qidxhdr, "HISTORY", "(end of quadidx command line)", NULL, NULL);
qfits_header_add(qidxhdr, "HISTORY", "** History entries copied from the input file:", NULL, NULL);
fits_copy_all_headers(quadhdr, qidxhdr, "HISTORY");
qfits_header_add(qidxhdr, "HISTORY", "** End of history entries.", NULL, NULL);
if (qidxfile_write_header(qidx)) {
logmsg("Couldn't write qidx header (%s).\n", idxfname);
exit(-1);
}
for (i=0; i<quads->numstars; i++) {
int thisnumq;
//int thisstar;
int* stars; // bad variable name - list of quads this star is in.
il* list = quadlist[i];
if (list) {
thisnumq = (uint)il_size(list);
stars = malloc(thisnumq * sizeof(uint));
il_copy(list, 0, thisnumq, (int*)stars);
} else {
thisnumq = 0;
stars = NULL;
}
//thisstar = i;
if (qidxfile_write_star(qidx, stars, thisnumq)) {
logmsg("Couldn't write star to qidx file (%s).\n", idxfname);
exit(-1);
}
if (list) {
free(stars);
il_free(list);
quadlist[i] = NULL;
}
}
free(quadlist);
quadfile_close(quads);
if (qidxfile_close(qidx)) {
logmsg("Failed to close qidx file.\n");
exit(-1);
}
logmsg(" done.\n");
return 0;
}
