static void add_columns(fitstable_t* tab, anbool write) { tfits_type any = fitscolumn_any_type(); tfits_type d = fitscolumn_double_type(); tfits_type f = fitscolumn_float_type(); tfits_type u8 = fitscolumn_u8_type(); tfits_type i16 = fitscolumn_i16_type(); tfits_type i32 = fitscolumn_i32_type(); tfits_type i64 = fitscolumn_i64_type(); tfits_type i = fitscolumn_int_type(); tfits_type logical = fitscolumn_boolean_type(); tfits_type b = fitscolumn_bool_type(); tfits_type c = fitscolumn_char_type(); char* nil = " "; MatchObj mo; ADDCOL(i, i32, "QUAD", nil, quadno); ADDCOL(u8, u8, "DIMQUADS", nil, dimquads); ADDARR(i, i32, "STARS", nil, star, DQMAX); ADDARR(i, i32, "FIELDOBJS", nil, field, DQMAX); ADDARR(i64,i64, "IDS", nil, ids, DQMAX); ADDCOL(f, f, "CODEERR", nil, code_err); ADDARR(d, d, "QUADPIX", nil, quadpix, 2*DQMAX); ADDARR(d, d, "QUADXYZ", nil, quadxyz, 3*DQMAX); ADDARR(d, d, "CENTERXYZ", nil, center, 3); ADDCOL(d, d, "RADIUS", "DEG", radius_deg); ADDCOL(i, i32, "NMATCH", nil, nmatch); ADDCOL(i ,i32, "NDISTRACT", nil, ndistractor); ADDCOL(i ,i32, "NCONFLICT", nil, nconflict); ADDCOL(i ,i32, "NFIELD", nil, nfield); ADDCOL(i ,i32, "NINDEX", nil, nindex); ADDCOL(i ,i32, "NAGREE", nil, nagree); //ADDCOL(i16,i16, "BESTI", nil, besti); ADDARR(d, d, "CRVAL", nil, wcstan.crval, 2); ADDARR(d, d, "CRPIX", nil, wcstan.crpix, 2); ADDARR(d, d, "CD", nil, wcstan.cd, 4); ADDCOL(b, logical, "WCS_VALID", nil, wcs_valid); ADDCOL(i,i32, "FIELDNUM", nil, fieldnum); ADDCOL(i,i32, "FIELDID", nil, fieldfile); ADDCOL(i16,i16, "INDEXID", nil, indexid); ADDCOL(i16,i16, "HEALPIX", nil, healpix); ADDCOL(i16,i16, "HPNSIDE", nil, hpnside); ADDARR(c, c, "FIELDNAME", nil, fieldname, sizeof(mo.fieldname)-1); ADDCOL(b, logical, "PARITY", nil, parity); ADDCOL(i,i32, "QTRIED", nil, quads_tried); ADDCOL(i,i32, "QMATCHED", nil, quads_matched); ADDCOL(i,i32, "QSCALEOK", nil, quads_scaleok); ADDCOL(i16,i16, "QPEERS", nil, quad_npeers); ADDCOL(i,i32, "NVERIFIED", nil, nverified); ADDCOL(f, f, "TIMEUSED", "s", timeused); ADDCOL(f, f, "LOGODDS", nil, logodds); ADDCOL(f, f, "WORSTLOGODDS", nil, worstlogodds); }
double* startree_get_data_column_array(startree_t* s, const char* colname, const int* indices, int N, int* arraysize) { fitstable_t* table; tfits_type dubl = fitscolumn_double_type(); double* arr; table = startree_get_tagalong(s); if (!table) { ERROR("No tag-along data found"); return NULL; } arr = fitstable_read_column_array_inds(table, colname, dubl, indices, N, arraysize); if (!arr) { ERROR("Failed to read tag-along data"); return NULL; } return arr; }
static void add_columns(fitstable_t* tab, anbool write) { tfits_type any = fitscolumn_any_type(); tfits_type d = fitscolumn_double_type(); tfits_type f = fitscolumn_float_type(); tfits_type u8 = fitscolumn_u8_type(); tfits_type i32 = fitscolumn_i32_type(); tfits_type J = TFITS_BIN_TYPE_J; tfits_type bitfield = fitscolumn_bitfield_type(); char* nil = " "; ADDCOL(d, d, "RA", "deg", ra); ADDCOL(d, d, "DEC", "deg", dec); ADDCOL(f, f, "SIGMA_RACOSDEC", "deg", sigma_racosdec); ADDCOL(f, f, "SIGMA_DEC", "deg", sigma_dec); ADDCOL(f, f, "PM_RACOSDEC", "arcsec/yr", pm_racosdec); ADDCOL(f, f, "PM_DEC", "arcsyc/yr", pm_dec); ADDCOL(f, f, "SIGMA_PM_RACOSDEC", "arcsec/yr", sigma_pm_racosdec); ADDCOL(f, f, "SIGMA_PM_DEC", "arcsyc/yr", sigma_pm_dec); ADDCOL(f, f, "EPOCH_RA", "yr", epoch_ra); ADDCOL(f, f, "EPOCH_DEC", "yr", epoch_dec); ADDCOL(f, f, "MAG_B", "mag", mag_B); ADDCOL(f, f, "MAG_V", "mag", mag_V); ADDCOL(f, f, "MAG_R", "mag", mag_R); ADDCOL(f, f, "MAG_J", "mag", mag_J); ADDCOL(f, f, "MAG_H", "mag", mag_H); ADDCOL(f, f, "MAG_K", "mag", mag_K); ADDCOL(i32,J, "USNOB_ID", nil, usnob_id); ADDCOL(i32,J, "TWOMASS_ID", nil, twomass_id); ADDCOL(i32,J, "YB6_ID", nil, yb6_id); ADDCOL(i32,J, "UCAC2_ID", nil, ucac2_id); ADDCOL(i32,J, "TYCHO2_ID", nil, tycho2_id); ADDCOL(u8, u8, "ASTROMETRY_SRC", nil, astrometry_src); ADDCOL(u8, u8, "BLUE_SRC", nil, blue_src); ADDCOL(u8, u8, "VISUAL_SRC", nil, visual_src); ADDCOL(u8, u8, "RED_SRC", nil, red_src); ADDCOL(i32, J, "NOMAD_ID", nil, nomad_id); ADDARR(bitfield, bitfield, "FLAGS", nil, flags, 2); }
int main(int argc, char *argv[]) { int argchar; char* progname = argv[0]; sl* infns = sl_new(16); char* outfnpat = NULL; char* racol = "RA"; char* deccol = "DEC"; char* tempdir = "/tmp"; anbool gzip = FALSE; sl* cols = sl_new(16); int loglvl = LOG_MSG; int nside = 1; double margin = 0.0; int NHP; double md; char* backref = NULL; fitstable_t* intable; fitstable_t** outtables; char** myargs; int nmyargs; int i; while ((argchar = getopt (argc, argv, OPTIONS)) != -1) switch (argchar) { case 'b': backref = optarg; break; case 't': tempdir = optarg; break; case 'c': sl_append(cols, optarg); break; case 'g': gzip = TRUE; break; case 'o': outfnpat = optarg; break; case 'r': racol = optarg; break; case 'd': deccol = optarg; break; case 'n': nside = atoi(optarg); break; case 'm': margin = atof(optarg); break; case 'v': loglvl++; break; case '?': fprintf(stderr, "Unknown option `-%c'.\n", optopt); case 'h': printHelp(progname); return 0; default: return -1; } if (sl_size(cols) == 0) { sl_free2(cols); cols = NULL; } nmyargs = argc - optind; myargs = argv + optind; for (i=0; i<nmyargs; i++) sl_append(infns, myargs[i]); if (!sl_size(infns)) { printHelp(progname); printf("Need input filenames!\n"); exit(-1); } log_init(loglvl); fits_use_error_system(); NHP = 12 * nside * nside; logmsg("%i output healpixes\n", NHP); outtables = calloc(NHP, sizeof(fitstable_t*)); assert(outtables); md = deg2dist(margin); /** About the mincaps/maxcaps: These have a center and radius-squared, describing the region inside a small circle on the sphere. The "mincaps" describe the regions that are definitely owned by a single healpix -- ie, more than MARGIN distance from any edge. That is, the mincap is the small circle centered at (0.5, 0.5) in the healpix and with radius = the distance to the closest healpix boundary, MINUS the margin distance. Below, we first check whether a new star is within the "mincap" of any healpix. If so, we stick it in that healpix and continue. Otherwise, we check all the "maxcaps" -- these are the healpixes it could *possibly* be in. We then refine with healpix_within_range_of_xyz. The maxcap distance is the distance to the furthest boundary point, PLUS the margin distance. */ cap_t* mincaps = malloc(NHP * sizeof(cap_t)); cap_t* maxcaps = malloc(NHP * sizeof(cap_t)); for (i=0; i<NHP; i++) { // center double r2; double xyz[3]; double* cxyz; double step = 1e-3; double v; double r2b, r2a; cxyz = mincaps[i].xyz; healpix_to_xyzarr(i, nside, 0.5, 0.5, mincaps[i].xyz); memcpy(maxcaps[i].xyz, cxyz, 3 * sizeof(double)); logverb("Center of HP %i: (%.3f, %.3f, %.3f)\n", i, cxyz[0], cxyz[1], cxyz[2]); // radius-squared: // max is the easy one: max of the four corners (I assume) r2 = 0.0; healpix_to_xyzarr(i, nside, 0.0, 0.0, xyz); logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3)); r2 = MAX(r2, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, 1.0, 0.0, xyz); logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3)); r2 = MAX(r2, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, 0.0, 1.0, xyz); logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3)); r2 = MAX(r2, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, 1.0, 1.0, xyz); logverb(" HP %i corner 1: (%.3f, %.3f, %.3f), distsq %.3f\n", i, xyz[0], xyz[1], xyz[2], distsq(xyz, cxyz, 3)); r2 = MAX(r2, distsq(xyz, cxyz, 3)); logverb(" max distsq: %.3f\n", r2); logverb(" margin dist: %.3f\n", md); maxcaps[i].r2 = square(sqrt(r2) + md); logverb(" max cap distsq: %.3f\n", maxcaps[i].r2); r2a = r2; r2 = 1.0; r2b = 0.0; for (v=0; v<=1.0; v+=step) { healpix_to_xyzarr(i, nside, 0.0, v, xyz); r2 = MIN(r2, distsq(xyz, cxyz, 3)); r2b = MAX(r2b, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, 1.0, v, xyz); r2 = MIN(r2, distsq(xyz, cxyz, 3)); r2b = MAX(r2b, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, v, 0.0, xyz); r2 = MIN(r2, distsq(xyz, cxyz, 3)); r2b = MAX(r2b, distsq(xyz, cxyz, 3)); healpix_to_xyzarr(i, nside, v, 1.0, xyz); r2 = MIN(r2, distsq(xyz, cxyz, 3)); r2b = MAX(r2b, distsq(xyz, cxyz, 3)); } mincaps[i].r2 = square(MAX(0, sqrt(r2) - md)); logverb("\nhealpix %i: min rad %g\n", i, sqrt(r2)); logverb("healpix %i: max rad %g\n", i, sqrt(r2a)); logverb("healpix %i: max rad(b) %g\n", i, sqrt(r2b)); assert(r2a >= r2b); } if (backref) { fitstable_t* tab = fitstable_open_for_writing(backref); int maxlen = 0; char* buf; for (i=0; i<sl_size(infns); i++) { char* infn = sl_get(infns, i); maxlen = MAX(maxlen, strlen(infn)); } fitstable_add_write_column_array(tab, fitscolumn_char_type(), maxlen, "filename", NULL); fitstable_add_write_column(tab, fitscolumn_i16_type(), "index", NULL); if (fitstable_write_primary_header(tab) || fitstable_write_header(tab)) { ERROR("Failed to write header of backref table \"%s\"", backref); exit(-1); } buf = malloc(maxlen+1); assert(buf); for (i=0; i<sl_size(infns); i++) { char* infn = sl_get(infns, i); int16_t ind; memset(buf, 0, maxlen); strcpy(buf, infn); ind = i; if (fitstable_write_row(tab, buf, &ind)) { ERROR("Failed to write row %i of backref table: %s = %i", i, buf, ind); exit(-1); } } if (fitstable_fix_header(tab) || fitstable_close(tab)) { ERROR("Failed to fix header & close backref table"); exit(-1); } logmsg("Wrote backref table %s\n", backref); free(buf); } for (i=0; i<sl_size(infns); i++) { char* infn = sl_get(infns, i); char* originfn = infn; int r, NR; tfits_type any, dubl; il* hps = NULL; bread_t* rowbuf; int R; char* tempfn = NULL; char* padrowdata = NULL; int ii; logmsg("Reading input \"%s\"...\n", infn); if (gzip) { char* cmd; int rtn; tempfn = create_temp_file("hpsplit", tempdir); asprintf_safe(&cmd, "gunzip -cd %s > %s", infn, tempfn); logmsg("Running: \"%s\"\n", cmd); rtn = run_command_get_outputs(cmd, NULL, NULL); if (rtn) { ERROR("Failed to run command: \"%s\"", cmd); exit(-1); } free(cmd); infn = tempfn; } intable = fitstable_open(infn); if (!intable) { ERROR("Couldn't read catalog %s", infn); exit(-1); } NR = fitstable_nrows(intable); logmsg("Got %i rows\n", NR); any = fitscolumn_any_type(); dubl = fitscolumn_double_type(); fitstable_add_read_column_struct(intable, dubl, 1, 0, any, racol, TRUE); fitstable_add_read_column_struct(intable, dubl, 1, sizeof(double), any, deccol, TRUE); fitstable_use_buffered_reading(intable, 2*sizeof(double), 1000); R = fitstable_row_size(intable); rowbuf = buffered_read_new(R, 1000, NR, refill_rowbuffer, intable); if (fitstable_read_extension(intable, 1)) { ERROR("Failed to find RA and DEC columns (called \"%s\" and \"%s\" in the FITS file)", racol, deccol); exit(-1); } for (r=0; r<NR; r++) { int hp = -1; double ra, dec; int j; double* rd; void* rowdata; void* rdata; if (r && ((r % 100000) == 0)) { logmsg("Reading row %i of %i\n", r, NR); } //printf("reading RA,Dec for row %i\n", r); rd = fitstable_next_struct(intable); ra = rd[0]; dec = rd[1]; logverb("row %i: ra,dec %g,%g\n", r, ra, dec); if (margin == 0) { hp = radecdegtohealpix(ra, dec, nside); logverb(" --> healpix %i\n", hp); } else { double xyz[3]; anbool gotit = FALSE; double d2; if (!hps) hps = il_new(4); radecdeg2xyzarr(ra, dec, xyz); for (j=0; j<NHP; j++) { d2 = distsq(xyz, mincaps[j].xyz, 3); if (d2 <= mincaps[j].r2) { logverb(" -> in mincap %i (dist %g vs %g)\n", j, sqrt(d2), sqrt(mincaps[j].r2)); il_append(hps, j); gotit = TRUE; break; } } if (!gotit) { for (j=0; j<NHP; j++) { d2 = distsq(xyz, maxcaps[j].xyz, 3); if (d2 <= maxcaps[j].r2) { logverb(" -> in maxcap %i (dist %g vs %g)\n", j, sqrt(d2), sqrt(maxcaps[j].r2)); if (healpix_within_range_of_xyz(j, nside, xyz, margin)) { logverb(" -> and within range.\n"); il_append(hps, j); } } } } //hps = healpix_rangesearch_radec(ra, dec, margin, nside, hps); logverb(" --> healpixes: ["); for (j=0; j<il_size(hps); j++) logverb(" %i", il_get(hps, j)); logverb(" ]\n"); } //printf("Reading rowdata for row %i\n", r); rowdata = buffered_read(rowbuf); assert(rowdata); j=0; while (1) { if (hps) { if (j >= il_size(hps)) break; hp = il_get(hps, j); j++; } assert(hp < NHP); assert(hp >= 0); if (!outtables[hp]) { char* outfn; fitstable_t* out; // MEMLEAK the output filename. You'll live. asprintf_safe(&outfn, outfnpat, hp); logmsg("Opening output file \"%s\"...\n", outfn); out = fitstable_open_for_writing(outfn); if (!out) { ERROR("Failed to open output table \"%s\"", outfn); exit(-1); } // Set the output table structure. if (cols) { fitstable_add_fits_columns_as_struct3(intable, out, cols, 0); } else fitstable_add_fits_columns_as_struct2(intable, out); if (backref) { tfits_type i16type; tfits_type i32type; // R = fitstable_row_size(intable); int off = R; i16type = fitscolumn_i16_type(); i32type = fitscolumn_i32_type(); fitstable_add_read_column_struct(out, i16type, 1, off, i16type, "backref_file", TRUE); off += sizeof(int16_t); fitstable_add_read_column_struct(out, i32type, 1, off, i32type, "backref_index", TRUE); } //printf("Output table:\n"); //fitstable_print_columns(out); if (fitstable_write_primary_header(out) || fitstable_write_header(out)) { ERROR("Failed to write output file headers for \"%s\"", outfn); exit(-1); } outtables[hp] = out; } if (backref) { int16_t brfile; int32_t brind; if (!padrowdata) { padrowdata = malloc(R + sizeof(int16_t) + sizeof(int32_t)); assert(padrowdata); } // convert to FITS endian brfile = htons(i); brind = htonl(r); // add backref data to rowdata memcpy(padrowdata, rowdata, R); memcpy(padrowdata + R, &brfile, sizeof(int16_t)); memcpy(padrowdata + R + sizeof(int16_t), &brind, sizeof(int32_t)); rdata = padrowdata; } else { rdata = rowdata; } if (cols) { if (fitstable_write_struct_noflip(outtables[hp], rdata)) { ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp); } } else { if (fitstable_write_row_data(outtables[hp], rdata)) { ERROR("Failed to copy a row of data from input table \"%s\" to output healpix %i", infn, hp); } } if (!hps) break; } if (hps) il_remove_all(hps); } buffered_read_free(rowbuf); // wack... buffered_read_free() just frees its internal buffer, // not the "rowbuf" struct itself. // who wrote this crazy code? Oh, me of 5 years ago. Jerk. free(rowbuf); fitstable_close(intable); il_free(hps); if (tempfn) { logverb("Removing temp file %s\n", tempfn); if (unlink(tempfn)) { SYSERROR("Failed to unlink() temp file \"%s\"", tempfn); } tempfn = NULL; } // fix headers so that the files are valid at this point. for (ii=0; ii<NHP; ii++) { if (!outtables[ii]) continue; off_t offset = ftello(outtables[ii]->fid); if (fitstable_fix_header(outtables[ii])) { ERROR("Failed to fix header for healpix %i after reading input file \"%s\"", ii, originfn); exit(-1); } fseeko(outtables[ii]->fid, offset, SEEK_SET); } if (padrowdata) { free(padrowdata); padrowdata = NULL; } } for (i=0; i<NHP; i++) { if (!outtables[i]) continue; if (fitstable_fix_header(outtables[i]) || fitstable_fix_primary_header(outtables[i]) || fitstable_close(outtables[i])) { ERROR("Failed to close output table for healpix %i", i); exit(-1); } } free(outtables); sl_free2(infns); sl_free2(cols); free(mincaps); free(maxcaps); return 0; }
int main(int argc, char **argv) { int argchar; double ra=HUGE_VAL, dec=HUGE_VAL, radius=HUGE_VAL; int loglvl = LOG_MSG; char** myargs; int nmyargs; int i; char* outfn = NULL; fitstable_t* table = NULL; while ((argchar = getopt (argc, argv, OPTIONS)) != -1) switch (argchar) { case 'o': outfn = optarg; break; case 'r': ra = atof(optarg); break; case 'd': dec = atof(optarg); break; case 'R': radius = atof(optarg); break; case 'v': loglvl++; break; case '?': fprintf(stderr, "Unknown option `-%c'.\n", optopt); case 'h': default: printHelp(argv[0]); return -1; } log_init(loglvl); nmyargs = argc - optind; myargs = argv + optind; if (nmyargs < 1) { printHelp(argv[0]); exit(-1); } if (ra == HUGE_VAL || dec == HUGE_VAL || radius == HUGE_VAL) { printHelp(argv[0]); exit(-1); } if (outfn) { table = fitstable_open_for_writing(outfn); if (!table) { ERROR("Failed to open output table"); exit(-1); } if (fitstable_write_primary_header(table)) { ERROR("Failed to write primary header of output table"); exit(-1); } } for (i=0; i<nmyargs; i++) { char* indexfn = myargs[i]; index_t index; sl* cols; int* inds; double* radecs; int N; int j; fitstable_t* tagtable = NULL; logmsg("Reading index \"%s\"...\n", indexfn); if (!index_load(indexfn, 0, &index)) { ERROR("Failed to read index \"%s\"", indexfn); continue; } logmsg("Index %s: id %i, healpix %i (nside %i), %i stars, %i quads, dimquads=%i, scales %g to %g arcmin.\n", index.indexname, index.indexid, index.healpix, index.hpnside, index.nstars, index.nquads, index.dimquads, arcsec2arcmin(index.index_scale_lower), arcsec2arcmin(index.index_scale_upper)); cols = startree_get_tagalong_column_names(index.starkd, NULL); { char* colstr = sl_join(cols, ", "); logmsg("Tag-along columns: %s\n", colstr); free(colstr); } logmsg("Searching for stars around RA,Dec (%g, %g), radius %g deg.\n", ra, dec, radius); startree_search_for_radec(index.starkd, ra, dec, radius, NULL, &radecs, &inds, &N); logmsg("Found %i stars\n", N); if (table) { int tagsize; int rowsize; char* rowbuf = NULL; if (i > 0) { fitstable_next_extension(table); fitstable_clear_table(table); } tagtable = startree_get_tagalong(index.starkd); if (tagtable) { fitstable_add_fits_columns_as_struct(tagtable); logverb("Input tag-along table:\n"); if (log_get_level() >= LOG_VERB) fitstable_print_columns(tagtable); fitstable_copy_columns(tagtable, table); } tagsize = fitstable_get_struct_size(table); debug("tagsize=%i\n", tagsize); // Add RA,Dec at the end of the row... fitstable_add_write_column_struct(table, fitscolumn_double_type(), 1, tagsize, fitscolumn_double_type(), "RA", "degrees"); fitstable_add_write_column_struct(table, fitscolumn_double_type(), 1, tagsize + sizeof(double), fitscolumn_double_type(), "DEC", "degrees"); rowsize = fitstable_get_struct_size(table); assert(rowsize == tagsize + 2*sizeof(double)); debug("rowsize=%i\n", rowsize); rowbuf = malloc(rowsize); logverb("Output table:\n"); if (log_get_level() >= LOG_VERB) fitstable_print_columns(table); if (fitstable_write_header(table)) { ERROR("Failed to write header of output table"); exit(-1); } for (j=0; j<N; j++) { if (tagtable) { if (fitstable_read_struct(tagtable, inds[j], rowbuf)) { ERROR("Failed to read row %i of tag-along table", inds[j]); exit(-1); } } // Add RA,Dec to end of struct... memcpy(rowbuf + tagsize, radecs+2*j+0, sizeof(double)); memcpy(rowbuf + tagsize + sizeof(double), radecs+2*j+1, sizeof(double)); if (fitstable_write_struct(table, rowbuf)) { ERROR("Failed to write row %i of output", j); exit(-1); } } free(rowbuf); if (fitstable_fix_header(table)) { ERROR("Failed to fix header of output table"); exit(-1); } } sl_free2(cols); free(radecs); free(inds); index_close(&index); } if (table) { if (fitstable_close(table)) { ERROR("Failed to close output table"); exit(-1); } } return 0; }
int uniformize_catalog(fitstable_t* intable, fitstable_t* outtable, const char* racol, const char* deccol, const char* sortcol, anbool sort_ascending, double sort_min_cut, // ? Or do this cut in a separate process? int bighp, int bignside, int nmargin, // uniformization nside. int Nside, double dedup_radius, int nsweeps, char** args, int argc) { anbool allsky; intmap_t* starlists; int NHP; anbool dense = FALSE; double dedupr2 = 0.0; tfits_type dubl; int N; int* inorder = NULL; int* outorder = NULL; int outi; double *ra = NULL, *dec = NULL; il* myhps = NULL; int i,j,k; int nkeep = nsweeps; int noob = 0; int ndup = 0; struct oh_token token; int* npersweep = NULL; qfits_header* outhdr = NULL; double *sortval = NULL; if (bignside == 0) bignside = 1; allsky = (bighp == -1); if (Nside % bignside) { ERROR("Fine healpixelization Nside must be a multiple of the coarse healpixelization Nside"); return -1; } if (Nside > HP_MAX_INT_NSIDE) { ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE); return -1; } NHP = 12 * Nside * Nside; logverb("Healpix Nside: %i, # healpixes on the whole sky: %i\n", Nside, NHP); if (!allsky) { logverb("Creating index for healpix %i, nside %i\n", bighp, bignside); logverb("Number of healpixes: %i\n", ((Nside/bignside)*(Nside/bignside))); } logverb("Healpix side length: %g arcmin.\n", healpix_side_length_arcmin(Nside)); dubl = fitscolumn_double_type(); if (!racol) racol = "RA"; ra = fitstable_read_column(intable, racol, dubl); if (!ra) { ERROR("Failed to find RA column (%s) in table", racol); return -1; } if (!deccol) deccol = "DEC"; dec = fitstable_read_column(intable, deccol, dubl); if (!dec) { ERROR("Failed to find DEC column (%s) in table", deccol); free(ra); return -1; } N = fitstable_nrows(intable); logverb("Have %i objects\n", N); // FIXME -- argsort and seek around the input table, and append to // starlists in order; OR read from the input table in sequence and // sort in the starlists? if (sortcol) { logverb("Sorting by %s...\n", sortcol); sortval = fitstable_read_column(intable, sortcol, dubl); if (!sortval) { ERROR("Failed to read sorting column \"%s\"", sortcol); free(ra); free(dec); return -1; } inorder = permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, NULL, N); if (sort_min_cut > -HUGE_VAL) { logverb("Cutting to %s > %g...\n", sortcol, sort_min_cut); // Cut objects with sortval < sort_min_cut. if (sort_ascending) { // skipped objects are at the front -- find the first obj // to keep for (i=0; i<N; i++) if (sortval[inorder[i]] > sort_min_cut) break; // move the "inorder" indices down. if (i) memmove(inorder, inorder+i, (N-i)*sizeof(int)); N -= i; } else { // skipped objects are at the end -- find the last obj to keep. for (i=N-1; i>=0; i--) if (sortval[inorder[i]] > sort_min_cut) break; N = i+1; } logverb("Cut to %i objects\n", N); } //free(sortval); } token.nside = bignside; token.finenside = Nside; token.hp = bighp; if (!allsky && nmargin) { int bigbighp, bighpx, bighpy; //int ninside; il* seeds = il_new(256); logverb("Finding healpixes in range...\n"); healpix_decompose_xy(bighp, &bigbighp, &bighpx, &bighpy, bignside); //ninside = (Nside/bignside)*(Nside/bignside); // Prime the queue with the fine healpixes that are on the // boundary of the big healpix. for (i=0; i<((Nside / bignside) - 1); i++) { // add (i,0), (i,max), (0,i), and (0,max) healpixes int xx = i + bighpx * (Nside / bignside); int yy = i + bighpy * (Nside / bignside); int y0 = bighpy * (Nside / bignside); // -1 prevents us from double-adding the corners. int y1 =(1 + bighpy)* (Nside / bignside) - 1; int x0 = bighpx * (Nside / bignside); int x1 =(1 + bighpx)* (Nside / bignside) - 1; assert(xx < Nside); assert(yy < Nside); assert(x0 < Nside); assert(x1 < Nside); assert(y0 < Nside); assert(y1 < Nside); il_append(seeds, healpix_compose_xy(bigbighp, xx, y0, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, xx, y1, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x0, yy, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x1, yy, Nside)); } logmsg("Number of boundary healpixes: %zu (Nside/bignside = %i)\n", il_size(seeds), Nside/bignside); myhps = healpix_region_search(-1, seeds, Nside, NULL, NULL, outside_healpix, &token, nmargin); logmsg("Number of margin healpixes: %zu\n", il_size(myhps)); il_free(seeds); il_sort(myhps, TRUE); // DEBUG il_check_consistency(myhps); il_check_sorted_ascending(myhps, TRUE); } dedupr2 = arcsec2distsq(dedup_radius); starlists = intmap_new(sizeof(int32_t), nkeep, 0, dense); logverb("Placing stars in grid cells...\n"); for (i=0; i<N; i++) { int hp; bl* lst; int32_t j32; anbool oob; if (inorder) { j = inorder[i]; //printf("Placing star %i (%i): sort value %s = %g, RA,Dec=%g,%g\n", i, j, sortcol, sortval[j], ra[j], dec[j]); } else j = i; hp = radecdegtohealpix(ra[j], dec[j], Nside); //printf("HP %i\n", hp); // in bounds? oob = FALSE; if (myhps) { oob = (outside_healpix(hp, &token) && !il_sorted_contains(myhps, hp)); } else if (!allsky) { oob = (outside_healpix(hp, &token)); } if (oob) { //printf("out of bounds.\n"); noob++; continue; } lst = intmap_find(starlists, hp, TRUE); /* printf("list has %i existing entries.\n", bl_size(lst)); for (k=0; k<bl_size(lst); k++) { bl_get(lst, k, &j32); printf(" %i: index %i, %s = %g\n", k, j32, sortcol, sortval[j32]); } */ // is this list full? if (nkeep && (bl_size(lst) >= nkeep)) { // Here we assume we're working in sorted order: once the list is full we're done. //printf("Skipping: list is full.\n"); continue; } if ((dedupr2 > 0.0) && is_duplicate(hp, ra[j], dec[j], Nside, starlists, ra, dec, dedupr2)) { //printf("Skipping: duplicate\n"); ndup++; continue; } // Add the new star (by index) j32 = j; bl_append(lst, &j32); } logverb("%i outside the healpix\n", noob); logverb("%i duplicates\n", ndup); il_free(myhps); myhps = NULL; free(inorder); inorder = NULL; free(ra); ra = NULL; free(dec); dec = NULL; outorder = malloc(N * sizeof(int)); outi = 0; npersweep = calloc(nsweeps, sizeof(int)); for (k=0; k<nsweeps; k++) { int starti = outi; int32_t j32; for (i=0;; i++) { bl* lst; int hp; if (!intmap_get_entry(starlists, i, &hp, &lst)) break; if (bl_size(lst) <= k) continue; bl_get(lst, k, &j32); outorder[outi] = j32; //printf("sweep %i, cell #%i, hp %i, star %i, %s = %g\n", k, i, hp, j32, sortcol, sortval[j32]); outi++; } logmsg("Sweep %i: %i stars\n", k+1, outi - starti); npersweep[k] = outi - starti; if (sortcol) { // Re-sort within this sweep. permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, outorder + starti, npersweep[k]); /* for (i=0; i<npersweep[k]; i++) { printf(" within sweep %i: star %i, j=%i, %s=%g\n", k, i, outorder[starti + i], sortcol, sortval[outorder[starti + i]]); } */ } } intmap_free(starlists); starlists = NULL; ////// free(sortval); sortval = NULL; logmsg("Total: %i stars\n", outi); N = outi; outhdr = fitstable_get_primary_header(outtable); if (allsky) qfits_header_add(outhdr, "ALLSKY", "T", "All-sky catalog.", NULL); BOILERPLATE_ADD_FITS_HEADERS(outhdr); qfits_header_add(outhdr, "HISTORY", "This file was generated by the command-line:", NULL, NULL); fits_add_args(outhdr, args, argc); qfits_header_add(outhdr, "HISTORY", "(end of command line)", NULL, NULL); fits_add_long_history(outhdr, "uniformize-catalog args:"); fits_add_long_history(outhdr, " RA,Dec columns: %s,%s", racol, deccol); fits_add_long_history(outhdr, " sort column: %s", sortcol); fits_add_long_history(outhdr, " sort direction: %s", sort_ascending ? "ascending" : "descending"); if (sort_ascending) fits_add_long_history(outhdr, " (ie, for mag-like sort columns)"); else fits_add_long_history(outhdr, " (ie, for flux-like sort columns)"); fits_add_long_history(outhdr, " uniformization nside: %i", Nside); fits_add_long_history(outhdr, " (ie, side length ~ %g arcmin)", healpix_side_length_arcmin(Nside)); fits_add_long_history(outhdr, " deduplication scale: %g arcsec", dedup_radius); fits_add_long_history(outhdr, " number of sweeps: %i", nsweeps); fits_header_add_int(outhdr, "NSTARS", N, "Number of stars."); fits_header_add_int(outhdr, "HEALPIX", bighp, "Healpix covered by this catalog, with Nside=HPNSIDE"); fits_header_add_int(outhdr, "HPNSIDE", bignside, "Nside of HEALPIX."); fits_header_add_int(outhdr, "CUTNSIDE", Nside, "uniformization scale (healpix nside)"); fits_header_add_int(outhdr, "CUTMARG", nmargin, "margin size, in healpixels"); //qfits_header_add(outhdr, "CUTBAND", cutband, "band on which the cut was made", NULL); fits_header_add_double(outhdr, "CUTDEDUP", dedup_radius, "deduplication radius [arcsec]"); fits_header_add_int(outhdr, "CUTNSWEP", nsweeps, "number of sweeps"); //fits_header_add_double(outhdr, "CUTMINMG", minmag, "minimum magnitude"); //fits_header_add_double(outhdr, "CUTMAXMG", maxmag, "maximum magnitude"); for (k=0; k<nsweeps; k++) { char key[64]; sprintf(key, "SWEEP%i", (k+1)); fits_header_add_int(outhdr, key, npersweep[k], "# stars added"); } free(npersweep); if (fitstable_write_primary_header(outtable)) { ERROR("Failed to write primary header"); return -1; } // Write output. fitstable_add_fits_columns_as_struct2(intable, outtable); if (fitstable_write_header(outtable)) { ERROR("Failed to write output table header"); return -1; } logmsg("Writing output...\n"); logverb("Row size: %i\n", fitstable_row_size(intable)); if (fitstable_copy_rows_data(intable, outorder, N, outtable)) { ERROR("Failed to copy rows from input table to output"); return -1; } if (fitstable_fix_header(outtable)) { ERROR("Failed to fix output table header"); return -1; } free(outorder); return 0; }
int loadphotondata(char *filename, char *passfile) { fitstable_t* tab; qfits_header* hdr; tfits_type flt = fitscolumn_float_type(), chtype = fitscolumn_char_type(), dbltype=fitscolumn_double_type(); float *localphotondata[NPHOTON_LOADDATA], *galdisrsp=NULL, *isodisrsp=NULL; double *localphotontime; char *conversion_type=NULL; char buffer[255]; int i, j, ncurr, ndiff, nresp=0, conv_type, psf_cnt, psf_ind; double rmax2, ehold, rad, r2hold; if ( (tab = fitstable_open(filename))==NULL) { printf("# Cannot open %s in loadphotondata() %s:%d\n",filename,__FILE__,__LINE__); return -1; } hdr = fitstable_get_primary_header(tab); hdr = fitstable_get_header(tab); ncurr = fitstable_nrows(tab); if (debug) printf("# filename= %s ncurr= %d\n",filename,ncurr); /* load in the energy, theta, ra and dec */ for (i=0; i<NPHOTON_LOADDATA; i++) { if ( (localphotondata[i]= (float *) fitstable_read_column_array(tab, photon_colname[i], flt))==NULL) { printf("# vector for %s is NULL in loadphotondata() %s:%d\n",photon_colname[i],__FILE__,__LINE__); for (j=0; j<i; j++) { SAFEFREE( localphotondata[i]); } fitstable_close(tab); return -1; } } /* load in conversion type */ if ( (conversion_type= (char *) fitstable_read_column(tab, "CONVERSION_TYPE", chtype))==NULL) { printf("# array for conversion_type is NULL in loadphotondata() %s:%d\n",__FILE__,__LINE__); for (i=0; i<NPHOTON_LOADDATA; i++) { SAFEFREE( localphotondata[i]); } fitstable_close(tab); return -1; } /* load in photon arrival time */ if ( (localphotontime= (double *) fitstable_read_column(tab, "TIME", dbltype))==NULL) { printf("# array for localphotontime is NULL in loadphotondata() %s:%d\n",__FILE__,__LINE__); for (i=0; i<NPHOTON_LOADDATA; i++) { SAFEFREE( localphotondata[i]); } SAFEFREE( conversion_type); fitstable_close(tab); return -1; } /* load in the diffuse response functions that match the passfile */ ndiff = qfits_header_getint(hdr, "NDIFRSP",-1); for (i=0; i<ndiff; i++) { sprintf(buffer,"DIFRSP%d",i); char *respname = qfits_header_getstr(hdr, buffer); if (debug) printf("# %d %s = %s\n",i,buffer,respname); if (strcasestr(respname,passfile)) { if (strcasestr(respname,"gll")) { if (debug) printf("# galactic is %s\n",buffer); if ( (galdisrsp= (float *) fitstable_read_column_array(tab, buffer, flt))==NULL) { printf("# array for %s is NULL in loadphotondata() %s:%d\n",buffer,__FILE__,__LINE__); for (j=0; j<i; j++) { SAFEFREE( localphotondata[i]); } SAFEFREE( conversion_type); SAFEFREE(localphotontime); SAFEFREE( galdisrsp); SAFEFREE( isodisrsp); fitstable_close(tab); return -1; } nresp++; } if (strcasestr(respname,"iso")) { if (debug) printf("# iso is %s\n",buffer); if ( (isodisrsp= (float *) fitstable_read_column_array(tab, buffer, flt))==NULL) { printf("# array for %s is NULL in loadphotondata() %s:%d\n",buffer,__FILE__,__LINE__); for (j=0; j<i; j++) { SAFEFREE( localphotondata[i]); } SAFEFREE( conversion_type); SAFEFREE(localphotontime); SAFEFREE( galdisrsp); SAFEFREE( isodisrsp); fitstable_close(tab); return -1; } nresp++; } } } if (nresp<2) { printf("Could not find the matching response functions in loadphotondata() %s:%d.\n",__FILE__,__LINE__); return -1; } if (alloc_globals(ncurr+ntot)) { SAFEFREE(localphotontime); SAFEFREE(conversion_type); SAFEFREE(data); for (i=0; i<NPHOTON_LOADDATA; i++) { SAFEFREE(localphotondata[i]); } return -1; } fitstable_close(tab); j=0; for (i=0; i<ncurr; i++) { ehold=localphotondata[ENERGY][i]; if (ehold>=e0 && ehold<=e1) { double ra, dec; __sincospi((ra=localphotondata[RA][i])/180.0,data+d*(j+ntot)+1,data+d*(j+ntot)); __sincospi((dec=localphotondata[DEC][i])/180.0,data+d*(j+ntot)+2,&rad); data[d*(ntot+j)]*=rad; data[d*(ntot+j)+1]*=rad; if (ehold<energy_min) energy_min=ehold; /* calculate RMAX2 for this photon */ conv_type=conversion_type[i]; rmax2=photondata[COSTHETA][j+ntot]=cos(localphotondata[THETA][i]*M_PI/180.0); if (debug>3) printf("# costheta= %g\n",rmax2); #if 1 /* fast way --- evenly spaced in log energy and costheta -- no interpolation */ // rmax2=psfds.psfdata[conv_type][RMAX2][(int) ((rmax2-psfds.mumin)/psfds.mustep)*psfds.nE + (int) ((log10(ehold)-psfds.lemin)/psfds.lestep)]; rmax2=rmax2_funk(psfds,conv_type,rmax2,ehold); #else /* slow way --- find the right bin -- no interpolation */ for (psf_cnt=0; psf_cnt<psfds.nE; psf_cnt++) { if (ehold>psfds.psfdata[conv_type][ENERG_LO][psf_cnt] && ehold<=psfds.psfdata[conv_type][ENERG_HI][psf_cnt]) { psf_ind=psf_cnt; break; } } for (psf_cnt=0; psf_cnt<psfds.nMu; psf_cnt++) { if (rmax2>psfds.psfdata[conv_type][CTHETA_LO][psf_cnt] && rmax2<=psfds.psfdata[conv_type][CTHETA_HI][psf_cnt]) { psf_ind+=psf_cnt*psfds.nE; break; } } rmax2=psfds.psfdata[conv_type][RMAX2][psf_ind]; #endif if (debug>3) printf("# rmax2= %g\n",rmax2); if (conv_type) { /* back conversion */ r2hold=SPE_squared_back(ehold)*rmax2; data[d*(ntot+j)+3]=-(r2hold>4 ? 4 : r2hold); } else { /* front conversion */ r2hold=SPE_squared_front(ehold)*rmax2; data[d*(ntot+j)+3]=(r2hold>4 ? 4 : r2hold); } if (debug>3) printf("# %g rmax2= %g %g %g\n",ehold,data[d*(ntot+j)+3],SPE_squared_front(ehold),localphotondata[THETA][i]); photondata[ENERGY][j+ntot]=ehold; photondata[DIFRSP_GAL][j+ntot]=galdisrsp[i]; photondata[DIFRSP_ISO][j+ntot]=isodisrsp[i]; r2hold=aeffds.aeffdata[conv_type][AEFF_EFF_AREA] [(int) ((photondata[COSTHETA][j+ntot]-aeffds.mumin)/aeffds.mustep)*aeffds.nE + (int) ((log10(ehold)-aeffds.lemin)/aeffds.lestep)]; if (r2hold<EFFAREAMIN) r2hold=EFFAREAMIN; photondata[EFFAREA][j+ntot]=r2hold; /* calculate the effective area integral */ r2hold=calcefft(ra,dec,ehold,&aeffds,<cubeds); if (r2hold<EFFAREATMIN) r2hold=EFFAREATMIN; photondata[EFFAREAT][j+ntot]=r2hold; photontime[j+ntot]=localphotontime[i]; j++; } } for (i=0; i<NPHOTON_LOADDATA; i++) { SAFEFREE(localphotondata[i]); } SAFEFREE( galdisrsp); SAFEFREE( isodisrsp); SAFEFREE(conversion_type); SAFEFREE(localphotontime); ntot+=j; if (debug) printf("# ntot= %d j= %d\n",ntot,j); if (j!=ncurr) { if (alloc_globals(ncurr+ntot)) { SAFEFREE(data); for (i=0; i<NPHOTON_LOADDATA; i++) { SAFEFREE(localphotondata[i]); } return -1; } } return 0; }
int main(int argc, char** args) { int argchar; char* basename; char* outfn = NULL; index_t* index; quadfile* qf; rdlist_t* rdls; startree_t* skdt = NULL; anbool addradius = FALSE; int i; int radcolumn = -1; while ((argchar = getopt (argc, args, OPTIONS)) != -1) switch (argchar) { case 'R': addradius = TRUE; break; case 'r': outfn = optarg; break; case 'h': print_help(args[0]); exit(0); } if (!outfn || (optind == argc)) { print_help(args[0]); exit(-1); } rdls = rdlist_open_for_writing(outfn); if (!rdls) { fprintf(stderr, "Failed to open RDLS file %s for output.\n", outfn); exit(-1); } if (rdlist_write_primary_header(rdls)) { fprintf(stderr, "Failed to write RDLS header.\n"); exit(-1); } if (addradius) { radcolumn = rdlist_add_tagalong_column(rdls, fitscolumn_double_type(), 1, fitscolumn_double_type(), "QUADRADIUS", "deg"); } for (; optind<argc; optind++) { //int Nstars; int dimquads; basename = args[optind]; printf("Reading files with basename %s\n", basename); index = index_load(basename, 0, NULL); if (!index) { fprintf(stderr, "Failed to read index with base name \"%s\"\n", basename); exit(-1); } qf = index->quads; skdt = index->starkd; //Nstars = startree_N(skdt); if (rdlist_write_header(rdls)) { fprintf(stderr, "Failed to write new RDLS field header.\n"); exit(-1); } dimquads = quadfile_dimquads(qf); printf("Reading quads...\n"); for (i=0; i<qf->numquads; i++) { unsigned int stars[dimquads]; double axyz[3], bxyz[3]; double midab[3]; double radec[2]; if (!(i % 200000)) { printf("."); fflush(stdout); } quadfile_get_stars(qf, i, stars); startree_get(skdt, stars[0], axyz); startree_get(skdt, stars[1], bxyz); star_midpoint(midab, axyz, bxyz); xyzarr2radecdegarr(midab, radec); if (rdlist_write_one_radec(rdls, radec[0], radec[1])) { fprintf(stderr, "Failed to write a RA,Dec entry.\n"); exit(-1); } if (addradius) { double rad = arcsec2deg(distsq2arcsec(distsq(midab, axyz, 3))); if (rdlist_write_tagalong_column(rdls, radcolumn, i, 1, &rad, 0)) { fprintf(stderr, "Failed to write quad radius.\n"); exit(-1); } } } printf("\n"); index_close(index); if (rdlist_fix_header(rdls)) { fprintf(stderr, "Failed to fix RDLS field header.\n"); exit(-1); } rdlist_next_field(rdls); } if (rdlist_fix_primary_header(rdls) || rdlist_close(rdls)) { fprintf(stderr, "Failed to close RDLS file.\n"); exit(-1); } return 0; }
double* startree_get_data_column(startree_t* s, const char* colname, const int* inds, int N) { return get_data_column(s, colname, inds, N, fitscolumn_double_type()); }
int main(int argc, char *argv[]) { int argchar; char* progname = argv[0]; index_t* index; fitstable_t* table; char* indexfn = NULL; char* outfn = NULL; int i, D, N; while ((argchar = getopt (argc, argv, OPTIONS)) != -1) { switch (argchar) { case 'i': indexfn = optarg; break; case 'o': outfn = optarg; break; default: case 'h': printHelp(progname); exit(0); } } if (!(indexfn && outfn)) { printHelp(progname); exit(-1); } index = index_load(indexfn, 0, NULL); if (!index) { ERROR("Failed to open index"); exit(-1); } table = fitstable_open_for_writing(outfn); if (!table) { ERROR("Failed to open output file for writing"); exit(-1); } D = index_get_quad_dim(index); fitstable_add_write_column_array(table, fitscolumn_i32_type(), D, "quads", ""); if (fitstable_write_primary_header(table) || fitstable_write_header(table)) { ERROR("Failed to write headers"); exit(-1); } N = index_nquads(index); for (i=0; i<N; i++) { unsigned int quad[D]; quadfile_get_stars(index->quads, i, quad); if (fitstable_write_row(table, quad)) { ERROR("Failed to write quad %i", i); exit(-1); } } if (fitstable_fix_header(table)) { ERROR("Failed to fix quad header"); exit(-1); } fitstable_next_extension(table); fitstable_clear_table(table); // write star RA,Dec s. fitstable_add_write_column(table, fitscolumn_double_type(), "RA", "deg"); fitstable_add_write_column(table, fitscolumn_double_type(), "Dec", "deg"); if (fitstable_write_header(table)) { ERROR("Failed to write star header"); exit(-1); } N = index_nstars(index); for (i=0; i<N; i++) { double xyz[3]; double ra, dec; startree_get(index->starkd, i, xyz); xyzarr2radecdeg(xyz, &ra, &dec); if (fitstable_write_row(table, &ra, &dec)) { ERROR("Failed to write star %i", i); exit(-1); } } if (fitstable_fix_header(table) || fitstable_close(table)) { ERROR("Failed to fix star header and close"); exit(-1); } index_close(index); return 0; }