int plot_healpix_plot(const char* command, cairo_t* cairo, plot_args_t* pargs, void* baton) { plothealpix_t* args = (plothealpix_t*)baton; double ra,dec,rad; il* hps; int i; double hpstep; int minx[12], maxx[12], miny[12], maxy[12]; plotstuff_builtin_apply(cairo, pargs); if (plotstuff_get_radec_center_and_radius(pargs, &ra, &dec, &rad)) { ERROR("Failed to get RA,Dec center and radius"); return -1; } hps = healpix_rangesearch_radec(ra, dec, rad, args->nside, NULL); logmsg("Found %zu healpixes in range.\n", il_size(hps)); hpstep = args->nside * args->stepsize * plotstuff_pixel_scale(pargs) / 60.0 / healpix_side_length_arcmin(args->nside); hpstep = MIN(1, hpstep); logmsg("Taking steps of %g in healpix space\n", hpstep); // For each of the 12 top-level healpixes, find the range of healpixes covered by this image. for (i=0; i<12; i++) { maxx[i] = maxy[i] = -1; minx[i] = miny[i] = args->nside+1; } for (i=0; i<il_size(hps); i++) { int hp = il_get(hps, i); int hpx, hpy; int bighp; healpix_decompose_xy(hp, &bighp, &hpx, &hpy, args->nside); logverb(" hp %i: bighp %i, x,y (%i,%i)\n", i, bighp, hpx, hpy); minx[bighp] = MIN(minx[bighp], hpx); maxx[bighp] = MAX(maxx[bighp], hpx); miny[bighp] = MIN(miny[bighp], hpy); maxy[bighp] = MAX(maxy[bighp], hpy); } il_free(hps); for (i=0; i<12; i++) { int hx,hy; int hp; double d, frac; double x,y; if (maxx[i] == -1) continue; logverb("Big healpix %i: x range [%i, %i], y range [%i, %i]\n", i, minx[i], maxx[i], miny[i], maxy[i]); for (hy = miny[i]; hy <= maxy[i]; hy++) { logverb(" y=%i\n", hy); for (d=minx[i]; d<=maxx[i]; d+=hpstep) { hx = floor(d); frac = d - hx; hp = healpix_compose_xy(i, hx, hy, args->nside); healpix_to_radecdeg(hp, args->nside, frac, 0.0, &ra, &dec); if (!plotstuff_radec2xy(pargs, ra, dec, &x, &y)) continue; if (d == minx[i]) cairo_move_to(pargs->cairo, x, y); else cairo_line_to(pargs->cairo, x, y); } cairo_stroke(pargs->cairo); } for (hx = minx[i]; hx <= maxx[i]; hx++) { for (d=miny[i]; d<=maxy[i]; d+=hpstep) { hy = floor(d); frac = d - hy; hp = healpix_compose_xy(i, hx, hy, args->nside); healpix_to_radecdeg(hp, args->nside, 0.0, frac, &ra, &dec); if (!plotstuff_radec2xy(pargs, ra, dec, &x, &y)) continue; if (d == miny[i]) cairo_move_to(pargs->cairo, x, y); else cairo_line_to(pargs->cairo, x, y); } cairo_stroke(pargs->cairo); } } return 0; }
int hpquads(startree_t* starkd, codefile_t* codes, quadfile_t* quads, int Nside, double scale_min_arcmin, double scale_max_arcmin, int dimquads, int passes, int Nreuses, int Nloosen, int id, anbool scanoccupied, void* sort_data, int (*sort_func)(const void*, const void*), int sort_size, char** args, int argc) { hpquads_t myhpquads; hpquads_t* me = &myhpquads; int i; int pass; anbool circle = TRUE; double radius2; il* hptotry; int Nhptotry = 0; int nquads; double hprad; double quadscale; int skhp, sknside; qfits_header* qhdr; qfits_header* chdr; int N; int dimcodes; int quadsize; int NHP; memset(me, 0, sizeof(hpquads_t)); if (Nside > HP_MAX_INT_NSIDE) { ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE); return -1; } if (Nreuses > 255) { ERROR("Error, reuse (-r) must be less than 256"); return -1; } me->Nside = Nside; me->dimquads = dimquads; NHP = 12 * Nside * Nside; dimcodes = dimquad2dimcode(dimquads); quadsize = sizeof(unsigned int) * dimquads; logmsg("Nside=%i. Nside^2=%i. Number of healpixes=%i. Healpix side length ~ %g arcmin.\n", me->Nside, me->Nside*me->Nside, NHP, healpix_side_length_arcmin(me->Nside)); me->sort_data = sort_data; me->sort_func = sort_func; me->sort_size = sort_size; tic(); me->starkd = starkd; N = startree_N(me->starkd); logmsg("Star tree contains %i objects.\n", N); // get the "HEALPIX" header from the skdt... skhp = qfits_header_getint(startree_header(me->starkd), "HEALPIX", -1); if (skhp == -1) { if (!qfits_header_getboolean(startree_header(me->starkd), "ALLSKY", FALSE)) { logmsg("Warning: skdt does not contain \"HEALPIX\" header. Code and quad files will not contain this header either.\n"); } } // likewise "HPNSIDE" sknside = qfits_header_getint(startree_header(me->starkd), "HPNSIDE", 1); if (sknside && Nside % sknside) { logerr("Error: Nside (-n) must be a multiple of the star kdtree healpixelisation: %i\n", sknside); return -1; } if (!scanoccupied && (N*(skhp == -1 ? 1 : sknside*sknside*12) < NHP)) { logmsg("\n\n"); logmsg("NOTE, your star kdtree is sparse (has only a fraction of the stars expected)\n"); logmsg(" so you probably will get much faster results by setting the \"-E\" command-line\n"); logmsg(" flag.\n"); logmsg("\n\n"); } quads->dimquads = me->dimquads; codes->dimcodes = dimcodes; quads->healpix = skhp; codes->healpix = skhp; quads->hpnside = sknside; codes->hpnside = sknside; if (id) { quads->indexid = id; codes->indexid = id; } qhdr = quadfile_get_header(quads); chdr = codefile_get_header(codes); add_headers(qhdr, args, argc, startree_header(me->starkd), circle, passes); add_headers(chdr, args, argc, startree_header(me->starkd), circle, passes); if (quadfile_write_header(quads)) { ERROR("Couldn't write headers to quad file"); return -1; } if (codefile_write_header(codes)) { ERROR("Couldn't write headers to code file"); return -1; } quads->numstars = codes->numstars = N; me->quad_dist2_upper = arcmin2distsq(scale_max_arcmin); me->quad_dist2_lower = arcmin2distsq(scale_min_arcmin); codes->index_scale_upper = quads->index_scale_upper = distsq2rad(me->quad_dist2_upper); codes->index_scale_lower = quads->index_scale_lower = distsq2rad(me->quad_dist2_lower); me->nuses = calloc(N, sizeof(unsigned char)); // hprad = sqrt(2) * (healpix side length / 2.) hprad = arcmin2dist(healpix_side_length_arcmin(Nside)) * M_SQRT1_2; quadscale = 0.5 * sqrt(me->quad_dist2_upper); // 1.01 for a bit of safety. we'll look at a few extra stars. radius2 = square(1.01 * (hprad + quadscale)); me->radius2 = radius2; logmsg("Healpix radius %g arcsec, quad scale %g arcsec, total %g arcsec\n", distsq2arcsec(hprad*hprad), distsq2arcsec(quadscale*quadscale), distsq2arcsec(radius2)); hptotry = il_new(1024); if (scanoccupied) { logmsg("Scanning %i input stars...\n", N); for (i=0; i<N; i++) { double xyz[3]; int j; if (startree_get(me->starkd, i, xyz)) { ERROR("Failed to get star %i", i); return -1; } j = xyzarrtohealpix(xyz, Nside); il_insert_unique_ascending(hptotry, j); if (log_get_level() > LOG_VERB) { double ra,dec; if (startree_get_radec(me->starkd, i, &ra, &dec)) { ERROR("Failed to get RA,Dec for star %i\n", i); return -1; } logdebug("star %i: RA,Dec %g,%g; xyz %g,%g,%g; hp %i\n", i, ra, dec, xyz[0], xyz[1], xyz[2], j); } } logmsg("Will check %zu healpixes.\n", il_size(hptotry)); if (log_get_level() > LOG_VERB) { logdebug("Checking healpixes: [ "); for (i=0; i<il_size(hptotry); i++) logdebug("%i ", il_get(hptotry, i)); logdebug("]\n"); } } else { if (skhp == -1) { // Try all healpixes. il_free(hptotry); hptotry = NULL; Nhptotry = NHP; } else { // The star kdtree may itself be healpixed int starhp, starx, stary; // In that case, the healpixes we are interested in form a rectangle // within a big healpix. These are the coords (in [0, Nside)) of // that rectangle. int x0, x1, y0, y1; int x, y; healpix_decompose_xy(skhp, &starhp, &starx, &stary, sknside); x0 = starx * (Nside / sknside); x1 = (starx+1) * (Nside / sknside); y0 = stary * (Nside / sknside); y1 = (stary+1) * (Nside / sknside); for (y=y0; y<y1; y++) { for (x=x0; x<x1; x++) { int j = healpix_compose_xy(starhp, x, y, Nside); il_append(hptotry, j); } } assert(il_size(hptotry) == (Nside/sknside) * (Nside/sknside)); } } if (hptotry) Nhptotry = il_size(hptotry); me->quadlist = bl_new(65536, quadsize); if (Nloosen) me->retryhps = il_new(1024); for (pass=0; pass<passes; pass++) { char key[64]; int nthispass; logmsg("Pass %i of %i.\n", pass+1, passes); logmsg("Trying %i healpixes.\n", Nhptotry); nthispass = build_quads(me, Nhptotry, hptotry, Nreuses); logmsg("Made %i quads (out of %i healpixes) this pass.\n", nthispass, Nhptotry); logmsg("Made %i quads so far.\n", (me->bigquadlist ? bt_size(me->bigquadlist) : 0) + (int)bl_size(me->quadlist)); sprintf(key, "PASS%i", pass+1); fits_header_mod_int(chdr, key, nthispass, "quads created in this pass"); fits_header_mod_int(qhdr, key, nthispass, "quads created in this pass"); logmsg("Merging quads...\n"); if (!me->bigquadlist) me->bigquadlist = bt_new(quadsize, 256); for (i=0; i<bl_size(me->quadlist); i++) { void* q = bl_access(me->quadlist, i); bt_insert2(me->bigquadlist, q, FALSE, compare_quads, &me->dimquads); } bl_remove_all(me->quadlist); } il_free(hptotry); hptotry = NULL; if (Nloosen) { int R; for (R=Nreuses+1; R<=Nloosen; R++) { il* trylist; int nthispass; logmsg("Loosening reuse maximum to %i...\n", R); logmsg("Trying %zu healpixes.\n", il_size(me->retryhps)); if (!il_size(me->retryhps)) break; trylist = me->retryhps; me->retryhps = il_new(1024); nthispass = build_quads(me, il_size(trylist), trylist, R); logmsg("Made %i quads (out of %zu healpixes) this pass.\n", nthispass, il_size(trylist)); il_free(trylist); for (i=0; i<bl_size(me->quadlist); i++) { void* q = bl_access(me->quadlist, i); bt_insert2(me->bigquadlist, q, FALSE, compare_quads, &me->dimquads); } bl_remove_all(me->quadlist); } } if (me->retryhps) il_free(me->retryhps); kdtree_free_query(me->res); me->res = NULL; me->inds = NULL; me->stars = NULL; free(me->nuses); me->nuses = NULL; logmsg("Writing quads...\n"); // add the quads from the big-quadlist nquads = bt_size(me->bigquadlist); for (i=0; i<nquads; i++) { unsigned int* q = bt_access(me->bigquadlist, i); quad_write(codes, quads, q, me->starkd, me->dimquads, dimcodes); } // add the quads that were made during the final round. for (i=0; i<bl_size(me->quadlist); i++) { unsigned int* q = bl_access(me->quadlist, i); quad_write(codes, quads, q, me->starkd, me->dimquads, dimcodes); } // fix output file headers. if (quadfile_fix_header(quads)) { ERROR("Failed to fix quadfile headers"); return -1; } if (codefile_fix_header(codes)) { ERROR("Failed to fix codefile headers"); return -1; } bl_free(me->quadlist); bt_free(me->bigquadlist); toc(); logmsg("Done.\n"); return 0; }
int uniformize_catalog(fitstable_t* intable, fitstable_t* outtable, const char* racol, const char* deccol, const char* sortcol, anbool sort_ascending, double sort_min_cut, // ? Or do this cut in a separate process? int bighp, int bignside, int nmargin, // uniformization nside. int Nside, double dedup_radius, int nsweeps, char** args, int argc) { anbool allsky; intmap_t* starlists; int NHP; anbool dense = FALSE; double dedupr2 = 0.0; tfits_type dubl; int N; int* inorder = NULL; int* outorder = NULL; int outi; double *ra = NULL, *dec = NULL; il* myhps = NULL; int i,j,k; int nkeep = nsweeps; int noob = 0; int ndup = 0; struct oh_token token; int* npersweep = NULL; qfits_header* outhdr = NULL; double *sortval = NULL; if (bignside == 0) bignside = 1; allsky = (bighp == -1); if (Nside % bignside) { ERROR("Fine healpixelization Nside must be a multiple of the coarse healpixelization Nside"); return -1; } if (Nside > HP_MAX_INT_NSIDE) { ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE); return -1; } NHP = 12 * Nside * Nside; logverb("Healpix Nside: %i, # healpixes on the whole sky: %i\n", Nside, NHP); if (!allsky) { logverb("Creating index for healpix %i, nside %i\n", bighp, bignside); logverb("Number of healpixes: %i\n", ((Nside/bignside)*(Nside/bignside))); } logverb("Healpix side length: %g arcmin.\n", healpix_side_length_arcmin(Nside)); dubl = fitscolumn_double_type(); if (!racol) racol = "RA"; ra = fitstable_read_column(intable, racol, dubl); if (!ra) { ERROR("Failed to find RA column (%s) in table", racol); return -1; } if (!deccol) deccol = "DEC"; dec = fitstable_read_column(intable, deccol, dubl); if (!dec) { ERROR("Failed to find DEC column (%s) in table", deccol); free(ra); return -1; } N = fitstable_nrows(intable); logverb("Have %i objects\n", N); // FIXME -- argsort and seek around the input table, and append to // starlists in order; OR read from the input table in sequence and // sort in the starlists? if (sortcol) { logverb("Sorting by %s...\n", sortcol); sortval = fitstable_read_column(intable, sortcol, dubl); if (!sortval) { ERROR("Failed to read sorting column \"%s\"", sortcol); free(ra); free(dec); return -1; } inorder = permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, NULL, N); if (sort_min_cut > -HUGE_VAL) { logverb("Cutting to %s > %g...\n", sortcol, sort_min_cut); // Cut objects with sortval < sort_min_cut. if (sort_ascending) { // skipped objects are at the front -- find the first obj // to keep for (i=0; i<N; i++) if (sortval[inorder[i]] > sort_min_cut) break; // move the "inorder" indices down. if (i) memmove(inorder, inorder+i, (N-i)*sizeof(int)); N -= i; } else { // skipped objects are at the end -- find the last obj to keep. for (i=N-1; i>=0; i--) if (sortval[inorder[i]] > sort_min_cut) break; N = i+1; } logverb("Cut to %i objects\n", N); } //free(sortval); } token.nside = bignside; token.finenside = Nside; token.hp = bighp; if (!allsky && nmargin) { int bigbighp, bighpx, bighpy; //int ninside; il* seeds = il_new(256); logverb("Finding healpixes in range...\n"); healpix_decompose_xy(bighp, &bigbighp, &bighpx, &bighpy, bignside); //ninside = (Nside/bignside)*(Nside/bignside); // Prime the queue with the fine healpixes that are on the // boundary of the big healpix. for (i=0; i<((Nside / bignside) - 1); i++) { // add (i,0), (i,max), (0,i), and (0,max) healpixes int xx = i + bighpx * (Nside / bignside); int yy = i + bighpy * (Nside / bignside); int y0 = bighpy * (Nside / bignside); // -1 prevents us from double-adding the corners. int y1 =(1 + bighpy)* (Nside / bignside) - 1; int x0 = bighpx * (Nside / bignside); int x1 =(1 + bighpx)* (Nside / bignside) - 1; assert(xx < Nside); assert(yy < Nside); assert(x0 < Nside); assert(x1 < Nside); assert(y0 < Nside); assert(y1 < Nside); il_append(seeds, healpix_compose_xy(bigbighp, xx, y0, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, xx, y1, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x0, yy, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x1, yy, Nside)); } logmsg("Number of boundary healpixes: %zu (Nside/bignside = %i)\n", il_size(seeds), Nside/bignside); myhps = healpix_region_search(-1, seeds, Nside, NULL, NULL, outside_healpix, &token, nmargin); logmsg("Number of margin healpixes: %zu\n", il_size(myhps)); il_free(seeds); il_sort(myhps, TRUE); // DEBUG il_check_consistency(myhps); il_check_sorted_ascending(myhps, TRUE); } dedupr2 = arcsec2distsq(dedup_radius); starlists = intmap_new(sizeof(int32_t), nkeep, 0, dense); logverb("Placing stars in grid cells...\n"); for (i=0; i<N; i++) { int hp; bl* lst; int32_t j32; anbool oob; if (inorder) { j = inorder[i]; //printf("Placing star %i (%i): sort value %s = %g, RA,Dec=%g,%g\n", i, j, sortcol, sortval[j], ra[j], dec[j]); } else j = i; hp = radecdegtohealpix(ra[j], dec[j], Nside); //printf("HP %i\n", hp); // in bounds? oob = FALSE; if (myhps) { oob = (outside_healpix(hp, &token) && !il_sorted_contains(myhps, hp)); } else if (!allsky) { oob = (outside_healpix(hp, &token)); } if (oob) { //printf("out of bounds.\n"); noob++; continue; } lst = intmap_find(starlists, hp, TRUE); /* printf("list has %i existing entries.\n", bl_size(lst)); for (k=0; k<bl_size(lst); k++) { bl_get(lst, k, &j32); printf(" %i: index %i, %s = %g\n", k, j32, sortcol, sortval[j32]); } */ // is this list full? if (nkeep && (bl_size(lst) >= nkeep)) { // Here we assume we're working in sorted order: once the list is full we're done. //printf("Skipping: list is full.\n"); continue; } if ((dedupr2 > 0.0) && is_duplicate(hp, ra[j], dec[j], Nside, starlists, ra, dec, dedupr2)) { //printf("Skipping: duplicate\n"); ndup++; continue; } // Add the new star (by index) j32 = j; bl_append(lst, &j32); } logverb("%i outside the healpix\n", noob); logverb("%i duplicates\n", ndup); il_free(myhps); myhps = NULL; free(inorder); inorder = NULL; free(ra); ra = NULL; free(dec); dec = NULL; outorder = malloc(N * sizeof(int)); outi = 0; npersweep = calloc(nsweeps, sizeof(int)); for (k=0; k<nsweeps; k++) { int starti = outi; int32_t j32; for (i=0;; i++) { bl* lst; int hp; if (!intmap_get_entry(starlists, i, &hp, &lst)) break; if (bl_size(lst) <= k) continue; bl_get(lst, k, &j32); outorder[outi] = j32; //printf("sweep %i, cell #%i, hp %i, star %i, %s = %g\n", k, i, hp, j32, sortcol, sortval[j32]); outi++; } logmsg("Sweep %i: %i stars\n", k+1, outi - starti); npersweep[k] = outi - starti; if (sortcol) { // Re-sort within this sweep. permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, outorder + starti, npersweep[k]); /* for (i=0; i<npersweep[k]; i++) { printf(" within sweep %i: star %i, j=%i, %s=%g\n", k, i, outorder[starti + i], sortcol, sortval[outorder[starti + i]]); } */ } } intmap_free(starlists); starlists = NULL; ////// free(sortval); sortval = NULL; logmsg("Total: %i stars\n", outi); N = outi; outhdr = fitstable_get_primary_header(outtable); if (allsky) qfits_header_add(outhdr, "ALLSKY", "T", "All-sky catalog.", NULL); BOILERPLATE_ADD_FITS_HEADERS(outhdr); qfits_header_add(outhdr, "HISTORY", "This file was generated by the command-line:", NULL, NULL); fits_add_args(outhdr, args, argc); qfits_header_add(outhdr, "HISTORY", "(end of command line)", NULL, NULL); fits_add_long_history(outhdr, "uniformize-catalog args:"); fits_add_long_history(outhdr, " RA,Dec columns: %s,%s", racol, deccol); fits_add_long_history(outhdr, " sort column: %s", sortcol); fits_add_long_history(outhdr, " sort direction: %s", sort_ascending ? "ascending" : "descending"); if (sort_ascending) fits_add_long_history(outhdr, " (ie, for mag-like sort columns)"); else fits_add_long_history(outhdr, " (ie, for flux-like sort columns)"); fits_add_long_history(outhdr, " uniformization nside: %i", Nside); fits_add_long_history(outhdr, " (ie, side length ~ %g arcmin)", healpix_side_length_arcmin(Nside)); fits_add_long_history(outhdr, " deduplication scale: %g arcsec", dedup_radius); fits_add_long_history(outhdr, " number of sweeps: %i", nsweeps); fits_header_add_int(outhdr, "NSTARS", N, "Number of stars."); fits_header_add_int(outhdr, "HEALPIX", bighp, "Healpix covered by this catalog, with Nside=HPNSIDE"); fits_header_add_int(outhdr, "HPNSIDE", bignside, "Nside of HEALPIX."); fits_header_add_int(outhdr, "CUTNSIDE", Nside, "uniformization scale (healpix nside)"); fits_header_add_int(outhdr, "CUTMARG", nmargin, "margin size, in healpixels"); //qfits_header_add(outhdr, "CUTBAND", cutband, "band on which the cut was made", NULL); fits_header_add_double(outhdr, "CUTDEDUP", dedup_radius, "deduplication radius [arcsec]"); fits_header_add_int(outhdr, "CUTNSWEP", nsweeps, "number of sweeps"); //fits_header_add_double(outhdr, "CUTMINMG", minmag, "minimum magnitude"); //fits_header_add_double(outhdr, "CUTMAXMG", maxmag, "maximum magnitude"); for (k=0; k<nsweeps; k++) { char key[64]; sprintf(key, "SWEEP%i", (k+1)); fits_header_add_int(outhdr, key, npersweep[k], "# stars added"); } free(npersweep); if (fitstable_write_primary_header(outtable)) { ERROR("Failed to write primary header"); return -1; } // Write output. fitstable_add_fits_columns_as_struct2(intable, outtable); if (fitstable_write_header(outtable)) { ERROR("Failed to write output table header"); return -1; } logmsg("Writing output...\n"); logverb("Row size: %i\n", fitstable_row_size(intable)); if (fitstable_copy_rows_data(intable, outorder, N, outtable)) { ERROR("Failed to copy rows from input table to output"); return -1; } if (fitstable_fix_header(outtable)) { ERROR("Failed to fix output table header"); return -1; } free(outorder); return 0; }