void plot_quad_xy(cairo_t* cairo, double* quadxy, int dimquads) { int k; double cx, cy; double theta[DQMAX]; int* perm; cx = cy = 0.0; for (k=0; k<dimquads; k++) { cx += quadxy[2*k+0]; cy += quadxy[2*k+1]; } cx /= dimquads; cy /= dimquads; for (k=0; k<dimquads; k++) theta[k] = atan2(quadxy[2*k+1] - cy, quadxy[2*k+0] - cx); perm = permuted_sort(theta, sizeof(double), compare_doubles_asc, NULL, dimquads); for (k=0; k<dimquads; k++) { double px,py; px = quadxy[2 * perm[k] + 0]; py = quadxy[2 * perm[k] + 1]; if (k == 0) { cairo_move_to(cairo, px, py); } else { cairo_line_to(cairo, px, py); } } free(perm); cairo_close_path(cairo); }
static anbool find_stars(hpquads_t* me, double radius2, int R) { int d, j, N; int destind; double centre[3]; int* perm; healpix_to_xyzarr(me->hp, me->Nside, 0.5, 0.5, centre); me->res = kdtree_rangesearch_options_reuse(me->starkd->tree, me->res, centre, radius2, KD_OPTIONS_RETURN_POINTS); // here we could check whether stars are in the box defined by the // healpix boundaries plus quad scale, rather than just the circle // containing that box. N = me->res->nres; me->Nstars = N; if (N < me->dimquads) return FALSE; // FIXME -- could merge this step with the sorting step... // remove stars that have been used up. if (R) { destind = 0; for (j=0; j<N; j++) { if (me->nuses[me->res->inds[j]] >= R) continue; me->res->inds[destind] = me->res->inds[j]; for (d=0; d<3; d++) me->res->results.d[destind*3+d] = me->res->results.d[j*3+d]; destind++; } N = destind; if (N < me->dimquads) return FALSE; } // sort the stars in increasing order of index - assume // that this corresponds to decreasing order of brightness. // UNLESS another sorting is provided! if (me->sort_data && me->sort_func && me->sort_size) { /* Two levels of indirection here! me->res->inds are indices into the "sort_data" array (since kdtree is assumed to be un-permuted) We want to produce "perm", which permutes me->res->inds to make sort_data sorted; need to do this because we also want to permute results.d. Alternatively, we could re-fetch the results.d ... */ int k; char* tempdata = malloc(me->sort_size * N); for (k=0; k<N; k++) memcpy(tempdata + k*me->sort_size, ((char*)me->sort_data) + me->sort_size * me->res->inds[k], me->sort_size); perm = permuted_sort(tempdata, me->sort_size, me->sort_func, NULL, N); free(tempdata); } else { // find permutation that sorts by index... perm = permuted_sort(me->res->inds, sizeof(int), compare_ints_asc, NULL, N); } // apply the permutation... permutation_apply(perm, N, me->res->inds, me->res->inds, sizeof(int)); permutation_apply(perm, N, me->res->results.d, me->res->results.d, 3 * sizeof(double)); free(perm); me->inds = (int*)me->res->inds; me->stars = me->res->results.d; me->Nstars = N; return TRUE; }
int uniformize_catalog(fitstable_t* intable, fitstable_t* outtable, const char* racol, const char* deccol, const char* sortcol, anbool sort_ascending, double sort_min_cut, // ? Or do this cut in a separate process? int bighp, int bignside, int nmargin, // uniformization nside. int Nside, double dedup_radius, int nsweeps, char** args, int argc) { anbool allsky; intmap_t* starlists; int NHP; anbool dense = FALSE; double dedupr2 = 0.0; tfits_type dubl; int N; int* inorder = NULL; int* outorder = NULL; int outi; double *ra = NULL, *dec = NULL; il* myhps = NULL; int i,j,k; int nkeep = nsweeps; int noob = 0; int ndup = 0; struct oh_token token; int* npersweep = NULL; qfits_header* outhdr = NULL; double *sortval = NULL; if (bignside == 0) bignside = 1; allsky = (bighp == -1); if (Nside % bignside) { ERROR("Fine healpixelization Nside must be a multiple of the coarse healpixelization Nside"); return -1; } if (Nside > HP_MAX_INT_NSIDE) { ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE); return -1; } NHP = 12 * Nside * Nside; logverb("Healpix Nside: %i, # healpixes on the whole sky: %i\n", Nside, NHP); if (!allsky) { logverb("Creating index for healpix %i, nside %i\n", bighp, bignside); logverb("Number of healpixes: %i\n", ((Nside/bignside)*(Nside/bignside))); } logverb("Healpix side length: %g arcmin.\n", healpix_side_length_arcmin(Nside)); dubl = fitscolumn_double_type(); if (!racol) racol = "RA"; ra = fitstable_read_column(intable, racol, dubl); if (!ra) { ERROR("Failed to find RA column (%s) in table", racol); return -1; } if (!deccol) deccol = "DEC"; dec = fitstable_read_column(intable, deccol, dubl); if (!dec) { ERROR("Failed to find DEC column (%s) in table", deccol); free(ra); return -1; } N = fitstable_nrows(intable); logverb("Have %i objects\n", N); // FIXME -- argsort and seek around the input table, and append to // starlists in order; OR read from the input table in sequence and // sort in the starlists? if (sortcol) { logverb("Sorting by %s...\n", sortcol); sortval = fitstable_read_column(intable, sortcol, dubl); if (!sortval) { ERROR("Failed to read sorting column \"%s\"", sortcol); free(ra); free(dec); return -1; } inorder = permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, NULL, N); if (sort_min_cut > -HUGE_VAL) { logverb("Cutting to %s > %g...\n", sortcol, sort_min_cut); // Cut objects with sortval < sort_min_cut. if (sort_ascending) { // skipped objects are at the front -- find the first obj // to keep for (i=0; i<N; i++) if (sortval[inorder[i]] > sort_min_cut) break; // move the "inorder" indices down. if (i) memmove(inorder, inorder+i, (N-i)*sizeof(int)); N -= i; } else { // skipped objects are at the end -- find the last obj to keep. for (i=N-1; i>=0; i--) if (sortval[inorder[i]] > sort_min_cut) break; N = i+1; } logverb("Cut to %i objects\n", N); } //free(sortval); } token.nside = bignside; token.finenside = Nside; token.hp = bighp; if (!allsky && nmargin) { int bigbighp, bighpx, bighpy; //int ninside; il* seeds = il_new(256); logverb("Finding healpixes in range...\n"); healpix_decompose_xy(bighp, &bigbighp, &bighpx, &bighpy, bignside); //ninside = (Nside/bignside)*(Nside/bignside); // Prime the queue with the fine healpixes that are on the // boundary of the big healpix. for (i=0; i<((Nside / bignside) - 1); i++) { // add (i,0), (i,max), (0,i), and (0,max) healpixes int xx = i + bighpx * (Nside / bignside); int yy = i + bighpy * (Nside / bignside); int y0 = bighpy * (Nside / bignside); // -1 prevents us from double-adding the corners. int y1 =(1 + bighpy)* (Nside / bignside) - 1; int x0 = bighpx * (Nside / bignside); int x1 =(1 + bighpx)* (Nside / bignside) - 1; assert(xx < Nside); assert(yy < Nside); assert(x0 < Nside); assert(x1 < Nside); assert(y0 < Nside); assert(y1 < Nside); il_append(seeds, healpix_compose_xy(bigbighp, xx, y0, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, xx, y1, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x0, yy, Nside)); il_append(seeds, healpix_compose_xy(bigbighp, x1, yy, Nside)); } logmsg("Number of boundary healpixes: %zu (Nside/bignside = %i)\n", il_size(seeds), Nside/bignside); myhps = healpix_region_search(-1, seeds, Nside, NULL, NULL, outside_healpix, &token, nmargin); logmsg("Number of margin healpixes: %zu\n", il_size(myhps)); il_free(seeds); il_sort(myhps, TRUE); // DEBUG il_check_consistency(myhps); il_check_sorted_ascending(myhps, TRUE); } dedupr2 = arcsec2distsq(dedup_radius); starlists = intmap_new(sizeof(int32_t), nkeep, 0, dense); logverb("Placing stars in grid cells...\n"); for (i=0; i<N; i++) { int hp; bl* lst; int32_t j32; anbool oob; if (inorder) { j = inorder[i]; //printf("Placing star %i (%i): sort value %s = %g, RA,Dec=%g,%g\n", i, j, sortcol, sortval[j], ra[j], dec[j]); } else j = i; hp = radecdegtohealpix(ra[j], dec[j], Nside); //printf("HP %i\n", hp); // in bounds? oob = FALSE; if (myhps) { oob = (outside_healpix(hp, &token) && !il_sorted_contains(myhps, hp)); } else if (!allsky) { oob = (outside_healpix(hp, &token)); } if (oob) { //printf("out of bounds.\n"); noob++; continue; } lst = intmap_find(starlists, hp, TRUE); /* printf("list has %i existing entries.\n", bl_size(lst)); for (k=0; k<bl_size(lst); k++) { bl_get(lst, k, &j32); printf(" %i: index %i, %s = %g\n", k, j32, sortcol, sortval[j32]); } */ // is this list full? if (nkeep && (bl_size(lst) >= nkeep)) { // Here we assume we're working in sorted order: once the list is full we're done. //printf("Skipping: list is full.\n"); continue; } if ((dedupr2 > 0.0) && is_duplicate(hp, ra[j], dec[j], Nside, starlists, ra, dec, dedupr2)) { //printf("Skipping: duplicate\n"); ndup++; continue; } // Add the new star (by index) j32 = j; bl_append(lst, &j32); } logverb("%i outside the healpix\n", noob); logverb("%i duplicates\n", ndup); il_free(myhps); myhps = NULL; free(inorder); inorder = NULL; free(ra); ra = NULL; free(dec); dec = NULL; outorder = malloc(N * sizeof(int)); outi = 0; npersweep = calloc(nsweeps, sizeof(int)); for (k=0; k<nsweeps; k++) { int starti = outi; int32_t j32; for (i=0;; i++) { bl* lst; int hp; if (!intmap_get_entry(starlists, i, &hp, &lst)) break; if (bl_size(lst) <= k) continue; bl_get(lst, k, &j32); outorder[outi] = j32; //printf("sweep %i, cell #%i, hp %i, star %i, %s = %g\n", k, i, hp, j32, sortcol, sortval[j32]); outi++; } logmsg("Sweep %i: %i stars\n", k+1, outi - starti); npersweep[k] = outi - starti; if (sortcol) { // Re-sort within this sweep. permuted_sort(sortval, sizeof(double), sort_ascending ? compare_doubles_asc : compare_doubles_desc, outorder + starti, npersweep[k]); /* for (i=0; i<npersweep[k]; i++) { printf(" within sweep %i: star %i, j=%i, %s=%g\n", k, i, outorder[starti + i], sortcol, sortval[outorder[starti + i]]); } */ } } intmap_free(starlists); starlists = NULL; ////// free(sortval); sortval = NULL; logmsg("Total: %i stars\n", outi); N = outi; outhdr = fitstable_get_primary_header(outtable); if (allsky) qfits_header_add(outhdr, "ALLSKY", "T", "All-sky catalog.", NULL); BOILERPLATE_ADD_FITS_HEADERS(outhdr); qfits_header_add(outhdr, "HISTORY", "This file was generated by the command-line:", NULL, NULL); fits_add_args(outhdr, args, argc); qfits_header_add(outhdr, "HISTORY", "(end of command line)", NULL, NULL); fits_add_long_history(outhdr, "uniformize-catalog args:"); fits_add_long_history(outhdr, " RA,Dec columns: %s,%s", racol, deccol); fits_add_long_history(outhdr, " sort column: %s", sortcol); fits_add_long_history(outhdr, " sort direction: %s", sort_ascending ? "ascending" : "descending"); if (sort_ascending) fits_add_long_history(outhdr, " (ie, for mag-like sort columns)"); else fits_add_long_history(outhdr, " (ie, for flux-like sort columns)"); fits_add_long_history(outhdr, " uniformization nside: %i", Nside); fits_add_long_history(outhdr, " (ie, side length ~ %g arcmin)", healpix_side_length_arcmin(Nside)); fits_add_long_history(outhdr, " deduplication scale: %g arcsec", dedup_radius); fits_add_long_history(outhdr, " number of sweeps: %i", nsweeps); fits_header_add_int(outhdr, "NSTARS", N, "Number of stars."); fits_header_add_int(outhdr, "HEALPIX", bighp, "Healpix covered by this catalog, with Nside=HPNSIDE"); fits_header_add_int(outhdr, "HPNSIDE", bignside, "Nside of HEALPIX."); fits_header_add_int(outhdr, "CUTNSIDE", Nside, "uniformization scale (healpix nside)"); fits_header_add_int(outhdr, "CUTMARG", nmargin, "margin size, in healpixels"); //qfits_header_add(outhdr, "CUTBAND", cutband, "band on which the cut was made", NULL); fits_header_add_double(outhdr, "CUTDEDUP", dedup_radius, "deduplication radius [arcsec]"); fits_header_add_int(outhdr, "CUTNSWEP", nsweeps, "number of sweeps"); //fits_header_add_double(outhdr, "CUTMINMG", minmag, "minimum magnitude"); //fits_header_add_double(outhdr, "CUTMAXMG", maxmag, "maximum magnitude"); for (k=0; k<nsweeps; k++) { char key[64]; sprintf(key, "SWEEP%i", (k+1)); fits_header_add_int(outhdr, key, npersweep[k], "# stars added"); } free(npersweep); if (fitstable_write_primary_header(outtable)) { ERROR("Failed to write primary header"); return -1; } // Write output. fitstable_add_fits_columns_as_struct2(intable, outtable); if (fitstable_write_header(outtable)) { ERROR("Failed to write output table header"); return -1; } logmsg("Writing output...\n"); logverb("Row size: %i\n", fitstable_row_size(intable)); if (fitstable_copy_rows_data(intable, outorder, N, outtable)) { ERROR("Failed to copy rows from input table to output"); return -1; } if (fitstable_fix_header(outtable)) { ERROR("Failed to fix output table header"); return -1; } free(outorder); return 0; }
int resort_xylist(const char* infn, const char* outfn, const char* fluxcol, const char* backcol, anbool ascending) { FILE* fin = NULL; FILE* fout = NULL; double *flux = NULL, *back = NULL; int *perm1 = NULL, *perm2 = NULL; anbool *used = NULL; int start, size, nextens, ext; int (*compare)(const void*, const void*); fitstable_t* tab = NULL; anqfits_t* anq = NULL; if (ascending) compare = compare_doubles_asc; else compare = compare_doubles_desc; if (!fluxcol) fluxcol = "FLUX"; if (!backcol) backcol = "BACKGROUND"; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open file \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); tab = fitstable_open(infn); if (!tab) { ERROR("Failed to open FITS table in file %s", infn); goto bailout; } for (ext=1; ext<nextens; ext++) { int hdrstart, hdrsize, datstart; int i, N; int rowsize; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extention %i isn't a table. Skipping", ext); continue; } // Copy the header as-is. if (pipe_file_offset(fin, hdrstart, hdrsize, fout)) { ERROR("Failed to copy the header of extension %i", ext); goto bailout; } if (fitstable_read_extension(tab, ext)) { ERROR("Failed to read FITS table from extension %i", ext); goto bailout; } rowsize = fitstable_row_size(tab); // read FLUX column as doubles. flux = fitstable_read_column(tab, fluxcol, TFITS_BIN_TYPE_D); if (!flux) { ERROR("Failed to read FLUX column from extension %i", ext); goto bailout; } // BACKGROUND back = fitstable_read_column(tab, backcol, TFITS_BIN_TYPE_D); if (!back) { ERROR("Failed to read BACKGROUND column from extension %i", ext); goto bailout; } debug("First 10 rows of input table:\n"); for (i=0; i<10; i++) debug("flux %g, background %g\n", flux[i], back[i]); N = fitstable_nrows(tab); // set back = flux + back (ie, non-background-subtracted flux) for (i=0; i<N; i++) back[i] += flux[i]; // Sort by flux... perm1 = permuted_sort(flux, sizeof(double), compare, NULL, N); // Sort by non-background-subtracted flux... perm2 = permuted_sort(back, sizeof(double), compare, NULL, N); used = malloc(N * sizeof(anbool)); memset(used, 0, N * sizeof(anbool)); // Check sort... for (i=0; i<N-1; i++) { if (ascending) { assert(flux[perm1[i]] <= flux[perm1[i+1]]); assert(back[perm2[i]] <= back[perm2[i+1]]); } else { assert(flux[perm1[i]] >= flux[perm1[i+1]]); assert(back[perm2[i]] >= back[perm2[i+1]]); } } for (i=0; i<N; i++) { int j; int inds[] = { perm1[i], perm2[i] }; for (j=0; j<2; j++) { int index = inds[j]; assert(index < N); if (used[index]) continue; used[index] = TRUE; debug("adding index %i: %s %g\n", index, j==0 ? "flux" : "bgsub", j==0 ? flux[index] : back[index]); if (pipe_file_offset(fin, datstart + index * rowsize, rowsize, fout)) { ERROR("Failed to copy row %i", index); goto bailout; } } } for (i=0; i<N; i++) assert(used[i]); if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } free(flux); flux = NULL; free(back); back = NULL; free(perm1); perm1 = NULL; free(perm2); perm2 = NULL; free(used); used = NULL; } fitstable_close(tab); tab = NULL; if (fclose(fout)) { SYSERROR("Failed to close output file %s", outfn); return -1; } fclose(fin); return 0; bailout: if (tab) fitstable_close(tab); if (fout) fclose(fout); if (fin) fclose(fin); free(flux); free(back); free(perm1); free(perm2); free(used); return -1; }
int tabsort(const char* infn, const char* outfn, const char* colname, int descending) { FILE* fin; FILE* fout; int ext, nextens; off_t start, size; void* data = NULL; int* perm = NULL; unsigned char* map = NULL; size_t mapsize = 0; anqfits_t* anq = NULL; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); //logverb("Sorting %i extensions.\n", nextens); for (ext=1; ext<nextens; ext++) { int c; qfits_table* table; qfits_col* col; int mgap; off_t mstart; size_t msize; int atomsize; int (*sort_func)(const void*, const void*); unsigned char* tabledata; unsigned char* tablehdr; off_t hdrstart, hdrsize, datsize, datstart; int i; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); datsize = anqfits_data_size (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extension %i isn't a table. Skipping.\n", ext); continue; } table = anqfits_get_table(anq, ext); if (!table) { ERROR("Failed to open table: file %s, extension %i. Skipping.", infn, ext); continue; } c = fits_find_column(table, colname); if (c == -1) { ERROR("Couldn't find column named \"%s\" in extension %i. Skipping.", colname, ext); continue; } col = table->col + c; switch (col->atom_type) { case TFITS_BIN_TYPE_D: data = realloc(data, table->nr * sizeof(double)); if (descending) sort_func = compare_doubles_desc; else sort_func = compare_doubles_asc; break; case TFITS_BIN_TYPE_E: data = realloc(data, table->nr * sizeof(float)); if (descending) sort_func = compare_floats_desc; else sort_func = compare_floats_asc; break; case TFITS_BIN_TYPE_K: data = realloc(data, table->nr * sizeof(int64_t)); if (descending) sort_func = compare_int64_desc; else sort_func = compare_int64_asc; break; default: ERROR("Column %s is neither FITS type D, E, nor K. Skipping.", colname); continue; } // Grab the sort column. atomsize = fits_get_atom_size(col->atom_type); printf("Reading sort column \"%s\"\n", colname); qfits_query_column_seq_to_array(table, c, 0, table->nr, data, atomsize); // Sort it. printf("Sorting sort column\n"); perm = permuted_sort(data, atomsize, sort_func, NULL, table->nr); // mmap the input file. printf("mmapping input file\n"); start = hdrstart; size = hdrsize + datsize; get_mmap_size(start, size, &mstart, &msize, &mgap); mapsize = msize; map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, fileno(fin), mstart); if (map == MAP_FAILED) { SYSERROR("Failed to mmap input file %s", infn); map = NULL; goto bailout; } tabledata = map + (off_t)mgap + (datstart - hdrstart); tablehdr = map + (off_t)mgap; // Copy the table header without change. printf("Copying table header.\n"); if (fwrite(tablehdr, 1, hdrsize, fout) != hdrsize) { SYSERROR("Failed to write FITS table header"); goto bailout; } for (i=0; i<table->nr; i++) { unsigned char* rowptr; if (i % 100000 == 0) printf("Writing row %i\n", i); rowptr = tabledata + (off_t)(perm[i]) * (off_t)table->tab_w; if (fwrite(rowptr, 1, table->tab_w, fout) != table->tab_w) { SYSERROR("Failed to write FITS table row"); goto bailout; } } munmap(map, mapsize); map = NULL; free(perm); perm = NULL; if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } qfits_table_close(table); } free(data); if (fclose(fout)) { SYSERROR("Error closing output file"); fout = NULL; goto bailout; } fclose(fin); anqfits_close(anq); printf("Done\n"); return 0; bailout: free(data); free(perm); if (fout) fclose(fout); fclose(fin); if (map) munmap(map, mapsize); return -1; }