void test_il_sorted(CuTest* tc) {
int vals[] = {34951,34950,34949,35049,35149,29951,29950,29949,34999,34998,5099,5199,39849,39949,4999,35249,29952,34952,5299,35349,29953,34953,5399,35449,29954,34954,5499,35549,29955,34955,5599,35649,29956,34956,5699,35749,29957,34957,5799,35849,29958,34958,5899,35949,29959,34959,5999,36049,29960,34960,6099,36149,29961,34961,6199,36249,29962,34962,6299,36349,29963,34963,6399,36449,29964,34964,6499,36549,29965,34965,6599,36649,29966,34966,6699,36749,29967,34967,6799,36849,29968,34968,6899,36949,29969,34969,6999,37049,29970,34970,7099,37149,29971,34971,7199,37249,29972,34972,7299,37349,29973,34973,7399,37449,29974,34974,7499,37549,29975,34975,7599,37649,29976,34976,7699,37749,29977,34977,7799,37849,29978,34978,7899,37949,29979,34979,7999,38049,29980,34980,8099,38149,29981,34981,8199,38249,29982,34982,8299,38349,29983,34983,8399,38449,29984,34984,8499,38549,29985,34985,8599,38649,29986,34986,8699,38749,29987,34987,8799,38849,29988,34988,8899,38949,29989,34989,8999,39049,29990,34990,9099,39149,29991,34991,9199,39249,29992,34992,9299,39349,29993,34993,9399,39449,29994,34994,9499,39549,29995,34995,9599,39649,29996,34996,9699,39749,29997,34997,9799,29998,9899,29999,9999};
int i, N;
il* lst;
N = sizeof(vals)/sizeof(int);
lst = il_new(256);
for (i=0; i<N; i++)
il_append(lst, vals[i]);
CuAssertIntEquals(tc, N, il_size(lst));
for (i=0; i<N; i++)
CuAssertIntEquals(tc, vals[i], il_get(lst, i));
printf("Before sorting:\n");
il_print(lst);
il_sort(lst, TRUE);
printf("After sorting:\n");
il_print(lst);
CuAssertIntEquals(tc, 0, il_check_consistency(lst));
CuAssertIntEquals(tc, 0, il_check_sorted_ascending(lst, TRUE));
for (i=1; i<N; i++)
CuAssertTrue(tc, il_get(lst, i-1) < il_get(lst, i));
for (i=0; i<N; i++)
CuAssertIntEquals(tc, 1, il_sorted_contains(lst, vals[i]));
}
void test_il_insert_unique_ascending(CuTest* tc) {
int i;
il* x = il_new(4);
il_insert_unique_ascending(x,2);
il_insert_unique_ascending(x,4);
il_insert_unique_ascending(x,4);
il_insert_unique_ascending(x,7);
il_insert_unique_ascending(x,4);
il_insert_unique_ascending(x,4);
il_insert_unique_ascending(x,8);
il_insert_unique_ascending(x,5);
il_insert_unique_ascending(x,0);
il_insert_unique_ascending(x,5);
il_insert_unique_ascending(x,5);
il_insert_unique_ascending(x,5);
il_insert_unique_ascending(x,4);
il_insert_unique_ascending(x,5);
il_insert_unique_ascending(x,6);
il_insert_unique_ascending(x,7);
il_insert_unique_ascending(x,7);
il_insert_unique_ascending(x,7);
il_insert_unique_ascending(x,7);
il_insert_unique_ascending(x,1);
il_insert_unique_ascending(x,1);
il_insert_unique_ascending(x,3);
il_insert_unique_ascending(x,1);
il_insert_unique_ascending(x,1);
il_insert_unique_ascending(x,0);
il_print(x);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
CuAssertIntEquals(tc, il_check_sorted_ascending(x, 1), 0);
for (i=0;i<il_size(x);i++)
CuAssertIntEquals(tc, i, il_get(x, i));
il_free(x);
}
void test_il_push_pop2(CuTest* tc) {
int i, N=100;
il* x = il_new(10);
for (i=0; i<N; i++)
il_push(x,i);
for (i=0; i<N; i++)
CuAssertIntEquals(tc, N-i-1, il_pop(x));
CuAssertIntEquals(tc, il_check_consistency(x), 0);
il_free(x);
}
void test_il_get_push(CuTest* tc) {
il* x = il_new(10);
il_push(x,10);
CuAssertIntEquals(tc, 10, il_get(x, 0));
il_push(x,20);
CuAssertIntEquals(tc, 10, il_get(x, 0));
CuAssertIntEquals(tc, 20, il_get(x, 1));
CuAssertIntEquals(tc, il_check_consistency(x), 0);
il_free(x);
}
void test_il_new(CuTest* tc) {
il* x = NULL;
x = il_new(10);
CuAssert(tc, "new", x != NULL);
CuAssertIntEquals(tc, il_size(x), 0);
CuAssertPtrEquals(tc, x->head, NULL);
CuAssertPtrEquals(tc, x->tail, NULL);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
CuAssertIntEquals(tc, il_check_sorted_ascending(x, 0), 0);
il_free(x);
}
void test_il_remove_value(CuTest* tc) {
il* x = il_new(5);
il_push(x,10);
il_push(x,20);
il_push(x,30);
il_push(x,87);
il_push(x,87);
il_push(x,87);
il_push(x,87);
il_push(x,87);
il_push(x,92);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
CuAssertIntEquals(tc, 8, il_remove_value(x, 92));
CuAssertIntEquals(tc, -1, il_remove_value(x, 37));
CuAssertIntEquals(tc, -1, il_remove_value(x, 0));
CuAssertIntEquals(tc, 2, il_remove_value(x, 30));
CuAssertIntEquals(tc, 0, il_remove_value(x, 10));
CuAssertIntEquals(tc, 0, il_remove_value(x, 20));
CuAssertIntEquals(tc, il_check_consistency(x), 0);
il_free(x);
}
void test_il_copy(CuTest* tc) {
int i, N=60, start=10, length=10;
int buf[N];
il* x = il_new(4);
memset(buf, 0, N);
for (i=0;i<N;i++)
il_push(x,i);
il_copy(x, start, length, buf);
for (i=0;i<length;i++)
CuAssertIntEquals(tc, start+i, buf[i]);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
il_free(x);
}
void test_il_contains(CuTest* tc) {
il* x = il_new(4);
il_push(x,10);
il_push(x,20);
il_push(x,30);
il_push(x,30);
il_push(x,30);
il_push(x,41);
il_push(x,30);
il_push(x,81);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
CuAssertIntEquals(tc, 1, il_contains(x, 10));
CuAssertIntEquals(tc, 1, il_contains(x, 20));
CuAssertIntEquals(tc, 1, il_contains(x, 30));
CuAssertIntEquals(tc, 1, il_contains(x, 81));
CuAssertIntEquals(tc, 1, il_contains(x, 41));
CuAssertIntEquals(tc, 0, il_contains(x, 42));
il_remove_value(x, 41);
CuAssertIntEquals(tc, il_check_consistency(x), 0);
CuAssertIntEquals(tc, 0, il_contains(x, 41));
il_free(x);
}
void test_il_dupe(CuTest* tc) {
int i, N=63;
il* x = il_new(4), *y;
for (i=0;i<N;i++)
il_push(x,i);
y = il_dupe(x);
for (i=0;i<N;i++)
CuAssertIntEquals(tc, i, il_get(y, i));
for (i=0;i<N;i++)
il_pop(x);
CuAssertIntEquals(tc, N, il_size(y));
CuAssertIntEquals(tc, il_check_consistency(x), 0);
il_free(x);
il_free(y);
}
void test_delete_2(CuTest* tc) {
il* bl = il_new(2);
il_push(bl, 42);
il_push(bl, 43);
il_push(bl, 47);
il_push(bl, 49);
il_remove(bl, 0);
il_remove(bl, 0);
il_remove(bl, 0);
il_remove(bl, 0);
CuAssertIntEquals(tc, il_size(bl), 0);
CuAssertPtrEquals(tc, bl->head, NULL);
CuAssertPtrEquals(tc, bl->tail, NULL);
CuAssertIntEquals(tc, il_check_consistency(bl), 0);
il_free(bl);
}
void test_set(CuTest* tc) {
il* bl;
bl = il_new(2);
CuAssertIntEquals(tc, il_size(bl), 0);
il_push(bl, 42);
il_push(bl, 43);
il_push(bl, 47);
il_push(bl, 49);
il_set(bl, 0, 0);
il_set(bl, 1, 1);
il_set(bl, 2, 2);
CuAssertIntEquals(tc, il_size(bl), 4);
CuAssertIntEquals(tc, 0, il_get(bl, 0));
CuAssertIntEquals(tc, 1, il_get(bl, 1));
CuAssertIntEquals(tc, 2, il_get(bl, 2));
CuAssertIntEquals(tc, il_check_consistency(bl), 0);
il_free(bl);
}
void test_delete_4(CuTest* tc) {
int i, j, N;
il* bl = il_new(20);
N = 100;
for (i=0; i<N; i++)
il_push(bl, i);
for (i=0; i<N; i++) {
int ind = rand() % il_size(bl);
il_remove(bl, ind);
for (j=1; j<il_size(bl); j++) {
CuAssert(tc, "mono", (il_get(bl, j) - il_get(bl, j-1)) > 0);
}
}
CuAssertIntEquals(tc, il_size(bl), 0);
CuAssertPtrEquals(tc, bl->head, NULL);
CuAssertPtrEquals(tc, bl->tail, NULL);
CuAssertIntEquals(tc, il_check_consistency(bl), 0);
il_free(bl);
}
int uniformize_catalog(fitstable_t* intable, fitstable_t* outtable,
const char* racol, const char* deccol,
const char* sortcol, anbool sort_ascending,
double sort_min_cut,
// ? Or do this cut in a separate process?
int bighp, int bignside,
int nmargin,
// uniformization nside.
int Nside,
double dedup_radius,
int nsweeps,
char** args, int argc) {
anbool allsky;
intmap_t* starlists;
int NHP;
anbool dense = FALSE;
double dedupr2 = 0.0;
tfits_type dubl;
int N;
int* inorder = NULL;
int* outorder = NULL;
int outi;
double *ra = NULL, *dec = NULL;
il* myhps = NULL;
int i,j,k;
int nkeep = nsweeps;
int noob = 0;
int ndup = 0;
struct oh_token token;
int* npersweep = NULL;
qfits_header* outhdr = NULL;
double *sortval = NULL;
if (bignside == 0)
bignside = 1;
allsky = (bighp == -1);
if (Nside % bignside) {
ERROR("Fine healpixelization Nside must be a multiple of the coarse healpixelization Nside");
return -1;
}
if (Nside > HP_MAX_INT_NSIDE) {
ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE);
return -1;
}
NHP = 12 * Nside * Nside;
logverb("Healpix Nside: %i, # healpixes on the whole sky: %i\n", Nside, NHP);
if (!allsky) {
logverb("Creating index for healpix %i, nside %i\n", bighp, bignside);
logverb("Number of healpixes: %i\n", ((Nside/bignside)*(Nside/bignside)));
}
logverb("Healpix side length: %g arcmin.\n", healpix_side_length_arcmin(Nside));
dubl = fitscolumn_double_type();
if (!racol)
racol = "RA";
ra = fitstable_read_column(intable, racol, dubl);
if (!ra) {
ERROR("Failed to find RA column (%s) in table", racol);
return -1;
}
if (!deccol)
deccol = "DEC";
dec = fitstable_read_column(intable, deccol, dubl);
if (!dec) {
ERROR("Failed to find DEC column (%s) in table", deccol);
free(ra);
return -1;
}
N = fitstable_nrows(intable);
logverb("Have %i objects\n", N);
// FIXME -- argsort and seek around the input table, and append to
// starlists in order; OR read from the input table in sequence and
// sort in the starlists?
if (sortcol) {
logverb("Sorting by %s...\n", sortcol);
sortval = fitstable_read_column(intable, sortcol, dubl);
if (!sortval) {
ERROR("Failed to read sorting column \"%s\"", sortcol);
free(ra);
free(dec);
return -1;
}
inorder = permuted_sort(sortval, sizeof(double),
sort_ascending ? compare_doubles_asc : compare_doubles_desc,
NULL, N);
if (sort_min_cut > -HUGE_VAL) {
logverb("Cutting to %s > %g...\n", sortcol, sort_min_cut);
// Cut objects with sortval < sort_min_cut.
if (sort_ascending) {
// skipped objects are at the front -- find the first obj
// to keep
for (i=0; i<N; i++)
if (sortval[inorder[i]] > sort_min_cut)
break;
// move the "inorder" indices down.
if (i)
memmove(inorder, inorder+i, (N-i)*sizeof(int));
N -= i;
} else {
// skipped objects are at the end -- find the last obj to keep.
for (i=N-1; i>=0; i--)
if (sortval[inorder[i]] > sort_min_cut)
break;
N = i+1;
}
logverb("Cut to %i objects\n", N);
}
//free(sortval);
}
token.nside = bignside;
token.finenside = Nside;
token.hp = bighp;
if (!allsky && nmargin) {
int bigbighp, bighpx, bighpy;
//int ninside;
il* seeds = il_new(256);
logverb("Finding healpixes in range...\n");
healpix_decompose_xy(bighp, &bigbighp, &bighpx, &bighpy, bignside);
//ninside = (Nside/bignside)*(Nside/bignside);
// Prime the queue with the fine healpixes that are on the
// boundary of the big healpix.
for (i=0; i<((Nside / bignside) - 1); i++) {
// add (i,0), (i,max), (0,i), and (0,max) healpixes
int xx = i + bighpx * (Nside / bignside);
int yy = i + bighpy * (Nside / bignside);
int y0 = bighpy * (Nside / bignside);
// -1 prevents us from double-adding the corners.
int y1 =(1 + bighpy)* (Nside / bignside) - 1;
int x0 = bighpx * (Nside / bignside);
int x1 =(1 + bighpx)* (Nside / bignside) - 1;
assert(xx < Nside);
assert(yy < Nside);
assert(x0 < Nside);
assert(x1 < Nside);
assert(y0 < Nside);
assert(y1 < Nside);
il_append(seeds, healpix_compose_xy(bigbighp, xx, y0, Nside));
il_append(seeds, healpix_compose_xy(bigbighp, xx, y1, Nside));
il_append(seeds, healpix_compose_xy(bigbighp, x0, yy, Nside));
il_append(seeds, healpix_compose_xy(bigbighp, x1, yy, Nside));
}
logmsg("Number of boundary healpixes: %zu (Nside/bignside = %i)\n", il_size(seeds), Nside/bignside);
myhps = healpix_region_search(-1, seeds, Nside, NULL, NULL,
outside_healpix, &token, nmargin);
logmsg("Number of margin healpixes: %zu\n", il_size(myhps));
il_free(seeds);
il_sort(myhps, TRUE);
// DEBUG
il_check_consistency(myhps);
il_check_sorted_ascending(myhps, TRUE);
}
dedupr2 = arcsec2distsq(dedup_radius);
starlists = intmap_new(sizeof(int32_t), nkeep, 0, dense);
logverb("Placing stars in grid cells...\n");
for (i=0; i<N; i++) {
int hp;
bl* lst;
int32_t j32;
anbool oob;
if (inorder) {
j = inorder[i];
//printf("Placing star %i (%i): sort value %s = %g, RA,Dec=%g,%g\n", i, j, sortcol, sortval[j], ra[j], dec[j]);
} else
j = i;
hp = radecdegtohealpix(ra[j], dec[j], Nside);
//printf("HP %i\n", hp);
// in bounds?
oob = FALSE;
if (myhps) {
oob = (outside_healpix(hp, &token) && !il_sorted_contains(myhps, hp));
} else if (!allsky) {
oob = (outside_healpix(hp, &token));
}
if (oob) {
//printf("out of bounds.\n");
noob++;
continue;
}
lst = intmap_find(starlists, hp, TRUE);
/*
printf("list has %i existing entries.\n", bl_size(lst));
for (k=0; k<bl_size(lst); k++) {
bl_get(lst, k, &j32);
printf(" %i: index %i, %s = %g\n", k, j32, sortcol, sortval[j32]);
}
*/
// is this list full?
if (nkeep && (bl_size(lst) >= nkeep)) {
// Here we assume we're working in sorted order: once the list is full we're done.
//printf("Skipping: list is full.\n");
continue;
}
if ((dedupr2 > 0.0) &&
is_duplicate(hp, ra[j], dec[j], Nside, starlists, ra, dec, dedupr2)) {
//printf("Skipping: duplicate\n");
ndup++;
continue;
}
// Add the new star (by index)
j32 = j;
bl_append(lst, &j32);
}
logverb("%i outside the healpix\n", noob);
logverb("%i duplicates\n", ndup);
il_free(myhps);
myhps = NULL;
free(inorder);
inorder = NULL;
free(ra);
ra = NULL;
free(dec);
dec = NULL;
outorder = malloc(N * sizeof(int));
outi = 0;
npersweep = calloc(nsweeps, sizeof(int));
for (k=0; k<nsweeps; k++) {
int starti = outi;
int32_t j32;
for (i=0;; i++) {
bl* lst;
int hp;
if (!intmap_get_entry(starlists, i, &hp, &lst))
break;
if (bl_size(lst) <= k)
continue;
bl_get(lst, k, &j32);
outorder[outi] = j32;
//printf("sweep %i, cell #%i, hp %i, star %i, %s = %g\n", k, i, hp, j32, sortcol, sortval[j32]);
outi++;
}
logmsg("Sweep %i: %i stars\n", k+1, outi - starti);
npersweep[k] = outi - starti;
if (sortcol) {
// Re-sort within this sweep.
permuted_sort(sortval, sizeof(double),
sort_ascending ? compare_doubles_asc : compare_doubles_desc,
outorder + starti, npersweep[k]);
/*
for (i=0; i<npersweep[k]; i++) {
printf(" within sweep %i: star %i, j=%i, %s=%g\n",
k, i, outorder[starti + i], sortcol, sortval[outorder[starti + i]]);
}
*/
}
}
intmap_free(starlists);
starlists = NULL;
//////
free(sortval);
sortval = NULL;
logmsg("Total: %i stars\n", outi);
N = outi;
outhdr = fitstable_get_primary_header(outtable);
if (allsky)
qfits_header_add(outhdr, "ALLSKY", "T", "All-sky catalog.", NULL);
BOILERPLATE_ADD_FITS_HEADERS(outhdr);
qfits_header_add(outhdr, "HISTORY", "This file was generated by the command-line:", NULL, NULL);
fits_add_args(outhdr, args, argc);
qfits_header_add(outhdr, "HISTORY", "(end of command line)", NULL, NULL);
fits_add_long_history(outhdr, "uniformize-catalog args:");
fits_add_long_history(outhdr, " RA,Dec columns: %s,%s", racol, deccol);
fits_add_long_history(outhdr, " sort column: %s", sortcol);
fits_add_long_history(outhdr, " sort direction: %s", sort_ascending ? "ascending" : "descending");
if (sort_ascending)
fits_add_long_history(outhdr, " (ie, for mag-like sort columns)");
else
fits_add_long_history(outhdr, " (ie, for flux-like sort columns)");
fits_add_long_history(outhdr, " uniformization nside: %i", Nside);
fits_add_long_history(outhdr, " (ie, side length ~ %g arcmin)", healpix_side_length_arcmin(Nside));
fits_add_long_history(outhdr, " deduplication scale: %g arcsec", dedup_radius);
fits_add_long_history(outhdr, " number of sweeps: %i", nsweeps);
fits_header_add_int(outhdr, "NSTARS", N, "Number of stars.");
fits_header_add_int(outhdr, "HEALPIX", bighp, "Healpix covered by this catalog, with Nside=HPNSIDE");
fits_header_add_int(outhdr, "HPNSIDE", bignside, "Nside of HEALPIX.");
fits_header_add_int(outhdr, "CUTNSIDE", Nside, "uniformization scale (healpix nside)");
fits_header_add_int(outhdr, "CUTMARG", nmargin, "margin size, in healpixels");
//qfits_header_add(outhdr, "CUTBAND", cutband, "band on which the cut was made", NULL);
fits_header_add_double(outhdr, "CUTDEDUP", dedup_radius, "deduplication radius [arcsec]");
fits_header_add_int(outhdr, "CUTNSWEP", nsweeps, "number of sweeps");
//fits_header_add_double(outhdr, "CUTMINMG", minmag, "minimum magnitude");
//fits_header_add_double(outhdr, "CUTMAXMG", maxmag, "maximum magnitude");
for (k=0; k<nsweeps; k++) {
char key[64];
sprintf(key, "SWEEP%i", (k+1));
fits_header_add_int(outhdr, key, npersweep[k], "# stars added");
}
free(npersweep);
if (fitstable_write_primary_header(outtable)) {
ERROR("Failed to write primary header");
return -1;
}
// Write output.
fitstable_add_fits_columns_as_struct2(intable, outtable);
if (fitstable_write_header(outtable)) {
ERROR("Failed to write output table header");
return -1;
}
logmsg("Writing output...\n");
logverb("Row size: %i\n", fitstable_row_size(intable));
if (fitstable_copy_rows_data(intable, outorder, N, outtable)) {
ERROR("Failed to copy rows from input table to output");
return -1;
}
if (fitstable_fix_header(outtable)) {
ERROR("Failed to fix output table header");
return -1;
}
free(outorder);
return 0;
}
