bl* henry_draper_get(hd_catalog_t* hdcat, double racenter, double deccenter, double r_arcsec) { double r2; double xyz[3]; kdtree_qres_t* q; bl* res; int i; hd_entry_t hd; radecdeg2xyzarr(racenter, deccenter, xyz); r2 = arcsec2distsq(r_arcsec); q = kdtree_rangesearch(hdcat->kd, xyz, r2); if (!q) { return NULL; } res = bl_new(256, sizeof(hd_entry_t)); for (i=0; i<q->nres; i++) { double* pt = q->results.d + i*3; xyzarr2radecdeg(pt, &(hd.ra), &(hd.dec)); hd.hd = q->inds[i] + 1; bl_append(res, &hd); } kdtree_free_query(q); return res; }
static void ensure_row_list_exists(fitstable_t* table) { if (!table->rows) { // how big are the rows? int rowsize = offset_of_column(table, bl_size(table->cols)); table->rows = bl_new(1024, rowsize); } }
static fitstable_t* fitstable_new() { fitstable_t* tab; tab = calloc(1, sizeof(fitstable_t)); if (!tab) return tab; tab->cols = bl_new(8, sizeof(fitscol_t)); return tab; }
bl* opts_from_array(const an_option_t* opts, int N, bl* lst) { int i; if (!lst) lst = bl_new(4, sizeof(an_option_t)); for (i=0; i<N; i++) bl_append(lst, opts + i); return lst; }
void testit(int* wanted, int Nwanted, int dimquads, int (*compar)(const void *, const void *), anbool cxdx) { int i; solver_t* solver; index_t index; starxy_t* starxy; starxy = field1(); quadlist = bl_new(16, dimquads*sizeof(int)); ninv = 0; solver = solver_new(); memset(&index, 0, sizeof(index_t)); index.index_scale_lower = 1; index.index_scale_upper = 10; index.dimquads = dimquads; index.cx_less_than_dx = index.meanx_less_than_half = cxdx; solver->funits_lower = 0.1; solver->funits_upper = 10; solver_add_index(solver, &index); solver_set_field(solver, starxy); solver_preprocess_field(solver); printf("Found:\n"); solver_run(solver); printf("\n"); fflush(NULL); solver_free_field(solver); solver_free(solver); // bl_sort(quadlist, compar); qsort(wanted, Nwanted, dimquads*sizeof(int), compar); printf("\n\n"); printf("Wanted:\n"); for (i=0; i<Nwanted; i++) { int j; printf("{"); for (j=0; j<dimquads; j++) printf("%s%i", (j?",":""), wanted[i*dimquads+j]); printf("}, "); } printf("\n"); printf("N found: %i; N wanted: %i\n", bl_size(quadlist), Nwanted); printf("N obeying invariants: %i\n", ninv); assert(bl_size(quadlist) == Nwanted); for (i=0; i<bl_size(quadlist); i++) { //int* i1 = bl_access(quadlist, i); //int* i2 = wanted[i]; //printf("[%i, %i, %i] vs [%i, %i, %i]\n", i1[0],i1[1],i1[2], i2[0],i2[1],i2[2]); assert(compar(bl_access(quadlist, i), wanted+i*dimquads) == 0); } bl_free(quadlist); }
void glBegin(GLenum mode) { if (! gl_valid_mode(mode)) { ERROR(GL_INVALID_ENUM); } ERROR_IN_BLOCK(); block_t *block = state.block.active = bl_new(mode); displaylist_t *list = state.list.active; if (list) { dl_append_block(list, block); } }
void* plot_annotations_init(plot_args_t* args) { plotann_t* ann = calloc(1, sizeof(plotann_t)); ann->ngc_fraction = 0.02; ann->targets = bl_new(4, sizeof(target_t)); ann->NGC = TRUE; ann->bright = TRUE; ann->bright_labels = TRUE; ann->constellation_lines = TRUE; ann->constellation_lines_offset = 5.0; return ann; }
void test_bl_extend(CuTest *tc) { bl* list = bl_new(10, sizeof(int)); CuAssertIntEquals(tc, bl_size(list), 0); int *new1 = bl_extend(list); CuAssertPtrNotNull(tc, new1); CuAssertIntEquals(tc, bl_size(list), 1); *new1 = 10; int *new2 = bl_access(list, 0); CuAssertPtrEquals(tc, new2, new1); bl_free(list); }
fitstable_t* fitstable_open_in_memory() { fitstable_t* tab; tab = fitstable_new(); if (!tab) { ERROR("Failed to allocate new FITS table structure"); goto bailout; } tab->fn = NULL; tab->fid = NULL; tab->primheader = qfits_table_prim_header_default(); tab->inmemory = TRUE; tab->extensions = bl_new(16, sizeof(fitsext_t)); return tab; bailout: if (tab) { fitstable_close(tab); } return NULL; }
static bl* get_chunks(startree_t* s, il* wordsizes) { bl* chunks = bl_new(4, sizeof(fitsbin_chunk_t)); fitsbin_chunk_t chunk; kdtree_t* kd = s->tree; fitsbin_chunk_init(&chunk); chunk.tablename = "sweep"; chunk.forced_type = fitscolumn_u8_type(); chunk.itemsize = sizeof(uint8_t); chunk.nrows = kd->ndata; chunk.data = s->sweep; chunk.userdata = &(s->sweep); chunk.required = FALSE; bl_append(chunks, &chunk); if (wordsizes) il_append(wordsizes, sizeof(uint8_t)); fitsbin_chunk_clean(&chunk); return chunks; }
static void* plot_builtin_init(plot_args_t* args) { parse_color_rgba("gray", args->rgba); parse_color_rgba("black", args->bg_rgba); args->text_bg_layer = 2; args->text_fg_layer = 3; args->marker_fg_layer = 3; args->bg_lw = 3.0; args->lw = 1.0; args->marker = CAIROUTIL_MARKER_CIRCLE; args->markersize = 5.0; args->linestep = 10; args->op = CAIRO_OPERATOR_OVER; args->fontsize = 20; args->halign = 'C'; args->valign = 'B'; args->cairocmds = bl_new(256, sizeof(cairocmd_t)); args->label_offset_x = 10.0; args->label_offset_y = 5.0; return NULL; }
static block_t *block_from_arrays(GLenum mode, GLsizei skip, GLsizei count) { block_t *block = bl_new(mode); block->artificial = true; block->len = block->cap = count; if (state.enable.vertex_array) { block->vert = gl_copy_pointer(&state.pointers.vertex, 3, skip, count, false); } if (state.enable.color_array) { block->color = gl_copy_pointer(&state.pointers.color, 4, skip, count, true); } if (state.enable.normal_array) { block->normal = gl_copy_pointer(&state.pointers.normal, 3, skip, count, false); } for (int i = 0; i < MAX_TEX; i++) { if (state.enable.tex_coord_array[i]) { block->tex[i] = gl_copy_pointer(&state.pointers.tex_coord[i], 2, skip, count, false); } } return block; }
int main(int argc, char *argv[]) { int argchar; char* progname = argv[0]; char** inputfiles = NULL; int ninputfiles = 0; int i; int firstfield=0, lastfield=INT_MAX-1; int firstfieldfile=1, lastfieldfile=INT_MAX-1; matchfile** mfs; MatchObj** mos; anbool* eofs; anbool* eofieldfile; int nread = 0; int f; int fieldfile; int totalsolved, totalunsolved; int mode = MODE_BEST; double logodds_tosolve = -HUGE_VAL; anbool agree = FALSE; MatchObj* bestmo; bl* keepers; bl* leftovers = NULL; while ((argchar = getopt (argc, argv, OPTIONS)) != -1) { switch (argchar) { case 'S': solvedfile = optarg; break; case 's': solvedserver = optarg; break; case 'F': mode = MODE_FIRST; break; case 'a': mode = MODE_ALL; break; case 'r': ratio_tosolve = atof(optarg); logodds_tosolve = log(ratio_tosolve); break; case 'f': ninfield_tosolve = atoi(optarg); break; case 'M': agreefname = optarg; break; case 'L': leftoverfname = optarg; break; case 'I': firstfieldfile = atoi(optarg); break; case 'J': lastfieldfile = atoi(optarg); break; case 'A': firstfield = atoi(optarg); break; case 'B': lastfield = atoi(optarg); break; case 'h': default: printHelp(progname); exit(-1); } } if (optind < argc) { ninputfiles = argc - optind; inputfiles = argv + optind; } else { printHelp(progname); exit(-1); } if (lastfield < firstfield) { fprintf(stderr, "Last field (-B) must be at least as big as first field (-A)\n"); exit(-1); } if (solvedserver) if (solvedclient_set_server(solvedserver)) { fprintf(stderr, "Failed to set solved server.\n"); exit(-1); } if (leftoverfname) { leftovermf = matchfile_open_for_writing(leftoverfname); if (!leftovermf) { fprintf(stderr, "Failed to open file %s to write leftover matches.\n", leftoverfname); exit(-1); } BOILERPLATE_ADD_FITS_HEADERS(leftovermf->header); qfits_header_add(leftovermf->header, "HISTORY", "This file was created by the program \"agreeable\".", NULL, NULL); if (matchfile_write_headers(leftovermf)) { fprintf(stderr, "Failed to write leftovers matchfile header.\n"); exit(-1); } leftovers = bl_new(256, sizeof(MatchObj)); } if (agreefname) { agreemf = matchfile_open_for_writing(agreefname); if (!agreemf) { fprintf(stderr, "Failed to open file %s to write agreeing matches.\n", agreefname); exit(-1); } BOILERPLATE_ADD_FITS_HEADERS(agreemf->header); qfits_header_add(agreemf->header, "HISTORY", "This file was created by the program \"agreeable\".", NULL, NULL); if (matchfile_write_headers(agreemf)) { fprintf(stderr, "Failed to write agreeing matchfile header.\n"); exit(-1); } agree = TRUE; } solved = il_new(256); unsolved = il_new(256); keepers = bl_new(256, sizeof(MatchObj)); totalsolved = totalunsolved = 0; mos = calloc(ninputfiles, sizeof(MatchObj*)); eofs = calloc(ninputfiles, sizeof(anbool)); eofieldfile = malloc(ninputfiles * sizeof(anbool)); mfs = malloc(ninputfiles * sizeof(matchfile*)); for (i=0; i<ninputfiles; i++) { fprintf(stderr, "Opening file %s...\n", inputfiles[i]); mfs[i] = matchfile_open(inputfiles[i]); if (!mfs[i]) { fprintf(stderr, "Failed to open matchfile %s.\n", inputfiles[i]); exit(-1); } } // we assume the matchfiles are sorted by field id and number. for (fieldfile=firstfieldfile; fieldfile<=lastfieldfile; fieldfile++) { anbool alldone = TRUE; memset(eofieldfile, 0, ninputfiles * sizeof(anbool)); for (f=firstfield; f<=lastfield; f++) { int fieldnum = f; anbool donefieldfile; anbool solved_it; bl* writematches = NULL; // quit if we've reached the end of all the input files. alldone = TRUE; for (i=0; i<ninputfiles; i++) if (!eofs[i]) { alldone = FALSE; break; } if (alldone) break; // move on to the next fieldfile if all the input files have been // exhausted. donefieldfile = TRUE; for (i=0; i<ninputfiles; i++) if (!eofieldfile[i] && !eofs[i]) { donefieldfile = FALSE; break; } if (donefieldfile) break; // start a new field. fprintf(stderr, "File %i, Field %i.\n", fieldfile, f); solved_it = FALSE; bestmo = NULL; for (i=0; i<ninputfiles; i++) { int nr = 0; int ns = 0; while (1) { if (eofs[i]) break; if (!mos[i]) mos[i] = matchfile_read_match(mfs[i]); if (unlikely(!mos[i])) { eofs[i] = TRUE; break; } // skip past entries that are out of range... if ((mos[i]->fieldfile < firstfieldfile) || (mos[i]->fieldfile > lastfieldfile) || (mos[i]->fieldnum < firstfield) || (mos[i]->fieldnum > lastfield)) { mos[i] = NULL; ns++; continue; } if (mos[i]->fieldfile > fieldfile) eofieldfile[i] = TRUE; if (mos[i]->fieldfile != fieldfile) break; assert(mos[i]->fieldnum >= fieldnum); if (mos[i]->fieldnum != fieldnum) break; nread++; if (nread % 10000 == 9999) { fprintf(stderr, "."); fflush(stderr); } // if we've already found a solution, skip past the // remaining matches in this file... if (solved_it && (mode == MODE_FIRST)) { ns++; mos[i] = NULL; continue; } nr++; if ((mos[i]->logodds >= logodds_tosolve) && (mos[i]->nindex >= ninfield_tosolve)) { solved_it = TRUE; // (note, we get a pointer to its position in the list) mos[i] = bl_append(keepers, mos[i]); if (!bestmo || mos[i]->logodds > bestmo->logodds) bestmo = mos[i]; } else if (leftovers) { bl_append(leftovers, mos[i]); } mos[i] = NULL; } if (nr || ns) fprintf(stderr, "File %s: read %i matches, skipped %i matches.\n", inputfiles[i], nr, ns); } // which matches do we want to write out? if (agree) { writematches = bl_new(256, sizeof(MatchObj)); switch (mode) { case MODE_BEST: case MODE_FIRST: if (bestmo) bl_append(writematches, bestmo); break; case MODE_ALL: bl_append_list(writematches, keepers); break; } } write_field(writematches, leftovers, fieldfile, fieldnum); if (agree) bl_free(writematches); if (leftovers) bl_remove_all(leftovers); if (keepers) bl_remove_all(keepers); fprintf(stderr, "This file: %i fields solved, %i unsolved.\n", il_size(solved), il_size(unsolved)); fprintf(stderr, "Grand total: %i solved, %i unsolved.\n", totalsolved + il_size(solved), totalunsolved + il_size(unsolved)); } totalsolved += il_size(solved); totalunsolved += il_size(unsolved); il_remove_all(solved); il_remove_all(unsolved); if (alldone) break; } for (i=0; i<ninputfiles; i++) matchfile_close(mfs[i]); free(mfs); free(mos); free(eofs); fprintf(stderr, "\nRead %i matches.\n", nread); fflush(stderr); if (keepers) bl_free(keepers); if (leftovers) bl_free(leftovers); il_free(solved); il_free(unsolved); if (leftovermf) { matchfile_fix_headers(leftovermf); matchfile_close(leftovermf); } if (agreemf) { matchfile_fix_headers(agreemf); matchfile_close(agreemf); } return 0; }
void test2() { int i; solver_t* solver; index_t index; starxy_t* starxy; int wanted[][3] = { { 0, 1, 3 }, { 0, 1, 4 }, { 0, 1, 5 }, { 0, 2, 3 }, { 0, 2, 4 }, { 0, 3, 4 }, { 0, 5, 4 }, { 0, 6, 4 }, { 0, 6, 5 }, { 1, 2, 3 }, { 1, 2, 4 }, { 1, 3, 4 }, { 1, 5, 4 }, { 1, 6, 4 }, { 1, 6, 5 }, { 2, 4, 3 }, { 2, 5, 0 }, { 2, 5, 1 }, { 2, 5, 3 }, { 2, 5, 4 }, { 2, 6, 0 }, { 2, 6, 1 }, { 2, 6, 3 }, { 2, 6, 4 }, { 2, 6, 5 }, { 3, 5, 4 }, { 3, 6, 0 }, { 3, 6, 1 }, { 3, 6, 4 }, { 3, 6, 5 }, { 4, 6, 5 }, }; starxy = field1(); quadlist = bl_new(16, 3*sizeof(uint)); solver = solver_new(); memset(&index, 0, sizeof(index_t)); index.index_scale_lower = 1; index.index_scale_upper = 10; index.dimquads = 3; solver->funits_lower = 0.1; solver->funits_upper = 10; solver_add_index(solver, &index); solver_set_field(solver, starxy); solver_preprocess_field(solver); solver_run(solver); solver_free_field(solver); solver_free(solver); // assert(bl_size(quadlist) == (sizeof(wanted) / (3*sizeof(uint)))); bl_sort(quadlist, compare_tri); for (i=0; i<bl_size(quadlist); i++) { assert(compare_tri(bl_access(quadlist, i), wanted[i]) == 0); } bl_free(quadlist); }
static void bl_sort_rec(bl* list, void* pivot, int (*compare)(const void* v1, const void* v2, void* userdata), void* userdata) { bl* less; bl* equal; bl* greater; int i; bl_node* node; // Empty case if (!list->head) return; // Base case: list with only one block. if (list->head == list->tail) { bl_node* node; struct funcandtoken ft; ft.compare = compare; ft.userdata = userdata; node = list->head; QSORT_R(NODE_DATA(node), node->N, list->datasize, &ft, qcompare); return; } less = bl_new(list->blocksize, list->datasize); equal = bl_new(list->blocksize, list->datasize); greater = bl_new(list->blocksize, list->datasize); for (node=list->head; node; node=node->next) { char* data = NODE_CHARDATA(node); for (i=0; i<node->N; i++) { int val = compare(data, pivot, userdata); if (val < 0) bl_append(less, data); else if (val > 0) bl_append(greater, data); else bl_append(equal, data); data += list->datasize; } } // recurse before freeing anything... bl_sort_with_userdata(less, compare, userdata); bl_sort_with_userdata(greater, compare, userdata); for (node=list->head; node;) { bl_node* next; next = node->next; bl_free_node(node); node = next; } list->head = NULL; list->tail = NULL; list->N = 0; list->last_access = NULL; list->last_access_n = 0; if (less->N) { list->head = less->head; list->tail = less->tail; list->N = less->N; } if (equal->N) { if (list->N) { list->tail->next = equal->head; list->tail = equal->tail; } else { list->head = equal->head; list->tail = equal->tail; } list->N += equal->N; } if (greater->N) { if (list->N) { list->tail->next = greater->head; list->tail = greater->tail; } else { list->head = greater->head; list->tail = greater->tail; } list->N += greater->N; } // note, these are supposed to be "free", not "bl_free"; we've stolen // the blocks, we're just freeing the headers. free(less); free(equal); free(greater); }
int main(int argc, char** args) { int argchar; char* infn = NULL; char* outfn = NULL; unsigned int row; int bits; FILE* fid = stdin; FILE* fout = stdout; int loglvl = LOG_MSG; char* progname = args[0]; int bzero = 0; int outformat; qfits_header* hdr; unsigned int plane; off_t datastart; anbool onepass = FALSE; bl* pixcache = NULL; #if HAVE_NETPBM struct pam img; tuple * tuplerow; #else void* rowbuf; #endif int W, H, depth, maxval; while ((argchar = getopt (argc, args, OPTIONS)) != -1) switch (argchar) { case '?': case 'h': printHelp(progname); exit(0); case 'v': loglvl++; break; case 'q': loglvl--; break; case 'o': outfn = optarg; break; } log_init(loglvl); log_to(stderr); fits_use_error_system(); if (optind == argc) { // ok, stdin to stdout. } else if (optind == argc-1) { infn = args[optind]; } else if (optind == argc-2) { infn = args[optind]; outfn = args[optind+1]; } else { printHelp(progname); exit(-1); } if (infn && !streq(infn, "-")) { fid = fopen(infn, "rb"); if (!fid) { SYSERROR("Failed to open input file %s", infn); exit(-1); } } if (outfn) { fout = fopen(outfn, "wb"); if (!fid) { SYSERROR("Failed to open output file %s", outfn); exit(-1); } } else outfn = "stdout"; #if HAVE_NETPBM pm_init(args[0], 0); pnm_readpaminit(fid, &img, // PAM_STRUCT_SIZE isn't defined until Netpbm 10.23 (July 2004) #if defined(PAM_STRUCT_SIZE) PAM_STRUCT_SIZE(tuple_type) #else sizeof(struct pam) #endif ); W = img.width; H = img.height; depth = img.depth; maxval = img.maxval; tuplerow = pnm_allocpamrow(&img); bits = pm_maxvaltobits(img.maxval); bits = (bits <= 8) ? 8 : 16; #else // No NETPBM if (parse_pnm_header(fid, &W, &H, &depth, &maxval)) { ERROR("Failed to parse PNM header from file: %s\n", infn ? infn : "<stdin>"); exit(-1); } bits = 8 * maxval_to_bytes(maxval); rowbuf = malloc(W * depth * (bits/8)); #endif logmsg("Read file %s: %i x %i pixels x %i color(s); maxval %i\n", infn ? infn : "stdin", W, H, depth, maxval); if (bits == 8) outformat = BPP_8_UNSIGNED; else { outformat = BPP_16_SIGNED; if (maxval >= INT16_MAX) bzero = 0x8000; } logmsg("Using %i-bit output\n", bits); hdr = fits_get_header_for_image3(W, H, outformat, depth, NULL); if (bzero) fits_header_add_int(hdr, "BZERO", bzero, "Number that has been subtracted from pixel values"); if (qfits_header_dump(hdr, fout)) { ERROR("Failed to write FITS header to file %s", outfn); exit(-1); } qfits_header_destroy(hdr); datastart = ftello(fid); // Figure out if we can seek backward in this input file... if ((fid == stdin) || (fseeko(fid, 0, SEEK_SET) || fseeko(fid, datastart, SEEK_SET))) // Nope! onepass = TRUE; if (onepass && depth > 1) { logmsg("Reading in one pass\n"); pixcache = bl_new(16384, bits/8); } for (plane=0; plane<depth; plane++) { if (plane > 0) { if (fseeko(fid, datastart, SEEK_SET)) { SYSERROR("Failed to seek back to start of image data"); exit(-1); } } for (row = 0; row<H; row++) { unsigned int column; #if HAVE_NETPBM pnm_readpamrow(&img, tuplerow); #else read_pnm_row(fid, W, depth, maxval, rowbuf); #endif for (column = 0; column<W; column++) { int rtn; int pixval; #if HAVE_NETPBM pixval = tuplerow[column][plane]; #else pixval = (bits == 8 ? ((uint8_t *)rowbuf)[column*depth + plane] : ((uint16_t*)rowbuf)[column*depth + plane]); #endif if (outformat == BPP_8_UNSIGNED) rtn = fits_write_data_B(fout, pixval); else rtn = fits_write_data_I(fout, pixval-bzero, TRUE); if (rtn) { ERROR("Failed to write FITS pixel"); exit(-1); } } if (onepass && depth > 1) { for (column = 0; column<W; column++) { for (plane=1; plane<depth; plane++) { int pixval; #if HAVE_NETPBM pixval = tuplerow[column][plane]; #else pixval = (bits == 8 ? ((uint8_t *)rowbuf)[column*depth + plane] : ((uint16_t*)rowbuf)[column*depth + plane]); #endif if (outformat == BPP_8_UNSIGNED) { uint8_t pix = pixval; bl_append(pixcache, &pix); } else { int16_t pix = pixval - bzero; bl_append(pixcache, &pix); } } } } } } #if HAVE_NETPBM pnm_freepamrow(tuplerow); #else free(rowbuf); #endif if (pixcache) { int i, j; int step = (depth - 1); logverb("Writing %zu queued pixels\n", bl_size(pixcache)); for (plane=1; plane<depth; plane++) { j = (plane - 1); for (i=0; i<(W * H); i++) { int rtn; if (outformat == BPP_8_UNSIGNED) { uint8_t* pix = bl_access(pixcache, j); rtn = fits_write_data_B(fout, *pix); } else { int16_t* pix = bl_access(pixcache, j); rtn = fits_write_data_I(fout, *pix, TRUE); } if (rtn) { ERROR("Failed to write FITS pixel"); exit(-1); } j += step; } } bl_free(pixcache); } if (fid != stdin) fclose(fid); if (fits_pad_file(fout)) { ERROR("Failed to pad output file \"%s\"", outfn); return -1; } if (fout != stdout) if (fclose(fout)) { SYSERROR("Failed to close output file %s", outfn); exit(-1); } return 0; }
int ark_check_persistence(_ARK *_arkp, uint64_t flags) { int32_t rc = -1; char *p_data_orig = NULL; char *p_data = NULL; ark_io_list_t *bl_array = NULL; p_cntr_t *pptr = NULL; P_ARK_t *pcfg = NULL; hash_t *htp = NULL; BL *blp = NULL; uint64_t rdblks = 0; if (flags & ARK_KV_PERSIST_LOAD) {KV_TRC(pAT, "PERSIST_LOAD");} // Ignore the persistence data and load from scratch if ( (!(flags & ARK_KV_PERSIST_LOAD)) || (flags & ARK_KV_VIRTUAL_LUN) ) { return -1; } p_data_orig = am_malloc(_arkp->bsize); if (p_data_orig == NULL) { KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" bytes for the first " "persistence block", _arkp->bsize); rc = ENOMEM; } else { p_data = ptr_align(p_data_orig); bl_array = bl_chain_no_bl(0, 1); rc = ea_async_io(_arkp->ea, ARK_EA_READ, (void *)p_data, bl_array, 1, 1); am_free(bl_array); } if (rc == 0) { // We've read the first block. We check to see if // persistence data is present and if so, then // read the rest of the data from the flash. pptr = (p_cntr_t *)p_data; _arkp->persdata = p_data_orig; if ( memcmp(pptr->p_cntr_magic, ARK_P_MAGIC, sizeof(pptr->p_cntr_magic) != 0)) { KV_TRC_FFDC(pAT, "No magic number found in persistence data: %d", EINVAL); // The magic number does not match so data is either // not present or is corrupted. rc = -1; } else { // Now we check version and the first persistence data // needs to be the ARK_PERSIST_CONFIG block if (pptr->p_cntr_version != ARK_P_VERSION_1 && pptr->p_cntr_version != ARK_P_VERSION_2) { KV_TRC_FFDC(pAT, "Invalid / unsupported version: %"PRIu64"", pptr->p_cntr_version); rc = EINVAL; } else { // Read in the rest of the persistence data pcfg = (P_ARK_t *)(pptr->p_cntr_data + pptr->p_cntr_cfg_offset); rdblks = pcfg->pblocks; if (rdblks > 1) { p_data_orig = am_realloc(p_data_orig, (rdblks * _arkp->bsize)); if (p_data_orig == NULL) { KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" bytes for " "full persistence block", (rdblks * _arkp->bsize)); rc = ENOMEM; } else { p_data = ptr_align(p_data_orig); bl_array = bl_chain_no_bl(0, rdblks); if (bl_array == NULL) { KV_TRC_FFDC(pAT, "Out of memory allocating %"PRIu64" blocks for " "full persistence data", rdblks); rc = ENOMEM; } } // We are still good to read the rest of the data // from the flash if (rc == 0) { KV_TRC(pAT, "PERSIST_RD rdblks:%ld", rdblks); rc = ea_async_io(_arkp->ea, ARK_EA_READ, (void *)p_data, bl_array, rdblks, 1); am_free(bl_array); pptr = (p_cntr_t *)p_data; pcfg = (P_ARK_t *)(pptr->p_cntr_data + pptr->p_cntr_cfg_offset); _arkp->persdata = p_data_orig; } } } } } // If rc == 0, that means we have persistence data if (rc == 0) { KV_TRC(pAT, "PERSIST_META size %ld bsize %ld hcount %ld bcount %ld " "nthrds %d nasyncs %d basyncs %d blkbits %ld version:%ld", pcfg->size, pcfg->bsize, pcfg->hcount, pcfg->bcount, pcfg->nthrds, pcfg->nasyncs, pcfg->basyncs, pcfg->blkbits, pptr->p_cntr_version); _arkp->persload = 1; _arkp->size = pcfg->size; _arkp->flags = flags; _arkp->bsize = pcfg->bsize; _arkp->bcount = pcfg->bcount; _arkp->blkbits = pcfg->blkbits; _arkp->grow = pcfg->grow; _arkp->hcount = pcfg->hcount; _arkp->vlimit = pcfg->vlimit; _arkp->blkused = pcfg->blkused; _arkp->pers_stats.kv_cnt = pcfg->pstats.kv_cnt; _arkp->pers_stats.blk_cnt = pcfg->pstats.blk_cnt; _arkp->pers_stats.byte_cnt = pcfg->pstats.byte_cnt; KV_TRC(pAT, "ARK_META size %ld bsize %ld hcount %ld bcount %ld " "nthrds %d nasyncs %ld basyncs %d blkbits %ld", _arkp->size, _arkp->bsize, _arkp->hcount, _arkp->bcount, _arkp->nthrds, _arkp->nasyncs, _arkp->basyncs, _arkp->blkbits); htp = (hash_t *)(pptr->p_cntr_data + pptr->p_cntr_ht_offset); _arkp->ht = hash_new(htp->n); if (_arkp->ht == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "ht_new failed: n:%ld rc:%d", htp->n, rc); goto error_exit; } memcpy(_arkp->ht, htp, pptr->p_cntr_ht_size); blp = (BL *)(pptr->p_cntr_data + pptr->p_cntr_bl_offset); _arkp->bl = bl_new(blp->n, blp->w); if (_arkp->bl == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "bl_new failed: n:%ld w:%ld rc:%d", blp->n, blp->w, rc); goto error_exit; } _arkp->bl->count = blp->count; _arkp->bl->head = blp->head; _arkp->bl->hold = blp->hold; _arkp->bl->top = blp->top; if (pptr->p_cntr_version == ARK_P_VERSION_1) { IV *piv = (IV *)(pptr->p_cntr_data + pptr->p_cntr_bliv_offset); KV_TRC(pAT, "PERSIST_VERSION_1 LOADED"); _arkp->bl->top = _arkp->bl->n; // copy IV->data from piv->data memcpy(_arkp->bl->list->data, piv->data, pptr->p_cntr_bliv_size); } else if (pptr->p_cntr_version == ARK_P_VERSION_2) { KV_TRC(pAT, "PERSIST_VERSION_2 LOADED"); // copy IV->data from bliv_offset memcpy(_arkp->bl->list->data, pptr->p_cntr_data + pptr->p_cntr_bliv_offset, pptr->p_cntr_bliv_size); } else { rc = EINVAL; KV_TRC_FFDC(pAT, "bad persistent version number: ver:%ld", pptr->p_cntr_version); goto error_exit; } KV_TRC(pAT, "BL_META: n:%ld count:%ld head:%ld hold:%ld top:%ld", _arkp->bl->n, _arkp->bl->count, _arkp->bl->head, _arkp->bl->hold, _arkp->bl->top); } error_exit: am_free(p_data_orig); return rc; }
int main(int argc, char** args) { int c; int rtn; int help_flag = 0; bl* opts; char* me; augment_xylist_t theargs; augment_xylist_t* axy = &theargs; me = find_executable(args[0], NULL); opts = bl_new(4, sizeof(an_option_t)); augment_xylist_add_options(opts); augment_xylist_init(axy); while (1) { c = opts_getopt(opts, argc, args); if (c == -1) break; switch (c) { case 0: fprintf(stderr, "Unknown option '-%c'\n", optopt); exit(-1); case '?': break; case 'h': help_flag = 1; break; default: if (augment_xylist_parse_option(c, optarg, axy)) { exit(-1); } break; } } rtn = 0; if (optind != argc) { int i; printf("Unknown arguments:\n "); for (i=optind; i<argc; i++) { printf("%s ", args[i]); } printf("\n"); help_flag = 1; rtn = -1; } if (!axy->axyfn) { printf("Output filename (-o / --out) is required.\n"); help_flag = 1; rtn = -1; } if (!(axy->imagefn || axy->xylsfn)) { printf("Require either an image (-i / --image) or an XYlist (-x / --xylist) input file.\n"); help_flag = 1; rtn = -1; } if (help_flag) { print_help(args[0], opts); exit(rtn); } bl_free(opts); rtn = augment_xylist(axy, me); augment_xylist_free_contents(axy); return rtn; }
int ark_create_verbose(char *path, ARK **arkret, uint64_t size, uint64_t bsize, uint64_t hcount, int nthrds, int nqueue, int basyncs, uint64_t flags) { int rc = 0; int p_rc = 0; uint64_t bcount = 0; uint64_t x = 0; int i = 0; int tnum = 0; int rnum = 0; scb_t *scbp = NULL; KV_TRC_OPEN(pAT, "arkdb"); if (NULL == arkret) { KV_TRC_FFDC(pAT, "Incorrect value for ARK control block: rc=EINVAL"); rc = EINVAL; goto ark_create_ark_err; } if ( (flags & (ARK_KV_PERSIST_LOAD|ARK_KV_PERSIST_STORE)) && (flags & ARK_KV_VIRTUAL_LUN) ) { KV_TRC_FFDC(pAT, "Invalid persistence combination with ARK flags: %016lx", flags); rc = EINVAL; goto ark_create_ark_err; } if (nthrds <= 0) { KV_TRC_FFDC(pAT, "invalid nthrds:%d", nthrds); rc = EINVAL; goto ark_create_ark_err; } _ARK *ark = am_malloc(sizeof(_ARK)); if (ark == NULL) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocating ARK control structure for %ld", sizeof(_ARK)); goto ark_create_ark_err; } KV_TRC(pAT, "%p path(%s) size %ld bsize %ld hcount %ld " "nthrds %d nqueue %d basyncs %d flags:%08lx", ark, path, size, bsize, hcount, nthrds, nqueue, basyncs, flags); ark->bsize = bsize; ark->rthread = 0; ark->persload = 0; ark->nasyncs = ((nqueue <= 0) ? ARK_MAX_ASYNC_OPS : nqueue); ark->basyncs = basyncs; ark->ntasks = ARK_MAX_TASK_OPS; ark->nthrds = ARK_VERBOSE_NTHRDS_DEF; // hardcode, perf requirement // Create the KV storage, whether that will be memory based // or flash ark->ea = ea_new(path, ark->bsize, basyncs, &size, &bcount, (flags & ARK_KV_VIRTUAL_LUN)); if (ark->ea == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "KV storage initialization failed: rc/errno:%d", rc); goto ark_create_ea_err; } // Now that the "connection" to the store has been established // we need to check to see if data was persisted from a previous // instantiation of the KV store. p_rc = ark_check_persistence(ark, flags); if (p_rc > 0) { // We ran into an error while trying to read from // the store. rc = p_rc; KV_TRC_FFDC(pAT, "Persistence check failed: %d", rc); goto ark_create_persist_err; } else if (p_rc == -1) { KV_TRC(pAT, "NO PERSIST LOAD FLAG"); // There was no persistence data, so we just build off // of what was passed into the API. ark->size = size; ark->bcount = bcount; ark->hcount = hcount; ark->vlimit = ARK_VERBOSE_VLIMIT_DEF; ark->blkbits = ARK_VERBOSE_BLKBITS_DEF; ark->grow = ARK_VERBOSE_GROW_DEF; ark->rthread = 0; ark->flags = flags; ark->astart = 0; ark->blkused = 1; ark->ark_exit = 0; ark->nactive = 0; ark->pers_stats.kv_cnt = 0; ark->pers_stats.blk_cnt = 0; ark->pers_stats.byte_cnt = 0; ark->pcmd = PT_IDLE; // Create the requests and tag control blocks and queues. x = ark->hcount / ark->nthrds; ark->npart = x + (ark->hcount % ark->nthrds ? 1 : 0); // Create the hash table ark->ht = hash_new(ark->hcount); if (ark->ht == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "Hash initialization failed: %d", rc); goto ark_create_ht_err; } // Create the block list ark->bl = bl_new(ark->bcount, ark->blkbits); if (ark->bl == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "Block list initialization failed: %d", rc); goto ark_create_bl_err; } if (flags & ARK_KV_PERSIST_STORE) { ark_persistence_calc(ark); if (bl_reserve(ark->bl, ark->pers_max_blocks)) {goto ark_create_bl_err;} } } else { KV_TRC(pAT, "PERSIST: %p path(%s) size %ld bsize %ld hcount %ld " "nthrds %d nqueue %ld basyncs %d bcount %ld blkbits %ld", ark, path, ark->size, ark->bsize, ark->hcount, ark->nthrds, ark->nasyncs, ark->basyncs, ark->bcount, ark->blkbits); } rc = pthread_mutex_init(&ark->mainmutex,NULL); if (rc != 0) { KV_TRC_FFDC(pAT, "pthread_mutex_init for main mutex failed: %d", rc); goto ark_create_pth_mutex_err; } ark->rtags = tag_new(ark->nasyncs); if ( NULL == ark->rtags ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Tag initialization for requests failed: %d", rc); goto ark_create_rtag_err; } ark->ttags = tag_new(ark->ntasks); if ( NULL == ark->ttags ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Tag initialization for tasks failed: %d", rc); goto ark_create_ttag_err; } ark->rcbs = am_malloc(ark->nasyncs * sizeof(rcb_t)); if ( NULL == ark->rcbs ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for request control blocks", (ark->nasyncs * sizeof(rcb_t))); goto ark_create_rcbs_err; } ark->tcbs = am_malloc(ark->ntasks * sizeof(tcb_t)); if ( NULL == ark->tcbs ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for task control blocks", (ark->ntasks * sizeof(rcb_t))); goto ark_create_tcbs_err; } ark->iocbs = am_malloc(ark->ntasks * sizeof(iocb_t)); if ( NULL == ark->iocbs ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for io control blocks", (ark->ntasks * sizeof(iocb_t))); goto ark_create_iocbs_err; } ark->poolthreads = am_malloc(ark->nthrds * sizeof(scb_t)); if ( NULL == ark->poolthreads ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation of %"PRIu64" bytes for server thread control blocks", (ark->nthrds * sizeof(scb_t))); goto ark_create_poolthreads_err; } for ( rnum = 0; rnum < ark->nasyncs ; rnum++ ) { ark->rcbs[rnum].stat = A_NULL; pthread_cond_init(&(ark->rcbs[rnum].acond), NULL); pthread_mutex_init(&(ark->rcbs[rnum].alock), NULL); } for ( tnum = 0; tnum < ark->ntasks; tnum++ ) { ark->tcbs[tnum].inb = bt_new(0, ark->vlimit, sizeof(uint64_t), &(ark->tcbs[tnum].inblen), &(ark->tcbs[tnum].inb_orig)); if (ark->tcbs[tnum].inb == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "Bucket allocation for inbuffer failed: %d", rc); goto ark_create_taskloop_err; } ark->tcbs[tnum].oub = bt_new(0, ark->vlimit, sizeof(uint64_t), &(ark->tcbs[tnum].oublen), &(ark->tcbs[tnum].oub_orig)); if (ark->tcbs[tnum].oub == NULL) { if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=ENOMEM;} rc = errno; KV_TRC_FFDC(pAT, "Bucket allocation for outbuffer failed: %d", rc); goto ark_create_taskloop_err; } //ark->tcbs[tnum].vbsize = bsize * 1024; ark->tcbs[tnum].vbsize = bsize * 256; ark->tcbs[tnum].vb_orig = am_malloc(ark->tcbs[tnum].vbsize); if (ark->tcbs[tnum].vb_orig == NULL) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation for %"PRIu64" bytes for variable size buffer", (bsize * 1024)); goto ark_create_taskloop_err; } ark->tcbs[tnum].vb = ptr_align(ark->tcbs[tnum].vb_orig); } *arkret = (void *)ark; ark->pts = (PT *)am_malloc(sizeof(PT) * ark->nthrds); if ( ark->pts == NULL ) { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocation for %"PRIu64" bytes for server thread data", (sizeof(PT) * ark->nthrds)); goto ark_create_taskloop_err; } for (i = 0; i < ark->nthrds; i++) { PT *pt = &(ark->pts[i]); scbp = &(ark->poolthreads[i]); memset(scbp, 0, sizeof(scb_t)); // Start off the random start point for this thread // at -1, to show that it has not been part of a // ark_random call. scbp->rlast = -1; scbp->holds = 0; scbp->poolstate = PT_RUN; scbp->poolstats.io_cnt = 0; scbp->poolstats.ops_cnt = 0; scbp->poolstats.kv_cnt = 0; scbp->poolstats.blk_cnt = 0; scbp->poolstats.byte_cnt = 0; pthread_mutex_init(&(scbp->poolmutex), NULL); pthread_cond_init(&(scbp->poolcond), NULL); scbp->rqueue = queue_new(ark->nasyncs); scbp->tqueue = queue_new(ark->ntasks); scbp->ioqueue = queue_new(ark->ntasks); pt->id = i; pt->ark = ark; rc = pthread_create(&(scbp->pooltid), NULL, pool_function, pt); if (rc != 0) { KV_TRC_FFDC(pAT, "pthread_create of server thread failed: %d", rc); goto ark_create_poolloop_err; } } #if 0 while (ark->nactive < ark->nthrds) { usleep(1); //printf("Create waiting %d/%d\n", ark->nactive, ark->nthrds); } #endif ark->pcmd = PT_RUN; goto ark_create_return; ark_create_poolloop_err: for (; i >= 0; i--) { scbp = &(ark->poolthreads[i]); if (scbp->pooltid != 0) { queue_lock(scbp->rqueue); queue_wakeup(scbp->rqueue); queue_unlock(scbp->rqueue); pthread_join(scbp->pooltid, NULL); pthread_mutex_destroy(&(scbp->poolmutex)); pthread_cond_destroy(&(scbp->poolcond)); if ( scbp->rqueue != NULL ) { queue_free(scbp->rqueue); } if ( scbp->tqueue != NULL ) { queue_free(scbp->tqueue); } if ( scbp->ioqueue != NULL ) { queue_free(scbp->ioqueue); } } } if ( ark->pts != NULL ) { am_free(ark->pts); } ark_create_taskloop_err: for ( tnum = 0; tnum < ark->ntasks; tnum++ ) { if (ark->tcbs[tnum].inb) { bt_delete(ark->tcbs[tnum].inb); } if (ark->tcbs[tnum].oub) { bt_delete(ark->tcbs[tnum].oub); } if (ark->tcbs[tnum].vb_orig) { am_free(ark->tcbs[tnum].vb_orig); } } for (rnum = 0; rnum < ark->nasyncs; rnum++) { pthread_cond_destroy(&(ark->rcbs[rnum].acond)); pthread_mutex_destroy(&(ark->rcbs[rnum].alock)); } if ( ark->poolthreads != NULL ) { am_free(ark->poolthreads); } ark_create_poolthreads_err: if (ark->iocbs) { am_free(ark->iocbs); } ark_create_iocbs_err: if (ark->tcbs) { am_free(ark->tcbs); } ark_create_tcbs_err: if (ark->rcbs) { am_free(ark->rcbs); } ark_create_rcbs_err: if (ark->ttags) { tag_free(ark->ttags); } ark_create_ttag_err: if (ark->rtags) { tag_free(ark->rtags); } ark_create_rtag_err: pthread_mutex_destroy(&ark->mainmutex); ark_create_pth_mutex_err: bl_delete(ark->bl); ark_create_bl_err: hash_free(ark->ht); ark_create_ht_err: ark_create_persist_err: ea_delete(ark->ea); ark_create_ea_err: am_free(ark); *arkret = NULL; ark_create_ark_err: KV_TRC_CLOSE(pAT); ark_create_return: return rc; }
int hpquads(startree_t* starkd, codefile_t* codes, quadfile_t* quads, int Nside, double scale_min_arcmin, double scale_max_arcmin, int dimquads, int passes, int Nreuses, int Nloosen, int id, anbool scanoccupied, void* sort_data, int (*sort_func)(const void*, const void*), int sort_size, char** args, int argc) { hpquads_t myhpquads; hpquads_t* me = &myhpquads; int i; int pass; anbool circle = TRUE; double radius2; il* hptotry; int Nhptotry = 0; int nquads; double hprad; double quadscale; int skhp, sknside; qfits_header* qhdr; qfits_header* chdr; int N; int dimcodes; int quadsize; int NHP; memset(me, 0, sizeof(hpquads_t)); if (Nside > HP_MAX_INT_NSIDE) { ERROR("Error: maximum healpix Nside = %i", HP_MAX_INT_NSIDE); return -1; } if (Nreuses > 255) { ERROR("Error, reuse (-r) must be less than 256"); return -1; } me->Nside = Nside; me->dimquads = dimquads; NHP = 12 * Nside * Nside; dimcodes = dimquad2dimcode(dimquads); quadsize = sizeof(unsigned int) * dimquads; logmsg("Nside=%i. Nside^2=%i. Number of healpixes=%i. Healpix side length ~ %g arcmin.\n", me->Nside, me->Nside*me->Nside, NHP, healpix_side_length_arcmin(me->Nside)); me->sort_data = sort_data; me->sort_func = sort_func; me->sort_size = sort_size; tic(); me->starkd = starkd; N = startree_N(me->starkd); logmsg("Star tree contains %i objects.\n", N); // get the "HEALPIX" header from the skdt... skhp = qfits_header_getint(startree_header(me->starkd), "HEALPIX", -1); if (skhp == -1) { if (!qfits_header_getboolean(startree_header(me->starkd), "ALLSKY", FALSE)) { logmsg("Warning: skdt does not contain \"HEALPIX\" header. Code and quad files will not contain this header either.\n"); } } // likewise "HPNSIDE" sknside = qfits_header_getint(startree_header(me->starkd), "HPNSIDE", 1); if (sknside && Nside % sknside) { logerr("Error: Nside (-n) must be a multiple of the star kdtree healpixelisation: %i\n", sknside); return -1; } if (!scanoccupied && (N*(skhp == -1 ? 1 : sknside*sknside*12) < NHP)) { logmsg("\n\n"); logmsg("NOTE, your star kdtree is sparse (has only a fraction of the stars expected)\n"); logmsg(" so you probably will get much faster results by setting the \"-E\" command-line\n"); logmsg(" flag.\n"); logmsg("\n\n"); } quads->dimquads = me->dimquads; codes->dimcodes = dimcodes; quads->healpix = skhp; codes->healpix = skhp; quads->hpnside = sknside; codes->hpnside = sknside; if (id) { quads->indexid = id; codes->indexid = id; } qhdr = quadfile_get_header(quads); chdr = codefile_get_header(codes); add_headers(qhdr, args, argc, startree_header(me->starkd), circle, passes); add_headers(chdr, args, argc, startree_header(me->starkd), circle, passes); if (quadfile_write_header(quads)) { ERROR("Couldn't write headers to quad file"); return -1; } if (codefile_write_header(codes)) { ERROR("Couldn't write headers to code file"); return -1; } quads->numstars = codes->numstars = N; me->quad_dist2_upper = arcmin2distsq(scale_max_arcmin); me->quad_dist2_lower = arcmin2distsq(scale_min_arcmin); codes->index_scale_upper = quads->index_scale_upper = distsq2rad(me->quad_dist2_upper); codes->index_scale_lower = quads->index_scale_lower = distsq2rad(me->quad_dist2_lower); me->nuses = calloc(N, sizeof(unsigned char)); // hprad = sqrt(2) * (healpix side length / 2.) hprad = arcmin2dist(healpix_side_length_arcmin(Nside)) * M_SQRT1_2; quadscale = 0.5 * sqrt(me->quad_dist2_upper); // 1.01 for a bit of safety. we'll look at a few extra stars. radius2 = square(1.01 * (hprad + quadscale)); me->radius2 = radius2; logmsg("Healpix radius %g arcsec, quad scale %g arcsec, total %g arcsec\n", distsq2arcsec(hprad*hprad), distsq2arcsec(quadscale*quadscale), distsq2arcsec(radius2)); hptotry = il_new(1024); if (scanoccupied) { logmsg("Scanning %i input stars...\n", N); for (i=0; i<N; i++) { double xyz[3]; int j; if (startree_get(me->starkd, i, xyz)) { ERROR("Failed to get star %i", i); return -1; } j = xyzarrtohealpix(xyz, Nside); il_insert_unique_ascending(hptotry, j); if (log_get_level() > LOG_VERB) { double ra,dec; if (startree_get_radec(me->starkd, i, &ra, &dec)) { ERROR("Failed to get RA,Dec for star %i\n", i); return -1; } logdebug("star %i: RA,Dec %g,%g; xyz %g,%g,%g; hp %i\n", i, ra, dec, xyz[0], xyz[1], xyz[2], j); } } logmsg("Will check %zu healpixes.\n", il_size(hptotry)); if (log_get_level() > LOG_VERB) { logdebug("Checking healpixes: [ "); for (i=0; i<il_size(hptotry); i++) logdebug("%i ", il_get(hptotry, i)); logdebug("]\n"); } } else { if (skhp == -1) { // Try all healpixes. il_free(hptotry); hptotry = NULL; Nhptotry = NHP; } else { // The star kdtree may itself be healpixed int starhp, starx, stary; // In that case, the healpixes we are interested in form a rectangle // within a big healpix. These are the coords (in [0, Nside)) of // that rectangle. int x0, x1, y0, y1; int x, y; healpix_decompose_xy(skhp, &starhp, &starx, &stary, sknside); x0 = starx * (Nside / sknside); x1 = (starx+1) * (Nside / sknside); y0 = stary * (Nside / sknside); y1 = (stary+1) * (Nside / sknside); for (y=y0; y<y1; y++) { for (x=x0; x<x1; x++) { int j = healpix_compose_xy(starhp, x, y, Nside); il_append(hptotry, j); } } assert(il_size(hptotry) == (Nside/sknside) * (Nside/sknside)); } } if (hptotry) Nhptotry = il_size(hptotry); me->quadlist = bl_new(65536, quadsize); if (Nloosen) me->retryhps = il_new(1024); for (pass=0; pass<passes; pass++) { char key[64]; int nthispass; logmsg("Pass %i of %i.\n", pass+1, passes); logmsg("Trying %i healpixes.\n", Nhptotry); nthispass = build_quads(me, Nhptotry, hptotry, Nreuses); logmsg("Made %i quads (out of %i healpixes) this pass.\n", nthispass, Nhptotry); logmsg("Made %i quads so far.\n", (me->bigquadlist ? bt_size(me->bigquadlist) : 0) + (int)bl_size(me->quadlist)); sprintf(key, "PASS%i", pass+1); fits_header_mod_int(chdr, key, nthispass, "quads created in this pass"); fits_header_mod_int(qhdr, key, nthispass, "quads created in this pass"); logmsg("Merging quads...\n"); if (!me->bigquadlist) me->bigquadlist = bt_new(quadsize, 256); for (i=0; i<bl_size(me->quadlist); i++) { void* q = bl_access(me->quadlist, i); bt_insert2(me->bigquadlist, q, FALSE, compare_quads, &me->dimquads); } bl_remove_all(me->quadlist); } il_free(hptotry); hptotry = NULL; if (Nloosen) { int R; for (R=Nreuses+1; R<=Nloosen; R++) { il* trylist; int nthispass; logmsg("Loosening reuse maximum to %i...\n", R); logmsg("Trying %zu healpixes.\n", il_size(me->retryhps)); if (!il_size(me->retryhps)) break; trylist = me->retryhps; me->retryhps = il_new(1024); nthispass = build_quads(me, il_size(trylist), trylist, R); logmsg("Made %i quads (out of %zu healpixes) this pass.\n", nthispass, il_size(trylist)); il_free(trylist); for (i=0; i<bl_size(me->quadlist); i++) { void* q = bl_access(me->quadlist, i); bt_insert2(me->bigquadlist, q, FALSE, compare_quads, &me->dimquads); } bl_remove_all(me->quadlist); } } if (me->retryhps) il_free(me->retryhps); kdtree_free_query(me->res); me->res = NULL; me->inds = NULL; me->stars = NULL; free(me->nuses); me->nuses = NULL; logmsg("Writing quads...\n"); // add the quads from the big-quadlist nquads = bt_size(me->bigquadlist); for (i=0; i<nquads; i++) { unsigned int* q = bt_access(me->bigquadlist, i); quad_write(codes, quads, q, me->starkd, me->dimquads, dimcodes); } // add the quads that were made during the final round. for (i=0; i<bl_size(me->quadlist); i++) { unsigned int* q = bl_access(me->quadlist, i); quad_write(codes, quads, q, me->starkd, me->dimquads, dimcodes); } // fix output file headers. if (quadfile_fix_header(quads)) { ERROR("Failed to fix quadfile headers"); return -1; } if (codefile_fix_header(codes)) { ERROR("Failed to fix codefile headers"); return -1; } bl_free(me->quadlist); bt_free(me->bigquadlist); toc(); logmsg("Done.\n"); return 0; }
int main(int argc, char** args) { int c; anbool help = FALSE; char* outdir = NULL; char* cmd; int i, j, f; int inputnum; int rtn; sl* engineargs; int nbeargs; anbool fromstdin = FALSE; anbool overwrite = FALSE; anbool cont = FALSE; anbool skip_solved = FALSE; anbool makeplots = TRUE; double plotscale = 1.0; char* inbgfn = NULL; char* bgfn = NULL; char* me; anbool verbose = FALSE; int loglvl = LOG_MSG; char* outbase = NULL; anbool usecurl = TRUE; bl* opts; augment_xylist_t theallaxy; augment_xylist_t* allaxy = &theallaxy; int nmyopts; char* removeopts = "ixo\x01"; char* newfits; char* kmz = NULL; char* scamp = NULL; char* scampconfig = NULL; char* index_xyls; anbool just_augment = FALSE; anbool engine_batch = FALSE; bl* batchaxy = NULL; bl* batchsf = NULL; sl* outfiles; sl* tempfiles; // these are deleted after the outer loop over input files sl* tempfiles2; sl* tempdirs; anbool timestamp = FALSE; anbool tempaxy = FALSE; errors_print_on_exit(stderr); fits_use_error_system(); me = find_executable(args[0], NULL); engineargs = sl_new(16); append_executable(engineargs, "astrometry-engine", me); // output filenames. outfiles = sl_new(16); tempfiles = sl_new(4); tempfiles2 = sl_new(4); tempdirs = sl_new(4); rtn = 0; nmyopts = sizeof(options)/sizeof(an_option_t); opts = opts_from_array(options, nmyopts, NULL); augment_xylist_add_options(opts); // remove duplicate short options. for (i=0; i<nmyopts; i++) { an_option_t* opt1 = bl_access(opts, i); for (j=nmyopts; j<bl_size(opts); j++) { an_option_t* opt2 = bl_access(opts, j); if (opt2->shortopt == opt1->shortopt) bl_remove_index(opts, j); } } // remove unwanted augment-xylist options. for (i=0; i<strlen(removeopts); i++) { for (j=nmyopts; j<bl_size(opts); j++) { an_option_t* opt2 = bl_access(opts, j); if (opt2->shortopt == removeopts[i]) bl_remove_index(opts, j); } } // which options are left? /*{ char options[256]; memset(options, 0, 256); printf("options:\n"); for (i=0; i<bl_size(opts); i++) { an_option_t* opt = bl_access(opts, i); printf(" %c (%i) %s\n", opt->shortopt, (int)opt->shortopt, opt->name); options[(int)((opt->shortopt + 256) % 256)] = 1; } printf("Remaining short opts:\n"); for (i=0; i<256; i++) { if (!options[i]) printf(" %c (%i, 0x%x)\n", (char)i, i, i); } }*/ augment_xylist_init(allaxy); // default output filename patterns. allaxy->axyfn = "%s.axy"; allaxy->matchfn = "%s.match"; allaxy->rdlsfn = "%s.rdls"; allaxy->solvedfn = "%s.solved"; allaxy->wcsfn = "%s.wcs"; allaxy->corrfn = "%s.corr"; newfits = "%s.new"; index_xyls = "%s-indx.xyls"; while (1) { int res; c = opts_getopt(opts, argc, args); //printf("option %c (%i)\n", c, (int)c); if (c == -1) break; switch (c) { case '\x91': allaxy->axyfn = optarg; break; case '\x90': tempaxy = TRUE; break; case '\x88': timestamp = TRUE; break; case '\x84': plotscale = atof(optarg); break; case '\x85': inbgfn = optarg; break; case '\x87': allaxy->assume_fits_image = TRUE; break; case '(': engine_batch = TRUE; break; case '@': just_augment = TRUE; break; case 'U': index_xyls = optarg; break; case 'n': scampconfig = optarg; break; case 'i': scamp = optarg; break; case 'Z': kmz = optarg; break; case 'N': newfits = optarg; break; case 'h': help = TRUE; break; case 'v': sl_append(engineargs, "--verbose"); verbose = TRUE; allaxy->verbosity++; loglvl++; break; case 'D': outdir = optarg; break; case 'o': outbase = optarg; break; case 'b': case '\x89': sl_append(engineargs, "--config"); append_escape(engineargs, optarg); break; case 'f': fromstdin = TRUE; break; case 'O': overwrite = TRUE; break; case 'p': makeplots = FALSE; break; case 'G': usecurl = FALSE; break; case 'K': cont = TRUE; break; case 'J': skip_solved = TRUE; break; default: res = augment_xylist_parse_option(c, optarg, allaxy); if (res) { rtn = -1; goto dohelp; } } } if ((optind == argc) && !fromstdin) { printf("ERROR: You didn't specify any files to process.\n"); help = TRUE; } if (help) { dohelp: print_help(args[0], opts); exit(rtn); } bl_free(opts); // --dont-augment: advertised as just write xy file, // so quit after doing that. if (allaxy->dont_augment) { just_augment = TRUE; } log_init(loglvl); if (timestamp) log_set_timestamp(TRUE); if (kmz && starts_with(kmz, "-")) logmsg("Do you really want to save KMZ to the file named \"%s\" ??\n", kmz); if (starts_with(newfits, "-")) { logmsg("Do you really want to save the new FITS file to the file named \"%s\" ??\n", newfits); } if (engine_batch) { batchaxy = bl_new(16, sizeof(augment_xylist_t)); batchsf = bl_new(16, sizeof(solve_field_args_t)); } // Allow (some of the) default filenames to be disabled by setting them to "none". allaxy->matchfn = none_is_null(allaxy->matchfn); allaxy->rdlsfn = none_is_null(allaxy->rdlsfn); allaxy->solvedfn = none_is_null(allaxy->solvedfn); allaxy->solvedinfn = none_is_null(allaxy->solvedinfn); allaxy->wcsfn = none_is_null(allaxy->wcsfn); allaxy->corrfn = none_is_null(allaxy->corrfn); newfits = none_is_null(newfits); index_xyls = none_is_null(index_xyls); if (outdir) { if (mkdir_p(outdir)) { ERROR("Failed to create output directory %s", outdir); exit(-1); } } // number of engine args not specific to a particular file nbeargs = sl_size(engineargs); f = optind; inputnum = 0; while (1) { char* infile = NULL; anbool isxyls; char* reason; int len; char* base; char* basedir; char* basefile = NULL; char *objsfn=NULL; char *ppmfn=NULL; char* downloadfn = NULL; char* suffix = NULL; sl* cmdline; anbool ctrlc; anbool isurl; augment_xylist_t theaxy; augment_xylist_t* axy = &theaxy; int j; solve_field_args_t thesf; solve_field_args_t* sf = &thesf; anbool want_pnm = FALSE; // reset augment-xylist args. memcpy(axy, allaxy, sizeof(augment_xylist_t)); memset(sf, 0, sizeof(solve_field_args_t)); if (fromstdin) { char fnbuf[1024]; if (!fgets(fnbuf, sizeof(fnbuf), stdin)) { if (ferror(stdin)) SYSERROR("Failed to read a filename from stdin"); break; } len = strlen(fnbuf); if (fnbuf[len-1] == '\n') fnbuf[len-1] = '\0'; infile = fnbuf; logmsg("Reading input file \"%s\"...\n", infile); } else { if (f == argc) break; infile = args[f]; f++; logmsg("Reading input file %i of %i: \"%s\"...\n", f - optind, argc - optind, infile); } inputnum++; cmdline = sl_new(16); if (!engine_batch) { // Remove arguments that might have been added in previous trips through this loop sl_remove_from(engineargs, nbeargs); } // Choose the base path/filename for output files. if (outbase) asprintf_safe(&basefile, outbase, inputnum, infile); else basefile = basename_safe(infile); //logverb("Base filename: %s\n", basefile); isurl = (!file_exists(infile) && (starts_with(infile, "http://") || starts_with(infile, "ftp://"))); if (outdir) basedir = strdup(outdir); else { if (isurl) basedir = strdup("."); else basedir = dirname_safe(infile); } //logverb("Base directory: %s\n", basedir); asprintf_safe(&base, "%s/%s", basedir, basefile); //logverb("Base name for output files: %s\n", base); // trim .gz, .bz2 // hmm, we drop the suffix in this case... len = strlen(base); if (ends_with(base, ".gz")) base[len-3] = '\0'; else if (ends_with(base, ".bz2")) base[len-4] = '\0'; len = strlen(base); // trim .xx / .xxx / .xxxx if (len >= 5) { for (j=3; j<=5; j++) { if (base[len - j] == '/') break; if (base[len - j] == '.') { base[len - j] = '\0'; suffix = base + len - j + 1; break; } } } logverb("Base: \"%s\", basefile \"%s\", basedir \"%s\", suffix \"%s\"\n", base, basefile, basedir, suffix); if (tempaxy) { axy->axyfn = create_temp_file("axy", axy->tempdir); sl_append_nocopy(tempfiles2, axy->axyfn); } else axy->axyfn = sl_appendf(outfiles, axy->axyfn, base); if (axy->matchfn) axy->matchfn = sl_appendf(outfiles, axy->matchfn, base); if (axy->rdlsfn) axy->rdlsfn = sl_appendf(outfiles, axy->rdlsfn, base); if (axy->solvedfn) axy->solvedfn = sl_appendf(outfiles, axy->solvedfn, base); if (axy->wcsfn) axy->wcsfn = sl_appendf(outfiles, axy->wcsfn, base); if (axy->corrfn) axy->corrfn = sl_appendf(outfiles, axy->corrfn, base); if (axy->cancelfn) axy->cancelfn = sl_appendf(outfiles, axy->cancelfn, base); if (axy->keepxylsfn) axy->keepxylsfn = sl_appendf(outfiles, axy->keepxylsfn, base); if (axy->pnmfn) axy->pnmfn = sl_appendf(outfiles, axy->pnmfn, base); if (newfits) sf->newfitsfn = sl_appendf(outfiles, newfits, base); if (kmz) sf->kmzfn = sl_appendf(outfiles, kmz, base); if (index_xyls) sf->indxylsfn = sl_appendf(outfiles, index_xyls, base); if (scamp) sf->scampfn = sl_appendf(outfiles, scamp, base); if (scampconfig) sf->scampconfigfn = sl_appendf(outfiles, scampconfig, base); if (makeplots) { objsfn = sl_appendf(outfiles, "%s-objs.png", base); sf->redgreenfn = sl_appendf(outfiles, "%s-indx.png", base); sf->ngcfn = sl_appendf(outfiles, "%s-ngc.png", base); } if (isurl) { if (suffix) downloadfn = sl_appendf(outfiles, "%s.%s", base, suffix); else downloadfn = sl_appendf(outfiles, "%s", base); } if (axy->solvedinfn) asprintf_safe(&axy->solvedinfn, axy->solvedinfn, base); // Do %s replacement on --verify-wcs entries... if (sl_size(axy->verifywcs)) { sl* newlist = sl_new(4); for (j=0; j<sl_size(axy->verifywcs); j++) sl_appendf(newlist, sl_get(axy->verifywcs, j), base); axy->verifywcs = newlist; } // ... and plot-bg if (inbgfn) asprintf_safe(&bgfn, inbgfn, base); if (axy->solvedinfn && axy->solvedfn && streq(axy->solvedfn, axy->solvedinfn)) { // solved input and output files are the same: don't delete the input! sl_remove_string(outfiles, axy->solvedfn); free(axy->solvedfn); axy->solvedfn = axy->solvedinfn; } free(basedir); free(basefile); if (skip_solved) { char* tocheck[] = { axy->solvedinfn, axy->solvedfn }; for (j=0; j<sizeof(tocheck)/sizeof(char*); j++) { if (!tocheck[j]) continue; logverb("Checking for solved file %s\n", tocheck[j]); if (file_exists(tocheck[j])) { logmsg("Solved file exists: %s; skipping this input file.\n", tocheck[j]); goto nextfile; } else { logverb("File \"%s\" does not exist.\n", tocheck[j]); } } } // Check for overlap between input and output filenames for (i = 0; i < sl_size(outfiles); i++) { char* fn = sl_get(outfiles, i); if (streq(fn, infile)) { logmsg("Output filename \"%s\" is the same as your input file.\n" "Refusing to continue.\n" "You can either choose a different output filename, or\n" "rename your input file to have a different extension.\n", fn); goto nextfile; } } // Check for (and possibly delete) existing output filenames. for (i = 0; i < sl_size(outfiles); i++) { char* fn = sl_get(outfiles, i); if (!file_exists(fn)) continue; if (cont) { } else if (overwrite) { if (unlink(fn)) { SYSERROR("Failed to delete an already-existing output file \"%s\"", fn); exit(-1); } } else { logmsg("Output file already exists: \"%s\".\n" "Use the --overwrite flag to overwrite existing files,\n" " or the --continue flag to not overwrite existing files but still try solving.\n", fn); logmsg("Continuing to next input file.\n"); goto nextfile; } } // if we're making "redgreen" plot, we need: if (sf->redgreenfn) { // -- index xylist if (!sf->indxylsfn) { sf->indxylsfn = create_temp_file("indxyls", axy->tempdir); sl_append_nocopy(tempfiles, sf->indxylsfn); } // -- match file. if (!axy->matchfn) { axy->matchfn = create_temp_file("match", axy->tempdir); sl_append_nocopy(tempfiles, axy->matchfn); } } // if index xyls file is needed, we need: if (sf->indxylsfn) { // -- wcs if (!axy->wcsfn) { axy->wcsfn = create_temp_file("wcs", axy->tempdir); sl_append_nocopy(tempfiles, axy->wcsfn); } // -- rdls if (!axy->rdlsfn) { axy->rdlsfn = create_temp_file("rdls", axy->tempdir); sl_append_nocopy(tempfiles, axy->rdlsfn); } } // Download URL... if (isurl) { sl_append(cmdline, usecurl ? "curl" : "wget"); if (!verbose) sl_append(cmdline, usecurl ? "--silent" : "--quiet"); sl_append(cmdline, usecurl ? "--output" : "-O"); append_escape(cmdline, downloadfn); append_escape(cmdline, infile); cmd = sl_implode(cmdline, " "); logmsg("Downloading...\n"); if (run_command(cmd, &ctrlc)) { ERROR("%s command %s", sl_get(cmdline, 0), (ctrlc ? "was cancelled" : "failed")); exit(-1); } sl_remove_all(cmdline); free(cmd); infile = downloadfn; } if (makeplots) want_pnm = TRUE; if (axy->assume_fits_image) { axy->imagefn = infile; if (axy->pnmfn) want_pnm = TRUE; } else { logverb("Checking if file \"%s\" ext %i is xylist or image: ", infile, axy->extension); fflush(NULL); reason = NULL; isxyls = xylist_is_file_xylist(infile, axy->extension, axy->xcol, axy->ycol, &reason); logverb(isxyls ? "xyls\n" : "image\n"); if (!isxyls) logverb(" (not xyls because: %s)\n", reason); free(reason); fflush(NULL); if (isxyls) axy->xylsfn = infile; else { axy->imagefn = infile; want_pnm = TRUE; } } if (want_pnm && !axy->pnmfn) { ppmfn = create_temp_file("ppm", axy->tempdir); sl_append_nocopy(tempfiles, ppmfn); axy->pnmfn = ppmfn; axy->force_ppm = TRUE; } axy->keep_fitsimg = (newfits || scamp); if (augment_xylist(axy, me)) { ERROR("augment-xylist failed"); exit(-1); } if (just_augment) goto nextfile; if (makeplots) { // Check that the plotting executables were built... char* exec = find_executable("plotxy", me); free(exec); if (!exec) { logmsg("Couldn't find \"plotxy\" executable - maybe you didn't build the plotting programs?\n"); logmsg("Disabling plots.\n"); makeplots = FALSE; } } if (makeplots) { // source extraction overlay if (plot_source_overlay(axy, me, objsfn, plotscale, bgfn)) makeplots = FALSE; } append_escape(engineargs, axy->axyfn); if (file_readable(axy->wcsfn)) axy->wcs_last_mod = file_get_last_modified_time(axy->wcsfn); else axy->wcs_last_mod = 0; if (!engine_batch) { run_engine(engineargs); after_solved(axy, sf, makeplots, me, verbose, axy->tempdir, tempdirs, tempfiles, plotscale, bgfn); } else { bl_append(batchaxy, axy); bl_append(batchsf, sf ); } fflush(NULL); // clean up and move on to the next file. nextfile: free(base); sl_free2(cmdline); if (!engine_batch) { free(axy->fitsimgfn); free(axy->solvedinfn); free(bgfn); // erm. if (axy->verifywcs != allaxy->verifywcs) sl_free2(axy->verifywcs); sl_remove_all(outfiles); if (!axy->no_delete_temp) delete_temp_files(tempfiles, tempdirs); } errors_print_stack(stdout); errors_clear_stack(); logmsg("\n"); } if (engine_batch) { run_engine(engineargs); for (i=0; i<bl_size(batchaxy); i++) { augment_xylist_t* axy = bl_access(batchaxy, i); solve_field_args_t* sf = bl_access(batchsf, i); after_solved(axy, sf, makeplots, me, verbose, axy->tempdir, tempdirs, tempfiles, plotscale, bgfn); errors_print_stack(stdout); errors_clear_stack(); logmsg("\n"); free(axy->fitsimgfn); free(axy->solvedinfn); // erm. if (axy->verifywcs != allaxy->verifywcs) sl_free2(axy->verifywcs); } if (!allaxy->no_delete_temp) delete_temp_files(tempfiles, tempdirs); bl_free(batchaxy); bl_free(batchsf); } if (!allaxy->no_delete_temp) delete_temp_files(tempfiles2, NULL); sl_free2(outfiles); sl_free2(tempfiles); sl_free2(tempfiles2); sl_free2(tempdirs); sl_free2(engineargs); free(me); augment_xylist_free_contents(allaxy); return 0; }