// Returns a newly-allocated array containing the raw header bytes for the // given extension. char* anqfits_header_get_data(const anqfits_t* qf, int ext, int* Nbytes) { FILE* fid; off_t N, nr; char* data; off_t start; start = anqfits_header_start(qf, ext); if (start == -1) return NULL; N = anqfits_header_size(qf, ext); if (N == -1) return NULL; fid = fopen(qf->filename, "rb"); if (!fid) { return NULL; } data = malloc(N + 1); if (start) { if (fseeko(fid, start, SEEK_SET)) { SYSERROR("Failed to seek to start of FITS header: byte %li in %s", (long int)start, qf->filename); return NULL; } } nr = fread(data, 1, N, fid); fclose(fid); if (nr != N) { free(data); return NULL; } data[N] = '\0'; if (Nbytes) *Nbytes = N; return data; }
int anqfits_get_header_start_and_size(const anqfits_t* qf, int ext, off_t* pstart, off_t* psize) { if (pstart) { *pstart = anqfits_header_start(qf, ext); if (*pstart == -1) return -1; } if (psize) { *psize = anqfits_header_size(qf, ext); if (*psize == -1) return -1; } return 0; }
const qfits_header* anqfits_get_header_const(const anqfits_t* qf, int ext) { assert(ext >= 0 && ext < qf->Nexts); if (!qf->exts[ext].header) { off_t start, size; char* str; start = anqfits_header_start(qf, ext); size = anqfits_header_size (qf, ext); if ((start == -1) || (size == -1)) { ERROR("failed to get header start + size for file \"%s\" extension %i", qf->filename, ext); return NULL; } str = file_get_contents_offset(qf->filename, (int)start, (int)size); if (!str) { ERROR("failed to read \"%s\" extension %i: offset %i size %i\n", qf->filename, ext, (int)start, (int)size); return NULL; } qf->exts[ext].header = qfits_header_read_hdr_string ((unsigned char*)str, (int)size); } return qf->exts[ext].header; }
int main(int argc, char *argv[]) { int argchar; char* infn = NULL; char* outfn = NULL; FILE* fin = NULL; FILE* fout = NULL; pl* cols; char* progname = argv[0]; int nextens; int ext; int NC; int start, size; anqfits_t* anq = NULL; qfits_table* outtable; unsigned char* buffer; cols = pl_new(16); while ((argchar = getopt (argc, argv, OPTIONS)) != -1) switch (argchar) { case 'c': pl_append(cols, optarg); break; case 'i': infn = optarg; break; case 'o': outfn = optarg; break; case '?': case 'h': printHelp(progname); return 0; default: return -1; } if (!infn || !outfn || !pl_size(cols)) { printHelp(progname); exit(-1); } fin = fopen(infn, "rb"); if (!fin) { ERROR("Failed to open input file %s: %s\n", infn, strerror(errno)); exit(-1); } fout = fopen(outfn, "wb"); if (!fout) { ERROR("Failed to open output file %s: %s\n", outfn, strerror(errno)); exit(-1); } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to read \"%s\"", infn); exit(-1); } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary header.\n"); exit(-1); } NC = pl_size(cols); nextens = anqfits_n_ext(anq); printf("Translating %i extensions.\n", nextens); buffer = NULL; for (ext=1; ext<nextens; ext++) { int c2, c; int columns[NC]; int sizes[NC]; int offsets[NC]; int offset = 0; int off, n; int totalsize = 0; const int BLOCK=1000; qfits_table* table; qfits_header* header; qfits_header* tablehdr; if (ext%100 == 0) { printf("Extension %i.\n", ext); fflush(stdout); } if (!anqfits_is_table(anq, ext)) { ERROR("extention %i isn't a table.\n", ext); // HACK - just copy it. return -1; } table = anqfits_get_table(anq, ext); if (!table) { ERROR("failed to open table: file %s, extension %i.\n", infn, ext); return -1; } header = anqfits_get_header(anq, ext); if (!header) { ERROR("failed to read header: extension %i\n", ext); exit(-1); } outtable = qfits_table_new(outfn, QFITS_BINTABLE, 0, NC, table->nr); outtable->tab_w = 0; for (c=0; c<pl_size(cols); c++) { columns[c] = -1; } for (c=0; c<pl_size(cols); c++) { char* colname = pl_get(cols, c); qfits_col* col; c2 = fits_find_column(table, colname); if (c2 == -1) { ERROR("Extension %i: failed to find column named %s\n", ext, colname); exit(-1); } col = table->col + c2; columns[c] = c2; sizes[c] = col->atom_nb * col->atom_size; offsets[c] = offset; offset += sizes[c]; qfits_col_fill(outtable->col + c, col->atom_nb, col->atom_dec_nb, col->atom_size, col->atom_type, col->tlabel, col->tunit, col->nullval, col->tdisp, col->zero_present, col->zero, col->scale_present, col->scale, outtable->tab_w); outtable->tab_w += sizes[c]; } totalsize = offset; tablehdr = qfits_table_ext_header_default(outtable); // add any headers from the original table that aren't part of the BINTABLE extension. fits_copy_non_table_headers(tablehdr, header); qfits_header_dump(tablehdr, fout); qfits_header_destroy(tablehdr); buffer = realloc(buffer, totalsize * BLOCK); for (off=0; off<table->nr; off+=n) { if (off + BLOCK > table->nr) n = table->nr - off; else n = BLOCK; for (c=0; c<pl_size(cols); c++) qfits_query_column_seq_to_array_no_endian_swap (table, columns[c], off, n, buffer + offsets[c], totalsize); if (fwrite(buffer, totalsize, n, fout) != n) { ERROR("Error writing a block of data: ext %i: %s\n", ext, strerror(errno)); exit(-1); } } qfits_table_close(outtable); fits_pad_file(fout); qfits_header_destroy(header); qfits_table_close(table); } free(buffer); if (fclose(fout)) { ERROR("Error closing output file: %s\n", strerror(errno)); } fclose(fin); anqfits_close(anq); pl_free(cols); return 0; }
int resort_xylist(const char* infn, const char* outfn, const char* fluxcol, const char* backcol, anbool ascending) { FILE* fin = NULL; FILE* fout = NULL; double *flux = NULL, *back = NULL; int *perm1 = NULL, *perm2 = NULL; anbool *used = NULL; int start, size, nextens, ext; int (*compare)(const void*, const void*); fitstable_t* tab = NULL; anqfits_t* anq = NULL; if (ascending) compare = compare_doubles_asc; else compare = compare_doubles_desc; if (!fluxcol) fluxcol = "FLUX"; if (!backcol) backcol = "BACKGROUND"; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open file \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); tab = fitstable_open(infn); if (!tab) { ERROR("Failed to open FITS table in file %s", infn); goto bailout; } for (ext=1; ext<nextens; ext++) { int hdrstart, hdrsize, datstart; int i, N; int rowsize; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extention %i isn't a table. Skipping", ext); continue; } // Copy the header as-is. if (pipe_file_offset(fin, hdrstart, hdrsize, fout)) { ERROR("Failed to copy the header of extension %i", ext); goto bailout; } if (fitstable_read_extension(tab, ext)) { ERROR("Failed to read FITS table from extension %i", ext); goto bailout; } rowsize = fitstable_row_size(tab); // read FLUX column as doubles. flux = fitstable_read_column(tab, fluxcol, TFITS_BIN_TYPE_D); if (!flux) { ERROR("Failed to read FLUX column from extension %i", ext); goto bailout; } // BACKGROUND back = fitstable_read_column(tab, backcol, TFITS_BIN_TYPE_D); if (!back) { ERROR("Failed to read BACKGROUND column from extension %i", ext); goto bailout; } debug("First 10 rows of input table:\n"); for (i=0; i<10; i++) debug("flux %g, background %g\n", flux[i], back[i]); N = fitstable_nrows(tab); // set back = flux + back (ie, non-background-subtracted flux) for (i=0; i<N; i++) back[i] += flux[i]; // Sort by flux... perm1 = permuted_sort(flux, sizeof(double), compare, NULL, N); // Sort by non-background-subtracted flux... perm2 = permuted_sort(back, sizeof(double), compare, NULL, N); used = malloc(N * sizeof(anbool)); memset(used, 0, N * sizeof(anbool)); // Check sort... for (i=0; i<N-1; i++) { if (ascending) { assert(flux[perm1[i]] <= flux[perm1[i+1]]); assert(back[perm2[i]] <= back[perm2[i+1]]); } else { assert(flux[perm1[i]] >= flux[perm1[i+1]]); assert(back[perm2[i]] >= back[perm2[i+1]]); } } for (i=0; i<N; i++) { int j; int inds[] = { perm1[i], perm2[i] }; for (j=0; j<2; j++) { int index = inds[j]; assert(index < N); if (used[index]) continue; used[index] = TRUE; debug("adding index %i: %s %g\n", index, j==0 ? "flux" : "bgsub", j==0 ? flux[index] : back[index]); if (pipe_file_offset(fin, datstart + index * rowsize, rowsize, fout)) { ERROR("Failed to copy row %i", index); goto bailout; } } } for (i=0; i<N; i++) assert(used[i]); if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } free(flux); flux = NULL; free(back); back = NULL; free(perm1); perm1 = NULL; free(perm2); perm2 = NULL; free(used); used = NULL; } fitstable_close(tab); tab = NULL; if (fclose(fout)) { SYSERROR("Failed to close output file %s", outfn); return -1; } fclose(fin); return 0; bailout: if (tab) fitstable_close(tab); if (fout) fclose(fout); if (fin) fclose(fin); free(flux); free(back); free(perm1); free(perm2); free(used); return -1; }
int tabsort(const char* infn, const char* outfn, const char* colname, int descending) { FILE* fin; FILE* fout; int ext, nextens; off_t start, size; void* data = NULL; int* perm = NULL; unsigned char* map = NULL; size_t mapsize = 0; anqfits_t* anq = NULL; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); //logverb("Sorting %i extensions.\n", nextens); for (ext=1; ext<nextens; ext++) { int c; qfits_table* table; qfits_col* col; int mgap; off_t mstart; size_t msize; int atomsize; int (*sort_func)(const void*, const void*); unsigned char* tabledata; unsigned char* tablehdr; off_t hdrstart, hdrsize, datsize, datstart; int i; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); datsize = anqfits_data_size (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extension %i isn't a table. Skipping.\n", ext); continue; } table = anqfits_get_table(anq, ext); if (!table) { ERROR("Failed to open table: file %s, extension %i. Skipping.", infn, ext); continue; } c = fits_find_column(table, colname); if (c == -1) { ERROR("Couldn't find column named \"%s\" in extension %i. Skipping.", colname, ext); continue; } col = table->col + c; switch (col->atom_type) { case TFITS_BIN_TYPE_D: data = realloc(data, table->nr * sizeof(double)); if (descending) sort_func = compare_doubles_desc; else sort_func = compare_doubles_asc; break; case TFITS_BIN_TYPE_E: data = realloc(data, table->nr * sizeof(float)); if (descending) sort_func = compare_floats_desc; else sort_func = compare_floats_asc; break; case TFITS_BIN_TYPE_K: data = realloc(data, table->nr * sizeof(int64_t)); if (descending) sort_func = compare_int64_desc; else sort_func = compare_int64_asc; break; default: ERROR("Column %s is neither FITS type D, E, nor K. Skipping.", colname); continue; } // Grab the sort column. atomsize = fits_get_atom_size(col->atom_type); printf("Reading sort column \"%s\"\n", colname); qfits_query_column_seq_to_array(table, c, 0, table->nr, data, atomsize); // Sort it. printf("Sorting sort column\n"); perm = permuted_sort(data, atomsize, sort_func, NULL, table->nr); // mmap the input file. printf("mmapping input file\n"); start = hdrstart; size = hdrsize + datsize; get_mmap_size(start, size, &mstart, &msize, &mgap); mapsize = msize; map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, fileno(fin), mstart); if (map == MAP_FAILED) { SYSERROR("Failed to mmap input file %s", infn); map = NULL; goto bailout; } tabledata = map + (off_t)mgap + (datstart - hdrstart); tablehdr = map + (off_t)mgap; // Copy the table header without change. printf("Copying table header.\n"); if (fwrite(tablehdr, 1, hdrsize, fout) != hdrsize) { SYSERROR("Failed to write FITS table header"); goto bailout; } for (i=0; i<table->nr; i++) { unsigned char* rowptr; if (i % 100000 == 0) printf("Writing row %i\n", i); rowptr = tabledata + (off_t)(perm[i]) * (off_t)table->tab_w; if (fwrite(rowptr, 1, table->tab_w, fout) != table->tab_w) { SYSERROR("Failed to write FITS table row"); goto bailout; } } munmap(map, mapsize); map = NULL; free(perm); perm = NULL; if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } qfits_table_close(table); } free(data); if (fclose(fout)) { SYSERROR("Error closing output file"); fout = NULL; goto bailout; } fclose(fin); anqfits_close(anq); printf("Done\n"); return 0; bailout: free(data); free(perm); if (fout) fclose(fout); fclose(fin); if (map) munmap(map, mapsize); return -1; }
int main(int argc, char *argv[]) { int argchar; char* infn = NULL; char* outfn = NULL; anbool tostdout = FALSE; FILE* fin = NULL; FILE* fout = NULL; il* exts; int i; char* progname = argv[0]; anbool inblocks = FALSE; anbool inmegs = FALSE; int allexts = 0; int Next = -1; anbool dataonly = FALSE; anbool headeronly = FALSE; anqfits_t* anq = NULL; int loglvl = LOG_MSG; exts = il_new(16); while ((argchar = getopt (argc, argv, OPTIONS)) != -1) switch (argchar) { case 'v': loglvl++; break; case 'D': dataonly = TRUE; break; case 'H': headeronly = TRUE; break; case 'a': allexts = 1; break; case 'b': inblocks = TRUE; break; case 'M': inmegs = TRUE; break; case 'e': il_append(exts, atoi(optarg)); break; case 'i': infn = optarg; break; case 'o': outfn = optarg; break; case '?': case 'h': printHelp(progname); return 0; default: return -1; } if (headeronly && dataonly) { fprintf(stderr, "Can't write data blocks only AND header blocks only!\n"); exit(-1); } if (inblocks && inmegs) { fprintf(stderr, "Can't write sizes in FITS blocks and megabytes.\n"); exit(-1); } fits_use_error_system(); log_init(loglvl); log_to(stderr); errors_log_to(stderr); if (infn) { anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open input file \"%s\"", infn); exit(-1); } Next = anqfits_n_ext(anq); fprintf(stderr, "File %s contains %i FITS extensions.\n", infn, Next); } if (infn && !outfn) { for (i=0; i<Next; i++) { off_t hdrstart, hdrlen, datastart, datalen; hdrstart = anqfits_header_start(anq, i); hdrlen = anqfits_header_size(anq, i); datastart = anqfits_data_start(anq, i); datalen = anqfits_data_size(anq, i); if (inblocks) { off_t block = (off_t)FITS_BLOCK_SIZE; fprintf(stderr, "Extension %i : header start %zu , length %zu ; data start %zu , length %zu blocks.\n", i, (size_t)(hdrstart / block), (size_t)(hdrlen / block), (size_t)(datastart / block), (size_t)(datalen / block)); } else if (inmegs) { off_t meg = 1024*1024; fprintf(stderr, "Extension %i : header start %zu , length %zu ; data start %zu , length %zu megabytes.\n", i, (size_t)(hdrstart/meg), (size_t)(hdrlen/meg), (size_t)(datastart/meg), (size_t)(datalen/meg)); } else { fprintf(stderr, "Extension %i : header start %zu , length %zu ; data start %zu , length %zu .\n", i, (size_t)hdrstart, (size_t)hdrlen, (size_t)datastart, (size_t)datalen); } } anqfits_close(anq); exit(0); } if (!infn || !outfn || !(il_size(exts) || allexts)) { printHelp(progname); exit(-1); } if (!strcmp(outfn, "-")) { tostdout = TRUE; if (allexts) { fprintf(stderr, "Specify all extensions (-a) and outputting to stdout (-o -) doesn't make much sense...\n"); exit(-1); } } if (infn) { fin = fopen(infn, "rb"); if (!fin) { fprintf(stderr, "Failed to open input file %s: %s\n", infn, strerror(errno)); exit(-1); } } if (tostdout) fout = stdout; else { if (allexts) for (i=0; i<Next; i++) il_append(exts, i); else { // open the (single) output file. fout = fopen(outfn, "wb"); if (!fout) { fprintf(stderr, "Failed to open output file %s: %s\n", outfn, strerror(errno)); exit(-1); } } } for (i=0; i<il_size(exts); i++) { off_t hdrstart, hdrlen, datastart, datalen; int ext = il_get(exts, i); if (allexts) { char fn[256]; snprintf(fn, sizeof(fn), outfn, ext); fout = fopen(fn, "wb"); if (!fout) { fprintf(stderr, "Failed to open output file %s: %s\n", fn, strerror(errno)); exit(-1); } } hdrstart = anqfits_header_start(anq, ext); hdrlen = anqfits_header_size(anq, ext); datastart = anqfits_data_start(anq, ext); datalen = anqfits_data_size(anq, ext); if (inblocks) { off_t block = (off_t)FITS_BLOCK_SIZE; fprintf(stderr, "Writing extension %i : header start %zu , length %zu ; data start %zu , length %zu blocks.\n", ext, (size_t)(hdrstart / block), (size_t)(hdrlen / block), (size_t)(datastart / block), (size_t)(datalen / block)); } else if (inmegs) { off_t meg = 1024*1024; fprintf(stderr, "Writing extension %i : header start %zu , length %zu ; data start %zu , length %zu megabytes.\n", ext, (size_t)(hdrstart/meg), (size_t)(hdrlen/meg), (size_t)(datastart/meg), (size_t)(datalen/meg)); } else { fprintf(stderr, "Writing extension %i : header start %zu , length %zu ; data start %zu , length %zu .\n", ext, (size_t)hdrstart, (size_t)hdrlen, (size_t)datastart, (size_t)datalen); } if (hdrlen && !dataonly) { if (pipe_file_offset(fin, hdrstart, hdrlen, fout)) { fprintf(stderr, "Failed to write header for extension %i: %s\n", ext, strerror(errno)); exit(-1); } } if (datalen && !headeronly) { if (pipe_file_offset(fin, datastart, datalen, fout)) { fprintf(stderr, "Failed to write data for extension %i: %s\n", ext, strerror(errno)); exit(-1); } } if (allexts) if (fclose(fout)) { fprintf(stderr, "Failed to close output file: %s\n", strerror(errno)); exit(-1); } } fclose(fin); if (!allexts && !tostdout) fclose(fout); il_free(exts); anqfits_close(anq); return 0; }
int main(int argc, char *argv[]) { int argchar; char* infn = NULL; char* outfn = NULL; anbool tostdout = FALSE; FILE* fin = NULL; FILE* fout = NULL; il* exts; il* sizes; int i; char* progname = argv[0]; int Next; anqfits_t* anq; exts = il_new(16); sizes = il_new(16); while ((argchar = getopt (argc, argv, OPTIONS)) != -1) switch (argchar) { case 'e': il_append(exts, atoi(optarg)); break; case 's': il_append(sizes, atoi(optarg)); break; case 'i': infn = optarg; break; case 'o': outfn = optarg; break; case '?': case 'h': printHelp(progname); return 0; default: return -1; } log_init(LOG_MSG); if (!infn || !outfn || !il_size(exts) || (il_size(exts) != il_size(sizes))) { printHelp(progname); exit(-1); } if (infn) { fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); exit(-1); } } anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open input file %s", infn); exit(-1); } Next = anqfits_n_ext(anq); if (Next == -1) { ERROR("Couldn't determine how many extensions are in file %s", infn); exit(-1); } else { logverb("File %s contains %i FITS extensions.\n", infn, Next); } for (i=0; i<il_size(exts); i++) { int e = il_get(exts, i); int s = il_get(sizes, i); if (e < 0 || e >= Next) { logerr("Extension %i is not valid: must be in [%i, %i]\n", e, 0, Next); exit(-1); } if (s != 2 && s != 4 && s != 8) { logerr("Invalid byte size %i: must be 2, 4, or 8.\n", s); exit(-1); } } if (!strcmp(outfn, "-")) tostdout = TRUE; if (tostdout) fout = stdout; else { fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); exit(-1); } } for (i=0; i<Next; i++) { int hdrstart, hdrlen, datastart, datalen; int ind; int size; ind = il_index_of(exts, i); if (ind == -1) { size = 0; } else { size = il_get(sizes, ind); } hdrstart = anqfits_header_start(anq, i); hdrlen = anqfits_header_size (anq, i); datastart = anqfits_data_start(anq, i); datalen = anqfits_data_size (anq, i); if (hdrlen) { if (pipe_file_offset(fin, hdrstart, hdrlen, fout)) { ERROR("Failed to write header for extension %i", i); exit(-1); } } if (!datalen) continue; if (size) { int Nitems = datalen / size; int j; char buf[size]; logmsg("Extension %i: flipping words of length %i bytes.\n", i, size); for (j=0; j<Nitems; j++) { if (fread(buf, size, 1, fin) != 1) { SYSERROR("Failed to read data element %i from extension %i", j, i); exit(-1); } endian_swap(buf, size); if (fwrite(buf, size, 1, fout) != 1) { SYSERROR("Failed to write data element %i to extension %i", j, i); exit(-1); } } } else { logmsg("Extension %i: copying verbatim.\n", i); // passthrough if (pipe_file_offset(fin, datastart, datalen, fout)) { ERROR("Failed to write data for extension %i", i); exit(-1); } } } fclose(fin); anqfits_close(anq); if (!tostdout) fclose(fout); il_free(exts); il_free(sizes); return 0; }