int fitstable_read_structs(fitstable_t* tab, void* struc, int strucstride, int offset, int N) { int i; void* tempdata = NULL; int highwater = 0; //printf("fitstable_read_structs: stride %i, offset %i, N %i\n",strucstride, offset, N); for (i=0; i<ncols(tab); i++) { void* dest; int stride; void* finaldest; int finalstride; fitscol_t* col = getcol(tab, i); if (col->col == -1) continue; if (!col->in_struct) continue; finaldest = ((char*)struc) + col->coffset; finalstride = strucstride; if (col->fitstype != col->ctype) { int NB = fitscolumn_get_size(col) * N; if (NB > highwater) { free(tempdata); tempdata = malloc(NB); highwater = NB; } dest = tempdata; stride = fitscolumn_get_size(col); } else { dest = finaldest; stride = finalstride; } if (in_memory(tab)) { int j; int off = offset_of_column(tab, i); int sz; if (!tab->rows) { ERROR("No data has been written to this fitstable"); return -1; } if (offset + N > bl_size(tab->rows)) { ERROR("Number of data items requested exceeds number of rows: offset %i, n %i, nrows %zu", offset, N, bl_size(tab->rows)); return -1; } //logverb("column %i: dest offset %i, stride %i, row offset %i, input offset %i, size %i (%ix%i)\n", i, (int)(dest - struc), stride, offset, off, fitscolumn_get_size(col), col->fitssize, col->arraysize); sz = fitscolumn_get_size(col); for (j=0; j<N; j++) memcpy(((char*)dest) + j * stride, ((char*)bl_access(tab->rows, offset+j)) + off, sz); } else { // Read from FITS file... qfits_query_column_seq_to_array(tab->table, col->col, offset, N, dest, stride); } if (col->fitstype != col->ctype) { fits_convert_data(finaldest, finalstride, col->ctype, dest, stride, col->fitstype, col->arraysize, N); } } free(tempdata); if (tab->postprocess_read_structs) return tab->postprocess_read_structs(tab, struc, strucstride, offset, N); return 0; }
/** If "inds" is non-NULL, it's a list of indices to read. */ static void* read_array_into(const fitstable_t* tab, const char* colname, tfits_type ctype, anbool array_ok, int offset, const int* inds, int Nread, void* dest, int deststride, int desired_arraysize, int* p_arraysize) { int colnum; qfits_col* col; int fitssize; int csize; int fitstype; int arraysize; char* tempdata = NULL; char* cdata; char* fitsdata; int cstride; int fitsstride; int N; colnum = fits_find_column(tab->table, colname); if (colnum == -1) { ERROR("Column \"%s\" not found in FITS table %s", colname, tab->fn); return NULL; } col = tab->table->col + colnum; if (!array_ok && (col->atom_nb != 1)) { ERROR("Column \"%s\" in FITS table %s is an array of size %i, not a scalar", colname, tab->fn, col->atom_nb); return NULL; } arraysize = col->atom_nb; if (p_arraysize) *p_arraysize = arraysize; if (desired_arraysize && arraysize != desired_arraysize) { ERROR("Column \"%s\" has array size %i but you wanted %i", colname, arraysize, desired_arraysize); return NULL; } fitstype = col->atom_type; fitssize = fits_get_atom_size(fitstype); csize = fits_get_atom_size(ctype); N = tab->table->nr; if (Nread == -1) Nread = N; if (offset == -1) offset = 0; if (dest) cdata = dest; else cdata = calloc(Nread * arraysize, csize); if (dest && deststride > 0) cstride = deststride; else cstride = csize * arraysize; fitsstride = fitssize * arraysize; if (csize < fitssize) { // Need to allocate a bigger temp array and down-convert the data. // HACK - could set data=tempdata and realloc after (if 'dest' is NULL) tempdata = calloc(Nread * arraysize, fitssize); fitsdata = tempdata; } else { // We'll read the data into the first fraction of the output array. fitsdata = cdata; } if (in_memory(tab)) { int i; int off; int sz; if (!tab->rows) { ERROR("No data has been written to this fitstable"); return NULL; } if (offset + Nread > bl_size(tab->rows)) { ERROR("Number of data items requested exceeds number of rows: offset %i, n %i, nrows %zu", offset, Nread, bl_size(tab->rows)); return NULL; } off = fits_offset_of_column(tab->table, colnum); sz = fitsstride; if (inds) { for (i=0; i<Nread; i++) memcpy(fitsdata + i * fitsstride, ((char*)bl_access(tab->rows, inds[i])) + off, sz); } else { for (i=0; i<Nread; i++) memcpy(fitsdata + i * fitsstride, ((char*)bl_access(tab->rows, offset+i)) + off, sz); } } else { int res; if (inds) { res = qfits_query_column_seq_to_array_inds(tab->table, colnum, inds, Nread, (unsigned char*)fitsdata, fitsstride); } else { res = qfits_query_column_seq_to_array(tab->table, colnum, offset, Nread, (unsigned char*)fitsdata, fitsstride); } if (res) { ERROR("Failed to read column from FITS file"); // MEMLEAK! return NULL; } } if (fitstype != ctype) { if (csize <= fitssize) { // work forward fits_convert_data(cdata, cstride, ctype, fitsdata, fitsstride, fitstype, arraysize, Nread); } else { // work backward from the end of the array fits_convert_data(cdata + (((off_t)Nread*(off_t)arraysize)-1) * (off_t)csize, -csize, ctype, fitsdata + (((off_t)Nread*(off_t)arraysize)-1) * (off_t)fitssize, -fitssize, fitstype, 1, Nread * arraysize); } } free(tempdata); return cdata; }
int tabsort(const char* infn, const char* outfn, const char* colname, int descending) { FILE* fin; FILE* fout; int ext, nextens; off_t start, size; void* data = NULL; int* perm = NULL; unsigned char* map = NULL; size_t mapsize = 0; anqfits_t* anq = NULL; fin = fopen(infn, "rb"); if (!fin) { SYSERROR("Failed to open input file %s", infn); return -1; } fout = fopen(outfn, "wb"); if (!fout) { SYSERROR("Failed to open output file %s", outfn); goto bailout; } // copy the main header exactly. anq = anqfits_open(infn); if (!anq) { ERROR("Failed to open \"%s\"", infn); goto bailout; } start = anqfits_header_start(anq, 0); size = anqfits_header_size (anq, 0); if (pipe_file_offset(fin, start, size, fout)) { ERROR("Failed to copy primary FITS header."); goto bailout; } nextens = anqfits_n_ext(anq); //logverb("Sorting %i extensions.\n", nextens); for (ext=1; ext<nextens; ext++) { int c; qfits_table* table; qfits_col* col; int mgap; off_t mstart; size_t msize; int atomsize; int (*sort_func)(const void*, const void*); unsigned char* tabledata; unsigned char* tablehdr; off_t hdrstart, hdrsize, datsize, datstart; int i; hdrstart = anqfits_header_start(anq, ext); hdrsize = anqfits_header_size (anq, ext); datstart = anqfits_data_start (anq, ext); datsize = anqfits_data_size (anq, ext); if (!anqfits_is_table(anq, ext)) { ERROR("Extension %i isn't a table. Skipping.\n", ext); continue; } table = anqfits_get_table(anq, ext); if (!table) { ERROR("Failed to open table: file %s, extension %i. Skipping.", infn, ext); continue; } c = fits_find_column(table, colname); if (c == -1) { ERROR("Couldn't find column named \"%s\" in extension %i. Skipping.", colname, ext); continue; } col = table->col + c; switch (col->atom_type) { case TFITS_BIN_TYPE_D: data = realloc(data, table->nr * sizeof(double)); if (descending) sort_func = compare_doubles_desc; else sort_func = compare_doubles_asc; break; case TFITS_BIN_TYPE_E: data = realloc(data, table->nr * sizeof(float)); if (descending) sort_func = compare_floats_desc; else sort_func = compare_floats_asc; break; case TFITS_BIN_TYPE_K: data = realloc(data, table->nr * sizeof(int64_t)); if (descending) sort_func = compare_int64_desc; else sort_func = compare_int64_asc; break; default: ERROR("Column %s is neither FITS type D, E, nor K. Skipping.", colname); continue; } // Grab the sort column. atomsize = fits_get_atom_size(col->atom_type); printf("Reading sort column \"%s\"\n", colname); qfits_query_column_seq_to_array(table, c, 0, table->nr, data, atomsize); // Sort it. printf("Sorting sort column\n"); perm = permuted_sort(data, atomsize, sort_func, NULL, table->nr); // mmap the input file. printf("mmapping input file\n"); start = hdrstart; size = hdrsize + datsize; get_mmap_size(start, size, &mstart, &msize, &mgap); mapsize = msize; map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, fileno(fin), mstart); if (map == MAP_FAILED) { SYSERROR("Failed to mmap input file %s", infn); map = NULL; goto bailout; } tabledata = map + (off_t)mgap + (datstart - hdrstart); tablehdr = map + (off_t)mgap; // Copy the table header without change. printf("Copying table header.\n"); if (fwrite(tablehdr, 1, hdrsize, fout) != hdrsize) { SYSERROR("Failed to write FITS table header"); goto bailout; } for (i=0; i<table->nr; i++) { unsigned char* rowptr; if (i % 100000 == 0) printf("Writing row %i\n", i); rowptr = tabledata + (off_t)(perm[i]) * (off_t)table->tab_w; if (fwrite(rowptr, 1, table->tab_w, fout) != table->tab_w) { SYSERROR("Failed to write FITS table row"); goto bailout; } } munmap(map, mapsize); map = NULL; free(perm); perm = NULL; if (fits_pad_file(fout)) { ERROR("Failed to add padding to extension %i", ext); goto bailout; } qfits_table_close(table); } free(data); if (fclose(fout)) { SYSERROR("Error closing output file"); fout = NULL; goto bailout; } fclose(fin); anqfits_close(anq); printf("Done\n"); return 0; bailout: free(data); free(perm); if (fout) fclose(fout); fclose(fin); if (map) munmap(map, mapsize); return -1; }