int fitstable_read_structs(fitstable_t* tab, void* struc,
int strucstride, int offset, int N) {
int i;
void* tempdata = NULL;
int highwater = 0;
//printf("fitstable_read_structs: stride %i, offset %i, N %i\n",strucstride, offset, N);
for (i=0; i<ncols(tab); i++) {
void* dest;
int stride;
void* finaldest;
int finalstride;
fitscol_t* col = getcol(tab, i);
if (col->col == -1)
continue;
if (!col->in_struct)
continue;
finaldest = ((char*)struc) + col->coffset;
finalstride = strucstride;
if (col->fitstype != col->ctype) {
int NB = fitscolumn_get_size(col) * N;
if (NB > highwater) {
free(tempdata);
tempdata = malloc(NB);
highwater = NB;
}
dest = tempdata;
stride = fitscolumn_get_size(col);
} else {
dest = finaldest;
stride = finalstride;
}
if (in_memory(tab)) {
int j;
int off = offset_of_column(tab, i);
int sz;
if (!tab->rows) {
ERROR("No data has been written to this fitstable");
return -1;
}
if (offset + N > bl_size(tab->rows)) {
ERROR("Number of data items requested exceeds number of rows: offset %i, n %i, nrows %zu", offset, N, bl_size(tab->rows));
return -1;
}
//logverb("column %i: dest offset %i, stride %i, row offset %i, input offset %i, size %i (%ix%i)\n", i, (int)(dest - struc), stride, offset, off, fitscolumn_get_size(col), col->fitssize, col->arraysize);
sz = fitscolumn_get_size(col);
for (j=0; j<N; j++)
memcpy(((char*)dest) + j * stride,
((char*)bl_access(tab->rows, offset+j)) + off,
sz);
} else {
// Read from FITS file...
qfits_query_column_seq_to_array(tab->table, col->col, offset, N, dest, stride);
}
if (col->fitstype != col->ctype) {
fits_convert_data(finaldest, finalstride, col->ctype,
dest, stride, col->fitstype,
col->arraysize, N);
}
}
free(tempdata);
if (tab->postprocess_read_structs)
return tab->postprocess_read_structs(tab, struc, strucstride, offset, N);
return 0;
}
/**
If "inds" is non-NULL, it's a list of indices to read.
*/
static void* read_array_into(const fitstable_t* tab,
const char* colname, tfits_type ctype,
anbool array_ok,
int offset, const int* inds, int Nread,
void* dest, int deststride,
int desired_arraysize,
int* p_arraysize) {
int colnum;
qfits_col* col;
int fitssize;
int csize;
int fitstype;
int arraysize;
char* tempdata = NULL;
char* cdata;
char* fitsdata;
int cstride;
int fitsstride;
int N;
colnum = fits_find_column(tab->table, colname);
if (colnum == -1) {
ERROR("Column \"%s\" not found in FITS table %s", colname, tab->fn);
return NULL;
}
col = tab->table->col + colnum;
if (!array_ok && (col->atom_nb != 1)) {
ERROR("Column \"%s\" in FITS table %s is an array of size %i, not a scalar",
colname, tab->fn, col->atom_nb);
return NULL;
}
arraysize = col->atom_nb;
if (p_arraysize)
*p_arraysize = arraysize;
if (desired_arraysize && arraysize != desired_arraysize) {
ERROR("Column \"%s\" has array size %i but you wanted %i", colname, arraysize, desired_arraysize);
return NULL;
}
fitstype = col->atom_type;
fitssize = fits_get_atom_size(fitstype);
csize = fits_get_atom_size(ctype);
N = tab->table->nr;
if (Nread == -1)
Nread = N;
if (offset == -1)
offset = 0;
if (dest)
cdata = dest;
else
cdata = calloc(Nread * arraysize, csize);
if (dest && deststride > 0)
cstride = deststride;
else
cstride = csize * arraysize;
fitsstride = fitssize * arraysize;
if (csize < fitssize) {
// Need to allocate a bigger temp array and down-convert the data.
// HACK - could set data=tempdata and realloc after (if 'dest' is NULL)
tempdata = calloc(Nread * arraysize, fitssize);
fitsdata = tempdata;
} else {
// We'll read the data into the first fraction of the output array.
fitsdata = cdata;
}
if (in_memory(tab)) {
int i;
int off;
int sz;
if (!tab->rows) {
ERROR("No data has been written to this fitstable");
return NULL;
}
if (offset + Nread > bl_size(tab->rows)) {
ERROR("Number of data items requested exceeds number of rows: offset %i, n %i, nrows %zu", offset, Nread, bl_size(tab->rows));
return NULL;
}
off = fits_offset_of_column(tab->table, colnum);
sz = fitsstride;
if (inds) {
for (i=0; i<Nread; i++)
memcpy(fitsdata + i * fitsstride,
((char*)bl_access(tab->rows, inds[i])) + off,
sz);
} else {
for (i=0; i<Nread; i++)
memcpy(fitsdata + i * fitsstride,
((char*)bl_access(tab->rows, offset+i)) + off,
sz);
}
} else {
int res;
if (inds) {
res = qfits_query_column_seq_to_array_inds(tab->table, colnum, inds, Nread,
(unsigned char*)fitsdata, fitsstride);
} else {
res = qfits_query_column_seq_to_array(tab->table, colnum, offset, Nread,
(unsigned char*)fitsdata, fitsstride);
}
if (res) {
ERROR("Failed to read column from FITS file");
// MEMLEAK!
return NULL;
}
}
if (fitstype != ctype) {
if (csize <= fitssize) {
// work forward
fits_convert_data(cdata, cstride, ctype,
fitsdata, fitsstride, fitstype,
arraysize, Nread);
} else {
// work backward from the end of the array
fits_convert_data(cdata + (((off_t)Nread*(off_t)arraysize)-1) * (off_t)csize,
-csize, ctype,
fitsdata + (((off_t)Nread*(off_t)arraysize)-1) * (off_t)fitssize,
-fitssize, fitstype,
1, Nread * arraysize);
}
}
free(tempdata);
return cdata;
}
int tabsort(const char* infn, const char* outfn, const char* colname,
int descending) {
FILE* fin;
FILE* fout;
int ext, nextens;
off_t start, size;
void* data = NULL;
int* perm = NULL;
unsigned char* map = NULL;
size_t mapsize = 0;
anqfits_t* anq = NULL;
fin = fopen(infn, "rb");
if (!fin) {
SYSERROR("Failed to open input file %s", infn);
return -1;
}
fout = fopen(outfn, "wb");
if (!fout) {
SYSERROR("Failed to open output file %s", outfn);
goto bailout;
}
// copy the main header exactly.
anq = anqfits_open(infn);
if (!anq) {
ERROR("Failed to open \"%s\"", infn);
goto bailout;
}
start = anqfits_header_start(anq, 0);
size = anqfits_header_size (anq, 0);
if (pipe_file_offset(fin, start, size, fout)) {
ERROR("Failed to copy primary FITS header.");
goto bailout;
}
nextens = anqfits_n_ext(anq);
//logverb("Sorting %i extensions.\n", nextens);
for (ext=1; ext<nextens; ext++) {
int c;
qfits_table* table;
qfits_col* col;
int mgap;
off_t mstart;
size_t msize;
int atomsize;
int (*sort_func)(const void*, const void*);
unsigned char* tabledata;
unsigned char* tablehdr;
off_t hdrstart, hdrsize, datsize, datstart;
int i;
hdrstart = anqfits_header_start(anq, ext);
hdrsize = anqfits_header_size (anq, ext);
datstart = anqfits_data_start (anq, ext);
datsize = anqfits_data_size (anq, ext);
if (!anqfits_is_table(anq, ext)) {
ERROR("Extension %i isn't a table. Skipping.\n", ext);
continue;
}
table = anqfits_get_table(anq, ext);
if (!table) {
ERROR("Failed to open table: file %s, extension %i. Skipping.", infn, ext);
continue;
}
c = fits_find_column(table, colname);
if (c == -1) {
ERROR("Couldn't find column named \"%s\" in extension %i. Skipping.", colname, ext);
continue;
}
col = table->col + c;
switch (col->atom_type) {
case TFITS_BIN_TYPE_D:
data = realloc(data, table->nr * sizeof(double));
if (descending)
sort_func = compare_doubles_desc;
else
sort_func = compare_doubles_asc;
break;
case TFITS_BIN_TYPE_E:
data = realloc(data, table->nr * sizeof(float));
if (descending)
sort_func = compare_floats_desc;
else
sort_func = compare_floats_asc;
break;
case TFITS_BIN_TYPE_K:
data = realloc(data, table->nr * sizeof(int64_t));
if (descending)
sort_func = compare_int64_desc;
else
sort_func = compare_int64_asc;
break;
default:
ERROR("Column %s is neither FITS type D, E, nor K. Skipping.", colname);
continue;
}
// Grab the sort column.
atomsize = fits_get_atom_size(col->atom_type);
printf("Reading sort column \"%s\"\n", colname);
qfits_query_column_seq_to_array(table, c, 0, table->nr, data, atomsize);
// Sort it.
printf("Sorting sort column\n");
perm = permuted_sort(data, atomsize, sort_func, NULL, table->nr);
// mmap the input file.
printf("mmapping input file\n");
start = hdrstart;
size = hdrsize + datsize;
get_mmap_size(start, size, &mstart, &msize, &mgap);
mapsize = msize;
map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, fileno(fin), mstart);
if (map == MAP_FAILED) {
SYSERROR("Failed to mmap input file %s", infn);
map = NULL;
goto bailout;
}
tabledata = map + (off_t)mgap + (datstart - hdrstart);
tablehdr = map + (off_t)mgap;
// Copy the table header without change.
printf("Copying table header.\n");
if (fwrite(tablehdr, 1, hdrsize, fout) != hdrsize) {
SYSERROR("Failed to write FITS table header");
goto bailout;
}
for (i=0; i<table->nr; i++) {
unsigned char* rowptr;
if (i % 100000 == 0)
printf("Writing row %i\n", i);
rowptr = tabledata + (off_t)(perm[i]) * (off_t)table->tab_w;
if (fwrite(rowptr, 1, table->tab_w, fout) != table->tab_w) {
SYSERROR("Failed to write FITS table row");
goto bailout;
}
}
munmap(map, mapsize);
map = NULL;
free(perm);
perm = NULL;
if (fits_pad_file(fout)) {
ERROR("Failed to add padding to extension %i", ext);
goto bailout;
}
qfits_table_close(table);
}
free(data);
if (fclose(fout)) {
SYSERROR("Error closing output file");
fout = NULL;
goto bailout;
}
fclose(fin);
anqfits_close(anq);
printf("Done\n");
return 0;
bailout:
free(data);
free(perm);
if (fout)
fclose(fout);
fclose(fin);
if (map)
munmap(map, mapsize);
return -1;
}
