str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch;
sql_table *fits_fl, *fits_tbl, *tbl = NULL;
sql_column *col;
sql_subtype tpe;
fitsfile *fptr;
str tname = *(str*)getArgReference(stk, pci, 1);
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
int status = 0, cnum = 0, fid, hdu, hdutype, i, j, anynull = 0, mtype;
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows;
char keywrd[80], **cname, nm[FLEN_VALUE];
ptr nilptr;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
sch = mvc_bind_schema(m, "sys");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
if (fits_tbl == NULL) {
msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n");
return msg;
}
tbl = mvc_bind_table(m, sch, tname);
if (tbl) {
msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
if (rid == oid_nil) {
msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
return msg;
}
/* Open FITS file and move to the table HDU */
col = mvc_bind_column(m, fits_tbl, "file_id");
fid = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_fl = mvc_bind_table(m, sch, "fits_files");
col = mvc_bind_column(m, fits_fl, "id");
frid = table_funcs.column_find_row(m->session->tr, col, (void *)&fid, NULL);
col = mvc_bind_column(m, fits_fl, "name");
fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid);
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "hdu");
hdu = *(int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_movabs_hdu(fptr, hdu, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", hdu);
fits_close_file(fptr, &status);
return msg;
}
/* create a SQL table to hold the FITS table */
/* col = mvc_bind_column(m, fits_tbl, "columns");
cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for (j = 1; j <= cnum; j++) {
/* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
snprintf(keywrd, 80, "TTYPE%d", j);
fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
if (status) {
snprintf(nm, FLEN_VALUE, "column_%d", j);
status = 0;
}
cname[j - 1] = GDKstrdup(toLower(nm));
fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
/* mnstr_printf(cntxt->fdout,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */
mvc_create_column(m, tbl, cname[j - 1], &tpe);
}
/* data load */
fits_get_num_rows(fptr, &rows, &status);
mnstr_printf(cntxt->fdout,"#Loading %ld rows in table %s\n", rows, tname);
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
mtype = fits2mtype(tpcode[j - 1]);
nilptr = ATOMnil(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = BATnew(TYPE_void, mtype, rows);
if ( tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", MAL_MALLOC_FAIL);
}
BATseqbase(tmp, 0);
if (rows > (long)REMAP_PAGE_MAXSIZE)
BATmmap(tmp, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0);
if (mtype != TYPE_str) {
fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status);
BATsetcount(tmp, rows);
tmp->tsorted = 0;
tmp->trevsorted = 0;
} else {
/* char *v = GDKzalloc(wid[j-1]);*/
int bsize = 50;
int tm0, tloadtm = 0, tattachtm = 0;
int batch = bsize, k;
char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
for(i = 0; i < bsize; i++)
v[i] = GDKzalloc(wid[j-1]);
for(i = 0; i < rows; i += batch) {
batch = rows - i < bsize ? rows - i: bsize;
tm0 = GDKms();
fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status);
tloadtm += GDKms() - tm0;
tm0 = GDKms();
for(k = 0; k < batch ; k++)
BUNappend(tmp, v[k], TRUE);
tattachtm += GDKms() - tm0;
}
for(i = 0; i < bsize ; i++)
GDKfree(v[i]);
GDKfree(v);
mnstr_printf(cntxt->fdout,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
}
if (status) {
char buf[FLEN_ERRMSG + 1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
break;
}
mnstr_printf(cntxt->fdout,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat);
mnstr_printf(cntxt->fdout,"#Total %d ms\n", GDKms() - time0);
BBPunfix(tmp->batCacheid);
}
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
return msg;
}
str
ALARMtime(int *res)
{
*res = GDKms();
return MAL_SUCCEED;
}
str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str tname = *(str*) getArgReference(stk, pci, 1);
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *tbl, *column, *tables = NULL;
sql_column *col;
oid rid = oid_nil;
str type, name, *colname, *tform;
fitsfile *fptr;
char filename[BUFSIZ];
long nrows = 0, optimal;
rids * rs;
int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlong=0, texportfloat=0, texportdouble=0;
int numberrow = 0, cc = 0, status = 0, j = 0, columns, fid, dimension = 0, block = 0;
int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, longcols = 0, floatcols = 0, doublecols = 0;
int hdutype;
char charvalue, *readcharrows;
str strvalue; char **readstrrows;
short shortvalue, *readshortrows;
int intvalue, *readintrows;
long longvalue, *readlongrows;
float realvalue, *readfloatrows;
double doublevalue, *readdoublerows;
_Bool boolvalue, *readboolrows;
struct list * set;
msg = getSQLContext(cntxt, mb, &m, NULL);
if (msg)
return msg;
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
/* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
tbl = mvc_bind_table(m, sch, tname);
if (tbl == NULL) {
msg = createException (MAL, "fits.exporttable", "Table %s is missing.\n", tname);
return msg;
}
set = (*tbl).columns.set;
columns = list_length(set);
colname = (str *) GDKmalloc(columns * sizeof(str));
tform = (str *) GDKmalloc(columns * sizeof(str));
/* mnstr_printf(GDKout,"Number of columns: %d\n", columns);*/
tables = mvc_bind_table(m, sch, "_tables");
col = mvc_bind_column(m, tables, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
col = mvc_bind_column(m, tables, "id");
fid = *(int*) table_funcs.column_find_value(m->session->tr, col, rid);
column = mvc_bind_table(m, sch, "_columns");
col = mvc_bind_column(m, column, "table_id");
rs = table_funcs.rids_select(m->session->tr, col, (void *) &fid, (void *) &fid, NULL);
while ((rid = table_funcs.rids_next(rs)) != oid_nil)
{
col = mvc_bind_column(m, column, "name");
name = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
colname[j] = toLower(name);
col = mvc_bind_column(m, column, "type");
type = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
if (strcmp(type,"boolean")==0) tform[j] = "1L";
if (strcmp(type,"char")==0) tform[j] = "1S";
if (strcmp(type,"varchar")==0) tform[j] = "8A";
if (strcmp(type,"smallint")==0) tform[j] = "1I";
if (strcmp(type,"int")==0) tform[j] = "1J";
if (strcmp(type,"bigint")==0) tform[j] = "1K";
if (strcmp(type,"real")==0) tform[j] = "1E";
if (strcmp(type,"double")==0) tform[j] = "1D";
j++;
}
col = mvc_bind_column(m, tbl, colname[0]);
nrows = store_funcs.count_col(tr, col, 1);
snprintf(filename,BUFSIZ,"\n%s.fit",tname);
mnstr_printf(GDKout, "Filename: %s\n", filename);
remove(filename);
status=0;
fits_create_file(&fptr, filename, &status);
fits_create_img(fptr, USHORT_IMG, 0, NULL, &status);
fits_close_file(fptr, &status);
fits_open_file(&fptr, filename, READWRITE, &status);
fits_movabs_hdu(fptr, 1, &hdutype, &status);
fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
for (cc = 0; cc < columns; cc++)
{
char * columntype;
col = mvc_bind_column(m, tbl, colname[cc]);
columntype = col -> type.type->sqlname;
if (strcmp(columntype,"boolean")==0)
{
boolcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
boolvalue = *(_Bool*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readboolrows[dimension] = boolvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
}
if (strcmp(columntype,"char")==0)
{
charcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
charvalue = *(char*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readcharrows[dimension] = charvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
}
if (strcmp(columntype,"varchar")==0)
{
strcols++; dimension=0; block=0;
fits_get_rowsize(fptr,&optimal,&status);
readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
strvalue = (char *) table_funcs.column_find_value(m->session->tr, col, numberrow);
readstrrows[dimension] = strvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
texportstring += GDKms() - tm0;
GDKfree(readstrrows);
readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
texportstring += GDKms() - tm0;
GDKfree(readstrrows);
}
if (strcmp(columntype,"smallint")==0)
{
shortcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
shortvalue = *(short*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readshortrows[dimension] = shortvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
}
if (strcmp(columntype,"int")==0)
{
intcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
intvalue = *(int*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readintrows[dimension] = intvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
}
if (strcmp(columntype,"bigint")==0)
{
longcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readlongrows = (long *) GDKmalloc (sizeof(long) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
longvalue = *(long*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readlongrows[dimension] = longvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlongrows, &status);
texportlong += GDKms() - tm0;
GDKfree(readlongrows);
readlongrows = (long *) GDKmalloc (sizeof(long) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlongrows, &status);
texportlong += GDKms() - tm0;
GDKfree(readlongrows);
}
if (strcmp(columntype,"real")==0)
{
floatcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
realvalue = *(float*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readfloatrows[dimension] = realvalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
}
if (strcmp(columntype,"double")==0)
{
doublecols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
doublevalue = *(double*) table_funcs.column_find_value(m->session->tr, col, numberrow);
readdoublerows[dimension] = doublevalue;
dimension++;
if (dimension == optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
}
}
/* print all the times that were needed to export each one of the columns
mnstr_printf(GDKout, "\n\n");
if (texportboolean > 0) mnstr_printf(GDKout, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean);
if (texportchar > 0) mnstr_printf(GDKout, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar);
if (texportstring > 0) mnstr_printf(GDKout, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring);
if (texportshort > 0) mnstr_printf(GDKout, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort);
if (texportint > 0) mnstr_printf(GDKout, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint);
if (texportlong > 0) mnstr_printf(GDKout, "%d Long\t\tcolumn(s) exported in %d ms\n", longcols, texportlong);
if (texportfloat > 0) mnstr_printf(GDKout, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat);
if (texportdouble > 0) mnstr_printf(GDKout, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble);
*/
fits_close_file(fptr, &status);
return msg;
}
