static str
SHPimportFile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bool partial) {
mvc *m = NULL;
sql_schema *sch = NULL;
char *sch_name = "sys";
sql_table *shps_table = NULL, *fls_table = NULL, *data_table = NULL;
char *shps_table_name = "shapefiles";
char *fls_table_name = "files";
char *data_table_name = NULL;
sql_column *col;
sql_column **cols;
BAT **colsBAT;
int colsNum = 2; //we will have at least the gid column and a geometry column
int rowsNum = 0; //the number of rows in the shape file that will be imported
//GIntBig rowsNum = 0;
int gidNum = 0;
char *nameToLowerCase = NULL;
str msg = MAL_SUCCEED;
str fname = NULL;
int vid = *(int*)getArgReference(stk, pci, 1);
ptr *p;
wkb *g;
OGRGeometryH geom;
OGREnvelope *mbb;
/* SHP-level descriptor */
OGRFieldDefnH hFieldDefn;
int i=0;
oid irid;
GDALWConnection shp_conn;
GDALWConnection * shp_conn_ptr = NULL;
GDALWSimpleFieldDef * field_definitions;
OGRFeatureH feature;
OGRFeatureDefnH featureDefn;
/* get table columns from shp and create the table */
if((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
if(!(sch = mvc_bind_schema(m, sch_name)))
return createException(MAL, "shp.import", SQLSTATE(38000) "Schema '%s' missing", sch_name);
/* find the name of the shape file corresponding to the given id */
if(!(fls_table = mvc_bind_table(m, sch, fls_table_name)))
return createException(MAL, "shp.import", SQLSTATE(38000) "Table '%s.%s' missing", sch_name, fls_table_name);
if(!(col = mvc_bind_column(m, fls_table, "id")))
return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(id)' missing", sch_name, fls_table_name);
irid = table_funcs.column_find_row(m->session->tr, col, (void *)&vid, NULL);
if (is_oid_nil(irid))
return createException(MAL, "shp.import", SQLSTATE(38000) "Shapefile with id %d not in the %s.%s table\n", vid, sch_name, fls_table_name);
if(!(col = mvc_bind_column(m, fls_table, "path")))
return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(path)' missing", sch_name, fls_table_name);
fname = (str)table_funcs.column_find_value(m->session->tr, col, irid);
/* find the name of the table that has been reserved for this shape file */
if(!(shps_table = mvc_bind_table(m, sch, shps_table_name)))
return createException(MAL, "shp.import", SQLSTATE(38000) "Table '%s.%s' missing", sch_name, shps_table_name);
if(!(col = mvc_bind_column(m, shps_table, "fileid")))
return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(fileid)' missing", sch_name, shps_table_name);
irid = table_funcs.column_find_row(m->session->tr, col, (void *)&vid, NULL);
if (is_oid_nil(irid))
return createException(MAL, "shp.import", SQLSTATE(38000) "Shapefile with id %d not in the Shapefile catalog\n", vid);
if(!(col = mvc_bind_column(m, shps_table, "datatable")))
return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(datatable)' missing", sch_name, shps_table_name);
data_table_name = (str)table_funcs.column_find_value(m->session->tr, col, irid);
/* add the data on the file to the table */
if(!(shp_conn_ptr = GDALWConnect((char *) fname)))
return createException(MAL, "shp.import", SQLSTATE(38000) "Missing shape file %s\n", fname);
shp_conn = *shp_conn_ptr;
/*count the number of lines in the shape file */
if ((rowsNum = OGR_L_GetFeatureCount(shp_conn.layer, false)) == -1) {
if ((rowsNum = OGR_L_GetFeatureCount(shp_conn.layer, true)) == -1) {
OGR_L_ResetReading(shp_conn.layer);
rowsNum = 0;
while ((feature = OGR_L_GetNextFeature(shp_conn.layer)) != NULL ) {
rowsNum++;
OGR_F_Destroy(feature);
}
}
}
/* calculate the mbb of the query geometry */
if (partial) {
p = (ptr*)getArgReference(stk, pci, 2);
g = (wkb*)*p;
geom = OGR_G_CreateGeometry(wkbPolygon);
if (OGR_G_ImportFromWkb(geom, (unsigned char*)g->data, g->len) != OGRERR_NONE) {
msg = createException(MAL, "shp.import", SQLSTATE(38000) "Could not intantiate the query polygon.");
OGR_F_Destroy(geom);
goto final;
}
if (!(mbb = (OGREnvelope*)GDKmalloc(sizeof(OGREnvelope)))) {
msg = createException(MAL, "shp.import", SQLSTATE(HY001) MAL_MALLOC_FAIL);
OGR_F_Destroy(geom);
goto final;
}
/* attach a single shp file given its name, fill in shp catalog tables */
str
SHPattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch = NULL;
sql_table *fls = NULL, *shps = NULL, *shps_dbf = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *(str*)getArgReference(stk, pci, 1);
/* SHP-level descriptor */
char buf[BUFSIZ], temp_buf[BUFSIZ], *s=buf;
int i=0, shpid = 0;
oid fid, rid = oid_nil;
GDALWConnection shp_conn;
GDALWConnection * shp_conn_ptr = NULL;
GDALWSimpleFieldDef * field_definitions;
GDALWSpatialInfo spatial_info;
char *nameToLowerCase = NULL;
if((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
if(!(sch = mvc_bind_schema(m, "sys")))
return createException(MAL, "shp.attach", SQLSTATE(38000) "Schema sys missing\n");
fls = mvc_bind_table(m, sch, "files");
shps = mvc_bind_table(m, sch, "shapefiles");
shps_dbf = mvc_bind_table(m, sch, "shapefiles_dbf");
if (fls == NULL || shps == NULL || shps_dbf == NULL )
return createException(MAL, "shp.attach", SQLSTATE(38000) "Catalog table missing\n");
if ((shp_conn_ptr = GDALWConnect((char *) fname)) == NULL) {
return createException(MAL, "shp.attach", SQLSTATE(38000) "Missing shape file %s\n", fname);
}
shp_conn = *shp_conn_ptr;
/* check if the file is already attached */
col = mvc_bind_column(m, fls, "path");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (!is_oid_nil(rid)) {
GDALWClose(shp_conn_ptr);
return createException(MAL, "shp.attach", SQLSTATE(38000) "File %s already attached\n", fname);
}
/* add row in the files(id, path) catalog table */
col = mvc_bind_column(m, fls, "id");
fid = store_funcs.count_col(m->session->tr, col, 1) + 1;
snprintf(buf, BUFSIZ, INSFILE, (int)fid, fname);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto finish;
/*if (shp_conn.layer == NULL || shp_conn.source == NULL || shp_conn.handler == NULL || shp_conn.driver == NULL) {
msg = createException(MAL, "shp.attach", SQLSTATE(38000) "lol-1\n");
return msg;
}*/
/* add row in the shapefiles catalog table (e.g. the name of the table that will store tha data of the shapefile) */
spatial_info = GDALWGetSpatialInfo(shp_conn);
col = mvc_bind_column(m, shps, "shapefileid");
shpid = store_funcs.count_col(m->session->tr, col, 1) + 1;
nameToLowerCase = toLower(shp_conn.layername);
snprintf(buf, BUFSIZ, INSSHP, shpid, (int)fid, spatial_info.epsg, nameToLowerCase);
GDKfree(nameToLowerCase);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto finish;
/* add information about the fields of the shape file
* one row for each field with info (shapefile_id, field_name, field_type) */
field_definitions = GDALWGetSimpleFieldDefinitions(shp_conn);
if (field_definitions == NULL) {
GDALWClose(&shp_conn);
return createException(MAL, "shp.attach", SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) {
snprintf(buf, BUFSIZ, INSSHPDBF, shpid, field_definitions[i].fieldName, field_definitions[i].fieldType);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto fin;
}
/* create the table that will store the data of the shape file */
temp_buf[0]='\0';
for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) {
nameToLowerCase = toLower(field_definitions[i].fieldName);
if (strcmp(field_definitions[i].fieldType, "Integer") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" INT, ", nameToLowerCase);
} else if (strcmp(field_definitions[i].fieldType, "Real") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" FLOAT, ", nameToLowerCase);
#if 0
} else if (strcmp(field_definitions[i].fieldType, "Date") == 0) {
sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase);
#endif
} else
sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase);
GDKfree(nameToLowerCase);
}
sprintf(temp_buf + strlen(temp_buf), "geom GEOMETRY ");
snprintf(buf, BUFSIZ, CRTTBL, shp_conn.layername, temp_buf);
if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.import",TRUE,FALSE,NULL)) != MAL_SUCCEED)
goto fin;
fin:
free(field_definitions);
finish:
/* if (msg != MAL_SUCCEED){
snprintf(buf, BUFSIZ,"ROLLBACK;");
SQLstatementIntern(cntxt,&s,"geotiff.attach",TRUE,FALSE));
}*/
GDALWClose(&shp_conn);
return msg;
}
str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *getArgReference_str(stk, pci, 1);
fitsfile *fptr; /* pointer to the FITS file */
int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
oid fid, tid, cid, rid = oid_nil;
char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ];
long tbcol; /* type long used by fits library */
char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
double tscal, tzero;
char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.attach", "Missing FITS file %s.\n", fname);
return msg;
}
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
fits_fl = mvc_bind_table(m, sch, "fits_files");
if (fits_fl == NULL)
FITSinitCatalog(m);
fits_fl = mvc_bind_table(m, sch, "fits_files");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
fits_col = mvc_bind_table(m, sch, "fits_columns");
fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
/* check if the file is already attached */
col = mvc_bind_column(m, fits_fl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (rid != oid_nil) {
fits_close_file(fptr, &status);
msg = createException(SQL, "fits.attach", "File %s already attached\n", fname);
return msg;
}
/* add row in the fits_files catalog table */
col = mvc_bind_column(m, fits_fl, "id");
fid = store_funcs.count_col(tr, col, 1) + 1;
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "name"), fname, TYPE_str);
col = mvc_bind_column(m, fits_tbl, "id");
tid = store_funcs.count_col(tr, col, 1) + 1;
if ((s = strrchr(fname, DIR_SEP)) == NULL)
s = fname;
else
s++;
strcpy(bname, s);
s = strrchr(bname, '.');
if (s) *s = 0;
fits_get_num_hdus(fptr, &hdunum, &status);
for (i = 1; i <= hdunum; i++) {
fits_movabs_hdu(fptr, i, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
continue;
/* SQL table name - the name of FITS extention */
fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
if (status) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
status = 0;
}else { /* check table name for existence in the fits catalog */
tname_low = toLower(tname);
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname_low, NULL);
/* or as regular SQL table */
tbl = mvc_bind_table(m, sch, tname_low);
if (rid != oid_nil || tbl) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
}
}
fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
if (status) {
status = 0;
}
fits_get_num_cols(fptr, &cnum, &status);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "name"), tname_low, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "columns"), &cnum, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "file_id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "date"), tdate, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "origin"), orig, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "comment"), comm, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "table_id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "xtension"), xtensionname, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "bitpix"), &bitpixnumber, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilvers"), stilversion, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilclas"), stilclass, TYPE_str);
/* read columns description */
s = stmt;
col = mvc_bind_column(m, fits_col, "id");
cid = store_funcs.count_col(tr, col, 1) + 1;
for (j = 1; j <= cnum; j++, cid++) {
fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname, tform, tunit, j, (int)tid);
msg = SQLstatementIntern(cntxt, &s, "fits.attach", TRUE, FALSE, NULL);
if (msg != MAL_SUCCEED) {
fits_close_file(fptr, &status);
return msg;
}
}
tid++;
}
fits_close_file(fptr, &status);
return MAL_SUCCEED;
}
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch;
sql_table *fits_fl, *fits_tbl, *tbl = NULL;
sql_column *col;
sql_subtype tpe;
fitsfile *fptr;
str tname = *getArgReference_str(stk, pci, 1);
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype;
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */
char keywrd[80], **cname, nm[FLEN_VALUE];
ptr nilptr;
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
sch = mvc_bind_schema(m, "sys");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
if (fits_tbl == NULL) {
msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n");
return msg;
}
tbl = mvc_bind_table(m, sch, tname);
if (tbl) {
msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
if (rid == oid_nil) {
msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
return msg;
}
/* Open FITS file and move to the table HDU */
col = mvc_bind_column(m, fits_tbl, "file_id");
fid = (int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_fl = mvc_bind_table(m, sch, "fits_files");
col = mvc_bind_column(m, fits_fl, "id");
frid = table_funcs.column_find_row(m->session->tr, col, (void *)fid, NULL);
GDKfree(fid);
col = mvc_bind_column(m, fits_fl, "name");
fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid);
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname);
GDKfree(fname);
return msg;
}
GDKfree(fname);
col = mvc_bind_column(m, fits_tbl, "hdu");
hdu = (int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_movabs_hdu(fptr, *hdu, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", *hdu);
GDKfree(hdu);
fits_close_file(fptr, &status);
return msg;
}
GDKfree(hdu);
/* create a SQL table to hold the FITS table */
/* col = mvc_bind_column(m, fits_tbl, "columns");
cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for (j = 1; j <= cnum; j++) {
/* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
snprintf(keywrd, 80, "TTYPE%d", j);
fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
if (status) {
snprintf(nm, FLEN_VALUE, "column_%d", j);
status = 0;
}
cname[j - 1] = toLower(nm);
fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
/* fprintf(stderr,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */
mvc_create_column(m, tbl, cname[j - 1], &tpe);
}
/* data load */
fits_get_num_rows(fptr, &rows, &status);
fprintf(stderr,"#Loading %ld rows in table %s\n", rows, tname);
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
mtype = fits2mtype(tpcode[j - 1]);
nilptr = ATOMnilptr(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = BATnew(TYPE_void, mtype, rows, TRANSIENT);
if ( tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", MAL_MALLOC_FAIL);
}
BATseqbase(tmp, 0);
if (mtype != TYPE_str) {
fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status);
BATsetcount(tmp, rows);
tmp->tsorted = 0;
tmp->trevsorted = 0;
} else {
/* char *v = GDKzalloc(wid[j-1]);*/
/* type long demanded by "rows", i.e., by fits library */
long bsize = 50, batch = bsize, k, i;
int tm0, tloadtm = 0, tattachtm = 0;
char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
for(i = 0; i < bsize; i++)
v[i] = GDKzalloc(wid[j-1]);
for(i = 0; i < rows; i += batch) {
batch = rows - i < bsize ? rows - i: bsize;
tm0 = GDKms();
fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status);
tloadtm += GDKms() - tm0;
tm0 = GDKms();
for(k = 0; k < batch ; k++)
BUNappend(tmp, v[k], TRUE);
tattachtm += GDKms() - tm0;
}
for(i = 0; i < bsize ; i++)
GDKfree(v[i]);
GDKfree(v);
fprintf(stderr,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
}
if (status) {
char buf[FLEN_ERRMSG + 1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
break;
}
fprintf(stderr,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat);
fprintf(stderr,"#Total %d ms\n", GDKms() - time0);
BBPunfix(tmp->batCacheid);
}
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
return msg;
}
str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str tname = *getArgReference_str(stk, pci, 1);
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *tbl, *column, *tables = NULL;
sql_column *col;
oid rid = oid_nil;
str type, name, *colname, *tform;
fitsfile *fptr;
char filename[BUFSIZ];
size_t nrows = 0;
long optimal; /* type long used by fits library */
rids * rs;
int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0;
size_t numberrow = 0, dimension = 0;
int cc = 0, status = 0, j = 0, columns, *fid, block = 0;
int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0;
int hdutype;
char *charvalue, *readcharrows;
str strvalue; char **readstrrows;
short *shortvalue, *readshortrows;
int *intvalue, *readintrows;
lng *lngvalue, *readlngrows;
float *realvalue, *readfloatrows;
double *doublevalue, *readdoublerows;
_Bool *boolvalue, *readboolrows;
struct list * set;
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
/* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
tbl = mvc_bind_table(m, sch, tname);
if (tbl == NULL) {
msg = createException (MAL, "fits.exporttable", "Table %s is missing.\n", tname);
return msg;
}
set = (*tbl).columns.set;
columns = list_length(set);
colname = (str *) GDKmalloc(columns * sizeof(str));
tform = (str *) GDKmalloc(columns * sizeof(str));
/* fprintf(stderr,"Number of columns: %d\n", columns);*/
tables = mvc_bind_table(m, sch, "_tables");
col = mvc_bind_column(m, tables, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
col = mvc_bind_column(m, tables, "id");
fid = (int*) table_funcs.column_find_value(m->session->tr, col, rid);
column = mvc_bind_table(m, sch, "_columns");
col = mvc_bind_column(m, column, "table_id");
rs = table_funcs.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL);
GDKfree(fid);
while ((rid = table_funcs.rids_next(rs)) != oid_nil)
{
col = mvc_bind_column(m, column, "name");
name = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
colname[j] = toLower(name);
GDKfree(name);
col = mvc_bind_column(m, column, "type");
type = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
if (strcmp(type,"boolean")==0) tform[j] = "1L";
if (strcmp(type,"char")==0) tform[j] = "1S";
if (strcmp(type,"varchar")==0) tform[j] = "8A";
if (strcmp(type,"smallint")==0) tform[j] = "1I";
if (strcmp(type,"int")==0) tform[j] = "1J";
if (strcmp(type,"bigint")==0) tform[j] = "1K";
if (strcmp(type,"real")==0) tform[j] = "1E";
if (strcmp(type,"double")==0) tform[j] = "1D";
GDKfree(type);
j++;
}
col = mvc_bind_column(m, tbl, colname[0]);
nrows = store_funcs.count_col(tr, col, 1);
assert(nrows <= (size_t) GDK_oid_max);
snprintf(filename,BUFSIZ,"\n%s.fit",tname);
fprintf(stderr, "Filename: %s\n", filename);
remove(filename);
status=0;
fits_create_file(&fptr, filename, &status);
fits_create_img(fptr, USHORT_IMG, 0, NULL, &status);
fits_close_file(fptr, &status);
fits_open_file(&fptr, filename, READWRITE, &status);
fits_movabs_hdu(fptr, 1, &hdutype, &status);
fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
for (cc = 0; cc < columns; cc++)
{
char * columntype;
col = mvc_bind_column(m, tbl, colname[cc]);
columntype = col -> type.type->sqlname;
if (strcmp(columntype,"boolean")==0)
{
boolcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
boolvalue = (_Bool*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readboolrows[dimension] = *boolvalue;
GDKfree(boolvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
}
if (strcmp(columntype,"char")==0)
{
charcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
charvalue = (char*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readcharrows[dimension] = *charvalue;
GDKfree(charvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
}
if (strcmp(columntype,"varchar")==0)
{
strcols++; dimension=0; block=0;
fits_get_rowsize(fptr,&optimal,&status);
readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
strvalue = (char *) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readstrrows[dimension] = strvalue;
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
texportstring += GDKms() - tm0;
for (dimension = 0; dimension < (size_t) optimal; dimension++)
GDKfree(readstrrows[dimension]);
dimension = 0;
GDKfree(readstrrows);
readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
texportstring += GDKms() - tm0;
for (numberrow = 0; numberrow < dimension; numberrow++)
GDKfree(readstrrows[numberrow]);
GDKfree(readstrrows);
}
if (strcmp(columntype,"smallint")==0)
{
shortcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
shortvalue = (short*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readshortrows[dimension] = *shortvalue;
GDKfree(shortvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
}
if (strcmp(columntype,"int")==0)
{
intcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
intvalue = (int*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readintrows[dimension] = *intvalue;
GDKfree(intvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
}
if (strcmp(columntype,"bigint")==0)
{
lngcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
lngvalue = (lng*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readlngrows[dimension] = *lngvalue;
GDKfree(lngvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status);
texportlng += GDKms() - tm0;
GDKfree(readlngrows);
readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status);
texportlng += GDKms() - tm0;
GDKfree(readlngrows);
}
if (strcmp(columntype,"real")==0)
{
floatcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
realvalue = (float*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readfloatrows[dimension] = *realvalue;
GDKfree(realvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
}
if (strcmp(columntype,"double")==0)
{
doublecols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
doublevalue = (double*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readdoublerows[dimension] = *doublevalue;
GDKfree(doublevalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
}
}
/* print all the times that were needed to export each one of the columns
fprintf(stderr, "\n\n");
if (texportboolean > 0) fprintf(stderr, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean);
if (texportchar > 0) fprintf(stderr, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar);
if (texportstring > 0) fprintf(stderr, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring);
if (texportshort > 0) fprintf(stderr, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort);
if (texportint > 0) fprintf(stderr, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint);
if (texportlng > 0) fprintf(stderr, "%d Long\t\tcolumn(s) exported in %d ms\n", lngcols, texportlng);
if (texportfloat > 0) fprintf(stderr, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat);
if (texportdouble > 0) fprintf(stderr, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble);
*/
fits_close_file(fptr, &status);
return msg;
}
