static str SHPimportFile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bool partial) { mvc *m = NULL; sql_schema *sch = NULL; char *sch_name = "sys"; sql_table *shps_table = NULL, *fls_table = NULL, *data_table = NULL; char *shps_table_name = "shapefiles"; char *fls_table_name = "files"; char *data_table_name = NULL; sql_column *col; sql_column **cols; BAT **colsBAT; int colsNum = 2; //we will have at least the gid column and a geometry column int rowsNum = 0; //the number of rows in the shape file that will be imported //GIntBig rowsNum = 0; int gidNum = 0; char *nameToLowerCase = NULL; str msg = MAL_SUCCEED; str fname = NULL; int vid = *(int*)getArgReference(stk, pci, 1); ptr *p; wkb *g; OGRGeometryH geom; OGREnvelope *mbb; /* SHP-level descriptor */ OGRFieldDefnH hFieldDefn; int i=0; oid irid; GDALWConnection shp_conn; GDALWConnection * shp_conn_ptr = NULL; GDALWSimpleFieldDef * field_definitions; OGRFeatureH feature; OGRFeatureDefnH featureDefn; /* get table columns from shp and create the table */ if((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; if(!(sch = mvc_bind_schema(m, sch_name))) return createException(MAL, "shp.import", SQLSTATE(38000) "Schema '%s' missing", sch_name); /* find the name of the shape file corresponding to the given id */ if(!(fls_table = mvc_bind_table(m, sch, fls_table_name))) return createException(MAL, "shp.import", SQLSTATE(38000) "Table '%s.%s' missing", sch_name, fls_table_name); if(!(col = mvc_bind_column(m, fls_table, "id"))) return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(id)' missing", sch_name, fls_table_name); irid = table_funcs.column_find_row(m->session->tr, col, (void *)&vid, NULL); if (is_oid_nil(irid)) return createException(MAL, "shp.import", SQLSTATE(38000) "Shapefile with id %d not in the %s.%s table\n", vid, sch_name, fls_table_name); if(!(col = mvc_bind_column(m, fls_table, "path"))) return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(path)' missing", sch_name, fls_table_name); fname = (str)table_funcs.column_find_value(m->session->tr, col, irid); /* find the name of the table that has been reserved for this shape file */ if(!(shps_table = mvc_bind_table(m, sch, shps_table_name))) return createException(MAL, "shp.import", SQLSTATE(38000) "Table '%s.%s' missing", sch_name, shps_table_name); if(!(col = mvc_bind_column(m, shps_table, "fileid"))) return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(fileid)' missing", sch_name, shps_table_name); irid = table_funcs.column_find_row(m->session->tr, col, (void *)&vid, NULL); if (is_oid_nil(irid)) return createException(MAL, "shp.import", SQLSTATE(38000) "Shapefile with id %d not in the Shapefile catalog\n", vid); if(!(col = mvc_bind_column(m, shps_table, "datatable"))) return createException(MAL, "shp.import", SQLSTATE(38000) "Column '%s.%s(datatable)' missing", sch_name, shps_table_name); data_table_name = (str)table_funcs.column_find_value(m->session->tr, col, irid); /* add the data on the file to the table */ if(!(shp_conn_ptr = GDALWConnect((char *) fname))) return createException(MAL, "shp.import", SQLSTATE(38000) "Missing shape file %s\n", fname); shp_conn = *shp_conn_ptr; /*count the number of lines in the shape file */ if ((rowsNum = OGR_L_GetFeatureCount(shp_conn.layer, false)) == -1) { if ((rowsNum = OGR_L_GetFeatureCount(shp_conn.layer, true)) == -1) { OGR_L_ResetReading(shp_conn.layer); rowsNum = 0; while ((feature = OGR_L_GetNextFeature(shp_conn.layer)) != NULL ) { rowsNum++; OGR_F_Destroy(feature); } } } /* calculate the mbb of the query geometry */ if (partial) { p = (ptr*)getArgReference(stk, pci, 2); g = (wkb*)*p; geom = OGR_G_CreateGeometry(wkbPolygon); if (OGR_G_ImportFromWkb(geom, (unsigned char*)g->data, g->len) != OGRERR_NONE) { msg = createException(MAL, "shp.import", SQLSTATE(38000) "Could not intantiate the query polygon."); OGR_F_Destroy(geom); goto final; } if (!(mbb = (OGREnvelope*)GDKmalloc(sizeof(OGREnvelope)))) { msg = createException(MAL, "shp.import", SQLSTATE(HY001) MAL_MALLOC_FAIL); OGR_F_Destroy(geom); goto final; }
/* attach a single shp file given its name, fill in shp catalog tables */ str SHPattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { mvc *m = NULL; sql_schema *sch = NULL; sql_table *fls = NULL, *shps = NULL, *shps_dbf = NULL; sql_column *col; str msg = MAL_SUCCEED; str fname = *(str*)getArgReference(stk, pci, 1); /* SHP-level descriptor */ char buf[BUFSIZ], temp_buf[BUFSIZ], *s=buf; int i=0, shpid = 0; oid fid, rid = oid_nil; GDALWConnection shp_conn; GDALWConnection * shp_conn_ptr = NULL; GDALWSimpleFieldDef * field_definitions; GDALWSpatialInfo spatial_info; char *nameToLowerCase = NULL; if((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; if(!(sch = mvc_bind_schema(m, "sys"))) return createException(MAL, "shp.attach", SQLSTATE(38000) "Schema sys missing\n"); fls = mvc_bind_table(m, sch, "files"); shps = mvc_bind_table(m, sch, "shapefiles"); shps_dbf = mvc_bind_table(m, sch, "shapefiles_dbf"); if (fls == NULL || shps == NULL || shps_dbf == NULL ) return createException(MAL, "shp.attach", SQLSTATE(38000) "Catalog table missing\n"); if ((shp_conn_ptr = GDALWConnect((char *) fname)) == NULL) { return createException(MAL, "shp.attach", SQLSTATE(38000) "Missing shape file %s\n", fname); } shp_conn = *shp_conn_ptr; /* check if the file is already attached */ col = mvc_bind_column(m, fls, "path"); rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL); if (!is_oid_nil(rid)) { GDALWClose(shp_conn_ptr); return createException(MAL, "shp.attach", SQLSTATE(38000) "File %s already attached\n", fname); } /* add row in the files(id, path) catalog table */ col = mvc_bind_column(m, fls, "id"); fid = store_funcs.count_col(m->session->tr, col, 1) + 1; snprintf(buf, BUFSIZ, INSFILE, (int)fid, fname); if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED) goto finish; /*if (shp_conn.layer == NULL || shp_conn.source == NULL || shp_conn.handler == NULL || shp_conn.driver == NULL) { msg = createException(MAL, "shp.attach", SQLSTATE(38000) "lol-1\n"); return msg; }*/ /* add row in the shapefiles catalog table (e.g. the name of the table that will store tha data of the shapefile) */ spatial_info = GDALWGetSpatialInfo(shp_conn); col = mvc_bind_column(m, shps, "shapefileid"); shpid = store_funcs.count_col(m->session->tr, col, 1) + 1; nameToLowerCase = toLower(shp_conn.layername); snprintf(buf, BUFSIZ, INSSHP, shpid, (int)fid, spatial_info.epsg, nameToLowerCase); GDKfree(nameToLowerCase); if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED) goto finish; /* add information about the fields of the shape file * one row for each field with info (shapefile_id, field_name, field_type) */ field_definitions = GDALWGetSimpleFieldDefinitions(shp_conn); if (field_definitions == NULL) { GDALWClose(&shp_conn); return createException(MAL, "shp.attach", SQLSTATE(HY001) MAL_MALLOC_FAIL); } for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) { snprintf(buf, BUFSIZ, INSSHPDBF, shpid, field_definitions[i].fieldName, field_definitions[i].fieldType); if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.attach",TRUE,FALSE,NULL)) != MAL_SUCCEED) goto fin; } /* create the table that will store the data of the shape file */ temp_buf[0]='\0'; for (i=0 ; i<shp_conn.numFieldDefinitions ; i++) { nameToLowerCase = toLower(field_definitions[i].fieldName); if (strcmp(field_definitions[i].fieldType, "Integer") == 0) { sprintf(temp_buf + strlen(temp_buf), "\"%s\" INT, ", nameToLowerCase); } else if (strcmp(field_definitions[i].fieldType, "Real") == 0) { sprintf(temp_buf + strlen(temp_buf), "\"%s\" FLOAT, ", nameToLowerCase); #if 0 } else if (strcmp(field_definitions[i].fieldType, "Date") == 0) { sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase); #endif } else sprintf(temp_buf + strlen(temp_buf), "\"%s\" STRING, ", nameToLowerCase); GDKfree(nameToLowerCase); } sprintf(temp_buf + strlen(temp_buf), "geom GEOMETRY "); snprintf(buf, BUFSIZ, CRTTBL, shp_conn.layername, temp_buf); if ( ( msg = SQLstatementIntern(cntxt, &s,"shp.import",TRUE,FALSE,NULL)) != MAL_SUCCEED) goto fin; fin: free(field_definitions); finish: /* if (msg != MAL_SUCCEED){ snprintf(buf, BUFSIZ,"ROLLBACK;"); SQLstatementIntern(cntxt,&s,"geotiff.attach",TRUE,FALSE)); }*/ GDALWClose(&shp_conn); return msg; }
str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { mvc *m = NULL; sql_trans *tr; sql_schema *sch; sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL; sql_column *col; str msg = MAL_SUCCEED; str fname = *getArgReference_str(stk, pci, 1); fitsfile *fptr; /* pointer to the FITS file */ int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0; oid fid, tid, cid, rid = oid_nil; char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ]; long tbcol; /* type long used by fits library */ char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ]; double tscal, tzero; char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = ""; char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = ""; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; if (fits_open_file(&fptr, fname, READONLY, &status)) { msg = createException(MAL, "fits.attach", "Missing FITS file %s.\n", fname); return msg; } tr = m->session->tr; sch = mvc_bind_schema(m, "sys"); fits_fl = mvc_bind_table(m, sch, "fits_files"); if (fits_fl == NULL) FITSinitCatalog(m); fits_fl = mvc_bind_table(m, sch, "fits_files"); fits_tbl = mvc_bind_table(m, sch, "fits_tables"); fits_col = mvc_bind_table(m, sch, "fits_columns"); fits_tp = mvc_bind_table(m, sch, "fits_table_properties"); /* check if the file is already attached */ col = mvc_bind_column(m, fits_fl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL); if (rid != oid_nil) { fits_close_file(fptr, &status); msg = createException(SQL, "fits.attach", "File %s already attached\n", fname); return msg; } /* add row in the fits_files catalog table */ col = mvc_bind_column(m, fits_fl, "id"); fid = store_funcs.count_col(tr, col, 1) + 1; store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_fl, "id"), &fid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_fl, "name"), fname, TYPE_str); col = mvc_bind_column(m, fits_tbl, "id"); tid = store_funcs.count_col(tr, col, 1) + 1; if ((s = strrchr(fname, DIR_SEP)) == NULL) s = fname; else s++; strcpy(bname, s); s = strrchr(bname, '.'); if (s) *s = 0; fits_get_num_hdus(fptr, &hdunum, &status); for (i = 1; i <= hdunum; i++) { fits_movabs_hdu(fptr, i, &hdutype, &status); if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) continue; /* SQL table name - the name of FITS extention */ fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status); if (status) { snprintf(tname, BUFSIZ, "%s_%d", bname, i); tname_low = toLower(tname); status = 0; }else { /* check table name for existence in the fits catalog */ tname_low = toLower(tname); col = mvc_bind_column(m, fits_tbl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname_low, NULL); /* or as regular SQL table */ tbl = mvc_bind_table(m, sch, tname_low); if (rid != oid_nil || tbl) { snprintf(tname, BUFSIZ, "%s_%d", bname, i); tname_low = toLower(tname); } } fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status); if (status) { status = 0; } fits_get_num_cols(fptr, &cnum, &status); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "name"), tname_low, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "columns"), &cnum, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "file_id"), &fid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "date"), tdate, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "origin"), orig, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "comment"), comm, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "table_id"), &tid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "xtension"), xtensionname, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "bitpix"), &bitpixnumber, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "stilvers"), stilversion, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "stilclas"), stilclass, TYPE_str); /* read columns description */ s = stmt; col = mvc_bind_column(m, fits_col, "id"); cid = store_funcs.count_col(tr, col, 1) + 1; for (j = 1; j <= cnum; j++, cid++) { fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname, tform, tunit, j, (int)tid); msg = SQLstatementIntern(cntxt, &s, "fits.attach", TRUE, FALSE, NULL); if (msg != MAL_SUCCEED) { fits_close_file(fptr, &status); return msg; } } tid++; } fits_close_file(fptr, &status); return MAL_SUCCEED; }
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { mvc *m = NULL; sql_schema *sch; sql_table *fits_fl, *fits_tbl, *tbl = NULL; sql_column *col; sql_subtype tpe; fitsfile *fptr; str tname = *getArgReference_str(stk, pci, 1); str fname; str msg = MAL_SUCCEED; oid rid = oid_nil, frid = oid_nil; int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype; int *tpcode = NULL; long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */ char keywrd[80], **cname, nm[FLEN_VALUE]; ptr nilptr; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; sch = mvc_bind_schema(m, "sys"); fits_tbl = mvc_bind_table(m, sch, "fits_tables"); if (fits_tbl == NULL) { msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n"); return msg; } tbl = mvc_bind_table(m, sch, tname); if (tbl) { msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname); return msg; } col = mvc_bind_column(m, fits_tbl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL); if (rid == oid_nil) { msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname); return msg; } /* Open FITS file and move to the table HDU */ col = mvc_bind_column(m, fits_tbl, "file_id"); fid = (int*)table_funcs.column_find_value(m->session->tr, col, rid); fits_fl = mvc_bind_table(m, sch, "fits_files"); col = mvc_bind_column(m, fits_fl, "id"); frid = table_funcs.column_find_row(m->session->tr, col, (void *)fid, NULL); GDKfree(fid); col = mvc_bind_column(m, fits_fl, "name"); fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid); if (fits_open_file(&fptr, fname, READONLY, &status)) { msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname); GDKfree(fname); return msg; } GDKfree(fname); col = mvc_bind_column(m, fits_tbl, "hdu"); hdu = (int*)table_funcs.column_find_value(m->session->tr, col, rid); fits_movabs_hdu(fptr, *hdu, &hdutype, &status); if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) { msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", *hdu); GDKfree(hdu); fits_close_file(fptr, &status); return msg; } GDKfree(hdu); /* create a SQL table to hold the FITS table */ /* col = mvc_bind_column(m, fits_tbl, "columns"); cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */ fits_get_num_cols(fptr, &cnum, &status); tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum); tpcode = (int *)GDKzalloc(sizeof(int) * cnum); rep = (long *)GDKzalloc(sizeof(long) * cnum); wid = (long *)GDKzalloc(sizeof(long) * cnum); cname = (char **)GDKzalloc(sizeof(char *) * cnum); for (j = 1; j <= cnum; j++) { /* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */ snprintf(keywrd, 80, "TTYPE%d", j); fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status); if (status) { snprintf(nm, FLEN_VALUE, "column_%d", j); status = 0; } cname[j - 1] = toLower(nm); fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status); fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]); /* fprintf(stderr,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */ mvc_create_column(m, tbl, cname[j - 1], &tpe); } /* data load */ fits_get_num_rows(fptr, &rows, &status); fprintf(stderr,"#Loading %ld rows in table %s\n", rows, tname); for (j = 1; j <= cnum; j++) { BAT *tmp = NULL; int time0 = GDKms(); mtype = fits2mtype(tpcode[j - 1]); nilptr = ATOMnilptr(mtype); col = mvc_bind_column(m, tbl, cname[j - 1]); tmp = BATnew(TYPE_void, mtype, rows, TRANSIENT); if ( tmp == NULL){ GDKfree(tpcode); GDKfree(rep); GDKfree(wid); GDKfree(cname); throw(MAL,"fits.load", MAL_MALLOC_FAIL); } BATseqbase(tmp, 0); if (mtype != TYPE_str) { fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status); BATsetcount(tmp, rows); tmp->tsorted = 0; tmp->trevsorted = 0; } else { /* char *v = GDKzalloc(wid[j-1]);*/ /* type long demanded by "rows", i.e., by fits library */ long bsize = 50, batch = bsize, k, i; int tm0, tloadtm = 0, tattachtm = 0; char **v = (char **) GDKzalloc(sizeof(char *) * bsize); for(i = 0; i < bsize; i++) v[i] = GDKzalloc(wid[j-1]); for(i = 0; i < rows; i += batch) { batch = rows - i < bsize ? rows - i: bsize; tm0 = GDKms(); fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status); tloadtm += GDKms() - tm0; tm0 = GDKms(); for(k = 0; k < batch ; k++) BUNappend(tmp, v[k], TRUE); tattachtm += GDKms() - tm0; } for(i = 0; i < bsize ; i++) GDKfree(v[i]); GDKfree(v); fprintf(stderr,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm); } if (status) { char buf[FLEN_ERRMSG + 1]; fits_read_errmsg(buf); msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf); break; } fprintf(stderr,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0); store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat); fprintf(stderr,"#Total %d ms\n", GDKms() - time0); BBPunfix(tmp->batCacheid); } GDKfree(tpcode); GDKfree(rep); GDKfree(wid); GDKfree(cname); fits_close_file(fptr, &status); return msg; }
str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str msg = MAL_SUCCEED; str tname = *getArgReference_str(stk, pci, 1); mvc *m = NULL; sql_trans *tr; sql_schema *sch; sql_table *tbl, *column, *tables = NULL; sql_column *col; oid rid = oid_nil; str type, name, *colname, *tform; fitsfile *fptr; char filename[BUFSIZ]; size_t nrows = 0; long optimal; /* type long used by fits library */ rids * rs; int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0; size_t numberrow = 0, dimension = 0; int cc = 0, status = 0, j = 0, columns, *fid, block = 0; int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0; int hdutype; char *charvalue, *readcharrows; str strvalue; char **readstrrows; short *shortvalue, *readshortrows; int *intvalue, *readintrows; lng *lngvalue, *readlngrows; float *realvalue, *readfloatrows; double *doublevalue, *readdoublerows; _Bool *boolvalue, *readboolrows; struct list * set; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; tr = m->session->tr; sch = mvc_bind_schema(m, "sys"); /* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */ tbl = mvc_bind_table(m, sch, tname); if (tbl == NULL) { msg = createException (MAL, "fits.exporttable", "Table %s is missing.\n", tname); return msg; } set = (*tbl).columns.set; columns = list_length(set); colname = (str *) GDKmalloc(columns * sizeof(str)); tform = (str *) GDKmalloc(columns * sizeof(str)); /* fprintf(stderr,"Number of columns: %d\n", columns);*/ tables = mvc_bind_table(m, sch, "_tables"); col = mvc_bind_column(m, tables, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL); col = mvc_bind_column(m, tables, "id"); fid = (int*) table_funcs.column_find_value(m->session->tr, col, rid); column = mvc_bind_table(m, sch, "_columns"); col = mvc_bind_column(m, column, "table_id"); rs = table_funcs.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL); GDKfree(fid); while ((rid = table_funcs.rids_next(rs)) != oid_nil) { col = mvc_bind_column(m, column, "name"); name = (char *) table_funcs.column_find_value(m->session->tr, col, rid); colname[j] = toLower(name); GDKfree(name); col = mvc_bind_column(m, column, "type"); type = (char *) table_funcs.column_find_value(m->session->tr, col, rid); if (strcmp(type,"boolean")==0) tform[j] = "1L"; if (strcmp(type,"char")==0) tform[j] = "1S"; if (strcmp(type,"varchar")==0) tform[j] = "8A"; if (strcmp(type,"smallint")==0) tform[j] = "1I"; if (strcmp(type,"int")==0) tform[j] = "1J"; if (strcmp(type,"bigint")==0) tform[j] = "1K"; if (strcmp(type,"real")==0) tform[j] = "1E"; if (strcmp(type,"double")==0) tform[j] = "1D"; GDKfree(type); j++; } col = mvc_bind_column(m, tbl, colname[0]); nrows = store_funcs.count_col(tr, col, 1); assert(nrows <= (size_t) GDK_oid_max); snprintf(filename,BUFSIZ,"\n%s.fit",tname); fprintf(stderr, "Filename: %s\n", filename); remove(filename); status=0; fits_create_file(&fptr, filename, &status); fits_create_img(fptr, USHORT_IMG, 0, NULL, &status); fits_close_file(fptr, &status); fits_open_file(&fptr, filename, READWRITE, &status); fits_movabs_hdu(fptr, 1, &hdutype, &status); fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status); for (cc = 0; cc < columns; cc++) { char * columntype; col = mvc_bind_column(m, tbl, colname[cc]); columntype = col -> type.type->sqlname; if (strcmp(columntype,"boolean")==0) { boolcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { boolvalue = (_Bool*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readboolrows[dimension] = *boolvalue; GDKfree(boolvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status); texportboolean += GDKms() - tm0; GDKfree(readboolrows); readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status); texportboolean += GDKms() - tm0; GDKfree(readboolrows); } if (strcmp(columntype,"char")==0) { charcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readcharrows = (char *) GDKmalloc (sizeof(char) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { charvalue = (char*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readcharrows[dimension] = *charvalue; GDKfree(charvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status); texportchar += GDKms() - tm0; GDKfree(readcharrows); readcharrows = (char *) GDKmalloc (sizeof(char) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status); texportchar += GDKms() - tm0; GDKfree(readcharrows); } if (strcmp(columntype,"varchar")==0) { strcols++; dimension=0; block=0; fits_get_rowsize(fptr,&optimal,&status); readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { strvalue = (char *) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readstrrows[dimension] = strvalue; dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status); texportstring += GDKms() - tm0; for (dimension = 0; dimension < (size_t) optimal; dimension++) GDKfree(readstrrows[dimension]); dimension = 0; GDKfree(readstrrows); readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal); block++; } } tm0 = GDKms(); fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status); texportstring += GDKms() - tm0; for (numberrow = 0; numberrow < dimension; numberrow++) GDKfree(readstrrows[numberrow]); GDKfree(readstrrows); } if (strcmp(columntype,"smallint")==0) { shortcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readshortrows = (short *) GDKmalloc (sizeof(short) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { shortvalue = (short*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readshortrows[dimension] = *shortvalue; GDKfree(shortvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status); texportshort += GDKms() - tm0; GDKfree(readshortrows); readshortrows = (short *) GDKmalloc (sizeof(short) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status); texportshort += GDKms() - tm0; GDKfree(readshortrows); } if (strcmp(columntype,"int")==0) { intcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readintrows = (int *) GDKmalloc (sizeof(int) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { intvalue = (int*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readintrows[dimension] = *intvalue; GDKfree(intvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status); texportint += GDKms() - tm0; GDKfree(readintrows); readintrows = (int *) GDKmalloc (sizeof(int) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status); texportint += GDKms() - tm0; GDKfree(readintrows); } if (strcmp(columntype,"bigint")==0) { lngcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { lngvalue = (lng*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readlngrows[dimension] = *lngvalue; GDKfree(lngvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status); texportlng += GDKms() - tm0; GDKfree(readlngrows); readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status); texportlng += GDKms() - tm0; GDKfree(readlngrows); } if (strcmp(columntype,"real")==0) { floatcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { realvalue = (float*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readfloatrows[dimension] = *realvalue; GDKfree(realvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status); texportfloat += GDKms() - tm0; GDKfree(readfloatrows); readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status); texportfloat += GDKms() - tm0; GDKfree(readfloatrows); } if (strcmp(columntype,"double")==0) { doublecols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { doublevalue = (double*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readdoublerows[dimension] = *doublevalue; GDKfree(doublevalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status); texportdouble += GDKms() - tm0; GDKfree(readdoublerows); readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status); texportdouble += GDKms() - tm0; GDKfree(readdoublerows); } } /* print all the times that were needed to export each one of the columns fprintf(stderr, "\n\n"); if (texportboolean > 0) fprintf(stderr, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean); if (texportchar > 0) fprintf(stderr, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar); if (texportstring > 0) fprintf(stderr, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring); if (texportshort > 0) fprintf(stderr, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort); if (texportint > 0) fprintf(stderr, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint); if (texportlng > 0) fprintf(stderr, "%d Long\t\tcolumn(s) exported in %d ms\n", lngcols, texportlng); if (texportfloat > 0) fprintf(stderr, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat); if (texportdouble > 0) fprintf(stderr, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble); */ fits_close_file(fptr, &status); return msg; }