str
QOTshowPlan(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str modnme;
str fcnnme;
Symbol s = NULL;
if (stk != 0) {
modnme = *getArgReference_str(stk, p, 1);
fcnnme = *getArgReference_str(stk, p, 2);
} else {
modnme = getArgDefault(mb, p, 1);
fcnnme = getArgDefault(mb, p, 2);
}
mnstr_printf(cntxt->fdout,"#showPlan()\n");
removeInstruction(mb, p);
if( modnme ) {
s = findSymbol(cntxt->nspace, putName(modnme, strlen(modnme)), putName(fcnnme, strlen(fcnnme)));
if (s == NULL) {
char buf[1024];
snprintf(buf,1024, "%s.%s", modnme, fcnnme);
throw(MAL, "optimizer.showPlan", RUNTIME_OBJECT_UNDEFINED ":%s", buf);
}
mb= s->def;
}
printFunction(cntxt->fdout, mb, 0, LIST_INPUT);
return MAL_SUCCEED;
}
str
QOTshowFlowGraph(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str fname;
str modnme;
str fcnnme;
Symbol s = NULL;
(void) cntxt;
if (stk != 0) {
modnme = *getArgReference_str(stk, p, 1);
fcnnme = *getArgReference_str(stk, p, 2);
fname = *getArgReference_str(stk, p, 3);
} else {
modnme = getArgDefault(mb, p, 1);
fcnnme = getArgDefault(mb, p, 2);
fname = getArgDefault(mb, p, 3);
}
s = findSymbol(cntxt->nspace,putName(modnme, strlen(modnme)), putName(fcnnme, strlen(fcnnme)));
if (s == NULL) {
char buf[1024];
snprintf(buf,1024, "%s.%s", modnme, fcnnme);
throw(MAL, "optimizer.showFlowGraph", RUNTIME_OBJECT_UNDEFINED ":%s", buf);
}
showFlowGraph(s->def, stk, fname);
return MAL_SUCCEED;
}
/*
* Display routines
*/
str
MDBlifespan(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
Lifespan span;
str modnme;
str fcnnme;
Symbol s = NULL;
(void) cntxt;
if (stk != 0) {
modnme = *getArgReference_str(stk, p, 1);
fcnnme = *getArgReference_str(stk, p, 2);
} else {
modnme = getArgDefault(mb, p, 1);
fcnnme = getArgDefault(mb, p, 2);
}
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL)
throw(MAL, "mdb.inspect", RUNTIME_SIGNATURE_MISSING);
span = setLifespan(s->def);
if( span == NULL)
throw(MAL,"mdb.inspect", MAL_MALLOC_FAIL);
debugLifespan(cntxt, s->def, span);
GDKfree(span);
(void) p;
(void) stk;
return MAL_SUCCEED;
}
str
MDBtrapFunction(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str modnme = *getArgReference_str(stk, pci, 1);
str fcnnme = *getArgReference_str(stk, pci, 2);
bit b= *getArgReference_bit(stk,pci,3);
(void) cntxt;
(void) mb;
if ( mdbSetTrap(cntxt,modnme,fcnnme,b) )
throw(MAL,"mdb.trap", RUNTIME_SIGNATURE_MISSING);
return MAL_SUCCEED;
}
str OPTsql_append(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){
str modnme;
str fcnnme;
str msg= MAL_SUCCEED;
Symbol s= NULL;
lng t,clk= GDKusec();
int actions = 0;
if( p )
removeInstruction(mb, p);
OPTDEBUGsql_append mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER sql_append\n");
if( p && p->argc > 1 ){
if( getArgType(mb,p,1) != TYPE_str ||
getArgType(mb,p,2) != TYPE_str ||
!isVarConstant(mb,getArg(p,1)) ||
!isVarConstant(mb,getArg(p,2))
) {
throw(MAL, "optimizer.sql_append", ILLARG_CONSTANTS);
}
if( stk != 0){
modnme= *getArgReference_str(stk,p,1);
fcnnme= *getArgReference_str(stk,p,2);
} else {
modnme= getArgDefault(mb,p,1);
fcnnme= getArgDefault(mb,p,2);
}
s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme)));
if( s == NULL) {
char buf[1024];
snprintf(buf,1024, "%s.%s",modnme,fcnnme);
throw(MAL, "optimizer.sql_append", RUNTIME_OBJECT_UNDEFINED ":%s", buf);
}
mb = s->def;
stk= 0;
}
if( mb->errors ){
/* when we have errors, we still want to see them */
addtoMalBlkHistory(mb,"sql_append");
return MAL_SUCCEED;
}
actions= OPTsql_appendImplementation(cntxt, mb,stk,p);
msg= optimizerCheck(cntxt, mb, "optimizer.sql_append", actions, t=(GDKusec() - clk));
OPTDEBUGsql_append {
mnstr_printf(cntxt->fdout,"=FINISHED sql_append %d\n",actions);
printFunction(cntxt->fdout,mb,0,LIST_MAL_ALL );
}
DEBUGoptimizers
mnstr_printf(cntxt->fdout,"#opt_reduce: " LLFMT " ms\n",t);
QOTupdateStatistics("sql_append",actions,t);
addtoMalBlkHistory(mb,"sql_append");
return msg;
}
str
MDBvar3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str modnme = *getArgReference_str(stk, p, 1);
str fcnnme = *getArgReference_str(stk, p, 2);
Symbol s = NULL;
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL)
throw(MAL,"mdb.var","Could not find %s.%s", modnme, fcnnme);
printStack(cntxt->fdout, s->def, (s->def == mb ? stk : 0));
(void) mb;
return NULL;
}
str
MDBlist3Detail(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str modnme = *getArgReference_str(stk, p, 1);
str fcnnme = *getArgReference_str(stk, p, 2);
Symbol s = NULL;
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL)
throw(MAL,"mdb.list","Could not find %s.%s", modnme, fcnnme);
debugFunction(cntxt->fdout, s->def, 0, LIST_MAL_NAME | LIST_MAL_VALUE | LIST_MAL_TYPE | LIST_MAL_PROPS , 0, s->def->stop);
(void) mb; /* fool compiler */
return NULL;
}
str
MDBlist3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str modnme = *getArgReference_str(stk, p, 1);
str fcnnme = *getArgReference_str(stk, p, 2);
Symbol s = NULL;
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL)
throw(MAL,"mdb.list","Could not find %s.%s", modnme, fcnnme);
printFunction(cntxt->fdout, s->def, 0, LIST_MAL_NAME );
(void) mb; /* fool compiler */
return MAL_SUCCEED;
}
str
FCTshutdown(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str mod = *getArgReference_str(stk, pci, 1);
str fcn = *getArgReference_str(stk, pci, 2);
Symbol s;
(void) mb;
s = findSymbol(cntxt->nspace, putName(mod), putName(fcn));
if (s == NULL)
throw(MAL, "factories.shutdown", RUNTIME_OBJECT_MISSING);
shutdownFactory(cntxt,s->def);
return MAL_SUCCEED;
}
str
SQLcompile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *ret = getArgReference_str(stk, pci, 0);
str *expr = getArgReference_str(stk, pci, 1);
str msg;
(void) mb;
*ret = NULL;
msg = SQLstatementIntern(cntxt, expr, "SQLcompile", FALSE, FALSE, NULL);
if (msg == MAL_SUCCEED)
*ret = _STRDUP("SQLcompile");
return msg;
}
str
CMDsetProfilerFile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *fnme = getArgReference_str(stk,pci,1);
(void) mb; /* fool compiler */
return setLogFile(cntxt->fdout,cntxt->nspace, *fnme);
}
str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *getArgReference_str(stk, pci, 1);
DIR *dp;
struct dirent *ep;
fitsfile *fptr;
char *s;
int status = 0;
(void)mb;
dp = opendir(dir);
if (dp != NULL) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
while ((ep = readdir(dp)) != NULL && !msg) {
snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdir", TRUE, FALSE, NULL);
fits_close_file(fptr, &status);
}
}
(void)closedir(dp);
} else
msg = createException(MAL, "listdir", "Couldn't open the directory");
return msg;
}
str
CMDsetProfilerStream (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *host = getArgReference_str(stk,pci,1);
int *port = getArgReference_int(stk,pci,2);
(void) mb; /* fool compiler */
return setLogStream(cntxt->nspace, *host, *port);
}
str
MDBsetVarTrace(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str v;
(void) cntxt;
v = *getArgReference_str(stk, p, 1);
mdbSetBreakRequest(cntxt, mb, v, 't');
stk->cmd = 'c';
cntxt->itrace = 'c';
return MAL_SUCCEED;
}
str
SQLstatement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *expr = getArgReference_str(stk, pci, 1);
bit output = TRUE;
(void) mb;
if (pci->argc == 3)
output = *getArgReference_bit(stk, pci, 2);
return SQLstatementIntern(cntxt, expr, "SQLstatement", TRUE, output, NULL);
}
str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str dir = *getArgReference_str(stk, pci, 1);
str pat = *getArgReference_str(stk, pci, 2);
fitsfile *fptr;
char *s;
int status = 0;
glob_t globbuf;
char fulldirectory[BUFSIZ];
size_t j = 0;
(void)mb;
globbuf.gl_offs = 0;
snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat);
glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
/* fprintf(stderr,"#fulldir: %s \nSize: %lu\n",fulldirectory, globbuf.gl_pathc);*/
if (globbuf.gl_pathc == 0)
throw(MAL, "listdir", "Couldn't open the directory or there are no files that match the pattern");
for (j = 0; j < globbuf.gl_pathc; j++) {
char stmt[BUFSIZ];
char fname[BUFSIZ];
s = stmt;
snprintf(fname, BUFSIZ, "%s", globbuf.gl_pathv[j]);
status = 0;
fits_open_file(&fptr, fname, READONLY, &status);
if (status == 0) {
snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
msg = SQLstatementIntern(cntxt, &s, "fits.listofdirpat", TRUE, FALSE, NULL);
fits_close_file(fptr, &status);
break;
}
}
return msg;
}
str
MDBinspect(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str modnme;
str fcnnme;
Symbol s = NULL;
(void) cntxt;
if (stk != 0) {
modnme = *getArgReference_str(stk, p, 1);
fcnnme = *getArgReference_str(stk, p, 2);
} else {
modnme = getArgDefault(mb, p, 1);
fcnnme = getArgDefault(mb, p, 2);
}
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL)
throw(MAL, "mdb.inspect", RUNTIME_SIGNATURE_MISSING);
return runMALDebugger(cntxt, s);
}
str
MDBshowFlowGraph(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
str fname;
str modnme;
str fcnnme;
Symbol s = NULL;
(void)cntxt;
if (stk != 0) {
if (p->argc == 2) {
modnme = fcnnme = NULL;
fname = *getArgReference_str(stk, p, 1);
} else {
modnme = *getArgReference_str(stk, p, 1);
fcnnme = *getArgReference_str(stk, p, 2);
fname = *getArgReference_str(stk, p, 3);
}
} else {
modnme = getArgDefault(mb, p, 1);
fcnnme = getArgDefault(mb, p, 2);
fname = getArgDefault(mb, p, 3);
}
if (modnme != NULL) {
s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme));
if (s == NULL) {
char buf[1024];
snprintf(buf,1024, "Could not find %s.%s\n", modnme, fcnnme);
throw(MAL, "mdb.dot", "%s", buf);
}
showFlowGraph(s->def, stk, fname);
} else {
showFlowGraph(mb, stk, fname);
}
return MAL_SUCCEED;
}
/**
* Calls the bam loader for a single file
*/
str
bam_loader_file(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
/* arg 1: path to bam stream */
str bam_file = *getArgReference_str(stk, pci, pci->retc);
/* arg 2: dbschema to use */
sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1);
(void) stk;
(void) pci;
return bam_loader(cntxt, mb, &bam_file, 1, dbschema, 1);
}
str
QOToptimize(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str modnme;
str fcnnme;
Symbol s;
(void) stk;
if (stk != 0) {
modnme = *getArgReference_str(stk, pci, 1);
fcnnme = *getArgReference_str(stk, pci, 2);
} else {
modnme = getArgDefault(mb, pci, 1);
fcnnme = getArgDefault(mb, pci, 2);
}
s = findSymbol(cntxt->usermodule, putName(modnme), fcnnme);
if (s == NULL)
throw(MAL, "optimizer.optimize", SQLSTATE(HY002) SEMANTIC_OPERATION_MISSING);
removeInstruction(mb, pci);
addtoMalBlkHistory(s->def);
return optimizeMALBlock(cntxt, s->def);
}
str
CMDbbpbind(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str name;
ValPtr lhs;
bat i;
int ht,tt;
BAT *b;
(void) cntxt;
(void) mb; /* fool compiler */
lhs = &stk->stk[pci->argv[0]];
name = *getArgReference_str(stk, pci, 1);
if (isIdentifier(name) < 0)
throw(MAL, "bbp.bind", IDENTIFIER_EXPECTED);
i = BBPindex(name);
if (i == 0)
throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);
/* make sure you load the descriptors and heaps */
b = (BAT *) BATdescriptor(i);
if (b == 0)
/* Simple ignore the binding if you can't find the bat */
throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);
/* check conformity of the actual type and the one requested */
ht= getHeadType(getArgType(mb,pci,0));
tt= getColumnType(getArgType(mb,pci,0));
if( b->htype == TYPE_void && ht== TYPE_oid) ht= TYPE_void;
if( b->ttype == TYPE_void && tt== TYPE_oid) tt= TYPE_void;
if( ht != b->htype || tt != b->ttype){
BBPunfix(i);
throw(MAL, "bbp.bind", SEMANTIC_TYPE_MISMATCH );
}
/* make sure we are not dealing with an about to be deleted bat */
if( BBP_refs(b->batCacheid) == 1 &&
BBP_lrefs(b->batCacheid) == 0){
BBPunfix(i);
throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);
}
BBPkeepref(b->batCacheid);
lhs->vtype = TYPE_bat;
lhs->val.bval = i;
return MAL_SUCCEED;
}
/*
* Locate a file with SQL commands and execute it. For the time being a 1MB
* file limit is implicitly imposed. If the file can not be located in the
* script library, we assume it is sufficiently self descriptive.
* (Respecting the file system context where the call is executed )
*/
str
SQLinclude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
stream *fd;
bstream *bfd;
str *name = getArgReference_str(stk, pci, 1);
str msg = MAL_SUCCEED, fullname;
str *expr;
mvc *m;
size_t sz;
fullname = MSP_locate_sqlscript(*name, 0);
if (fullname == NULL)
fullname = *name;
fd = open_rastream(fullname);
if (mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
mnstr_destroy(fd);
throw(MAL, "sql.include", "could not open file: %s\n", *name);
}
sz = getFileSize(fd);
if (sz > (size_t) 1 << 29) {
mnstr_destroy(fd);
throw(MAL, "sql.include", "file %s too large to process", fullname);
}
bfd = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz);
if (bstream_next(bfd) < 0) {
bstream_destroy(bfd);
throw(MAL, "sql.include", "could not read %s\n", *name);
}
expr = &bfd->buf;
msg = SQLstatementIntern(cntxt, expr, "sql.include", TRUE, FALSE, NULL);
bstream_destroy(bfd);
m = ((backend *) cntxt->sqlcontext)->mvc;
if (m->sa)
sa_destroy(m->sa);
m->sa = NULL;
(void) mb;
return msg;
}
/*
* THe choice operator first searches the next one to identify
* the fragment to be optimized and to gain access to the variables
* without the need to declare them upfront.
*/
str
RUNchoice(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int target;
lng cost, mincost;
int i, j, pc;
char *nme;
InstrPtr q;
pc = getPC(mb, p);
for (i = pc + 1; i < mb->stop; i++) {
q = getInstrPtr(mb, i);
if (getModuleId(p) == getModuleId(q) &&
getFunctionId(p) == getFunctionId(q)) {
p = q;
break;
}
}
if (i == mb->stop)
return MAL_SUCCEED;
target = getArg(p, 2);
if (getArgType(mb, p, 1) == TYPE_int && p->argc >= 3 && (p->argc - 1) % 2 == 0) {
/* choice pairs */
mincost = *getArgReference_int(stk, p, 1);
for (i = 3; i < p->argc; i += 2) {
cost = *getArgReference_int(stk, p, i);
if (cost < mincost && !isVarDisabled(mb, getArg(p, i + 1))) {
mincost = cost;
target = getArg(p, i + 1);
}
}
} else if (getArgType(mb, p, 1) == TYPE_str) {
nme = *getArgReference_str(stk, p, 1);
/* should be generalized to allow an arbitrary user defined function */
if (strcmp(nme, "getVolume") != 0)
throw(MAL, "scheduler.choice", ILLEGAL_ARGUMENT "Illegal cost function");
mincost = -1;
for (j = 2; j < p->argc; j++) {
if (!isVarDisabled(mb, getArg(p, j)))
for (i = pc + 1; i < mb->stop; i++) {
InstrPtr q = getInstrPtr(mb, i);
if (p->token >= 0 && getArg(q, 0) == getArg(p, j)) {
cost = getVolume(stk, q, 1);
if (cost > 0 && (cost < mincost || mincost == -1)) {
mincost = cost;
target = getArg(p, j);
}
break;
}
}
}
}
#ifdef DEBUG_RUN_MEMORUN
mnstr_printf(cntxt->fdout, "#function target %s cost %d\n", getVarName(mb, target), mincost);
#else
(void) cntxt;
#endif
/* remove non-qualifying variables */
for (i = 2; i < p->argc; i += 2)
if (getArg(p, i) != target) {
setVarDisabled(mb, getArg(p, i - 1));
setVarDisabled(mb, getArg(p, i));
}
propagateNonTarget(mb, pc + 1);
#ifdef DEBUG_RUN_MEMORUN
mnstr_printf(cntxt->fdout, "#cost choice selected %s %d\n",
getVarName(mb, target), mincost);
printFunction(cntxt->fdout, mb, 1);
#endif
return MAL_SUCCEED;
}
/**
* Gathers all BAM files that are listed in the given file and calls
* bam_loader for these files
*/
str
bam_loader_files(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
/* arg 1: path to file containing bam file names separated by \n */
str bam_files = *getArgReference_str(stk, pci, pci->retc);
/* arg 2: dbschema to use */
sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1);
/* For now, we hard code the number of threads, since the SQL
level should not bother with this */
sht nr_threads = GDKnr_threads;
str *filenames = NULL;
int nr_files = 0;
FILE *f = NULL;
int cur;
int line_size;
str line = NULL;
size_t line_buf_size = 0;
int nr_lines = 1;
str msg = MAL_SUCCEED;
int i;
/* Open the file */
f = fopen(bam_files, "r");
if (f == NULL) {
msg = createException(MAL, "bam_loader_files",
"Error on opening file list '%s': %s",
bam_files, strerror(errno));
goto cleanup;
}
/* Start with counting the number of lines in the file (is nr
* of newlines + 1) */
while ((cur = fgetc(f)) != EOF) {
if (cur == '\n')
++nr_lines;
}
if (ferror(f)) {
msg = createException(MAL, "bam_loader_files",
"Error while processing file '%s'",
bam_files);
goto cleanup;
}
/* Now we can malloc the filenames array */
if ((filenames = (str *) GDKmalloc(nr_lines * sizeof(str))) == NULL) {
msg = createException(MAL, "bam_loader_files",
MAL_MALLOC_FAIL);
goto cleanup;
}
/* Enables cleanup to check individual files */
memset(filenames, 0, nr_lines * sizeof(str));
/* Reset file pointer to beginning of file */
rewind(f);
/* Read BAM filenames into files array */
while ((line_size =
getline(&line, &line_buf_size, f)) >= 0) {
if (line_size == 0 || line[0] == '\n' || line[0] == '\r' || line[0] == '#') {
/* Skip empty lines or lines that start with a hash */
free(line);
line = NULL;
/* We need to reset buf size to 0, since getline will have updated it */
line_buf_size = 0;
continue;
}
if (line[line_size - 1] == '\n') {
/* Newline character is not part of the filepath */
line[line_size - 1] = '\0';
}
if(line_size > 1 && line[line_size - 2] == '\r') {
/* Return character is also not a part of the filepath */
line[line_size - 2] = '\0';
}
filenames[nr_files++] = line;
line = NULL;
/* We need to reset it to 0, since getline will have updated this val */
line_buf_size = 0;
}
if (ferror(f)) {
msg = createException(MAL, "bam_loader_files",
"Error while processing file '%s'",
bam_files);
goto cleanup;
}
if (nr_files == 0) {
msg = createException(MAL, "bam_loader_files",
"No valid entries found in file '%s'",
bam_files);
goto cleanup;
}
/* And call the bam loader */
msg = bam_loader(cntxt, mb, filenames, nr_files, dbschema,
nr_threads);
cleanup:
if(line) {
//if line still contains anything, this is something that was put there
//by getline on the last, failed attempt to read a line
free(line);
}
if (filenames) {
for (i = 0; i < nr_files; ++i) {
if (filenames[i]) {
free(filenames[i]);
}
}
GDKfree(filenames);
}
if (f)
fclose(f);
(void) stk;
(void) pci;
return msg;
}
/**
* Gathers all BAM files in the given repository and calls bam_loader for these files
*/
str
bam_loader_repos(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
/* arg 1: path to bam file repository */
str bam_repos = *getArgReference_str(stk, pci, pci->retc);
/* arg 2: dbschema to use */
sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1);
/* For now, we hard code the number of threads, since the SQL
level should not bother with this */
sht nr_threads = GDKnr_threads;
str *filenames = NULL;
int nr_files = 0;
DIR *d = NULL;
struct dirent *direntry = NULL;
char path[4096];
int i, filecount = 0;
str msg = MAL_SUCCEED;
if ((d = opendir(bam_repos)) == NULL) {
if (errno == ENOENT) {
msg = createException(MAL, "bam_loader_repos",
"Could not find directory '%s'",
bam_repos);
} else {
msg = createException(MAL, "bam_loader_repos",
"Could not open directory '%s'",
bam_repos);
}
goto cleanup;
}
/* First, count number of SAM/BAM files */
while ((direntry = readdir(d)) != NULL) {
int len = strlen(direntry->d_name);
if (IS_SAMORBAM(direntry->d_name, len))
++filecount;
}
if (filecount == 0) {
msg = createException(MAL, "bam_loader_repos",
"No SAM or BAM files found in directory '%s'",
bam_repos);
goto cleanup;
}
/* Now malloc enough memory for filenames array */
if ((filenames = (str *) GDKmalloc(filecount * sizeof(str))) == NULL) {
msg = createException(MAL, "bam_loader_repos",
MAL_MALLOC_FAIL);
goto cleanup;
}
/* Enables cleanup to check individual filenames */
memset(filenames, 0, filecount * sizeof(str));
rewinddir(d);
/* Now loop through dir; we add a slash in between the given
* bam_repos and the filename. This could lead to double
* slashes in a path, but as far as I know, this is no problem
* on all OS's */
while ((direntry = readdir(d)) != NULL) {
/* Check if d_name has the .sam or .bam extension (case
* insensitive) */
int len = strlen(direntry->d_name);
if (IS_SAMORBAM(direntry->d_name, len)) {
/* This is a SAM or BAM file, construct its path and
* add that to the files array */
if (snprintf
(path, 4096, "%s/%s", bam_repos,
direntry->d_name) < 0) {
msg = createException(MAL, "bam_loader_repos",
"Could not construct filepath for SAM/BAM file '%s'",
direntry->d_name);
goto cleanup;
}
if (nr_files >= filecount) {
/* This should not happen. If it does,
* it might be the case that a BAM
* file was added to the directory in
* the meantime */
msg = createException(MAL, "bam_loader_repos",
"An error occurred while reading directory '%s'",
bam_repos);
goto cleanup;
}
filenames[nr_files++] = GDKstrdup(path);
}
}
/* And call the bam loader */
msg = bam_loader(cntxt, mb, filenames, nr_files, dbschema,
nr_threads);
cleanup:
if (filenames) {
for (i = 0; i < nr_files; ++i) {
if (filenames[i])
GDKfree(filenames[i]);
}
GDKfree(filenames);
}
if (d)
closedir(d);
(void) stk;
(void) pci;
return msg;
}
str OPTwrapper (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){
str modnme = "(NONE)";
str fcnnme = 0;
str msg= MAL_SUCCEED;
Symbol s= NULL;
lng t,clk= GDKusec();
int i, actions = 0;
char optimizer[256];
InstrPtr q;
if( p == NULL)
throw(MAL, "opt_wrapper", "missing optimizer statement");
snprintf(optimizer,256,"%s", fcnnme = getFunctionId(p));
q= copyInstruction(p);
OPTIMIZERDEBUG
mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER %s\n",fcnnme);
if( p && p->argc > 1 ){
if( getArgType(mb,p,1) != TYPE_str ||
getArgType(mb,p,2) != TYPE_str ||
!isVarConstant(mb,getArg(p,1)) ||
!isVarConstant(mb,getArg(p,2))
) {
freeInstruction(q);
throw(MAL, optimizer, ILLARG_CONSTANTS);
}
if( stk != 0){
modnme= *getArgReference_str(stk,p,1);
fcnnme= *getArgReference_str(stk,p,2);
} else {
modnme= getArgDefault(mb,p,1);
fcnnme= getArgDefault(mb,p,2);
}
removeInstruction(mb, p);
s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme)));
if( s == NULL) {
freeInstruction(q);
throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme);
}
mb = s->def;
stk= 0;
} else if( p )
removeInstruction(mb, p);
if( mb->errors ){
/* when we have errors, we still want to see them */
addtoMalBlkHistory(mb,getModuleId(q));
freeInstruction(q);
return MAL_SUCCEED;
}
for ( i=0; codes[i].nme; i++)
if ( strcmp(codes[i].nme, optimizer)== 0 ){
actions = (int)(*(codes[i].fcn))(cntxt, mb, stk,0);
break;
}
if ( codes[i].nme == 0){
freeInstruction(q);
throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme);
}
msg= optimizerCheck(cntxt, mb, optimizer, actions, t=(GDKusec() - clk));
OPTIMIZERDEBUG {
mnstr_printf(cntxt->fdout,"=FINISHED %s %d\n",optimizer, actions);
printFunction(cntxt->fdout,mb,0,LIST_MAL_DEBUG );
}
DEBUGoptimizers
mnstr_printf(cntxt->fdout,"#optimizer %-11s %3d actions %5d MAL instructions ("SZFMT" K) " LLFMT" usec\n", optimizer, actions, mb->stop,
((sizeof( MalBlkRecord) +mb->ssize * offsetof(InstrRecord, argv)+ mb->vtop * sizeof(int) /* argv estimate */ +mb->vtop* sizeof(VarRecord) + mb->vsize*sizeof(VarPtr)+1023)/1024),
t);
QOTupdateStatistics(getModuleId(q),actions,t);
addtoMalBlkHistory(mb,getModuleId(q));
freeInstruction(q);
return msg;
}
str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str msg = MAL_SUCCEED;
str tname = *getArgReference_str(stk, pci, 1);
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *tbl, *column, *tables = NULL;
sql_column *col;
oid rid = oid_nil;
str type, name, *colname, *tform;
fitsfile *fptr;
char filename[BUFSIZ];
size_t nrows = 0;
long optimal; /* type long used by fits library */
rids * rs;
int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0;
size_t numberrow = 0, dimension = 0;
int cc = 0, status = 0, j = 0, columns, *fid, block = 0;
int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0;
int hdutype;
char *charvalue, *readcharrows;
str strvalue; char **readstrrows;
short *shortvalue, *readshortrows;
int *intvalue, *readintrows;
lng *lngvalue, *readlngrows;
float *realvalue, *readfloatrows;
double *doublevalue, *readdoublerows;
_Bool *boolvalue, *readboolrows;
struct list * set;
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
/* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
tbl = mvc_bind_table(m, sch, tname);
if (tbl == NULL) {
msg = createException (MAL, "fits.exporttable", "Table %s is missing.\n", tname);
return msg;
}
set = (*tbl).columns.set;
columns = list_length(set);
colname = (str *) GDKmalloc(columns * sizeof(str));
tform = (str *) GDKmalloc(columns * sizeof(str));
/* fprintf(stderr,"Number of columns: %d\n", columns);*/
tables = mvc_bind_table(m, sch, "_tables");
col = mvc_bind_column(m, tables, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
col = mvc_bind_column(m, tables, "id");
fid = (int*) table_funcs.column_find_value(m->session->tr, col, rid);
column = mvc_bind_table(m, sch, "_columns");
col = mvc_bind_column(m, column, "table_id");
rs = table_funcs.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL);
GDKfree(fid);
while ((rid = table_funcs.rids_next(rs)) != oid_nil)
{
col = mvc_bind_column(m, column, "name");
name = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
colname[j] = toLower(name);
GDKfree(name);
col = mvc_bind_column(m, column, "type");
type = (char *) table_funcs.column_find_value(m->session->tr, col, rid);
if (strcmp(type,"boolean")==0) tform[j] = "1L";
if (strcmp(type,"char")==0) tform[j] = "1S";
if (strcmp(type,"varchar")==0) tform[j] = "8A";
if (strcmp(type,"smallint")==0) tform[j] = "1I";
if (strcmp(type,"int")==0) tform[j] = "1J";
if (strcmp(type,"bigint")==0) tform[j] = "1K";
if (strcmp(type,"real")==0) tform[j] = "1E";
if (strcmp(type,"double")==0) tform[j] = "1D";
GDKfree(type);
j++;
}
col = mvc_bind_column(m, tbl, colname[0]);
nrows = store_funcs.count_col(tr, col, 1);
assert(nrows <= (size_t) GDK_oid_max);
snprintf(filename,BUFSIZ,"\n%s.fit",tname);
fprintf(stderr, "Filename: %s\n", filename);
remove(filename);
status=0;
fits_create_file(&fptr, filename, &status);
fits_create_img(fptr, USHORT_IMG, 0, NULL, &status);
fits_close_file(fptr, &status);
fits_open_file(&fptr, filename, READWRITE, &status);
fits_movabs_hdu(fptr, 1, &hdutype, &status);
fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
for (cc = 0; cc < columns; cc++)
{
char * columntype;
col = mvc_bind_column(m, tbl, colname[cc]);
columntype = col -> type.type->sqlname;
if (strcmp(columntype,"boolean")==0)
{
boolcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
boolvalue = (_Bool*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readboolrows[dimension] = *boolvalue;
GDKfree(boolvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
texportboolean += GDKms() - tm0;
GDKfree(readboolrows);
}
if (strcmp(columntype,"char")==0)
{
charcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
charvalue = (char*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readcharrows[dimension] = *charvalue;
GDKfree(charvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
texportchar += GDKms() - tm0;
GDKfree(readcharrows);
}
if (strcmp(columntype,"varchar")==0)
{
strcols++; dimension=0; block=0;
fits_get_rowsize(fptr,&optimal,&status);
readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
strvalue = (char *) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readstrrows[dimension] = strvalue;
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
texportstring += GDKms() - tm0;
for (dimension = 0; dimension < (size_t) optimal; dimension++)
GDKfree(readstrrows[dimension]);
dimension = 0;
GDKfree(readstrrows);
readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
texportstring += GDKms() - tm0;
for (numberrow = 0; numberrow < dimension; numberrow++)
GDKfree(readstrrows[numberrow]);
GDKfree(readstrrows);
}
if (strcmp(columntype,"smallint")==0)
{
shortcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
shortvalue = (short*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readshortrows[dimension] = *shortvalue;
GDKfree(shortvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
texportshort += GDKms() - tm0;
GDKfree(readshortrows);
}
if (strcmp(columntype,"int")==0)
{
intcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
intvalue = (int*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readintrows[dimension] = *intvalue;
GDKfree(intvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
texportint += GDKms() - tm0;
GDKfree(readintrows);
}
if (strcmp(columntype,"bigint")==0)
{
lngcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
lngvalue = (lng*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readlngrows[dimension] = *lngvalue;
GDKfree(lngvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status);
texportlng += GDKms() - tm0;
GDKfree(readlngrows);
readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status);
texportlng += GDKms() - tm0;
GDKfree(readlngrows);
}
if (strcmp(columntype,"real")==0)
{
floatcols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
realvalue = (float*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readfloatrows[dimension] = *realvalue;
GDKfree(realvalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
texportfloat += GDKms() - tm0;
GDKfree(readfloatrows);
}
if (strcmp(columntype,"double")==0)
{
doublecols++; dimension = 0; block = 0;
fits_get_rowsize(fptr,&optimal,&status);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
for (numberrow = 0; numberrow < nrows ; numberrow++)
{
doublevalue = (double*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow);
readdoublerows[dimension] = *doublevalue;
GDKfree(doublevalue);
dimension++;
if (dimension == (size_t) optimal)
{
dimension = 0;
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
block++;
}
}
tm0 = GDKms();
fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
texportdouble += GDKms() - tm0;
GDKfree(readdoublerows);
}
}
/* print all the times that were needed to export each one of the columns
fprintf(stderr, "\n\n");
if (texportboolean > 0) fprintf(stderr, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean);
if (texportchar > 0) fprintf(stderr, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar);
if (texportstring > 0) fprintf(stderr, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring);
if (texportshort > 0) fprintf(stderr, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort);
if (texportint > 0) fprintf(stderr, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint);
if (texportlng > 0) fprintf(stderr, "%d Long\t\tcolumn(s) exported in %d ms\n", lngcols, texportlng);
if (texportfloat > 0) fprintf(stderr, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat);
if (texportdouble > 0) fprintf(stderr, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble);
*/
fits_close_file(fptr, &status);
return msg;
}
str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_trans *tr;
sql_schema *sch;
sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
sql_column *col;
str msg = MAL_SUCCEED;
str fname = *getArgReference_str(stk, pci, 1);
fitsfile *fptr; /* pointer to the FITS file */
int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
oid fid, tid, cid, rid = oid_nil;
char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ];
long tbcol; /* type long used by fits library */
char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
double tscal, tzero;
char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.attach", "Missing FITS file %s.\n", fname);
return msg;
}
tr = m->session->tr;
sch = mvc_bind_schema(m, "sys");
fits_fl = mvc_bind_table(m, sch, "fits_files");
if (fits_fl == NULL)
FITSinitCatalog(m);
fits_fl = mvc_bind_table(m, sch, "fits_files");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
fits_col = mvc_bind_table(m, sch, "fits_columns");
fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
/* check if the file is already attached */
col = mvc_bind_column(m, fits_fl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL);
if (rid != oid_nil) {
fits_close_file(fptr, &status);
msg = createException(SQL, "fits.attach", "File %s already attached\n", fname);
return msg;
}
/* add row in the fits_files catalog table */
col = mvc_bind_column(m, fits_fl, "id");
fid = store_funcs.count_col(tr, col, 1) + 1;
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_fl, "name"), fname, TYPE_str);
col = mvc_bind_column(m, fits_tbl, "id");
tid = store_funcs.count_col(tr, col, 1) + 1;
if ((s = strrchr(fname, DIR_SEP)) == NULL)
s = fname;
else
s++;
strcpy(bname, s);
s = strrchr(bname, '.');
if (s) *s = 0;
fits_get_num_hdus(fptr, &hdunum, &status);
for (i = 1; i <= hdunum; i++) {
fits_movabs_hdu(fptr, i, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
continue;
/* SQL table name - the name of FITS extention */
fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
if (status) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
status = 0;
}else { /* check table name for existence in the fits catalog */
tname_low = toLower(tname);
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname_low, NULL);
/* or as regular SQL table */
tbl = mvc_bind_table(m, sch, tname_low);
if (rid != oid_nil || tbl) {
snprintf(tname, BUFSIZ, "%s_%d", bname, i);
tname_low = toLower(tname);
}
}
fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
if (status) {
status = 0;
}
fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
if (status) {
status = 0;
}
fits_get_num_cols(fptr, &cnum, &status);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "name"), tname_low, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "columns"), &cnum, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "file_id"), &fid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "date"), tdate, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "origin"), orig, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "comment"), comm, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "table_id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "xtension"), xtensionname, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "bitpix"), &bitpixnumber, TYPE_int);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilvers"), stilversion, TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tp, "stilclas"), stilclass, TYPE_str);
/* read columns description */
s = stmt;
col = mvc_bind_column(m, fits_col, "id");
cid = store_funcs.count_col(tr, col, 1) + 1;
for (j = 1; j <= cnum; j++, cid++) {
fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname, tform, tunit, j, (int)tid);
msg = SQLstatementIntern(cntxt, &s, "fits.attach", TRUE, FALSE, NULL);
if (msg != MAL_SUCCEED) {
fits_close_file(fptr, &status);
return msg;
}
}
tid++;
}
fits_close_file(fptr, &status);
return MAL_SUCCEED;
}
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
mvc *m = NULL;
sql_schema *sch;
sql_table *fits_fl, *fits_tbl, *tbl = NULL;
sql_column *col;
sql_subtype tpe;
fitsfile *fptr;
str tname = *getArgReference_str(stk, pci, 1);
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype;
int *tpcode = NULL;
long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */
char keywrd[80], **cname, nm[FLEN_VALUE];
ptr nilptr;
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
return msg;
if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
return msg;
sch = mvc_bind_schema(m, "sys");
fits_tbl = mvc_bind_table(m, sch, "fits_tables");
if (fits_tbl == NULL) {
msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n");
return msg;
}
tbl = mvc_bind_table(m, sch, tname);
if (tbl) {
msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname);
return msg;
}
col = mvc_bind_column(m, fits_tbl, "name");
rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL);
if (rid == oid_nil) {
msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
return msg;
}
/* Open FITS file and move to the table HDU */
col = mvc_bind_column(m, fits_tbl, "file_id");
fid = (int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_fl = mvc_bind_table(m, sch, "fits_files");
col = mvc_bind_column(m, fits_fl, "id");
frid = table_funcs.column_find_row(m->session->tr, col, (void *)fid, NULL);
GDKfree(fid);
col = mvc_bind_column(m, fits_fl, "name");
fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid);
if (fits_open_file(&fptr, fname, READONLY, &status)) {
msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname);
GDKfree(fname);
return msg;
}
GDKfree(fname);
col = mvc_bind_column(m, fits_tbl, "hdu");
hdu = (int*)table_funcs.column_find_value(m->session->tr, col, rid);
fits_movabs_hdu(fptr, *hdu, &hdutype, &status);
if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", *hdu);
GDKfree(hdu);
fits_close_file(fptr, &status);
return msg;
}
GDKfree(hdu);
/* create a SQL table to hold the FITS table */
/* col = mvc_bind_column(m, fits_tbl, "columns");
cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */
fits_get_num_cols(fptr, &cnum, &status);
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
rep = (long *)GDKzalloc(sizeof(long) * cnum);
wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for (j = 1; j <= cnum; j++) {
/* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
snprintf(keywrd, 80, "TTYPE%d", j);
fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
if (status) {
snprintf(nm, FLEN_VALUE, "column_%d", j);
status = 0;
}
cname[j - 1] = toLower(nm);
fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
/* fprintf(stderr,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */
mvc_create_column(m, tbl, cname[j - 1], &tpe);
}
/* data load */
fits_get_num_rows(fptr, &rows, &status);
fprintf(stderr,"#Loading %ld rows in table %s\n", rows, tname);
for (j = 1; j <= cnum; j++) {
BAT *tmp = NULL;
int time0 = GDKms();
mtype = fits2mtype(tpcode[j - 1]);
nilptr = ATOMnilptr(mtype);
col = mvc_bind_column(m, tbl, cname[j - 1]);
tmp = BATnew(TYPE_void, mtype, rows, TRANSIENT);
if ( tmp == NULL){
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
throw(MAL,"fits.load", MAL_MALLOC_FAIL);
}
BATseqbase(tmp, 0);
if (mtype != TYPE_str) {
fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status);
BATsetcount(tmp, rows);
tmp->tsorted = 0;
tmp->trevsorted = 0;
} else {
/* char *v = GDKzalloc(wid[j-1]);*/
/* type long demanded by "rows", i.e., by fits library */
long bsize = 50, batch = bsize, k, i;
int tm0, tloadtm = 0, tattachtm = 0;
char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
for(i = 0; i < bsize; i++)
v[i] = GDKzalloc(wid[j-1]);
for(i = 0; i < rows; i += batch) {
batch = rows - i < bsize ? rows - i: bsize;
tm0 = GDKms();
fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status);
tloadtm += GDKms() - tm0;
tm0 = GDKms();
for(k = 0; k < batch ; k++)
BUNappend(tmp, v[k], TRUE);
tattachtm += GDKms() - tm0;
}
for(i = 0; i < bsize ; i++)
GDKfree(v[i]);
GDKfree(v);
fprintf(stderr,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
}
if (status) {
char buf[FLEN_ERRMSG + 1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
break;
}
fprintf(stderr,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat);
fprintf(stderr,"#Total %d ms\n", GDKms() - time0);
BBPunfix(tmp->batCacheid);
}
GDKfree(tpcode);
GDKfree(rep);
GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
return msg;
}
str RAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) {
sql_func * sqlfun = NULL;
str exprStr = *getArgReference_str(stk, pci, pci->retc + 1);
SEXP x, env, retval;
SEXP varname = R_NilValue;
SEXP varvalue = R_NilValue;
ParseStatus status;
int i = 0;
char argbuf[64];
char *argnames = NULL;
size_t argnameslen;
size_t pos;
char* rcall = NULL;
size_t rcalllen;
int ret_cols = 0; /* int because pci->retc is int, too*/
str *args;
int evalErr;
char *msg = MAL_SUCCEED;
BAT *b;
node * argnode;
int seengrp = FALSE;
rapiClient = cntxt;
if (!RAPIEnabled()) {
throw(MAL, "rapi.eval",
"Embedded R has not been enabled. Start server with --set %s=true",
rapi_enableflag);
}
if (!rapiInitialized) {
throw(MAL, "rapi.eval",
"Embedded R initialization has failed");
}
if (!grouped) {
sql_subfunc *sqlmorefun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc));
if (sqlmorefun) sqlfun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc))->func;
} else {
sqlfun = *(sql_func**) getArgReference(stk, pci, pci->retc);
}
args = (str*) GDKzalloc(sizeof(str) * pci->argc);
if (args == NULL) {
throw(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
}
// get the lock even before initialization of the R interpreter, as this can take a second and must be done only once.
MT_lock_set(&rapiLock);
env = PROTECT(eval(lang1(install("new.env")), R_GlobalEnv));
assert(env != NULL);
// first argument after the return contains the pointer to the sql_func structure
// NEW macro temporarily renamed to MNEW to allow including sql_catalog.h
if (sqlfun != NULL && sqlfun->ops->cnt > 0) {
int carg = pci->retc + 2;
argnode = sqlfun->ops->h;
while (argnode) {
char* argname = ((sql_arg*) argnode->data)->name;
args[carg] = GDKstrdup(argname);
carg++;
argnode = argnode->next;
}
}
// the first unknown argument is the group, we don't really care for the rest.
argnameslen = 2;
for (i = pci->retc + 2; i < pci->argc; i++) {
if (args[i] == NULL) {
if (!seengrp && grouped) {
args[i] = GDKstrdup("aggr_group");
seengrp = TRUE;
} else {
snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc - 1);
args[i] = GDKstrdup(argbuf);
}
}
argnameslen += strlen(args[i]) + 2; /* extra for ", " */
}
// install the MAL variables into the R environment
// we can basically map values to int ("INTEGER") or double ("REAL")
for (i = pci->retc + 2; i < pci->argc; i++) {
int bat_type = getBatType(getArgType(mb,pci,i));
// check for BAT or scalar first, keep code left
if (!isaBatType(getArgType(mb,pci,i))) {
b = COLnew(0, getArgType(mb, pci, i), 0, TRANSIENT);
if (b == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
if ( getArgType(mb,pci,i) == TYPE_str) {
if (BUNappend(b, *getArgReference_str(stk, pci, i), false) != GDK_SUCCEED) {
BBPreclaim(b);
b = NULL;
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
} else {
if (BUNappend(b, getArgReference(stk, pci, i), false) != GDK_SUCCEED) {
BBPreclaim(b);
b = NULL;
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
} else {
b = BATdescriptor(*getArgReference_bat(stk, pci, i));
if (b == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
// check the BAT count, if it is bigger than RAPI_MAX_TUPLES, fail
if (BATcount(b) > RAPI_MAX_TUPLES) {
msg = createException(MAL, "rapi.eval",
"Got "BUNFMT" rows, but can only handle "LLFMT". Sorry.",
BATcount(b), (lng) RAPI_MAX_TUPLES);
BBPunfix(b->batCacheid);
goto wrapup;
}
varname = PROTECT(Rf_install(args[i]));
varvalue = bat_to_sexp(b, bat_type);
if (varvalue == NULL) {
msg = createException(MAL, "rapi.eval", "unknown argument type ");
goto wrapup;
}
BBPunfix(b->batCacheid);
// install vector into R environment
Rf_defineVar(varname, varvalue, env);
UNPROTECT(2);
}
/* we are going to evaluate the user function within an anonymous function call:
* ret <- (function(arg1){return(arg1*2)})(42)
* the user code is put inside the {}, this keeps our environment clean (TM) and gives
* a clear path for return values, namely using the builtin return() function
* this is also compatible with PL/R
*/
pos = 0;
argnames = malloc(argnameslen);
if (argnames == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
argnames[0] = '\0';
for (i = pci->retc + 2; i < pci->argc; i++) {
pos += snprintf(argnames + pos, argnameslen - pos, "%s%s",
args[i], i < pci->argc - 1 ? ", " : "");
}
rcalllen = 2 * pos + strlen(exprStr) + 100;
rcall = malloc(rcalllen);
if (rcall == NULL) {
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
snprintf(rcall, rcalllen,
"ret <- as.data.frame((function(%s){%s})(%s), nm=NA, stringsAsFactors=F)\n",
argnames, exprStr, argnames);
free(argnames);
argnames = NULL;
#ifdef _RAPI_DEBUG_
printf("# R call %s\n",rcall);
#endif
x = R_ParseVector(mkString(rcall), 1, &status, R_NilValue);
if (LENGTH(x) != 1 || status != PARSE_OK) {
msg = createException(MAL, "rapi.eval",
"Error parsing R expression '%s'. ", exprStr);
goto wrapup;
}
retval = R_tryEval(VECTOR_ELT(x, 0), env, &evalErr);
if (evalErr != FALSE) {
char* errormsg = strdup(R_curErrorBuf());
size_t c;
if (errormsg == NULL) {
msg = createException(MAL, "rapi.eval", "Error running R expression.");
goto wrapup;
}
// remove newlines from error message so it fits into a MAPI error (lol)
for (c = 0; c < strlen(errormsg); c++) {
if (errormsg[c] == '\r' || errormsg[c] == '\n') {
errormsg[c] = ' ';
}
}
msg = createException(MAL, "rapi.eval",
"Error running R expression: %s", errormsg);
free(errormsg);
goto wrapup;
}
// ret should be a data frame with exactly as many columns as we need from retc
ret_cols = LENGTH(retval);
if (ret_cols != pci->retc) {
msg = createException(MAL, "rapi.eval",
"Expected result of %d columns, got %d", pci->retc, ret_cols);
goto wrapup;
}
// collect the return values
for (i = 0; i < pci->retc; i++) {
SEXP ret_col = VECTOR_ELT(retval, i);
int bat_type = getBatType(getArgType(mb,pci,i));
if (bat_type == TYPE_any || bat_type == TYPE_void) {
getArgType(mb,pci,i) = bat_type;
msg = createException(MAL, "rapi.eval",
"Unknown return value, possibly projecting with no parameters.");
goto wrapup;
}
// hand over the vector into a BAT
b = sexp_to_bat(ret_col, bat_type);
if (b == NULL) {
msg = createException(MAL, "rapi.eval",
"Failed to convert column %i", i);
goto wrapup;
}
// bat return
if (isaBatType(getArgType(mb,pci,i))) {
*getArgReference_bat(stk, pci, i) = b->batCacheid;
} else { // single value return, only for non-grouped aggregations
BATiter li = bat_iterator(b);
if (VALinit(&stk->stk[pci->argv[i]], bat_type,
BUNtail(li, 0)) == NULL) { // TODO BUNtail here
msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL);
goto wrapup;
}
}
msg = MAL_SUCCEED;
}
/* unprotect environment, so it will be eaten by the GC. */
UNPROTECT(1);
wrapup:
MT_lock_unset(&rapiLock);
if (argnames)
free(argnames);
if (rcall)
free(rcall);
for (i = 0; i < pci->argc; i++)
GDKfree(args[i]);
GDKfree(args);
return msg;
}
