str QOTshowPlan(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str modnme; str fcnnme; Symbol s = NULL; if (stk != 0) { modnme = *getArgReference_str(stk, p, 1); fcnnme = *getArgReference_str(stk, p, 2); } else { modnme = getArgDefault(mb, p, 1); fcnnme = getArgDefault(mb, p, 2); } mnstr_printf(cntxt->fdout,"#showPlan()\n"); removeInstruction(mb, p); if( modnme ) { s = findSymbol(cntxt->nspace, putName(modnme, strlen(modnme)), putName(fcnnme, strlen(fcnnme))); if (s == NULL) { char buf[1024]; snprintf(buf,1024, "%s.%s", modnme, fcnnme); throw(MAL, "optimizer.showPlan", RUNTIME_OBJECT_UNDEFINED ":%s", buf); } mb= s->def; } printFunction(cntxt->fdout, mb, 0, LIST_INPUT); return MAL_SUCCEED; }
str QOTshowFlowGraph(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str fname; str modnme; str fcnnme; Symbol s = NULL; (void) cntxt; if (stk != 0) { modnme = *getArgReference_str(stk, p, 1); fcnnme = *getArgReference_str(stk, p, 2); fname = *getArgReference_str(stk, p, 3); } else { modnme = getArgDefault(mb, p, 1); fcnnme = getArgDefault(mb, p, 2); fname = getArgDefault(mb, p, 3); } s = findSymbol(cntxt->nspace,putName(modnme, strlen(modnme)), putName(fcnnme, strlen(fcnnme))); if (s == NULL) { char buf[1024]; snprintf(buf,1024, "%s.%s", modnme, fcnnme); throw(MAL, "optimizer.showFlowGraph", RUNTIME_OBJECT_UNDEFINED ":%s", buf); } showFlowGraph(s->def, stk, fname); return MAL_SUCCEED; }
/* * Display routines */ str MDBlifespan(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { Lifespan span; str modnme; str fcnnme; Symbol s = NULL; (void) cntxt; if (stk != 0) { modnme = *getArgReference_str(stk, p, 1); fcnnme = *getArgReference_str(stk, p, 2); } else { modnme = getArgDefault(mb, p, 1); fcnnme = getArgDefault(mb, p, 2); } s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) throw(MAL, "mdb.inspect", RUNTIME_SIGNATURE_MISSING); span = setLifespan(s->def); if( span == NULL) throw(MAL,"mdb.inspect", MAL_MALLOC_FAIL); debugLifespan(cntxt, s->def, span); GDKfree(span); (void) p; (void) stk; return MAL_SUCCEED; }
str MDBtrapFunction(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str modnme = *getArgReference_str(stk, pci, 1); str fcnnme = *getArgReference_str(stk, pci, 2); bit b= *getArgReference_bit(stk,pci,3); (void) cntxt; (void) mb; if ( mdbSetTrap(cntxt,modnme,fcnnme,b) ) throw(MAL,"mdb.trap", RUNTIME_SIGNATURE_MISSING); return MAL_SUCCEED; }
str OPTsql_append(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){ str modnme; str fcnnme; str msg= MAL_SUCCEED; Symbol s= NULL; lng t,clk= GDKusec(); int actions = 0; if( p ) removeInstruction(mb, p); OPTDEBUGsql_append mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER sql_append\n"); if( p && p->argc > 1 ){ if( getArgType(mb,p,1) != TYPE_str || getArgType(mb,p,2) != TYPE_str || !isVarConstant(mb,getArg(p,1)) || !isVarConstant(mb,getArg(p,2)) ) { throw(MAL, "optimizer.sql_append", ILLARG_CONSTANTS); } if( stk != 0){ modnme= *getArgReference_str(stk,p,1); fcnnme= *getArgReference_str(stk,p,2); } else { modnme= getArgDefault(mb,p,1); fcnnme= getArgDefault(mb,p,2); } s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme))); if( s == NULL) { char buf[1024]; snprintf(buf,1024, "%s.%s",modnme,fcnnme); throw(MAL, "optimizer.sql_append", RUNTIME_OBJECT_UNDEFINED ":%s", buf); } mb = s->def; stk= 0; } if( mb->errors ){ /* when we have errors, we still want to see them */ addtoMalBlkHistory(mb,"sql_append"); return MAL_SUCCEED; } actions= OPTsql_appendImplementation(cntxt, mb,stk,p); msg= optimizerCheck(cntxt, mb, "optimizer.sql_append", actions, t=(GDKusec() - clk)); OPTDEBUGsql_append { mnstr_printf(cntxt->fdout,"=FINISHED sql_append %d\n",actions); printFunction(cntxt->fdout,mb,0,LIST_MAL_ALL ); } DEBUGoptimizers mnstr_printf(cntxt->fdout,"#opt_reduce: " LLFMT " ms\n",t); QOTupdateStatistics("sql_append",actions,t); addtoMalBlkHistory(mb,"sql_append"); return msg; }
str MDBvar3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str modnme = *getArgReference_str(stk, p, 1); str fcnnme = *getArgReference_str(stk, p, 2); Symbol s = NULL; s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) throw(MAL,"mdb.var","Could not find %s.%s", modnme, fcnnme); printStack(cntxt->fdout, s->def, (s->def == mb ? stk : 0)); (void) mb; return NULL; }
str MDBlist3Detail(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str modnme = *getArgReference_str(stk, p, 1); str fcnnme = *getArgReference_str(stk, p, 2); Symbol s = NULL; s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) throw(MAL,"mdb.list","Could not find %s.%s", modnme, fcnnme); debugFunction(cntxt->fdout, s->def, 0, LIST_MAL_NAME | LIST_MAL_VALUE | LIST_MAL_TYPE | LIST_MAL_PROPS , 0, s->def->stop); (void) mb; /* fool compiler */ return NULL; }
str MDBlist3(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str modnme = *getArgReference_str(stk, p, 1); str fcnnme = *getArgReference_str(stk, p, 2); Symbol s = NULL; s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) throw(MAL,"mdb.list","Could not find %s.%s", modnme, fcnnme); printFunction(cntxt->fdout, s->def, 0, LIST_MAL_NAME ); (void) mb; /* fool compiler */ return MAL_SUCCEED; }
str FCTshutdown(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str mod = *getArgReference_str(stk, pci, 1); str fcn = *getArgReference_str(stk, pci, 2); Symbol s; (void) mb; s = findSymbol(cntxt->nspace, putName(mod), putName(fcn)); if (s == NULL) throw(MAL, "factories.shutdown", RUNTIME_OBJECT_MISSING); shutdownFactory(cntxt,s->def); return MAL_SUCCEED; }
str SQLcompile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str *ret = getArgReference_str(stk, pci, 0); str *expr = getArgReference_str(stk, pci, 1); str msg; (void) mb; *ret = NULL; msg = SQLstatementIntern(cntxt, expr, "SQLcompile", FALSE, FALSE, NULL); if (msg == MAL_SUCCEED) *ret = _STRDUP("SQLcompile"); return msg; }
str CMDsetProfilerFile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str *fnme = getArgReference_str(stk,pci,1); (void) mb; /* fool compiler */ return setLogFile(cntxt->fdout,cntxt->nspace, *fnme); }
str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str msg = MAL_SUCCEED; str dir = *getArgReference_str(stk, pci, 1); DIR *dp; struct dirent *ep; fitsfile *fptr; char *s; int status = 0; (void)mb; dp = opendir(dir); if (dp != NULL) { char stmt[BUFSIZ]; char fname[BUFSIZ]; s = stmt; while ((ep = readdir(dp)) != NULL && !msg) { snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name); status = 0; fits_open_file(&fptr, fname, READONLY, &status); if (status == 0) { snprintf(stmt, BUFSIZ, ATTACHDIR, fname); msg = SQLstatementIntern(cntxt, &s, "fits.listofdir", TRUE, FALSE, NULL); fits_close_file(fptr, &status); } } (void)closedir(dp); } else msg = createException(MAL, "listdir", "Couldn't open the directory"); return msg; }
str CMDsetProfilerStream (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str *host = getArgReference_str(stk,pci,1); int *port = getArgReference_int(stk,pci,2); (void) mb; /* fool compiler */ return setLogStream(cntxt->nspace, *host, *port); }
str MDBsetVarTrace(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str v; (void) cntxt; v = *getArgReference_str(stk, p, 1); mdbSetBreakRequest(cntxt, mb, v, 't'); stk->cmd = 'c'; cntxt->itrace = 'c'; return MAL_SUCCEED; }
str SQLstatement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str *expr = getArgReference_str(stk, pci, 1); bit output = TRUE; (void) mb; if (pci->argc == 3) output = *getArgReference_bit(stk, pci, 2); return SQLstatementIntern(cntxt, expr, "SQLstatement", TRUE, output, NULL); }
str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str msg = MAL_SUCCEED; str dir = *getArgReference_str(stk, pci, 1); str pat = *getArgReference_str(stk, pci, 2); fitsfile *fptr; char *s; int status = 0; glob_t globbuf; char fulldirectory[BUFSIZ]; size_t j = 0; (void)mb; globbuf.gl_offs = 0; snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat); glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf); /* fprintf(stderr,"#fulldir: %s \nSize: %lu\n",fulldirectory, globbuf.gl_pathc);*/ if (globbuf.gl_pathc == 0) throw(MAL, "listdir", "Couldn't open the directory or there are no files that match the pattern"); for (j = 0; j < globbuf.gl_pathc; j++) { char stmt[BUFSIZ]; char fname[BUFSIZ]; s = stmt; snprintf(fname, BUFSIZ, "%s", globbuf.gl_pathv[j]); status = 0; fits_open_file(&fptr, fname, READONLY, &status); if (status == 0) { snprintf(stmt, BUFSIZ, ATTACHDIR, fname); msg = SQLstatementIntern(cntxt, &s, "fits.listofdirpat", TRUE, FALSE, NULL); fits_close_file(fptr, &status); break; } } return msg; }
str MDBinspect(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str modnme; str fcnnme; Symbol s = NULL; (void) cntxt; if (stk != 0) { modnme = *getArgReference_str(stk, p, 1); fcnnme = *getArgReference_str(stk, p, 2); } else { modnme = getArgDefault(mb, p, 1); fcnnme = getArgDefault(mb, p, 2); } s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) throw(MAL, "mdb.inspect", RUNTIME_SIGNATURE_MISSING); return runMALDebugger(cntxt, s); }
str MDBshowFlowGraph(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { str fname; str modnme; str fcnnme; Symbol s = NULL; (void)cntxt; if (stk != 0) { if (p->argc == 2) { modnme = fcnnme = NULL; fname = *getArgReference_str(stk, p, 1); } else { modnme = *getArgReference_str(stk, p, 1); fcnnme = *getArgReference_str(stk, p, 2); fname = *getArgReference_str(stk, p, 3); } } else { modnme = getArgDefault(mb, p, 1); fcnnme = getArgDefault(mb, p, 2); fname = getArgDefault(mb, p, 3); } if (modnme != NULL) { s = findSymbol(cntxt->nspace, putName(modnme), putName(fcnnme)); if (s == NULL) { char buf[1024]; snprintf(buf,1024, "Could not find %s.%s\n", modnme, fcnnme); throw(MAL, "mdb.dot", "%s", buf); } showFlowGraph(s->def, stk, fname); } else { showFlowGraph(mb, stk, fname); } return MAL_SUCCEED; }
/** * Calls the bam loader for a single file */ str bam_loader_file(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { /* arg 1: path to bam stream */ str bam_file = *getArgReference_str(stk, pci, pci->retc); /* arg 2: dbschema to use */ sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1); (void) stk; (void) pci; return bam_loader(cntxt, mb, &bam_file, 1, dbschema, 1); }
str QOToptimize(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str modnme; str fcnnme; Symbol s; (void) stk; if (stk != 0) { modnme = *getArgReference_str(stk, pci, 1); fcnnme = *getArgReference_str(stk, pci, 2); } else { modnme = getArgDefault(mb, pci, 1); fcnnme = getArgDefault(mb, pci, 2); } s = findSymbol(cntxt->usermodule, putName(modnme), fcnnme); if (s == NULL) throw(MAL, "optimizer.optimize", SQLSTATE(HY002) SEMANTIC_OPERATION_MISSING); removeInstruction(mb, pci); addtoMalBlkHistory(s->def); return optimizeMALBlock(cntxt, s->def); }
str CMDbbpbind(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str name; ValPtr lhs; bat i; int ht,tt; BAT *b; (void) cntxt; (void) mb; /* fool compiler */ lhs = &stk->stk[pci->argv[0]]; name = *getArgReference_str(stk, pci, 1); if (isIdentifier(name) < 0) throw(MAL, "bbp.bind", IDENTIFIER_EXPECTED); i = BBPindex(name); if (i == 0) throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING); /* make sure you load the descriptors and heaps */ b = (BAT *) BATdescriptor(i); if (b == 0) /* Simple ignore the binding if you can't find the bat */ throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING); /* check conformity of the actual type and the one requested */ ht= getHeadType(getArgType(mb,pci,0)); tt= getColumnType(getArgType(mb,pci,0)); if( b->htype == TYPE_void && ht== TYPE_oid) ht= TYPE_void; if( b->ttype == TYPE_void && tt== TYPE_oid) tt= TYPE_void; if( ht != b->htype || tt != b->ttype){ BBPunfix(i); throw(MAL, "bbp.bind", SEMANTIC_TYPE_MISMATCH ); } /* make sure we are not dealing with an about to be deleted bat */ if( BBP_refs(b->batCacheid) == 1 && BBP_lrefs(b->batCacheid) == 0){ BBPunfix(i); throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING); } BBPkeepref(b->batCacheid); lhs->vtype = TYPE_bat; lhs->val.bval = i; return MAL_SUCCEED; }
/* * Locate a file with SQL commands and execute it. For the time being a 1MB * file limit is implicitly imposed. If the file can not be located in the * script library, we assume it is sufficiently self descriptive. * (Respecting the file system context where the call is executed ) */ str SQLinclude(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { stream *fd; bstream *bfd; str *name = getArgReference_str(stk, pci, 1); str msg = MAL_SUCCEED, fullname; str *expr; mvc *m; size_t sz; fullname = MSP_locate_sqlscript(*name, 0); if (fullname == NULL) fullname = *name; fd = open_rastream(fullname); if (mnstr_errnr(fd) == MNSTR_OPEN_ERROR) { mnstr_destroy(fd); throw(MAL, "sql.include", "could not open file: %s\n", *name); } sz = getFileSize(fd); if (sz > (size_t) 1 << 29) { mnstr_destroy(fd); throw(MAL, "sql.include", "file %s too large to process", fullname); } bfd = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz); if (bstream_next(bfd) < 0) { bstream_destroy(bfd); throw(MAL, "sql.include", "could not read %s\n", *name); } expr = &bfd->buf; msg = SQLstatementIntern(cntxt, expr, "sql.include", TRUE, FALSE, NULL); bstream_destroy(bfd); m = ((backend *) cntxt->sqlcontext)->mvc; if (m->sa) sa_destroy(m->sa); m->sa = NULL; (void) mb; return msg; }
/* * THe choice operator first searches the next one to identify * the fragment to be optimized and to gain access to the variables * without the need to declare them upfront. */ str RUNchoice(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { int target; lng cost, mincost; int i, j, pc; char *nme; InstrPtr q; pc = getPC(mb, p); for (i = pc + 1; i < mb->stop; i++) { q = getInstrPtr(mb, i); if (getModuleId(p) == getModuleId(q) && getFunctionId(p) == getFunctionId(q)) { p = q; break; } } if (i == mb->stop) return MAL_SUCCEED; target = getArg(p, 2); if (getArgType(mb, p, 1) == TYPE_int && p->argc >= 3 && (p->argc - 1) % 2 == 0) { /* choice pairs */ mincost = *getArgReference_int(stk, p, 1); for (i = 3; i < p->argc; i += 2) { cost = *getArgReference_int(stk, p, i); if (cost < mincost && !isVarDisabled(mb, getArg(p, i + 1))) { mincost = cost; target = getArg(p, i + 1); } } } else if (getArgType(mb, p, 1) == TYPE_str) { nme = *getArgReference_str(stk, p, 1); /* should be generalized to allow an arbitrary user defined function */ if (strcmp(nme, "getVolume") != 0) throw(MAL, "scheduler.choice", ILLEGAL_ARGUMENT "Illegal cost function"); mincost = -1; for (j = 2; j < p->argc; j++) { if (!isVarDisabled(mb, getArg(p, j))) for (i = pc + 1; i < mb->stop; i++) { InstrPtr q = getInstrPtr(mb, i); if (p->token >= 0 && getArg(q, 0) == getArg(p, j)) { cost = getVolume(stk, q, 1); if (cost > 0 && (cost < mincost || mincost == -1)) { mincost = cost; target = getArg(p, j); } break; } } } } #ifdef DEBUG_RUN_MEMORUN mnstr_printf(cntxt->fdout, "#function target %s cost %d\n", getVarName(mb, target), mincost); #else (void) cntxt; #endif /* remove non-qualifying variables */ for (i = 2; i < p->argc; i += 2) if (getArg(p, i) != target) { setVarDisabled(mb, getArg(p, i - 1)); setVarDisabled(mb, getArg(p, i)); } propagateNonTarget(mb, pc + 1); #ifdef DEBUG_RUN_MEMORUN mnstr_printf(cntxt->fdout, "#cost choice selected %s %d\n", getVarName(mb, target), mincost); printFunction(cntxt->fdout, mb, 1); #endif return MAL_SUCCEED; }
/** * Gathers all BAM files that are listed in the given file and calls * bam_loader for these files */ str bam_loader_files(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { /* arg 1: path to file containing bam file names separated by \n */ str bam_files = *getArgReference_str(stk, pci, pci->retc); /* arg 2: dbschema to use */ sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1); /* For now, we hard code the number of threads, since the SQL level should not bother with this */ sht nr_threads = GDKnr_threads; str *filenames = NULL; int nr_files = 0; FILE *f = NULL; int cur; int line_size; str line = NULL; size_t line_buf_size = 0; int nr_lines = 1; str msg = MAL_SUCCEED; int i; /* Open the file */ f = fopen(bam_files, "r"); if (f == NULL) { msg = createException(MAL, "bam_loader_files", "Error on opening file list '%s': %s", bam_files, strerror(errno)); goto cleanup; } /* Start with counting the number of lines in the file (is nr * of newlines + 1) */ while ((cur = fgetc(f)) != EOF) { if (cur == '\n') ++nr_lines; } if (ferror(f)) { msg = createException(MAL, "bam_loader_files", "Error while processing file '%s'", bam_files); goto cleanup; } /* Now we can malloc the filenames array */ if ((filenames = (str *) GDKmalloc(nr_lines * sizeof(str))) == NULL) { msg = createException(MAL, "bam_loader_files", MAL_MALLOC_FAIL); goto cleanup; } /* Enables cleanup to check individual files */ memset(filenames, 0, nr_lines * sizeof(str)); /* Reset file pointer to beginning of file */ rewind(f); /* Read BAM filenames into files array */ while ((line_size = getline(&line, &line_buf_size, f)) >= 0) { if (line_size == 0 || line[0] == '\n' || line[0] == '\r' || line[0] == '#') { /* Skip empty lines or lines that start with a hash */ free(line); line = NULL; /* We need to reset buf size to 0, since getline will have updated it */ line_buf_size = 0; continue; } if (line[line_size - 1] == '\n') { /* Newline character is not part of the filepath */ line[line_size - 1] = '\0'; } if(line_size > 1 && line[line_size - 2] == '\r') { /* Return character is also not a part of the filepath */ line[line_size - 2] = '\0'; } filenames[nr_files++] = line; line = NULL; /* We need to reset it to 0, since getline will have updated this val */ line_buf_size = 0; } if (ferror(f)) { msg = createException(MAL, "bam_loader_files", "Error while processing file '%s'", bam_files); goto cleanup; } if (nr_files == 0) { msg = createException(MAL, "bam_loader_files", "No valid entries found in file '%s'", bam_files); goto cleanup; } /* And call the bam loader */ msg = bam_loader(cntxt, mb, filenames, nr_files, dbschema, nr_threads); cleanup: if(line) { //if line still contains anything, this is something that was put there //by getline on the last, failed attempt to read a line free(line); } if (filenames) { for (i = 0; i < nr_files; ++i) { if (filenames[i]) { free(filenames[i]); } } GDKfree(filenames); } if (f) fclose(f); (void) stk; (void) pci; return msg; }
/** * Gathers all BAM files in the given repository and calls bam_loader for these files */ str bam_loader_repos(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { /* arg 1: path to bam file repository */ str bam_repos = *getArgReference_str(stk, pci, pci->retc); /* arg 2: dbschema to use */ sht dbschema = *getArgReference_sht(stk, pci, pci->retc + 1); /* For now, we hard code the number of threads, since the SQL level should not bother with this */ sht nr_threads = GDKnr_threads; str *filenames = NULL; int nr_files = 0; DIR *d = NULL; struct dirent *direntry = NULL; char path[4096]; int i, filecount = 0; str msg = MAL_SUCCEED; if ((d = opendir(bam_repos)) == NULL) { if (errno == ENOENT) { msg = createException(MAL, "bam_loader_repos", "Could not find directory '%s'", bam_repos); } else { msg = createException(MAL, "bam_loader_repos", "Could not open directory '%s'", bam_repos); } goto cleanup; } /* First, count number of SAM/BAM files */ while ((direntry = readdir(d)) != NULL) { int len = strlen(direntry->d_name); if (IS_SAMORBAM(direntry->d_name, len)) ++filecount; } if (filecount == 0) { msg = createException(MAL, "bam_loader_repos", "No SAM or BAM files found in directory '%s'", bam_repos); goto cleanup; } /* Now malloc enough memory for filenames array */ if ((filenames = (str *) GDKmalloc(filecount * sizeof(str))) == NULL) { msg = createException(MAL, "bam_loader_repos", MAL_MALLOC_FAIL); goto cleanup; } /* Enables cleanup to check individual filenames */ memset(filenames, 0, filecount * sizeof(str)); rewinddir(d); /* Now loop through dir; we add a slash in between the given * bam_repos and the filename. This could lead to double * slashes in a path, but as far as I know, this is no problem * on all OS's */ while ((direntry = readdir(d)) != NULL) { /* Check if d_name has the .sam or .bam extension (case * insensitive) */ int len = strlen(direntry->d_name); if (IS_SAMORBAM(direntry->d_name, len)) { /* This is a SAM or BAM file, construct its path and * add that to the files array */ if (snprintf (path, 4096, "%s/%s", bam_repos, direntry->d_name) < 0) { msg = createException(MAL, "bam_loader_repos", "Could not construct filepath for SAM/BAM file '%s'", direntry->d_name); goto cleanup; } if (nr_files >= filecount) { /* This should not happen. If it does, * it might be the case that a BAM * file was added to the directory in * the meantime */ msg = createException(MAL, "bam_loader_repos", "An error occurred while reading directory '%s'", bam_repos); goto cleanup; } filenames[nr_files++] = GDKstrdup(path); } } /* And call the bam loader */ msg = bam_loader(cntxt, mb, filenames, nr_files, dbschema, nr_threads); cleanup: if (filenames) { for (i = 0; i < nr_files; ++i) { if (filenames[i]) GDKfree(filenames[i]); } GDKfree(filenames); } if (d) closedir(d); (void) stk; (void) pci; return msg; }
str OPTwrapper (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){ str modnme = "(NONE)"; str fcnnme = 0; str msg= MAL_SUCCEED; Symbol s= NULL; lng t,clk= GDKusec(); int i, actions = 0; char optimizer[256]; InstrPtr q; if( p == NULL) throw(MAL, "opt_wrapper", "missing optimizer statement"); snprintf(optimizer,256,"%s", fcnnme = getFunctionId(p)); q= copyInstruction(p); OPTIMIZERDEBUG mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER %s\n",fcnnme); if( p && p->argc > 1 ){ if( getArgType(mb,p,1) != TYPE_str || getArgType(mb,p,2) != TYPE_str || !isVarConstant(mb,getArg(p,1)) || !isVarConstant(mb,getArg(p,2)) ) { freeInstruction(q); throw(MAL, optimizer, ILLARG_CONSTANTS); } if( stk != 0){ modnme= *getArgReference_str(stk,p,1); fcnnme= *getArgReference_str(stk,p,2); } else { modnme= getArgDefault(mb,p,1); fcnnme= getArgDefault(mb,p,2); } removeInstruction(mb, p); s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme))); if( s == NULL) { freeInstruction(q); throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme); } mb = s->def; stk= 0; } else if( p ) removeInstruction(mb, p); if( mb->errors ){ /* when we have errors, we still want to see them */ addtoMalBlkHistory(mb,getModuleId(q)); freeInstruction(q); return MAL_SUCCEED; } for ( i=0; codes[i].nme; i++) if ( strcmp(codes[i].nme, optimizer)== 0 ){ actions = (int)(*(codes[i].fcn))(cntxt, mb, stk,0); break; } if ( codes[i].nme == 0){ freeInstruction(q); throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme); } msg= optimizerCheck(cntxt, mb, optimizer, actions, t=(GDKusec() - clk)); OPTIMIZERDEBUG { mnstr_printf(cntxt->fdout,"=FINISHED %s %d\n",optimizer, actions); printFunction(cntxt->fdout,mb,0,LIST_MAL_DEBUG ); } DEBUGoptimizers mnstr_printf(cntxt->fdout,"#optimizer %-11s %3d actions %5d MAL instructions ("SZFMT" K) " LLFMT" usec\n", optimizer, actions, mb->stop, ((sizeof( MalBlkRecord) +mb->ssize * offsetof(InstrRecord, argv)+ mb->vtop * sizeof(int) /* argv estimate */ +mb->vtop* sizeof(VarRecord) + mb->vsize*sizeof(VarPtr)+1023)/1024), t); QOTupdateStatistics(getModuleId(q),actions,t); addtoMalBlkHistory(mb,getModuleId(q)); freeInstruction(q); return msg; }
str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { str msg = MAL_SUCCEED; str tname = *getArgReference_str(stk, pci, 1); mvc *m = NULL; sql_trans *tr; sql_schema *sch; sql_table *tbl, *column, *tables = NULL; sql_column *col; oid rid = oid_nil; str type, name, *colname, *tform; fitsfile *fptr; char filename[BUFSIZ]; size_t nrows = 0; long optimal; /* type long used by fits library */ rids * rs; int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0; size_t numberrow = 0, dimension = 0; int cc = 0, status = 0, j = 0, columns, *fid, block = 0; int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0; int hdutype; char *charvalue, *readcharrows; str strvalue; char **readstrrows; short *shortvalue, *readshortrows; int *intvalue, *readintrows; lng *lngvalue, *readlngrows; float *realvalue, *readfloatrows; double *doublevalue, *readdoublerows; _Bool *boolvalue, *readboolrows; struct list * set; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; tr = m->session->tr; sch = mvc_bind_schema(m, "sys"); /* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */ tbl = mvc_bind_table(m, sch, tname); if (tbl == NULL) { msg = createException (MAL, "fits.exporttable", "Table %s is missing.\n", tname); return msg; } set = (*tbl).columns.set; columns = list_length(set); colname = (str *) GDKmalloc(columns * sizeof(str)); tform = (str *) GDKmalloc(columns * sizeof(str)); /* fprintf(stderr,"Number of columns: %d\n", columns);*/ tables = mvc_bind_table(m, sch, "_tables"); col = mvc_bind_column(m, tables, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL); col = mvc_bind_column(m, tables, "id"); fid = (int*) table_funcs.column_find_value(m->session->tr, col, rid); column = mvc_bind_table(m, sch, "_columns"); col = mvc_bind_column(m, column, "table_id"); rs = table_funcs.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL); GDKfree(fid); while ((rid = table_funcs.rids_next(rs)) != oid_nil) { col = mvc_bind_column(m, column, "name"); name = (char *) table_funcs.column_find_value(m->session->tr, col, rid); colname[j] = toLower(name); GDKfree(name); col = mvc_bind_column(m, column, "type"); type = (char *) table_funcs.column_find_value(m->session->tr, col, rid); if (strcmp(type,"boolean")==0) tform[j] = "1L"; if (strcmp(type,"char")==0) tform[j] = "1S"; if (strcmp(type,"varchar")==0) tform[j] = "8A"; if (strcmp(type,"smallint")==0) tform[j] = "1I"; if (strcmp(type,"int")==0) tform[j] = "1J"; if (strcmp(type,"bigint")==0) tform[j] = "1K"; if (strcmp(type,"real")==0) tform[j] = "1E"; if (strcmp(type,"double")==0) tform[j] = "1D"; GDKfree(type); j++; } col = mvc_bind_column(m, tbl, colname[0]); nrows = store_funcs.count_col(tr, col, 1); assert(nrows <= (size_t) GDK_oid_max); snprintf(filename,BUFSIZ,"\n%s.fit",tname); fprintf(stderr, "Filename: %s\n", filename); remove(filename); status=0; fits_create_file(&fptr, filename, &status); fits_create_img(fptr, USHORT_IMG, 0, NULL, &status); fits_close_file(fptr, &status); fits_open_file(&fptr, filename, READWRITE, &status); fits_movabs_hdu(fptr, 1, &hdutype, &status); fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status); for (cc = 0; cc < columns; cc++) { char * columntype; col = mvc_bind_column(m, tbl, colname[cc]); columntype = col -> type.type->sqlname; if (strcmp(columntype,"boolean")==0) { boolcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { boolvalue = (_Bool*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readboolrows[dimension] = *boolvalue; GDKfree(boolvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status); texportboolean += GDKms() - tm0; GDKfree(readboolrows); readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status); texportboolean += GDKms() - tm0; GDKfree(readboolrows); } if (strcmp(columntype,"char")==0) { charcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readcharrows = (char *) GDKmalloc (sizeof(char) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { charvalue = (char*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readcharrows[dimension] = *charvalue; GDKfree(charvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status); texportchar += GDKms() - tm0; GDKfree(readcharrows); readcharrows = (char *) GDKmalloc (sizeof(char) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status); texportchar += GDKms() - tm0; GDKfree(readcharrows); } if (strcmp(columntype,"varchar")==0) { strcols++; dimension=0; block=0; fits_get_rowsize(fptr,&optimal,&status); readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { strvalue = (char *) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readstrrows[dimension] = strvalue; dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status); texportstring += GDKms() - tm0; for (dimension = 0; dimension < (size_t) optimal; dimension++) GDKfree(readstrrows[dimension]); dimension = 0; GDKfree(readstrrows); readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal); block++; } } tm0 = GDKms(); fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status); texportstring += GDKms() - tm0; for (numberrow = 0; numberrow < dimension; numberrow++) GDKfree(readstrrows[numberrow]); GDKfree(readstrrows); } if (strcmp(columntype,"smallint")==0) { shortcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readshortrows = (short *) GDKmalloc (sizeof(short) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { shortvalue = (short*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readshortrows[dimension] = *shortvalue; GDKfree(shortvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status); texportshort += GDKms() - tm0; GDKfree(readshortrows); readshortrows = (short *) GDKmalloc (sizeof(short) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status); texportshort += GDKms() - tm0; GDKfree(readshortrows); } if (strcmp(columntype,"int")==0) { intcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readintrows = (int *) GDKmalloc (sizeof(int) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { intvalue = (int*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readintrows[dimension] = *intvalue; GDKfree(intvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status); texportint += GDKms() - tm0; GDKfree(readintrows); readintrows = (int *) GDKmalloc (sizeof(int) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status); texportint += GDKms() - tm0; GDKfree(readintrows); } if (strcmp(columntype,"bigint")==0) { lngcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { lngvalue = (lng*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readlngrows[dimension] = *lngvalue; GDKfree(lngvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status); texportlng += GDKms() - tm0; GDKfree(readlngrows); readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status); texportlng += GDKms() - tm0; GDKfree(readlngrows); } if (strcmp(columntype,"real")==0) { floatcols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { realvalue = (float*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readfloatrows[dimension] = *realvalue; GDKfree(realvalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status); texportfloat += GDKms() - tm0; GDKfree(readfloatrows); readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status); texportfloat += GDKms() - tm0; GDKfree(readfloatrows); } if (strcmp(columntype,"double")==0) { doublecols++; dimension = 0; block = 0; fits_get_rowsize(fptr,&optimal,&status); readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal); for (numberrow = 0; numberrow < nrows ; numberrow++) { doublevalue = (double*) table_funcs.column_find_value(m->session->tr, col, (oid) numberrow); readdoublerows[dimension] = *doublevalue; GDKfree(doublevalue); dimension++; if (dimension == (size_t) optimal) { dimension = 0; tm0 = GDKms(); fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status); texportdouble += GDKms() - tm0; GDKfree(readdoublerows); readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal); block++; } } tm0 = GDKms(); fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status); texportdouble += GDKms() - tm0; GDKfree(readdoublerows); } } /* print all the times that were needed to export each one of the columns fprintf(stderr, "\n\n"); if (texportboolean > 0) fprintf(stderr, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean); if (texportchar > 0) fprintf(stderr, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar); if (texportstring > 0) fprintf(stderr, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring); if (texportshort > 0) fprintf(stderr, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort); if (texportint > 0) fprintf(stderr, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint); if (texportlng > 0) fprintf(stderr, "%d Long\t\tcolumn(s) exported in %d ms\n", lngcols, texportlng); if (texportfloat > 0) fprintf(stderr, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat); if (texportdouble > 0) fprintf(stderr, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble); */ fits_close_file(fptr, &status); return msg; }
str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { mvc *m = NULL; sql_trans *tr; sql_schema *sch; sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL; sql_column *col; str msg = MAL_SUCCEED; str fname = *getArgReference_str(stk, pci, 1); fitsfile *fptr; /* pointer to the FITS file */ int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0; oid fid, tid, cid, rid = oid_nil; char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ]; long tbcol; /* type long used by fits library */ char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ]; double tscal, tzero; char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = ""; char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = ""; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; if (fits_open_file(&fptr, fname, READONLY, &status)) { msg = createException(MAL, "fits.attach", "Missing FITS file %s.\n", fname); return msg; } tr = m->session->tr; sch = mvc_bind_schema(m, "sys"); fits_fl = mvc_bind_table(m, sch, "fits_files"); if (fits_fl == NULL) FITSinitCatalog(m); fits_fl = mvc_bind_table(m, sch, "fits_files"); fits_tbl = mvc_bind_table(m, sch, "fits_tables"); fits_col = mvc_bind_table(m, sch, "fits_columns"); fits_tp = mvc_bind_table(m, sch, "fits_table_properties"); /* check if the file is already attached */ col = mvc_bind_column(m, fits_fl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, fname, NULL); if (rid != oid_nil) { fits_close_file(fptr, &status); msg = createException(SQL, "fits.attach", "File %s already attached\n", fname); return msg; } /* add row in the fits_files catalog table */ col = mvc_bind_column(m, fits_fl, "id"); fid = store_funcs.count_col(tr, col, 1) + 1; store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_fl, "id"), &fid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_fl, "name"), fname, TYPE_str); col = mvc_bind_column(m, fits_tbl, "id"); tid = store_funcs.count_col(tr, col, 1) + 1; if ((s = strrchr(fname, DIR_SEP)) == NULL) s = fname; else s++; strcpy(bname, s); s = strrchr(bname, '.'); if (s) *s = 0; fits_get_num_hdus(fptr, &hdunum, &status); for (i = 1; i <= hdunum; i++) { fits_movabs_hdu(fptr, i, &hdutype, &status); if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) continue; /* SQL table name - the name of FITS extention */ fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status); if (status) { snprintf(tname, BUFSIZ, "%s_%d", bname, i); tname_low = toLower(tname); status = 0; }else { /* check table name for existence in the fits catalog */ tname_low = toLower(tname); col = mvc_bind_column(m, fits_tbl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname_low, NULL); /* or as regular SQL table */ tbl = mvc_bind_table(m, sch, tname_low); if (rid != oid_nil || tbl) { snprintf(tname, BUFSIZ, "%s_%d", bname, i); tname_low = toLower(tname); } } fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status); if (status) { status = 0; } fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status); if (status) { status = 0; } fits_get_num_cols(fptr, &cnum, &status); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "name"), tname_low, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "columns"), &cnum, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "file_id"), &fid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "date"), tdate, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "origin"), orig, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tbl, "comment"), comm, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "table_id"), &tid, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "xtension"), xtensionname, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "bitpix"), &bitpixnumber, TYPE_int); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "stilvers"), stilversion, TYPE_str); store_funcs.append_col(m->session->tr, mvc_bind_column(m, fits_tp, "stilclas"), stilclass, TYPE_str); /* read columns description */ s = stmt; col = mvc_bind_column(m, fits_col, "id"); cid = store_funcs.count_col(tr, col, 1) + 1; for (j = 1; j <= cnum; j++, cid++) { fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname, tform, tunit, j, (int)tid); msg = SQLstatementIntern(cntxt, &s, "fits.attach", TRUE, FALSE, NULL); if (msg != MAL_SUCCEED) { fits_close_file(fptr, &status); return msg; } } tid++; } fits_close_file(fptr, &status); return MAL_SUCCEED; }
str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { mvc *m = NULL; sql_schema *sch; sql_table *fits_fl, *fits_tbl, *tbl = NULL; sql_column *col; sql_subtype tpe; fitsfile *fptr; str tname = *getArgReference_str(stk, pci, 1); str fname; str msg = MAL_SUCCEED; oid rid = oid_nil, frid = oid_nil; int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype; int *tpcode = NULL; long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */ char keywrd[80], **cname, nm[FLEN_VALUE]; ptr nilptr; if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) return msg; if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED) return msg; sch = mvc_bind_schema(m, "sys"); fits_tbl = mvc_bind_table(m, sch, "fits_tables"); if (fits_tbl == NULL) { msg = createException(MAL, "fits.loadtable", "FITS catalog is missing.\n"); return msg; } tbl = mvc_bind_table(m, sch, tname); if (tbl) { msg = createException(MAL, "fits.loadtable", "Table %s is already created.\n", tname); return msg; } col = mvc_bind_column(m, fits_tbl, "name"); rid = table_funcs.column_find_row(m->session->tr, col, tname, NULL); if (rid == oid_nil) { msg = createException(MAL, "fits.loadtable", "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname); return msg; } /* Open FITS file and move to the table HDU */ col = mvc_bind_column(m, fits_tbl, "file_id"); fid = (int*)table_funcs.column_find_value(m->session->tr, col, rid); fits_fl = mvc_bind_table(m, sch, "fits_files"); col = mvc_bind_column(m, fits_fl, "id"); frid = table_funcs.column_find_row(m->session->tr, col, (void *)fid, NULL); GDKfree(fid); col = mvc_bind_column(m, fits_fl, "name"); fname = (char *)table_funcs.column_find_value(m->session->tr, col, frid); if (fits_open_file(&fptr, fname, READONLY, &status)) { msg = createException(MAL, "fits.loadtable", "Missing FITS file %s.\n", fname); GDKfree(fname); return msg; } GDKfree(fname); col = mvc_bind_column(m, fits_tbl, "hdu"); hdu = (int*)table_funcs.column_find_value(m->session->tr, col, rid); fits_movabs_hdu(fptr, *hdu, &hdutype, &status); if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) { msg = createException(MAL, "fits.loadtable", "HDU %d is not a table.\n", *hdu); GDKfree(hdu); fits_close_file(fptr, &status); return msg; } GDKfree(hdu); /* create a SQL table to hold the FITS table */ /* col = mvc_bind_column(m, fits_tbl, "columns"); cnum = *(int*) table_funcs.column_find_value(m->session->tr, col, rid); */ fits_get_num_cols(fptr, &cnum, &status); tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum); tpcode = (int *)GDKzalloc(sizeof(int) * cnum); rep = (long *)GDKzalloc(sizeof(long) * cnum); wid = (long *)GDKzalloc(sizeof(long) * cnum); cname = (char **)GDKzalloc(sizeof(char *) * cnum); for (j = 1; j <= cnum; j++) { /* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */ snprintf(keywrd, 80, "TTYPE%d", j); fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status); if (status) { snprintf(nm, FLEN_VALUE, "column_%d", j); status = 0; } cname[j - 1] = toLower(nm); fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status); fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]); /* fprintf(stderr,"#%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->sqlname); */ mvc_create_column(m, tbl, cname[j - 1], &tpe); } /* data load */ fits_get_num_rows(fptr, &rows, &status); fprintf(stderr,"#Loading %ld rows in table %s\n", rows, tname); for (j = 1; j <= cnum; j++) { BAT *tmp = NULL; int time0 = GDKms(); mtype = fits2mtype(tpcode[j - 1]); nilptr = ATOMnilptr(mtype); col = mvc_bind_column(m, tbl, cname[j - 1]); tmp = BATnew(TYPE_void, mtype, rows, TRANSIENT); if ( tmp == NULL){ GDKfree(tpcode); GDKfree(rep); GDKfree(wid); GDKfree(cname); throw(MAL,"fits.load", MAL_MALLOC_FAIL); } BATseqbase(tmp, 0); if (mtype != TYPE_str) { fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, nilptr, (void *)BUNtloc(bat_iterator(tmp), BUNfirst(tmp)), &anynull, &status); BATsetcount(tmp, rows); tmp->tsorted = 0; tmp->trevsorted = 0; } else { /* char *v = GDKzalloc(wid[j-1]);*/ /* type long demanded by "rows", i.e., by fits library */ long bsize = 50, batch = bsize, k, i; int tm0, tloadtm = 0, tattachtm = 0; char **v = (char **) GDKzalloc(sizeof(char *) * bsize); for(i = 0; i < bsize; i++) v[i] = GDKzalloc(wid[j-1]); for(i = 0; i < rows; i += batch) { batch = rows - i < bsize ? rows - i: bsize; tm0 = GDKms(); fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, nilptr, (void *)v, &anynull, &status); tloadtm += GDKms() - tm0; tm0 = GDKms(); for(k = 0; k < batch ; k++) BUNappend(tmp, v[k], TRUE); tattachtm += GDKms() - tm0; } for(i = 0; i < bsize ; i++) GDKfree(v[i]); GDKfree(v); fprintf(stderr,"#String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm); } if (status) { char buf[FLEN_ERRMSG + 1]; fits_read_errmsg(buf); msg = createException(MAL, "fits.loadtable", "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf); break; } fprintf(stderr,"#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0); store_funcs.append_col(m->session->tr, col, tmp, TYPE_bat); fprintf(stderr,"#Total %d ms\n", GDKms() - time0); BBPunfix(tmp->batCacheid); } GDKfree(tpcode); GDKfree(rep); GDKfree(wid); GDKfree(cname); fits_close_file(fptr, &status); return msg; }
str RAPIeval(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, bit grouped) { sql_func * sqlfun = NULL; str exprStr = *getArgReference_str(stk, pci, pci->retc + 1); SEXP x, env, retval; SEXP varname = R_NilValue; SEXP varvalue = R_NilValue; ParseStatus status; int i = 0; char argbuf[64]; char *argnames = NULL; size_t argnameslen; size_t pos; char* rcall = NULL; size_t rcalllen; int ret_cols = 0; /* int because pci->retc is int, too*/ str *args; int evalErr; char *msg = MAL_SUCCEED; BAT *b; node * argnode; int seengrp = FALSE; rapiClient = cntxt; if (!RAPIEnabled()) { throw(MAL, "rapi.eval", "Embedded R has not been enabled. Start server with --set %s=true", rapi_enableflag); } if (!rapiInitialized) { throw(MAL, "rapi.eval", "Embedded R initialization has failed"); } if (!grouped) { sql_subfunc *sqlmorefun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc)); if (sqlmorefun) sqlfun = (*(sql_subfunc**) getArgReference(stk, pci, pci->retc))->func; } else { sqlfun = *(sql_func**) getArgReference(stk, pci, pci->retc); } args = (str*) GDKzalloc(sizeof(str) * pci->argc); if (args == NULL) { throw(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); } // get the lock even before initialization of the R interpreter, as this can take a second and must be done only once. MT_lock_set(&rapiLock); env = PROTECT(eval(lang1(install("new.env")), R_GlobalEnv)); assert(env != NULL); // first argument after the return contains the pointer to the sql_func structure // NEW macro temporarily renamed to MNEW to allow including sql_catalog.h if (sqlfun != NULL && sqlfun->ops->cnt > 0) { int carg = pci->retc + 2; argnode = sqlfun->ops->h; while (argnode) { char* argname = ((sql_arg*) argnode->data)->name; args[carg] = GDKstrdup(argname); carg++; argnode = argnode->next; } } // the first unknown argument is the group, we don't really care for the rest. argnameslen = 2; for (i = pci->retc + 2; i < pci->argc; i++) { if (args[i] == NULL) { if (!seengrp && grouped) { args[i] = GDKstrdup("aggr_group"); seengrp = TRUE; } else { snprintf(argbuf, sizeof(argbuf), "arg%i", i - pci->retc - 1); args[i] = GDKstrdup(argbuf); } } argnameslen += strlen(args[i]) + 2; /* extra for ", " */ } // install the MAL variables into the R environment // we can basically map values to int ("INTEGER") or double ("REAL") for (i = pci->retc + 2; i < pci->argc; i++) { int bat_type = getBatType(getArgType(mb,pci,i)); // check for BAT or scalar first, keep code left if (!isaBatType(getArgType(mb,pci,i))) { b = COLnew(0, getArgType(mb, pci, i), 0, TRANSIENT); if (b == NULL) { msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } if ( getArgType(mb,pci,i) == TYPE_str) { if (BUNappend(b, *getArgReference_str(stk, pci, i), false) != GDK_SUCCEED) { BBPreclaim(b); b = NULL; msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } } else { if (BUNappend(b, getArgReference(stk, pci, i), false) != GDK_SUCCEED) { BBPreclaim(b); b = NULL; msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } } } else { b = BATdescriptor(*getArgReference_bat(stk, pci, i)); if (b == NULL) { msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } } // check the BAT count, if it is bigger than RAPI_MAX_TUPLES, fail if (BATcount(b) > RAPI_MAX_TUPLES) { msg = createException(MAL, "rapi.eval", "Got "BUNFMT" rows, but can only handle "LLFMT". Sorry.", BATcount(b), (lng) RAPI_MAX_TUPLES); BBPunfix(b->batCacheid); goto wrapup; } varname = PROTECT(Rf_install(args[i])); varvalue = bat_to_sexp(b, bat_type); if (varvalue == NULL) { msg = createException(MAL, "rapi.eval", "unknown argument type "); goto wrapup; } BBPunfix(b->batCacheid); // install vector into R environment Rf_defineVar(varname, varvalue, env); UNPROTECT(2); } /* we are going to evaluate the user function within an anonymous function call: * ret <- (function(arg1){return(arg1*2)})(42) * the user code is put inside the {}, this keeps our environment clean (TM) and gives * a clear path for return values, namely using the builtin return() function * this is also compatible with PL/R */ pos = 0; argnames = malloc(argnameslen); if (argnames == NULL) { msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } argnames[0] = '\0'; for (i = pci->retc + 2; i < pci->argc; i++) { pos += snprintf(argnames + pos, argnameslen - pos, "%s%s", args[i], i < pci->argc - 1 ? ", " : ""); } rcalllen = 2 * pos + strlen(exprStr) + 100; rcall = malloc(rcalllen); if (rcall == NULL) { msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } snprintf(rcall, rcalllen, "ret <- as.data.frame((function(%s){%s})(%s), nm=NA, stringsAsFactors=F)\n", argnames, exprStr, argnames); free(argnames); argnames = NULL; #ifdef _RAPI_DEBUG_ printf("# R call %s\n",rcall); #endif x = R_ParseVector(mkString(rcall), 1, &status, R_NilValue); if (LENGTH(x) != 1 || status != PARSE_OK) { msg = createException(MAL, "rapi.eval", "Error parsing R expression '%s'. ", exprStr); goto wrapup; } retval = R_tryEval(VECTOR_ELT(x, 0), env, &evalErr); if (evalErr != FALSE) { char* errormsg = strdup(R_curErrorBuf()); size_t c; if (errormsg == NULL) { msg = createException(MAL, "rapi.eval", "Error running R expression."); goto wrapup; } // remove newlines from error message so it fits into a MAPI error (lol) for (c = 0; c < strlen(errormsg); c++) { if (errormsg[c] == '\r' || errormsg[c] == '\n') { errormsg[c] = ' '; } } msg = createException(MAL, "rapi.eval", "Error running R expression: %s", errormsg); free(errormsg); goto wrapup; } // ret should be a data frame with exactly as many columns as we need from retc ret_cols = LENGTH(retval); if (ret_cols != pci->retc) { msg = createException(MAL, "rapi.eval", "Expected result of %d columns, got %d", pci->retc, ret_cols); goto wrapup; } // collect the return values for (i = 0; i < pci->retc; i++) { SEXP ret_col = VECTOR_ELT(retval, i); int bat_type = getBatType(getArgType(mb,pci,i)); if (bat_type == TYPE_any || bat_type == TYPE_void) { getArgType(mb,pci,i) = bat_type; msg = createException(MAL, "rapi.eval", "Unknown return value, possibly projecting with no parameters."); goto wrapup; } // hand over the vector into a BAT b = sexp_to_bat(ret_col, bat_type); if (b == NULL) { msg = createException(MAL, "rapi.eval", "Failed to convert column %i", i); goto wrapup; } // bat return if (isaBatType(getArgType(mb,pci,i))) { *getArgReference_bat(stk, pci, i) = b->batCacheid; } else { // single value return, only for non-grouped aggregations BATiter li = bat_iterator(b); if (VALinit(&stk->stk[pci->argv[i]], bat_type, BUNtail(li, 0)) == NULL) { // TODO BUNtail here msg = createException(MAL, "rapi.eval", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto wrapup; } } msg = MAL_SUCCEED; } /* unprotect environment, so it will be eaten by the GC. */ UNPROTECT(1); wrapup: MT_lock_unset(&rapiLock); if (argnames) free(argnames); if (rcall) free(rcall); for (i = 0; i < pci->argc; i++) GDKfree(args[i]); GDKfree(args); return msg; }