static QEP
QEPnew(int p, int c){
QEP qep;
qep = (QEP) GDKmalloc( sizeof(struct QEPrecord));
if (qep == NULL)
return NULL;
qep->mb= NULL;
qep->p = NULL;
qep->plimit = p;
if( p ) {
qep->parents = (QEP*) GDKzalloc( sizeof(QEP) * p);
if( qep->parents == NULL){
GDKfree(qep);
return NULL;
}
}
qep->climit = c;
if( c){
qep->children = (QEP *) GDKzalloc( sizeof(QEP) * c);
if( qep->children == NULL){
GDKfree(qep);
return NULL;
}
}
return qep;
}
static AGGRtask*
GROUPcollect( Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci){
AGGRtask *a;
int i;
BAT *b, *bs, *bh = NULL;
BUN sample;
(void) mb;
(void) cntxt;
a= (AGGRtask *) GDKzalloc(sizeof(*a));
if ( a == NULL)
return NULL;
a->bid = (bat*) GDKzalloc(pci->argc * sizeof(bat));
a->cols = (BAT**) GDKzalloc(pci->argc * sizeof(BAT*));
a->unique = (BUN *) GDKzalloc(pci->argc * sizeof(BUN));
if ( a->cols == NULL || a->bid == NULL || a->unique == NULL){
if(a->cols) GDKfree(a->cols);
if(a->bid) GDKfree(a->bid);
if(a->unique) GDKfree(a->unique);
GDKfree(a);
return NULL;
}
for ( i= pci->retc; i< pci->argc; i++, a->last++) {
a->bid[a->last] = *getArgReference_bat(stk,pci,i);
b = a->cols[a->last]= BATdescriptor(a->bid[a->last]);
if ( a->cols[a->last] == NULL){
for(a->last--; a->last>=0; a->last--)
BBPunfix(a->cols[a->last]->batCacheid);
GDKfree(a->cols);
GDKfree(a->bid);
GDKfree(a->unique);
GDKfree(a);
return NULL;
}
sample = BATcount(b) < 1000 ? BATcount(b): 1000;
bs = BATsample( b, sample);
if (bs) {
bh = BATunique(b, bs);
if (bh) {
a->unique[a->last] = BATcount(bh);
BBPunfix(bh->batCacheid);
}
BBPunfix(bs->batCacheid);
}
if ( b->tsorted)
a->unique[a->last] = 1000; /* sorting helps grouping */
a->size = BATcount(b);
}
#ifdef _DEBUG_GROUPBY_
for(i=0; i<a->last; i++)
fprintf(stderr,"#group %d unique "BUNFMT "\n", i, a->unique[i]);
#endif
return a;
}
static JSON *
JSONnewtree(int size)
{
JSON *js;
if (size == 0)
size = jsonhint;
js = (JSON *) GDKzalloc(sizeof(JSON));
js->elm = (JSONterm *) GDKzalloc(sizeof(JSONterm) * size);
js->size = size;
return js;
}
void getModuleList(Module** out, int* length) {
int i;
int moduleCount = 0;
int currentIndex = 0;
for(i = 0; i < MODULE_HASH_SIZE; i++) {
Module m = moduleIndex[i];
while(m) {
moduleCount++;
m = m->link;
}
}
*out = GDKzalloc(moduleCount * sizeof(Module));
if (*out == NULL) {
return;
}
*length = moduleCount;
for(i = 0; i < MODULE_HASH_SIZE; i++) {
Module m = moduleIndex[i];
while(m) {
(*out)[currentIndex++] = m;
m = m->link;
}
}
}
static void *
column_find_value(sql_trans *tr, sql_column *c, oid rid)
{
BUN q;
BAT *b, *d = NULL;
void *res = NULL;
sql_dbat *bat = c->t->data;
if (bat->dbid)
d = store_funcs.bind_del(tr, c->t, RDONLY);
b = full_column(c, d, NULL);
if (d)
bat_destroy(d);
q = BUNfnd(b, (ptr) &rid);
if (q != BUN_NONE) {
BATiter bi = bat_iterator(b);
void *r;
int sz;
res = BUNtail(bi, q);
sz = ATOMlen(b->ttype, res);
r = GDKzalloc(sz);
memcpy(r,res,sz);
res = r;
}
bat_destroy(b);
return res;
}
static str
JSONglue(str res, str r, char sep)
{
size_t len, l;
str n;
if (r == 0 || *r == 0)
return res;
len = strlen(r);
if (res == 0)
res = GDKstrdup(r);
else {
l = strlen(res);
n = GDKzalloc(l + len + 3);
strcpy(n, res);
if (sep) {
n[l] = ',';
strncpy(n + l + 1, r, len);
n[l + 1 + len] = 0;
} else {
strncpy(n + l, r, len);
n[l + len] = 0;
}
GDKfree(res);
res = n;
}
if (r)
GDKfree(r);
return res;
}
/*
* The administrator should be initialized to enable interpretation of
* the command line arguments, before it starts serviceing statements
*/
int
MCinitClientThread(Client c)
{
Thread t;
char cname[11 + 1];
snprintf(cname, 11, OIDFMT, c->user);
cname[11] = '\0';
t = THRnew(cname);
if (t == 0) {
showException(c->fdout, MAL, "initClientThread",
"Failed to initialize client");
MPresetProfiler(c->fdout);
return -1;
}
/*
* The GDK thread administration should be set to reflect use of
* the proper IO descriptors.
*/
t->data[1] = c->fdin;
t->data[0] = c->fdout;
c->mythread = t;
c->errbuf = GDKerrbuf;
if (c->errbuf == NULL) {
GDKsetbuf(GDKzalloc(GDKMAXERRLEN));
c->errbuf = GDKerrbuf;
} else
c->errbuf[0] = 0;
return 0;
}
str
JSONkeyArray(json *ret, json *js)
{
char *result = NULL;
str r;
int i;
JSON *jt;
jt = JSONparse(*js, FALSE); // already validated
CHECK_JSON(jt);
if (jt->elm[0].kind == JSON_OBJECT)
for (i = jt->elm[0].next; i; i = jt->elm[i].next) {
r = GDKzalloc(jt->elm[i].valuelen + 3);
if (jt->elm[i].valuelen)
strncpy(r, jt->elm[i].value - 1, jt->elm[i].valuelen + 2);
result = JSONglue(result, r, ',');
} else
throw(MAL, "json.keyarray", "Object expected");
r = (char *) GDKstrdup("[");
result = JSONglue(r, result, 0);
r = (char *) GDKstrdup("]");
*ret = JSONglue(result, r, 0);
return MAL_SUCCEED;
}
/*
* Determine the variables being used and clear non-used onces.
*/
void
MSresetVariables(Client cntxt, MalBlkPtr mb, MalStkPtr glb, int start)
{
int i;
bit *used = GDKzalloc(mb->vtop * sizeof(bit));
if( used == NULL){
GDKerror("MSresetVariables" MAL_MALLOC_FAIL);
return;
}
for (i = 0; i < start && start < mb->vtop; i++)
used[i] = 1;
if (mb->errors == 0)
for (i = start; i < mb->vtop; i++) {
if (used[i] || !isTmpVar(mb, i)) {
assert(!mb->var[i]->value.vtype || isVarConstant(mb, i));
used[i] = 1;
}
if (glb && !used[i]) {
if (isVarConstant(mb, i))
garbageElement(cntxt, &glb->stk[i]);
/* clean stack entry */
glb->stk[i].vtype = TYPE_int;
glb->stk[i].len = 0;
glb->stk[i].val.pval = 0;
}
}
if (mb->errors == 0)
trimMalVariables_(mb, used, glb);
GDKfree(used);
}
bool
MCinit(void)
{
const char *max_clients = GDKgetenv("max_clients");
int maxclients = 0;
if (max_clients != NULL)
maxclients = atoi(max_clients);
if (maxclients <= 0) {
maxclients = 64;
if (GDKsetenv("max_clients", "64") != GDK_SUCCEED) {
fprintf(stderr,"#MCinit: GDKsetenv failed");
return false;
}
}
MAL_MAXCLIENTS =
/* console */ 1 +
/* client connections */ maxclients;
mal_clients = GDKzalloc(sizeof(ClientRec) * MAL_MAXCLIENTS);
if( mal_clients == NULL){
fprintf(stderr,"#MCinit:" MAL_MALLOC_FAIL);
return false;
}
for (int i = 0; i < MAL_MAXCLIENTS; i++)
ATOMIC_INIT(&mal_clients[i].lastprint, 0);
return true;
}
/*
* The administrator should be initialized to enable interpretation of
* the command line arguments, before it starts servicing statements
*/
int
MCinitClientThread(Client c)
{
Thread t;
t = MT_thread_getdata(); /* should succeed */
if (t == NULL) {
MPresetProfiler(c->fdout);
return -1;
}
/*
* The GDK thread administration should be set to reflect use of
* the proper IO descriptors.
*/
t->data[1] = c->fdin;
t->data[0] = c->fdout;
c->mythread = t;
c->errbuf = GDKerrbuf;
if (c->errbuf == NULL) {
char *n = GDKzalloc(GDKMAXERRLEN);
if ( n == NULL){
MPresetProfiler(c->fdout);
return -1;
}
GDKsetbuf(n);
c->errbuf = GDKerrbuf;
} else
c->errbuf[0] = 0;
return 0;
}
str
MRgetCloud(int *ret, str *mrcluster)
{
str msg;
BAT *cloud;
BUN p, q;
BATiter bi;
char nodes[BUFSIZ];
char *n = nodes;
int mapcount = 0;
snprintf(nodes, sizeof(nodes), "*/%s/node/*", *mrcluster);
if ((msg = RMTresolve(ret, &n)) != MAL_SUCCEED)
return msg;
MT_lock_set(&mal_contextLock, "mapreduce");
cloud = BATdescriptor(*ret); /* should succeed */
mapnodes = (mapnode*)GDKzalloc(sizeof(mapnode) * (BATcount(cloud) + 1));
if (mapnodes == NULL) {
BBPreleaseref(*ret);
throw(MAL, "mapreduce.getCloud", MAL_MALLOC_FAIL);
}
bi = bat_iterator(cloud);
BATloop(cloud, p, q) {
str t = (str)BUNtail(bi, p);
mapnodes[mapcount].uri = GDKstrdup(t);
mapnodes[mapcount].user = GDKstrdup("monetdb");
mapnodes[mapcount].pass = GDKstrdup("monetdb");
mapcount++;
}
/* Every client record has a private module name 'user'
* for keeping around non-shared functions */
Module userModule(void){
Module cur;
cur = (Module) GDKzalloc(sizeof(ModuleRecord));
if (cur == NULL)
return NULL;
cur->name = putName("user");
cur->link = NULL;
cur->space = NULL;
cur->isAtomModule = FALSE;
cur->space = (Symbol *) GDKzalloc(MAXSCOPE * sizeof(Symbol));
if (cur->space == NULL) {
GDKfree(cur);
return NULL;
}
return cur;
}
int
OPTgroupsImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
{
int i, actions=0;
InstrPtr q;
InstrPtr *old, *ref;
int limit,slimit;
(void) cntxt;
(void) stk;
if (varGetProp(mb, getArg(mb->stmt[0], 0), inlineProp) != NULL) {
return 0;
}
/* beware, new variables and instructions are introduced */
ref= (InstrPtr*) GDKzalloc(sizeof(InstrPtr) * mb->vtop); /* to find last assignment */
if ( ref == NULL) {
return 0;
}
old= mb->stmt;
limit= mb->stop;
slimit= mb->ssize;
if ( newMalBlkStmt(mb,mb->ssize) <0) {
GDKfree(ref);
return 0;
}
for (i = 0; i<limit; i++){
p= old[i];
if (getModuleId(p) == groupRef && p->argc == 4 && getFunctionId(p) == subgroupRef ){
setFunctionId(p, multicolumnsRef);
ref[getArg(p,0)] = p;
actions++;
OPTDEBUGgroups {
mnstr_printf(cntxt->fdout,"#new groups instruction\n");
printInstruction(cntxt->fdout,mb, 0, p, LIST_MAL_ALL);
}
}
if (getModuleId(p) == groupRef && p->argc == 5 && getFunctionId(p) == subgroupdoneRef && ref[getArg(p,4)] != NULL){
/*
* Try to expand its argument list with what we have found so far.
* This creates a series of derive paths, many of which will be removed during deadcode elimination.
*/
q= copyInstruction(ref[getArg(p,4)]);
q= pushArgument(mb, q, getArg(p,3));
getArg(q,0) = getArg(p,0);
getArg(q,1) = getArg(p,1);
getArg(q,2) = getArg(p,2);
ref[getArg(q,0)] = q;
freeInstruction(p);
p= q;
OPTDEBUGgroups{
mnstr_printf(cntxt->fdout,"#new groups instruction extension\n");
printInstruction(cntxt->fdout,mb, 0, p, LIST_MAL_ALL);
}
}
static Sensor
SEnew(str nme)
{
Sensor se;
se = (Sensor) GDKzalloc(sizeof(SErecord));
se->name = GDKstrdup(nme);
se->nxt = seAnchor;
seAnchor = se;
return se;
}
static Actuator
ACnew(str nme)
{
Actuator ac;
ac = (Actuator) GDKzalloc(sizeof(ACrecord));
ac->name = GDKstrdup(nme);
ac->nxt = acAnchor;
acAnchor = ac;
return ac;
}
MalStkPtr
newGlobalStack(int size)
{
MalStkPtr s;
s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk));
if (!s)
return NULL;
s->stksize = size;
return s;
}
MalStkPtr
newGlobalStack(int size)
{
MalStkPtr s;
s = (MalStkPtr) GDKzalloc(stackSize(size) + offsetof(MalStack, stk));
if (s == NULL)
GDKfatal("newGlobalStack:can not obtain memory\n");
s->stksize = size;
return s;
}
Client
MCinitClientRecord(Client c, oid user, bstream *fin, stream *fout)
{
str prompt;
c->user = user;
c->scenario = NULL;
c->oldscenario = NULL;
c->srcFile = NULL;
c->blkmode = 0;
c->fdin = fin ? fin : bstream_create(GDKin, 0);
c->yycur = 0;
c->bak = NULL;
c->listing = 0;
c->fdout = fout ? fout : GDKstdout;
c->mdb = 0;
c->history = 0;
c->curprg = c->backup = 0;
c->glb = 0;
/* remove garbage from previous connection */
if (c->nspace) {
freeModule(c->nspace);
c->nspace = 0;
}
c->father = NULL;
c->login = c->lastcmd = time(0);
c->qtimeout = 0;
c->stimeout = 0;
c->stage = 0;
c->itrace = 0;
c->debugOptimizer = c->debugScheduler = 0;
c->flags = MCdefault;
c->timer = 0;
c->memory = 0;
c->errbuf = 0;
prompt = !fin ? GDKgetenv("monet_prompt") : PROMPT1;
c->prompt = GDKstrdup(prompt);
c->promptlength = strlen(prompt);
c->actions = 0;
c->totaltime = 0;
c->rcc = (RecPtr) GDKzalloc(sizeof(RecStat));
c->rcc->curQ = -1;
c->exception_buf_initialized = 0;
MT_sema_init(&c->s, 0, "MCinitClient");
return c;
}
static str
JSONgetValue(JSON *jt, int idx)
{
str s;
if (jt->elm[idx].valuelen == 0)
return GDKstrdup(str_nil);
if (strncmp(jt->elm[idx].value, "null", 4) == 0)
return GDKstrdup(str_nil);
s = GDKzalloc(jt->elm[idx].valuelen + 1);
if (s)
strncpy(s, jt->elm[idx].value, jt->elm[idx].valuelen);
return s;
}
/*
* Module scope management
* It will contain the symbol table of all globally accessible functions.
*/
Module globalModule(str nme)
{ Module cur;
// Global modules are not named 'user'
assert (strcmp(nme, "user"));
#ifdef _DEBUG_MODULE_
fprintf(stderr,"#create new global module %s\n",nme);
#endif
nme = putName(nme);
cur = (Module) GDKzalloc(sizeof(ModuleRecord));
if (cur == NULL)
return NULL;
cur->name = nme;
cur->link = NULL;
cur->isAtomModule = FALSE;
cur->space = (Symbol *) GDKzalloc(MAXSCOPE * sizeof(Symbol));
if (cur->space == NULL) {
GDKfree(cur);
return NULL;
}
addModuleToIndex(cur);
return cur;
}
/*
* After filling in a structure it is added to the multi-level symbol
* table. We keep a skip list of similarly named function symbols.
* This speeds up searching provided the modules adhere to the
* structure and group the functions as well.
*/
void insertSymbol(Module scope, Symbol prg){
InstrPtr sig;
int t;
Module c;
assert(scope);
sig = getSignature(prg);
#ifdef _DEBUG_MODULE_
fprintf(stderr,"#insertSymbol: %s.%s in %s ", getModuleId(sig), getFunctionId(sig), scope->name);
#endif
if(getModuleId(sig) && getModuleId(sig)!= scope->name){
/* move the definition to the proper place */
/* default scope is the last resort */
c= findModule(scope,getModuleId(sig));
if ( c )
scope = c;
#ifdef _DEBUG_MODULE_
fprintf(stderr," found alternative module %s ", scope->name);
#endif
}
t = getSymbolIndex(getFunctionId(sig));
if( scope->space == NULL) {
scope->space = (Symbol *) GDKzalloc(MAXSCOPE * sizeof(Symbol));
if (scope->space == NULL)
return;
}
assert(scope->space);
if (scope->space[t] == prg){
/* already known, last inserted */
#ifdef _DEBUG_MODULE_
fprintf(stderr," unexpected double insert ");
#endif
} else {
prg->peer= scope->space[t];
scope->space[t] = prg;
if( prg->peer &&
idcmp(prg->name,prg->peer->name) == 0)
prg->skip = prg->peer->skip;
else
prg->skip = prg->peer;
}
assert(prg != prg->peer);
#ifdef _DEBUG_MODULE_
fprintf(stderr,"\n");
#endif
}
static str
JSONfilterInternal(json *ret, json *js, str *expr, str other)
{
pattern terms[MAXTERMS];
int tidx = 0;
JSON *jt;
str j = *js, msg = MAL_SUCCEED, s;
json result = 0;
size_t l;
(void) other;
if (strNil(j)) {
*ret = GDKstrdup(j);
return MAL_SUCCEED;
}
memset((char *) terms, 0, MAXTERMS * sizeof(pattern));
msg = JSONcompile(*expr, terms);
if (msg)
return msg;
jt = JSONparse(j, FALSE);
CHECK_JSON(jt);
result = s = JSONmatch(jt, 0, terms, tidx);
// process all other PATH expression
for (tidx++; tidx < MAXTERMS && terms[tidx].token; tidx++)
if (terms[tidx].token == END_STEP && tidx + 1 < MAXTERMS && terms[tidx + 1].token) {
s = JSONmatch(jt, 0, terms, ++tidx);
result = JSONglue(result, s, ',');
}
if (result) {
l = strlen(result);
if (result[l - 1] == ',')
result[l - 1] = 0;
} else
l = 3;
s = GDKzalloc(l + 3);
snprintf(s, l + 3, "[%s]", (result ? result : ""));
GDKfree(result);
for (l = 0; terms[l].token; l++)
if (terms[l].name)
GDKfree(terms[l].name);
JSONfree(jt);
*ret = s;
return msg;
}
str
MATproject(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
bat *res_id = (bat*) getArgReference(stk,pci,0);
bat map_id = *(bat*) getArgReference(stk,pci,1);
BAT *res = NULL, *map;
/* rest of the args are parts, (excluding result and map) */
BAT **bats = GDKzalloc(sizeof(BAT*) * pci->argc - 2);
BUN bcnt = 0;
int i, len = pci->argc-2, sorted = 1;
(void) cntxt; (void) mb; (void) stk;
if( bats == NULL)
throw(SQL, "mat.project",MAL_MALLOC_FAIL);
map = BATdescriptor(map_id);
if (!map)
goto error;
for (i=2; i<pci->argc; i++) {
bat id = *(bat*) getArgReference(stk,pci,i);
bats[i-2] = BATdescriptor(id);
if (!bats[i-2])
goto error;
bcnt += BATcount(bats[i-2]);
if (!bats[i-2]->T->sorted)
sorted = 0;
}
assert(bcnt == BATcount(map));
res = MATproject_(map, bats, len );
if (sorted && res)
BATordered(BATmirror(res));
error:
if (map) BBPunfix(map->batCacheid);
if (bats) {
for (i=0; i<len && bats[i]; i++)
BBPunfix(bats[i]->batCacheid);
GDKfree(bats);
}
if (res) {
BATsettrivprop(res);
BBPkeepref( *res_id = res->batCacheid);
return MAL_SUCCEED;
}
throw(SQL, "mat.project","Cannot access descriptor");
}
void
MCinit(void)
{
char *max_clients = GDKgetenv("max_clients");
int maxclients = 0;
if (max_clients != NULL)
maxclients = atoi(max_clients);
if (maxclients <= 0) {
maxclients = 64;
GDKsetenv("max_clients", "64");
}
MAL_MAXCLIENTS =
/* console */ 1 +
/* client connections */ maxclients;
mal_clients = GDKzalloc(sizeof(ClientRec) * MAL_MAXCLIENTS);
}
void
printSignature(stream *fd, Symbol s, int flg)
{
InstrPtr p;
str txt;
if ( s->def == 0 ){
mnstr_printf(fd, "missing definition of %s\n", s->name);
return;
}
txt = GDKzalloc(MAXLISTING); /* some slack for large blocks */
if( txt){
p = getSignature(s);
(void) fcnDefinition(s->def, p, txt, flg, txt, MAXLISTING);
mnstr_printf(fd, "%s\n", txt);
GDKfree(txt);
} else mnstr_printf(fd, "printSignature: " MAL_MALLOC_FAIL);
}
static int
table_dump(sql_trans *tr, sql_table *t)
{
node *n = cs_first_node(&t->columns);
int i, l = cs_size(&t->columns);
BAT **b = (BAT**)GDKzalloc(sizeof(BAT*) * l);
(void)tr;
for (i = 0; n; n = n->next, i++) {
sql_column *c = n->data;
sql_delta *bat = c->data;
b[i] = temp_descriptor(bat->bid);
}
BATmultiprintf(GDKstdout, l +1, b, TRUE, 0, 1);
for (i = 0; i < l; i++)
bat_destroy(b[i]);
GDKfree(b);
return 0;
}
void
MCinit(void)
{
char *max_clients = GDKgetenv("max_clients");
int maxclients = 0;
if (max_clients != NULL)
maxclients = atoi(max_clients);
if (maxclients <= 0) {
maxclients = 64;
GDKsetenv("max_clients", "64");
}
MAL_MAXCLIENTS =
/* console */ 1 +
/* client connections */ maxclients;
mal_clients = GDKzalloc(sizeof(ClientRec) * MAL_MAXCLIENTS);
if( mal_clients == NULL){
showException(GDKout, MAL, "MCinit",MAL_MALLOC_FAIL);
mal_exit();
}
}
int
OPTpushselectImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
int i, j, limit, slimit, actions=0, *vars, push_down_delta = 0, nr_topn = 0, nr_likes = 0;
InstrPtr p, *old;
subselect_t subselects;
memset(&subselects, 0, sizeof(subselects));
if( mb->errors)
return 0;
OPTDEBUGpushselect
mnstr_printf(cntxt->fdout,"#Push select optimizer started\n");
(void) stk;
(void) pci;
vars= (int*) GDKzalloc(sizeof(int)* mb->vtop);
if( vars == NULL)
return 0;
limit = mb->stop;
slimit= mb->ssize;
old = mb->stmt;
/* check for bailout conditions */
for (i = 1; i < limit; i++) {
int lastbat;
p = old[i];
for (j = 0; j<p->retc; j++) {
int res = getArg(p, j);
vars[res] = i;
}
if (getModuleId(p) == algebraRef &&
(getFunctionId(p) == tintersectRef || getFunctionId(p) == tinterRef ||
getFunctionId(p) == tdifferenceRef || getFunctionId(p) == tdiffRef)) {
GDKfree(vars);
return 0;
}
if (getModuleId(p) == algebraRef && getFunctionId(p) == sliceRef)
nr_topn++;
if (isLikeOp(p))
nr_likes++;
if (getModuleId(p) == sqlRef && getFunctionId(p) == deltaRef)
push_down_delta++;
if (getModuleId(p) == sqlRef && getFunctionId(p) == tidRef) { /* rewrite equal table ids */
int sname = getArg(p, 2), tname = getArg(p, 3), s;
for (s = 0; s < subselects.nr; s++) {
InstrPtr q = old[vars[subselects.tid[s]]];
int Qsname = getArg(q, 2), Qtname = getArg(q, 3);
if (no_updates(old, vars, getArg(q,1), getArg(p,1)) &&
((sname == Qsname && tname == Qtname) ||
(0 && strcmp(getVarConstant(mb, sname).val.sval, getVarConstant(mb, Qsname).val.sval) == 0 &&
strcmp(getVarConstant(mb, tname).val.sval, getVarConstant(mb, Qtname).val.sval) == 0))) {
clrFunction(p);
p->retc = 1;
p->argc = 2;
getArg(p, 1) = getArg(q, 0);
break;
}
}
}
lastbat = lastbat_arg(mb, p);
if (isSubSelect(p) && p->retc == 1 &&
/* no cand list */ getArgType(mb, p, lastbat) != newBatType(TYPE_oid, TYPE_oid)) {
int i1 = getArg(p, 1), tid = 0;
InstrPtr q = old[vars[i1]];
/* find the table ids */
while(!tid) {
if (getModuleId(q) == algebraRef && getFunctionId(q) == leftfetchjoinRef) {
int i1 = getArg(q, 1);
InstrPtr s = old[vars[i1]];
if (getModuleId(s) == sqlRef && getFunctionId(s) == tidRef)
tid = getArg(q, 1);
if (s->argc == 2 && s->retc == 1) {
int i1 = getArg(s, 1);
InstrPtr s = old[vars[i1]];
if (getModuleId(s) == sqlRef && getFunctionId(s) == tidRef)
tid = getArg(q, 1);
}
break;
} else if (isMapOp(q) && q->argc >= 2 && isaBatType(getArgType(mb, q, 1))) {
int i1 = getArg(q, 1);
q = old[vars[i1]];
} else if (isMapOp(q) && q->argc >= 3 && isaBatType(getArgType(mb, q, 2))) {
int i2 = getArg(q, 2);
q = old[vars[i2]];
} else {
break;
}
}
if (tid && subselect_add(&subselects, tid, getArg(p, 0)) < 0) {
GDKfree(vars);
return 0;
}
}
/* left hand side */
if ( (GDKdebug & (1<<15)) &&
isMatJoinOp(p) && p->retc == 2) {
int i1 = getArg(p, 2), tid = 0;
InstrPtr q = old[vars[i1]];
/* find the table ids */
while(!tid) {
if (getModuleId(q) == algebraRef && getFunctionId(q) == leftfetchjoinRef) {
int i1 = getArg(q, 1);
InstrPtr s = old[vars[i1]];
if (getModuleId(s) == sqlRef && getFunctionId(s) == tidRef)
tid = getArg(q, 1);
break;
} else if (isMapOp(q) && q->argc >= 2 && isaBatType(getArgType(mb, q, 1))) {
int i1 = getArg(q, 1);
q = old[vars[i1]];
} else if (isMapOp(q) && q->argc >= 3 && isaBatType(getArgType(mb, q, 2))) {
int i2 = getArg(q, 2);
q = old[vars[i2]];
} else {
break;
}
}
if (tid && subselect_add(&subselects, tid, getArg(p, 0)) < 0) {
GDKfree(vars);
return 0;
}
}
/* right hand side */
if ( (GDKdebug & (1<<15)) &&
isMatJoinOp(p) && p->retc == 2) {
int i1 = getArg(p, 3), tid = 0;
InstrPtr q = old[vars[i1]];
/* find the table ids */
while(!tid) {
if (getModuleId(q) == algebraRef && getFunctionId(q) == leftfetchjoinRef) {
int i1 = getArg(q, 1);
InstrPtr s = old[vars[i1]];
if (getModuleId(s) == sqlRef && getFunctionId(s) == tidRef)
tid = getArg(q, 1);
break;
} else if (isMapOp(q) && q->argc >= 2 && isaBatType(getArgType(mb, q, 1))) {
int i1 = getArg(q, 1);
q = old[vars[i1]];
} else if (isMapOp(q) && q->argc >= 3 && isaBatType(getArgType(mb, q, 2))) {
int i2 = getArg(q, 2);
q = old[vars[i2]];
} else {
break;
}
}
if (tid && subselect_add(&subselects, tid, getArg(p, 1)) < 0) {
GDKfree(vars);
return 0;
}
}
}
if ((!subselects.nr && !nr_topn && !nr_likes) || newMalBlkStmt(mb, mb->ssize) <0 ) {
GDKfree(vars);
return 0;
}
pushInstruction(mb,old[0]);
for (i = 1; i < limit; i++) {
p = old[i];
/* rewrite batalgebra.like + subselect -> likesubselect */
if (getModuleId(p) == algebraRef && p->retc == 1 && getFunctionId(p) == subselectRef) {
int var = getArg(p, 1);
InstrPtr q = mb->stmt[vars[var]]; /* BEWARE: the optimizer may not add or remove statements ! */
if (isLikeOp(q)) { /* TODO check if getArg(p, 3) value == TRUE */
InstrPtr r = newInstruction(mb, ASSIGNsymbol);
int has_cand = (getArgType(mb, p, 2) == newBatType(TYPE_oid, TYPE_oid));
int a, anti = (getFunctionId(q)[0] == 'n'), ignore_case = (getFunctionId(q)[anti?4:0] == 'i');
setModuleId(r, algebraRef);
setFunctionId(r, likesubselectRef);
getArg(r,0) = getArg(p,0);
r = pushArgument(mb, r, getArg(q, 1));
if (has_cand)
r = pushArgument(mb, r, getArg(p, 2));
for(a = 2; a<q->argc; a++)
r = pushArgument(mb, r, getArg(q, a));
if (r->argc < (4+has_cand))
r = pushStr(mb, r, ""); /* default esc */
if (r->argc < (5+has_cand))
r = pushBit(mb, r, ignore_case);
if (r->argc < (6+has_cand))
r = pushBit(mb, r, anti);
freeInstruction(p);
p = r;
actions++;
}
}
/* inject table ids into subselect
* s = subselect(c, C1..) => subselect(c, t, C1..)
*/
if (isSubSelect(p) && p->retc == 1) {
int tid = 0;
if ((tid = subselect_find_tids(&subselects, getArg(p, 0))) >= 0) {
int lastbat = lastbat_arg(mb, p);
if (getArgType(mb, p, lastbat) == TYPE_bat) /* empty candidate list bat_nil */
getArg(p, lastbat) = tid;
else
p = PushArgument(mb, p, tid, lastbat+1);
/* make sure to resolve again */
p->token = ASSIGNsymbol;
p->typechk = TYPE_UNKNOWN;
p->fcn = NULL;
p->blk = NULL;
actions++;
}
}
else if ( (GDKdebug & (1<<15)) &&
isMatJoinOp(p) && p->retc == 2
&& !(getFunctionId(p) == joinRef && p->argc > 4)
) {
int ltid = 0, rtid = 0, done = 0;
int range = 0;
if(getFunctionId(p) == joinRef)
range = (p->argc >= 4);
if ((ltid = subselect_find_tids(&subselects, getArg(p, 0))) >= 0 &&
(rtid = subselect_find_tids(&subselects, getArg(p, 1))) >= 0) {
p = PushArgument(mb, p, ltid, 4+range);
p = PushArgument(mb, p, rtid, 5+range);
done = 1;
} else if ((ltid = subselect_find_tids(&subselects, getArg(p, 0))) >= 0) {
p = PushArgument(mb, p, ltid, 4+range);
p = PushNil(mb, p, 5+range, TYPE_bat);
done = 1;
} else if ((rtid = subselect_find_tids(&subselects, getArg(p, 1))) >= 0) {
p = PushNil(mb, p, 4+range, TYPE_bat);
p = PushArgument(mb, p, rtid, 5+range);
done = 1;
}
if (done) {
if(getFunctionId(p) == antijoinRef)
p = pushInt(mb, p, JOIN_NE);
p = pushBit(mb, p, FALSE); /* do not match nils */
p = pushNil(mb, p, TYPE_lng); /* no estimate */
/* TODO join* -> subjoin* */
if(getFunctionId(p) == joinRef)
getFunctionId(p) = subjoinRef;
else if(getFunctionId(p) == antijoinRef)
getFunctionId(p) = subthetajoinRef;
else if(getFunctionId(p) == thetajoinRef)
getFunctionId(p) = subthetajoinRef;
else if(getFunctionId(p) == bandjoinRef)
getFunctionId(p) = subbandjoinRef;
/* make sure to resolve again */
p->token = ASSIGNsymbol;
p->typechk = TYPE_UNKNOWN;
p->fcn = NULL;
p->blk = NULL;
actions++;
}
}
/* Leftfetchjoins involving rewriten table ids need to be flattend
* l = leftfetchjoin(t, c); => l = c;
* and
* l = leftfetchjoin(s, ntids); => l = s;
*/
else if (getModuleId(p) == algebraRef && getFunctionId(p) == leftfetchjoinRef) {
int var = getArg(p, 1);
if (subselect_find_subselect(&subselects, var) > 0) {
InstrPtr q = newAssignment(mb);
getArg(q, 0) = getArg(p, 0);
(void) pushArgument(mb, q, getArg(p, 2));
actions++;
freeInstruction(p);
continue;
} else { /* deletes/updates use table ids */
int var = getArg(p, 2);
InstrPtr q = mb->stmt[vars[var]]; /* BEWARE: the optimizer may not add or remove statements ! */
if (q->token == ASSIGNsymbol) {
var = getArg(q, 1);
q = mb->stmt[vars[var]];
}
if (subselect_find_subselect(&subselects, var) > 0) {
InstrPtr qq = newAssignment(mb);
/* TODO: check result */
getArg(qq, 0) = getArg(p, 0);
(void) pushArgument(mb, qq, getArg(p, 1));
actions++;
freeInstruction(p);
continue;
}
/* c = sql.delta(b,uid,uval,ins);
* l = leftfetchjoin(x, c);
* into
* l = sql.projectdelta(x,b,uid,uval,ins);
*/
else if (getModuleId(q) == sqlRef && getFunctionId(q) == deltaRef && q->argc == 5) {
q = copyInstruction(q);
setFunctionId(q, projectdeltaRef);
getArg(q, 0) = getArg(p, 0);
q = PushArgument(mb, q, getArg(p, 1), 1);
freeInstruction(p);
p = q;
actions++;
}
}
}
pushInstruction(mb,p);
}
for (; i<limit; i++)
if (old[i])
pushInstruction(mb,old[i]);
for (; i<slimit; i++)
if (old[i])
freeInstruction(old[i]);
GDKfree(old);
if (!push_down_delta) {
GDKfree(vars);
return actions;
}
/* now push selects through delta's */
limit = mb->stop;
slimit= mb->ssize;
old = mb->stmt;
if (newMalBlkStmt(mb, mb->stop+(5*push_down_delta)) <0 ) {
mb->stmt = old;
GDKfree(vars);
return actions;
}
pushInstruction(mb,old[0]);
for (i = 1; i < limit; i++) {
int lastbat;
p = old[i];
for (j = 0; j<p->retc; j++) {
int res = getArg(p, j);
vars[res] = i;
}
/* push subslice under projectdelta */
if (isSlice(p) && p->retc == 1) {
int var = getArg(p, 1);
InstrPtr q = old[vars[var]];
if (getModuleId(q) == sqlRef && getFunctionId(q) == projectdeltaRef) {
InstrPtr r = copyInstruction(p);
InstrPtr s = copyInstruction(q);
ValRecord cst;
/* slice the candidates */
setFunctionId(r, sliceRef);
getArg(r, 0) = newTmpVariable(mb, newBatType(TYPE_oid, TYPE_oid));
getArg(r, 1) = getArg(s, 1);
cst.vtype = getArgType(mb, r, 2);
cst.val.wval = 0;
getArg(r, 2) = defConstant(mb, cst.vtype, &cst); /* start from zero */
pushInstruction(mb,r);
/* dummy result for the old q, will be removed by deadcode optimizer */
getArg(q, 0) = newTmpVariable(mb, getArgType(mb, q, 0));
getArg(s, 1) = getArg(r, 0); /* use result of subslice */
pushInstruction(mb, s);
}
}
/* c = delta(b, uid, uvl, ins)
* s = subselect(c, C1..)
*
* nc = subselect(b, C1..)
* ni = subselect(ins, C1..)
* nu = subselect(uvl, C1..)
* s = subdelta(nc, uid, nu, ni);
*
* doesn't handle Xsubselect(x, .. z, C1.. cases) ie multicolumn selects
*/
lastbat = lastbat_arg(mb, p);
if (isSubSelect(p) && p->retc == 1 && lastbat == 2) {
int var = getArg(p, 1);
InstrPtr q = old[vars[var]];
if (q->token == ASSIGNsymbol) {
var = getArg(q, 1);
q = old[vars[var]];
}
if (getModuleId(q) == sqlRef && getFunctionId(q) == deltaRef) {
InstrPtr r = copyInstruction(p);
InstrPtr s = copyInstruction(p);
InstrPtr t = copyInstruction(p);
InstrPtr u = copyInstruction(q);
getArg(r, 0) = newTmpVariable(mb, newBatType(TYPE_oid, TYPE_oid));
getArg(r, 1) = getArg(q, 1); /* column */
pushInstruction(mb,r);
getArg(s, 0) = newTmpVariable(mb, newBatType(TYPE_oid, TYPE_oid));
getArg(s, 1) = getArg(q, 3); /* updates */
s = ReplaceWithNil(mb, s, 2, TYPE_bat); /* no candidate list */
setArgType(mb, s, 2, newBatType(TYPE_oid,TYPE_oid));
/* make sure to resolve again */
s->token = ASSIGNsymbol;
s->typechk = TYPE_UNKNOWN;
s->fcn = NULL;
s->blk = NULL;
pushInstruction(mb,s);
getArg(t, 0) = newTmpVariable(mb, newBatType(TYPE_oid, TYPE_oid));
getArg(t, 1) = getArg(q, 4); /* inserts */
pushInstruction(mb,t);
setFunctionId(u, subdeltaRef);
getArg(u, 0) = getArg(p,0);
getArg(u, 1) = getArg(r,0);
getArg(u, 2) = getArg(p,2); /* pre-cands */
getArg(u, 3) = getArg(q,2); /* update ids */
getArg(u, 4) = getArg(s,0);
u = pushArgument(mb, u, getArg(t,0));
pushInstruction(mb,u);
freeInstruction(p);
continue;
}
}
pushInstruction(mb,p);
}
for (; i<limit; i++)
if (old[i])
pushInstruction(mb,old[i]);
GDKfree(vars);
GDKfree(old);
return actions;
}
/*
* The prime routine for the BAT layer is to create a new hash index.
* Its argument is the element type and the maximum number of BUNs be
* stored under the hash function.
*/
BAT *
BAThash(BAT *b, BUN masksize)
{
BAT *o = NULL;
lng t0,t1;
(void) t0;
(void) t1;
if (VIEWhparent(b)) {
bat p = VIEWhparent(b);
o = b;
b = BATdescriptor(p);
if (!ALIGNsynced(o, b) || BUNfirst(o) != BUNfirst(b)) {
BBPunfix(b->batCacheid);
b = o;
o = NULL;
}
}
MT_lock_set(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
if (b->H->hash == NULL) {
unsigned int tpe = ATOMstorage(b->htype);
BUN cnt = BATcount(b);
BUN mask;
BUN p = BUNfirst(b), q = BUNlast(b), r;
Hash *h = NULL;
Heap *hp = NULL;
str nme = BBP_physical(b->batCacheid);
BATiter bi = bat_iterator(b);
ALGODEBUG fprintf(stderr, "#BAThash: create hash(" BUNFMT ");\n", BATcount(b));
/* cnt = 0, hopefully there is a proper capacity from
* which we can derive enough information */
if (!cnt)
cnt = BATcapacity(b);
if (b->htype == TYPE_void) {
if (b->hseqbase == oid_nil) {
MT_lock_unset(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
ALGODEBUG fprintf(stderr, "#BAThash: cannot create hash-table on void-NIL column.\n");
return NULL;
}
ALGODEBUG fprintf(stderr, "#BAThash: creating hash-table on void column..\n");
tpe = TYPE_void;
}
/* determine hash mask size p = first; then no dynamic
* scheme */
if (masksize > 0) {
mask = HASHmask(masksize);
} else if (ATOMsize(ATOMstorage(tpe)) == 1) {
mask = (1 << 8);
} else if (ATOMsize(ATOMstorage(tpe)) == 2) {
mask = (1 << 12);
} else if (b->hkey) {
mask = HASHmask(cnt);
} else {
/* dynamic hash: we start with
* HASHmask(cnt/64); if there are too many
* collisions we try HASHmask(cnt/16), then
* HASHmask(cnt/4), and finally
* HASHmask(cnt). */
mask = HASHmask(cnt >> 6);
p += (cnt >> 2); /* try out on first 25% of b */
if (p > q)
p = q;
}
if (mask < 1024)
mask = 1024;
t0 = GDKusec();
do {
BUN nslots = mask >> 3; /* 1/8 full is too full */
r = BUNfirst(b);
if (hp) {
HEAPfree(hp);
GDKfree(hp);
}
if (h) {
ALGODEBUG fprintf(stderr, "#BAThash: retry hash construction\n");
GDKfree(h);
}
/* create the hash structures */
hp = (Heap *) GDKzalloc(sizeof(Heap));
if (hp &&
(hp->filename = GDKmalloc(strlen(nme) + 12)) != NULL)
sprintf(hp->filename, "%s.%chash", nme, b->batCacheid > 0 ? 'h' : 't');
if (hp == NULL ||
hp->filename == NULL ||
(h = HASHnew(hp, ATOMtype(b->htype), BATcapacity(b), mask)) == NULL) {
MT_lock_unset(&GDKhashLock(ABS(b->batCacheid)), "BAThash");
if (hp != NULL) {
GDKfree(hp->filename);
GDKfree(hp);
}
return NULL;
}
switch (tpe) {
case TYPE_bte:
starthash(bte);
break;
case TYPE_sht:
starthash(sht);
break;
case TYPE_int:
case TYPE_flt:
starthash(int);
break;
case TYPE_dbl:
case TYPE_lng:
starthash(lng);
break;
default:
for (; r < p; r++) {
ptr v = BUNhead(bi, r);
BUN c = (BUN) heap_hash_any(b->H->vheap, h, v);
if ( HASHget(h,c) == HASHnil(h) &&
nslots-- == 0)
break; /* mask too full */
HASHputlink(h,r, HASHget(h,c));
HASHput(h,c, r);
}
break;
}
} while (r < p && mask < cnt && (mask <<= 2));
/* finish the hashtable with the current mask */
p = r;
switch (tpe) {
case TYPE_bte:
finishhash(bte);
break;
case TYPE_sht:
finishhash(sht);
break;
case TYPE_int:
case TYPE_flt:
finishhash(int);
break;
case TYPE_dbl:
case TYPE_lng:
finishhash(lng);
break;
default:
for (; p < q; p++) {
ptr v = BUNhead(bi, p);
BUN c = (BUN) heap_hash_any(b->H->vheap, h, v);
HASHputlink(h,p, HASHget(h,c));
HASHput(h,c,p);
}
break;
}
b->H->hash = h;
t1 = GDKusec();
ALGODEBUG
fprintf(stderr, "#BAThash: hash construction "LLFMT" usec\n", t1-t0);
ALGODEBUG HASHcollisions(b,b->H->hash);
}