str OPTorcam(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){
Symbol t;
str msg,mod,fcn;
lng clk= GDKusec();
int actions = 0;
if( p ==NULL )
return 0;
removeInstruction(mb, p);
if( p->argc == 3){
mod= getArgDefault(mb,p,1);
fcn= getArgDefault(mb,p,2);
} else {
mod= getArgDefault(mb,p,3);
fcn= getArgDefault(mb,p,4);
}
t= findSymbol(cntxt->nspace, putName(mod, strlen(mod)), fcn);
if( t == 0)
return 0;
msg = MACROvalidate(t->def);
if( msg)
return msg;
if( mb->errors == 0)
actions= OPTorcamImplementation(cntxt,mb,stk,p);
return optimizerCheck(cntxt,mb, "optimizer.orcam", actions, GDKusec() - clk, OPT_CHECK_ALL);
}
str OPTsql_append(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){
str modnme;
str fcnnme;
str msg= MAL_SUCCEED;
Symbol s= NULL;
lng t,clk= GDKusec();
int actions = 0;
if( p )
removeInstruction(mb, p);
OPTDEBUGsql_append mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER sql_append\n");
if( p && p->argc > 1 ){
if( getArgType(mb,p,1) != TYPE_str ||
getArgType(mb,p,2) != TYPE_str ||
!isVarConstant(mb,getArg(p,1)) ||
!isVarConstant(mb,getArg(p,2))
) {
throw(MAL, "optimizer.sql_append", ILLARG_CONSTANTS);
}
if( stk != 0){
modnme= *getArgReference_str(stk,p,1);
fcnnme= *getArgReference_str(stk,p,2);
} else {
modnme= getArgDefault(mb,p,1);
fcnnme= getArgDefault(mb,p,2);
}
s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme)));
if( s == NULL) {
char buf[1024];
snprintf(buf,1024, "%s.%s",modnme,fcnnme);
throw(MAL, "optimizer.sql_append", RUNTIME_OBJECT_UNDEFINED ":%s", buf);
}
mb = s->def;
stk= 0;
}
if( mb->errors ){
/* when we have errors, we still want to see them */
addtoMalBlkHistory(mb,"sql_append");
return MAL_SUCCEED;
}
actions= OPTsql_appendImplementation(cntxt, mb,stk,p);
msg= optimizerCheck(cntxt, mb, "optimizer.sql_append", actions, t=(GDKusec() - clk));
OPTDEBUGsql_append {
mnstr_printf(cntxt->fdout,"=FINISHED sql_append %d\n",actions);
printFunction(cntxt->fdout,mb,0,LIST_MAL_ALL );
}
DEBUGoptimizers
mnstr_printf(cntxt->fdout,"#opt_reduce: " LLFMT " ms\n",t);
QOTupdateStatistics("sql_append",actions,t);
addtoMalBlkHistory(mb,"sql_append");
return msg;
}
void
addQueryToCache(Client c)
{
MalBlkPtr mb;
backend *be;
str msg = 0, pipe;
be = (backend *) c->sqlcontext;
assert(be && be->mvc); /* SQL clients should always have their state set */
pipe = getSQLoptimizer(be->mvc);
insertSymbol(c->nspace, c->curprg);
trimMalBlk(c->curprg->def);
c->blkmode = 0;
mb = c->curprg->def;
chkProgram(c->fdout, c->nspace, mb);
#ifdef _SQL_OPTIMIZER_DEBUG
mnstr_printf(GDKout, "ADD QUERY TO CACHE\n");
printFunction(GDKout, mb, 0, LIST_MAL_ALL);
#endif
/*
* An error in the compilation should be reported to the user.
* And if the debugging option is set, the debugger is called
* to allow inspection.
*/
if (mb->errors) {
showErrors(c);
if (c->listing)
printFunction(c->fdout, mb, 0, c->listing);
if (be->mvc->debug) {
msg = runMALDebugger(c, c->curprg);
if (msg != MAL_SUCCEED)
GDKfree(msg); /* ignore error */
}
return;
}
addOptimizers(c, mb, pipe);
msg = optimizeMALBlock(c, mb);
if (msg != MAL_SUCCEED) {
showScriptException(c->fdout, mb, 0, MAL, "%s", msg);
GDKfree(msg);
return;
}
/* time to execute the optimizers */
if (c->debug)
optimizerCheck(c, mb, "sql.baseline", -1, 0);
#ifdef _SQL_OPTIMIZER_DEBUG
mnstr_printf(GDKout, "ADD optimized QUERY TO CACHE\n");
printFunction(GDKout, mb, 0, LIST_MAL_ALL);
#endif
}
str
optimizeQuery(Client c)
{
MalBlkPtr mb;
backend *be;
str msg = 0, pipe;
be = (backend *) c->sqlcontext;
assert(be && be->mvc); /* SQL clients should always have their state set */
pipe = getSQLoptimizer(be->mvc);
trimMalBlk(c->curprg->def);
c->blkmode = 0;
mb = c->curprg->def;
chkProgram(c->fdout, c->nspace, mb);
#ifdef _SQL_OPTIMIZER_DEBUG
mnstr_printf(GDKout, "Optimize query\n");
printFunction(GDKout, mb, 0, LIST_MAL_ALL);
#endif
/*
* An error in the compilation should be reported to the user.
* And if the debugging option is set, the debugger is called
* to allow inspection.
*/
if (mb->errors) {
showErrors(c);
if (c->listing)
printFunction(c->fdout, mb, 0, c->listing);
return NULL;
}
addOptimizers(c, mb, pipe);
msg = optimizeMALBlock(c, mb);
if (msg)
return msg;
/* time to execute the optimizers */
if (c->debug)
optimizerCheck(c, mb, "sql.baseline", -1, 0);
#ifdef _SQL_OPTIMIZER_DEBUG
mnstr_printf(GDKout, "End Optimize Query\n");
printFunction(GDKout, mb, 0, LIST_MAL_ALL);
#endif
return NULL;
}
static void
SQLgetStatistics(Client cntxt, mvc *m, MalBlkPtr mb)
{
InstrPtr *old = NULL;
int oldtop, i, actions = 0, size = 0;
lng clk = GDKusec();
sql_trans *tr = m->session->tr;
str msg;
old = mb->stmt;
oldtop = mb->stop;
size = (mb->stop * 1.2 < mb->ssize) ? mb->ssize : (int) (mb->stop * 1.2);
mb->stmt = (InstrPtr *) GDKzalloc(size * sizeof(InstrPtr));
mb->ssize = size;
mb->stop = 0;
for (i = 0; i < oldtop; i++) {
InstrPtr p = old[i];
char *f = getFunctionId(p);
ValRecord vr;
if (getModuleId(p) == sqlRef && f == tidRef) {
char *sname = getVarConstant(mb, getArg(p, 2)).val.sval;
char *tname = getVarConstant(mb, getArg(p, 3)).val.sval;
sql_schema *s = mvc_bind_schema(m, sname);
sql_table *t;
if (!s || strcmp(s->base.name, dt_schema) == 0) {
pushInstruction(mb, p);
continue;
}
t = mvc_bind_table(m, s, tname);
if (t && (!isRemote(t) && !isMergeTable(t)) && t->p) {
int k = getArg(p, 0), mt_member = t->p->base.id;
varSetProp(mb, k, mtProp, op_eq, VALset(&vr, TYPE_int, &mt_member));
}
}
if (getModuleId(p) == sqlRef && (f == bindRef || f == bindidxRef)) {
int upd = (p->argc == 7 || p->argc == 9);
char *sname = getVarConstant(mb, getArg(p, 2 + upd)).val.sval;
char *tname = getVarConstant(mb, getArg(p, 3 + upd)).val.sval;
char *cname = NULL;
int not_null = 0, mt_member = 0;
wrd rows = 1; /* default to cope with delta bats */
int mode = 0;
int k = getArg(p, 0);
sql_schema *s = mvc_bind_schema(m, sname);
BAT *b;
if (!s || strcmp(s->base.name, dt_schema) == 0) {
pushInstruction(mb, p);
continue;
}
cname = getVarConstant(mb, getArg(p, 4 + upd)).val.sval;
mode = getVarConstant(mb, getArg(p, 5 + upd)).val.ival;
if (s && f == bindidxRef && cname) {
size_t cnt;
sql_idx *i = mvc_bind_idx(m, s, cname);
if (i && (!isRemote(i->t) && !isMergeTable(i->t))) {
cnt = store_funcs.count_idx(tr, i, 1);
assert(cnt <= (size_t) GDK_oid_max);
b = store_funcs.bind_idx(m->session->tr, i, RDONLY);
if (b) {
str loc;
if (b->batPersistence == PERSISTENT && BATlocation(&loc, &b->batCacheid) && loc)
varSetProp(mb, k, fileProp, op_eq, VALset(&vr, TYPE_str, loc));
cnt = BATcount(b);
BBPunfix(b->batCacheid);
}
rows = (wrd) cnt;
if (i->t->p)
mt_member = i->t->p->base.id;
}
} else if (s && f == bindRef && cname) {
size_t cnt;
sql_table *t = mvc_bind_table(m, s, tname);
sql_column *c = mvc_bind_column(m, t, cname);
if (c && (!isRemote(c->t) && !isMergeTable(c->t))) {
not_null = !c->null;
cnt = store_funcs.count_col(tr, c, 1);
assert(cnt <= (size_t) GDK_oid_max);
b = store_funcs.bind_col(m->session->tr, c, RDONLY);
if (b) {
str loc;
if (b->batPersistence == PERSISTENT && BATlocation(&loc, &b->batCacheid) && loc)
varSetProp(mb, k, fileProp, op_eq, VALset(&vr, TYPE_str, loc));
cnt = BATcount(b);
BBPunfix(b->batCacheid);
}
rows = (wrd) cnt;
if (c->t->p)
mt_member = c->t->p->base.id;
}
}
if (rows > 1 && mode != RD_INS)
varSetProp(mb, k, rowsProp, op_eq, VALset(&vr, TYPE_wrd, &rows));
if (not_null)
varSetProp(mb, k, notnilProp, op_eq, NULL);
if (mt_member && mode != RD_INS)
varSetProp(mb, k, mtProp, op_eq, VALset(&vr, TYPE_int, &mt_member));
{
int lowprop = hlbProp, highprop = hubProp;
/* rows == cnt has been checked above to be <= GDK_oid_max */
oid low = 0, high = low + (oid) rows;
pushInstruction(mb, p);
if (mode == RD_INS) {
low = high;
high += 1024 * 1024;
}
varSetProp(mb, getArg(p, 0), lowprop, op_gte, VALset(&vr, TYPE_oid, &low));
varSetProp(mb, getArg(p, 0), highprop, op_lt, VALset(&vr, TYPE_oid, &high));
}
if (not_null)
actions++;
} else {
pushInstruction(mb, p);
}
}
GDKfree(old);
msg = optimizerCheck(cntxt, mb, "optimizer.SQLgetstatistics", actions, GDKusec() - clk);
if (msg) /* what to do with an error? */
GDKfree(msg);
}
static void SQLgetStatistics(Client cntxt, mvc *m, MalBlkPtr mb)
{
InstrPtr *old = NULL;
int oldtop, i, actions = 0, size = 0;
lng clk = GDKusec();
sql_trans *tr = m->session->tr;
str msg;
old = mb->stmt;
oldtop = mb->stop;
size = (mb->stop * 1.2 < mb->ssize) ? mb->ssize : (int) (mb->stop * 1.2);
mb->stmt = (InstrPtr *) GDKzalloc(size * sizeof(InstrPtr));
mb->ssize = size;
mb->stop = 0;
for (i = 0; i < oldtop; i++) {
InstrPtr p = old[i];
char *f = getFunctionId(p);
if (getModuleId(p) == sqlRef && f == tidRef) {
char *sname = getVarConstant(mb, getArg(p, 2)).val.sval;
char *tname = getVarConstant(mb, getArg(p, 3)).val.sval;
sql_schema *s = mvc_bind_schema(m, sname);
sql_table *t;
if (!s || strcmp(s->base.name, dt_schema) == 0) {
pushInstruction(mb, p);
continue;
}
t = mvc_bind_table(m, s, tname);
if (t && (!isRemote(t) && !isMergeTable(t)) && t->p) {
int mt_member = t->p->base.id;
setMitosisPartition(p,mt_member);
}
}
if (getModuleId(p) == sqlRef && (f == bindRef || f == bindidxRef)) {
int upd = (p->argc == 7 || p->argc == 9);
char *sname = getVarConstant(mb, getArg(p, 2 + upd)).val.sval;
char *tname = getVarConstant(mb, getArg(p, 3 + upd)).val.sval;
char *cname = NULL;
int mt_member = 0;
BUN rows = 1; /* default to cope with delta bats */
int mode = 0;
int k = getArg(p, 0);
sql_schema *s = mvc_bind_schema(m, sname);
BAT *b;
if (!s || strcmp(s->base.name, dt_schema) == 0) {
pushInstruction(mb, p);
continue;
}
cname = getVarConstant(mb, getArg(p, 4 + upd)).val.sval;
mode = getVarConstant(mb, getArg(p, 5 + upd)).val.ival;
if (s && f == bindidxRef && cname) {
size_t cnt;
sql_idx *i = mvc_bind_idx(m, s, cname);
if (i && (!isRemote(i->t) && !isMergeTable(i->t))) {
cnt = store_funcs.count_idx(tr, i, 1);
assert(cnt <= (size_t) GDK_oid_max);
b = store_funcs.bind_idx(m->session->tr, i, RDONLY);
if (b) {
cnt = BATcount(b);
BBPunfix(b->batCacheid);
}
rows = (BUN) cnt;
if (i->t->p)
mt_member = i->t->p->base.id;
}
} else if (s && f == bindRef && cname) {
size_t cnt;
sql_table *t = mvc_bind_table(m, s, tname);
sql_column *c = mvc_bind_column(m, t, cname);
if (c && (!isRemote(c->t) && !isMergeTable(c->t))) {
cnt = store_funcs.count_col(tr, c, 1);
assert(cnt <= (size_t) GDK_oid_max);
b = store_funcs.bind_col(m->session->tr, c, RDONLY);
if (b) {
cnt = BATcount(b);
BBPunfix(b->batCacheid);
}
rows = (BUN) cnt;
if (c->t->p)
mt_member = c->t->p->base.id;
}
}
if (rows > 1 && mode != RD_INS)
setRowCnt(mb,k,rows);
if (mt_member && mode != RD_INS)
setMitosisPartition(p,mt_member);
pushInstruction(mb, p);
} else {
pushInstruction(mb, p);
}
}
GDKfree(old);
msg = optimizerCheck(cntxt, mb, "optimizer.SQLgetstatistics", actions, GDKusec() - clk);
if (msg) /* what to do with an error? */
GDKfree(msg);
}
str OPTwrapper (Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p){
str modnme = "(NONE)";
str fcnnme = 0;
str msg= MAL_SUCCEED;
Symbol s= NULL;
lng t,clk= GDKusec();
int i, actions = 0;
char optimizer[256];
InstrPtr q;
if( p == NULL)
throw(MAL, "opt_wrapper", "missing optimizer statement");
snprintf(optimizer,256,"%s", fcnnme = getFunctionId(p));
q= copyInstruction(p);
OPTIMIZERDEBUG
mnstr_printf(cntxt->fdout,"=APPLY OPTIMIZER %s\n",fcnnme);
if( p && p->argc > 1 ){
if( getArgType(mb,p,1) != TYPE_str ||
getArgType(mb,p,2) != TYPE_str ||
!isVarConstant(mb,getArg(p,1)) ||
!isVarConstant(mb,getArg(p,2))
) {
freeInstruction(q);
throw(MAL, optimizer, ILLARG_CONSTANTS);
}
if( stk != 0){
modnme= *getArgReference_str(stk,p,1);
fcnnme= *getArgReference_str(stk,p,2);
} else {
modnme= getArgDefault(mb,p,1);
fcnnme= getArgDefault(mb,p,2);
}
removeInstruction(mb, p);
s= findSymbol(cntxt->nspace, putName(modnme,strlen(modnme)),putName(fcnnme,strlen(fcnnme)));
if( s == NULL) {
freeInstruction(q);
throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme);
}
mb = s->def;
stk= 0;
} else if( p )
removeInstruction(mb, p);
if( mb->errors ){
/* when we have errors, we still want to see them */
addtoMalBlkHistory(mb,getModuleId(q));
freeInstruction(q);
return MAL_SUCCEED;
}
for ( i=0; codes[i].nme; i++)
if ( strcmp(codes[i].nme, optimizer)== 0 ){
actions = (int)(*(codes[i].fcn))(cntxt, mb, stk,0);
break;
}
if ( codes[i].nme == 0){
freeInstruction(q);
throw(MAL, optimizer, RUNTIME_OBJECT_UNDEFINED ":%s.%s", modnme, fcnnme);
}
msg= optimizerCheck(cntxt, mb, optimizer, actions, t=(GDKusec() - clk));
OPTIMIZERDEBUG {
mnstr_printf(cntxt->fdout,"=FINISHED %s %d\n",optimizer, actions);
printFunction(cntxt->fdout,mb,0,LIST_MAL_DEBUG );
}
DEBUGoptimizers
mnstr_printf(cntxt->fdout,"#optimizer %-11s %3d actions %5d MAL instructions ("SZFMT" K) " LLFMT" usec\n", optimizer, actions, mb->stop,
((sizeof( MalBlkRecord) +mb->ssize * offsetof(InstrRecord, argv)+ mb->vtop * sizeof(int) /* argv estimate */ +mb->vtop* sizeof(VarRecord) + mb->vsize*sizeof(VarPtr)+1023)/1024),
t);
QOTupdateStatistics(getModuleId(q),actions,t);
addtoMalBlkHistory(mb,getModuleId(q));
freeInstruction(q);
return msg;
}
