str DCsliceStrict(int *ret, bat *bid, lng *start, lng *end) { BAT *b, *bn = NULL; if ((b = BATdescriptor(*bid)) == NULL) { throw(MAL, "dcoperator.sliceStrict", "Cannot access descriptor"); } assert(*start >= 0); assert(*end >= 0); assert(*start <= (lng) BUN_MAX); assert(*end < (lng) BUN_MAX); assert(*start <= *end); if ((BUN) ((*end - *start) + 1) > BATcount(b)) { bn = BATnew(b->htype, b->ttype, 0); BATsetcount(bn, 0); *ret = bn->batCacheid; BBPkeepref(*ret); return MAL_SUCCEED; } bn = BATslice(b, (BUN) *start, (BUN) *end + 1); BBPreleaseref(b->batCacheid); if (bn != NULL) { if (!(bn->batDirty & 2)) bn = BATsetaccess(bn, BAT_READ); *ret = bn->batCacheid; BBPkeepref(*ret); return MAL_SUCCEED; } throw(MAL, "dcoperator.sliceStrict", "GDKerror"); }
str CMDbbpHeat(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_int, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpHeat", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpHeat"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) { if (BBP_cache(i) && !monet_modulesilent) { int heat = BBP_lastused(i); BUNins(b, &i, &heat, FALSE); } else if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { int zero = 0; BUNins(b, &i, &zero, FALSE); } } BBPunlock("CMDbbpHeat"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","heat"); return MAL_SUCCEED; }
str CMDbbpNames(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_str, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpNames", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpNames"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) { if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i)) ) { BUNappend(b, BBP_logical(i), FALSE); if (BBP_logical(-i) && (BBP_refs(-i) || BBP_lrefs(-i)) && !BBPtmpcheck(BBP_logical(-i))) BUNappend(b, BBP_logical(-i), FALSE); } } BBPunlock("CMDbbpNames"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","name"); return MAL_SUCCEED; }
str CMDbbpKind(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_str, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpKind", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpKind"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { char *mode = NULL; if ((BBP_status(i) & BBPDELETED) || !(BBP_status(i) & BBPPERSISTENT)) mode = "transient"; else mode = "persistent"; if (mode) BUNappend(b, mode, FALSE); } BBPunlock("CMDbbpKind"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","kind"); return MAL_SUCCEED; }
str CMDbbpCount(bat *ret) { BAT *b, *bn; int i; lng l; b = BATnew(TYPE_void, TYPE_lng, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpCount", MAL_MALLOC_FAIL); BATseqbase(b,0); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) { if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { bn = BATdescriptor(i); if (bn) { l = BATcount(bn); BUNappend(b, &l, FALSE); BBPunfix(bn->batCacheid); } } } if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","count"); return MAL_SUCCEED; }
/* * The BAT status is redundantly stored in CMDbat_info. */ str CMDbbpLocation(bat *ret) { BAT *b; int i; char buf[MAXPATHLEN]; char cwd[MAXPATHLEN]; if (getcwd(cwd, MAXPATHLEN) == NULL) throw(MAL, "catalog.bbpLocation", RUNTIME_DIR_ERROR); b = BATnew(TYPE_void, TYPE_str, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpLocation", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpLocation"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) { if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { snprintf(buf,MAXPATHLEN,"%s/bat/%s",cwd,BBP_physical(i)); BUNappend(b, buf, FALSE); } } BBPunlock("CMDbbpLocation"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","location"); return MAL_SUCCEED; }
/* set access mode to bat, replacing input with output */ static BAT * setaccess(BAT *b, int mode) { BAT *bn = b; if (BATsetaccess(b, mode) != GDK_SUCCEED) { if (b->batSharecnt && mode != BAT_READ) { bn = BATcopy(b, b->htype, b->ttype, TRUE, TRANSIENT); if (bn != NULL) BATsetaccess(bn, mode); } else { bn = NULL; } BBPunfix(b->batCacheid); } return bn; }
/* * Enable incremental packing. The SQL front-end requires * fixed oid sequences. */ str MATpackIncrement(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p) { bat *ret = getArgReference_bat(stk,p,0); int pieces; BAT *b, *bb, *bn; size_t newsize; (void) cntxt; b = BATdescriptor( stk->stk[getArg(p,1)].val.ival); if ( b == NULL) throw(MAL, "mat.pack", RUNTIME_OBJECT_MISSING); if ( getArgType(mb,p,2) == TYPE_int){ /* first step, estimate with some slack */ pieces = stk->stk[getArg(p,2)].val.ival; bn = BATnew(TYPE_void, b->ttype?b->ttype:TYPE_oid, (BUN)(1.2 * BATcount(b) * pieces), TRANSIENT); if (bn == NULL) throw(MAL, "mat.pack", MAL_MALLOC_FAIL); /* allocate enough space for the vheap, but not for strings, * since BATappend does clever things for strings */ if ( b->T->vheap && bn->T->vheap && ATOMstorage(b->ttype) != TYPE_str){ newsize = b->T->vheap->size * pieces; if (HEAPextend(bn->T->vheap, newsize, TRUE) != GDK_SUCCEED) throw(MAL, "mat.pack", MAL_MALLOC_FAIL); } BATseqbase(bn, b->H->seq); BATseqbase(BATmirror(bn), b->T->seq); BATappend(bn,b,FALSE); assert(!bn->H->nil || !bn->H->nonil); assert(!bn->T->nil || !bn->T->nonil); bn->H->align = (pieces-1); BBPkeepref(*ret = bn->batCacheid); BBPunfix(b->batCacheid); } else { /* remaining steps */ bb = BATdescriptor(stk->stk[getArg(p,2)].val.ival); if ( bb ){ if (BATcount(b) == 0) BATseqbase(b, bb->H->seq); if (BATcount(b) == 0) BATseqbase(BATmirror(b), bb->T->seq); BATappend(b,bb,FALSE); } b->H->align--; if(b->H->align == 0) BATsetaccess(b, BAT_READ); assert(!b->H->nil || !b->H->nonil); assert(!b->T->nil || !b->T->nonil); BBPkeepref(*ret = b->batCacheid); if( bb) BBPunfix(bb->batCacheid); } return MAL_SUCCEED; }
str CMDbbpLRefCount(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_int, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpLRefCount", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpLRefCount"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid && BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { int refs = BBP_lrefs(i); BUNappend(b, &refs, FALSE); } BBPunlock("CMDbbpLRefCount"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","lrefcnt"); return MAL_SUCCEED; }
/* * It is illustrative to dump the code when you * have encountered an error. */ str MDBgetDefinition(Client cntxt, MalBlkPtr m, MalStkPtr stk, InstrPtr p) { int i; bat *ret = getArgReference_bat(stk, p, 0); str ps; BAT *b = BATnew(TYPE_void, TYPE_str, 256, TRANSIENT); (void) cntxt; if (b == 0) throw(MAL, "mdb.getDefinition", MAL_MALLOC_FAIL); BATseqbase(b,0); for (i = 0; i < m->stop; i++) { ps = instruction2str(m,0, getInstrPtr(m, i), 1); BUNappend(b, ps, FALSE); GDKfree(ps); } if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"view","fcn","stmt"); return MAL_SUCCEED; }
/* * The BAT status is redundantly stored in CMDbat_info. */ str CMDbbpStatus(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_str, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpStatus", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpStatus"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { char *loc = BBP_cache(i) ? "load" : "disk"; BUNappend(b, loc, FALSE); } BBPunlock("CMDbbpStatus"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","status"); return MAL_SUCCEED; }
/* * The BAT dirty status:dirty => (mem != disk); diffs = not-committed */ str CMDbbpDirty(bat *ret) { BAT *b; int i; b = BATnew(TYPE_void, TYPE_str, getBBPsize(), TRANSIENT); if (b == 0) throw(MAL, "catalog.bbpDirty", MAL_MALLOC_FAIL); BATseqbase(b,0); BBPlock("CMDbbpDirty"); for (i = 1; i < getBBPsize(); i++) if (i != b->batCacheid) if (BBP_logical(i) && (BBP_refs(i) || BBP_lrefs(i))) { BAT *bn = BBP_cache(i); BUNappend(b, bn ? BATdirty(bn) ? "dirty" : DELTAdirty(bn) ? "diffs" : "clean" : (BBP_status(i) & BBPSWAPPED) ? "diffs" : "clean", FALSE); } BBPunlock("CMDbbpDirty"); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); pseudo(ret,b,"bbp","status"); return MAL_SUCCEED; }
static BAT * ALGjoinPathBody(Client cntxt, int top, BAT **joins, int flag) { BAT *b = NULL; BUN estimate, e = 0; int i, j, k; int *postpone= (int*) GDKzalloc(sizeof(int) *top); int postponed=0; if(postpone == NULL){ GDKerror("joinPathBody" MAL_MALLOC_FAIL); return NULL; } /* solve the join by pairing the smallest first */ while (top > 1) { j = 0; estimate = ALGjoinCost(cntxt,joins[0],joins[1],flag); ALGODEBUG fprintf(stderr,"#joinPath estimate join(%d,%d) %d cnt="BUNFMT" %s\n", joins[0]->batCacheid, joins[1]->batCacheid,(int)estimate, BATcount(joins[0]), postpone[0]?"postpone":""); for (i = 1; i < top - 1; i++) { e = ALGjoinCost(cntxt,joins[i], joins[i + 1],flag); ALGODEBUG fprintf(stderr,"#joinPath estimate join(%d,%d) %d cnt="BUNFMT" %s\n", joins[i]->batCacheid, joins[i+1]->batCacheid,(int)e,BATcount(joins[i]), postpone[i]?"postpone":""); if (e < estimate && ( !(postpone[i] && postpone[i+1]) || postponed<top)) { estimate = e; j = i; } } /* * BEWARE. you may not use a size estimation, because it * may fire a BATproperty check in a few cases. * In case a join fails, we may try another order first before * abandoning the task. It can handle cases where a Cartesian product emerges. * * A left-join sequence only requires the result to be sorted * against the first operand. For all others operand pairs, the cheapest join suffice. */ switch(flag){ case 0: if ( j == 0) { b = BATleftjoin(joins[j], joins[j + 1], BATcount(joins[j])); ALGODEBUG{ fprintf(stderr,"#joinpath step produces "BUNFMT"\n", BATcount(b)); } break; } case 1: b = BATjoin(joins[j], joins[j + 1], (BATcount(joins[j]) < BATcount(joins[j + 1])? BATcount(joins[j]):BATcount(joins[ j + 1]))); break; case 3: b = BATproject(joins[j], joins[j + 1]); ALGODEBUG{ fprintf(stderr,"#joinpath step produces "BUNFMT"\n", BATcount(b)); } break; } if (b==NULL){ if ( postpone[j] && postpone[j+1]){ for( --top; top>=0; top--) BBPunfix(joins[top]->batCacheid); GDKfree(postpone); return NULL; } postpone[j] = TRUE; postpone[j+1] = TRUE; postponed = 0; for( k=0; k<top; k++) postponed += postpone[k]== TRUE; if ( postponed == top){ for( --top; top>=0; top--) BBPunfix(joins[top]->batCacheid); GDKfree(postpone); return NULL; } /* clear the GDKerrors and retry */ if( cntxt->errbuf ) cntxt->errbuf[0]=0; continue; } else { /* reset the postponed joins */ for( k=0; k<top; k++) postpone[k]=FALSE; if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); postponed = 0; } ALGODEBUG{ if (b ) { fprintf(stderr, "#joinPath %d:= join(%d,%d)" " arguments %d (cnt= "BUNFMT") against (cnt "BUNFMT") cost "BUNFMT"\n", b->batCacheid, joins[j]->batCacheid, joins[j + 1]->batCacheid, j, BATcount(joins[j]), BATcount(joins[j+1]), e); } } if ( b == 0 ){ for( --top; top>=0; top--) BBPunfix(joins[top]->batCacheid); GDKfree(postpone); return 0; } BBPunfix(joins[j]->batCacheid); BBPunfix(joins[j+1]->batCacheid); joins[j] = b; top--; for (i = j + 1; i < top; i++) joins[i] = joins[i + 1]; }
str BKCreuseBATmap(bat *ret, const bat *bid, const bat *did) { BAT *b, *d, *bn, *bs; oid bidx, oidx = 0, *o, *ol; oid *r; gdk_return res; if ((b = BATdescriptor(*bid)) == NULL) { throw(MAL, "bat.shrinkMap", RUNTIME_OBJECT_MISSING); } if ( b->htype != TYPE_void) { BBPunfix(b->batCacheid); throw(MAL, "bat.shrinkMap", SEMANTIC_TYPE_MISMATCH); } if ((d = BATdescriptor(*did)) == NULL) { BBPunfix(b->batCacheid); throw(MAL, "bat.shrinkMap", RUNTIME_OBJECT_MISSING); } bn= BATnew(TYPE_void, TYPE_oid, BATcount(b) - BATcount(d), TRANSIENT); if (bn == NULL) { BBPunfix(b->batCacheid); BBPunfix(d->batCacheid); throw(MAL, "bat.shrinkMap", MAL_MALLOC_FAIL ); } res = BATsubsort(&bs, NULL, NULL, d, NULL, NULL, 0, 0); BBPunfix(d->batCacheid); if (res != GDK_SUCCEED) { BBPunfix(b->batCacheid); BBPunfix(bn->batCacheid); throw(MAL, "bat.shrinkMap", MAL_MALLOC_FAIL ); } oidx = b->hseqbase; bidx = oidx + BATcount(b)-1; o = (oid*)Tloc(bs, BUNfirst(bs)); ol = (oid*)Tloc(bs, BUNlast(bs)); r = (oid*)Tloc(bn, BUNfirst(bn)); for (; oidx <= bidx; oidx++) { if ( *o == oidx ){ while ( ol > o && ol[-1] == bidx) { bidx--; ol--; } *r++ = bidx; o += (o < ol); bidx--; } else { *r++ = oidx; } } BATsetcount(bn, BATcount(b)-BATcount(bs)); BATseqbase(bn, b->hseqbase); bn->tsorted = 0; bn->trevsorted = 0; bn->tdense = 0; if (!(bn->batDirty&2)) BATsetaccess(bn, BAT_READ); BBPunfix(b->batCacheid); BBPunfix(bs->batCacheid); BBPkeepref(*ret= bn->batCacheid); return MAL_SUCCEED; }
str BKCreuseBAT(bat *ret, const bat *bid, const bat *did) { BAT *b, *d, *bn, *bs; oid oidx = 0, bidx, *o, *ol; gdk_return res; if ((b = BATdescriptor(*bid)) == NULL) { throw(MAL, "bat.reuse", RUNTIME_OBJECT_MISSING); } if ( b->htype != TYPE_void) { BBPunfix(b->batCacheid); throw(MAL, "bat.reuse", SEMANTIC_TYPE_MISMATCH); } if ((d = BATdescriptor(*did)) == NULL) { BBPunfix(b->batCacheid); throw(MAL, "bat.reuse", RUNTIME_OBJECT_MISSING); } bn= BATnew(b->htype, b->ttype, BATcount(b) - BATcount(d), TRANSIENT); if (bn == NULL) { BBPunfix(b->batCacheid); BBPunfix(d->batCacheid); throw(MAL, "bat.reuse", MAL_MALLOC_FAIL ); } res = BATsubsort(&bs, NULL, NULL, d, NULL, NULL, 0, 0); BBPunfix(d->batCacheid); if (res != GDK_SUCCEED) { BBPunfix(b->batCacheid); BBPunfix(bn->batCacheid); throw(MAL, "bat.reuse", MAL_MALLOC_FAIL ); } oidx = b->hseqbase; bidx = oidx + BATcount(b)-1; o = (oid*)Tloc(bs, BUNfirst(bs)); ol= (oid*)Tloc(bs, BUNlast(bs)); switch(ATOMstorage(b->ttype) ){ case TYPE_bte: reuseloop(bte); break; case TYPE_sht: reuseloop(sht); break; case TYPE_int: reuseloop(int); break; case TYPE_lng: reuseloop(lng); break; #ifdef HAVE_HGE case TYPE_hge: reuseloop(hge); break; #endif case TYPE_flt: reuseloop(flt); break; case TYPE_dbl: reuseloop(dbl); break; case TYPE_oid: reuseloop(oid); break; case TYPE_str: /* to be done based on its index width */ default: if (ATOMvarsized(bn->ttype)) { BUN p = BUNfirst(b); BUN q = BUNlast(b); BATiter bi = bat_iterator(b); for (;p<q; oidx++, p++) { if ( *o == oidx ){ while ( ol > o && ol[-1] == bidx) { bidx--; q--; ol--; } BUNappend(bn, BUNtail(bi, --q), FALSE); o += (o < ol); bidx--; } else { BUNappend(bn, BUNtail(bi, p), FALSE); } } } else { switch( b->T->width){ case 1:reuseloop(bte); break; case 2:reuseloop(sht); break; case 4:reuseloop(int); break; case 8:reuseloop(lng); break; #ifdef HAVE_HGE case 16:reuseloop(hge); break; #endif default: throw(MAL, "bat.shrink", "Illegal argument type"); } } } BATsetcount(bn, BATcount(b) - BATcount(bs)); BATseqbase(bn, b->hseqbase); bn->tsorted = 0; bn->trevsorted = 0; bn->tdense = 0; bn->tkey = b->tkey; if (!(bn->batDirty&2)) BATsetaccess(bn, BAT_READ); BBPunfix(b->batCacheid); BBPunfix(bs->batCacheid); BBPkeepref(*ret= bn->batCacheid); return MAL_SUCCEED; }
str MDBStkTrace(Client cntxt, MalBlkPtr m, MalStkPtr s, InstrPtr p) { BAT *b, *bn; str msg; char *buf; bat *ret = getArgReference_bat(s, p, 0); bat *ret2 = getArgReference_bat(s, p, 1); int k = 0; size_t len,l; b = BATnew(TYPE_void, TYPE_int, 256, TRANSIENT); if ( b== NULL) throw(MAL, "mdb.getStackTrace", MAL_MALLOC_FAIL); bn = BATnew(TYPE_void, TYPE_str, 256, TRANSIENT); if ( bn== NULL) { BBPreclaim(b); throw(MAL, "mdb.getStackTrace", MAL_MALLOC_FAIL); } BATseqbase(b,0); BATseqbase(bn,0); (void) cntxt; msg = instruction2str(s->blk, s, p, LIST_MAL_DEBUG); len = strlen(msg); buf = (char*) GDKmalloc(len +1024); if ( buf == NULL){ GDKfree(msg); throw(MAL,"mdb.setTrace",MAL_MALLOC_FAIL); } snprintf(buf,len+1024,"%s at %s.%s[%d]", msg, getModuleId(getInstrPtr(m,0)), getFunctionId(getInstrPtr(m,0)), getPC(m, p)); BUNappend(b, &k, FALSE); BUNappend(bn, buf, FALSE); GDKfree(msg); for (s = s->up, k++; s != NULL; s = s->up, k++) { msg = instruction2str(s->blk, s, getInstrPtr(s->blk,s->pcup),LIST_MAL_DEBUG); l = strlen(msg); if (l>len){ GDKfree(buf); len=l; buf = (char*) GDKmalloc(len +1024); if ( buf == NULL){ GDKfree(msg); BBPunfix(b->batCacheid); BBPunfix(bn->batCacheid); throw(MAL,"mdb.setTrace",MAL_MALLOC_FAIL); } } snprintf(buf,len+1024,"%s at %s.%s[%d]", msg, getModuleId(getInstrPtr(s->blk,0)), getFunctionId(getInstrPtr(s->blk,0)), s->pcup); BUNappend(b, &k, FALSE); BUNappend(bn, buf, FALSE); GDKfree(msg); } GDKfree(buf); if (!(b->batDirty&2)) BATsetaccess(b, BAT_READ); if (!(bn->batDirty&2)) BATsetaccess(bn, BAT_READ); pseudo(ret,b,"view","stk","trace"); pseudo(ret2,bn,"view","stk","traceB"); return MAL_SUCCEED; }