/*! * insert a value at position i of the mfv sketch * * we do not overwrite the previous value at position i. * instead we place the new value at the next_offset. * * <i>Note: we do not currently garbage collection the old value's storage. * This wastes space, with the worst-case scenario being a column with values * of increasing size and frequency!</i> * * \param transblob the transition value packed into a bytea * \param dat the value to be inserted * \param i the position to insert at */ bytea *mfv_transval_insert_at(bytea *transblob, Datum dat, uint32 i) { mfvtransval *transval = (mfvtransval *)VARDATA(transblob); bytea * tmpblob; size_t datumLen = ExtractDatumLen(dat, transval->typLen, transval->typByVal, -1); if (i > transval->next_mfv) elog( ERROR, "attempt to insert frequent value at illegal index %d in mfv sketch", i); if (MFV_TRANSVAL_CAPACITY(transblob) < datumLen) { /* allocate a copy with room for this, and double the current space for values */ size_t curspace = VARSIZE(transblob) - transval->mfvs[0].offset - VARHDRSZ; tmpblob = (bytea *)palloc0(VARSIZE(transblob) + curspace + datumLen); memmove(tmpblob, transblob, VARSIZE(transblob)); SET_VARSIZE(tmpblob, VARSIZE(transblob) + curspace + datumLen); /* * PG won't let us pfree the old transblob * pfree(transblob); */ transblob = tmpblob; transval = (mfvtransval *)VARDATA(transblob); } transval->mfvs[i].offset = transval->next_offset; mfv_copy_datum(transblob, i, dat); transval->next_offset += datumLen; return(transblob); }
/* check whether the content in the given bytea is safe for mfvtransval */ void check_mfvtransval(bytea *storage) { size_t left_len = VARSIZE(storage); size_t cur_size = 0; size_t cur_capacity = 0; Oid outFuncOid; bool typIsVarLen; mfvtransval *mfv = NULL; if (left_len < VARHDRSZ + sizeof(mfvtransval)) { elog(ERROR, "invalid transition state for mfvsketch"); } mfv = (mfvtransval*)VARDATA(storage); left_len -= VARHDRSZ + sizeof(mfvtransval); if (mfv->next_mfv > mfv->max_mfvs) { elog(ERROR, "invalid transition state for mfvsketch"); } if (mfv->next_offset + VARHDRSZ > VARSIZE(storage)) { elog(ERROR, "invalid transition state for mfvsketch"); } if (InvalidOid == mfv->typOid) { elog(ERROR, "invalid transition state for mfvsketch"); } getTypeOutputInfo(mfv->typOid, &outFuncOid, &typIsVarLen); if (mfv->outFuncOid != outFuncOid || mfv->typLen != get_typlen(mfv->typOid) || mfv->typByVal != get_typbyval(mfv->typOid)) { elog(ERROR, "invalid transition state for mfvsketch"); } if (left_len < sizeof(offsetcnt)*mfv->max_mfvs) { elog(ERROR, "invalid transition state for mfvsketch"); } /* offset is relative to mfvtransval */ left_len = VARSIZE(storage) - VARHDRSZ; /* * the following checking may be inefficiency, but by doing this centrally, * we can avoid spreading the checking code everywhere. */ for (unsigned i = 0; i < mfv->next_mfv; i ++) { cur_capacity = left_len - mfv->mfvs[i].offset; if (mfv->mfvs[i].offset > left_len) { elog(ERROR, "invalid transition state for mfvsketch"); } cur_size = ExtractDatumLen(PointerGetDatum(MFV_DATA(mfv) + mfv->mfvs[i].offset), mfv->typLen, mfv->typByVal, cur_capacity); if (cur_size > cur_capacity) { elog(ERROR, "invalid transition state for mfvsketch"); } } }
/*! * copy datum <c>dat</c> into the offset of position <c>index</c> of * the mfv sketch stored in <c>transblob</c>. * * <i>Caller beware: this helper return assumes that * <c>dat</c> is small enough to fit in the storage * currently used by the datum at position <c>index</c>.</i> * * \param transblob a bytea holding and mfv transval * \param index the index of the destination for copying * \param dat the datum to be copied into the transval */ void mfv_copy_datum(bytea *transblob, int index, Datum dat) { mfvtransval *transval = (mfvtransval *)VARDATA(transblob); size_t datumLen = ExtractDatumLen(dat, transval->typLen, transval->typByVal, -1); void * curval = (char*)transval + transval->mfvs[index].offset; memmove(curval, (void *)DatumExtractPointer(dat, transval->typByVal), datumLen); }
/*! * copy datum <c>dat</c> into the offset of position <c>index</c> of * the mfv sketch stored in <c>transblob</c>. * * <i>Caller beware: this helper return assumes that * <c>dat</c> is small enough to fit in the storage * currently used by the datum at position <c>index</c>.</i> * * \param transblob a bytea holding and mfv transval * \param index the index of the destination for copying * \param dat the datum to be copied into the transval */ void mfv_copy_datum(bytea *transblob, int index, Datum dat) { mfvtransval *transval = (mfvtransval *)VARDATA(transblob); size_t datumLen = ExtractDatumLen(dat, transval->typLen, transval->typByVal, -1); void * curval = mfv_transval_getval(transblob,index); memmove(curval, (void *)DatumExtractPointer(dat, transval->typByVal), datumLen); }
/*! * replace the value at position i of the mfvsketch with dat * * \param transblob the transition value packed into a bytea * \param dat the value to be inserted * \param i the position to replace */ bytea *mfv_transval_replace(bytea *transblob, Datum dat, int i) { /* * if new value is smaller than old, we overwrite at the old offset. * otherwise we call mfv_transval_insert_at which will take care of * space allocation for the new value */ mfvtransval *transval = (mfvtransval *)VARDATA(transblob); size_t datumLen = ExtractDatumLen(dat, transval->typLen, transval->typByVal, -1); void * tmpp = mfv_transval_getval(transblob,i); Datum oldDat = PointerExtractDatum(tmpp, transval->typByVal); size_t oldLen = ExtractDatumLen(oldDat, transval->typLen, transval->typByVal, -1); if (datumLen <= oldLen) { mfv_copy_datum(transblob, i, dat); return transblob; } else return(mfv_transval_insert_at(transblob, dat, i)); }
/*! * look to see if the mfvsketch currently has <c>val</c> * stored as one of its most-frequent values. * Returns the offset in the <c>mfvs</c> array, or -1 * if not found. * NOTE: a 0 return value means the item <i>was found</i> * at offset 0! * \param blob a bytea holding an mfv transval * \param val the datum to search for */ int mfv_find(bytea *blob, Datum val) { mfvtransval *transval = (mfvtransval *)VARDATA(blob); unsigned i; uint32 len; void * datp; Datum iDat; void *valp = DatumExtractPointer(val, transval->typByVal); /* look for existing entry for this value */ for (i = 0; i < transval->next_mfv; i++) { /* if they're the same */ datp = mfv_transval_getval(blob,i); iDat = PointerExtractDatum(datp, transval->typByVal); if ((len = ExtractDatumLen(iDat, transval->typLen, transval->typByVal, -1)) == ExtractDatumLen(val, transval->typLen, transval->typByVal, -1)) { if (!memcmp(datp, valp, len)) /* arg is an mfv */ return(i); } } return(-1); }
/*! * \param blob a bytea holding an mfv transval * \param i index of the mfv to look up * \returns pointer to the datum associated with the i'th mfv */ void *mfv_transval_getval(bytea *blob, uint32 i) { mfvtransval *tvp = (mfvtransval *)VARDATA(blob); void * retval = (void *)(((char*)tvp) + tvp->mfvs[i].offset); Datum dat = PointerExtractDatum(retval, tvp->typByVal); if (i > tvp->next_mfv) elog(ERROR, "attempt to get frequent value at illegal index %d in mfv sketch", i); if (tvp->mfvs[i].offset > VARSIZE(blob) - VARHDRSZ || tvp->mfvs[i].offset < MFV_TRANSVAL_SZ(tvp->max_mfvs)-VARHDRSZ) elog(ERROR, "illegal offset %u in mfv sketch", tvp->mfvs[i].offset); if (tvp->mfvs[i].offset + ExtractDatumLen(dat, tvp->typLen, tvp->typByVal, -1) > VARSIZE(blob) - VARHDRSZ) elog(ERROR, "value overruns size of mfv sketch"); return (retval); }
/*! * \param blob a bytea holding an mfv transval * \param i index of the mfv to look up * \returns pointer to the datum associated with the i'th mfv */ void *mfv_transval_getval(bytea *blob, uint32 i) { mfvtransval *tvp = (mfvtransval *)VARDATA(blob); void * retval = (void *)(((char*)tvp) + tvp->mfvs[i].offset); Datum dat = PointerExtractDatum(retval, tvp->typByVal); if (i >= tvp->next_mfv) elog(ERROR, "attempt to get frequent value at illegal index %d in mfv sketch", i); if (tvp->mfvs[i].offset > VARSIZE(blob) - VARHDRSZ || tvp->mfvs[i].offset < MFV_TRANSVAL_SZ(tvp->max_mfvs)-VARHDRSZ) elog(ERROR, "illegal offset %u in mfv sketch", tvp->mfvs[i].offset); /* * call ExtractDatumLen to make sure enough space, this checking is unnecessary, * it is used to prevent gcc from optimizing out the ExtractDatumLen function call. */ if (tvp->mfvs[i].offset + ExtractDatumLen(dat, tvp->typLen, tvp->typByVal, VARSIZE(blob) - VARHDRSZ - tvp->mfvs[i].offset) > VARSIZE(blob) - VARHDRSZ) elog(ERROR, "value overruns size of mfv sketch"); return (retval); }
/*! UDA transition function for the fmsketch aggregate. */ Datum __fmsketch_trans(PG_FUNCTION_ARGS) { bytea * transblob = (bytea *)PG_GETARG_BYTEA_P(0); fmtransval *transval; Oid element_type = get_fn_expr_argtype(fcinfo->flinfo, 1); Oid funcOid; bool typIsVarlena; Datum retval; Datum inval; if (!OidIsValid(element_type)) elog(ERROR, "could not determine data type of input"); /* * This is Postgres boilerplate for UDFs that modify the data in their own context. * Such UDFs can only be correctly called in an agg context since regular scalar * UDFs are essentially stateless across invocations. */ if (!(fcinfo->context && (IsA(fcinfo->context, AggState) #ifdef NOTGP || IsA(fcinfo->context, WindowAggState) #endif ))) elog( ERROR, "UDF call to a function that only works for aggs (destructive pass by reference)"); /* get the provided element, being careful in case it's NULL */ if (!PG_ARGISNULL(1)) { inval = PG_GETARG_DATUM(1); /* * if this is the first call, initialize transval to hold a sortasort * on the first call, we should have the empty string (if the agg was declared properly!) */ if (VARSIZE(transblob) <= VARHDRSZ) { size_t blobsz = VARHDRSZ + sizeof(fmtransval) + SORTASORT_INITIAL_STORAGE; transblob = (bytea *)palloc0(blobsz); SET_VARSIZE(transblob, blobsz); transval = (fmtransval *)VARDATA(transblob); transval->typOid = element_type; /* figure out the outfunc for this type */ getTypeOutputInfo(element_type, &funcOid, &typIsVarlena); get_typlenbyval(element_type, &(transval->typLen), &(transval->typByVal)); transval->status = SMALL; sortasort_init((sortasort *)transval->storage, MINVALS, SORTASORT_INITIAL_STORAGE, transval->typLen, transval->typByVal); } else { check_fmtransval(transblob); /* extract the existing transval from the transblob */ transval = (fmtransval *)VARDATA(transblob); if (transval->typOid != element_type) { elog(ERROR, "cannot aggregate on elements with different types"); } } /* * if we've seen < MINVALS distinct values, place datum into the sortasort * XXXX Would be cleaner to try the sortasort insert and if it fails, then continue. */ if (transval->status == SMALL && ((sortasort *)(transval->storage))->num_vals < MINVALS) { int len = ExtractDatumLen(inval, transval->typLen, transval->typByVal, -1); retval = PointerGetDatum(fmsketch_sortasort_insert( transblob, inval, len)); PG_RETURN_DATUM(retval); } /* * if we've seen exactly MINVALS distinct values, create FM bitmaps * and load the contents of the sortasort into the FM sketch */ else if (transval->status == SMALL && ((sortasort *)(transval->storage))->num_vals == MINVALS) { int i; sortasort *s = (sortasort *)(transval->storage); bytea *newblob = fm_new(transval); transval = (fmtransval *)VARDATA(newblob); /* * "catch up" on the past as if we were doing FM from the beginning: * apply the FM sketching algorithm to each value previously stored in the sortasort */ for (i = 0; i < MINVALS; i++) __fmsketch_trans_c(newblob, PointerExtractDatum(sortasort_getval(s,i), s->typByVal)); /* * XXXX would like to pfree the old transblob, but the memory allocator doesn't like it * XXXX Meanwhile we know that this memory "leak" is of fixed size and will get * XXXX deallocated "soon" when the memory context is destroyed. */ /* drop through to insert the current datum in "BIG" mode */ transblob = newblob; } /* * if we're here we've seen >=MINVALS distinct values and are in BIG mode. * Just for sanity, let's check. */ if (transval->status != BIG) elog( ERROR, "FM sketch failed internal sanity check"); /* Apply FM algorithm to this datum */ retval = __fmsketch_trans_c(transblob, inval); PG_RETURN_DATUM(retval); } else PG_RETURN_NULL(); }