StreamBatch *StreamBatchCreate(Bitmapset *readers, int num_tuples)
{
char *ptr = ShmemDynAlloc0(sizeof(StreamBatch) + BITMAPSET_SIZE(readers->nwords) + (bms_num_members(readers) * sizeof(int)));
StreamBatch *batch = (StreamBatch *) ptr;
int cq_id;
int i = 0;
batch->id = rand() ^ (int) MyProcPid;
batch->num_tups = num_tuples;
batch->num_wtups = bms_num_members(readers) * num_tuples;
SpinLockInit(&batch->mutex);
ptr += sizeof(StreamBatch);
batch->readers = (Bitmapset *) ptr;
memcpy(batch->readers, readers, BITMAPSET_SIZE(readers->nwords));
ptr += BITMAPSET_SIZE(readers->nwords);
batch->proc_runs = (int *) ptr;
readers = bms_copy(readers);
while ((cq_id = bms_first_member(readers)) != -1)
{
CQProcEntry *pentry = GetCQProcEntry(cq_id);
batch->proc_runs[i] = Max(pentry->proc_runs, pentry->pg_size);
i++;
}
pfree(readers);
return batch;
}
void StreamBatchWaitAndRemove(StreamBatch *batch)
{
int cycle = 0;
while (!StreamBatchAllAcked(batch))
{
if (cycle % CHECK_CRASH_CYCLES == 0)
{
int num_crashes = num_cq_crashes(batch);
cycle = 0;
if (num_crashes == 0)
continue;
// All tuples have been read, and we've received acks from all workers that didn't crash.
if (StreamBatchAllRead(batch) &&
batch->num_wacks >= (batch->num_tups * (bms_num_members(batch->readers) - num_crashes)))
break;
}
pg_usleep(SLEEP_MS * 1000);
cycle++;
}
ShmemDynFree(batch);
}
/*
* statext_ndistinct_build
* Compute ndistinct coefficient for the combination of attributes.
*
* This computes the ndistinct estimate using the same estimator used
* in analyze.c and then computes the coefficient.
*/
MVNDistinct *
statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
Bitmapset *attrs, VacAttrStats **stats)
{
MVNDistinct *result;
int k;
int itemcnt;
int numattrs = bms_num_members(attrs);
int numcombs = num_combinations(numattrs);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
result->magic = STATS_NDISTINCT_MAGIC;
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
itemcnt = 0;
for (k = 2; k <= numattrs; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
generator = generator_init(numattrs, k);
while ((combination = generator_next(generator)))
{
MVNDistinctItem *item = &result->items[itemcnt];
int j;
item->attrs = NULL;
for (j = 0; j < k; j++)
item->attrs = bms_add_member(item->attrs,
stats[combination[j]]->attr->attnum);
item->ndistinct =
ndistinct_for_combination(totalrows, numrows, rows,
stats, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
}
generator_free(generator);
}
/* must consume exactly the whole output array */
Assert(itemcnt == result->nitems);
return result;
}
/*
* find_strongest_dependency
* find the strongest dependency on the attributes
*
* When applying functional dependencies, we start with the strongest
* dependencies. That is, we select the dependency that:
*
* (a) has all attributes covered by equality clauses
*
* (b) has the most attributes
*
* (c) has the highest degree of validity
*
* This guarantees that we eliminate the most redundant conditions first
* (see the comment in dependencies_clauselist_selectivity).
*/
static MVDependency *
find_strongest_dependency(StatisticExtInfo *stats, MVDependencies *dependencies,
Bitmapset *attnums)
{
int i;
MVDependency *strongest = NULL;
/* number of attnums in clauses */
int nattnums = bms_num_members(attnums);
/*
* Iterate over the MVDependency items and find the strongest one from the
* fully-matched dependencies. We do the cheap checks first, before
* matching it against the attnums.
*/
for (i = 0; i < dependencies->ndeps; i++)
{
MVDependency *dependency = dependencies->deps[i];
/*
* Skip dependencies referencing more attributes than available
* clauses, as those can't be fully matched.
*/
if (dependency->nattributes > nattnums)
continue;
if (strongest)
{
/* skip dependencies on fewer attributes than the strongest. */
if (dependency->nattributes < strongest->nattributes)
continue;
/* also skip weaker dependencies when attribute count matches */
if (strongest->nattributes == dependency->nattributes &&
strongest->degree > dependency->degree)
continue;
}
/*
* this dependency is stronger, but we must still check that it's
* fully matched to these attnums. We perform this check last as it's
* slightly more expensive than the previous checks.
*/
if (dependency_is_fully_matched(dependency, attnums))
strongest = dependency; /* save new best match */
}
return strongest;
}
/*
* statext_ndistinct_serialize
* serialize ndistinct to the on-disk bytea format
*/
bytea *
statext_ndistinct_serialize(MVNDistinct *ndistinct)
{
int i;
bytea *output;
char *tmp;
Size len;
Assert(ndistinct->magic == STATS_NDISTINCT_MAGIC);
Assert(ndistinct->type == STATS_NDISTINCT_TYPE_BASIC);
/*
* Base size is size of scalar fields in the struct, plus one base struct
* for each item, including number of items for each.
*/
len = VARHDRSZ + SizeOfMVNDistinct +
ndistinct->nitems * (offsetof(MVNDistinctItem, attrs) + sizeof(int));
/* and also include space for the actual attribute numbers */
for (i = 0; i < ndistinct->nitems; i++)
{
int nmembers;
nmembers = bms_num_members(ndistinct->items[i].attrs);
Assert(nmembers >= 2);
len += sizeof(AttrNumber) * nmembers;
}
output = (bytea *) palloc(len);
SET_VARSIZE(output, len);
tmp = VARDATA(output);
/* Store the base struct values (magic, type, nitems) */
memcpy(tmp, &ndistinct->magic, sizeof(uint32));
tmp += sizeof(uint32);
memcpy(tmp, &ndistinct->type, sizeof(uint32));
tmp += sizeof(uint32);
memcpy(tmp, &ndistinct->nitems, sizeof(uint32));
tmp += sizeof(uint32);
/*
* store number of attributes and attribute numbers for each ndistinct
* entry
*/
for (i = 0; i < ndistinct->nitems; i++)
{
MVNDistinctItem item = ndistinct->items[i];
int nmembers = bms_num_members(item.attrs);
int x;
memcpy(tmp, &item.ndistinct, sizeof(double));
tmp += sizeof(double);
memcpy(tmp, &nmembers, sizeof(int));
tmp += sizeof(int);
x = -1;
while ((x = bms_next_member(item.attrs, x)) >= 0)
{
AttrNumber value = (AttrNumber) x;
memcpy(tmp, &value, sizeof(AttrNumber));
tmp += sizeof(AttrNumber);
}
Assert(tmp <= ((char *) output + len));
}
return output;
}
/*
* detects functional dependencies between groups of columns
*
* Generates all possible subsets of columns (variations) and computes
* the degree of validity for each one. For example when creating statistics
* on three columns (a,b,c) there are 9 possible dependencies
*
* two columns three columns
* ----------- -------------
* (a) -> b (a,b) -> c
* (a) -> c (a,c) -> b
* (b) -> a (b,c) -> a
* (b) -> c
* (c) -> a
* (c) -> b
*/
MVDependencies *
statext_dependencies_build(int numrows, HeapTuple *rows, Bitmapset *attrs,
VacAttrStats **stats)
{
int i,
j,
k;
int numattrs;
int *attnums;
/* result */
MVDependencies *dependencies = NULL;
numattrs = bms_num_members(attrs);
/*
* Transform the bms into an array, to make accessing i-th member easier.
*/
attnums = palloc(sizeof(int) * bms_num_members(attrs));
i = 0;
j = -1;
while ((j = bms_next_member(attrs, j)) >= 0)
attnums[i++] = j;
Assert(numattrs >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
* covering just 2 columns, to the largest ones, covering all columns
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
for (k = 2; k <= numattrs; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
{
double degree;
MVDependency *d;
/* compute how valid the dependency seems */
degree = dependency_degree(numrows, rows, k, dependency, stats, attrs);
/*
* if the dependency seems entirely invalid, don't store it
*/
if (degree == 0.0)
continue;
d = (MVDependency *) palloc0(offsetof(MVDependency, attributes)
+ k * sizeof(AttrNumber));
/* copy the dependency (and keep the indexes into stxkeys) */
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
d->attributes[i] = attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
{
dependencies
= (MVDependencies *) palloc0(sizeof(MVDependencies));
dependencies->magic = STATS_DEPS_MAGIC;
dependencies->type = STATS_DEPS_TYPE_BASIC;
dependencies->ndeps = 0;
}
dependencies->ndeps++;
dependencies = (MVDependencies *) repalloc(dependencies,
offsetof(MVDependencies, deps)
+ dependencies->ndeps * sizeof(MVDependency));
dependencies->deps[dependencies->ndeps - 1] = d;
}
/*
* we're done with variations of k elements, so free the
* DependencyGenerator
*/
DependencyGenerator_free(DependencyGenerator);
}
return dependencies;
}
/*
* validates functional dependency on the data
*
* An actual work horse of detecting functional dependencies. Given a variation
* of k attributes, it checks that the first (k-1) are sufficient to determine
* the last one.
*/
static double
dependency_degree(int numrows, HeapTuple *rows, int k, AttrNumber *dependency,
VacAttrStats **stats, Bitmapset *attrs)
{
int i,
j;
int nvalues = numrows * k;
MultiSortSupport mss;
SortItem *items;
Datum *values;
bool *isnull;
int *attnums;
/* counters valid within a group */
int group_size = 0;
int n_violations = 0;
/* total number of rows supporting (consistent with) the dependency */
int n_supporting_rows = 0;
/* Make sure we have at least two input attributes. */
Assert(k >= 2);
/* sort info for all attributes columns */
mss = multi_sort_init(k);
/* data for the sort */
items = (SortItem *) palloc(numrows * sizeof(SortItem));
values = (Datum *) palloc(sizeof(Datum) * nvalues);
isnull = (bool *) palloc(sizeof(bool) * nvalues);
/* fix the pointers to values/isnull */
for (i = 0; i < numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
}
/*
* Transform the bms into an array, to make accessing i-th member easier.
*/
attnums = (int *) palloc(sizeof(int) * bms_num_members(attrs));
i = 0;
j = -1;
while ((j = bms_next_member(attrs, j)) >= 0)
attnums[i++] = j;
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
*
* (a) sort the data lexicographically
*
* (b) split the data into groups by first (k-1) columns
*
* (c) for each group count different values in the last column
*/
/* prepare the sort function for the first dimension, and SortItem array */
for (i = 0; i < k; i++)
{
VacAttrStats *colstat = stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
elog(ERROR, "cache lookup failed for ordering operator for type %u",
colstat->attrtypid);
/* prepare the sort function for this dimension */
multi_sort_add_dimension(mss, i, type->lt_opr);
/* accumulate all the data for both columns into an array and sort it */
for (j = 0; j < numrows; j++)
{
items[j].values[i] =
heap_getattr(rows[j], attnums[dependency[i]],
stats[i]->tupDesc, &items[j].isnull[i]);
}
}
/* sort the items so that we can detect the groups */
qsort_arg((void *) items, numrows, sizeof(SortItem),
multi_sort_compare, mss);
/*
* Walk through the sorted array, split it into rows according to the
* first (k-1) columns. If there's a single value in the last column, we
* count the group as 'supporting' the functional dependency. Otherwise we
* count it as contradicting.
*/
/* start with the first row forming a group */
group_size = 1;
/* loop 1 beyond the end of the array so that we count the final group */
for (i = 1; i <= numrows; i++)
{
/*
* Check if the group ended, which may be either because we processed
* all the items (i==numrows), or because the i-th item is not equal
* to the preceding one.
*/
if (i == numrows ||
multi_sort_compare_dims(0, k - 2, &items[i - 1], &items[i], mss) != 0)
{
/*
* If no violations were found in the group then track the rows of
* the group as supporting the functional dependency.
*/
if (n_violations == 0)
n_supporting_rows += group_size;
/* Reset counters for the new group */
n_violations = 0;
group_size = 1;
continue;
}
/* first columns match, but the last one does not (so contradicting) */
else if (multi_sort_compare_dim(k - 1, &items[i - 1], &items[i], mss) != 0)
n_violations++;
group_size++;
}
pfree(items);
pfree(values);
pfree(isnull);
pfree(mss);
/* Compute the 'degree of validity' as (supporting/total). */
return (n_supporting_rows * 1.0 / numrows);
}