/*
* _bt_mergeload - Merge two streams of index tuples into new index files.
*/
static void
_bt_mergeload(Spooler *self, BTWriteState *wstate, BTSpool *btspool, BTReader *btspool2, Relation heapRel)
{
BTPageState *state = NULL;
IndexTuple itup,
itup2;
bool should_free = false;
TupleDesc tupdes = RelationGetDescr(wstate->index);
int keysz = RelationGetNumberOfAttributes(wstate->index);
ScanKey indexScanKey;
ON_DUPLICATE on_duplicate = self->on_duplicate;
Assert(btspool != NULL);
/* the preparation of merge */
itup = BTSpoolGetNextItem(btspool, NULL, &should_free);
itup2 = BTReaderGetNextItem(btspool2);
indexScanKey = _bt_mkscankey_nodata(wstate->index);
for (;;)
{
bool load1 = true; /* load BTSpool next ? */
bool hasnull;
int32 compare;
if (self->dup_old + self->dup_new > self->max_dup_errors)
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("Maximum duplicate error count exceeded")));
if (itup2 == NULL)
{
if (itup == NULL)
break;
}
else if (itup != NULL)
{
compare = compare_indextuple(itup, itup2, indexScanKey,
keysz, tupdes, &hasnull);
if (compare == 0 && !hasnull && btspool->isunique)
{
ItemPointerData t_tid2;
/*
* t_tid is update by heap_is_visible(), because use it for an
* index, t_tid backup
*/
ItemPointerCopy(&itup2->t_tid, &t_tid2);
/* The tuple pointed by the old index should not be visible. */
if (!heap_is_visible(heapRel, &itup->t_tid))
{
itup = BTSpoolGetNextItem(btspool, itup, &should_free);
}
else if (!heap_is_visible(heapRel, &itup2->t_tid))
{
itup2 = BTReaderGetNextItem(btspool2);
}
else
{
if (on_duplicate == ON_DUPLICATE_KEEP_NEW)
{
self->dup_old++;
remove_duplicate(self, heapRel, itup2,
RelationGetRelationName(wstate->index));
itup2 = BTReaderGetNextItem(btspool2);
}
else
{
ItemPointerCopy(&t_tid2, &itup2->t_tid);
self->dup_new++;
remove_duplicate(self, heapRel, itup,
RelationGetRelationName(wstate->index));
itup = BTSpoolGetNextItem(btspool, itup, &should_free);
}
}
continue;
}
else if (compare > 0)
load1 = false;
}
else
load1 = false;
BULKLOAD_PROFILE(&prof_merge_unique);
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
if (load1)
{
IndexTuple next_itup = NULL;
bool next_should_free = false;
for (;;)
{
/* get next item */
next_itup = BTSpoolGetNextItem(btspool, next_itup,
&next_should_free);
if (!btspool->isunique || next_itup == NULL)
break;
compare = compare_indextuple(itup, next_itup, indexScanKey,
keysz, tupdes, &hasnull);
if (compare < 0 || hasnull)
break;
if (compare > 0)
{
/* shouldn't happen */
elog(ERROR, "faild in tuplesort_performsort");
}
/*
* If tupple is deleted by other unique indexes, not visible
*/
if (!heap_is_visible(heapRel, &next_itup->t_tid))
{
continue;
}
if (!heap_is_visible(heapRel, &itup->t_tid))
{
if (should_free)
pfree(itup);
itup = next_itup;
should_free = next_should_free;
next_should_free = false;
continue;
}
/* not unique between input files */
self->dup_new++;
remove_duplicate(self, heapRel, next_itup,
RelationGetRelationName(wstate->index));
if (self->dup_old + self->dup_new > self->max_dup_errors)
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("Maximum duplicate error count exceeded")));
}
_bt_buildadd(wstate, state, itup);
if (should_free)
pfree(itup);
itup = next_itup;
should_free = next_should_free;
}
else
{
_bt_buildadd(wstate, state, itup2);
itup2 = BTReaderGetNextItem(btspool2);
}
BULKLOAD_PROFILE(&prof_merge_insert);
}
_bt_freeskey(indexScanKey);
/* Close down final pages and write the metapage */
_bt_uppershutdown(wstate, state);
/*
* If the index isn't temp, we must fsync it down to disk before it's safe
* to commit the transaction. (For a temp index we don't care since the
* index will be uninteresting after a crash anyway.)
*
* It's obvious that we must do this when not WAL-logging the build. It's
* less obvious that we have to do it even if we did WAL-log the index
* pages. The reason is that since we're building outside shared buffers,
* a CHECKPOINT occurring during the build has no way to flush the
* previously written data to disk (indeed it won't know the index even
* exists). A crash later on would replay WAL from the checkpoint,
* therefore it wouldn't replay our earlier WAL entries. If we do not
* fsync those pages here, they might still not be on disk when the crash
* occurs.
*/
if (!RELATION_IS_LOCAL(wstate->index))
{
RelationOpenSmgr(wstate->index);
smgrimmedsync(wstate->index->rd_smgr, MAIN_FORKNUM);
}
BULKLOAD_PROFILE(&prof_merge_term);
}
/*
* Read tuples in correct sort order from tuplesort, and load them into
* btree leaves.
*/
static void
_bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
{
BTPageState *state = NULL;
bool merge = (btspool2 != NULL);
IndexTuple itup,
itup2 = NULL;
bool load1;
TupleDesc tupdes = RelationGetDescr(wstate->index);
int i,
keysz = RelationGetNumberOfAttributes(wstate->index);
ScanKey indexScanKey = NULL;
SortSupport sortKeys;
if (merge)
{
/*
* Another BTSpool for dead tuples exists. Now we have to merge
* btspool and btspool2.
*/
/* the preparation of merge */
itup = tuplesort_getindextuple(btspool->sortstate, true);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
indexScanKey = _bt_mkscankey_nodata(wstate->index);
/* Prepare SortSupport data for each column */
sortKeys = (SortSupport) palloc0(keysz * sizeof(SortSupportData));
for (i = 0; i < keysz; i++)
{
SortSupport sortKey = sortKeys + i;
ScanKey scanKey = indexScanKey + i;
int16 strategy;
sortKey->ssup_cxt = CurrentMemoryContext;
sortKey->ssup_collation = scanKey->sk_collation;
sortKey->ssup_nulls_first =
(scanKey->sk_flags & SK_BT_NULLS_FIRST) != 0;
sortKey->ssup_attno = scanKey->sk_attno;
/* Abbreviation is not supported here */
sortKey->abbreviate = false;
AssertState(sortKey->ssup_attno != 0);
strategy = (scanKey->sk_flags & SK_BT_DESC) != 0 ?
BTGreaterStrategyNumber : BTLessStrategyNumber;
PrepareSortSupportFromIndexRel(wstate->index, strategy, sortKey);
}
_bt_freeskey(indexScanKey);
for (;;)
{
load1 = true; /* load BTSpool next ? */
if (itup2 == NULL)
{
if (itup == NULL)
break;
}
else if (itup != NULL)
{
for (i = 1; i <= keysz; i++)
{
SortSupport entry;
Datum attrDatum1,
attrDatum2;
bool isNull1,
isNull2;
int32 compare;
entry = sortKeys + i - 1;
attrDatum1 = index_getattr(itup, i, tupdes, &isNull1);
attrDatum2 = index_getattr(itup2, i, tupdes, &isNull2);
compare = ApplySortComparator(attrDatum1, isNull1,
attrDatum2, isNull2,
entry);
if (compare > 0)
{
load1 = false;
break;
}
else if (compare < 0)
break;
}
}
else
load1 = false;
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
if (load1)
{
_bt_buildadd(wstate, state, itup);
itup = tuplesort_getindextuple(btspool->sortstate, true);
}
else
{
_bt_buildadd(wstate, state, itup2);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
}
}
pfree(sortKeys);
}
else
{
/* merge is unnecessary */
while ((itup = tuplesort_getindextuple(btspool->sortstate,
true)) != NULL)
{
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
_bt_buildadd(wstate, state, itup);
}
}
/* Close down final pages and write the metapage */
_bt_uppershutdown(wstate, state);
/*
* If the index is WAL-logged, we must fsync it down to disk before it's
* safe to commit the transaction. (For a non-WAL-logged index we don't
* care since the index will be uninteresting after a crash anyway.)
*
* It's obvious that we must do this when not WAL-logging the build. It's
* less obvious that we have to do it even if we did WAL-log the index
* pages. The reason is that since we're building outside shared buffers,
* a CHECKPOINT occurring during the build has no way to flush the
* previously written data to disk (indeed it won't know the index even
* exists). A crash later on would replay WAL from the checkpoint,
* therefore it wouldn't replay our earlier WAL entries. If we do not
* fsync those pages here, they might still not be on disk when the crash
* occurs.
*/
if (RelationNeedsWAL(wstate->index))
{
RelationOpenSmgr(wstate->index);
smgrimmedsync(wstate->index->rd_smgr, MAIN_FORKNUM);
}
}
/*
* Read tuples in correct sort order from tuplesort, and load them into
* btree leaves.
*/
static void
_bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
{
BTPageState *state = NULL;
bool merge = (btspool2 != NULL);
IndexTuple itup,
itup2 = NULL;
bool should_free,
should_free2,
load1;
TupleDesc tupdes = RelationGetDescr(wstate->index);
int i,
keysz = RelationGetNumberOfAttributes(wstate->index);
ScanKey indexScanKey = NULL;
if (merge)
{
/*
* Another BTSpool for dead tuples exists. Now we have to merge
* btspool and btspool2.
*/
/* the preparation of merge */
itup = tuplesort_getindextuple(btspool->sortstate,
true, &should_free);
itup2 = tuplesort_getindextuple(btspool2->sortstate,
true, &should_free2);
indexScanKey = _bt_mkscankey_nodata(wstate->index);
for (;;)
{
load1 = true; /* load BTSpool next ? */
if (itup2 == NULL)
{
if (itup == NULL)
break;
}
else if (itup != NULL)
{
for (i = 1; i <= keysz; i++)
{
ScanKey entry;
Datum attrDatum1,
attrDatum2;
bool isNull1,
isNull2;
int32 compare;
entry = indexScanKey + i - 1;
attrDatum1 = index_getattr(itup, i, tupdes, &isNull1);
attrDatum2 = index_getattr(itup2, i, tupdes, &isNull2);
if (isNull1)
{
if (isNull2)
compare = 0; /* NULL "=" NULL */
else if (entry->sk_flags & SK_BT_NULLS_FIRST)
compare = -1; /* NULL "<" NOT_NULL */
else
compare = 1; /* NULL ">" NOT_NULL */
}
else if (isNull2)
{
if (entry->sk_flags & SK_BT_NULLS_FIRST)
compare = 1; /* NOT_NULL ">" NULL */
else
compare = -1; /* NOT_NULL "<" NULL */
}
else
{
compare =
DatumGetInt32(FunctionCall2Coll(&entry->sk_func,
entry->sk_collation,
attrDatum1,
attrDatum2));
if (entry->sk_flags & SK_BT_DESC)
compare = -compare;
}
if (compare > 0)
{
load1 = false;
break;
}
else if (compare < 0)
break;
}
}
else
load1 = false;
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
if (load1)
{
_bt_buildadd(wstate, state, itup);
if (should_free)
pfree(itup);
itup = tuplesort_getindextuple(btspool->sortstate,
true, &should_free);
}
else
{
_bt_buildadd(wstate, state, itup2);
if (should_free2)
pfree(itup2);
itup2 = tuplesort_getindextuple(btspool2->sortstate,
true, &should_free2);
}
}
_bt_freeskey(indexScanKey);
}
else
{
/* merge is unnecessary */
while ((itup = tuplesort_getindextuple(btspool->sortstate,
true, &should_free)) != NULL)
{
/* When we see first tuple, create first index page */
if (state == NULL)
state = _bt_pagestate(wstate, 0);
_bt_buildadd(wstate, state, itup);
if (should_free)
pfree(itup);
}
}
/* Close down final pages and write the metapage */
_bt_uppershutdown(wstate, state);
/*
* If the index is WAL-logged, we must fsync it down to disk before it's
* safe to commit the transaction. (For a non-WAL-logged index we don't
* care since the index will be uninteresting after a crash anyway.)
*
* It's obvious that we must do this when not WAL-logging the build. It's
* less obvious that we have to do it even if we did WAL-log the index
* pages. The reason is that since we're building outside shared buffers,
* a CHECKPOINT occurring during the build has no way to flush the
* previously written data to disk (indeed it won't know the index even
* exists). A crash later on would replay WAL from the checkpoint,
* therefore it wouldn't replay our earlier WAL entries. If we do not
* fsync those pages here, they might still not be on disk when the crash
* occurs.
*/
if (RelationNeedsWAL(wstate->index))
{
RelationOpenSmgr(wstate->index);
smgrimmedsync(wstate->index->rd_smgr, MAIN_FORKNUM);
}
}