/* ----------
* toast_fetch_datum_slice -
*
* Reconstruct a segment of a Datum from the chunks saved
* in the toast relation
* ----------
*/
static struct varlena *
toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length)
{
Relation toastrel;
Relation toastidx;
ScanKeyData toastkey[3];
int nscankeys;
SysScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
struct varlena *result;
struct varatt_external toast_pointer;
int32 attrsize;
int32 residx;
int32 nextidx;
int numchunks;
int startchunk;
int endchunk;
int32 startoffset;
int32 endoffset;
int totalchunks;
Pointer chunk;
bool isnull;
char *chunkdata;
int32 chunksize;
int32 chcpystrt;
int32 chcpyend;
Assert(VARATT_IS_EXTERNAL(attr));
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/*
* It's nonsense to fetch slices of a compressed datum -- this isn't lo_*
* we can't return a compressed datum which is meaningful to toast later
*/
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
attrsize = toast_pointer.va_extsize;
totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
length = 0;
}
if (((sliceoffset + length) > attrsize) || length < 0)
length = attrsize - sliceoffset;
result = (struct varlena *) palloc(length + VARHDRSZ);
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
SET_VARSIZE_COMPRESSED(result, length + VARHDRSZ);
else
SET_VARSIZE(result, length + VARHDRSZ);
if (length == 0)
return result; /* Can save a lot of work at this point! */
startchunk = sliceoffset / TOAST_MAX_CHUNK_SIZE;
endchunk = (sliceoffset + length - 1) / TOAST_MAX_CHUNK_SIZE;
numchunks = (endchunk - startchunk) + 1;
startoffset = sliceoffset % TOAST_MAX_CHUNK_SIZE;
endoffset = (sliceoffset + length - 1) % TOAST_MAX_CHUNK_SIZE;
/*
* Open the toast relation and its index
*/
toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
* Setup a scan key to fetch from the index. This is either two keys or
* three depending on the number of chunks.
*/
ScanKeyInit(&toastkey[0],
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(toast_pointer.va_valueid));
/*
* Use equality condition for one chunk, a range condition otherwise:
*/
if (numchunks == 1)
{
ScanKeyInit(&toastkey[1],
(AttrNumber) 2,
BTEqualStrategyNumber, F_INT4EQ,
Int32GetDatum(startchunk));
nscankeys = 2;
}
else
{
ScanKeyInit(&toastkey[1],
(AttrNumber) 2,
BTGreaterEqualStrategyNumber, F_INT4GE,
Int32GetDatum(startchunk));
ScanKeyInit(&toastkey[2],
(AttrNumber) 2,
BTLessEqualStrategyNumber, F_INT4LE,
Int32GetDatum(endchunk));
nscankeys = 3;
}
/*
* Read the chunks by index
*
* The index is on (valueid, chunkidx) so they will come in order
*/
nextidx = startchunk;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
SnapshotToast, nscankeys, toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
/*
* Have a chunk, extract the sequence number and the data
*/
residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
if (!VARATT_IS_EXTENDED(chunk))
{
chunksize = VARSIZE(chunk) - VARHDRSZ;
chunkdata = VARDATA(chunk);
}
else if (VARATT_IS_SHORT(chunk))
{
/* could happen due to heap_form_tuple doing its thing */
chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
chunkdata = VARDATA_SHORT(chunk);
}
else
{
/* should never happen */
elog(ERROR, "found toasted toast chunk for toast value %u in %s",
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
chunksize = 0; /* keep compiler quiet */
chunkdata = NULL;
}
/*
* Some checks on the data we've found
*/
if ((residx != nextidx) || (residx > endchunk) || (residx < startchunk))
elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u in %s",
residx, nextidx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
if (residx < totalchunks - 1)
{
if (chunksize != TOAST_MAX_CHUNK_SIZE)
elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s when fetching slice",
chunksize, (int) TOAST_MAX_CHUNK_SIZE,
residx, totalchunks,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
}
else if (residx == totalchunks - 1)
{
if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != attrsize)
elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u in %s when fetching slice",
chunksize,
(int) (attrsize - residx * TOAST_MAX_CHUNK_SIZE),
residx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
}
else
elog(ERROR, "unexpected chunk number %d (out of range %d..%d) for toast value %u in %s",
residx,
0, totalchunks - 1,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* Copy the data into proper place in our result
*/
chcpystrt = 0;
chcpyend = chunksize - 1;
if (residx == startchunk)
chcpystrt = startoffset;
if (residx == endchunk)
chcpyend = endoffset;
memcpy(VARDATA(result) +
(residx * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt,
chunkdata + chcpystrt,
(chcpyend - chcpystrt) + 1);
nextidx++;
}
/*
* Final checks that we successfully fetched the datum
*/
if (nextidx != (endchunk + 1))
elog(ERROR, "missing chunk number %d for toast value %u in %s",
nextidx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* End scan and close relations
*/
systable_endscan_ordered(toastscan);
index_close(toastidx, AccessShareLock);
heap_close(toastrel, AccessShareLock);
return result;
}
/* ----------
* heap_tuple_untoast_attr_slice -
*
* Public entry point to get back part of a toasted value
* from compression or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr_slice(struct varlena * attr,
int32 sliceoffset, int32 slicelength)
{
struct varlena *preslice;
struct varlena *result;
char *attrdata;
int32 attrsize;
if (VARATT_IS_EXTERNAL(attr))
{
struct varatt_external toast_pointer;
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/* fetch it back (compressed marker will get set automatically) */
preslice = toast_fetch_datum(attr);
}
else
preslice = attr;
if (VARATT_IS_COMPRESSED(preslice))
{
PGLZ_Header *tmp = (PGLZ_Header *) preslice;
Size size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ;
preslice = (struct varlena *) palloc(size);
SET_VARSIZE(preslice, size);
pglz_decompress(tmp, VARDATA(preslice));
if (tmp != (PGLZ_Header *) attr)
pfree(tmp);
}
if (VARATT_IS_SHORT(preslice))
{
attrdata = VARDATA_SHORT(preslice);
attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
}
else
{
attrdata = VARDATA(preslice);
attrsize = VARSIZE(preslice) - VARHDRSZ;
}
/* slicing of datum for compressed cases and plain value */
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
slicelength = 0;
}
if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
slicelength = attrsize - sliceoffset;
result = (struct varlena *) palloc(slicelength + VARHDRSZ);
SET_VARSIZE(result, slicelength + VARHDRSZ);
memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
if (preslice != attr)
pfree(preslice);
return result;
}
/* ----------
* toast_save_datum -
*
* Save one single datum into the secondary relation and return
* a Datum reference for it.
* ----------
*/
static Datum
toast_save_datum(Relation rel, Datum value, int options)
{
Relation toastrel;
Relation toastidx;
HeapTuple toasttup;
TupleDesc toasttupDesc;
Datum t_values[3];
bool t_isnull[3];
CommandId mycid = GetCurrentCommandId(true);
struct varlena *result;
struct varatt_external toast_pointer;
struct
{
struct varlena hdr;
char data[TOAST_MAX_CHUNK_SIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
/*
* Open the toast relation and its index. We can use the index to check
* uniqueness of the OID we assign to the toasted item, even though it has
* additional columns besides OID.
*/
toastrel = heap_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock);
/*
* Get the data pointer and length, and compute va_rawsize and va_extsize.
*
* va_rawsize is the size of the equivalent fully uncompressed datum, so
* we have to adjust for short headers.
*
* va_extsize is the actual size of the data payload in the toast records.
*/
if (VARATT_IS_SHORT(dval))
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
toast_pointer.va_extsize = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
toast_pointer.va_rawsize = VARRAWSIZE_4B_C(dval) + VARHDRSZ;
toast_pointer.va_extsize = data_todo;
/* Assert that the numbers look like it's compressed */
Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
toast_pointer.va_rawsize = VARSIZE(dval);
toast_pointer.va_extsize = data_todo;
}
/*
* Insert the correct table OID into the result TOAST pointer.
*
* Normally this is the actual OID of the target toast table, but during
* table-rewriting operations such as CLUSTER, we have to insert the OID
* of the table's real permanent toast table instead. rd_toastoid is set
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
toast_pointer.va_toastrelid = rel->rd_toastoid;
else
toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
/*
* Choose an unused OID within the toast table for this toast value.
*/
toast_pointer.va_valueid = GetNewOidWithIndex(toastrel,
RelationGetRelid(toastidx),
(AttrNumber) 1);
/*
* Initialize constant parts of the tuple data
*/
t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
t_values[2] = PointerGetDatum(&chunk_data);
t_isnull[0] = false;
t_isnull[1] = false;
t_isnull[2] = false;
/*
* Split up the item into chunks
*/
while (data_todo > 0)
{
/*
* Calculate the size of this chunk
*/
chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
/*
* Build a tuple and store it
*/
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
heap_insert(toastrel, toasttup, mycid, options, NULL);
/*
* Create the index entry. We cheat a little here by not using
* FormIndexDatum: this relies on the knowledge that the index columns
* are the same as the initial columns of the table.
*
* Note also that there had better not be any user-created index on
* the TOAST table, since we don't bother to update anything else.
*/
index_insert(toastidx, t_values, t_isnull,
&(toasttup->t_self),
toastrel,
toastidx->rd_index->indisunique ?
UNIQUE_CHECK_YES : UNIQUE_CHECK_NO);
/*
* Free memory
*/
heap_freetuple(toasttup);
/*
* Move on to next chunk
*/
data_todo -= chunk_size;
data_p += chunk_size;
}
/*
* Done - close toast relation
*/
index_close(toastidx, RowExclusiveLock);
heap_close(toastrel, RowExclusiveLock);
/*
* Create the TOAST pointer value that we'll return
*/
result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE);
memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
return PointerGetDatum(result);
}
/* ----------
* toast_fetch_datum -
*
* Reconstruct an in memory Datum from the chunks saved
* in the toast relation
* ----------
*/
static struct varlena *
toast_fetch_datum(struct varlena * attr)
{
Relation toastrel;
Relation toastidx;
ScanKeyData toastkey;
SysScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
struct varlena *result;
struct varatt_external toast_pointer;
int32 ressize;
int32 residx,
nextidx;
int32 numchunks;
Pointer chunk;
bool isnull;
char *chunkdata;
int32 chunksize;
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
ressize = toast_pointer.va_extsize;
numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
result = (struct varlena *) palloc(ressize + VARHDRSZ);
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
SET_VARSIZE_COMPRESSED(result, ressize + VARHDRSZ);
else
SET_VARSIZE(result, ressize + VARHDRSZ);
/*
* Open the toast relation and its index
*/
toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
* Setup a scan key to fetch from the index by va_valueid
*/
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(toast_pointer.va_valueid));
/*
* Read the chunks by index
*
* Note that because the index is actually on (valueid, chunkidx) we will
* see the chunks in chunkidx order, even though we didn't explicitly ask
* for it.
*/
nextidx = 0;
toastscan = systable_beginscan_ordered(toastrel, toastidx,
SnapshotToast, 1, &toastkey);
while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL)
{
/*
* Have a chunk, extract the sequence number and the data
*/
residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
if (!VARATT_IS_EXTENDED(chunk))
{
chunksize = VARSIZE(chunk) - VARHDRSZ;
chunkdata = VARDATA(chunk);
}
else if (VARATT_IS_SHORT(chunk))
{
/* could happen due to heap_form_tuple doing its thing */
chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
chunkdata = VARDATA_SHORT(chunk);
}
else
{
/* should never happen */
elog(ERROR, "found toasted toast chunk for toast value %u in %s",
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
chunksize = 0; /* keep compiler quiet */
chunkdata = NULL;
}
/*
* Some checks on the data we've found
*/
if (residx != nextidx)
elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u in %s",
residx, nextidx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
if (residx < numchunks - 1)
{
if (chunksize != TOAST_MAX_CHUNK_SIZE)
elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s",
chunksize, (int) TOAST_MAX_CHUNK_SIZE,
residx, numchunks,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
}
else if (residx == numchunks - 1)
{
if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize)
elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u in %s",
chunksize,
(int) (ressize - residx * TOAST_MAX_CHUNK_SIZE),
residx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
}
else
elog(ERROR, "unexpected chunk number %d (out of range %d..%d) for toast value %u in %s",
residx,
0, numchunks - 1,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* Copy the data into proper place in our result
*/
memcpy(VARDATA(result) + residx * TOAST_MAX_CHUNK_SIZE,
chunkdata,
chunksize);
nextidx++;
}
/*
* Final checks that we successfully fetched the datum
*/
if (nextidx != numchunks)
elog(ERROR, "missing chunk number %d for toast value %u in %s",
nextidx,
toast_pointer.va_valueid,
RelationGetRelationName(toastrel));
/*
* End scan and close relations
*/
systable_endscan_ordered(toastscan);
index_close(toastidx, AccessShareLock);
heap_close(toastrel, AccessShareLock);
return result;
}
/* ----------
* heap_tuple_untoast_attr_slice -
*
* Public entry point to get back part of a toasted value
* from compression or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr_slice(struct varlena * attr,
int32 sliceoffset, int32 slicelength)
{
varattrib *preslice;
varattrib *result;
char *attrdata;
int32 attrsize;
if (VARATT_IS_EXTERNAL(attr))
{
/* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(attr))
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/* this automatically sets the compressed flag if appropriate */
preslice = (varattrib *)toast_fetch_datum(attr);
}
else
preslice = (varattrib *)attr;
if (VARATT_IS_COMPRESSED(preslice))
{
unsigned size;
PGLZ_Header *tmp;
tmp = (PGLZ_Header *) preslice;
size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ;
preslice = (varattrib *) palloc(size);
SET_VARSIZE(preslice, size);
pglz_decompress(tmp, VARDATA(preslice));
if (tmp != (PGLZ_Header *) attr)
pfree(tmp);
}
if (VARATT_IS_SHORT(preslice))
{
attrdata = VARDATA_SHORT(preslice);
attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
}
else
{
attrdata = VARDATA(preslice);
attrsize = VARSIZE(preslice) - VARHDRSZ;
}
/* slicing of datum for compressed cases and plain value */
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
slicelength = 0;
}
if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
slicelength = attrsize - sliceoffset;
result = (varattrib *) palloc(slicelength + VARHDRSZ);
SET_VARSIZE(result, slicelength + VARHDRSZ);
memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
if ((struct varlena *)preslice != (struct varlena *)attr)
pfree(preslice);
return (struct varlena *)result;
}
/* ----------
* toast_fetch_datum_slice -
*
* Reconstruct a segment of a Datum from the chunks saved
* in the toast relation
* ----------
*/
static struct varlena *
toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
{
Relation toastrel;
Relation toastidx;
ScanKeyData toastkey[3];
int nscankeys;
IndexScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
varattrib *result;
int32 attrsize;
int32 residx;
int32 nextidx;
int numchunks;
int startchunk;
int endchunk;
int32 startoffset;
int32 endoffset;
int totalchunks;
Pointer chunk;
bool isnull;
int32 chunksize;
int32 chcpystrt;
int32 chcpyend;
attrsize = ((varattrib *)attr)->va_external.va_extsize;
totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
length = 0;
}
if (((sliceoffset + length) > attrsize) || length < 0)
length = attrsize - sliceoffset;
result = (varattrib *) palloc(length + VARHDRSZ);
SET_VARSIZE(result, length + VARHDRSZ);
if (VARATT_EXTERNAL_IS_COMPRESSED(attr))
VARATT_SET_COMPRESSED(result);
if (length == 0)
return (struct varlena *)result; /* Can save a lot of work at this point! */
startchunk = sliceoffset / TOAST_MAX_CHUNK_SIZE;
endchunk = (sliceoffset + length - 1) / TOAST_MAX_CHUNK_SIZE;
numchunks = (endchunk - startchunk) + 1;
startoffset = sliceoffset % TOAST_MAX_CHUNK_SIZE;
endoffset = (sliceoffset + length - 1) % TOAST_MAX_CHUNK_SIZE;
/*
* Open the toast relation and its index
*/
toastrel = heap_open(((varattrib *)attr)->va_external.va_toastrelid, AccessShareLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
* Setup a scan key to fetch from the index. This is either two keys or
* three depending on the number of chunks.
*/
ScanKeyInit(&toastkey[0],
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(((varattrib *)attr)->va_external.va_valueid));
/*
* Use equality condition for one chunk, a range condition otherwise:
*/
if (numchunks == 1)
{
ScanKeyInit(&toastkey[1],
(AttrNumber) 2,
BTEqualStrategyNumber, F_INT4EQ,
Int32GetDatum(startchunk));
nscankeys = 2;
}
else
{
ScanKeyInit(&toastkey[1],
(AttrNumber) 2,
BTGreaterEqualStrategyNumber, F_INT4GE,
Int32GetDatum(startchunk));
ScanKeyInit(&toastkey[2],
(AttrNumber) 2,
BTLessEqualStrategyNumber, F_INT4LE,
Int32GetDatum(endchunk));
nscankeys = 3;
}
/*
* Read the chunks by index
*
* The index is on (valueid, chunkidx) so they will come in order
*/
nextidx = startchunk;
toastscan = index_beginscan(toastrel, toastidx,
SnapshotToast, nscankeys, toastkey);
while ((ttup = index_getnext(toastscan, ForwardScanDirection)) != NULL)
{
/*
* Have a chunk, extract the sequence number and the data
*/
residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
if (VARATT_IS_SHORT((varattrib *)chunk))
chunksize = VARSIZE_SHORT((varattrib *)chunk) - VARHDRSZ_SHORT;
else if (!VARATT_IS_EXTENDED((varattrib *)chunk))
chunksize = VARSIZE((varattrib *)chunk) - VARHDRSZ;
else {
elog(ERROR, "found toasted toast chunk?");
chunksize = 0; /* shut compiler up */
}
/*
* Some checks on the data we've found
*/
if ((residx != nextidx) || (residx > endchunk) || (residx < startchunk))
elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u",
residx, nextidx,
((varattrib *)attr)->va_external.va_valueid);
if (residx < totalchunks - 1)
{
if (chunksize != TOAST_MAX_CHUNK_SIZE)
elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %u of %d when fetching slice (expected %d)",
chunksize, residx,
((varattrib *)attr)->va_external.va_valueid, totalchunks-1,
(int)TOAST_MAX_CHUNK_SIZE);
}
else if (residx == totalchunks-1)
{
if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != attrsize)
elog(ERROR, "unexpected chunk size %d in chunk %d for final toast value %u when fetching slice (expected %d)",
chunksize, residx,
((varattrib *)attr)->va_external.va_valueid,
attrsize - residx * (int)TOAST_MAX_CHUNK_SIZE);
}
else
{
elog(ERROR, "unexpected chunk");
}
/*
* Copy the data into proper place in our result
*/
chcpystrt = 0;
chcpyend = chunksize - 1;
if (residx == startchunk)
chcpystrt = startoffset;
if (residx == endchunk)
chcpyend = endoffset;
memcpy(((char *) VARDATA(result)) +
(residx * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt,
VARDATA((varattrib *)chunk) + chcpystrt,
(chcpyend - chcpystrt) + 1);
nextidx++;
}
/*
* Final checks that we successfully fetched the datum
*/
if (nextidx != (endchunk + 1))
elog(ERROR, "missing chunk number %d for toast value %u",
nextidx,
((varattrib *)attr)->va_external.va_valueid);
/*
* End scan and close relations
*/
index_endscan(toastscan);
index_close(toastidx, AccessShareLock);
heap_close(toastrel, AccessShareLock);
return (struct varlena *)result;
}
/* ----------
* toast_fetch_datum -
*
* Reconstruct an in memory Datum from the chunks saved
* in the toast relation
* ----------
*/
static struct varlena *
toast_fetch_datum(struct varlena *attr)
{
Relation toastrel;
Relation toastidx;
ScanKeyData toastkey;
IndexScanDesc toastscan;
HeapTuple ttup;
TupleDesc toasttupDesc;
varattrib *result;
int32 ressize;
int32 residx,
nextidx;
int32 numchunks;
Pointer chunk;
bool isnull;
int32 chunksize;
void *chunkdata;
ressize = ((varattrib *)attr)->va_external.va_extsize;
numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
result = (varattrib *) palloc(ressize + VARHDRSZ);
SET_VARSIZE(result, ressize + VARHDRSZ);
if (VARATT_EXTERNAL_IS_COMPRESSED(attr))
VARATT_SET_COMPRESSED(result);
/*
* Open the toast relation and its index
*/
toastrel = heap_open(((varattrib *)attr)->va_external.va_toastrelid, AccessShareLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock);
/*
* Setup a scan key to fetch from the index by va_valueid
*/
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(((varattrib *)attr)->va_external.va_valueid));
/*
* Read the chunks by index
*
* Note that because the index is actually on (valueid, chunkidx) we will
* see the chunks in chunkidx order, even though we didn't explicitly ask
* for it.
*/
nextidx = 0;
toastscan = index_beginscan(toastrel, toastidx,
SnapshotToast, 1, &toastkey);
while ((ttup = index_getnext(toastscan, ForwardScanDirection)) != NULL)
{
/*
* Have a chunk, extract the sequence number and the data
*/
residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
Assert(!isnull);
chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
Assert(!isnull);
if (VARATT_IS_SHORT(chunk))
{
chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
chunkdata = VARDATA_SHORT(chunk);
}
else if (!VARATT_IS_EXTENDED(chunk))
{
chunksize = VARSIZE(chunk) - VARHDRSZ;
chunkdata = VARDATA(chunk);
}
else
{
elog(ERROR, "found toasted toast chunk?");
chunksize = 0; /* shut compiler up */
chunkdata = NULL;
}
/*
* Some checks on the data we've found
*/
if (residx != nextidx)
elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u",
residx, nextidx,
((varattrib *)attr)->va_external.va_valueid);
if (residx < numchunks - 1)
{
if (chunksize != TOAST_MAX_CHUNK_SIZE)
elog(ERROR, "unexpected chunk size %d in chunk %d of %d for toast value %u (expected %d)",
chunksize, residx,
((varattrib *)attr)->va_external.va_valueid, numchunks-1,
(int)TOAST_MAX_CHUNK_SIZE);
}
else if (residx == numchunks-1)
{
if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize)
elog(ERROR, "unexpected chunk size %d in final chunk %d for toast value %u (expected %d)",
chunksize, residx,
((varattrib *)attr)->va_external.va_valueid,
ressize - residx*(int)TOAST_MAX_CHUNK_SIZE);
}
else
elog(ERROR, "unexpected chunk number %d for toast value %u (expected in %d..%d)",
residx,
((varattrib *)attr)->va_external.va_valueid,
0, numchunks-1);
/*
* Copy the data into proper place in our result
*/
memcpy(((char *) VARDATA(result)) + residx * TOAST_MAX_CHUNK_SIZE,
chunkdata,
chunksize);
nextidx++;
}
/*
* Final checks that we successfully fetched the datum
*/
if (nextidx != numchunks)
elog(ERROR, "missing chunk number %d for toast value %u",
nextidx,
((varattrib *)attr)->va_external.va_valueid);
/*
* End scan and close relations
*/
index_endscan(toastscan);
index_close(toastidx, AccessShareLock);
heap_close(toastrel, AccessShareLock);
return (struct varlena *)result;
}
/* ----------
* toast_save_datum -
*
* Save one single datum into the secondary relation and return
* a Datum reference for it.
* ----------
*/
static Datum
toast_save_datum(Relation rel, Datum value, bool isFrozen)
{
Relation toastrel;
Relation toastidx;
HeapTuple toasttup;
TupleDesc toasttupDesc;
Datum t_values[3];
bool t_isnull[3];
varattrib *result;
struct
{
struct varlena hdr;
char data[TOAST_MAX_CHUNK_SIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
int32 rawsize, extsize;
/*
* Open the toast relation and its index. We can use the index to check
* uniqueness of the OID we assign to the toasted item, even though it has
* additional columns besides OID.
*/
toastrel = heap_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock);
/*
* Create the varattrib reference
*/
result = (varattrib *) palloc(sizeof(varattrib));
/* rawsize is the size of the datum that will result after decompression --
* including the full header. so we have to adjust for short headers.
*
* extsize is the actual size of the data payload in the toast records
* without any headers
*/
if (VARATT_IS_SHORT_D(value))
{
rawsize = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT + VARHDRSZ;
extsize = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT;
data_p = VARDATA_SHORT_D(value);
data_todo = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT;
}
else if (VARATT_IS_COMPRESSED_D(value))
{
/* rawsize in a compressed datum is the just the size of the payload */
rawsize = ((varattrib *) DatumGetPointer(value))->va_compressed.va_rawsize + VARHDRSZ;
extsize = VARSIZE_D(value) - VARHDRSZ;
data_p = VARDATA_D(value);
data_todo = VARSIZE_D(value) - VARHDRSZ;
/* we used to set result->va_header |= VARATT_FLAG_COMPRESSED; down
* below. we don't any longer and depend on the equality holding:
* extsize = rawsize + VARHDRSZ*/
}
else
{
rawsize = VARSIZE_D(value);
extsize = VARSIZE_D(value) - VARHDRSZ;
data_p = VARDATA_D(value);
data_todo = VARSIZE_D(value) - VARHDRSZ;
}
SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE);
result->va_external.va_rawsize = rawsize;
result->va_external.va_extsize = extsize;
result->va_external.va_valueid = GetNewOidWithIndex(toastrel, toastidx);
result->va_external.va_toastrelid = rel->rd_rel->reltoastrelid;
#ifdef USE_ASSERT_CHECKING
Assert( (VARATT_IS_COMPRESSED_D(value)||0) == (VARATT_EXTERNAL_IS_COMPRESSED(result)||0) );
if (VARATT_IS_COMPRESSED_D(value))
{
Assert(VARATT_EXTERNAL_IS_COMPRESSED(result));
elog(DEBUG4,
"saved toast datum, original varsize %ud rawsize %ud new extsize %ud rawsize %uld\n",
VARSIZE_D(value), ((varattrib *) DatumGetPointer(value))->va_compressed.va_rawsize,
result->va_external.va_extsize, result->va_external.va_rawsize);
}
else
{
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(result));
elog(DEBUG4,
"saved toast datum, original varsize %ud new extsize %ud rawsize %ud\n",
VARSIZE_D(value),
result->va_external.va_extsize, result->va_external.va_rawsize);
}
#endif
/*
* Initialize constant parts of the tuple data
*/
t_values[0] = ObjectIdGetDatum(result->va_external.va_valueid);
t_values[2] = PointerGetDatum(&chunk_data);
t_isnull[0] = false;
t_isnull[1] = false;
t_isnull[2] = false;
/*
* Split up the item into chunks
*/
while (data_todo > 0)
{
/*
* Calculate the size of this chunk
*/
chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
/*
* Build a tuple and store it
*/
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
if (!HeapTupleIsValid(toasttup))
elog(ERROR, "failed to build TOAST tuple");
if(!isFrozen)
{
/* the normal case. regular insert */
simple_heap_insert(toastrel, toasttup);
}
else
{
/* insert and freeze the tuple. used for errtables and their related toast data */
frozen_heap_insert(toastrel, toasttup);
}
//heap_insert(relation, tup, GetCurrentCommandId(),
// true, true, GetCurrentTransactionId());
/*
* Create the index entry. We cheat a little here by not using
* FormIndexDatum: this relies on the knowledge that the index columns
* are the same as the initial columns of the table.
*
* Note also that there had better not be any user-created index on
* the TOAST table, since we don't bother to update anything else.
*/
index_insert(toastidx, t_values, t_isnull,
&(toasttup->t_self),
toastrel, toastidx->rd_index->indisunique);
/*
* Free memory
*/
heap_freetuple(toasttup);
/*
* Move on to next chunk
*/
data_todo -= chunk_size;
data_p += chunk_size;
}
/*
* Done - close toast relation
*/
index_close(toastidx, RowExclusiveLock);
heap_close(toastrel, RowExclusiveLock);
return PointerGetDatum(result);
}
static int
ReadStringFromToast(const char *buffer,
unsigned int buff_size,
unsigned int* out_size)
{
int result = 0;
/* If toasted value is on disk, we'll try to restore it. */
if (VARATT_IS_EXTERNAL_ONDISK(buffer))
{
varatt_external toast_ptr;
char *toast_data = NULL;
/* Number of chunks the TOAST data is divided into */
int32 num_chunks;
/* Actual size of external TOASTed value */
int32 toast_ext_size;
/* Path to directory with TOAST realtion file */
char *toast_relation_path;
/* Filename of TOAST relation file */
char toast_relation_filename[MAXPGPATH];
FILE *toast_rel_fp;
unsigned int block_options = 0;
unsigned int control_options = 0;
VARATT_EXTERNAL_GET_POINTER(toast_ptr, buffer);
printf(" TOAST value. Raw size: %8d, external size: %8d, "
"value id: %6d, toast relation id: %6d\n",
toast_ptr.va_rawsize,
toast_ptr.va_extsize,
toast_ptr.va_valueid,
toast_ptr.va_toastrelid);
/* Extract TOASTed value */
toast_ext_size = toast_ptr.va_extsize;
num_chunks = (toast_ext_size - 1) / TOAST_MAX_CHUNK_SIZE + 1;
printf(" Number of chunks: %d\n", num_chunks);
/* Open TOAST relation file */
toast_relation_path = strdup(fileName);
get_parent_directory(toast_relation_path);
sprintf(toast_relation_filename, "%s/%d", toast_relation_path,
toast_ptr.va_toastrelid);
printf(" Read TOAST relation %s\n", toast_relation_filename);
toast_rel_fp = fopen(toast_relation_filename, "rb");
if (!toast_rel_fp) {
printf("Cannot open TOAST relation %s\n",
toast_relation_filename);
result = -1;
}
if (result == 0)
{
unsigned int toast_relation_block_size = GetBlockSize(toast_rel_fp);
fseek(toast_rel_fp, 0, SEEK_SET);
toast_data = malloc(toast_ptr.va_rawsize);
result = DumpFileContents(block_options,
control_options,
toast_rel_fp,
toast_relation_block_size,
-1, /* no start block */
-1, /* no end block */
true, /* is toast relation */
toast_ptr.va_valueid,
toast_ptr.va_extsize,
toast_data);
if (result == 0)
{
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_ptr))
result = DumpCompressedString(toast_data, toast_ext_size);
else
CopyAppendEncode(toast_data, toast_ext_size);
}
else
{
printf("Error in TOAST file.\n");
}
free(toast_data);
}
fclose(toast_rel_fp);
free(toast_relation_path);
}
/* If tag is indirect or expanded, it was stored in memory. */
else
{
CopyAppend("(TOASTED IN MEMORY)");
}
return result;
}
