/* ----------
* toast_save_datum -
*
* Save one single datum into the secondary relation and return
* a Datum reference for it.
* ----------
*/
static Datum
toast_save_datum(Relation rel, Datum value, bool isFrozen)
{
Relation toastrel;
Relation toastidx;
HeapTuple toasttup;
TupleDesc toasttupDesc;
Datum t_values[3];
bool t_isnull[3];
varattrib *result;
struct
{
struct varlena hdr;
char data[TOAST_MAX_CHUNK_SIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
int32 rawsize, extsize;
/*
* Open the toast relation and its index. We can use the index to check
* uniqueness of the OID we assign to the toasted item, even though it has
* additional columns besides OID.
*/
toastrel = heap_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock);
/*
* Create the varattrib reference
*/
result = (varattrib *) palloc(sizeof(varattrib));
/* rawsize is the size of the datum that will result after decompression --
* including the full header. so we have to adjust for short headers.
*
* extsize is the actual size of the data payload in the toast records
* without any headers
*/
if (VARATT_IS_SHORT_D(value))
{
rawsize = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT + VARHDRSZ;
extsize = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT;
data_p = VARDATA_SHORT_D(value);
data_todo = VARSIZE_SHORT_D(value) - VARHDRSZ_SHORT;
}
else if (VARATT_IS_COMPRESSED_D(value))
{
/* rawsize in a compressed datum is the just the size of the payload */
rawsize = ((varattrib *) DatumGetPointer(value))->va_compressed.va_rawsize + VARHDRSZ;
extsize = VARSIZE_D(value) - VARHDRSZ;
data_p = VARDATA_D(value);
data_todo = VARSIZE_D(value) - VARHDRSZ;
/* we used to set result->va_header |= VARATT_FLAG_COMPRESSED; down
* below. we don't any longer and depend on the equality holding:
* extsize = rawsize + VARHDRSZ*/
}
else
{
rawsize = VARSIZE_D(value);
extsize = VARSIZE_D(value) - VARHDRSZ;
data_p = VARDATA_D(value);
data_todo = VARSIZE_D(value) - VARHDRSZ;
}
SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE);
result->va_external.va_rawsize = rawsize;
result->va_external.va_extsize = extsize;
result->va_external.va_valueid = GetNewOidWithIndex(toastrel, toastidx);
result->va_external.va_toastrelid = rel->rd_rel->reltoastrelid;
#ifdef USE_ASSERT_CHECKING
Assert( (VARATT_IS_COMPRESSED_D(value)||0) == (VARATT_EXTERNAL_IS_COMPRESSED(result)||0) );
if (VARATT_IS_COMPRESSED_D(value))
{
Assert(VARATT_EXTERNAL_IS_COMPRESSED(result));
elog(DEBUG4,
"saved toast datum, original varsize %ud rawsize %ud new extsize %ud rawsize %uld\n",
VARSIZE_D(value), ((varattrib *) DatumGetPointer(value))->va_compressed.va_rawsize,
result->va_external.va_extsize, result->va_external.va_rawsize);
}
else
{
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(result));
elog(DEBUG4,
"saved toast datum, original varsize %ud new extsize %ud rawsize %ud\n",
VARSIZE_D(value),
result->va_external.va_extsize, result->va_external.va_rawsize);
}
#endif
/*
* Initialize constant parts of the tuple data
*/
t_values[0] = ObjectIdGetDatum(result->va_external.va_valueid);
t_values[2] = PointerGetDatum(&chunk_data);
t_isnull[0] = false;
t_isnull[1] = false;
t_isnull[2] = false;
/*
* Split up the item into chunks
*/
while (data_todo > 0)
{
/*
* Calculate the size of this chunk
*/
chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
/*
* Build a tuple and store it
*/
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
if (!HeapTupleIsValid(toasttup))
elog(ERROR, "failed to build TOAST tuple");
if(!isFrozen)
{
/* the normal case. regular insert */
simple_heap_insert(toastrel, toasttup);
}
else
{
/* insert and freeze the tuple. used for errtables and their related toast data */
frozen_heap_insert(toastrel, toasttup);
}
//heap_insert(relation, tup, GetCurrentCommandId(),
// true, true, GetCurrentTransactionId());
/*
* Create the index entry. We cheat a little here by not using
* FormIndexDatum: this relies on the knowledge that the index columns
* are the same as the initial columns of the table.
*
* Note also that there had better not be any user-created index on
* the TOAST table, since we don't bother to update anything else.
*/
index_insert(toastidx, t_values, t_isnull,
&(toasttup->t_self),
toastrel, toastidx->rd_index->indisunique);
/*
* Free memory
*/
heap_freetuple(toasttup);
/*
* Move on to next chunk
*/
data_todo -= chunk_size;
data_p += chunk_size;
}
/*
* Done - close toast relation
*/
index_close(toastidx, RowExclusiveLock);
heap_close(toastrel, RowExclusiveLock);
return PointerGetDatum(result);
}
/*
* heap_deform_tuple
* Given a tuple, extract data into values/isnull arrays; this is
* the inverse of heap_form_tuple.
*
* Storage for the values/isnull arrays is provided by the caller;
* it should be sized according to tupleDesc->natts not tuple->t_natts.
*
* Note that for pass-by-reference datatypes, the pointer placed
* in the Datum will point into the given tuple.
*
* When all or most of a tuple's fields need to be extracted,
* this routine will be significantly quicker than a loop around
* heap_getattr; the loop will become O(N^2) as soon as any
* noncacheable attribute offsets are involved.
*/
void
heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc,
Datum *values, bool *isnull)
{
HeapTupleHeader tup = tuple->t_data;
bool hasnulls = HeapTupleHasNulls(tuple);
Form_pg_attribute *att = tupleDesc->attrs;
int tdesc_natts = tupleDesc->natts;
int natts; /* number of atts to extract */
int attnum;
char *tp; /* ptr to tuple data */
long off; /* offset in tuple data */
bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */
bool slow = false; /* can we use/set attcacheoff? */
Assert(!is_heaptuple_memtuple(tuple));
natts = HeapTupleHeaderGetNatts(tup);
/*
* In inheritance situations, it is possible that the given tuple actually
* has more fields than the caller is expecting. Don't run off the end of
* the caller's arrays.
*/
natts = Min(natts, tdesc_natts);
tp = (char *) tup + tup->t_hoff;
off = 0;
for (attnum = 0; attnum < natts; attnum++)
{
Form_pg_attribute thisatt = att[attnum];
if (hasnulls && att_isnull(attnum, bp))
{
values[attnum] = (Datum) 0;
isnull[attnum] = true;
slow = true; /* can't use attcacheoff anymore */
continue;
}
isnull[attnum] = false;
if (!slow && thisatt->attcacheoff >= 0)
off = thisatt->attcacheoff;
else
{
/* if it's a varlena it may or may not be aligned, so check for
* something that looks like a padding byte before aligning. If
* we're already aligned it may be the leading byte of a 4-byte
* header but then the att_align is harmless. Don't bother looking
* if it's not a varlena though.*/
if (thisatt->attlen != -1 || !tp[off])
off = att_align(off, thisatt->attalign);
if (!slow && thisatt->attlen != -1)
thisatt->attcacheoff = off;
}
if (!slow && thisatt->attlen < 0)
slow = true;
values[attnum] = fetchatt(thisatt, tp + off);
#ifdef USE_ASSERT_CHECKING
/* Ignore attributes with dropped types */
if (thisatt->attlen == -1 && !thisatt->attisdropped)
{
Assert(VARATT_IS_SHORT_D(values[attnum]) ||
!VARATT_COULD_SHORT_D(values[attnum]) ||
thisatt->atttypid == OIDVECTOROID ||
thisatt->atttypid == INT2VECTOROID ||
thisatt->atttypid >= FirstNormalObjectId);
}
#endif
off = att_addlength(off, thisatt->attlen, PointerGetDatum(tp + off));
}
/*
* If tuple doesn't have all the atts indicated by tupleDesc, read the
* rest as null
*/
for (; attnum < tdesc_natts; attnum++)
{
values[attnum] = (Datum) 0;
isnull[attnum] = true;
}
}
/* ----------------
* index_form_tuple
* ----------------
*/
IndexTuple
index_form_tuple(TupleDesc tupleDescriptor,
Datum *values,
bool *isnull)
{
char *tp; /* tuple pointer */
IndexTuple tuple; /* return tuple */
Size size,
hoff;
int i;
unsigned short infomask = 0;
bool hasnull = false;
uint16 tupmask = 0;
int numberOfAttributes = tupleDescriptor->natts;
#ifdef TOAST_INDEX_HACK
Datum untoasted_values[INDEX_MAX_KEYS];
bool untoasted_free[INDEX_MAX_KEYS];
#endif
if (numberOfAttributes > INDEX_MAX_KEYS)
ereport(ERROR,
(errcode(ERRCODE_TOO_MANY_COLUMNS),
errmsg("number of index columns (%d) exceeds limit (%d)",
numberOfAttributes, INDEX_MAX_KEYS)));
#ifdef TOAST_INDEX_HACK
for (i = 0; i < numberOfAttributes; i++)
{
Form_pg_attribute att = tupleDescriptor->attrs[i];
untoasted_values[i] = values[i];
untoasted_free[i] = false;
/* Do nothing if value is NULL or not of varlena type */
if (isnull[i] || att->attlen != -1)
continue;
/*
* If value is stored EXTERNAL, must fetch it so we are not depending
* on outside storage. This should be improved someday.
*/
if (VARATT_IS_EXTERNAL_D(values[i]))
{
untoasted_values[i] =
PointerGetDatum(heap_tuple_fetch_attr(DatumGetPointer(values[i])));
untoasted_free[i] = true;
}
/*
* If value is above size target, and is of a compressible datatype,
* try to compress it in-line.
*/
if (!VARATT_IS_SHORT_D(untoasted_values[i]) &&
VARSIZE_D(untoasted_values[i]) > TOAST_INDEX_TARGET &&
!VARATT_IS_COMPRESSED_D(untoasted_values[i]) &&
(att->attstorage == 'x' || att->attstorage == 'm'))
{
Datum cvalue = toast_compress_datum(untoasted_values[i]);
if (DatumGetPointer(cvalue) != NULL)
{
/* successful compression */
if (untoasted_free[i])
pfree(DatumGetPointer(untoasted_values[i]));
untoasted_values[i] = cvalue;
untoasted_free[i] = true;
}
}
}
#endif
for (i = 0; i < numberOfAttributes; i++)
{
if (isnull[i])
{
hasnull = true;
break;
}
}
if (hasnull)
infomask |= INDEX_NULL_MASK;
hoff = IndexInfoFindDataOffset(infomask);
#ifdef TOAST_INDEX_HACK
size = hoff + heap_compute_data_size(tupleDescriptor,
untoasted_values, isnull);
#else
size = hoff + heap_compute_data_size(tupleDescriptor,
values, isnull);
#endif
size = MAXALIGN(size); /* be conservative */
tp = (char *) palloc0(size);
tuple = (IndexTuple) tp;
heap_fill_tuple(tupleDescriptor,
#ifdef TOAST_INDEX_HACK
untoasted_values,
#else
values,
#endif
isnull,
(char *) tp + hoff,
&tupmask,
(hasnull ? (bits8 *) tp + sizeof(IndexTupleData) : NULL));
#ifdef TOAST_INDEX_HACK
for (i = 0; i < numberOfAttributes; i++)
{
if (untoasted_free[i])
pfree(DatumGetPointer(untoasted_values[i]));
}
#endif
/*
* We do this because heap_fill_tuple wants to initialize a "tupmask"
* which is used for HeapTuples, but we want an indextuple infomask. The
* only relevant info is the "has variable attributes" field. We have
* already set the hasnull bit above.
*/
if (tupmask & HEAP_HASVARWIDTH)
infomask |= INDEX_VAR_MASK;
/*
* Here we make sure that the size will fit in the field reserved for it
* in t_info.
*/
if ((size & INDEX_SIZE_MASK) != size)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("index row requires %lu bytes, maximum size is %lu",
(unsigned long) size,
(unsigned long) INDEX_SIZE_MASK)));
infomask |= size;
/*
* initialize metadata
*/
tuple->t_info = infomask;
return tuple;
}
/*
* heap_fill_tuple
* Load data portion of a tuple from values/isnull arrays
*
* We also fill the null bitmap (if any) and set the infomask bits
* that reflect the tuple's data contents.
*
* NOTE: it is now REQUIRED that the caller have pre-zeroed the data area.
*
*
* @param isnull will only be used if <code>bit</code> is non-NULL
* @param bit should be non-NULL (refer to td->t_bits) if isnull is set and contains non-null values
*/
Size
heap_fill_tuple(TupleDesc tupleDesc,
Datum *values, bool *isnull,
char *data, uint16 *infomask, bits8 *bit)
{
char *start = data;
bits8 *bitP;
int bitmask;
int i;
int numberOfAttributes = tupleDesc->natts;
Form_pg_attribute *att = tupleDesc->attrs;
if (bit != NULL)
{
bitP = &bit[-1];
bitmask = HIGHBIT;
}
else
{
/* just to keep compiler quiet */
bitP = NULL;
bitmask = 0;
}
*infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTENDED);
for (i = 0; i < numberOfAttributes; i++)
{
Size data_length;
if (bit != NULL)
{
if (bitmask != HIGHBIT)
bitmask <<= 1;
else
{
bitP += 1;
*bitP = 0x0;
bitmask = 1;
}
if (isnull[i])
{
*infomask |= HEAP_HASNULL;
continue;
}
*bitP |= bitmask;
}
/*
* XXX we use the att_align macros on the pointer value itself, not on
* an offset. This is a bit of a hack.
*/
if (att[i]->attbyval)
{
/* pass-by-value */
data = (char *) att_align_zero(data, att[i]->attalign);
store_att_byval(data, values[i], att[i]->attlen);
data_length = att[i]->attlen;
}
else if (att[i]->attlen == -1)
{
/* varlena */
*infomask |= HEAP_HASVARWIDTH;
if (VARATT_IS_COMPRESSED_D(values[i]))
*infomask |= HEAP_HASCOMPRESSED;
if (VARATT_IS_EXTERNAL_D(values[i]))
{
*infomask |= HEAP_HASEXTERNAL;
data = (char *) att_align_zero(data, att[i]->attalign);
data_length = VARSIZE_EXTERNAL(DatumGetPointer(values[i]));
memcpy(data, DatumGetPointer(values[i]), data_length);
}
else if (VARATT_IS_SHORT_D(values[i]))
{
/* no alignment for short varlenas */
data_length = VARSIZE_SHORT(DatumGetPointer(values[i]));
memcpy(data, DatumGetPointer(values[i]), data_length);
}
else if (VARATT_COULD_SHORT_D(values[i]) &&
att[i]->atttypid != INT2VECTOROID &&
att[i]->atttypid != OIDVECTOROID &&
att[i]->atttypid < FirstNormalObjectId)
{
/* convert to short varlena -- no alignment */
data_length = VARSIZE_D(values[i]) - VARHDRSZ + VARHDRSZ_SHORT;
*data = VARSIZE_TO_SHORT_D(values[i]);
memcpy(data+1,
VARDATA_D(values[i]),
data_length-1);
}
else
{
/* must store full 4-byte header varlena */
data = (char *) att_align_zero(data, att[i]->attalign);
data_length = VARSIZE(DatumGetPointer(values[i]));
memcpy(data, DatumGetPointer(values[i]), data_length);
}
}
else if (att[i]->attlen == -2)
{
/* cstring */
data = (char *) att_align_zero(data, att[i]->attalign);
*infomask |= HEAP_HASVARWIDTH;
data_length = strlen(DatumGetCString(values[i])) + 1;
memcpy(data, DatumGetPointer(values[i]), data_length);
}
else
{
/* fixed-length pass-by-reference */
data = (char *) att_align_zero(data, att[i]->attalign);
Assert(att[i]->attlen > 0);
data_length = att[i]->attlen;
memcpy(data, DatumGetPointer(values[i]), data_length);
}
data += data_length;
}
return data - start;
}
