/* ----------
* toast_raw_datum_size -
*
* Return the raw (detoasted) size of a varlena datum
* (including the VARHDRSZ header)
* ----------
*/
Size
toast_raw_datum_size(Datum value)
{
struct varlena *attr = (struct varlena *) DatumGetPointer(value);
Size result;
if (VARATT_IS_EXTERNAL(attr))
{
/* va_rawsize is the size of the original datum -- including header */
struct varatt_external toast_pointer;
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_rawsize;
}
else if (VARATT_IS_COMPRESSED(attr))
{
/* here, va_rawsize is just the payload size */
result = VARRAWSIZE_4B_C(attr) + VARHDRSZ;
}
else if (VARATT_IS_SHORT(attr))
{
/*
* we have to normalize the header length to VARHDRSZ or else the
* callers of this function will be confused.
*/
result = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT + VARHDRSZ;
}
else
{
/* plain untoasted datum */
result = VARSIZE(attr);
}
return result;
}
/* checks the individual attributes of the tuple */
uint32 check_index_tuple_attributes(Relation rel, PageHeader header, int block, int i, char *buffer) {
IndexTuple tuple;
uint32 nerrs = 0;
int j, off;
bits8 * bitmap;
BTPageOpaque opaque;
ereport(DEBUG2,(errmsg("[%d:%d] checking attributes for the tuple", block, i)));
/* get the index tuple and info about the page */
tuple = (IndexTuple)(buffer + header->pd_linp[i].lp_off);
opaque = (BTPageOpaque)(buffer + header->pd_special);
/* current attribute offset - always starts at (buffer + off) */
off = header->pd_linp[i].lp_off + IndexInfoFindDataOffset(tuple->t_info);
ereport(DEBUG3,(errmsg("[%d:%d] tuple has %d attributes", block, (i+1),
RelationGetNumberOfAttributes(rel))));
bitmap = (bits8*)(buffer + header->pd_linp[i].lp_off + sizeof(IndexTupleData));
/* TODO This is mostly copy'n'paste from check_heap_tuple_attributes,
so maybe it could be refactored to share the code. */
/* For left-most tuples on non-leaf pages, there are no data actually
(see src/backend/access/nbtree/README, last paragraph in section "Notes
About Data Representation")
Use P_LEFTMOST/P_ISLEAF to identify such cases (for the leftmost item only)
and set len = 0.
*/
if (P_LEFTMOST(opaque) && (! P_ISLEAF(opaque)) && (i == 0)) {
ereport(DEBUG3, (errmsg("[%d:%d] leftmost tuple on non-leaf block => no data, skipping", block, i)));
return nerrs;
}
/* check all the index attributes */
for (j = 0; j < rel->rd_att->natts; j++) {
/* default length of the attribute */
int len = rel->rd_att->attrs[j]->attlen;
/* copy from src/backend/commands/analyze.c */
bool is_varlena = (!rel->rd_att->attrs[j]->attbyval && len == -1);
bool is_varwidth = (!rel->rd_att->attrs[j]->attbyval && len < 0); /* thus it's "len = -2" */
/* if the attribute is marked as NULL (in the tuple header), skip to the next attribute */
if (IndexTupleHasNulls(tuple) && att_isnull(j, bitmap)) {
ereport(DEBUG3, (errmsg("[%d:%d] attribute '%s' is NULL (skipping)", block, (i+1), rel->rd_att->attrs[j]->attname.data)));
continue;
}
/* fix the alignment (see src/include/access/tupmacs.h) */
off = att_align_pointer(off, rel->rd_att->attrs[j]->attalign, rel->rd_att->attrs[j]->attlen, buffer+off);
if (is_varlena) {
/*
other interesting macros (see postgres.h) - should do something about those ...
VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
*/
len = VARSIZE_ANY(buffer + off);
if (len < 0) {
ereport(WARNING, (errmsg("[%d:%d] attribute '%s' has negative length < 0 (%d)", block, (i+1), rel->rd_att->attrs[j]->attname.data, len)));
++nerrs;
break;
}
if (VARATT_IS_COMPRESSED(buffer + off)) {
/* the raw length should be less than 1G (and positive) */
if ((VARRAWSIZE_4B_C(buffer + off) < 0) || (VARRAWSIZE_4B_C(buffer + off) > 1024*1024)) {
ereport(WARNING, (errmsg("[%d:%d] attribute '%s' has invalid length %d (should be between 0 and 1G)", block, (i+1), rel->rd_att->attrs[j]->attname.data, VARRAWSIZE_4B_C(buffer + off))));
++nerrs;
/* no break here, this does not break the page structure - we may check the other attributes */
}
}
/* FIXME Check if the varlena value may be detoasted. */
} else if (is_varwidth) {
/* get the C-string length (at most to the end of tuple), +1 as it does not include '\0' at the end */
/* if the string is not properly terminated, then this returns 'remaining space + 1' so it's detected */
len = strnlen(buffer + off, header->pd_linp[i].lp_off + len + header->pd_linp[i].lp_len - off) + 1;
}
/* Check if the length makes sense (is not negative and does not overflow
* the tuple end, stop validating the other rows (we don't know where to
* continue anyway). */
if (off + len > (header->pd_linp[i].lp_off + header->pd_linp[i].lp_len)) {
ereport(WARNING, (errmsg("[%d:%d] attribute '%s' (off=%d len=%d) overflows tuple end (off=%d, len=%d)", block, (i+1), rel->rd_att->attrs[j]->attname.data, off, len, header->pd_linp[i].lp_off, header->pd_linp[i].lp_len)));
++nerrs;
break;
}
/* skip to the next attribute */
off += len;
ereport(DEBUG3,(errmsg("[%d:%d] attribute '%s' len=%d", block, (i+1), rel->rd_att->attrs[j]->attname.data, len)));
}
ereport(DEBUG3,(errmsg("[%d:%d] last attribute ends at %d, tuple ends at %d", block, (i+1), off, header->pd_linp[i].lp_off + header->pd_linp[i].lp_len)));
/* after the last attribute, the offset should be exactly the same as the end of the tuple */
if (MAXALIGN(off) != header->pd_linp[i].lp_off + header->pd_linp[i].lp_len) {
ereport(WARNING, (errmsg("[%d:%d] the last attribute ends at %d but the tuple ends at %d", block, (i+1), off, header->pd_linp[i].lp_off + header->pd_linp[i].lp_len)));
++nerrs;
}
return nerrs;
}
/* ----------
* toast_save_datum -
*
* Save one single datum into the secondary relation and return
* a Datum reference for it.
* ----------
*/
static Datum
toast_save_datum(Relation rel, Datum value, int options)
{
Relation toastrel;
Relation toastidx;
HeapTuple toasttup;
TupleDesc toasttupDesc;
Datum t_values[3];
bool t_isnull[3];
CommandId mycid = GetCurrentCommandId(true);
struct varlena *result;
struct varatt_external toast_pointer;
struct
{
struct varlena hdr;
char data[TOAST_MAX_CHUNK_SIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} chunk_data;
int32 chunk_size;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
/*
* Open the toast relation and its index. We can use the index to check
* uniqueness of the OID we assign to the toasted item, even though it has
* additional columns besides OID.
*/
toastrel = heap_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
toasttupDesc = toastrel->rd_att;
toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock);
/*
* Get the data pointer and length, and compute va_rawsize and va_extsize.
*
* va_rawsize is the size of the equivalent fully uncompressed datum, so
* we have to adjust for short headers.
*
* va_extsize is the actual size of the data payload in the toast records.
*/
if (VARATT_IS_SHORT(dval))
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
toast_pointer.va_extsize = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
toast_pointer.va_rawsize = VARRAWSIZE_4B_C(dval) + VARHDRSZ;
toast_pointer.va_extsize = data_todo;
/* Assert that the numbers look like it's compressed */
Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
toast_pointer.va_rawsize = VARSIZE(dval);
toast_pointer.va_extsize = data_todo;
}
/*
* Insert the correct table OID into the result TOAST pointer.
*
* Normally this is the actual OID of the target toast table, but during
* table-rewriting operations such as CLUSTER, we have to insert the OID
* of the table's real permanent toast table instead. rd_toastoid is set
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
toast_pointer.va_toastrelid = rel->rd_toastoid;
else
toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
/*
* Choose an unused OID within the toast table for this toast value.
*/
toast_pointer.va_valueid = GetNewOidWithIndex(toastrel,
RelationGetRelid(toastidx),
(AttrNumber) 1);
/*
* Initialize constant parts of the tuple data
*/
t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
t_values[2] = PointerGetDatum(&chunk_data);
t_isnull[0] = false;
t_isnull[1] = false;
t_isnull[2] = false;
/*
* Split up the item into chunks
*/
while (data_todo > 0)
{
/*
* Calculate the size of this chunk
*/
chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
/*
* Build a tuple and store it
*/
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
toasttup = heap_form_tuple(toasttupDesc, t_values, t_isnull);
heap_insert(toastrel, toasttup, mycid, options, NULL);
/*
* Create the index entry. We cheat a little here by not using
* FormIndexDatum: this relies on the knowledge that the index columns
* are the same as the initial columns of the table.
*
* Note also that there had better not be any user-created index on
* the TOAST table, since we don't bother to update anything else.
*/
index_insert(toastidx, t_values, t_isnull,
&(toasttup->t_self),
toastrel,
toastidx->rd_index->indisunique ?
UNIQUE_CHECK_YES : UNIQUE_CHECK_NO);
/*
* Free memory
*/
heap_freetuple(toasttup);
/*
* Move on to next chunk
*/
data_todo -= chunk_size;
data_p += chunk_size;
}
/*
* Done - close toast relation
*/
index_close(toastidx, RowExclusiveLock);
heap_close(toastrel, RowExclusiveLock);
/*
* Create the TOAST pointer value that we'll return
*/
result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
SET_VARSIZE_EXTERNAL(result, TOAST_POINTER_SIZE);
memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
return PointerGetDatum(result);
}
/* Decode char(N), varchar(N), text, json or xml types */
static int
decode_string(const char *buffer, unsigned int buff_size, unsigned int *out_size)
{
int padding = 0;
/* Skip padding bytes. */
while (*buffer == 0x00)
{
if (buff_size == 0)
return -1;
buff_size--;
buffer++;
padding++;
}
if (VARATT_IS_1B_E(buffer))
{
/*
* 00000001 1-byte length word, unaligned, TOAST pointer
*/
uint32 len = VARSIZE_EXTERNAL(buffer);
int result = 0;
if (len > buff_size)
return -1;
if (blockOptions & BLOCK_DECODE_TOAST)
{
result = ReadStringFromToast(buffer, buff_size, out_size);
}
else
{
CopyAppend("(TOASTED)");
}
*out_size = padding + len;
return result;
}
if (VARATT_IS_1B(buffer))
{
/*
* xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to
* 126b) xxxxxxx is 1 + string length
*/
uint8 len = VARSIZE_1B(buffer);
if (len > buff_size)
return -1;
CopyAppendEncode(buffer + 1, len - 1);
*out_size = padding + len;
return 0;
}
if (VARATT_IS_4B_U(buffer) && buff_size >= 4)
{
/*
* xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
*/
uint32 len = VARSIZE_4B(buffer);
if (len > buff_size)
return -1;
CopyAppendEncode(buffer + 4, len - 4);
*out_size = padding + len;
return 0;
}
if (VARATT_IS_4B_C(buffer) && buff_size >= 8)
{
/*
* xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
*/
int decompress_ret;
uint32 len = VARSIZE_4B(buffer);
uint32 decompressed_len = VARRAWSIZE_4B_C(buffer);
if (len > buff_size)
return -1;
if (decompressed_len > sizeof(decompress_tmp_buff))
{
printf("WARNING: Unable to decompress a string since it's too "
"large (%d bytes after decompressing). Consider increasing "
"decompress_tmp_buff size.\n", decompressed_len);
CopyAppend("(COMPRESSED)");
*out_size = padding + len;
return 0;
}
decompress_ret = pglz_decompress(VARDATA_4B_C(buffer), len - 2 * sizeof(uint32),
decompress_tmp_buff, decompressed_len);
if ((decompress_ret != decompressed_len) || (decompress_ret < 0))
{
printf("WARNING: Unable to decompress a string. Data is corrupted.\n");
CopyAppend("(COMPRESSED)");
*out_size = padding + len;
return 0;
}
CopyAppendEncode(decompress_tmp_buff, decompressed_len);
*out_size = padding + len;
return 0;
}
return -9;
}
