/* ----------
* heap_tuple_untoast_attr -
*
* Public entry point to get back a toasted value from compression
* or external storage.
* ----------
*/
varattrib *
heap_tuple_untoast_attr(varattrib *attr)
{
varattrib *result;
if (VARATT_IS_EXTERNAL(attr))
{
if (VARATT_IS_COMPRESSED(attr))
{
/* ----------
* This is an external stored compressed value
* Fetch it from the toast heap and decompress.
* ----------
*/
varattrib *tmp;
tmp = toast_fetch_datum(attr);
result = (varattrib *) palloc(attr->va_content.va_external.va_rawsize
+ VARHDRSZ);
VARATT_SIZEP(result) = attr->va_content.va_external.va_rawsize
+ VARHDRSZ;
pglz_decompress((PGLZ_Header *) tmp, VARATT_DATA(result));
pfree(tmp);
}
else
{
/*
* This is an external stored plain value
*/
result = toast_fetch_datum(attr);
}
}
else if (VARATT_IS_COMPRESSED(attr))
{
/*
* This is a compressed value inside of the main tuple
*/
result = (varattrib *) palloc(attr->va_content.va_compressed.va_rawsize
+ VARHDRSZ);
VARATT_SIZEP(result) = attr->va_content.va_compressed.va_rawsize
+ VARHDRSZ;
pglz_decompress((PGLZ_Header *) attr, VARATT_DATA(result));
}
else
/*
* This is a plain value inside of the main tuple - why am I
* called?
*/
return attr;
return result;
}
/* ----------
* heap_tuple_untoast_attr -
*
* Public entry point to get back a toasted value from compression
* or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr(struct varlena * attr)
{
if (VARATT_IS_EXTERNAL(attr))
{
/*
* This is an externally stored datum --- fetch it back from there
*/
attr = toast_fetch_datum(attr);
/* If it's compressed, decompress it */
if (VARATT_IS_COMPRESSED(attr))
{
PGLZ_Header *tmp = (PGLZ_Header *) attr;
attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
pglz_decompress(tmp, VARDATA(attr));
pfree(tmp);
}
}
else if (VARATT_IS_COMPRESSED(attr))
{
/*
* This is a compressed value inside of the main tuple
*/
PGLZ_Header *tmp = (PGLZ_Header *) attr;
attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
pglz_decompress(tmp, VARDATA(attr));
}
else if (VARATT_IS_SHORT(attr))
{
/*
* This is a short-header varlena --- convert to 4-byte header format
*/
Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT;
Size new_size = data_size + VARHDRSZ;
struct varlena *new_attr;
new_attr = (struct varlena *) palloc(new_size);
SET_VARSIZE(new_attr, new_size);
memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size);
attr = new_attr;
}
return attr;
}
/* ----------
* heap_tuple_untoast_attr_slice -
*
* Public entry point to get back part of a toasted value
* from compression or external storage.
* ----------
*/
varattrib *
heap_tuple_untoast_attr_slice(varattrib *attr, int32 sliceoffset, int32 slicelength)
{
varattrib *preslice;
varattrib *result;
int32 attrsize;
if (VARATT_IS_COMPRESSED(attr))
{
varattrib *tmp;
if (VARATT_IS_EXTERNAL(attr))
tmp = toast_fetch_datum(attr);
else
{
tmp = attr; /* compressed in main tuple */
}
preslice = (varattrib *) palloc(attr->va_content.va_external.va_rawsize
+ VARHDRSZ);
VARATT_SIZEP(preslice) = attr->va_content.va_external.va_rawsize + VARHDRSZ;
pglz_decompress((PGLZ_Header *) tmp, VARATT_DATA(preslice));
if (tmp != attr)
pfree(tmp);
}
else
{
/* Plain value */
if (VARATT_IS_EXTERNAL(attr))
{
/* fast path */
return (toast_fetch_datum_slice(attr, sliceoffset, slicelength));
}
else
preslice = attr;
}
/* slicing of datum for compressed cases and plain value */
attrsize = VARSIZE(preslice) - VARHDRSZ;
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
slicelength = 0;
}
if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
slicelength = attrsize - sliceoffset;
result = (varattrib *) palloc(slicelength + VARHDRSZ);
VARATT_SIZEP(result) = slicelength + VARHDRSZ;
memcpy(VARDATA(result), VARDATA(preslice) + sliceoffset, slicelength);
if (preslice != attr)
pfree(preslice);
return result;
}
/**
* If this function is changed then update varattrib_untoast_len as well
*/
void varattrib_untoast_ptr_len(Datum d, char **datastart, int *len, void **tofree)
{
if (DatumGetPointer(d) == NULL)
{
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg(" Unable to detoast datum "),
errprintstack(true)));
}
struct varlena *va = (struct varlena *) DatumGetPointer(d);
varattrib *attr = (varattrib *) va;
*len = -1;
*tofree = NULL;
if(VARATT_IS_EXTENDED(attr))
{
if(VARATT_IS_EXTERNAL(attr))
{
attr = (varattrib *)toast_fetch_datum((struct varlena *)attr);
/* toast_fetch_datum will palloc, so set it up for free */
*tofree = attr;
}
if(VARATT_IS_COMPRESSED(attr))
{
PGLZ_Header *tmp = (PGLZ_Header *) attr;
attr = (varattrib *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
pglz_decompress(tmp, VARDATA(attr));
/* If tofree is set, that is, we get it from toast_fetch_datum.
* We need to free it here
*/
if(*tofree)
pfree(*tofree);
*tofree = attr;
}
else if(VARATT_IS_SHORT(attr))
{
/* Warning! Return unaligned pointer! */
*len = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT;
*datastart = VARDATA_SHORT(attr);
attr = NULL;
}
}
if(*len == -1)
{
*datastart = VARDATA(attr);
*len = VARSIZE(attr) - VARHDRSZ;
}
Assert(*len >= 0);
}
/* Decompresses sparse counters */
static HLLCounter
hll_decompress_dense_V1(HLLCounter hloglog)
{
char * dest;
int m,i;
HLLCounter htemp;
/* reset b to positive value for calcs and to indicate data is
* decompressed */
hloglog->b = -1 * (hloglog->b);
/* allocate and zero an array large enough to hold all the decompressed
* bins */
m = (int) pow(2,hloglog->b);
dest = malloc(m);
memset(dest,0,m);
/* decompress the data */
pglz_decompress((PGLZ_Header *)hloglog->data,dest);
/* copy the struct internals but not the data into a counter with enough
* space for the uncompressed data */
htemp = palloc(sizeof(HLLData) + (int)ceil((m * hloglog->binbits / 8.0)));
memcpy(htemp,hloglog,sizeof(HLLData));
hloglog = htemp;
/* set the registers to the appropriate value based on the decompressed
* data */
for (i=0; i<m; i++){
HLL_DENSE_SET_REGISTER(hloglog->data,i,dest[i],hloglog->binbits);
}
/* set the varsize to the appropriate length */
SET_VARSIZE(hloglog,sizeof(HLLData) + (int)ceil((m * hloglog->binbits / 8.0)) );
/* free allocated memory */
if (dest){
free(dest);
}
return hloglog;
}
/* ----------
* heap_tuple_untoast_attr -
*
* Public entry point to get back a toasted value from compression
* or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr(struct varlena *attr)
{
if (VARATT_IS_EXTERNAL(attr))
{
/*
* This is an externally stored datum --- fetch it back from there
*/
attr = toast_fetch_datum(attr);
/* fall through to IS_COMPRESSED if it's a compressed external datum */
}
if (VARATT_IS_COMPRESSED(attr))
{
/*
* This is a compressed value inside of the main tuple
*/
PGLZ_Header *tmp = (PGLZ_Header *) attr;
attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
pglz_decompress(tmp, VARDATA(attr));
}
else if (VARATT_IS_SHORT(attr))
{
/*
* This is a short-header varlena --- convert to 4-byte header format
*/
Size data_size = VARSIZE_SHORT(attr);
Size new_size = data_size - VARHDRSZ_SHORT + VARHDRSZ;
varattrib *tmp = (varattrib *)attr;
/* This is a "short" varlena header but is otherwise a normal varlena */
attr = (struct varlena *) palloc(new_size);
SET_VARSIZE(attr, new_size);
memcpy(VARDATA(attr), VARDATA_SHORT(tmp), data_size - VARHDRSZ_SHORT);
}
return attr;
}
/*
* decompress_data
*
* Decompress the bytea buffer and return result as bytea, this may be a page
* with its hole filled with zeros or a page without a hole.
*/
Datum
decompress_data(PG_FUNCTION_ARGS)
{
bytea *compress_data = PG_GETARG_BYTEA_P(0);
int16 raw_len = PG_GETARG_INT16(1);
bytea *res;
char *uncompress_buffer;
uncompress_buffer = palloc(raw_len);
if (pglz_decompress(VARDATA(compress_data),
VARSIZE(compress_data) - VARHDRSZ,
uncompress_buffer, raw_len) < 0)
ereport(ERROR, (errmsg("Decompression failed...")));
/* Build result */
res = (bytea *) palloc(raw_len + VARHDRSZ);
SET_VARSIZE(res, raw_len + VARHDRSZ);
memcpy(VARDATA(res), uncompress_buffer, raw_len);
pfree(uncompress_buffer);
PG_RETURN_BYTEA_P(res);
}
static int DumpCompressedString(const char *data, int32 decompressed_size)
{
int decompress_ret;
char *decompress_tmp_buff = malloc(TOAST_COMPRESS_RAWSIZE(data));
decompress_ret = pglz_decompress(TOAST_COMPRESS_RAWDATA(data),
decompressed_size - TOAST_COMPRESS_HEADER_SIZE,
decompress_tmp_buff, TOAST_COMPRESS_RAWSIZE(data));
if ((decompress_ret != TOAST_COMPRESS_RAWSIZE(data)) ||
(decompress_ret < 0))
{
printf("WARNING: Unable to decompress a string. Data is corrupted.\n");
printf("Returned %d while expected %d.\n", decompress_ret,
decompressed_size);
}
else
{
CopyAppendEncode(decompress_tmp_buff, *((uint32 *)data));
}
free(decompress_tmp_buff);
return decompress_ret;
}
/* ----------
* heap_tuple_untoast_attr_slice -
*
* Public entry point to get back part of a toasted value
* from compression or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr_slice(struct varlena * attr,
int32 sliceoffset, int32 slicelength)
{
struct varlena *preslice;
struct varlena *result;
char *attrdata;
int32 attrsize;
if (VARATT_IS_EXTERNAL(attr))
{
struct varatt_external toast_pointer;
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/* fetch it back (compressed marker will get set automatically) */
preslice = toast_fetch_datum(attr);
}
else
preslice = attr;
if (VARATT_IS_COMPRESSED(preslice))
{
PGLZ_Header *tmp = (PGLZ_Header *) preslice;
Size size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ;
preslice = (struct varlena *) palloc(size);
SET_VARSIZE(preslice, size);
pglz_decompress(tmp, VARDATA(preslice));
if (tmp != (PGLZ_Header *) attr)
pfree(tmp);
}
if (VARATT_IS_SHORT(preslice))
{
attrdata = VARDATA_SHORT(preslice);
attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
}
else
{
attrdata = VARDATA(preslice);
attrsize = VARSIZE(preslice) - VARHDRSZ;
}
/* slicing of datum for compressed cases and plain value */
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
slicelength = 0;
}
if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
slicelength = attrsize - sliceoffset;
result = (struct varlena *) palloc(slicelength + VARHDRSZ);
SET_VARSIZE(result, slicelength + VARHDRSZ);
memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
if (preslice != attr)
pfree(preslice);
return result;
}
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
{
return pglz_decompress(src, src_size, dst, dst_size);
}
/* ----------
* heap_tuple_untoast_attr_slice -
*
* Public entry point to get back part of a toasted value
* from compression or external storage.
* ----------
*/
struct varlena *
heap_tuple_untoast_attr_slice(struct varlena * attr,
int32 sliceoffset, int32 slicelength)
{
varattrib *preslice;
varattrib *result;
char *attrdata;
int32 attrsize;
if (VARATT_IS_EXTERNAL(attr))
{
/* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(attr))
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/* this automatically sets the compressed flag if appropriate */
preslice = (varattrib *)toast_fetch_datum(attr);
}
else
preslice = (varattrib *)attr;
if (VARATT_IS_COMPRESSED(preslice))
{
unsigned size;
PGLZ_Header *tmp;
tmp = (PGLZ_Header *) preslice;
size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ;
preslice = (varattrib *) palloc(size);
SET_VARSIZE(preslice, size);
pglz_decompress(tmp, VARDATA(preslice));
if (tmp != (PGLZ_Header *) attr)
pfree(tmp);
}
if (VARATT_IS_SHORT(preslice))
{
attrdata = VARDATA_SHORT(preslice);
attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
}
else
{
attrdata = VARDATA(preslice);
attrsize = VARSIZE(preslice) - VARHDRSZ;
}
/* slicing of datum for compressed cases and plain value */
if (sliceoffset >= attrsize)
{
sliceoffset = 0;
slicelength = 0;
}
if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
slicelength = attrsize - sliceoffset;
result = (varattrib *) palloc(slicelength + VARHDRSZ);
SET_VARSIZE(result, slicelength + VARHDRSZ);
memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
if ((struct varlena *)preslice != (struct varlena *)attr)
pfree(preslice);
return (struct varlena *)result;
}
/* Decode char(N), varchar(N), text, json or xml types */
static int
decode_string(const char *buffer, unsigned int buff_size, unsigned int *out_size)
{
int padding = 0;
/* Skip padding bytes. */
while (*buffer == 0x00)
{
if (buff_size == 0)
return -1;
buff_size--;
buffer++;
padding++;
}
if (VARATT_IS_1B_E(buffer))
{
/*
* 00000001 1-byte length word, unaligned, TOAST pointer
*/
uint32 len = VARSIZE_EXTERNAL(buffer);
int result = 0;
if (len > buff_size)
return -1;
if (blockOptions & BLOCK_DECODE_TOAST)
{
result = ReadStringFromToast(buffer, buff_size, out_size);
}
else
{
CopyAppend("(TOASTED)");
}
*out_size = padding + len;
return result;
}
if (VARATT_IS_1B(buffer))
{
/*
* xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to
* 126b) xxxxxxx is 1 + string length
*/
uint8 len = VARSIZE_1B(buffer);
if (len > buff_size)
return -1;
CopyAppendEncode(buffer + 1, len - 1);
*out_size = padding + len;
return 0;
}
if (VARATT_IS_4B_U(buffer) && buff_size >= 4)
{
/*
* xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
*/
uint32 len = VARSIZE_4B(buffer);
if (len > buff_size)
return -1;
CopyAppendEncode(buffer + 4, len - 4);
*out_size = padding + len;
return 0;
}
if (VARATT_IS_4B_C(buffer) && buff_size >= 8)
{
/*
* xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
*/
int decompress_ret;
uint32 len = VARSIZE_4B(buffer);
uint32 decompressed_len = VARRAWSIZE_4B_C(buffer);
if (len > buff_size)
return -1;
if (decompressed_len > sizeof(decompress_tmp_buff))
{
printf("WARNING: Unable to decompress a string since it's too "
"large (%d bytes after decompressing). Consider increasing "
"decompress_tmp_buff size.\n", decompressed_len);
CopyAppend("(COMPRESSED)");
*out_size = padding + len;
return 0;
}
decompress_ret = pglz_decompress(VARDATA_4B_C(buffer), len - 2 * sizeof(uint32),
decompress_tmp_buff, decompressed_len);
if ((decompress_ret != decompressed_len) || (decompress_ret < 0))
{
printf("WARNING: Unable to decompress a string. Data is corrupted.\n");
CopyAppend("(COMPRESSED)");
*out_size = padding + len;
return 0;
}
CopyAppendEncode(decompress_tmp_buff, decompressed_len);
*out_size = padding + len;
return 0;
}
return -9;
}
