/* * Print a value into the StringInfo provided by caller. */ static void print_value(StringInfo s, Datum origval, Oid typid, bool isnull) { Oid typoutput; bool typisvarlena; /* Query output function */ getTypeOutputInfo(typid, &typoutput, &typisvarlena); /* Print value */ if (isnull) appendStringInfoString(s, "null"); else if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) appendStringInfoString(s, "unchanged-toast-datum"); else if (!typisvarlena) print_literal(s, typid, OidOutputFunctionCall(typoutput, origval)); else { /* Definitely detoasted Datum */ Datum val; val = PointerGetDatum(PG_DETOAST_DATUM(origval)); print_literal(s, typid, OidOutputFunctionCall(typoutput, val)); } }
static void print_value(StringInfo s, TupleDesc tupdesc, HeapTuple tuple, int i) { bool typisvarlena; bool isnull; Oid typoutput; Form_pg_attribute attr = tupdesc->attrs[i]; Datum origval = fastgetattr(tuple, i + 1, tupdesc, &isnull); Oid typid = attr->atttypid; getTypeOutputInfo(typid, &typoutput, &typisvarlena); if (isnull) { appendStringInfoString(s, "null"); } else if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) { appendStringInfoString(s, "\"???unchanged-toast-datum???\""); } else if (!typisvarlena) { print_literal(s, typid, OidOutputFunctionCall(typoutput, origval)); } else { Datum val = PointerGetDatum(PG_DETOAST_DATUM(origval)); print_literal(s, typid, OidOutputFunctionCall(typoutput, val)); } }
Datum make_tuple_indirect(PG_FUNCTION_ARGS) { HeapTupleHeader rec = PG_GETARG_HEAPTUPLEHEADER(0); HeapTupleData tuple; int ncolumns; Datum *values; bool *nulls; Oid tupType; int32 tupTypmod; TupleDesc tupdesc; HeapTuple newtup; int i; MemoryContext old_context; /* Extract type info from the tuple itself */ tupType = HeapTupleHeaderGetTypeId(rec); tupTypmod = HeapTupleHeaderGetTypMod(rec); tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod); ncolumns = tupdesc->natts; /* Build a temporary HeapTuple control structure */ tuple.t_len = HeapTupleHeaderGetDatumLength(rec); ItemPointerSetInvalid(&(tuple.t_self)); tuple.t_tableOid = InvalidOid; tuple.t_data = rec; values = (Datum *) palloc(ncolumns * sizeof(Datum)); nulls = (bool *) palloc(ncolumns * sizeof(bool)); heap_deform_tuple(&tuple, tupdesc, values, nulls); old_context = MemoryContextSwitchTo(TopTransactionContext); for (i = 0; i < ncolumns; i++) { struct varlena *attr; struct varlena *new_attr; struct varatt_indirect redirect_pointer; /* only work on existing, not-null varlenas */ if (TupleDescAttr(tupdesc, i)->attisdropped || nulls[i] || TupleDescAttr(tupdesc, i)->attlen != -1) continue; attr = (struct varlena *) DatumGetPointer(values[i]); /* don't recursively indirect */ if (VARATT_IS_EXTERNAL_INDIRECT(attr)) continue; /* copy datum, so it still lives later */ if (VARATT_IS_EXTERNAL_ONDISK(attr)) attr = heap_tuple_fetch_attr(attr); else { struct varlena *oldattr = attr; attr = palloc0(VARSIZE_ANY(oldattr)); memcpy(attr, oldattr, VARSIZE_ANY(oldattr)); } /* build indirection Datum */ new_attr = (struct varlena *) palloc0(INDIRECT_POINTER_SIZE); redirect_pointer.pointer = attr; SET_VARTAG_EXTERNAL(new_attr, VARTAG_INDIRECT); memcpy(VARDATA_EXTERNAL(new_attr), &redirect_pointer, sizeof(redirect_pointer)); values[i] = PointerGetDatum(new_attr); } newtup = heap_form_tuple(tupdesc, values, nulls); pfree(values); pfree(nulls); ReleaseTupleDesc(tupdesc); MemoryContextSwitchTo(old_context); /* * We intentionally don't use PG_RETURN_HEAPTUPLEHEADER here, because that * would cause the indirect toast pointers to be flattened out of the * tuple immediately, rendering subsequent testing irrelevant. So just * return the HeapTupleHeader pointer as-is. This violates the general * rule that composite Datums shouldn't contain toast pointers, but so * long as the regression test scripts don't insert the result of this * function into a container type (record, array, etc) it should be OK. */ PG_RETURN_POINTER(newtup->t_data); }
/* * tuple_data_split_internal * * Split raw tuple data taken directly from a page into an array of bytea * elements. This routine does a lookup on NULL values and creates array * elements accordingly. This is a reimplementation of nocachegetattr() * in heaptuple.c simplified for educational purposes. */ static Datum tuple_data_split_internal(Oid relid, char *tupdata, uint16 tupdata_len, uint16 t_infomask, uint16 t_infomask2, bits8 *t_bits, bool do_detoast) { ArrayBuildState *raw_attrs; int nattrs; int i; int off = 0; Relation rel; TupleDesc tupdesc; /* Get tuple descriptor from relation OID */ rel = relation_open(relid, AccessShareLock); tupdesc = RelationGetDescr(rel); raw_attrs = initArrayResult(BYTEAOID, CurrentMemoryContext, false); nattrs = tupdesc->natts; if (nattrs < (t_infomask2 & HEAP_NATTS_MASK)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("number of attributes in tuple header is greater than number of attributes in tuple descriptor"))); for (i = 0; i < nattrs; i++) { Form_pg_attribute attr; bool is_null; bytea *attr_data = NULL; attr = TupleDescAttr(tupdesc, i); /* * Tuple header can specify less attributes than tuple descriptor as * ALTER TABLE ADD COLUMN without DEFAULT keyword does not actually * change tuples in pages, so attributes with numbers greater than * (t_infomask2 & HEAP_NATTS_MASK) should be treated as NULL. */ if (i >= (t_infomask2 & HEAP_NATTS_MASK)) is_null = true; else is_null = (t_infomask & HEAP_HASNULL) && att_isnull(i, t_bits); if (!is_null) { int len; if (attr->attlen == -1) { off = att_align_pointer(off, attr->attalign, -1, tupdata + off); /* * As VARSIZE_ANY throws an exception if it can't properly * detect the type of external storage in macros VARTAG_SIZE, * this check is repeated to have a nicer error handling. */ if (VARATT_IS_EXTERNAL(tupdata + off) && !VARATT_IS_EXTERNAL_ONDISK(tupdata + off) && !VARATT_IS_EXTERNAL_INDIRECT(tupdata + off)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("first byte of varlena attribute is incorrect for attribute %d", i))); len = VARSIZE_ANY(tupdata + off); } else { off = att_align_nominal(off, attr->attalign); len = attr->attlen; } if (tupdata_len < off + len) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("unexpected end of tuple data"))); if (attr->attlen == -1 && do_detoast) attr_data = DatumGetByteaPCopy(tupdata + off); else { attr_data = (bytea *) palloc(len + VARHDRSZ); SET_VARSIZE(attr_data, len + VARHDRSZ); memcpy(VARDATA(attr_data), tupdata + off, len); } off = att_addlength_pointer(off, attr->attlen, tupdata + off); } raw_attrs = accumArrayResult(raw_attrs, PointerGetDatum(attr_data), is_null, BYTEAOID, CurrentMemoryContext); if (attr_data) pfree(attr_data); } if (tupdata_len != off) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("end of tuple reached without looking at all its data"))); relation_close(rel, AccessShareLock); return makeArrayResult(raw_attrs, CurrentMemoryContext); }
/* * Write a tuple to the outputstream, in the most efficient format possible. */ static void pglogical_write_tuple(StringInfo out, PGLogicalOutputData *data, Relation rel, HeapTuple tuple) { TupleDesc desc; Datum values[MaxTupleAttributeNumber]; bool isnull[MaxTupleAttributeNumber]; int i; uint16 nliveatts = 0; desc = RelationGetDescr(rel); pq_sendbyte(out, 'T'); /* sending TUPLE */ for (i = 0; i < desc->natts; i++) { if (desc->attrs[i]->attisdropped) continue; nliveatts++; } pq_sendint(out, nliveatts, 2); /* try to allocate enough memory from the get go */ enlargeStringInfo(out, tuple->t_len + nliveatts * (1 + 4)); /* * XXX: should this prove to be a relevant bottleneck, it might be * interesting to inline heap_deform_tuple() here, we don't actually need * the information in the form we get from it. */ heap_deform_tuple(tuple, desc, values, isnull); for (i = 0; i < desc->natts; i++) { HeapTuple typtup; Form_pg_type typclass; Form_pg_attribute att = desc->attrs[i]; char transfer_type; /* skip dropped columns */ if (att->attisdropped) continue; if (isnull[i]) { pq_sendbyte(out, 'n'); /* null column */ continue; } else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i])) { pq_sendbyte(out, 'u'); /* unchanged toast column */ continue; } typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid)); if (!HeapTupleIsValid(typtup)) elog(ERROR, "cache lookup failed for type %u", att->atttypid); typclass = (Form_pg_type) GETSTRUCT(typtup); transfer_type = decide_datum_transfer(att, typclass, data->allow_internal_basetypes, data->allow_binary_basetypes); pq_sendbyte(out, transfer_type); switch (transfer_type) { case 'b': /* internal-format binary data follows */ /* pass by value */ if (att->attbyval) { pq_sendint(out, att->attlen, 4); /* length */ enlargeStringInfo(out, att->attlen); store_att_byval(out->data + out->len, values[i], att->attlen); out->len += att->attlen; out->data[out->len] = '\0'; } /* fixed length non-varlena pass-by-reference type */ else if (att->attlen > 0) { pq_sendint(out, att->attlen, 4); /* length */ appendBinaryStringInfo(out, DatumGetPointer(values[i]), att->attlen); } /* varlena type */ else if (att->attlen == -1) { char *data = DatumGetPointer(values[i]); /* send indirect datums inline */ if (VARATT_IS_EXTERNAL_INDIRECT(values[i])) { struct varatt_indirect redirect; VARATT_EXTERNAL_GET_POINTER(redirect, data); data = (char *) redirect.pointer; } Assert(!VARATT_IS_EXTERNAL(data)); pq_sendint(out, VARSIZE_ANY(data), 4); /* length */ appendBinaryStringInfo(out, data, VARSIZE_ANY(data)); } else elog(ERROR, "unsupported tuple type"); break; case 's': /* binary send/recv data follows */ { bytea *outputbytes; int len; outputbytes = OidSendFunctionCall(typclass->typsend, values[i]); len = VARSIZE(outputbytes) - VARHDRSZ; pq_sendint(out, len, 4); /* length */ pq_sendbytes(out, VARDATA(outputbytes), len); /* data */ pfree(outputbytes); } break; default: { char *outputstr; int len; outputstr = OidOutputFunctionCall(typclass->typoutput, values[i]); len = strlen(outputstr) + 1; pq_sendint(out, len, 4); /* length */ appendBinaryStringInfo(out, outputstr, len); /* data */ pfree(outputstr); } } ReleaseSysCache(typtup); } }
Datum make_tuple_indirect(PG_FUNCTION_ARGS) { HeapTupleHeader rec = PG_GETARG_HEAPTUPLEHEADER(0); HeapTupleData tuple; int ncolumns; Datum *values; bool *nulls; Oid tupType; int32 tupTypmod; TupleDesc tupdesc; HeapTuple newtup; int i; MemoryContext old_context; /* Extract type info from the tuple itself */ tupType = HeapTupleHeaderGetTypeId(rec); tupTypmod = HeapTupleHeaderGetTypMod(rec); tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod); ncolumns = tupdesc->natts; /* Build a temporary HeapTuple control structure */ tuple.t_len = HeapTupleHeaderGetDatumLength(rec); ItemPointerSetInvalid(&(tuple.t_self)); tuple.t_tableOid = InvalidOid; tuple.t_data = rec; values = (Datum *) palloc(ncolumns * sizeof(Datum)); nulls = (bool *) palloc(ncolumns * sizeof(bool)); heap_deform_tuple(&tuple, tupdesc, values, nulls); old_context = MemoryContextSwitchTo(TopTransactionContext); for (i = 0; i < ncolumns; i++) { struct varlena *attr; struct varlena *new_attr; struct varatt_indirect redirect_pointer; /* only work on existing, not-null varlenas */ if (tupdesc->attrs[i]->attisdropped || nulls[i] || tupdesc->attrs[i]->attlen != -1) continue; attr = (struct varlena *) DatumGetPointer(values[i]); /* don't recursively indirect */ if (VARATT_IS_EXTERNAL_INDIRECT(attr)) continue; /* copy datum, so it still lives later */ if (VARATT_IS_EXTERNAL_ONDISK(attr)) attr = heap_tuple_fetch_attr(attr); else { struct varlena *oldattr = attr; attr = palloc0(VARSIZE_ANY(oldattr)); memcpy(attr, oldattr, VARSIZE_ANY(oldattr)); } /* build indirection Datum */ new_attr = (struct varlena *) palloc0(INDIRECT_POINTER_SIZE); redirect_pointer.pointer = attr; SET_VARTAG_EXTERNAL(new_attr, VARTAG_INDIRECT); memcpy(VARDATA_EXTERNAL(new_attr), &redirect_pointer, sizeof(redirect_pointer)); values[i] = PointerGetDatum(new_attr); } newtup = heap_form_tuple(tupdesc, values, nulls); pfree(values); pfree(nulls); ReleaseTupleDesc(tupdesc); MemoryContextSwitchTo(old_context); PG_RETURN_HEAPTUPLEHEADER(newtup->t_data); }
/* * Write a tuple to the outputstream, in the most efficient format possible. */ static void logicalrep_write_tuple(StringInfo out, Relation rel, HeapTuple tuple) { TupleDesc desc; Datum values[MaxTupleAttributeNumber]; bool isnull[MaxTupleAttributeNumber]; int i; uint16 nliveatts = 0; desc = RelationGetDescr(rel); for (i = 0; i < desc->natts; i++) { if (TupleDescAttr(desc, i)->attisdropped) continue; nliveatts++; } pq_sendint(out, nliveatts, 2); /* try to allocate enough memory from the get-go */ enlargeStringInfo(out, tuple->t_len + nliveatts * (1 + 4)); heap_deform_tuple(tuple, desc, values, isnull); /* Write the values */ for (i = 0; i < desc->natts; i++) { HeapTuple typtup; Form_pg_type typclass; Form_pg_attribute att = TupleDescAttr(desc, i); char *outputstr; /* skip dropped columns */ if (att->attisdropped) continue; if (isnull[i]) { pq_sendbyte(out, 'n'); /* null column */ continue; } else if (att->attlen == -1 && VARATT_IS_EXTERNAL_ONDISK(values[i])) { pq_sendbyte(out, 'u'); /* unchanged toast column */ continue; } typtup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(att->atttypid)); if (!HeapTupleIsValid(typtup)) elog(ERROR, "cache lookup failed for type %u", att->atttypid); typclass = (Form_pg_type) GETSTRUCT(typtup); pq_sendbyte(out, 't'); /* 'text' data follows */ outputstr = OidOutputFunctionCall(typclass->typoutput, values[i]); pq_sendcountedtext(out, outputstr, strlen(outputstr), false); pfree(outputstr); ReleaseSysCache(typtup); } }
static int ReadStringFromToast(const char *buffer, unsigned int buff_size, unsigned int* out_size) { int result = 0; /* If toasted value is on disk, we'll try to restore it. */ if (VARATT_IS_EXTERNAL_ONDISK(buffer)) { varatt_external toast_ptr; char *toast_data = NULL; /* Number of chunks the TOAST data is divided into */ int32 num_chunks; /* Actual size of external TOASTed value */ int32 toast_ext_size; /* Path to directory with TOAST realtion file */ char *toast_relation_path; /* Filename of TOAST relation file */ char toast_relation_filename[MAXPGPATH]; FILE *toast_rel_fp; unsigned int block_options = 0; unsigned int control_options = 0; VARATT_EXTERNAL_GET_POINTER(toast_ptr, buffer); printf(" TOAST value. Raw size: %8d, external size: %8d, " "value id: %6d, toast relation id: %6d\n", toast_ptr.va_rawsize, toast_ptr.va_extsize, toast_ptr.va_valueid, toast_ptr.va_toastrelid); /* Extract TOASTed value */ toast_ext_size = toast_ptr.va_extsize; num_chunks = (toast_ext_size - 1) / TOAST_MAX_CHUNK_SIZE + 1; printf(" Number of chunks: %d\n", num_chunks); /* Open TOAST relation file */ toast_relation_path = strdup(fileName); get_parent_directory(toast_relation_path); sprintf(toast_relation_filename, "%s/%d", toast_relation_path, toast_ptr.va_toastrelid); printf(" Read TOAST relation %s\n", toast_relation_filename); toast_rel_fp = fopen(toast_relation_filename, "rb"); if (!toast_rel_fp) { printf("Cannot open TOAST relation %s\n", toast_relation_filename); result = -1; } if (result == 0) { unsigned int toast_relation_block_size = GetBlockSize(toast_rel_fp); fseek(toast_rel_fp, 0, SEEK_SET); toast_data = malloc(toast_ptr.va_rawsize); result = DumpFileContents(block_options, control_options, toast_rel_fp, toast_relation_block_size, -1, /* no start block */ -1, /* no end block */ true, /* is toast relation */ toast_ptr.va_valueid, toast_ptr.va_extsize, toast_data); if (result == 0) { if (VARATT_EXTERNAL_IS_COMPRESSED(toast_ptr)) result = DumpCompressedString(toast_data, toast_ext_size); else CopyAppendEncode(toast_data, toast_ext_size); } else { printf("Error in TOAST file.\n"); } free(toast_data); } fclose(toast_rel_fp); free(toast_relation_path); } /* If tag is indirect or expanded, it was stored in memory. */ else { CopyAppend("(TOASTED IN MEMORY)"); } return result; }