/* * Determine size of a large object * * NOTE: LOs can contain gaps, just like Unix files. We actually return * the offset of the last byte + 1. */ static uint32 inv_getsize(LargeObjectDesc *obj_desc) { bool found = false; uint32 lastbyte = 0; ScanKeyData skey[1]; IndexScanDesc sd; HeapTuple tuple; Assert(PointerIsValid(obj_desc)); open_lo_relation(); ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); sd = index_beginscan(lo_heap_r, lo_index_r, obj_desc->snapshot, 1, skey); /* * Because the pg_largeobject index is on both loid and pageno, but we * constrain only loid, a backwards scan should visit all pages of the * large object in reverse pageno order. So, it's sufficient to examine * the first valid tuple (== last valid page). */ while ((tuple = index_getnext(sd, BackwardScanDirection)) != NULL) { Form_pg_largeobject data; bytea *datafield; bool pfreeit; found = true; if (HeapTupleHasNulls(tuple)) /* paranoia */ elog(ERROR, "null field found in pg_largeobject"); data = (Form_pg_largeobject) GETSTRUCT(tuple); datafield = &(data->data); /* see note at top of file */ pfreeit = false; if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } lastbyte = data->pageno * LOBLKSIZE + getbytealen(datafield); if (pfreeit) pfree(datafield); break; } index_endscan(sd); if (!found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u does not exist", obj_desc->id))); return lastbyte; }
/* * Determine size of a large object * * NOTE: LOs can contain gaps, just like Unix files. We actually return * the offset of the last byte + 1. */ static uint64 inv_getsize(LargeObjectDesc *obj_desc) { uint64 lastbyte = 0; ScanKeyData skey[1]; SysScanDesc sd; HeapTuple tuple; Assert(PointerIsValid(obj_desc)); open_lo_relation(); ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); sd = systable_beginscan_ordered(lo_heap_r, lo_index_r, obj_desc->snapshot, 1, skey); /* * Because the pg_largeobject index is on both loid and pageno, but we * constrain only loid, a backwards scan should visit all pages of the * large object in reverse pageno order. So, it's sufficient to examine * the first valid tuple (== last valid page). */ tuple = systable_getnext_ordered(sd, BackwardScanDirection); if (HeapTupleIsValid(tuple)) { Form_pg_largeobject data; bytea *datafield; bool pfreeit; if (HeapTupleHasNulls(tuple)) /* paranoia */ elog(ERROR, "null field found in pg_largeobject"); data = (Form_pg_largeobject) GETSTRUCT(tuple); datafield = &(data->data); /* see note at top of file */ pfreeit = false; if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } lastbyte = (uint64) data->pageno * LOBLKSIZE + getbytealen(datafield); if (pfreeit) pfree(datafield); } systable_endscan_ordered(sd); return lastbyte; }
HeapTuple CheckerConstraints(Checker *checker, HeapTuple tuple, int *parsing_field) { if (checker->has_constraints) { *parsing_field = 0; /* Place tuple in tuple slot */ ExecStoreTuple(tuple, checker->slot, InvalidBuffer, false); /* Check the constraints of the tuple */ ExecConstraints(checker->resultRelInfo, checker->slot, checker->estate); } else if (checker->has_not_null && HeapTupleHasNulls(tuple)) { /* * Even if CHECK_CONSTRAINTS is not specified, check NOT NULL constraint */ TupleDesc desc = checker->desc; int i; for (i = 0; i < desc->natts; i++) { if (desc->attrs[i]->attnotnull && att_isnull(i, tuple->t_data->t_bits)) { *parsing_field = i + 1; /* 1 origin */ ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("null value in column \"%s\" violates not-null constraint", NameStr(desc->attrs[i]->attname)))); } } } return tuple; }
void inv_truncate(LargeObjectDesc *obj_desc, int len) { int32 pageno = (int32) (len / LOBLKSIZE); int off; ScanKeyData skey[2]; SysScanDesc sd; HeapTuple oldtuple; Form_pg_largeobject olddata; struct { bytea hdr; char data[LOBLKSIZE]; /* make struct big enough */ int32 align_it; /* ensure struct is aligned well enough */ } workbuf; char *workb = VARDATA(&workbuf.hdr); HeapTuple newtup; Datum values[Natts_pg_largeobject]; bool nulls[Natts_pg_largeobject]; bool replace[Natts_pg_largeobject]; CatalogIndexState indstate; Assert(PointerIsValid(obj_desc)); /* enforce writability because snapshot is probably wrong otherwise */ if ((obj_desc->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("large object %u was not opened for writing", obj_desc->id))); /* check existence of the target largeobject */ if (!LargeObjectExists(obj_desc->id)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u was already dropped", obj_desc->id))); open_lo_relation(); indstate = CatalogOpenIndexes(lo_heap_r); /* * Set up to find all pages with desired loid and pageno >= target */ ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); sd = systable_beginscan_ordered(lo_heap_r, lo_index_r, obj_desc->snapshot, 2, skey); /* * If possible, get the page the truncation point is in. The truncation * point may be beyond the end of the LO or in a hole. */ olddata = NULL; if ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { if (HeapTupleHasNulls(oldtuple)) /* paranoia */ elog(ERROR, "null field found in pg_largeobject"); olddata = (Form_pg_largeobject) GETSTRUCT(oldtuple); Assert(olddata->pageno >= pageno); } /* * If we found the page of the truncation point we need to truncate the * data in it. Otherwise if we're in a hole, we need to create a page to * mark the end of data. */ if (olddata != NULL && olddata->pageno == pageno) { /* First, load old data into workbuf */ bytea *datafield = &(olddata->data); /* see note at top of * file */ bool pfreeit = false; int pagelen; if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } pagelen = getbytealen(datafield); Assert(pagelen <= LOBLKSIZE); memcpy(workb, VARDATA(datafield), pagelen); if (pfreeit) pfree(datafield); /* * Fill any hole */ off = len % LOBLKSIZE; if (off > pagelen) MemSet(workb + pagelen, 0, off - pagelen); /* compute length of new page */ SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); /* * Form and insert updated tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); memset(replace, false, sizeof(replace)); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); replace[Anum_pg_largeobject_data - 1] = true; newtup = heap_modify_tuple(oldtuple, RelationGetDescr(lo_heap_r), values, nulls, replace); simple_heap_update(lo_heap_r, &newtup->t_self, newtup); CatalogIndexInsert(indstate, newtup); heap_freetuple(newtup); } else { /* * If the first page we found was after the truncation point, we're in * a hole that we'll fill, but we need to delete the later page * because the loop below won't visit it again. */ if (olddata != NULL) { Assert(olddata->pageno > pageno); simple_heap_delete(lo_heap_r, &oldtuple->t_self); } /* * Write a brand new page. * * Fill the hole up to the truncation point */ off = len % LOBLKSIZE; if (off > 0) MemSet(workb, 0, off); /* compute length of new page */ SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); /* * Form and insert new tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id); values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); newtup = heap_form_tuple(lo_heap_r->rd_att, values, nulls); simple_heap_insert(lo_heap_r, newtup); CatalogIndexInsert(indstate, newtup); heap_freetuple(newtup); } /* * Delete any pages after the truncation point. If the initial search * didn't find a page, then of course there's nothing more to do. */ if (olddata != NULL) { while ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { simple_heap_delete(lo_heap_r, &oldtuple->t_self); } } systable_endscan_ordered(sd); CatalogCloseIndexes(indstate); /* * Advance command counter so that tuple updates will be seen by later * large-object operations in this transaction. */ CommandCounterIncrement(); }
int inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes) { int nwritten = 0; int n; int off; int len; int32 pageno = (int32) (obj_desc->offset / LOBLKSIZE); ScanKeyData skey[2]; SysScanDesc sd; HeapTuple oldtuple; Form_pg_largeobject olddata; bool neednextpage; bytea *datafield; bool pfreeit; struct { bytea hdr; char data[LOBLKSIZE]; /* make struct big enough */ int32 align_it; /* ensure struct is aligned well enough */ } workbuf; char *workb = VARDATA(&workbuf.hdr); HeapTuple newtup; Datum values[Natts_pg_largeobject]; bool nulls[Natts_pg_largeobject]; bool replace[Natts_pg_largeobject]; CatalogIndexState indstate; Assert(PointerIsValid(obj_desc)); Assert(buf != NULL); /* enforce writability because snapshot is probably wrong otherwise */ if ((obj_desc->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("large object %u was not opened for writing", obj_desc->id))); /* check existence of the target largeobject */ if (!LargeObjectExists(obj_desc->id)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u was already dropped", obj_desc->id))); if (nbytes <= 0) return 0; open_lo_relation(); indstate = CatalogOpenIndexes(lo_heap_r); ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); sd = systable_beginscan_ordered(lo_heap_r, lo_index_r, obj_desc->snapshot, 2, skey); oldtuple = NULL; olddata = NULL; neednextpage = true; while (nwritten < nbytes) { /* * If possible, get next pre-existing page of the LO. We expect the * indexscan will deliver these in order --- but there may be holes. */ if (neednextpage) { if ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { if (HeapTupleHasNulls(oldtuple)) /* paranoia */ elog(ERROR, "null field found in pg_largeobject"); olddata = (Form_pg_largeobject) GETSTRUCT(oldtuple); Assert(olddata->pageno >= pageno); } neednextpage = false; } /* * If we have a pre-existing page, see if it is the page we want to * write, or a later one. */ if (olddata != NULL && olddata->pageno == pageno) { /* * Update an existing page with fresh data. * * First, load old data into workbuf */ datafield = &(olddata->data); /* see note at top of file */ pfreeit = false; if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } len = getbytealen(datafield); Assert(len <= LOBLKSIZE); memcpy(workb, VARDATA(datafield), len); if (pfreeit) pfree(datafield); /* * Fill any hole */ off = (int) (obj_desc->offset % LOBLKSIZE); if (off > len) MemSet(workb + len, 0, off - len); /* * Insert appropriate portion of new data */ n = LOBLKSIZE - off; n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); memcpy(workb + off, buf + nwritten, n); nwritten += n; obj_desc->offset += n; off += n; /* compute valid length of new page */ len = (len >= off) ? len : off; SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); /* * Form and insert updated tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); memset(replace, false, sizeof(replace)); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); replace[Anum_pg_largeobject_data - 1] = true; newtup = heap_modify_tuple(oldtuple, RelationGetDescr(lo_heap_r), values, nulls, replace); simple_heap_update(lo_heap_r, &newtup->t_self, newtup); CatalogIndexInsert(indstate, newtup); heap_freetuple(newtup); /* * We're done with this old page. */ oldtuple = NULL; olddata = NULL; neednextpage = true; } else { /* * Write a brand new page. * * First, fill any hole */ off = (int) (obj_desc->offset % LOBLKSIZE); if (off > 0) MemSet(workb, 0, off); /* * Insert appropriate portion of new data */ n = LOBLKSIZE - off; n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); memcpy(workb + off, buf + nwritten, n); nwritten += n; obj_desc->offset += n; /* compute valid length of new page */ len = off + n; SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); /* * Form and insert updated tuple */ memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id); values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); newtup = heap_form_tuple(lo_heap_r->rd_att, values, nulls); simple_heap_insert(lo_heap_r, newtup); CatalogIndexInsert(indstate, newtup); heap_freetuple(newtup); } pageno++; } systable_endscan_ordered(sd); CatalogCloseIndexes(indstate); /* * Advance command counter so that my tuple updates will be seen by later * large-object operations in this transaction. */ CommandCounterIncrement(); return nwritten; }
int inv_read(LargeObjectDesc *obj_desc, char *buf, int nbytes) { int nread = 0; int n; int off; int len; int32 pageno = (int32) (obj_desc->offset / LOBLKSIZE); uint32 pageoff; ScanKeyData skey[2]; SysScanDesc sd; HeapTuple tuple; Assert(PointerIsValid(obj_desc)); Assert(buf != NULL); if (nbytes <= 0) return 0; open_lo_relation(); ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); sd = systable_beginscan_ordered(lo_heap_r, lo_index_r, obj_desc->snapshot, 2, skey); while ((tuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { Form_pg_largeobject data; bytea *datafield; bool pfreeit; if (HeapTupleHasNulls(tuple)) /* paranoia */ elog(ERROR, "null field found in pg_largeobject"); data = (Form_pg_largeobject) GETSTRUCT(tuple); /* * We expect the indexscan will deliver pages in order. However, * there may be missing pages if the LO contains unwritten "holes". We * want missing sections to read out as zeroes. */ pageoff = ((uint32) data->pageno) * LOBLKSIZE; if (pageoff > obj_desc->offset) { n = pageoff - obj_desc->offset; n = (n <= (nbytes - nread)) ? n : (nbytes - nread); MemSet(buf + nread, 0, n); nread += n; obj_desc->offset += n; } if (nread < nbytes) { Assert(obj_desc->offset >= pageoff); off = (int) (obj_desc->offset - pageoff); Assert(off >= 0 && off < LOBLKSIZE); datafield = &(data->data); /* see note at top of file */ pfreeit = false; if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } len = getbytealen(datafield); if (len > off) { n = len - off; n = (n <= (nbytes - nread)) ? n : (nbytes - nread); memcpy(buf + nread, VARDATA(datafield) + off, n); nread += n; obj_desc->offset += n; } if (pfreeit) pfree(datafield); } if (nread >= nbytes) break; } systable_endscan_ordered(sd); return nread; }
/* * Convert a HeapTuple into a byte-sequence, and store it directly * into a chunklist for transmission. * * This code is based on the printtup_internal_20() function in printtup.c. */ void SerializeTupleIntoChunks(GenericTuple gtuple, SerTupInfo *pSerInfo, TupleChunkList tcList) { TupleChunkListItem tcItem = NULL; MemoryContext oldCtxt; TupleDesc tupdesc; int i, natts; AssertArg(tcList != NULL); AssertArg(gtuple != NULL); AssertArg(pSerInfo != NULL); tupdesc = pSerInfo->tupdesc; natts = tupdesc->natts; /* get ready to go */ tcList->p_first = NULL; tcList->p_last = NULL; tcList->num_chunks = 0; tcList->serialized_data_length = 0; tcList->max_chunk_length = Gp_max_tuple_chunk_size; if (natts == 0) { tcItem = getChunkFromCache(&pSerInfo->chunkCache); if (tcItem == NULL) { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Could not allocate space for first chunk item in new chunk list."))); } /* TC_EMTPY is just one chunk */ SetChunkType(tcItem->chunk_data, TC_EMPTY); tcItem->chunk_length = TUPLE_CHUNK_HEADER_SIZE; appendChunkToTCList(tcList, tcItem); return; } tcItem = getChunkFromCache(&pSerInfo->chunkCache); if (tcItem == NULL) { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Could not allocate space for first chunk item in new chunk list."))); } /* assume that we'll take a single chunk */ SetChunkType(tcItem->chunk_data, TC_WHOLE); tcItem->chunk_length = TUPLE_CHUNK_HEADER_SIZE; appendChunkToTCList(tcList, tcItem); AssertState(s_tupSerMemCtxt != NULL); if (is_memtuple(gtuple)) { MemTuple mtuple = (MemTuple) gtuple; addByteStringToChunkList(tcList, (char *) mtuple, memtuple_get_size(mtuple), &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, memtuple_get_size(mtuple)); } else { HeapTuple tuple = (HeapTuple) gtuple; HeapTupleHeader t_data = tuple->t_data; TupSerHeader tsh; unsigned int datalen; unsigned int nullslen; datalen = tuple->t_len - t_data->t_hoff; if (HeapTupleHasNulls(tuple)) nullslen = BITMAPLEN(HeapTupleHeaderGetNatts(t_data)); else nullslen = 0; tsh.tuplen = sizeof(TupSerHeader) + TYPEALIGN(TUPLE_CHUNK_ALIGN,nullslen) + datalen; tsh.natts = HeapTupleHeaderGetNatts(t_data); tsh.infomask = t_data->t_infomask; addByteStringToChunkList(tcList, (char *)&tsh, sizeof(TupSerHeader), &pSerInfo->chunkCache); /* If we don't have any attributes which have been toasted, we * can be very very simple: just send the raw data. */ if ((tsh.infomask & HEAP_HASEXTERNAL) == 0) { if (nullslen) { addByteStringToChunkList(tcList, (char *)t_data->t_bits, nullslen, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,nullslen); } addByteStringToChunkList(tcList, (char *)t_data + t_data->t_hoff, datalen, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,datalen); } else { /* We have to be more careful when we have tuples that * have been toasted. Ideally we'd like to send the * untoasted attributes in as "raw" a format as possible * but that makes rebuilding the tuple harder . */ oldCtxt = MemoryContextSwitchTo(s_tupSerMemCtxt); /* deconstruct the tuple (faster than a heap_getattr loop) */ heap_deform_tuple(tuple, tupdesc, pSerInfo->values, pSerInfo->nulls); MemoryContextSwitchTo(oldCtxt); /* Send the nulls character-array. */ addByteStringToChunkList(tcList, pSerInfo->nulls, natts, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,natts); /* * send the attributes of this tuple: NOTE anything which allocates * temporary space (e.g. could result in a PG_DETOAST_DATUM) should be * executed with the memory context set to s_tupSerMemCtxt */ for (i = 0; i < natts; ++i) { SerAttrInfo *attrInfo = pSerInfo->myinfo + i; Datum origattr = pSerInfo->values[i], attr; /* skip null attributes (already taken care of above) */ if (pSerInfo->nulls[i]) continue; if (attrInfo->typlen == -1) { int32 sz; char *data; /* * If we have a toasted datum, forcibly detoast it here to avoid * memory leakage: we want to force the detoast allocation(s) to * happen in our reset-able serialization context. */ oldCtxt = MemoryContextSwitchTo(s_tupSerMemCtxt); attr = PointerGetDatum(PG_DETOAST_DATUM_PACKED(origattr)); MemoryContextSwitchTo(oldCtxt); sz = VARSIZE_ANY_EXHDR(attr); data = VARDATA_ANY(attr); /* Send length first, then data */ addInt32ToChunkList(tcList, sz, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, data, sz, &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, sz); } else if (attrInfo->typlen == -2) { int32 sz; char *data; /* CString, we would send the string with the terminating '\0' */ data = DatumGetCString(origattr); sz = strlen(data) + 1; /* Send length first, then data */ addInt32ToChunkList(tcList, sz, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, data, sz, &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, sz); } else if (attrInfo->typbyval) { /* * We send a full-width Datum for all pass-by-value types, regardless of * the actual size. */ addByteStringToChunkList(tcList, (char *) &origattr, sizeof(Datum), &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, sizeof(Datum)); } else { addByteStringToChunkList(tcList, DatumGetPointer(origattr), attrInfo->typlen, &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, attrInfo->typlen); attr = origattr; } } MemoryContextReset(s_tupSerMemCtxt); } } /* * if we have more than 1 chunk we have to set the chunk types on our * first chunk and last chunk */ if (tcList->num_chunks > 1) { TupleChunkListItem first, last; first = tcList->p_first; last = tcList->p_last; Assert(first != NULL); Assert(first != last); Assert(last != NULL); SetChunkType(first->chunk_data, TC_PARTIAL_START); SetChunkType(last->chunk_data, TC_PARTIAL_END); /* * any intervening chunks are already set to TC_PARTIAL_MID when * allocated */ } return; }
/* ---------------- * nocachegetattr * * This only gets called from fastgetattr() macro, in cases where * we can't use a cacheoffset and the value is not null. * * This caches attribute offsets in the attribute descriptor. * * An alternative way to speed things up would be to cache offsets * with the tuple, but that seems more difficult unless you take * the storage hit of actually putting those offsets into the * tuple you send to disk. Yuck. * * This scheme will be slightly slower than that, but should * perform well for queries which hit large #'s of tuples. After * you cache the offsets once, examining all the other tuples using * the same attribute descriptor will go much quicker. -cim 5/4/91 * * NOTE: if you need to change this code, see also heap_deform_tuple. * Also see nocache_index_getattr, which is the same code for index * tuples. * ---------------- */ Datum nocachegetattr(HeapTuple tuple, int attnum, TupleDesc tupleDesc) { HeapTupleHeader tup = tuple->t_data; Form_pg_attribute *att = tupleDesc->attrs; char *tp; /* ptr to data part of tuple */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* do we have to walk attrs? */ int off; /* current offset within data */ Assert(!is_heaptuple_memtuple(tuple)); /* ---------------- * Three cases: * * 1: No nulls and no variable-width attributes. * 2: Has a null or a var-width AFTER att. * 3: Has nulls or var-widths BEFORE att. * ---------------- */ #ifdef IN_MACRO /* This is handled in the macro */ Assert(attnum > 0); if (isnull) *isnull = false; #endif attnum--; if (HeapTupleNoNulls(tuple)) { #ifdef IN_MACRO /* This is handled in the macro */ if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], (char *) tup + tup->t_hoff + att[attnum]->attcacheoff); } #endif } else { /* * there's a null somewhere in the tuple * * check to see if desired att is null */ #ifdef IN_MACRO /* This is handled in the macro */ if (att_isnull(attnum, bp)) { if (isnull) *isnull = true; return (Datum) NULL; } #endif /* * Now check to see if any preceding bits are null... */ { int byte = attnum >> 3; int finalbit = attnum & 0x07; /* check for nulls "before" final bit of last byte */ if ((~bp[byte]) & ((1 << finalbit) - 1)) slow = true; else { /* check for nulls in any "earlier" bytes */ int i; for (i = 0; i < byte; i++) { if (bp[i] != 0xFF) { slow = true; break; } } } } } tp = (char *) tup + tup->t_hoff; if (!slow) { /* * If we get here, there are no nulls up to and including the target * attribute. If we have a cached offset, we can use it. */ if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); } /* * Otherwise, check for non-fixed-length attrs up to and including * target. If there aren't any, it's safe to cheaply initialize the * cached offsets for these attrs. */ if (HeapTupleHasVarWidth(tuple)) { int j; for (j = 0; j <= attnum; j++) { if (att[j]->attlen <= 0) { slow = true; break; } } } } if (!slow) { int natts = tupleDesc->natts; int j = 1; /* * If we get here, we have a tuple with no nulls or var-widths up to * and including the target attribute, so we can use the cached offset * ... only we don't have it yet, or we'd not have got here. Since * it's cheap to compute offsets for fixed-width columns, we take the * opportunity to initialize the cached offsets for *all* the leading * fixed-width columns, in hope of avoiding future visits to this * routine. */ att[0]->attcacheoff = 0; /* we might have set some offsets in the slow path previously */ while (j < natts && att[j]->attcacheoff > 0) j++; off = att[j - 1]->attcacheoff + att[j - 1]->attlen; for (; j < natts; j++) { if (att[j]->attlen <= 0) break; off = att_align_nominal(off, att[j]->attalign); att[j]->attcacheoff = off; off += att[j]->attlen; } Assert(j > attnum); off = att[attnum]->attcacheoff; } else { bool usecache = true; int i; /* this is always true */ att[0]->attcacheoff = 0; /* * Now we know that we have to walk the tuple CAREFULLY. But we still * might be able to cache some offsets for next time. * * Note - This loop is a little tricky. For each non-null attribute, * we have to first account for alignment padding before the attr, * then advance over the attr based on its length. Nulls have no * storage and no alignment padding either. We can use/set * attcacheoff until we reach either a null or a var-width attribute. */ off = 0; for (i = 0;; i++) /* loop exit is at "break" */ { if (HeapTupleHasNulls(tuple) && att_isnull(i, bp)) { usecache = false; continue; /* this cannot be the target att */ } /* If we know the next offset, we can skip the rest */ if (usecache && att[i]->attcacheoff >= 0) off = att[i]->attcacheoff; else if (att[i]->attlen == -1) { /* * We can only cache the offset for a varlena attribute if the * offset is already suitably aligned, so that there would be * no pad bytes in any case: then the offset will be valid for * either an aligned or unaligned value. */ if (usecache && off == att_align_nominal(off, att[i]->attalign)) att[i]->attcacheoff = off; else { off = att_align_pointer(off, att[i]->attalign, -1, tp + off); usecache = false; } } else { /* not varlena, so safe to use att_align_nominal */ off = att_align_nominal(off, att[i]->attalign); if (usecache) att[i]->attcacheoff = off; } if (i == attnum) break; off = att_addlength_pointer(off, att[i]->attlen, tp + off); if (usecache && att[i]->attlen <= 0) usecache = false; } } return fetchatt(att[attnum], tp + off); }
/* * slot_deform_tuple * Given a TupleTableSlot, extract data from the slot's physical tuple * into its Datum/isnull arrays. Data is extracted up through the * natts'th column (caller must ensure this is a legal column number). * * This is essentially an incremental version of heap_deform_tuple: * on each call we extract attributes up to the one needed, without * re-computing information about previously extracted attributes. * slot->tts_nvalid is the number of attributes already extracted. */ static void slot_deform_tuple(TupleTableSlot *slot, int natts) { HeapTuple tuple = TupGetHeapTuple(slot); TupleDesc tupleDesc = slot->tts_tupleDescriptor; Datum *values = slot->PRIVATE_tts_values; bool *isnull = slot->PRIVATE_tts_isnull; HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; int attnum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow; /* can we use/set attcacheoff? */ /* * Check whether the first call for this tuple, and initialize or restore * loop state. */ attnum = slot->PRIVATE_tts_nvalid; if (attnum == 0) { /* Start from the first attribute */ off = 0; slow = false; } else { /* Restore state from previous execution */ off = slot->PRIVATE_tts_off; slow = slot->PRIVATE_tts_slow; } tp = (char *) tup + tup->t_hoff; for (; attnum < natts; attnum++) { Form_pg_attribute thisatt = att[attnum]; if (hasnulls && att_isnull(attnum, bp)) { values[attnum] = (Datum) 0; isnull[attnum] = true; slow = true; /* can't use attcacheoff anymore */ continue; } isnull[attnum] = false; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; else { /* if it's a varlena it may or may not be aligned, so check for * something that looks like a padding byte before aligning. If * we're already aligned it may be the leading byte of a 4-byte * header but then the att_align is harmless. Don't bother looking * if it's not a varlena though.*/ if (thisatt->attlen != -1 || !tp[off]) off = att_align(off, thisatt->attalign); if (!slow && thisatt->attlen != -1) thisatt->attcacheoff = off; } if (!slow && thisatt->attlen < 0) slow = true; values[attnum] = fetchatt(thisatt, tp + off); off = att_addlength(off, thisatt->attlen, PointerGetDatum(tp + off)); } /* * Save state for next execution */ slot->PRIVATE_tts_nvalid = attnum; slot->PRIVATE_tts_off = off; slot->PRIVATE_tts_slow = slow; }
/* * heap_deformtuple * * Given a tuple, extract data into values/nulls arrays; this is * the inverse of heap_formtuple. * * Storage for the values/nulls arrays is provided by the caller; * it should be sized according to tupleDesc->natts not tuple->t_natts. * * Note that for pass-by-reference datatypes, the pointer placed * in the Datum will point into the given tuple. * * When all or most of a tuple's fields need to be extracted, * this routine will be significantly quicker than a loop around * heap_getattr; the loop will become O(N^2) as soon as any * noncacheable attribute offsets are involved. * * OLD API with char 'n'/' ' convention for indicating nulls. * This is deprecated and should not be used in new code, but we keep it * around for use by old add-on modules. */ void heap_deformtuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, char *nulls) { HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; int tdesc_natts = tupleDesc->natts; int natts; /* number of atts to extract */ int attnum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* can we use/set attcacheoff? */ natts = HeapTupleHeaderGetNatts(tup); /* * In inheritance situations, it is possible that the given tuple actually * has more fields than the caller is expecting. Don't run off the end of * the caller's arrays. */ natts = Min(natts, tdesc_natts); tp = (char *) tup + tup->t_hoff; off = 0; for (attnum = 0; attnum < natts; attnum++) { Form_pg_attribute thisatt = att[attnum]; if (hasnulls && att_isnull(attnum, bp)) { values[attnum] = (Datum) 0; nulls[attnum] = 'n'; slow = true; /* can't use attcacheoff anymore */ continue; } nulls[attnum] = ' '; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; else if (thisatt->attlen == -1) { /* * We can only cache the offset for a varlena attribute if the * offset is already suitably aligned, so that there would be no * pad bytes in any case: then the offset will be valid for either * an aligned or unaligned value. */ if (!slow && off == att_align_nominal(off, thisatt->attalign)) thisatt->attcacheoff = off; else { off = att_align_pointer(off, thisatt->attalign, -1, tp + off); slow = true; } } else { /* not varlena, so safe to use att_align_nominal */ off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; } values[attnum] = fetchatt(thisatt, tp + off); off = att_addlength_pointer(off, thisatt->attlen, tp + off); if (thisatt->attlen <= 0) slow = true; /* can't use attcacheoff anymore */ } /* * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ for (; attnum < tdesc_natts; attnum++) { values[attnum] = (Datum) 0; nulls[attnum] = 'n'; } }
/* * heap_deform_tuple * Given a tuple, extract data into values/isnull arrays; this is * the inverse of heap_form_tuple. * * Storage for the values/isnull arrays is provided by the caller; * it should be sized according to tupleDesc->natts not tuple->t_natts. * * Note that for pass-by-reference datatypes, the pointer placed * in the Datum will point into the given tuple. * * When all or most of a tuple's fields need to be extracted, * this routine will be significantly quicker than a loop around * heap_getattr; the loop will become O(N^2) as soon as any * noncacheable attribute offsets are involved. */ void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull) { HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; int tdesc_natts = tupleDesc->natts; int natts; /* number of atts to extract */ int attnum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* can we use/set attcacheoff? */ Assert(!is_heaptuple_memtuple(tuple)); natts = HeapTupleHeaderGetNatts(tup); /* * In inheritance situations, it is possible that the given tuple actually * has more fields than the caller is expecting. Don't run off the end of * the caller's arrays. */ natts = Min(natts, tdesc_natts); tp = (char *) tup + tup->t_hoff; off = 0; for (attnum = 0; attnum < natts; attnum++) { Form_pg_attribute thisatt = att[attnum]; if (hasnulls && att_isnull(attnum, bp)) { values[attnum] = (Datum) 0; isnull[attnum] = true; slow = true; /* can't use attcacheoff anymore */ continue; } isnull[attnum] = false; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; else if (thisatt->attlen == -1) { /* * We can only cache the offset for a varlena attribute if the * offset is already suitably aligned, so that there would be no * pad bytes in any case: then the offset will be valid for either * an aligned or unaligned value. */ if (!slow && off == att_align_nominal(off, thisatt->attalign)) thisatt->attcacheoff = off; else { off = att_align_pointer(off, thisatt->attalign, -1, tp + off); slow = true; } } else { /* not varlena, so safe to use att_align_nominal */ off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; } if (!slow && thisatt->attlen < 0) slow = true; values[attnum] = fetchatt(thisatt, tp + off); #ifdef USE_ASSERT_CHECKING /* Ignore attributes with dropped types */ if (thisatt->attlen == -1 && !thisatt->attisdropped) { Assert(VARATT_IS_SHORT(DatumGetPointer(values[attnum])) || !VARATT_CAN_MAKE_SHORT(DatumGetPointer(values[attnum])) || thisatt->atttypid == OIDVECTOROID || thisatt->atttypid == INT2VECTOROID || thisatt->atttypid >= FirstNormalObjectId); } #endif off = att_addlength_pointer(off, thisatt->attlen, tp + off); } /* * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ for (; attnum < tdesc_natts; attnum++) { values[attnum] = (Datum) 0; isnull[attnum] = true; } }
/* * load_auth_entries: read pg_authid into auth_entry[] * * auth_info_out: pointer to auth_entry * where address to auth_entry[] should be stored * total_roles_out: pointer to int where num of total roles should be stored */ static void load_auth_entries(Relation rel_authid, auth_entry **auth_info_out, int *total_roles_out) { BlockNumber totalblocks; HeapScanDesc scan; HeapTuple tuple; int curr_role = 0; int total_roles = 0; int est_rows; auth_entry *auth_info; /* * Read pg_authid and fill temporary data structures. Note we must read * all roles, even those without rolcanlogin. */ totalblocks = RelationGetNumberOfBlocks(rel_authid); totalblocks = totalblocks ? totalblocks : 1; est_rows = totalblocks * (BLCKSZ / (sizeof(HeapTupleHeaderData) + sizeof(FormData_pg_authid))); auth_info = (auth_entry *) palloc(est_rows * sizeof(auth_entry)); scan = heap_beginscan(rel_authid, SnapshotNow, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_authid aform = (Form_pg_authid) GETSTRUCT(tuple); HeapTupleHeader tup = tuple->t_data; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmask in tuple */ Datum datum; if (curr_role >= est_rows) { est_rows *= 2; auth_info = (auth_entry *) repalloc(auth_info, est_rows * sizeof(auth_entry)); } auth_info[curr_role].roleid = HeapTupleGetOid(tuple); auth_info[curr_role].rolsuper = aform->rolsuper; auth_info[curr_role].rolcanlogin = aform->rolcanlogin; auth_info[curr_role].rolname = pstrdup(NameStr(aform->rolname)); auth_info[curr_role].member_of = NIL; /* * We can't use heap_getattr() here because during startup we will not * have any tupdesc for pg_authid. Fortunately it's not too hard to * work around this. rolpassword is the first possibly-null field so * we can compute its offset directly. */ tp = (char *) tup + tup->t_hoff; off = offsetof(FormData_pg_authid, rolpassword); if (HeapTupleHasNulls(tuple) && att_isnull(Anum_pg_authid_rolpassword - 1, bp)) { /* passwd is null, emit as an empty string */ auth_info[curr_role].rolpassword = pstrdup(""); } else { /* assume passwd is pass-by-ref */ datum = PointerGetDatum(tp + off); /* * The password probably shouldn't ever be out-of-line toasted; if * it is, ignore it, since we can't handle that in startup mode. */ if (VARATT_IS_EXTERNAL(DatumGetPointer(datum))) auth_info[curr_role].rolpassword = pstrdup(""); else auth_info[curr_role].rolpassword = DatumGetCString(DirectFunctionCall1(textout, datum)); /* assume passwd has attlen -1 */ off = att_addlength(off, -1, PointerGetDatum(tp + off)); } if (HeapTupleHasNulls(tuple) && att_isnull(Anum_pg_authid_rolvaliduntil - 1, bp)) { /* rolvaliduntil is null, emit as an empty string */ auth_info[curr_role].rolvaliduntil = pstrdup(""); } else { /* * rolvaliduntil is timestamptz, which we assume is double * alignment and pass-by-value. */ off = att_align(off, 'd'); datum = fetch_att(tp + off, true, sizeof(TimestampTz)); auth_info[curr_role].rolvaliduntil = DatumGetCString(DirectFunctionCall1(timestamptz_out, datum)); } /* * Check for illegal characters in the user name and password. */ if (!name_okay(auth_info[curr_role].rolname)) { ereport(LOG, (errmsg("invalid role name \"%s\"", auth_info[curr_role].rolname))); continue; } if (!name_okay(auth_info[curr_role].rolpassword)) { ereport(LOG, (errmsg("invalid role password \"%s\"", auth_info[curr_role].rolpassword))); continue; } curr_role++; total_roles++; } heap_endscan(scan); *auth_info_out = auth_info; *total_roles_out = total_roles; }
/* * heap_deform_tuple * Given a tuple, extract data into values/isnull arrays; this is * the inverse of heap_form_tuple. * * Storage for the values/isnull arrays is provided by the caller; * it should be sized according to tupleDesc->natts not tuple->t_natts. * * Note that for pass-by-reference datatypes, the pointer placed * in the Datum will point into the given tuple. * * When all or most of a tuple's fields need to be extracted, * this routine will be significantly quicker than a loop around * heap_getattr; the loop will become O(N^2) as soon as any * noncacheable attribute offsets are involved. */ void heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *values, bool *isnull) { HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; int tdesc_natts = tupleDesc->natts; int natts; /* number of atts to extract */ int attnum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* can we use/set attcacheoff? */ Assert(!is_heaptuple_memtuple(tuple)); natts = HeapTupleHeaderGetNatts(tup); /* * In inheritance situations, it is possible that the given tuple actually * has more fields than the caller is expecting. Don't run off the end of * the caller's arrays. */ natts = Min(natts, tdesc_natts); tp = (char *) tup + tup->t_hoff; off = 0; for (attnum = 0; attnum < natts; attnum++) { Form_pg_attribute thisatt = att[attnum]; if (hasnulls && att_isnull(attnum, bp)) { values[attnum] = (Datum) 0; isnull[attnum] = true; slow = true; /* can't use attcacheoff anymore */ continue; } isnull[attnum] = false; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; else { /* if it's a varlena it may or may not be aligned, so check for * something that looks like a padding byte before aligning. If * we're already aligned it may be the leading byte of a 4-byte * header but then the att_align is harmless. Don't bother looking * if it's not a varlena though.*/ if (thisatt->attlen != -1 || !tp[off]) off = att_align(off, thisatt->attalign); if (!slow && thisatt->attlen != -1) thisatt->attcacheoff = off; } if (!slow && thisatt->attlen < 0) slow = true; values[attnum] = fetchatt(thisatt, tp + off); #ifdef USE_ASSERT_CHECKING /* Ignore attributes with dropped types */ if (thisatt->attlen == -1 && !thisatt->attisdropped) { Assert(VARATT_IS_SHORT_D(values[attnum]) || !VARATT_COULD_SHORT_D(values[attnum]) || thisatt->atttypid == OIDVECTOROID || thisatt->atttypid == INT2VECTOROID || thisatt->atttypid >= FirstNormalObjectId); } #endif off = att_addlength(off, thisatt->attlen, PointerGetDatum(tp + off)); } /* * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ for (; attnum < tdesc_natts; attnum++) { values[attnum] = (Datum) 0; isnull[attnum] = true; } }
/* ---------------- * nocachegetattr * * This only gets called from fastgetattr() macro, in cases where * we can't use a cacheoffset and the value is not null. * * This caches attribute offsets in the attribute descriptor. * * An alternative way to speed things up would be to cache offsets * with the tuple, but that seems more difficult unless you take * the storage hit of actually putting those offsets into the * tuple you send to disk. Yuck. * * This scheme will be slightly slower than that, but should * perform well for queries which hit large #'s of tuples. After * you cache the offsets once, examining all the other tuples using * the same attribute descriptor will go much quicker. -cim 5/4/91 * * NOTE: if you need to change this code, see also heap_deform_tuple. * Also see nocache_index_getattr, which is the same code for index * tuples. * ---------------- */ Datum nocachegetattr(HeapTuple tuple, int attnum, TupleDesc tupleDesc) { HeapTupleHeader tup = tuple->t_data; Form_pg_attribute *att = tupleDesc->attrs; char *tp; /* ptr to att in tuple */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* do we have to walk nulls? */ Assert(!is_heaptuple_memtuple(tuple)); /* If any cached offsets are there we can check that they make sense, but * there may not be any at all, so pass -1 for the attnum we know is valid */ #ifdef IN_MACRO /* This is handled in the macro */ Assert(attnum > 0); if (isnull) *isnull = false; #endif attnum--; /* ---------------- * Three cases: * * 1: No nulls and no variable-width attributes. * 2: Has a null or a var-width AFTER att. * 3: Has nulls or var-widths BEFORE att. * ---------------- */ if (HeapTupleNoNulls(tuple)) { #ifdef IN_MACRO /* This is handled in the macro */ if (att[attnum]->attcacheoff != -1) { return fetchatt(att[attnum], (char *) tup + tup->t_hoff + att[attnum]->attcacheoff); } #endif } else { /* * there's a null somewhere in the tuple * * check to see if desired att is null */ #ifdef IN_MACRO /* This is handled in the macro */ if (att_isnull(attnum, bp)) { if (isnull) *isnull = true; return (Datum) NULL; } #endif /* * Now check to see if any preceding bits are null... */ { int byte = attnum >> 3; int finalbit = attnum & 0x07; /* check for nulls "before" final bit of last byte */ if ((~bp[byte]) & ((1 << finalbit) - 1)) slow = true; else { /* check for nulls in any "earlier" bytes */ int i; for (i = 0; i < byte; i++) { if (bp[i] != 0xFF) { slow = true; break; } } } } } tp = (char *) tup + tup->t_hoff; /* * now check for any non-fixed length attrs before our attribute */ if (!slow) { /* * If we get here, there are no nulls up to and including the target * attribute. If we have a cached offset, we can use it. */ if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); } /* * Otherwise, check for non-fixed-length attrs up to and including * target. If there aren't any, it's safe to cheaply initialize the * cached offsets for these attrs. */ if (HeapTupleHasVarWidth(tuple)) { int j; /* * In for(), we test <= and not < because we want to see if we can * go past it in initializing offsets. */ for (j = 0; j <= attnum; j++) { if (att[j]->attlen <= 0) { slow = true; break; } } } } /* * If slow is false, and we got here, we know that we have a tuple with no * nulls or var-widths before the target attribute. If possible, we also * want to initialize the remainder of the attribute cached offset values. */ if (!slow) { int j = 1; long off; int natts = HeapTupleHeaderGetNatts(tup); /* * If we get here, we have a tuple with no nulls or var-widths up to * and including the target attribute, so we can use the cached offset * ... only we don't have it yet, or we'd not have got here. Since * it's cheap to compute offsets for fixed-width columns, we take the * opportunity to initialize the cached offsets for *all* the leading * fixed-width columns, in hope of avoiding future visits to this * routine. */ /* this is always true */ att[0]->attcacheoff = 0; while (j < attnum && att[j]->attcacheoff > 0) j++; off = att[j - 1]->attcacheoff + att[j - 1]->attlen; for (; j <= attnum || /* Can we compute more? We will probably need them */ (j < natts && att[j]->attcacheoff == -1 && (HeapTupleNoNulls(tuple) || !att_isnull(j, bp)) && (HeapTupleAllFixed(tuple) || att[j]->attlen > 0)); j++) { /* don't need to worry about shortvarlenas here since we're only * looking at non-varlenas. Note that it's important that we check * that the target attribute itself is a nonvarlena too since we * can't use cached offsets for even the first varlena any more. */ off = att_align(off, att[j]->attalign); att[j]->attcacheoff = off; off = att_addlength(off, att[j]->attlen, PointerGetDatum(tp + off)); } return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); } else { bool usecache = true; int off = 0; int i; /* this is always true */ att[0]->attcacheoff = 0; /* * Now we know that we have to walk the tuple CAREFULLY. * * Note - This loop is a little tricky. For each non-null attribute, * we have to first account for alignment padding before the attr, * then advance over the attr based on its length. Nulls have no * storage and no alignment padding either. We can use/set * attcacheoff until we reach either a null or a var-width attribute. */ for (i = 0; i < attnum; i++) { if (HeapTupleHasNulls(tuple) && att_isnull(i, bp)) { usecache = false; continue; } /* If we know the next offset, we can skip the alignment calc */ if (usecache && att[i]->attcacheoff != -1) off = att[i]->attcacheoff; else { /* if it's a varlena it may or may not be aligned, so check for * something that looks like a padding byte before aligning. If * we're already aligned it may be the leading byte of a 4-byte * header but then the att_align is harmless. Don't bother * looking if it's not a varlena though.*/ if (att[i]->attlen != -1 || !tp[off]) off = att_align(off, att[i]->attalign); if (usecache && att[i]->attlen != -1) att[i]->attcacheoff = off; } if (att[i]->attlen < 0) usecache = false; off = att_addlength(off, att[i]->attlen, PointerGetDatum(tp + off)); } if (att[attnum]->attlen != -1 || !tp[off]) off = att_align(off, att[attnum]->attalign); return fetchatt(att[attnum], tp + off); } }
/* * Convert a HeapTuple into a byte-sequence, and store it directly * into a chunklist for transmission. * * This code is based on the printtup_internal_20() function in printtup.c. */ void SerializeTupleIntoChunks(HeapTuple tuple, SerTupInfo * pSerInfo, TupleChunkList tcList) { TupleChunkListItem tcItem = NULL; MemoryContext oldCtxt; TupleDesc tupdesc; int i, natts; bool fHandled; AssertArg(tcList != NULL); AssertArg(tuple != NULL); AssertArg(pSerInfo != NULL); tupdesc = pSerInfo->tupdesc; natts = tupdesc->natts; /* get ready to go */ tcList->p_first = NULL; tcList->p_last = NULL; tcList->num_chunks = 0; tcList->serialized_data_length = 0; tcList->max_chunk_length = Gp_max_tuple_chunk_size; if (natts == 0) { tcItem = getChunkFromCache(&pSerInfo->chunkCache); if (tcItem == NULL) { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Could not allocate space for first chunk item in new chunk list."))); } /* TC_EMTPY is just one chunk */ SetChunkType(tcItem->chunk_data, TC_EMPTY); tcItem->chunk_length = TUPLE_CHUNK_HEADER_SIZE; appendChunkToTCList(tcList, tcItem); return; } tcItem = getChunkFromCache(&pSerInfo->chunkCache); if (tcItem == NULL) { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Could not allocate space for first chunk item in new chunk list."))); } /* assume that we'll take a single chunk */ SetChunkType(tcItem->chunk_data, TC_WHOLE); tcItem->chunk_length = TUPLE_CHUNK_HEADER_SIZE; appendChunkToTCList(tcList, tcItem); AssertState(s_tupSerMemCtxt != NULL); if (is_heaptuple_memtuple(tuple)) { addByteStringToChunkList(tcList, (char *)tuple, memtuple_get_size((MemTuple)tuple, NULL), &pSerInfo->chunkCache); addPadding(tcList, &pSerInfo->chunkCache, memtuple_get_size((MemTuple)tuple, NULL)); } else { TupSerHeader tsh; unsigned int datalen; unsigned int nullslen; HeapTupleHeader t_data = tuple->t_data; datalen = tuple->t_len - t_data->t_hoff; if (HeapTupleHasNulls(tuple)) nullslen = BITMAPLEN(HeapTupleHeaderGetNatts(t_data)); else nullslen = 0; tsh.tuplen = sizeof(TupSerHeader) + TYPEALIGN(TUPLE_CHUNK_ALIGN,nullslen) + datalen; tsh.natts = HeapTupleHeaderGetNatts(t_data); tsh.infomask = t_data->t_infomask; addByteStringToChunkList(tcList, (char *)&tsh, sizeof(TupSerHeader), &pSerInfo->chunkCache); /* If we don't have any attributes which have been toasted, we * can be very very simple: just send the raw data. */ if ((tsh.infomask & HEAP_HASEXTERNAL) == 0) { if (nullslen) { addByteStringToChunkList(tcList, (char *)t_data->t_bits, nullslen, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,nullslen); } addByteStringToChunkList(tcList, (char *)t_data + t_data->t_hoff, datalen, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,datalen); } else { /* We have to be more careful when we have tuples that * have been toasted. Ideally we'd like to send the * untoasted attributes in as "raw" a format as possible * but that makes rebuilding the tuple harder . */ oldCtxt = MemoryContextSwitchTo(s_tupSerMemCtxt); /* deconstruct the tuple (faster than a heap_getattr loop) */ heap_deform_tuple(tuple, tupdesc, pSerInfo->values, pSerInfo->nulls); MemoryContextSwitchTo(oldCtxt); /* Send the nulls character-array. */ addByteStringToChunkList(tcList, pSerInfo->nulls, natts, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,natts); /* * send the attributes of this tuple: NOTE anything which allocates * temporary space (e.g. could result in a PG_DETOAST_DATUM) should be * executed with the memory context set to s_tupSerMemCtxt */ for (i = 0; i < natts; ++i) { SerAttrInfo *attrInfo = pSerInfo->myinfo + i; Datum origattr = pSerInfo->values[i], attr; bytea *outputbytes=0; /* skip null attributes (already taken care of above) */ if (pSerInfo->nulls[i]) continue; /* * If we have a toasted datum, forcibly detoast it here to avoid * memory leakage: we want to force the detoast allocation(s) to * happen in our reset-able serialization context. */ if (attrInfo->typisvarlena) { oldCtxt = MemoryContextSwitchTo(s_tupSerMemCtxt); /* we want to detoast but leave compressed, if * possible, but we have to handle varlena * attributes (and others ?) differently than we * currently do (first step is to use * heap_tuple_fetch_attr() instead of * PG_DETOAST_DATUM()). */ attr = PointerGetDatum(PG_DETOAST_DATUM(origattr)); MemoryContextSwitchTo(oldCtxt); } else attr = origattr; /* * Assume that the data's output will be handled by the special IO * code, and if not then we can handle it the slow way. */ fHandled = true; switch (attrInfo->atttypid) { case INT4OID: addInt32ToChunkList(tcList, DatumGetInt32(attr), &pSerInfo->chunkCache); break; case CHAROID: addCharToChunkList(tcList, DatumGetChar(attr), &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,1); break; case BPCHAROID: case VARCHAROID: case INT2VECTOROID: /* postgres serialization logic broken, use our own */ case OIDVECTOROID: /* postgres serialization logic broken, use our own */ case ANYARRAYOID: { text *pText = DatumGetTextP(attr); int32 textSize = VARSIZE(pText) - VARHDRSZ; addInt32ToChunkList(tcList, textSize, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, (char *) VARDATA(pText), textSize, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,textSize); break; } case DATEOID: { DateADT date = DatumGetDateADT(attr); addByteStringToChunkList(tcList, (char *) &date, sizeof(DateADT), &pSerInfo->chunkCache); break; } case NUMERICOID: { /* * Treat the numeric as a varlena variable, and just push * the whole shebang to the output-buffer. We don't care * about the guts of the numeric. */ Numeric num = DatumGetNumeric(attr); int32 numSize = VARSIZE(num) - VARHDRSZ; addInt32ToChunkList(tcList, numSize, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, (char *) VARDATA(num), numSize, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,numSize); break; } case ACLITEMOID: { AclItem *aip = DatumGetAclItemP(attr); char *outputstring; int32 aclSize ; outputstring = DatumGetCString(DirectFunctionCall1(aclitemout, PointerGetDatum(aip))); aclSize = strlen(outputstring); addInt32ToChunkList(tcList, aclSize, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, outputstring,aclSize, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,aclSize); break; } case 210: /* storage manager */ { char *smgrstr; int32 strsize; smgrstr = DatumGetCString(DirectFunctionCall1(smgrout, 0)); strsize = strlen(smgrstr); addInt32ToChunkList(tcList, strsize, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, smgrstr, strsize, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,strsize); break; } default: fHandled = false; } if (fHandled) continue; /* * the FunctionCall2 call into the send function may result in some * allocations which we'd like to have contained by our reset-able * context */ oldCtxt = MemoryContextSwitchTo(s_tupSerMemCtxt); /* Call the attribute type's binary input converter. */ if (attrInfo->send_finfo.fn_nargs == 1) outputbytes = DatumGetByteaP(FunctionCall1(&attrInfo->send_finfo, attr)); else if (attrInfo->send_finfo.fn_nargs == 2) outputbytes = DatumGetByteaP(FunctionCall2(&attrInfo->send_finfo, attr, ObjectIdGetDatum(attrInfo->send_typio_param))); else if (attrInfo->send_finfo.fn_nargs == 3) outputbytes = DatumGetByteaP(FunctionCall3(&attrInfo->send_finfo, attr, ObjectIdGetDatum(attrInfo->send_typio_param), Int32GetDatum(tupdesc->attrs[i]->atttypmod))); else { ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("Conversion function takes %d args",attrInfo->recv_finfo.fn_nargs))); } MemoryContextSwitchTo(oldCtxt); /* We assume the result will not have been toasted */ addInt32ToChunkList(tcList, VARSIZE(outputbytes) - VARHDRSZ, &pSerInfo->chunkCache); addByteStringToChunkList(tcList, VARDATA(outputbytes), VARSIZE(outputbytes) - VARHDRSZ, &pSerInfo->chunkCache); addPadding(tcList,&pSerInfo->chunkCache,VARSIZE(outputbytes) - VARHDRSZ); /* * this was allocated in our reset-able context, but we *are* done * with it; and for tuples with several large columns it'd be nice to * free the memory back to the context */ pfree(outputbytes); } MemoryContextReset(s_tupSerMemCtxt); } } /* * if we have more than 1 chunk we have to set the chunk types on our * first chunk and last chunk */ if (tcList->num_chunks > 1) { TupleChunkListItem first, last; first = tcList->p_first; last = tcList->p_last; Assert(first != NULL); Assert(first != last); Assert(last != NULL); SetChunkType(first->chunk_data, TC_PARTIAL_START); SetChunkType(last->chunk_data, TC_PARTIAL_END); /* * any intervening chunks are already set to TC_PARTIAL_MID when * allocated */ } return; }
/* * Serialize a tuple directly into a buffer. * * We're called with at least enough space for a tuple-chunk-header. */ int SerializeTupleDirect(HeapTuple tuple, SerTupInfo * pSerInfo, struct directTransportBuffer *b) { int natts; int dataSize = TUPLE_CHUNK_HEADER_SIZE; TupleDesc tupdesc; AssertArg(tuple != NULL); AssertArg(pSerInfo != NULL); AssertArg(b != NULL); tupdesc = pSerInfo->tupdesc; natts = tupdesc->natts; do { if (natts == 0) { /* TC_EMTPY is just one chunk */ SetChunkType(b->pri, TC_EMPTY); SetChunkDataSize(b->pri, 0); break; } /* easy case */ if (is_heaptuple_memtuple(tuple)) { int tupleSize; int paddedSize; tupleSize = memtuple_get_size((MemTuple)tuple, NULL); paddedSize = TYPEALIGN(TUPLE_CHUNK_ALIGN, tupleSize); if (paddedSize + TUPLE_CHUNK_HEADER_SIZE > b->prilen) return 0; /* will fit. */ memcpy(b->pri + TUPLE_CHUNK_HEADER_SIZE, tuple, tupleSize); memset(b->pri + TUPLE_CHUNK_HEADER_SIZE + tupleSize, 0, paddedSize - tupleSize); dataSize += paddedSize; SetChunkType(b->pri, TC_WHOLE); SetChunkDataSize(b->pri, dataSize - TUPLE_CHUNK_HEADER_SIZE); break; } else { TupSerHeader tsh; unsigned int datalen; unsigned int nullslen; HeapTupleHeader t_data = tuple->t_data; unsigned char *pos; datalen = tuple->t_len - t_data->t_hoff; if (HeapTupleHasNulls(tuple)) nullslen = BITMAPLEN(HeapTupleHeaderGetNatts(t_data)); else nullslen = 0; tsh.tuplen = sizeof(TupSerHeader) + TYPEALIGN(TUPLE_CHUNK_ALIGN, nullslen) + TYPEALIGN(TUPLE_CHUNK_ALIGN, datalen); tsh.natts = HeapTupleHeaderGetNatts(t_data); tsh.infomask = t_data->t_infomask; if (dataSize + tsh.tuplen > b->prilen || (tsh.infomask & HEAP_HASEXTERNAL) != 0) return 0; pos = b->pri + TUPLE_CHUNK_HEADER_SIZE; memcpy(pos, (char *)&tsh, sizeof(TupSerHeader)); pos += sizeof(TupSerHeader); if (nullslen) { memcpy(pos, (char *)t_data->t_bits, nullslen); pos += nullslen; memset(pos, 0, TYPEALIGN(TUPLE_CHUNK_ALIGN, nullslen) - nullslen); pos += TYPEALIGN(TUPLE_CHUNK_ALIGN, nullslen) - nullslen; } memcpy(pos, (char *)t_data + t_data->t_hoff, datalen); pos += datalen; memset(pos, 0, TYPEALIGN(TUPLE_CHUNK_ALIGN, datalen) - datalen); pos += TYPEALIGN(TUPLE_CHUNK_ALIGN, datalen) - datalen; dataSize += tsh.tuplen; SetChunkType(b->pri, TC_WHOLE); SetChunkDataSize(b->pri, dataSize - TUPLE_CHUNK_HEADER_SIZE); break; } /* tuple that we can't handle here (big ?) -- do the older "out-of-line" serialization */ return 0; } while (0); return dataSize; }
/* * slot_deform_tuple * Given a TupleTableSlot, extract data from the slot's physical tuple * into its Datum/isnull arrays. Data is extracted up through the * natts'th column (caller must ensure this is a legal column number). * * This is essentially an incremental version of heap_deform_tuple: * on each call we extract attributes up to the one needed, without * re-computing information about previously extracted attributes. * slot->tts_nvalid is the number of attributes already extracted. */ static void slot_deform_tuple(TupleTableSlot *slot, int natts) { HeapTuple tuple = TupGetHeapTuple(slot); TupleDesc tupleDesc = slot->tts_tupleDescriptor; Datum *values = slot->PRIVATE_tts_values; bool *isnull = slot->PRIVATE_tts_isnull; HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; int attnum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow; /* can we use/set attcacheoff? */ /* * Check whether the first call for this tuple, and initialize or restore * loop state. */ attnum = slot->PRIVATE_tts_nvalid; if (attnum == 0) { /* Start from the first attribute */ off = 0; slow = false; } else { /* Restore state from previous execution */ off = slot->PRIVATE_tts_off; slow = slot->PRIVATE_tts_slow; } tp = (char *) tup + tup->t_hoff; for (; attnum < natts; attnum++) { Form_pg_attribute thisatt = att[attnum]; if (hasnulls && att_isnull(attnum, bp)) { values[attnum] = (Datum) 0; isnull[attnum] = true; slow = true; /* can't use attcacheoff anymore */ continue; } isnull[attnum] = false; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; else if (thisatt->attlen == -1) { /* * We can only cache the offset for a varlena attribute if the * offset is already suitably aligned, so that there would be no * pad bytes in any case: then the offset will be valid for either * an aligned or unaligned value. */ if (!slow && off == att_align_nominal(off, thisatt->attalign)) thisatt->attcacheoff = off; else { off = att_align_pointer(off, thisatt->attalign, -1, tp + off); slow = true; } } else { /* not varlena, so safe to use att_align_nominal */ off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; } if (!slow && thisatt->attlen < 0) slow = true; values[attnum] = fetchatt(thisatt, tp + off); off = att_addlength_pointer(off, thisatt->attlen, tp + off); if (thisatt->attlen <= 0) slow = true; /* can't use attcacheoff anymore */ } /* * Save state for next execution */ slot->PRIVATE_tts_nvalid = attnum; slot->PRIVATE_tts_off = off; slot->PRIVATE_tts_slow = slow; }