Datum popc_or(PG_FUNCTION_ARGS) { VarBit *arg1 = PG_GETARG_VARBIT_P(0); VarBit *arg2 = PG_GETARG_VARBIT_P(1); bits8 *ptr1 = VARBITS(arg1); bits8 *ptr2 = VARBITS(arg2); int32 count = 0; int offset = 0; int bitlen; if (VARBITLEN(arg1) != VARBITLEN(arg2)) ereport(ERROR, (errcode(ERRCODE_STRING_DATA_LENGTH_MISMATCH), errmsg("cannot AND bit strings of different sizes"))); #ifdef __GNUC__ bitlen = LONGALIGN_DOWN(VARBITBYTES(arg1)); for (offset=0; offset < bitlen; offset += sizeof(long)) count += __builtin_popcountl(*((long *)(ptr1 + offset)) | *((long *)(ptr2 + offset))); #endif bitlen = VARBITBYTES(arg1); while (offset < bitlen) { count += bitcount_map[ptr1[offset] & ptr2[offset]]; offset++; } PG_RETURN_INT32(count); }
static bytea * gbt_bit_xfrm(bytea *leaf) { bytea *out = leaf; int s = INTALIGN(VARBITBYTES(leaf) + VARHDRSZ); out = palloc(s); SET_VARSIZE(out, s); memcpy((void *) VARDATA(out), (void *) VARBITS(leaf), VARBITBYTES(leaf)); return out; }
static bytea * gbt_bit_xfrm(bytea *leaf) { bytea *out = leaf; int sz = VARBITBYTES(leaf) + VARHDRSZ; int padded_sz = INTALIGN(sz); out = (bytea *) palloc(padded_sz); /* initialize the padding bytes to zero */ while (sz < padded_sz) ((char *) out)[sz++] = 0; SET_VARSIZE(out, padded_sz); memcpy((void *) VARDATA(out), (void *) VARBITS(leaf), VARBITBYTES(leaf)); return out; }
Datum popc(PG_FUNCTION_ARGS) { VarBit *arg = PG_GETARG_VARBIT_P(0); bits8 *ptr = VARBITS(arg); int32 count = 0; int offset = 0; int bitlen; #ifdef __GNUC__ bitlen = LONGALIGN_DOWN(VARBITBYTES(arg)); for (offset=0; offset < bitlen; offset += sizeof(long)) count += __builtin_popcountl(*((long *)(ptr + offset))); #endif bitlen = VARBITBYTES(arg); while (offset < bitlen) count += bitcount_map[ptr[offset++]]; PG_RETURN_INT32(count); }
/* * Add an attribute to the hash calculation. * **IMPORTANT: any new hard coded support for a data type in here * must be added to isGreenplumDbHashable() below! * * Note that the caller should provide the base type if the datum is * of a domain type. It is quite expensive to call get_typtype() and * getBaseType() here since this function gets called a lot for the * same set of Datums. * * @param hashFn called to update the hash value. * @param clientData passed to hashFn. */ void hashDatum(Datum datum, Oid type, datumHashFunction hashFn, void *clientData) { void *buf = NULL; /* pointer to the data */ size_t len = 0; /* length for the data buffer */ int64 intbuf; /* an 8 byte buffer for all integer sizes */ float4 buf_f4; float8 buf_f8; Timestamp tsbuf; /* timestamp data dype is either a double or * int8 (determined in compile time) */ TimestampTz tstzbuf; DateADT datebuf; TimeADT timebuf; TimeTzADT *timetzptr; Interval *intervalptr; AbsoluteTime abstime_buf; RelativeTime reltime_buf; TimeInterval tinterval; AbsoluteTime tinterval_len; Numeric num; bool bool_buf; char char_buf; Name namebuf; ArrayType *arrbuf; inet *inetptr; /* inet/cidr */ unsigned char inet_hkey[sizeof(inet_struct)]; macaddr *macptr; /* MAC address */ VarBit *vbitptr; int2vector *i2vec_buf; oidvector *oidvec_buf; Cash cash_buf; AclItem *aclitem_ptr; uint32 aclitem_buf; /* * special case buffers */ uint32 nanbuf; uint32 invalidbuf; void *tofree = NULL; /* * Select the hash to be performed according to the field type we are adding to the * hash. */ switch (type) { /* * ======= NUMERIC TYPES ======== */ case INT2OID: /* -32 thousand to 32 thousand, 2-byte storage */ intbuf = (int64) DatumGetInt16(datum); /* cast to 8 byte before * hashing */ buf = &intbuf; len = sizeof(intbuf); break; case INT4OID: /* -2 billion to 2 billion integer, 4-byte * storage */ intbuf = (int64) DatumGetInt32(datum); /* cast to 8 byte before * hashing */ buf = &intbuf; len = sizeof(intbuf); break; case INT8OID: /* ~18 digit integer, 8-byte storage */ intbuf = DatumGetInt64(datum); /* cast to 8 byte before * hashing */ buf = &intbuf; len = sizeof(intbuf); break; case FLOAT4OID: /* single-precision floating point number, * 4-byte storage */ buf_f4 = DatumGetFloat4(datum); /* * On IEEE-float machines, minus zero and zero have different bit * patterns but should compare as equal. We must ensure that they * have the same hash value, which is most easily done this way: */ if (buf_f4 == (float4) 0) buf_f4 = 0.0; buf = &buf_f4; len = sizeof(buf_f4); break; case FLOAT8OID: /* double-precision floating point number, * 8-byte storage */ buf_f8 = DatumGetFloat8(datum); /* * On IEEE-float machines, minus zero and zero have different bit * patterns but should compare as equal. We must ensure that they * have the same hash value, which is most easily done this way: */ if (buf_f8 == (float8) 0) buf_f8 = 0.0; buf = &buf_f8; len = sizeof(buf_f8); break; case NUMERICOID: num = DatumGetNumeric(datum); if (NUMERIC_IS_NAN(num)) { nanbuf = NAN_VAL; buf = &nanbuf; len = sizeof(nanbuf); } else /* not a nan */ { buf = num->n_data; len = (VARSIZE(num) - NUMERIC_HDRSZ); } /* * If we did a pg_detoast_datum, we need to remember to pfree, * or we will leak memory. Because of the 1-byte varlena header stuff. */ if (num != DatumGetPointer(datum)) tofree = num; break; /* * ====== CHARACTER TYPES ======= */ case CHAROID: /* char(1), single character */ char_buf = DatumGetChar(datum); buf = &char_buf; len = 1; break; case BPCHAROID: /* char(n), blank-padded string, fixed storage */ case TEXTOID: /* text */ case VARCHAROID: /* varchar */ case BYTEAOID: /* bytea */ { int tmplen; varattrib_untoast_ptr_len(datum, (char **) &buf, &tmplen, &tofree); /* adjust length to not include trailing blanks */ if (type != BYTEAOID && tmplen > 1) tmplen = ignoreblanks((char *) buf, tmplen); len = tmplen; break; } case NAMEOID: namebuf = DatumGetName(datum); len = NAMEDATALEN; buf = NameStr(*namebuf); /* adjust length to not include trailing blanks */ if (len > 1) len = ignoreblanks((char *) buf, len); break; /* * ====== OBJECT IDENTIFIER TYPES ====== */ case OIDOID: /* object identifier(oid), maximum 4 billion */ case REGPROCOID: /* function name */ case REGPROCEDUREOID: /* function name with argument types */ case REGOPEROID: /* operator name */ case REGOPERATOROID: /* operator with argument types */ case REGCLASSOID: /* relation name */ case REGTYPEOID: /* data type name */ intbuf = (int64) DatumGetUInt32(datum); /* cast to 8 byte before hashing */ buf = &intbuf; len = sizeof(intbuf); break; case TIDOID: /* tuple id (6 bytes) */ buf = DatumGetPointer(datum); len = SizeOfIptrData; break; /* * ====== DATE/TIME TYPES ====== */ case TIMESTAMPOID: /* date and time */ tsbuf = DatumGetTimestamp(datum); buf = &tsbuf; len = sizeof(tsbuf); break; case TIMESTAMPTZOID: /* date and time with time zone */ tstzbuf = DatumGetTimestampTz(datum); buf = &tstzbuf; len = sizeof(tstzbuf); break; case DATEOID: /* ANSI SQL date */ datebuf = DatumGetDateADT(datum); buf = &datebuf; len = sizeof(datebuf); break; case TIMEOID: /* hh:mm:ss, ANSI SQL time */ timebuf = DatumGetTimeADT(datum); buf = &timebuf; len = sizeof(timebuf); break; case TIMETZOID: /* time with time zone */ /* * will not compare to TIMEOID on equal values. * Postgres never attempts to compare the two as well. */ timetzptr = DatumGetTimeTzADTP(datum); buf = (unsigned char *) timetzptr; /* * Specify hash length as sizeof(double) + sizeof(int4), not as * sizeof(TimeTzADT), so that any garbage pad bytes in the structure * won't be included in the hash! */ len = sizeof(timetzptr->time) + sizeof(timetzptr->zone); break; case INTERVALOID: /* @ <number> <units>, time interval */ intervalptr = DatumGetIntervalP(datum); buf = (unsigned char *) intervalptr; /* * Specify hash length as sizeof(double) + sizeof(int4), not as * sizeof(Interval), so that any garbage pad bytes in the structure * won't be included in the hash! */ len = sizeof(intervalptr->time) + sizeof(intervalptr->month); break; case ABSTIMEOID: abstime_buf = DatumGetAbsoluteTime(datum); if (abstime_buf == INVALID_ABSTIME) { /* hash to a constant value */ invalidbuf = INVALID_VAL; len = sizeof(invalidbuf); buf = &invalidbuf; } else { len = sizeof(abstime_buf); buf = &abstime_buf; } break; case RELTIMEOID: reltime_buf = DatumGetRelativeTime(datum); if (reltime_buf == INVALID_RELTIME) { /* hash to a constant value */ invalidbuf = INVALID_VAL; len = sizeof(invalidbuf); buf = &invalidbuf; } else { len = sizeof(reltime_buf); buf = &reltime_buf; } break; case TINTERVALOID: tinterval = DatumGetTimeInterval(datum); /* * check if a valid interval. the '0' status code * stands for T_INTERVAL_INVAL which is defined in * nabstime.c. We use the actual value instead * of defining it again here. */ if(tinterval->status == 0 || tinterval->data[0] == INVALID_ABSTIME || tinterval->data[1] == INVALID_ABSTIME) { /* hash to a constant value */ invalidbuf = INVALID_VAL; len = sizeof(invalidbuf); buf = &invalidbuf; } else { /* normalize on length of the time interval */ tinterval_len = tinterval->data[1] - tinterval->data[0]; len = sizeof(tinterval_len); buf = &tinterval_len; } break; /* * ======= NETWORK TYPES ======== */ case INETOID: case CIDROID: inetptr = DatumGetInetP(datum); len = inet_getkey(inetptr, inet_hkey, sizeof(inet_hkey)); /* fill-in inet_key & get len */ buf = inet_hkey; break; case MACADDROID: macptr = DatumGetMacaddrP(datum); len = sizeof(macaddr); buf = (unsigned char *) macptr; break; /* * ======== BIT STRINGS ======== */ case BITOID: case VARBITOID: /* * Note that these are essentially strings. * we don't need to worry about '10' and '010' * to compare, b/c they will not, by design. * (see SQL standard, and varbit.c) */ vbitptr = DatumGetVarBitP(datum); len = VARBITBYTES(vbitptr); buf = (char *) VARBITS(vbitptr); break; /* * ======= other types ======= */ case BOOLOID: /* boolean, 'true'/'false' */ bool_buf = DatumGetBool(datum); buf = &bool_buf; len = sizeof(bool_buf); break; /* * We prepare the hash key for aclitems just like postgresql does. * (see code and comment in acl.c: hash_aclitem() ). */ case ACLITEMOID: aclitem_ptr = DatumGetAclItemP(datum); aclitem_buf = (uint32) (aclitem_ptr->ai_privs + aclitem_ptr->ai_grantee + aclitem_ptr->ai_grantor); buf = &aclitem_buf; len = sizeof(aclitem_buf); break; /* * ANYARRAY is a pseudo-type. We use it to include * any of the array types (OIDs 1007-1033 in pg_type.h). * caller needs to be sure the type is ANYARRAYOID * before calling cdbhash on an array (INSERT and COPY do so). */ case ANYARRAYOID: arrbuf = DatumGetArrayTypeP(datum); len = VARSIZE(arrbuf) - VARHDRSZ; buf = VARDATA(arrbuf); break; case INT2VECTOROID: i2vec_buf = (int2vector *) DatumGetPointer(datum); len = i2vec_buf->dim1 * sizeof(int2); buf = (void *)i2vec_buf->values; break; case OIDVECTOROID: oidvec_buf = (oidvector *) DatumGetPointer(datum); len = oidvec_buf->dim1 * sizeof(Oid); buf = oidvec_buf->values; break; case CASHOID: /* cash is stored in int32 internally */ cash_buf = (* (Cash *)DatumGetPointer(datum)); len = sizeof(Cash); buf = &cash_buf; break; default: ereport(ERROR, (errcode(ERRCODE_CDB_FEATURE_NOT_YET), errmsg("Type %u is not hashable.", type))); } /* switch(type) */ /* do the hash using the selected algorithm */ hashFn(clientData, buf, len); if(tofree) pfree(tofree); }
/* * pgfincore_file handle the mmaping, mincore process (and access file, etc.) */ static int pgfincore_file(char *filename, pgfincoreStruct *pgfncr) { int flag=1; int flag_dirty=1; int len, bitlen; bits8 *r; bits8 x = 0; register int64 pageIndex; /* * We use the AllocateFile(2) provided by PostgreSQL. We're going to * close it ourselves even if PostgreSQL close it anyway at transaction * end. */ FILE *fp; int fd; struct stat st; #ifndef HAVE_FINCORE void *pa = (char *) 0; #endif unsigned char *vec = (unsigned char *) 0; /* * OS Page size */ pgfncr->pageSize = sysconf(_SC_PAGESIZE); /* * Initialize counters */ pgfncr->pages_mem = 0; pgfncr->group_mem = 0; pgfncr->pages_dirty = 0; pgfncr->group_dirty = 0; pgfncr->rel_os_pages = 0; /* * Fopen and fstat file * fd will be provided to posix_fadvise * if there is no file, just return 1, it is expected to leave the SRF */ fp = AllocateFile(filename, "rb"); if (fp == NULL) return 1; fd = fileno(fp); if (fstat(fd, &st) == -1) { FreeFile(fp); elog(ERROR, "Can not stat object file : %s", filename); return 2; } /* * if file ok * then process */ if (st.st_size != 0) { /* number of pages in the current file */ pgfncr->rel_os_pages = (st.st_size+pgfncr->pageSize-1)/pgfncr->pageSize; #ifndef HAVE_FINCORE pa = mmap(NULL, st.st_size, PROT_NONE, MAP_SHARED, fd, 0); if (pa == MAP_FAILED) { FreeFile(fp); elog(ERROR, "Can not mmap object file : %s, errno = %i,%s\nThis error can happen if there is not enought space in memory to do the projection. Please mail [email protected] with '[pgfincore] ENOMEM' as subject.", filename, errno, strerror(errno)); return 3; } #endif /* Prepare our vector containing all blocks information */ vec = calloc(1, (st.st_size+pgfncr->pageSize-1)/pgfncr->pageSize); if ((void *)0 == vec) { #ifndef HAVE_FINCORE munmap(pa, st.st_size); #endif FreeFile(fp); elog(ERROR, "Can not calloc object file : %s", filename); return 4; } #ifndef HAVE_FINCORE /* Affect vec with mincore */ if (mincore(pa, st.st_size, vec) != 0) { munmap(pa, st.st_size); elog(ERROR, "mincore(%p, %lld, %p): %s\n", pa, (long long int)st.st_size, vec, strerror(errno)); #else /* Affect vec with fincore */ if (fincore(fd, 0, st.st_size, vec) != 0) { elog(ERROR, "fincore(%u, 0, %lld, %p): %s\n", fd, (long long int)st.st_size, vec, strerror(errno)); #endif free(vec); FreeFile(fp); return 5; } /* * prepare the bit string */ bitlen = FINCORE_BITS * ((st.st_size+pgfncr->pageSize-1)/pgfncr->pageSize); len = VARBITTOTALLEN(bitlen); /* * set to 0 so that *r is always initialised and string is zero-padded * XXX: do we need to free that ? */ pgfncr->databit = (VarBit *) palloc0(len); SET_VARSIZE(pgfncr->databit, len); VARBITLEN(pgfncr->databit) = bitlen; r = VARBITS(pgfncr->databit); x = HIGHBIT; /* handle the results */ for (pageIndex = 0; pageIndex <= pgfncr->rel_os_pages; pageIndex++) { // block in memory if (vec[pageIndex] & FINCORE_PRESENT) { pgfncr->pages_mem++; *r |= x; if (FINCORE_BITS > 1) { if (vec[pageIndex] & FINCORE_DIRTY) { pgfncr->pages_dirty++; *r |= (x >> 1); /* we flag to detect contigous blocks in the same state */ if (flag_dirty) pgfncr->group_dirty++; flag_dirty = 0; } else flag_dirty = 1; }
/* * pgfadvise_file */ static int pgfadvise_loader_file(char *filename, bool willneed, bool dontneed, VarBit *databit, pgfloaderStruct *pgfloader) { bits8 *sp; int bitlen; bits8 x; int i, k; /* * We use the AllocateFile(2) provided by PostgreSQL. We're going to * close it ourselves even if PostgreSQL close it anyway at transaction * end. */ FILE *fp; int fd; struct stat st; /* * OS things : Page size */ pgfloader->pageSize = sysconf(_SC_PAGESIZE); /* * we count the action we perform * both are theorical : we don't know if the page was or not in memory * when we call posix_fadvise */ pgfloader->pagesLoaded = 0; pgfloader->pagesUnloaded = 0; /* * Fopen and fstat file * fd will be provided to posix_fadvise * if there is no file, just return 1, it is expected to leave the SRF */ fp = AllocateFile(filename, "rb"); if (fp == NULL) return 1; fd = fileno(fp); if (fstat(fd, &st) == -1) { FreeFile(fp); elog(ERROR, "pgfadvise_loader: Can not stat object file: %s", filename); return 2; } elog(DEBUG1, "pgfadvise_loader: working on %s", filename); bitlen = VARBITLEN(databit); sp = VARBITS(databit); for (i = 0; i < bitlen - BITS_PER_BYTE; i += BITS_PER_BYTE, sp++) { x = *sp; /* Is this bit set ? */ for (k = 0; k < BITS_PER_BYTE; k++) { if (IS_HIGHBIT_SET(x)) { if (willneed) { (void) posix_fadvise(fd, ((i+k) * pgfloader->pageSize), pgfloader->pageSize, POSIX_FADV_WILLNEED); pgfloader->pagesLoaded++; } } else if (dontneed) { (void) posix_fadvise(fd, ((i+k) * pgfloader->pageSize), pgfloader->pageSize, POSIX_FADV_DONTNEED); pgfloader->pagesUnloaded++; } x <<= 1; } } /* * XXX this copy/paste of code to finnish to walk the bits is not pretty */ if (i < bitlen) { /* print the last partial byte */ x = *sp; for (k = i; k < bitlen; k++) { if (IS_HIGHBIT_SET(x)) { if (willneed) { (void) posix_fadvise(fd, (k * pgfloader->pageSize), pgfloader->pageSize, POSIX_FADV_WILLNEED); pgfloader->pagesLoaded++; } } else if (dontneed) { (void) posix_fadvise(fd, (k * pgfloader->pageSize), pgfloader->pageSize, POSIX_FADV_DONTNEED); pgfloader->pagesUnloaded++; } x <<= 1; } } FreeFile(fp); /* * OS things : Pages free */ pgfloader->pagesFree = sysconf(_SC_AVPHYS_PAGES); return 0; }