/* Case 2 - unmapped memory is unaddressable+undefined */
static void test2()
{
char *m = mm(0, pgsz * 5, PROT_READ|PROT_WRITE);
VALGRIND_CHECK_MEM_IS_DEFINED(m, pgsz*5); /* all OK */
munmap(&m[pgsz*2], pgsz);
VALGRIND_CHECK_MEM_IS_DEFINED(&m[pgsz*2], pgsz); /* undefined */
/* XXX need a memcheck request to test addressability */
m[pgsz*2] = 'x'; /* unmapped fault */
}
/* Case 5 - mprotect doesn't affect definedness */
static void test5()
{
char *m = mm(0, pgsz * 5, PROT_READ|PROT_WRITE);
VALGRIND_MAKE_MEM_UNDEFINED(m, pgsz*5);
memset(m, 'x', 10);
VALGRIND_CHECK_MEM_IS_DEFINED(m, 10); /* OK */
VALGRIND_CHECK_MEM_IS_DEFINED(m+10, 10); /* BAD */
mprotect(m, pgsz*5, PROT_NONE);
mprotect(m, pgsz*5, PROT_READ);
VALGRIND_CHECK_MEM_IS_DEFINED(m, 10); /* still OK */
VALGRIND_CHECK_MEM_IS_DEFINED(m+20, 10); /* BAD */
}
// . returns -1 if no boundary found
int32_t HttpMime::getMimeLen ( char *buf , int32_t bufLen , int32_t *bsize ) {
#ifdef _VALGRIND_
VALGRIND_CHECK_MEM_IS_DEFINED(buf,bufLen);
#endif
// size of the boundary
*bsize = 0;
// find the boundary
int32_t i;
for ( i = 0 ; i < bufLen ; i++ ) {
// continue until we hit a \r or \n
if ( buf[i] != '\r' && buf[i] != '\n' ) continue;
// boundary check
if ( i + 1 >= bufLen ) continue;
// prepare for a smaller mime size
*bsize = 1;
// \r\r
if ( buf[i ] == '\r' && buf[i+1] == '\r' ) break;
// \n\n
if ( buf[i ] == '\n' && buf[i+1] == '\n' ) break;
// boundary check
if ( i + 3 >= bufLen ) continue;
// prepare for a larger mime size
*bsize = 2;
// \r\n\r\n
if ( buf[i ] == '\r' && buf[i+1] == '\n' &&
buf[i+2] == '\r' && buf[i+3] == '\n' ) break;
// \n\r\n\r
if ( buf[i ] == '\n' && buf[i+1] == '\r' &&
buf[i+2] == '\n' && buf[i+3] == '\r' ) break;
}
// return false if could not find the end of the MIME
if ( i == bufLen ) return -1;
return i + *bsize * 2;
}
void valgrindFreeObject(MM_GCExtensionsBase *extensions, uintptr_t baseAddress)
{
int objSize;
if (MM_ForwardedHeader((omrobjectptr_t)baseAddress).isForwardedPointer())
{
/* In scavanger an object may act as pointer to another object(it's replica in another region).
In this case, getConsumedSizeInBytesWithHeader returns some junk value.
So instead we calculate the size of the object (replica) it is pointing to
and use it for freeing original object.
*/
omrobjectptr_t fwObject = MM_ForwardedHeader((omrobjectptr_t)baseAddress).getForwardedObject();
objSize = (int)((GC_ObjectModel)extensions->objectModel).getConsumedSizeInBytesWithHeader(fwObject);
}
else
{
objSize = (int)((GC_ObjectModel)extensions->objectModel).getConsumedSizeInBytesWithHeader((omrobjectptr_t)baseAddress);
}
#if defined(VALGRIND_REQUEST_LOGS)
VALGRIND_PRINTF_BACKTRACE("Clearing an object at 0x%lx of size %d\n", baseAddress, objSize);
#endif /* defined(VALGRIND_REQUEST_LOGS) */
VALGRIND_CHECK_MEM_IS_DEFINED(baseAddress, objSize);
VALGRIND_MEMPOOL_FREE(extensions->valgrindMempoolAddr, baseAddress);
MUTEX_ENTER(extensions->memcheckHashTableMutex);
hashTableRemove(extensions->memcheckHashTable, &baseAddress);
MUTEX_EXIT(extensions->memcheckHashTableMutex);
}
/* Case 3 - memory definedness doesn't survive remapping */
static void test3()
{
char *m = mm(0, pgsz * 5, PROT_READ|PROT_WRITE);
VALGRIND_MAKE_MEM_UNDEFINED(&m[pgsz], pgsz);
mm(&m[pgsz], pgsz, PROT_READ);
VALGRIND_CHECK_MEM_IS_DEFINED(&m[pgsz], pgsz); /* OK */
}
TEST(GetErrorString, Success) {
cudaError_t in = cudaSuccess;
const char * out = cudaGetErrorString(in);
ASSERT_FALSE(NULL == out);
size_t len = strlen(out);
int valgrind = VALGRIND_CHECK_MEM_IS_DEFINED(out, len + 1);
EXPECT_EQ(0, valgrind);
}
/* Case 4 - mprotect doesn't affect addressability */
static void test4()
{
char *m = mm(0, pgsz * 5, PROT_READ|PROT_WRITE);
mprotect(m, pgsz, PROT_WRITE);
VALGRIND_CHECK_MEM_IS_DEFINED(m, pgsz); /* OK */
m[44] = 'y'; /* OK */
mprotect(m, pgsz*5, PROT_NONE);
m[55] = 'x'; /* permission fault, but no tool complaint */
}
MMINLINE void valgrindFreeObjectDirect(MM_GCExtensionsBase *extensions, uintptr_t baseAddress)
{
int objSize = (int)((GC_ObjectModel)extensions->objectModel).getConsumedSizeInBytesWithHeader((omrobjectptr_t)baseAddress);
#if defined(VALGRIND_REQUEST_LOGS)
VALGRIND_PRINTF_BACKTRACE("Clearing an object at 0x%lx of size %d\n", baseAddress, objSize);
#endif /* defined(VALGRIND_REQUEST_LOGS) */
VALGRIND_CHECK_MEM_IS_DEFINED(baseAddress, objSize);
VALGRIND_MEMPOOL_FREE(extensions->valgrindMempoolAddr, baseAddress);
}
void croak ( void* aV )
{
char* a = (char*)aV;
char* undefp = malloc(1);
char saved = *a;
assert(undefp);
*a = *undefp;
VALGRIND_CHECK_MEM_IS_DEFINED(a, 1);
*a = saved;
free(undefp);
}
static void ucp_address_memchek(void *ptr, size_t size,
const uct_tl_resource_desc_t *rsc)
{
void *undef_ptr;
undef_ptr = (void*)VALGRIND_CHECK_MEM_IS_DEFINED(ptr, size);
if (undef_ptr != NULL) {
ucs_error(UCT_TL_RESOURCE_DESC_FMT
" address contains undefined bytes at offset %zd",
UCT_TL_RESOURCE_DESC_ARG(rsc), undef_ptr - ptr);
}
}
bool verify_mutable_item(
std::pair<char const*, int> v,
std::pair<char const*, int> salt,
boost::uint64_t seq,
char const* pk,
char const* sig)
{
#ifdef TORRENT_USE_VALGRIND
VALGRIND_CHECK_MEM_IS_DEFINED(v.first, v.second);
VALGRIND_CHECK_MEM_IS_DEFINED(pk, item_pk_len);
VALGRIND_CHECK_MEM_IS_DEFINED(sig, item_sig_len);
#endif
char str[canonical_length];
int len = canonical_string(v, seq, salt, str);
return ed25519_verify((unsigned char const*)sig,
(unsigned char const*)str,
len,
(unsigned char const*)pk) == 1;
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpyDtoDAsync)(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) {
int error = 0;
long vgErrorAddress;
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&hStream, sizeof(CUstream));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'hStream' in call to cuMemcpyDtoDAsync not defined.\n");
}
cgLock();
CUcontext ctx = NULL;
cgCtxListType *nodeCtx;
cgMemListType *nodeMemDst, *nodeMemSrc;
// Get current context ..
cgGetCtx(&ctx);
nodeCtx = cgFindCtx(ctx);
// .. and locate memory if we are handling device memory
nodeMemDst = cgFindMem(nodeCtx, dstDevice);
nodeMemSrc = cgFindMem(nodeCtx, srcDevice);
if (nodeMemDst && nodeMemDst->locked & 2 && nodeMemDst->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access by different streams.\n");
}
if (nodeMemSrc && nodeMemSrc->locked && nodeMemSrc->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access by different streams.\n");
}
if (nodeMemDst) {
nodeMemDst->locked = nodeMemDst->locked | 2;
nodeMemDst->stream = hStream;
}
if (nodeMemSrc) {
nodeMemSrc->locked = nodeMemSrc->locked | 1;
nodeMemSrc->stream = hStream;
}
cgUnlock();
if (error) {
VALGRIND_PRINTF_BACKTRACE("");
}
return cuMemcpyDtoD(dstDevice, srcDevice, ByteCount);
}
// . returns false if could not get a valid mime
// . we need the url in case there's a Location: mime that's base-relative
bool HttpMime::set ( char *buf , int32_t bufLen , Url *url ) {
#ifdef _VALGRIND_
VALGRIND_CHECK_MEM_IS_DEFINED(buf,bufLen);
#endif
// reset some stuff
m_mime = NULL;
m_currentLine = NULL;
m_currentLineLen = 0;
m_valueStartPos = 0;
m_nextLineStartPos = 0;
m_attributeStartPos = 0;
if (!m_fakeCurrentTime) {
m_currentTime = time(NULL);
}
m_contentLen = -1;
m_content = NULL;
m_mimeLen = 0;
m_contentType = CT_HTML;
m_contentEncoding = ET_IDENTITY;
m_charset = NULL;
m_charsetLen = 0;
m_cookies.clear();
// at the very least we should have a "HTTP/x.x 404\[nc]"
if ( bufLen < 13 ) {
return false;
}
// . get the length of the Mime, must end in \r\n\r\n , ...
// . m_bufLen is used as the mime length
m_mime = buf;
m_mimeLen = getMimeLen(buf, bufLen);
// . return false if we had no mime boundary
// . but set m_bufLen to 0 so getMimeLen() will return 0 instead of -1
// thus avoiding a potential buffer overflow
if (m_mimeLen == 0) {
log(LOG_WARN, "mime: no rnrn boundary detected");
return false;
}
// set this
m_content = buf + m_mimeLen;
// . parse out m_status, m_contentLen, m_lastModifiedData, contentType
// . returns false on bad mime
return parse ( buf , m_mimeLen , url );
}
ucs_status_t ucp_address_pack(ucp_worker_h worker, ucp_ep_h ep, uint64_t tl_bitmap,
unsigned *order, size_t *size_p, void **buffer_p)
{
ucp_address_packed_device_t *devices;
ucp_rsc_index_t num_devices;
ucs_status_t status;
void *buffer;
size_t size;
/* Collect all devices we want to pack */
status = ucp_address_gather_devices(worker, tl_bitmap, ep != NULL,
&devices, &num_devices);
if (status != UCS_OK) {
goto out;
}
/* Calculate packed size */
size = ucp_address_packed_size(worker, devices, num_devices);
/* Allocate address */
buffer = ucs_malloc(size, "ucp_address");
if (buffer == NULL) {
status = UCS_ERR_NO_MEMORY;
goto out_free_devices;
}
memset(buffer, 0, size);
/* Pack the address */
status = ucp_address_do_pack(worker, ep, buffer, size, tl_bitmap, order,
devices, num_devices);
if (status != UCS_OK) {
ucs_free(buffer);
goto out_free_devices;
}
VALGRIND_CHECK_MEM_IS_DEFINED(buffer, size);
*size_p = size;
*buffer_p = buffer;
status = UCS_OK;
out_free_devices:
ucs_free(devices);
out:
return status;
}
void
_cairo_debug_check_image_surface_is_defined (const cairo_surface_t *surface)
{
const cairo_image_surface_t *image = (cairo_image_surface_t *) surface;
const uint8_t *bits;
int row, width;
if (surface == NULL)
return;
if (! RUNNING_ON_VALGRIND)
return;
bits = image->data;
switch (image->format) {
case CAIRO_FORMAT_A1:
width = (image->width + 7)/8;
break;
case CAIRO_FORMAT_A8:
width = image->width;
break;
case CAIRO_FORMAT_RGB16_565:
width = image->width*2;
break;
case CAIRO_FORMAT_RGB24:
case CAIRO_FORMAT_RGB30:
case CAIRO_FORMAT_ARGB32:
width = image->width*4;
break;
case CAIRO_FORMAT_INVALID:
default:
/* XXX compute width from pixman bpp */
return;
}
for (row = 0; row < image->height; row++) {
VALGRIND_CHECK_MEM_IS_DEFINED (bits, width);
/* and then silence any future valgrind warnings */
VALGRIND_MAKE_MEM_DEFINED (bits, width);
bits += image->stride;
}
}
// . returns 0 if no boundary found
size_t HttpMime::getMimeLen(char *buf, size_t bufLen) {
#ifdef _VALGRIND_
VALGRIND_CHECK_MEM_IS_DEFINED(buf,bufLen);
#endif
// the size of the terminating boundary, either 1 or 2 bytes.
// just the last \n in the case of a \n\n or \r in the case
// of a \r\r, but it is the full \r\n in the case of a last \r\n\r\n
size_t bsize = 0;
// find the boundary
size_t i;
for ( i = 0 ; i < bufLen ; i++ ) {
// continue until we hit a \r or \n
if ( buf[i] != '\r' && buf[i] != '\n' ) continue;
// boundary check
if ( i + 1 >= bufLen ) continue;
// prepare for a smaller mime size
bsize = 1;
// \r\r
if ( buf[i ] == '\r' && buf[i+1] == '\r' ) break;
// \n\n
if ( buf[i ] == '\n' && buf[i+1] == '\n' ) break;
// boundary check
if ( i + 3 >= bufLen ) continue;
// prepare for a larger mime size
bsize = 2;
// \r\n\r\n
if ( buf[i ] == '\r' && buf[i+1] == '\n' &&
buf[i+2] == '\r' && buf[i+3] == '\n' ) break;
// \n\r\n\r
if ( buf[i ] == '\n' && buf[i+1] == '\r' &&
buf[i+2] == '\n' && buf[i+3] == '\r' ) break;
}
// return false if could not find the end of the MIME
if ( i == bufLen ) {
return 0;
}
return i + bsize * 2;
}
/*
* PageAddItemExtended
*
* Add an item to a page. Return value is the offset at which it was
* inserted, or InvalidOffsetNumber if the item is not inserted for any
* reason. A WARNING is issued indicating the reason for the refusal.
*
* offsetNumber must be either InvalidOffsetNumber to specify finding a
* free item pointer, or a value between FirstOffsetNumber and one past
* the last existing item, to specify using that particular item pointer.
*
* If offsetNumber is valid and flag PAI_OVERWRITE is set, we just store
* the item at the specified offsetNumber, which must be either a
* currently-unused item pointer, or one past the last existing item.
*
* If offsetNumber is valid and flag PAI_OVERWRITE is not set, insert
* the item at the specified offsetNumber, moving existing items later
* in the array to make room.
*
* If offsetNumber is not valid, then assign a slot by finding the first
* one that is both unused and deallocated.
*
* If flag PAI_IS_HEAP is set, we enforce that there can't be more than
* MaxHeapTuplesPerPage line pointers on the page.
*
* !!! EREPORT(ERROR) IS DISALLOWED HERE !!!
*/
OffsetNumber
PageAddItemExtended(Page page,
Item item,
Size size,
OffsetNumber offsetNumber,
int flags)
{
PageHeader phdr = (PageHeader) page;
Size alignedSize;
int lower;
int upper;
ItemId itemId;
OffsetNumber limit;
bool needshuffle = false;
/*
* Be wary about corrupted page pointers
*/
if (phdr->pd_lower < SizeOfPageHeaderData ||
phdr->pd_lower > phdr->pd_upper ||
phdr->pd_upper > phdr->pd_special ||
phdr->pd_special > BLCKSZ)
ereport(PANIC,
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("corrupted page pointers: lower = %u, upper = %u, special = %u",
phdr->pd_lower, phdr->pd_upper, phdr->pd_special)));
/*
* Select offsetNumber to place the new item at
*/
limit = OffsetNumberNext(PageGetMaxOffsetNumber(page));
/* was offsetNumber passed in? */
if (OffsetNumberIsValid(offsetNumber))
{
/* yes, check it */
if ((flags & PAI_OVERWRITE) != 0)
{
if (offsetNumber < limit)
{
itemId = PageGetItemId(phdr, offsetNumber);
if (ItemIdIsUsed(itemId) || ItemIdHasStorage(itemId))
{
elog(WARNING, "will not overwrite a used ItemId");
return InvalidOffsetNumber;
}
}
}
else
{
if (offsetNumber < limit)
needshuffle = true; /* need to move existing linp's */
}
}
else
{
/* offsetNumber was not passed in, so find a free slot */
/* if no free slot, we'll put it at limit (1st open slot) */
if (PageHasFreeLinePointers(phdr))
{
/*
* Look for "recyclable" (unused) ItemId. We check for no storage
* as well, just to be paranoid --- unused items should never have
* storage.
*/
for (offsetNumber = 1; offsetNumber < limit; offsetNumber++)
{
itemId = PageGetItemId(phdr, offsetNumber);
if (!ItemIdIsUsed(itemId) && !ItemIdHasStorage(itemId))
break;
}
if (offsetNumber >= limit)
{
/* the hint is wrong, so reset it */
PageClearHasFreeLinePointers(phdr);
}
}
else
{
/* don't bother searching if hint says there's no free slot */
offsetNumber = limit;
}
}
/* Reject placing items beyond the first unused line pointer */
if (offsetNumber > limit)
{
elog(WARNING, "specified item offset is too large");
return InvalidOffsetNumber;
}
/* Reject placing items beyond heap boundary, if heap */
if ((flags & PAI_IS_HEAP) != 0 && offsetNumber > MaxHeapTuplesPerPage)
{
elog(WARNING, "can't put more than MaxHeapTuplesPerPage items in a heap page");
return InvalidOffsetNumber;
}
/*
* Compute new lower and upper pointers for page, see if it'll fit.
*
* Note: do arithmetic as signed ints, to avoid mistakes if, say,
* alignedSize > pd_upper.
*/
if (offsetNumber == limit || needshuffle)
lower = phdr->pd_lower + sizeof(ItemIdData);
else
lower = phdr->pd_lower;
alignedSize = MAXALIGN(size);
upper = (int) phdr->pd_upper - (int) alignedSize;
if (lower > upper)
return InvalidOffsetNumber;
/*
* OK to insert the item. First, shuffle the existing pointers if needed.
*/
itemId = PageGetItemId(phdr, offsetNumber);
if (needshuffle)
memmove(itemId + 1, itemId,
(limit - offsetNumber) * sizeof(ItemIdData));
/* set the item pointer */
ItemIdSetNormal(itemId, upper, size);
/*
* Items normally contain no uninitialized bytes. Core bufpage consumers
* conform, but this is not a necessary coding rule; a new index AM could
* opt to depart from it. However, data type input functions and other
* C-language functions that synthesize datums should initialize all
* bytes; datumIsEqual() relies on this. Testing here, along with the
* similar check in printtup(), helps to catch such mistakes.
*
* Values of the "name" type retrieved via index-only scans may contain
* uninitialized bytes; see comment in btrescan(). Valgrind will report
* this as an error, but it is safe to ignore.
*/
VALGRIND_CHECK_MEM_IS_DEFINED(item, size);
/* copy the item's data onto the page */
memcpy((char *) page + upper, item, size);
/* adjust page header */
phdr->pd_lower = (LocationIndex) lower;
phdr->pd_upper = (LocationIndex) upper;
return offsetNumber;
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemsetD16)(CUdeviceptr dstDevice, unsigned short us, size_t N) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgMemListType *nodeMemDst;
int error = 0;
long vgErrorAddress;
size_t dstSize;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_WWW(result, fn, dstDevice, us, N);
// Check if function parameters are defined.
// TODO: Warning or error in case of a partially undefined us?
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&dstDevice, sizeof(CUdeviceptr));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'dstDevice' in call to cuMemsetD16 not defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&us, sizeof(us));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Warning: 'us' in call to cuMemsetD16 is not fully defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&N, sizeof(size_t));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'N' in call to cuMemsetD16 not defined.\n");
}
// Fetch current context
cgGetCtx(&ctx);
nodeMemDst = cgFindMem(cgFindCtx(ctx), dstDevice);
// Check if memory has been allocated
if (!nodeMemDst) {
error++;
VALGRIND_PRINTF("Error: Destination device memory not allocated in call to cuMemsetD16.\n");
} else {
// If memory is allocated, check size of available memory
dstSize = nodeMemDst->size - (dstDevice - nodeMemDst->dptr);
if (dstSize < sizeof(unsigned short) * N) {
error++;
VALGRIND_PRINTF("Error: Allocated device memory too small in call to cuMemsetD16.\n"
" Expected %lu allocated bytes but only found %lu.\n",
sizeof(unsigned short) * N, dstSize);
}
// Check if pointer is properly two byte aligned.
// TODO: Is this a valid check?
if (dstDevice % 2) {
error++;
VALGRIND_PRINTF("Error: Pointer dstDevice in call to cuMemsetD16 not two byte aligned.\n");
}
}
if (error) {
VALGRIND_PRINTF_BACKTRACE("");
}
cgUnlock();
return result;
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemsetD2D32)(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgMemListType *nodeMemDst;
int error = 0;
long vgErrorAddress;
size_t dstSize;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_5W(result, fn, dstDevice, dstPitch, ui, Width, Height);
// Check if function parameters are defined.
// TODO: Warning or error in case of a partially undefined ui?
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&dstDevice, sizeof(CUdeviceptr));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'dstDevice' in call to cuMemsetD2D32 not defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&dstPitch, sizeof(size_t));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'dstPitch' in call to cuMemsetD2D32 not defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&ui, sizeof(ui));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Warning: 'ui' in call to cuMemsetD2D32 is not fully defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&Width, sizeof(size_t));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'Width' in call to cuMemsetD2D32 not defined.\n");
}
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&Height, sizeof(size_t));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'Height' in call to cuMemsetD2D32 not defined.\n");
}
// Fetch current context
cgGetCtx(&ctx);
nodeMemDst = cgFindMem(cgFindCtx(ctx), dstDevice);
// Check if memory has been allocated
if (!nodeMemDst) {
error++;
VALGRIND_PRINTF("Error: Destination device memory not allocated in call to cuMemsetD2D32.\n");
} else {
// If memory is allocated, check size of available memory
dstSize = nodeMemDst->size - (dstDevice - nodeMemDst->dptr);
// The whole memory block of dstPitch*Height must fit into memory
if (dstSize < sizeof(ui) * dstPitch * Height) {
error++;
VALGRIND_PRINTF("Error: Allocated device memory too small in call to cuMemsetD2D32.\n"
" Expected %lu allocated bytes but only found %lu.\n",
sizeof(ui) * dstPitch * Height, dstSize);
}
// Check if dstDevice and dstPitch are both properly aligned
// TODO: Is this a valid check? (see also cuMemsetD32)
if (dstDevice % 4) {
error++;
VALGRIND_PRINTF("Error: Pointer dstDevice in call to cuMemsetD2D32 not four byte aligned.\n");
}
if (dstPitch % 4) {
error++;
VALGRIND_PRINTF("Error: Destination pitch in call to cuMemsetD2D32 not four byte aligned.\n");
}
}
// Make sure pitch is big enough to accommodate asked for Width
if (dstPitch < Width) {
error++;
VALGRIND_PRINTF("Error: dstPitch smaller than Width in call to cuMemsetD2D32.\n");
}
if (error) {
VALGRIND_PRINTF_BACKTRACE("");
}
cgUnlock();
return result;
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpy3D)(const CUDA_MEMCPY3D *pCopy) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
int error = 0, error_addressable, error_defined;
long vgErrorAddress = 0, vgErrorAddressDefined = 0;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_W(result, fn, pCopy);
// Check if pCopy is null, not allocated or undefined.
// For obvious reasons we skip the following checks if either condition is true.
if (!pCopy) {
error++;
VALGRIND_PRINTF_BACKTRACE("Error: pCopy in call to cuMemcpy3D is NULL.\n");
cgUnlock();
return result;
} else if ( vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(pCopy, sizeof(CUDA_MEMCPY3D)) ) {
error++;
VALGRIND_PRINTF_BACKTRACE("Error: pCopy in call to cuMemcpy3D points to unallocated memory.\n");
cgUnlock();
return result;
} // It makes no sense to check _IS_DEFINED on the whole structure, since only part of it is used!
// General checks of constaints imposed by reference manual
if (pCopy->srcMemoryType != CU_MEMORYTYPE_ARRAY) {
if (pCopy->srcPitch && pCopy->srcPitch < pCopy->WidthInBytes + pCopy->srcXInBytes) {
error++;
VALGRIND_PRINTF("Error: srcPitch < WidthInBytes+srcXInBytes in cuMemcpy3D.\n");
}
if (pCopy->srcHeight && pCopy->srcHeight < pCopy->Height + pCopy->srcY) {
error++;
VALGRIND_PRINTF("Error: srcHeight < Height+srcY in cuMemcpy3D.\n");
}
}
if (pCopy->dstMemoryType != CU_MEMORYTYPE_ARRAY) {
if (pCopy->dstPitch && pCopy->dstPitch < pCopy->WidthInBytes + pCopy->dstXInBytes) {
error++;
VALGRIND_PRINTF("Error: dstPitch < WidthInBytes+dstXInBytes in cuMemcpy3D.\n");
}
if (pCopy->dstHeight && pCopy->dstHeight < pCopy->Height + pCopy->dstY) {
error++;
VALGRIND_PRINTF("Error: dstHeight < Height+dstY in cuMemcpy3D.\n");
}
}
switch (pCopy->srcMemoryType) {
case CU_MEMORYTYPE_UNIFIED:
// TODO: How do we handle unified memory?
break;
case CU_MEMORYTYPE_HOST: {
void *line;
error_addressable = 0;
error_defined = 0;
// TODO: Is Height, Depth > 1, even for 1D/2D copy operations?
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (void*)(
(char*)pCopy->srcHost
+ ((pCopy->srcZ + j) * pCopy->srcHeight + (pCopy->srcY + i))*pCopy->srcPitch
+ pCopy->srcXInBytes
);
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_addressable++;
} else {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_defined++;
}
}
}
}
// TODO: Can we give precise information about location of error?
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) source host memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
if (error_defined) {
error++;
VALGRIND_PRINTF("Error: (Part of) source host memory not defined\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_DEVICE: {
// ptrEnd points to the end of the memory area which pCopy->srcDevice points into
CUdeviceptr line, ptrEnd;
cgMemListType *nodeMem;
// TODO: Check if pCopy->srcDevice is defined?
cgGetCtx(&ctx);
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->srcDevice);
// We only track addressable status (whether memory is allocated) for device memory regions
error_addressable = 0;
if (nodeMem) {
ptrEnd = nodeMem->dptr + nodeMem->size;
/*
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (CUdeviceptr)(
pCopy->srcDevice
+ ((pCopy->srcZ + j) * pCopy->srcHeight + (pCopy->srcY + i)) * pCopy->srcPitch
+ pCopy->srcXInBytes
);
// Is there enough allocated memory left to statisfy the current line?
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
}
}
*/
// Device memory should not be fragmented, so we only check the very last slice of memory
line = (CUdeviceptr)(
pCopy->srcDevice
+ (
(pCopy->srcZ + pCopy->Depth - 1) * pCopy->srcHeight
+ (pCopy->srcY + pCopy->Height - 1)
) * pCopy->srcPitch
+ pCopy->srcXInBytes);
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
} else {
error_addressable++;
}
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) source device memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
CUDA_ARRAY3D_DESCRIPTOR descriptor;
int bytesPerElement;
int widthInBytes;
// Fetch array descriptor ..
cuArray3DGetDescriptor(&descriptor, pCopy->srcArray);
bytesPerElement = cgArrDescBytesPerElement(&descriptor);
if (!bytesPerElement) {
error++;
VALGRIND_PRINTF("Error: Unknown Format value in src array descriptor in cuMemcpy3D.\n");
}
widthInBytes = bytesPerElement * descriptor.Width;
// .. and check if dimensions are conform to the ones requested in pCopy
if (widthInBytes - pCopy->srcXInBytes < pCopy->WidthInBytes) {
error++;
VALGRIND_PRINTF("Error: Available width of %u bytes in source array is smaller than\n"
" requested Width of %u bytes in pCopy of cuMemcpy3D.\n",
widthInBytes - pCopy->srcXInBytes, pCopy->WidthInBytes);
}
if (pCopy->Height > 1 && descriptor.Height - pCopy->srcY < pCopy->Height) {
error++;
VALGRIND_PRINTF("Error: Available Height of %u in source array is smaller than\n"
" requested Height of %u in pCopy of cuMemcpy3D.\n",
descriptor.Height - pCopy->srcY, pCopy->Height);
}
if (pCopy->Depth > 1 && descriptor.Depth - pCopy->srcZ < pCopy->Depth) {
error++;
VALGRIND_PRINTF("Error: Available Depth of %u in source array is smaller than\n"
" requested Depth of %u in pCopy of cuMemcpy3D.\n",
descriptor.Depth - pCopy->srcY, pCopy->Height);
}
break;
}
default:
error++;
VALGRIND_PRINTF("Error: Unknown source memory type %d in cuMemcpy3D\n");
break;
}
switch (pCopy->dstMemoryType) {
case CU_MEMORYTYPE_UNIFIED:
// TODO: How do we handle unified memory?
break;
case CU_MEMORYTYPE_HOST: {
void *line;
error_addressable = 0;
error_defined = 0;
// TODO: Is Height, Depth > 1, even for 1D/2D copy operations?
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (void*)(
(char*)pCopy->dstHost
+ ((pCopy->dstZ + j) * pCopy->dstHeight + (pCopy->dstY + i))*pCopy->dstPitch
+ pCopy->dstXInBytes
);
// Unlike for the source operand we only need to check allocation status here
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_addressable++;
}
}
}
// TODO: Can we give precise information about location of error?
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) destination host memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_DEVICE: {
// ptrEnd points to the end of the memory area which pCopy->dstDevice points into
CUdeviceptr line, ptrEnd;
cgMemListType *nodeMem;
// TODO: Check if pCopy->dstDevice is defined?
cgGetCtx(&ctx);
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->dstDevice);
// We only track addressable status (whether memory is allocated) for device memory regions
error_addressable = 0;
if (nodeMem) {
ptrEnd = nodeMem->dptr + nodeMem->size;
/*
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (CUdeviceptr)(
pCopy->dstDevice
+ ((pCopy->dstZ + j) * pCopy->dstHeight + (pCopy->dstY + i)) * pCopy->dstPitch
+ pCopy->dstXInBytes
);
// Is there enough allocated memory left to statisfy the current line?
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
}
}
*/
// Device memory should not be fragmented, so we only check the very last slice of memory
line = (CUdeviceptr)(
pCopy->dstDevice
+ (
(pCopy->dstZ + pCopy->Depth - 1) * pCopy->dstHeight
+ (pCopy->dstY + pCopy->Height - 1)
) * pCopy->dstPitch
+ pCopy->dstXInBytes);
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
} else {
error_addressable++;
}
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) destination device memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
CUDA_ARRAY3D_DESCRIPTOR descriptor;
int bytesPerElement;
int widthInBytes;
// Fetch array descriptor ..
cuArray3DGetDescriptor(&descriptor, pCopy->dstArray);
bytesPerElement = cgArrDescBytesPerElement(&descriptor);
if (!bytesPerElement) {
error++;
VALGRIND_PRINTF("Error: Unknown Format value in dst array descriptor in cuMemcpy3D.\n");
}
widthInBytes = bytesPerElement * descriptor.Width;
// .. and check if dimensions are conform to the ones requested in pCopy
if (widthInBytes - pCopy->dstXInBytes < pCopy->WidthInBytes) {
error++;
VALGRIND_PRINTF("Error: Available width of %u bytes in destination array is smaller than\n"
" requested Width of %u bytes in pCopy of cuMemcpy3D.\n",
widthInBytes - pCopy->dstXInBytes, pCopy->WidthInBytes);
}
if (pCopy->Height > 1 && descriptor.Height - pCopy->dstY < pCopy->Height) {
error++;
VALGRIND_PRINTF("Error: Available Height of %u in destination array is smaller than\n"
" requested Height of %u in pCopy of cuMemcpy3D.\n",
descriptor.Height - pCopy->dstY, pCopy->Height);
}
if (pCopy->Depth > 1 && descriptor.Depth - pCopy->dstZ < pCopy->Depth) {
error++;
VALGRIND_PRINTF("Error: Available Depth of %u in destination array is smaller than\n"
" requested Depth of %u in pCopy of cuMemcpy3D.\n",
descriptor.Depth - pCopy->dstZ, pCopy->Depth);
}
break;
}
default:
error++;
VALGRIND_PRINTF("Error: Unknown destination memory type %d in cuMemcpy3D\n");
break;
}
if (error) {
VALGRIND_PRINTF_BACKTRACE(" %d errors detected in call to cuMemcpy3D.", error);
}
cgUnlock();
return result;
}
// returns false and sets g_errno on error
bool Title::setTitle ( Xml *xml, Words *words, int32_t maxTitleLen, Query *query,
LinkInfo *linkInfo, Url *firstUrl, const char *filteredRootTitleBuf, int32_t filteredRootTitleBufSize,
uint8_t contentType, uint8_t langId, int32_t niceness ) {
// make Msg20.cpp faster if it is just has
// Msg20Request::m_setForLinkInfo set to true, no need to extricate a title.
if ( maxTitleLen <= 0 ) {
return true;
}
m_niceness = niceness;
m_maxTitleLen = maxTitleLen;
// if this is too big the "first line" algo can be huge!!!
// and really slow everything way down with a huge title candidate
int32_t maxTitleWords = 128;
// assume no title
reset();
int32_t NW = words->getNumWords();
//
// now get all the candidates
//
// . allow up to 100 title CANDIDATES
// . "as" is the word # of the first word in the candidate
// . "bs" is the word # of the last word IN the candidate PLUS ONE
int32_t n = 0;
int32_t as[MAX_TIT_CANDIDATES];
int32_t bs[MAX_TIT_CANDIDATES];
float scores[MAX_TIT_CANDIDATES];
Words *cptrs[MAX_TIT_CANDIDATES];
int32_t types[MAX_TIT_CANDIDATES];
int32_t parent[MAX_TIT_CANDIDATES];
// record the scoring algos effects
float baseScore [MAX_TIT_CANDIDATES];
float noCapsBoost [MAX_TIT_CANDIDATES];
float qtermsBoost [MAX_TIT_CANDIDATES];
float inCommonCandBoost[MAX_TIT_CANDIDATES];
// reset these
for ( int32_t i = 0 ; i < MAX_TIT_CANDIDATES ; i++ ) {
// assume no parent
parent[i] = -1;
}
// xml and words class for each link info, rss item
Xml tx[MAX_TIT_CANDIDATES];
Words tw[MAX_TIT_CANDIDATES];
int32_t ti = 0;
// restrict how many link texts and rss blobs we check for titles
// because title recs like www.google.com have hundreds and can
// really slow things down to like 50ms for title generation
int32_t kcount = 0;
int32_t rcount = 0;
//int64_t x = gettimeofdayInMilliseconds();
// . get every link text
// . TODO: repeat for linkInfo2, the imported link text
for ( Inlink *k = NULL; linkInfo && (k = linkInfo->getNextInlink(k)) ; ) {
// breathe
QUICKPOLL(m_niceness);
// fast skip check for link text
if ( k->size_linkText >= 3 && ++kcount >= 20 ) continue;
// fast skip check for rss item
if ( k->size_rssItem > 10 && ++rcount >= 20 ) continue;
// set Url
Url u;
u.set( k->getUrl(), k->size_urlBuf );
// is it the same host as us?
bool sh = true;
// skip if not from same host and should be
if ( firstUrl->getHostLen() != u.getHostLen() ) {
sh = false;
}
// skip if not from same host and should be
if ( strncmp( firstUrl->getHost(), u.getHost(), u.getHostLen() ) ) {
sh = false;
}
// get the link text
if ( k->size_linkText >= 3 ) {
char *p = k->getLinkText();
int32_t plen = k->size_linkText - 1;
if ( ! verifyUtf8 ( p , plen ) ) {
log("title: set4 bad link text from url=%s", k->getUrl());
continue;
}
// now the words.
if ( !tw[ti].set( k->getLinkText(), k->size_linkText - 1, true, 0 ) ) {
return false;
}
// set the bookends, it is the whole thing
cptrs [n] = &tw[ti];
as [n] = 0;
bs [n] = tw[ti].getNumWords();
// score higher if same host
if ( sh ) scores[n] = 1.05;
// do not count so high if remote!
else scores[n] = 0.80;
// set the type
if ( sh ) types [n] = TT_LINKTEXTLOCAL;
else types [n] = TT_LINKTEXTREMOTE;
// another candidate
n++;
// use xml and words
ti++;
// break out if too many already. save some for below.
if ( n + 30 >= MAX_TIT_CANDIDATES ) break;
}
// get the rss item
if ( k->size_rssItem <= 10 ) continue;
// . returns false and sets g_errno on error
// . use a 0 for niceness
if ( ! k->setXmlFromRSS ( &tx[ti] , 0 ) ) return false;
// get the word range
int32_t tslen;
bool isHtmlEnc;
char *ts = tx[ti].getRSSTitle ( &tslen , &isHtmlEnc );
// skip if not in the rss
if ( ! ts ) continue;
// skip if empty
if ( tslen <= 0 ) continue;
// now set words to that
if ( !tw[ti].set( ts, tslen, true, 0 ) ) {
return false;
}
// point to that
cptrs [n] = &tw[ti];
as [n] = 0;
bs [n] = tw[ti].getNumWords();
// increment since we are using it
ti++;
// base score for rss title
if ( sh ) scores[n] = 5.0;
// if not same host, treat like link text
else scores[n] = 2.0;
// set the type
if ( sh ) types [n] = TT_RSSITEMLOCAL;
else types [n] = TT_RSSITEMREMOTE;
// advance
n++;
// break out if too many already. save some for below.
if ( n + 30 >= MAX_TIT_CANDIDATES ) break;
}
//logf(LOG_DEBUG,"title: took1=%" PRId64,gettimeofdayInMilliseconds()-x);
//x = gettimeofdayInMilliseconds();
// . set the flags array
// . indicates what words are in title candidates already, but
// that is set below
// . up here we set words that are not allowed to be in candidates,
// like words that are in a link that is not a self link
// . alloc for it
char *flags = NULL;
char localBuf[10000];
int32_t need = words->getNumWords();
if ( need <= 10000 ) {
flags = (char *)localBuf;
} else {
flags = (char *)mmalloc(need,"TITLEflags");
}
if ( ! flags ) {
return false;
}
// clear it
memset ( flags , 0 , need );
// check tags in body
nodeid_t *tids = words->getTagIds();
// scan to set link text flags
// loop over all "words" in the html body
char inLink = false;
char selfLink = false;
for ( int32_t i = 0 ; i < NW ; i++ ) {
// breathe
QUICKPOLL(m_niceness);
// if in a link that is not self link, cannot be in a candidate
if ( inLink && ! selfLink ) {
flags[i] |= 0x02;
}
// out of a link
if ( tids[i] == (TAG_A | BACKBIT) ) {
inLink = false;
}
// if not start of <a> tag, skip it
if ( tids[i] != TAG_A ) {
continue;
}
// flag it
inLink = true;
// get the node in the xml
int32_t xn = words->getNodes()[i];
// is it a self link?
int32_t len;
char *link = xml->getString(xn,"href",&len);
// . set the url class to this
// . TODO: use the base url in the doc
Url u;
u.set( link, len, true, false );
// compare
selfLink = u.equals ( firstUrl );
// skip if not selfLink
if ( ! selfLink ) {
continue;
}
// if it is a selflink , check for an "onClick" tag in the
// anchor tag to fix that Mixx issue for:
// http://www.npr.org/templates/story/story.php?storyId=5417137
int32_t oclen;
char *oc = xml->getString(xn,"onclick",&oclen);
if ( ! oc ) {
oc = xml->getString(xn,"onClick",&oclen);
}
// assume not a self link if we see that...
if ( oc ) {
selfLink = false;
}
// if this <a href> link has a "title" attribute, use that
// instead! that thing is solid gold.
int32_t atlen;
char *atitle = xml->getString(xn,"title",&atlen);
// stop and use that, this thing is gold!
if ( ! atitle || atlen <= 0 ) {
continue;
}
// craziness? ignore it...
if ( atlen > 400 ) {
continue;
}
// if it contains permanent, permalink or share, ignore it!
if ( strncasestr ( atitle, "permalink", atlen ) ||
strncasestr ( atitle,"permanent", atlen) ||
strncasestr ( atitle,"share", atlen) ) {
continue;
}
// do not count the link text as viable
selfLink = false;
// aw, dammit
if ( ti >= MAX_TIT_CANDIDATES ) {
continue;
}
// other dammit
if ( n >= MAX_TIT_CANDIDATES ) {
break;
}
// ok, process it
if ( ! tw[ti].set ( atitle, atlen, true, 0 )) {
return false;
}
// set the bookends, it is the whole thing
cptrs [n] = &tw[ti];
as [n] = 0;
bs [n] = tw[ti].getNumWords();
scores [n] = 3.0; // not ALWAYS solid gold!
types [n] = TT_TITLEATT;
// we are using the words class
ti++;
// advance
n++;
// break out if too many already. save some for below.
if ( n + 20 >= MAX_TIT_CANDIDATES ) {
break;
}
}
//logf(LOG_DEBUG,"title: took2=%" PRId64,gettimeofdayInMilliseconds()-x);
//x = gettimeofdayInMilliseconds();
//int64_t *wids = WW->getWordIds();
// . find the last positive scoring guy
// . do not consider title candidates after "r" if "r" is non-zero
// . FIXES http://larvatusprodeo.net/2009/01/07/partisanship-politics-and-participation/
// the candidate # of the title tag
int32_t tti = -1;
// allow up to 4 tags from each type
char table[512];
// sanity check
if ( getNumXmlNodes() > 512 ) { char *xx=NULL;*xx=0; }
// clear table counts
memset ( table , 0 , 512 );
// the first word
char *wstart = NULL;
if ( NW > 0 ) {
wstart = words->getWord(0);
}
// loop over all "words" in the html body
for ( int32_t i = 0 ; i < NW ; i++ ) {
// come back up here if we encounter another "title-ish" tag
// within our first alleged "title-ish" tag
subloop:
// stop after 30k of text
if ( words->getWord(i) - wstart > 200000 ) {
break; // 1106
}
// get the tag id minus the back tag bit
nodeid_t tid = tids[i] & BACKBITCOMP;
// pen up and pen down for these comment like tags
if ( tid == TAG_SCRIPT || tid == TAG_STYLE ) {
// ignore "titles" in script or style tags
if ( ! (tids[i] & BACKBIT) ) {
continue;
}
}
/// @todo ALC we should allow more tags than just title/link
// skip if not a good tag.
if (tid != TAG_TITLE && tid != TAG_A) {
continue;
}
// must NOT be a back tag
if ( tids[i] & BACKBIT ) {
continue;
}
// skip if we hit our limit
if ( table[tid] >= 4 ) {
continue;
}
// skip over tag/word #i
i++;
// no words in links, unless it is a self link
if ( i < NW && (flags[i] & 0x02) ) {
continue;
}
// the start should be here
int32_t start = -1;
// do not go too far
int32_t max = i + 200;
// find the corresponding back tag for it
for ( ; i < NW && i < max ; i++ ) {
// hey we got it, BUT we got no alnum word first
// so the thing was empty, so loop back to subloop
if ( (tids[i] & BACKBITCOMP) == tid &&
(tids[i] & BACKBIT ) &&
start == -1 ) {
goto subloop;
}
// if we hit another title-ish tag, loop back up
if ( (tids[i] & BACKBITCOMP) == TAG_TITLE || (tids[i] & BACKBITCOMP) == TAG_A ) {
// if no alnum text, restart at the top
if ( start == -1 ) {
goto subloop;
}
// otherwise, break out and see if title works
break;
}
// if we hit a breaking tag...
if ( isBreakingTagId ( tids[i] & BACKBITCOMP ) &&
// do not consider <span> tags breaking for
// our purposes. i saw a <h1><span> setup before.
tids[i] != TAG_SPAN ) {
break;
}
// skip if not alnum word
if ( ! words->isAlnum(i) ) {
continue;
}
// if we hit an alnum word, break out
if ( start == -1 ) {
start = i;
}
}
// if no start was found, must have had a 0 score in there
if ( start == -1 ) {
continue;
}
// if we exhausted the doc, we are done
if ( i >= NW ) {
break;
}
// skip if way too big!
if ( i >= max ) {
continue;
}
// if was too long do not consider a title
if ( i - start > 300 ) {
continue;
}
// . skip if too many bytes
// . this does not include the length of word #i, but #(i-1)
if ( words->getStringSize ( start , i ) > 1000 ) {
continue;
}
// when using pdftohtml, the title tag is the filename when PDF property does not have title tag
if ( tid == TAG_TITLE && contentType == CT_PDF ) {
// skip if title == '/in.[0-9]*'
char* title_start = words->getWord(start);
char* title_end = words->getWord(i);
size_t title_size = title_end - title_start;
const char* result = strnstr( title_start, "/in.", title_size );
if (result != NULL) {
char* endp = NULL;
// do some further verification to avoid screwing up title
if ((strtoll(result + 4, &endp, 10) > 0) && (endp == title_end)) {
continue;
}
}
}
// count it
table[tid]++;
// max it out if we are positive scoring. stop after the
// first positive scoring guy in a section. this might
// hurt the "Hamlet" thing though...
// store a point to the title tag guy. Msg20.cpp needs this
// because the zak's proximity algo uses it in Summary.cpp
// and in Msg20.cpp
// only get the first one! often the 2nd on is in an iframe!! which we now expand into here.
if ( tid == TAG_TITLE && m_titleTagStart == -1 ) {
m_titleTagStart = start;
m_titleTagEnd = i;
// save the candidate # because we always use this
// as the title if we are a root
if ( tti < 0 ) {
tti = n;
}
}
// point to words class of the body that was passed in to us
cptrs[n] = words;
as[n] = start;
bs[n] = i;
if ( tid == TAG_B ) {
types[n] = TT_BOLDTAG;
scores[n] = 1.0;
} else if ( tid == TAG_H1 ) {
types[n] = TT_HTAG;
scores[n] = 1.8;
} else if ( tid == TAG_H2 ) {
types[n] = TT_HTAG;
scores[n] = 1.7;
} else if ( tid == TAG_H3 ) {
types[n] = TT_HTAG;
scores[n] = 1.6;
} else if ( tid == TAG_TITLE ) {
types[n] = TT_TITLETAG;
scores[n] = 3.0;
} else if ( tid == TAG_DIV ) {
types[n] = TT_DIVTAG;
scores[n] = 1.0;
} else if ( tid == TAG_TD ) {
types[n] = TT_TDTAG;
scores[n] = 1.0;
} else if ( tid == TAG_P ) {
types[n] = TT_PTAG;
scores[n] = 1.0;
} else if ( tid == TAG_FONT ) {
types[n] = TT_FONTTAG;
scores[n] = 1.0;
} else if ( tid == TAG_A ) {
types[n] = TT_ATAG;
// . self link is very powerful BUT
// http://www.npr.org/templates/story/story.php?storyId=5417137
// doesn't use it right! so use
// 1.3 instead of 3.0. that has an "onClick" thing in the
// <a> tag, so check for that!
// this was bad for
// http://www.spiritualwoman.net/?cat=191
// so i am demoting from 3.0 to 1.5
scores[n] = 1.5;
}
// count it
n++;
// start loop over at tag #i, for loop does an i++, so negate
// that so this will work
i--;
// break out if too many already. save some for below.
if ( n + 10 >= MAX_TIT_CANDIDATES ) {
break;
}
}
//logf(LOG_DEBUG,"title: took3=%" PRId64,gettimeofdayInMilliseconds()-x);
//x = gettimeofdayInMilliseconds();
// to handle text documents, throw in the first line of text
// as a title candidate, just make the score really low
bool textDoc = (contentType == CT_UNKNOWN || contentType == CT_TEXT);
if (textDoc) {
// make "i" point to first alphabetical word in the document
int32_t i ;
for ( i = 0 ; i < NW && !words->isAlpha(i) ; i++);
// if we got a first alphabetical word, then assume that to be the start of our title
if ( i < NW && n < MAX_TIT_CANDIDATES ) {
// first word in title is "t0"
int32_t t0 = i;
// find end of first line
int32_t numWords = 0;
// set i to the end now. we MUST find a \n to terminate the
// title, otherwise we will not have a valid title
while (i < NW && numWords < maxTitleWords && (words->isAlnum(i) || !words->hasChar(i, '\n'))) {
if(words->isAlnum(i)) {
numWords++;
}
++i;
}
// "t1" is the end
int32_t t1 = -1;
// we must have found our \n in order to set "t1"
if (i <= NW && numWords < maxTitleWords ) {
t1 = i;
}
// set the ptrs
cptrs [n] = words;
// this is the last resort i guess...
scores [n] = 0.5;
types [n] = TT_FIRSTLINE;
as [n] = t0;
bs [n] = t1;
// add it as a candidate if t0 and t1 were valid
if (t0 >= 0 && t1 > t0) {
n++;
}
}
}
//logf(LOG_DEBUG,"title: took4=%" PRId64,gettimeofdayInMilliseconds()-x);
//x = gettimeofdayInMilliseconds();
{
// now add the last url path to contain underscores or hyphens
char *pstart = firstUrl->getPath();
// get first url
Url *fu = firstUrl;
// start at the end
char *p = fu->getUrl() + fu->getUrlLen();
// end pointer
char *pend = NULL;
// come up here for each path component
while ( p >= pstart ) {
// save end
pend = p;
// skip over /
if ( *p == '/' ) {
p--;
}
// now go back to next /
int32_t count = 0;
for ( ; p >= pstart && *p !='/' ; p-- ) {
if ( *p == '_' || *p == '-' ) {
count++;
}
}
// did we get it?
if ( count > 0 ) {
break;
}
}
// did we get any?
if ( p > pstart && n < MAX_TIT_CANDIDATES ) {
// now set words to that
if ( ! tw[ti].set ( p, (pend - p), true, 0 )) {
return false;
}
// point to that
cptrs [n] = &tw[ti];
as [n] = 0;
bs [n] = tw[ti].getNumWords();
scores [n] = 1.0;
types [n] = TT_URLPATH;
// increment since we are using it
ti++;
// advance
n++;
}
}
// save old n
int32_t oldn = n;
// . do not split titles if we are a root url maps.yahoo.com was getting "Maps" for the title
if ( firstUrl->isRoot() ) {
oldn = -2;
}
// point to list of \0 separated titles
const char *rootTitleBuf = NULL;
const char *rootTitleBufEnd = NULL;
// get the root title if we are not root!
if (filteredRootTitleBuf) {
#ifdef _VALGRIND_
VALGRIND_CHECK_MEM_IS_DEFINED(filteredRootTitleBuf,filteredRootTitleBufSize);
#endif
// point to list of \0 separated titles
rootTitleBuf = filteredRootTitleBuf;
rootTitleBufEnd = filteredRootTitleBuf + filteredRootTitleBufSize;
}
{
Matches m;
if ( rootTitleBuf && query ) {
m.setQuery ( query );
}
// convert into an array
int32_t nr = 0;
const char *pr = rootTitleBuf;
const char *rootTitles[20];
int32_t rootTitleLens[20];
// loop over each root title segment
for ( ; pr && pr < rootTitleBufEnd ; pr += strnlen(pr,rootTitleBufEnd-pr) + 1 ) {
// if we had a query...
if ( query ) {
// reset it
m.reset();
// see if root title segment has query terms in it
m.addMatches ( const_cast<char*>(pr), strnlen(pr,rootTitleBufEnd-pr), MF_TITLEGEN, m_niceness );
// if matches query, do NOT add it, we only add it for
// removing from the title of the page...
if ( m.getNumMatches() ) {
continue;
}
}
// point to it. it should start with an alnum already
// since it is the "filtered" list of root titles...
// if not, fix it in xmldoc then.
rootTitles [nr] = pr;
rootTitleLens[nr] = gbstrlen(pr);
// advance
nr++;
// no breaching
if ( nr >= 20 ) break;
}
// now split up candidates in children candidates by tokenizing
// using :, | and - as delimters.
// the hyphen must have a space on at least one side, so "cd-rom" does
// not create a pair of tokens...
// FIX: for the title:
// Best Careers 2009: Librarian - US News and World Report
// we need to recognize "Best Careers 2009: Librarian" as a subtitle
// otherwise we don't get it as the title. so my question is are we
// going to have to do all the permutations at some point? for now
// let's just add in pairs...
for ( int32_t i = 0 ; i < oldn && n + 3 < MAX_TIT_CANDIDATES ; i++ ) {
// stop if no root title segments
if ( nr <= 0 ) break;
// get the word info
Words *w = cptrs[i];
int32_t a = as[i];
int32_t b = bs[i];
// init
int32_t lasta = a;
char prev = false;
// char length in bytes
//int32_t charlen = 1;
// see how many we add
int32_t added = 0;
char *skipTo = NULL;
bool qualified = true;
// . scan the words looking for a token
// . sometimes the candidates end in ": " so put in "k < b-1"
// . made this from k<b-1 to k<b to fix
// "Hot Tub Time Machine (2010) - IMDb" to strip IMDb
for ( int32_t k = a ; k < b && n + 3 < MAX_TIT_CANDIDATES; k++){
// get word
char *wp = w->getWord(k);
// skip if not alnum
if ( ! w->isAlnum(k) ) {
// in order for next alnum word to
// qualify for "clipping" if it matches
// the root title, there has to be more
// than just spaces here, some punct.
// otherwise title
// "T. D. Jakes: Biography from Answers.com"
// becomes
// "T. D. Jakes: Biography from"
qualified=isWordQualified(wp,w->getWordLen(k));
continue;
}
// gotta be qualified!
if ( ! qualified ) continue;
// skip if in root title
if ( skipTo && wp < skipTo ) continue;
// does this match any root page title segments?
int32_t j;
for ( j = 0 ; j < nr ; j++ ) {
// . compare to root title
// . break out if we matched!
if ( ! strncmp( wp, rootTitles[j], rootTitleLens[j] ) ) {
break;
}
}
// if we did not match a root title segment,
// keep on chugging
if ( j >= nr ) continue;
// . we got a root title match!
// . skip over
skipTo = wp + rootTitleLens[j];
// must land on qualified punct then!!
int32_t e = k+1;
for ( ; e<b && w->getWord(e)<skipTo ; e++ );
// ok, word #e must be a qualified punct
if ( e<b &&
! isWordQualified(w->getWord(e),w->getWordLen(e)))
// assume no match then!!
continue;
// if we had a previous guy, reset the end of the
// previous candidate
if ( prev ) {
bs[n-2] = k;
bs[n-1] = k;
}
// . ok, we got two more candidates
// . well, only one more if this is not the 1st time
if ( ! prev ) {
cptrs [n] = cptrs [i];
scores [n] = scores [i];
types [n] = types [i];
as [n] = lasta;
bs [n] = k;
parent [n] = i;
n++;
added++;
}
// the 2nd one
cptrs [n] = cptrs [i];
scores [n] = scores [i];
types [n] = types [i];
as [n] = e + 1;
bs [n] = bs [i];
parent [n] = i;
n++;
added++;
// now add in the last pair as a whole token
cptrs [n] = cptrs [i];
scores [n] = scores [i];
types [n] = types [i];
as [n] = lasta;
bs [n] = bs [i];
parent [n] = i;
n++;
added++;
// nuke the current candidate then since it got
// split up to not contain the root title...
//cptrs[i] = NULL;
// update this
lasta = k+1;
// if we encounter another delimeter we will have to revise bs[n-1], so note that
prev = true;
}
// nuke the current candidate then since it got
// split up to not contain the root title...
if ( added ) {
scores[i] = 0.001;
//cptrs[i] = NULL;
}
// erase the pair if that there was only one token
if ( added == 3 ) n--;
}
}
for ( int32_t i = 0 ; i < n ; i++ ) baseScore[i] = scores[i];
//
// . now punish by 0.85 for every lower case non-stop word it has
// . reward by 1.1 if has a non-stopword in the query
//
for ( int32_t i = 0 ; i < n ; i++ ) {
// point to the words
Words *w = cptrs[i];
// skip if got nuked above
if ( ! w ) {
continue;
}
// the word ptrs
char **wptrs = w->getWordPtrs();
// skip if empty
if ( w->getNumWords() <= 0 ) {
continue;
}
// get the word boundaries
int32_t a = as[i];
int32_t b = bs[i];
// record the boosts
float ncb = 1.0;
float qtb = 1.0;
// a flag
char uncapped = false;
// scan the words in this title candidate
for ( int32_t j = a ; j < b ; j++ ) {
// skip stop words
if ( w->isQueryStopWord( j, langId ) ) {
continue;
}
// punish if uncapitalized non-stopword
if ( ! w->isCapitalized(j) ) {
uncapped = true;
}
// skip if no query
if ( ! query ) {
continue;
}
int64_t wid = w->getWordId(j);
// reward if in the query
if ( query->getWordNum(wid) >= 0 ) {
qtb *= 1.5;
scores[i] *= 1.5;
}
}
// . only punish once if missing a capitalized word hurts us for:
// http://content-uk.cricinfo.com/ausvrsa2008_09/engine/current/match/351682.html
if ( uncapped ) {
ncb *= 1.00;
scores[i] *= 1.00;
}
// punish if a http:// title thingy
char *s = wptrs[a];
int32_t size = w->getStringSize(a,b);
if ( size > 9 && memcmp("http://", s, 7) == 0 ) {
ncb *= .10;
}
if ( size > 14 && memcmp("h\0t\0t\0p\0:\0/\0/", s, 14) == 0 ) {
ncb *= .10;
}
// set these guys
scores[i] *= ncb;
noCapsBoost[i] = ncb;
qtermsBoost[i] = qtb;
}
// . now compare each candidate to the other candidates
// . give a boost if matches
for ( int32_t i = 0 ; i < n ; i++ ) {
// point to the words
Words *w1 = cptrs[i];
// skip if got nuked above
if ( ! w1 ) {
continue;
}
int32_t a1 = as[i];
int32_t b1 = bs[i];
// reset some flags
char localFlag1 = 0;
char localFlag2 = 0;
// record the boost
float iccb = 1.0;
// total boost
float total = 1.0;
// to each other candidate
for ( int32_t j = 0 ; j < n ; j++ ) {
// not to ourselves
if ( j == i ) {
continue;
}
// or our derivatives
if ( parent[j] == i ) {
continue;
}
// or derivates to their parent
if ( parent[i] == j ) {
continue;
}
// only check parents now. do not check kids.
// this was only for when doing percent contained
// not getSimilarity() per se
//if ( parent[j] != -1 ) continue;
// TODO: do not accumulate boosts from a parent
// and its kids, subtitles...
//
// do not compare type X to type Y
if ( types[i] == TT_TITLETAG ) {
if ( types[j] == TT_TITLETAG ) {
continue;
}
}
// do not compare a div candidate to another div cand
// http://friendfeed.com/foxiewire?start=30
// likewise, a TD to another TD
// http://content-uk.cricinfo.com/ausvrsa2008_09/engine/match/351681.html
// ... etc.
if ( types[i] == TT_BOLDTAG ||
types[i] == TT_HTAG ||
types[i] == TT_DIVTAG ||
types[i] == TT_TDTAG ||
types[i] == TT_FONTTAG ) {
if ( types[j] == types[i] ) continue;
}
// . do not compare one kid to another kid
// . i.e. if we got "x | y" as a title and "x | z"
// as a link text, it will emphasize "x" too much
// http://content-uk.cricinfo.com/ausvrsa2008_09/engine/current/match/351682.html
if ( parent[j] != -1 && parent[i] != -1 ) continue;
// . body type tags are mostly mutually exclusive
// . for the legacy.com url mentioned below, we have
// good stuff in <td> tags, so this hurts us...
// . but for the sake of
// http://larvatusprodeo.net/2009/01/07/partisanship-politics-and-participation/
// i put bold tags back
if ( types[i] == TT_LINKTEXTLOCAL ) {
if ( types[j] == TT_LINKTEXTLOCAL ) continue;
}
if ( types[i] == TT_RSSITEMLOCAL ) {
if ( types[j] == TT_RSSITEMLOCAL ) continue;
}
// only compare to one local link text for each i
if ( types[j] == TT_LINKTEXTLOCAL && localFlag1 ) {
continue;
}
if ( types[j] == TT_RSSITEMLOCAL && localFlag2 ) {
continue;
}
if ( types[j] == TT_LINKTEXTLOCAL ) {
localFlag1 = 1;
}
if ( types[j] == TT_RSSITEMLOCAL ) {
localFlag2 = 1;
}
// not link title attr to link title attr either
// fixes http://www.spiritualwoman.net/?cat=191
if ( types[i] == TT_TITLEATT &&
types[j] == TT_TITLEATT )
continue;
// get our words
Words *w2 = cptrs[j];
// skip if got nuked above
if ( ! w2 ) continue;
int32_t a2 = as [j];
int32_t b2 = bs [j];
// how similar is title #i to title #j ?
float fp = getSimilarity ( w2 , a2 , b2 , w1 , a1 , b1 );
// error?
if ( fp == -1.0 ) return false;
// custom boosting...
float boost = 1.0;
if ( fp >= .95 ) boost = 3.0;
else if ( fp >= .90 ) boost = 2.0;
else if ( fp >= .85 ) boost = 1.5;
else if ( fp >= .80 ) boost = 1.4;
else if ( fp >= .75 ) boost = 1.3;
else if ( fp >= .70 ) boost = 1.2;
else if ( fp >= .60 ) boost = 1.1;
else if ( fp >= .50 ) boost = 1.08;
else if ( fp >= .40 ) boost = 1.04;
// limit total
total *= boost;
if ( total > 100.0 ) break;
// if you are matching the url path, that is pretty
// good so give more!
// actually, that would hurt:
// http://michellemalkin.com/2008/12/29/gag-worthy/
// custom boosting!
if ( fp > 0.0 && g_conf.m_logDebugTitle )
logf(LOG_DEBUG,"title: i=%" PRId32" j=%" PRId32" fp=%.02f "
"b=%.02f", i,j,fp,boost);
// apply it
scores[i] *= boost;
iccb *= boost;
}
inCommonCandBoost[i] = iccb;
}
//logf(LOG_DEBUG,"title: took7=%" PRId64,gettimeofdayInMilliseconds()-x);
//x = gettimeofdayInMilliseconds();
// loop over all n candidates
for ( int32_t i = 0 ; i < n ; i++ ) {
// skip if not in the document body
if ( cptrs[i] != words ) continue;
// point to the words
int32_t a1 = as [i];
int32_t b1 = bs [i];
// . loop through this candidates words
// . TODO: use memset here?
for ( int32_t j = a1 ; j <= b1 && j < NW ; j++ ) {
// flag it
flags[j] |= 0x01;
}
}
// free our stuff
if ( flags!=localBuf ) {
mfree (flags, need, "TITLEflags");
}
// now get the highest scoring candidate title
float max = -1.0;
int32_t winner = -1;
for ( int32_t i = 0 ; i < n ; i++ ) {
// skip if got nuked
if ( ! cptrs[i] ) {
continue;
}
if ( winner != -1 && scores[i] <= max ) {
continue;
}
// url path's cannot be titles in and of themselves
if ( types[i] == TT_URLPATH ) {
continue;
}
// skip if empty basically, like if title was exact
// copy of root, then the whole thing got nuked and
// some empty string added, where a > b
if ( as[i] >= bs[i] ) {
continue;
}
// got one
max = scores[i];
// save it
winner = i;
}
// if we are a root, always pick the title tag as the title
if ( oldn == -2 && tti >= 0 ) {
winner = tti;
}
// if no winner, all done. no title
if ( winner == -1 ) {
// last resort use file name
if ((contentType == CT_PDF) && (firstUrl->getFilenameLen() != 0)) {
Words w;
w.set(firstUrl->getFilename(), firstUrl->getFilenameLen(), true);
if (!copyTitle(&w, 0, w.getNumWords())) {
return false;
}
}
return true;
}
// point to the words class of the winner
Words *w = cptrs[winner];
// skip if got nuked above
if ( ! w ) { char *xx=NULL;*xx=0; }
// need to make our own Pos class if title not from body
Pos tp;
if ( w != words ) {
// set "Scores" ptr to NULL. we assume all are positive scores
if ( ! tp.set ( w ) ) {
return false;
}
}
// the string ranges from word #a up to and including word #b
int32_t a = as[winner];
int32_t b = bs[winner];
// sanity check
if ( a < 0 || b > w->getNumWords() ) { char*xx=NULL;*xx=0; }
// save the title
if ( ! copyTitle(w, a, b) ) {
return false;
}
/*
// debug logging
SafeBuf sb;
SafeBuf *pbuf = &sb;
log("title: candidates for %s",xd->getFirstUrl()->getUrl() );
pbuf->safePrintf("<div stype=\"border:1px solid black\">");
pbuf->safePrintf("<b>***Finding Title***</b><br>\n");
pbuf->safePrintf("<table cellpadding=5 border=2><tr>"
"<td colspan=20><center><b>Title Generation</b>"
"</center></td>"
"</tr>\n<tr>"
"<td>#</td>"
"<td>type</td>"
"<td>parent</td>"
"<td>base score</td>"
"<td>format penalty</td>"
"<td>query term boost</td>"
"<td>candidate intersection boost</td>"
"<td>FINAL SCORE</td>"
"<td>title</td>"
"</tr>\n" );
// print out all candidates
for ( int32_t i = 0 ; i < n ; i++ ) {
char *ts = "unknown";
if ( types[i] == TT_LINKTEXTLOCAL ) ts = "local inlink text";
if ( types[i] == TT_LINKTEXTREMOTE ) ts = "remote inlink text";
if ( types[i] == TT_RSSITEMLOCAL ) ts = "local rss title";
if ( types[i] == TT_RSSITEMREMOTE ) ts = "remote rss title";
if ( types[i] == TT_BOLDTAG ) ts = "bold tag";
if ( types[i] == TT_HTAG ) ts = "header tag";
if ( types[i] == TT_TITLETAG ) ts = "title tag";
if ( types[i] == TT_FIRSTLINE ) ts = "first line in text";
if ( types[i] == TT_FONTTAG ) ts = "font tag";
if ( types[i] == TT_ATAG ) ts = "anchor tag";
if ( types[i] == TT_DIVTAG ) ts = "div tag";
if ( types[i] == TT_TDTAG ) ts = "td tag";
if ( types[i] == TT_PTAG ) ts = "p tag";
if ( types[i] == TT_URLPATH ) ts = "url path";
if ( types[i] == TT_TITLEATT ) ts = "title attribute";
// get the title
pbuf->safePrintf(
"<tr>"
"<td>#%" PRId32"</td>"
"<td><nobr>%s</nobr></td>"
"<td>%" PRId32"</td>"
"<td>%0.2f</td>" // baseScore
"<td>%0.2f</td>"
"<td>%0.2f</td>"
"<td>%0.2f</td>"
"<td>%0.2f</td>"
"<td>",
i,
ts ,
parent[i],
baseScore[i],
noCapsBoost[i],
qtermsBoost[i],
inCommonCandBoost[i],
scores[i]);
// ptrs
Words *w = cptrs[i];
int32_t a = as[i];
int32_t b = bs[i];
// skip if no words
if ( w->getNumWords() <= 0 ) continue;
// the word ptrs
char **wptrs = w->getWordPtrs();
// string ptrs
char *ptr = wptrs[a];//w->getWord(a);
int32_t size = w->getStringSize(a,b);
// it is utf8
pbuf->safeMemcpy ( ptr , size );
// end the line
pbuf->safePrintf("</td></tr>\n");
}
pbuf->safePrintf("</table>\n<br>\n");
// log these for now
log("title: %s",sb.getBufStart());
*/
return true;
}
/* ----------------
* printtup --- print a tuple in protocol 3.0
* ----------------
*/
static void
printtup(TupleTableSlot *slot, DestReceiver *self)
{
TupleDesc typeinfo = slot->tts_tupleDescriptor;
DR_printtup *myState = (DR_printtup *) self;
StringInfoData buf;
int natts = typeinfo->natts;
int i;
/* Set or update my derived attribute info, if needed */
if (myState->attrinfo != typeinfo || myState->nattrs != natts)
printtup_prepare_info(myState, typeinfo, natts);
/* Make sure the tuple is fully deconstructed */
slot_getallattrs(slot);
/*
* Prepare a DataRow message
*/
pq_beginmessage(&buf, 'D');
pq_sendint(&buf, natts, 2);
/*
* send the attributes of this tuple
*/
for (i = 0; i < natts; ++i)
{
PrinttupAttrInfo *thisState = myState->myinfo + i;
Datum origattr = slot->tts_values[i],
attr;
if (slot->tts_isnull[i])
{
pq_sendint(&buf, -1, 4);
continue;
}
/*
* If we have a toasted datum, forcibly detoast it here to avoid
* memory leakage inside the type's output routine.
*
* Here we catch undefined bytes in tuples that are returned to the
* client without hitting disk; see comments at the related check in
* PageAddItem(). Whether to test before or after detoast is somewhat
* arbitrary, as is whether to test external/compressed data at all.
* Undefined bytes in the pre-toast datum will have triggered Valgrind
* errors in the compressor or toaster; any error detected here for
* such datums would indicate an (unlikely) bug in a type-independent
* facility. Therefore, this test is most useful for uncompressed,
* non-external datums.
*
* We don't presently bother checking non-varlena datums for undefined
* data. PageAddItem() does check them.
*/
if (thisState->typisvarlena)
{
VALGRIND_CHECK_MEM_IS_DEFINED(origattr, VARSIZE_ANY(origattr));
attr = PointerGetDatum(PG_DETOAST_DATUM(origattr));
}
else
attr = origattr;
if (thisState->format == 0)
{
/* Text output */
char *outputstr;
outputstr = OutputFunctionCall(&thisState->finfo, attr);
pq_sendcountedtext(&buf, outputstr, strlen(outputstr), false);
pfree(outputstr);
}
else
{
/* Binary output */
bytea *outputbytes;
outputbytes = SendFunctionCall(&thisState->finfo, attr);
pq_sendint(&buf, VARSIZE(outputbytes) - VARHDRSZ, 4);
pq_sendbytes(&buf, VARDATA(outputbytes),
VARSIZE(outputbytes) - VARHDRSZ);
pfree(outputbytes);
}
/* Clean up detoasted copy, if any */
if (DatumGetPointer(attr) != DatumGetPointer(origattr))
pfree(DatumGetPointer(attr));
}
pq_endmessage(&buf);
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpyHtoA)(CUarray dstArray, size_t dstOffset, const void *srcHost, size_t ByteCount) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgCtxListType *nodeCtx;
cgArrListType *nodeArrDst;
cgMemListType *nodeMemSrc;
long vgErrorAddress;
int error = 0;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_WWWW(result, fn, dstArray, dstOffset, srcHost, ByteCount);
// Check if actual function parameters are defined
if (VALGRIND_CHECK_MEM_IS_DEFINED(&dstArray, sizeof(CUarray))) {
error++;
VALGRIND_PRINTF("Error: dstArray in call to cuMemcpyAtoD is not defined.\n");
} else if (!dstArray) {
error++;
VALGRIND_PRINTF("Error: dstArray in call to cuMemcpyAtoD is NULL.\n");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&dstOffset, sizeof(size_t))) {
error++;
VALGRIND_PRINTF("Error: dstOffset in call to cuMemcpyAtoD is not defined.\n");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&srcHost, sizeof(CUdeviceptr))) {
error++;
VALGRIND_PRINTF("Error: srcHost in call to cuMemcpyAtoD is not defined.\n");
} else if (!srcHost) {
error++;
VALGRIND_PRINTF("Error: srcDevice in call to cuMemcpyAtoD is NULL");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&ByteCount, sizeof(size_t))) {
error++;
VALGRIND_PRINTF("Error: ByteCount in call to cuMemcpyAtoD is not defined.\n");
}
cgGetCtx(&ctx);
nodeCtx = cgFindCtx(ctx);
nodeArrDst = cgFindArr(nodeCtx, dstArray);
if (srcHost) {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(srcHost, ByteCount);
// Check if memory referenced by srcHost has been allocated.
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: Source host memory in cuMemcpyHtoA is not not allocated.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, vgErrorAddress - (long)srcHost);
} else {
// If allocated, now check if host memory is defined.
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(srcHost, ByteCount);
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: Source host memory in cuMemcpyHtoA is not defined.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, vgErrorAddress - (long)srcHost);
}
}
}
if (nodeArrDst) {
// Check if array is 1-dimensional or big enough in first dimension
if (nodeArrDst->desc.Height > 1 || nodeArrDst->desc.Depth > 1) {
if (nodeArrDst->desc.Width - dstOffset < ByteCount) {
error++;
VALGRIND_PRINTF("Error: Destination array in cuMemcpyAtoD is 2-dimensional\n"
" and ByteCount bigger than available width in first dimension.\n");
} else {
VALGRIND_PRINTF("Warning: Destination array in cuMemcpyAtoD is 2-dimensional.\n");
}
} else if (nodeArrDst->desc.Width - dstOffset < ByteCount) {
// If array is 1D, check size.
VALGRIND_PRINTF("Error: Destination array in cuMemcpyAtoD is too small.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, nodeArrDst->desc.Width - dstOffset);
error++;
}
} else {
error++;
VALGRIND_PRINTF("Error: Destination array not allocated in call to cuMemcpyAtoD.\n");
}
cgUnlock();
return result;
}
* int i;
* Valgrind::CheckDefined(i); // Valgrind complains!
* \endcode
*
* \tparam T The type of the object which ought to be checked
*
* \param value the object which valgrind should check
*
* \return true iff there are no undefined bytes in the memory
* occupied by the object.
*/
template <class T>
inline bool CheckDefined(const T& value OPM_VALGRIND_OPTIM_UNUSED)
{
#if !defined NDEBUG && HAVE_VALGRIND
auto tmp = VALGRIND_CHECK_MEM_IS_DEFINED(&value, sizeof(T));
return tmp == 0;
#else
return true;
#endif
}
/*!
* \ingroup Valgrind
* \brief Make valgrind complain if any of the memory occupied by an object
* is not addressable.
*
* Example:
*
/* Case 1 - mmaped memory is defined */
static void test1()
{
char *m = mm(0, pgsz * 5, PROT_READ);
VALGRIND_CHECK_MEM_IS_DEFINED(m, pgsz*5); /* all defined */
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpy2DAsync)(const CUDA_MEMCPY2D *pCopy, CUstream hStream) {
int error = 0;
long vgErrorAddress;
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(&hStream, sizeof(CUstream));
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: 'hStream' in call to cuMemcpy2DAsync not defined.\n");
}
cgLock();
CUcontext ctx = NULL;
cgGetCtx(&ctx);
// Check if destination (device) memory/array is already being written to.
switch (pCopy->dstMemoryType) {
case CU_MEMORYTYPE_DEVICE: {
cgMemListType *nodeMem;
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->dstDevice);
if (nodeMem) {
// Are we trying to read a memory region that's being written by diffrent stream?
if (nodeMem->locked & 2 && nodeMem->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access by different streams.\n");
}
nodeMem->locked = nodeMem->locked | 1;
nodeMem->stream = hStream;
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
cgArrListType *nodeArr;
nodeArr = cgFindArr(cgFindCtx(ctx), pCopy->dstArray);
if (nodeArr) {
// Are we trying to read an array that's being written by different stream?
if (nodeArr->locked & 2 && nodeArr->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access to array by different streams.\n");
}
nodeArr->locked = nodeArr->locked | 1;
nodeArr->stream = hStream;
}
break;
}
}
// Check if source (device) memory/array is already being written to/read from.
switch (pCopy->srcMemoryType) {
case CU_MEMORYTYPE_DEVICE: {
cgMemListType *nodeMem;
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->srcDevice);
if (nodeMem) {
// Are we trying to read a memory region that's being written by diffrent stream?
if (nodeMem->locked && nodeMem->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access by different streams.\n");
}
nodeMem->locked = nodeMem->locked | 2;
nodeMem->stream = hStream;
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
cgArrListType *nodeArr;
nodeArr = cgFindArr(cgFindCtx(ctx), pCopy->srcArray);
if (nodeArr) {
// Are we trying to read an array that's being written by different stream?
if (nodeArr->locked && nodeArr->stream != hStream) {
error++;
VALGRIND_PRINTF("Error: Concurrent write and read access to array by different streams.\n");
}
nodeArr->locked = nodeArr->locked | 2;
nodeArr->stream = hStream;
}
break;
}
}
cgUnlock();
if (error) {
VALGRIND_PRINTF_BACKTRACE("");
}
return cuMemcpy2D(pCopy);
}
/* ----------------
* printtup --- print a tuple in protocol 3.0
* ----------------
*/
static bool
printtup(TupleTableSlot *slot, DestReceiver *self)
{
TupleDesc typeinfo = slot->tts_tupleDescriptor;
DR_printtup *myState = (DR_printtup *) self;
MemoryContext oldcontext;
StringInfoData buf;
int natts = typeinfo->natts;
int i;
/* Set or update my derived attribute info, if needed */
if (myState->attrinfo != typeinfo || myState->nattrs != natts)
printtup_prepare_info(myState, typeinfo, natts);
/* Make sure the tuple is fully deconstructed */
slot_getallattrs(slot);
/* Switch into per-row context so we can recover memory below */
oldcontext = MemoryContextSwitchTo(myState->tmpcontext);
/*
* Prepare a DataRow message (note buffer is in per-row context)
*/
pq_beginmessage(&buf, 'D');
pq_sendint(&buf, natts, 2);
/*
* send the attributes of this tuple
*/
for (i = 0; i < natts; ++i)
{
PrinttupAttrInfo *thisState = myState->myinfo + i;
Datum attr = slot->tts_values[i];
if (slot->tts_isnull[i])
{
pq_sendint(&buf, -1, 4);
continue;
}
/*
* Here we catch undefined bytes in datums that are returned to the
* client without hitting disk; see comments at the related check in
* PageAddItem(). This test is most useful for uncompressed,
* non-external datums, but we're quite likely to see such here when
* testing new C functions.
*/
if (thisState->typisvarlena)
VALGRIND_CHECK_MEM_IS_DEFINED(DatumGetPointer(attr),
VARSIZE_ANY(attr));
if (thisState->format == 0)
{
/* Text output */
char *outputstr;
outputstr = OutputFunctionCall(&thisState->finfo, attr);
pq_sendcountedtext(&buf, outputstr, strlen(outputstr), false);
}
else
{
/* Binary output */
bytea *outputbytes;
outputbytes = SendFunctionCall(&thisState->finfo, attr);
pq_sendint(&buf, VARSIZE(outputbytes) - VARHDRSZ, 4);
pq_sendbytes(&buf, VARDATA(outputbytes),
VARSIZE(outputbytes) - VARHDRSZ);
}
}
pq_endmessage(&buf);
/* Return to caller's context, and flush row's temporary memory */
MemoryContextSwitchTo(oldcontext);
MemoryContextReset(myState->tmpcontext);
return true;
}
// returns NULL and set g_errno on error
int32_t Msg20Reply::serialize(char *buf, int32_t bufSize) const {
#ifdef _VALGRIND_
VALGRIND_CHECK_MEM_IS_DEFINED(this,sizeof(*this));
if(ptr_htag)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_htag,size_htag);
if(ptr_ubuf)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_ubuf,size_ubuf);
if(ptr_rubuf)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_rubuf,size_rubuf);
if(ptr_displaySum)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_displaySum,size_displaySum);
if(ptr_dbuf)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_dbuf,size_dbuf);
if(ptr_vbuf)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_vbuf,size_vbuf);
if(ptr_imgData)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_imgData,size_imgData);
if(ptr_site)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_site,size_site);
if(ptr_linkInfo)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_linkInfo,size_linkInfo);
if(ptr_outlinks)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_outlinks,size_outlinks);
if(ptr_vector1)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_vector1,size_vector1);
if(ptr_vector2)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_vector2,size_vector2);
if(ptr_vector3)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_vector3,size_vector3);
if(ptr_linkText)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_linkText,size_linkText);
if(ptr_surroundingText)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_surroundingText,size_surroundingText);
if(ptr_linkUrl)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_linkUrl,size_linkUrl);
if(ptr_rssItem)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_rssItem,size_rssItem);
if(ptr_categories)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_categories,size_categories);
if(ptr_content)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_content,size_content);
if(ptr_templateVector)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_templateVector,size_templateVector);
if(ptr_metadataBuf)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_metadataBuf,size_metadataBuf);
if(ptr_note)
VALGRIND_CHECK_MEM_IS_DEFINED(ptr_note,size_note);
#endif
int32_t retSize;
serializeMsg(sizeof(*this),
&size_tbuf, &size_note,
&ptr_tbuf,
this,
&retSize,
buf, bufSize);
if ( retSize > bufSize ) { g_process.shutdownAbort(true); }
// return it
return retSize;
}
/*
* pmemobj_vg_check_no_undef -- (internal) check whether there are any undefined
* regions
*/
static void
pmemobj_vg_check_no_undef(struct pmemobjpool *pop)
{
LOG(4, "pop %p", pop);
struct {
void *start, *end;
} undefs[MAX_UNDEFS];
int num_undefs = 0;
VALGRIND_DO_DISABLE_ERROR_REPORTING;
char *addr_start = pop->addr;
char *addr_end = addr_start + pop->size;
while (addr_start < addr_end) {
char *noaccess = (char *)VALGRIND_CHECK_MEM_IS_ADDRESSABLE(
addr_start, addr_end - addr_start);
if (noaccess == NULL)
noaccess = addr_end;
while (addr_start < noaccess) {
char *undefined =
(char *)VALGRIND_CHECK_MEM_IS_DEFINED(
addr_start, noaccess - addr_start);
if (undefined) {
addr_start = undefined;
#ifdef VALGRIND_CHECK_MEM_IS_UNDEFINED
addr_start = (char *)
VALGRIND_CHECK_MEM_IS_UNDEFINED(
addr_start, noaccess - addr_start);
if (addr_start == NULL)
addr_start = noaccess;
#else
while (addr_start < noaccess &&
VALGRIND_CHECK_MEM_IS_DEFINED(
addr_start, 1))
addr_start++;
#endif
if (num_undefs < MAX_UNDEFS) {
undefs[num_undefs].start = undefined;
undefs[num_undefs].end = addr_start - 1;
num_undefs++;
}
} else
addr_start = noaccess;
}
#ifdef VALGRIND_CHECK_MEM_IS_UNADDRESSABLE
addr_start = (char *)VALGRIND_CHECK_MEM_IS_UNADDRESSABLE(
addr_start, addr_end - addr_start);
if (addr_start == NULL)
addr_start = addr_end;
#else
while (addr_start < addr_end &&
(char *)VALGRIND_CHECK_MEM_IS_ADDRESSABLE(
addr_start, 1) == addr_start)
addr_start++;
#endif
}
VALGRIND_DO_ENABLE_ERROR_REPORTING;
if (num_undefs) {
/*
* How to resolve this error:
* If it's part of the free space Valgrind should be told about
* it by VALGRIND_DO_MAKE_MEM_NOACCESS request. If it's
* allocated - initialize it or use VALGRIND_DO_MAKE_MEM_DEFINED
* request.
*/
VALGRIND_PRINTF("Part of the pool is left in undefined state on"
" boot. This is pmemobj's bug.\nUndefined"
" regions:\n");
for (int i = 0; i < num_undefs; ++i)
VALGRIND_PRINTF(" [%p, %p]\n", undefs[i].start,
undefs[i].end);
if (num_undefs == MAX_UNDEFS)
VALGRIND_PRINTF(" ...\n");
/* Trigger error. */
VALGRIND_CHECK_MEM_IS_DEFINED(undefs[0].start, 1);
}
}
// Copy Host->Device
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpyHtoD)(CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgCtxListType *nodeCtx;
cgMemListType *nodeMem;
size_t dstSize;
long vgErrorAddress, vgErrorAddressDstDevice, vgErrorAddressSrcHost;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
vgErrorAddressDstDevice = VALGRIND_CHECK_MEM_IS_DEFINED(&dstDevice, sizeof(void*));
vgErrorAddressSrcHost = VALGRIND_CHECK_MEM_IS_DEFINED(&srcHost, sizeof(CUdeviceptr));
// TODO: Currently errors are exclusive .. i.e. with undefined src and NULL
// dst pointer, only the undefined pointer is reported.
if (vgErrorAddressDstDevice || vgErrorAddressSrcHost) {
VALGRIND_PRINTF("Error:");
if (vgErrorAddressDstDevice) {
VALGRIND_PRINTF(" destination device");
if (vgErrorAddressSrcHost) {
VALGRIND_PRINTF(" and");
}
}
if (vgErrorAddressSrcHost) {
VALGRIND_PRINTF(" source host");
}
VALGRIND_PRINTF_BACKTRACE(" pointer in cuMemcpyHtoD not defined.\n");
} else if (dstDevice != 0 && srcHost != NULL) {
cgGetCtx(&ctx);
// Check allocation status and available size on device
nodeCtx = cgFindCtx(ctx);
nodeMem = cgFindMem(nodeCtx, dstDevice);
if (!nodeMem) {
VALGRIND_PRINTF("Error: Device memory during host->device memory copy is not allocated.");
} else {
dstSize = nodeMem->size - (dstDevice - nodeMem->dptr);
if (dstSize < ByteCount) {
VALGRIND_PRINTF("Error: Allocated device memory too small for host->device memory copy.\n");
VALGRIND_PRINTF(" Expected %lu allocated bytes but only found %lu.", ByteCount, dstSize);
}
}
if (!nodeMem || dstSize < ByteCount) {
VALGRIND_PRINTF_BACKTRACE("\n");
}
// Check allocation and definedness for host memory
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(srcHost, ByteCount);
if (vgErrorAddress) {
VALGRIND_PRINTF("Error: Host memory during host->device memory copy is not allocated.\n");
VALGRIND_PRINTF(" Expected %lu allocated bytes but only found %lu.", ByteCount, vgErrorAddress - (long)srcHost);
} else {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(srcHost, ByteCount);
if (vgErrorAddress) {
VALGRIND_PRINTF("Error: Host memory during host->device memory copy is not defined.\n");
VALGRIND_PRINTF(" Expected %lu defined bytes but only found %lu.", ByteCount, vgErrorAddress - (long)srcHost);
}
}
if (vgErrorAddress) {
VALGRIND_PRINTF_BACKTRACE("\n");
}
} else {
VALGRIND_PRINTF("Error: cuMemcpyHtoD called with NULL");
if (dstDevice == 0) {
VALGRIND_PRINTF(" device");
if (srcHost == NULL) VALGRIND_PRINTF(" and");
}
if (srcHost == NULL) {
VALGRIND_PRINTF(" host");
}
VALGRIND_PRINTF_BACKTRACE(" pointer.\n");
}
CALL_FN_W_WWW(result, fn, dstDevice, srcHost, ByteCount);
cgUnlock();
return result;
}
