void setDataFromHex(byte *out, int len, const char *hex) {
#ifdef HAVE_VALGRIND_MEMCHECK_H
VALGRIND_CHECK_MEM_IS_ADDRESSABLE(out, len);
#endif
bool odd = false;
unsigned int last = 0;
while (len > 0 && *hex != '\0') {
byte nibble = *hex++;
if (nibble >= '0' && nibble <= '9')
nibble -= '0';
else if (nibble >= 'A' && nibble <= 'F')
nibble -= 'A' - 10;
else if (nibble >= 'a' && nibble <= 'f')
nibble -= 'a' - 10;
else
nibble = 0;
last |= (unsigned int)nibble;
if (odd) {
*out++ = (byte)last;
--len;
last = 0;
odd = false;
} else {
last <<= 4;
odd = true;
}
}
}
/*
* pmemobj_vg_check_no_undef -- (internal) check whether there are any undefined
* regions
*/
static void
pmemobj_vg_check_no_undef(struct pmemobjpool *pop)
{
LOG(4, "pop %p", pop);
struct {
void *start, *end;
} undefs[MAX_UNDEFS];
int num_undefs = 0;
VALGRIND_DO_DISABLE_ERROR_REPORTING;
char *addr_start = pop->addr;
char *addr_end = addr_start + pop->size;
while (addr_start < addr_end) {
char *noaccess = (char *)VALGRIND_CHECK_MEM_IS_ADDRESSABLE(
addr_start, addr_end - addr_start);
if (noaccess == NULL)
noaccess = addr_end;
while (addr_start < noaccess) {
char *undefined =
(char *)VALGRIND_CHECK_MEM_IS_DEFINED(
addr_start, noaccess - addr_start);
if (undefined) {
addr_start = undefined;
#ifdef VALGRIND_CHECK_MEM_IS_UNDEFINED
addr_start = (char *)
VALGRIND_CHECK_MEM_IS_UNDEFINED(
addr_start, noaccess - addr_start);
if (addr_start == NULL)
addr_start = noaccess;
#else
while (addr_start < noaccess &&
VALGRIND_CHECK_MEM_IS_DEFINED(
addr_start, 1))
addr_start++;
#endif
if (num_undefs < MAX_UNDEFS) {
undefs[num_undefs].start = undefined;
undefs[num_undefs].end = addr_start - 1;
num_undefs++;
}
} else
addr_start = noaccess;
}
#ifdef VALGRIND_CHECK_MEM_IS_UNADDRESSABLE
addr_start = (char *)VALGRIND_CHECK_MEM_IS_UNADDRESSABLE(
addr_start, addr_end - addr_start);
if (addr_start == NULL)
addr_start = addr_end;
#else
while (addr_start < addr_end &&
(char *)VALGRIND_CHECK_MEM_IS_ADDRESSABLE(
addr_start, 1) == addr_start)
addr_start++;
#endif
}
VALGRIND_DO_ENABLE_ERROR_REPORTING;
if (num_undefs) {
/*
* How to resolve this error:
* If it's part of the free space Valgrind should be told about
* it by VALGRIND_DO_MAKE_MEM_NOACCESS request. If it's
* allocated - initialize it or use VALGRIND_DO_MAKE_MEM_DEFINED
* request.
*/
VALGRIND_PRINTF("Part of the pool is left in undefined state on"
" boot. This is pmemobj's bug.\nUndefined"
" regions:\n");
for (int i = 0; i < num_undefs; ++i)
VALGRIND_PRINTF(" [%p, %p]\n", undefs[i].start,
undefs[i].end);
if (num_undefs == MAX_UNDEFS)
VALGRIND_PRINTF(" ...\n");
/* Trigger error. */
VALGRIND_CHECK_MEM_IS_DEFINED(undefs[0].start, 1);
}
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpy3D)(const CUDA_MEMCPY3D *pCopy) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
int error = 0, error_addressable, error_defined;
long vgErrorAddress = 0, vgErrorAddressDefined = 0;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_W(result, fn, pCopy);
// Check if pCopy is null, not allocated or undefined.
// For obvious reasons we skip the following checks if either condition is true.
if (!pCopy) {
error++;
VALGRIND_PRINTF_BACKTRACE("Error: pCopy in call to cuMemcpy3D is NULL.\n");
cgUnlock();
return result;
} else if ( vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(pCopy, sizeof(CUDA_MEMCPY3D)) ) {
error++;
VALGRIND_PRINTF_BACKTRACE("Error: pCopy in call to cuMemcpy3D points to unallocated memory.\n");
cgUnlock();
return result;
} // It makes no sense to check _IS_DEFINED on the whole structure, since only part of it is used!
// General checks of constaints imposed by reference manual
if (pCopy->srcMemoryType != CU_MEMORYTYPE_ARRAY) {
if (pCopy->srcPitch && pCopy->srcPitch < pCopy->WidthInBytes + pCopy->srcXInBytes) {
error++;
VALGRIND_PRINTF("Error: srcPitch < WidthInBytes+srcXInBytes in cuMemcpy3D.\n");
}
if (pCopy->srcHeight && pCopy->srcHeight < pCopy->Height + pCopy->srcY) {
error++;
VALGRIND_PRINTF("Error: srcHeight < Height+srcY in cuMemcpy3D.\n");
}
}
if (pCopy->dstMemoryType != CU_MEMORYTYPE_ARRAY) {
if (pCopy->dstPitch && pCopy->dstPitch < pCopy->WidthInBytes + pCopy->dstXInBytes) {
error++;
VALGRIND_PRINTF("Error: dstPitch < WidthInBytes+dstXInBytes in cuMemcpy3D.\n");
}
if (pCopy->dstHeight && pCopy->dstHeight < pCopy->Height + pCopy->dstY) {
error++;
VALGRIND_PRINTF("Error: dstHeight < Height+dstY in cuMemcpy3D.\n");
}
}
switch (pCopy->srcMemoryType) {
case CU_MEMORYTYPE_UNIFIED:
// TODO: How do we handle unified memory?
break;
case CU_MEMORYTYPE_HOST: {
void *line;
error_addressable = 0;
error_defined = 0;
// TODO: Is Height, Depth > 1, even for 1D/2D copy operations?
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (void*)(
(char*)pCopy->srcHost
+ ((pCopy->srcZ + j) * pCopy->srcHeight + (pCopy->srcY + i))*pCopy->srcPitch
+ pCopy->srcXInBytes
);
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_addressable++;
} else {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_defined++;
}
}
}
}
// TODO: Can we give precise information about location of error?
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) source host memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
if (error_defined) {
error++;
VALGRIND_PRINTF("Error: (Part of) source host memory not defined\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_DEVICE: {
// ptrEnd points to the end of the memory area which pCopy->srcDevice points into
CUdeviceptr line, ptrEnd;
cgMemListType *nodeMem;
// TODO: Check if pCopy->srcDevice is defined?
cgGetCtx(&ctx);
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->srcDevice);
// We only track addressable status (whether memory is allocated) for device memory regions
error_addressable = 0;
if (nodeMem) {
ptrEnd = nodeMem->dptr + nodeMem->size;
/*
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (CUdeviceptr)(
pCopy->srcDevice
+ ((pCopy->srcZ + j) * pCopy->srcHeight + (pCopy->srcY + i)) * pCopy->srcPitch
+ pCopy->srcXInBytes
);
// Is there enough allocated memory left to statisfy the current line?
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
}
}
*/
// Device memory should not be fragmented, so we only check the very last slice of memory
line = (CUdeviceptr)(
pCopy->srcDevice
+ (
(pCopy->srcZ + pCopy->Depth - 1) * pCopy->srcHeight
+ (pCopy->srcY + pCopy->Height - 1)
) * pCopy->srcPitch
+ pCopy->srcXInBytes);
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
} else {
error_addressable++;
}
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) source device memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
CUDA_ARRAY3D_DESCRIPTOR descriptor;
int bytesPerElement;
int widthInBytes;
// Fetch array descriptor ..
cuArray3DGetDescriptor(&descriptor, pCopy->srcArray);
bytesPerElement = cgArrDescBytesPerElement(&descriptor);
if (!bytesPerElement) {
error++;
VALGRIND_PRINTF("Error: Unknown Format value in src array descriptor in cuMemcpy3D.\n");
}
widthInBytes = bytesPerElement * descriptor.Width;
// .. and check if dimensions are conform to the ones requested in pCopy
if (widthInBytes - pCopy->srcXInBytes < pCopy->WidthInBytes) {
error++;
VALGRIND_PRINTF("Error: Available width of %u bytes in source array is smaller than\n"
" requested Width of %u bytes in pCopy of cuMemcpy3D.\n",
widthInBytes - pCopy->srcXInBytes, pCopy->WidthInBytes);
}
if (pCopy->Height > 1 && descriptor.Height - pCopy->srcY < pCopy->Height) {
error++;
VALGRIND_PRINTF("Error: Available Height of %u in source array is smaller than\n"
" requested Height of %u in pCopy of cuMemcpy3D.\n",
descriptor.Height - pCopy->srcY, pCopy->Height);
}
if (pCopy->Depth > 1 && descriptor.Depth - pCopy->srcZ < pCopy->Depth) {
error++;
VALGRIND_PRINTF("Error: Available Depth of %u in source array is smaller than\n"
" requested Depth of %u in pCopy of cuMemcpy3D.\n",
descriptor.Depth - pCopy->srcY, pCopy->Height);
}
break;
}
default:
error++;
VALGRIND_PRINTF("Error: Unknown source memory type %d in cuMemcpy3D\n");
break;
}
switch (pCopy->dstMemoryType) {
case CU_MEMORYTYPE_UNIFIED:
// TODO: How do we handle unified memory?
break;
case CU_MEMORYTYPE_HOST: {
void *line;
error_addressable = 0;
error_defined = 0;
// TODO: Is Height, Depth > 1, even for 1D/2D copy operations?
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (void*)(
(char*)pCopy->dstHost
+ ((pCopy->dstZ + j) * pCopy->dstHeight + (pCopy->dstY + i))*pCopy->dstPitch
+ pCopy->dstXInBytes
);
// Unlike for the source operand we only need to check allocation status here
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(line, (size_t)pCopy->WidthInBytes);
if (vgErrorAddress) {
error_addressable++;
}
}
}
// TODO: Can we give precise information about location of error?
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) destination host memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_DEVICE: {
// ptrEnd points to the end of the memory area which pCopy->dstDevice points into
CUdeviceptr line, ptrEnd;
cgMemListType *nodeMem;
// TODO: Check if pCopy->dstDevice is defined?
cgGetCtx(&ctx);
nodeMem = cgFindMem(cgFindCtx(ctx), pCopy->dstDevice);
// We only track addressable status (whether memory is allocated) for device memory regions
error_addressable = 0;
if (nodeMem) {
ptrEnd = nodeMem->dptr + nodeMem->size;
/*
for (int i = 0 ; i < pCopy->Height ; i++) {
for (int j = 0 ; j < pCopy->Depth ; j++) {
line = (CUdeviceptr)(
pCopy->dstDevice
+ ((pCopy->dstZ + j) * pCopy->dstHeight + (pCopy->dstY + i)) * pCopy->dstPitch
+ pCopy->dstXInBytes
);
// Is there enough allocated memory left to statisfy the current line?
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
}
}
*/
// Device memory should not be fragmented, so we only check the very last slice of memory
line = (CUdeviceptr)(
pCopy->dstDevice
+ (
(pCopy->dstZ + pCopy->Depth - 1) * pCopy->dstHeight
+ (pCopy->dstY + pCopy->Height - 1)
) * pCopy->dstPitch
+ pCopy->dstXInBytes);
if (ptrEnd - line < pCopy->WidthInBytes) {
error_addressable++;
}
} else {
error_addressable++;
}
if (error_addressable) {
error++;
VALGRIND_PRINTF("Error: (Part of) destination device memory not allocated\n"
" in call to cuMemcpy3D.\n");
}
break;
}
case CU_MEMORYTYPE_ARRAY: {
CUDA_ARRAY3D_DESCRIPTOR descriptor;
int bytesPerElement;
int widthInBytes;
// Fetch array descriptor ..
cuArray3DGetDescriptor(&descriptor, pCopy->dstArray);
bytesPerElement = cgArrDescBytesPerElement(&descriptor);
if (!bytesPerElement) {
error++;
VALGRIND_PRINTF("Error: Unknown Format value in dst array descriptor in cuMemcpy3D.\n");
}
widthInBytes = bytesPerElement * descriptor.Width;
// .. and check if dimensions are conform to the ones requested in pCopy
if (widthInBytes - pCopy->dstXInBytes < pCopy->WidthInBytes) {
error++;
VALGRIND_PRINTF("Error: Available width of %u bytes in destination array is smaller than\n"
" requested Width of %u bytes in pCopy of cuMemcpy3D.\n",
widthInBytes - pCopy->dstXInBytes, pCopy->WidthInBytes);
}
if (pCopy->Height > 1 && descriptor.Height - pCopy->dstY < pCopy->Height) {
error++;
VALGRIND_PRINTF("Error: Available Height of %u in destination array is smaller than\n"
" requested Height of %u in pCopy of cuMemcpy3D.\n",
descriptor.Height - pCopy->dstY, pCopy->Height);
}
if (pCopy->Depth > 1 && descriptor.Depth - pCopy->dstZ < pCopy->Depth) {
error++;
VALGRIND_PRINTF("Error: Available Depth of %u in destination array is smaller than\n"
" requested Depth of %u in pCopy of cuMemcpy3D.\n",
descriptor.Depth - pCopy->dstZ, pCopy->Depth);
}
break;
}
default:
error++;
VALGRIND_PRINTF("Error: Unknown destination memory type %d in cuMemcpy3D\n");
break;
}
if (error) {
VALGRIND_PRINTF_BACKTRACE(" %d errors detected in call to cuMemcpy3D.", error);
}
cgUnlock();
return result;
}
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpyHtoA)(CUarray dstArray, size_t dstOffset, const void *srcHost, size_t ByteCount) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgCtxListType *nodeCtx;
cgArrListType *nodeArrDst;
cgMemListType *nodeMemSrc;
long vgErrorAddress;
int error = 0;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
CALL_FN_W_WWWW(result, fn, dstArray, dstOffset, srcHost, ByteCount);
// Check if actual function parameters are defined
if (VALGRIND_CHECK_MEM_IS_DEFINED(&dstArray, sizeof(CUarray))) {
error++;
VALGRIND_PRINTF("Error: dstArray in call to cuMemcpyAtoD is not defined.\n");
} else if (!dstArray) {
error++;
VALGRIND_PRINTF("Error: dstArray in call to cuMemcpyAtoD is NULL.\n");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&dstOffset, sizeof(size_t))) {
error++;
VALGRIND_PRINTF("Error: dstOffset in call to cuMemcpyAtoD is not defined.\n");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&srcHost, sizeof(CUdeviceptr))) {
error++;
VALGRIND_PRINTF("Error: srcHost in call to cuMemcpyAtoD is not defined.\n");
} else if (!srcHost) {
error++;
VALGRIND_PRINTF("Error: srcDevice in call to cuMemcpyAtoD is NULL");
}
if (VALGRIND_CHECK_MEM_IS_DEFINED(&ByteCount, sizeof(size_t))) {
error++;
VALGRIND_PRINTF("Error: ByteCount in call to cuMemcpyAtoD is not defined.\n");
}
cgGetCtx(&ctx);
nodeCtx = cgFindCtx(ctx);
nodeArrDst = cgFindArr(nodeCtx, dstArray);
if (srcHost) {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(srcHost, ByteCount);
// Check if memory referenced by srcHost has been allocated.
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: Source host memory in cuMemcpyHtoA is not not allocated.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, vgErrorAddress - (long)srcHost);
} else {
// If allocated, now check if host memory is defined.
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(srcHost, ByteCount);
if (vgErrorAddress) {
error++;
VALGRIND_PRINTF("Error: Source host memory in cuMemcpyHtoA is not defined.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, vgErrorAddress - (long)srcHost);
}
}
}
if (nodeArrDst) {
// Check if array is 1-dimensional or big enough in first dimension
if (nodeArrDst->desc.Height > 1 || nodeArrDst->desc.Depth > 1) {
if (nodeArrDst->desc.Width - dstOffset < ByteCount) {
error++;
VALGRIND_PRINTF("Error: Destination array in cuMemcpyAtoD is 2-dimensional\n"
" and ByteCount bigger than available width in first dimension.\n");
} else {
VALGRIND_PRINTF("Warning: Destination array in cuMemcpyAtoD is 2-dimensional.\n");
}
} else if (nodeArrDst->desc.Width - dstOffset < ByteCount) {
// If array is 1D, check size.
VALGRIND_PRINTF("Error: Destination array in cuMemcpyAtoD is too small.\n"
" Expected %l bytes but only found %l.\n",
ByteCount, nodeArrDst->desc.Width - dstOffset);
error++;
}
} else {
error++;
VALGRIND_PRINTF("Error: Destination array not allocated in call to cuMemcpyAtoD.\n");
}
cgUnlock();
return result;
}
// Copy Host->Device
CUresult I_WRAP_SONAME_FNNAME_ZZ(libcudaZdsoZa, cuMemcpyHtoD)(CUdeviceptr dstDevice, const void *srcHost, size_t ByteCount) {
OrigFn fn;
CUresult result;
CUcontext ctx = NULL;
cgCtxListType *nodeCtx;
cgMemListType *nodeMem;
size_t dstSize;
long vgErrorAddress, vgErrorAddressDstDevice, vgErrorAddressSrcHost;
VALGRIND_GET_ORIG_FN(fn);
cgLock();
vgErrorAddressDstDevice = VALGRIND_CHECK_MEM_IS_DEFINED(&dstDevice, sizeof(void*));
vgErrorAddressSrcHost = VALGRIND_CHECK_MEM_IS_DEFINED(&srcHost, sizeof(CUdeviceptr));
// TODO: Currently errors are exclusive .. i.e. with undefined src and NULL
// dst pointer, only the undefined pointer is reported.
if (vgErrorAddressDstDevice || vgErrorAddressSrcHost) {
VALGRIND_PRINTF("Error:");
if (vgErrorAddressDstDevice) {
VALGRIND_PRINTF(" destination device");
if (vgErrorAddressSrcHost) {
VALGRIND_PRINTF(" and");
}
}
if (vgErrorAddressSrcHost) {
VALGRIND_PRINTF(" source host");
}
VALGRIND_PRINTF_BACKTRACE(" pointer in cuMemcpyHtoD not defined.\n");
} else if (dstDevice != 0 && srcHost != NULL) {
cgGetCtx(&ctx);
// Check allocation status and available size on device
nodeCtx = cgFindCtx(ctx);
nodeMem = cgFindMem(nodeCtx, dstDevice);
if (!nodeMem) {
VALGRIND_PRINTF("Error: Device memory during host->device memory copy is not allocated.");
} else {
dstSize = nodeMem->size - (dstDevice - nodeMem->dptr);
if (dstSize < ByteCount) {
VALGRIND_PRINTF("Error: Allocated device memory too small for host->device memory copy.\n");
VALGRIND_PRINTF(" Expected %lu allocated bytes but only found %lu.", ByteCount, dstSize);
}
}
if (!nodeMem || dstSize < ByteCount) {
VALGRIND_PRINTF_BACKTRACE("\n");
}
// Check allocation and definedness for host memory
vgErrorAddress = VALGRIND_CHECK_MEM_IS_ADDRESSABLE(srcHost, ByteCount);
if (vgErrorAddress) {
VALGRIND_PRINTF("Error: Host memory during host->device memory copy is not allocated.\n");
VALGRIND_PRINTF(" Expected %lu allocated bytes but only found %lu.", ByteCount, vgErrorAddress - (long)srcHost);
} else {
vgErrorAddress = VALGRIND_CHECK_MEM_IS_DEFINED(srcHost, ByteCount);
if (vgErrorAddress) {
VALGRIND_PRINTF("Error: Host memory during host->device memory copy is not defined.\n");
VALGRIND_PRINTF(" Expected %lu defined bytes but only found %lu.", ByteCount, vgErrorAddress - (long)srcHost);
}
}
if (vgErrorAddress) {
VALGRIND_PRINTF_BACKTRACE("\n");
}
} else {
VALGRIND_PRINTF("Error: cuMemcpyHtoD called with NULL");
if (dstDevice == 0) {
VALGRIND_PRINTF(" device");
if (srcHost == NULL) VALGRIND_PRINTF(" and");
}
if (srcHost == NULL) {
VALGRIND_PRINTF(" host");
}
VALGRIND_PRINTF_BACKTRACE(" pointer.\n");
}
CALL_FN_W_WWW(result, fn, dstDevice, srcHost, ByteCount);
cgUnlock();
return result;
}
