// Store a blob. Make sure it doesn't show up until/unless the repo tx is committed.
void MyFn(one_blob)(SG_context * pCtx, SG_repo* pRepo)
{
SG_byte* pBufIn = NULL;
SG_uint32 lenBufIn = 0;
SG_byte* pBufOut = NULL;
SG_uint64 lenBufOut = 0;
SG_repo_tx_handle* pTx;
char* pszHidReturned = NULL;
VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pBufIn, &lenBufIn) );
// Start writing blob.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) );
// Should fail: tx not committed.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut),
SG_ERR_BLOB_NOT_FOUND ); // Blob visible before repo tx committed
SG_NULLFREE(pCtx, pBufOut);
// Abort repo tx.
VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__abort_tx should null/free the repo transaction.", !pTx);
// Should fail: tx aborted.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut),
SG_ERR_BLOB_NOT_FOUND ); // Blob exists after repo tx abort
SG_NULLFREE(pCtx, pBufOut);
SG_NULLFREE(pCtx, pszHidReturned);
// Write blob, commit tx.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) );
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__commit_tx should null/free the repo transaction.", !pTx);
// Read back the blob. It should exist now.
VERIFY_ERR_CHECK( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut) );
// Just verify the length. It's another test's job to roundtrip blobs and verify data.
VERIFY_COND( "blob length mismatch", lenBufOut == lenBufIn );
// Fall through to common cleanup.
fail:
SG_NULLFREE(pCtx, pszHidReturned);
SG_NULLFREE(pCtx, pBufIn);
SG_NULLFREE(pCtx, pBufOut);
}
void MyFn(create_zero_byte_blob)(SG_context* pCtx,
SG_repo* pRepo)
{
struct z
{
const char * pszHashMethod;
const char * pszTrivialHash;
};
struct z az[] = { { "SHA1/160", "da39a3ee5e6b4b0d3255bfef95601890afd80709" },
{ "SHA2/256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" },
{ "SHA2/384", "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b" },
{ "SHA2/512", "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" },
};
char* pszidHidBlob1 = NULL;
char * pbuf1 = NULL;
char * pbuf2 = NULL;
SG_uint64 lenBuf2;
SG_repo_tx_handle* pTx = NULL;
SG_uint32 lenBuf1 = 0;
char * pszHashMethod = NULL;
SG_uint32 k, kLimit;
pbuf1 = (char *)SG_calloc(1,lenBuf1+1);
VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
INFOP("create_zero_byte_blob",("Created blob [%s]",(pszidHidBlob1)));
//////////////////////////////////////////////////////////////////
// fetch blob into a new buffer and verify that it matches.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) );
VERIFY_COND("create_zero_byte_blob(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1));
VERIFY_COND("create_zero_byte_blob(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0));
VERIFY_ERR_CHECK_DISCARD( SG_repo__get_hash_method(pCtx, pRepo, &pszHashMethod) );
kLimit = SG_NrElements(az);
for (k=0; k<kLimit; k++)
{
if (strcmp(pszHashMethod,az[k].pszHashMethod) == 0)
{
// The empty blob should always have this hid
VERIFY_COND("zero byte blob hid mismatch",
strcmp(pszidHidBlob1, az[k].pszTrivialHash) == 0);
}
}
//////////////////////////////////////////////////////////////////
// cleanup
SG_NULLFREE(pCtx, pbuf1);
SG_NULLFREE(pCtx, pbuf2);
SG_NULLFREE(pCtx, pszidHidBlob1);
SG_NULLFREE(pCtx, pszHashMethod);
}
void SG_dbrecord__save_to_repo(SG_context* pCtx, SG_dbrecord * pRecord, SG_repo * pRepo, SG_repo_tx_handle* pRepoTx, SG_uint64* iBlobFullLength)
{
SG_string * pString = NULL;
char* pszHidComputed = NULL;
SG_uint32 iLength = 0;
SG_NULLARGCHECK_RETURN(pRepo);
SG_NULLARGCHECK_RETURN(pRecord);
// serialize the dbrecord into JSON string.
SG_ERR_CHECK( SG_STRING__ALLOC(pCtx, &pString) );
SG_ERR_CHECK( SG_dbrecord__to_json(pCtx, pRecord,pString) );
#if 0
printf("%s\n", SG_string__sz(pString)); // TODO remove this
#endif
// remember the length of the blob
iLength = SG_string__length_in_bytes(pString);
*iBlobFullLength = (SG_uint64) iLength;
// create a blob in the repository using the JSON string. this computes the HID and returns it.
SG_repo__store_blob_from_memory(pCtx,
pRepo,pRepoTx,
NULL, // TODO pass the record id into here
SG_FALSE,
(const SG_byte *)SG_string__sz(pString),
iLength,
&pszHidComputed);
if (!SG_context__has_err(pCtx) || SG_context__err_equals(pCtx, SG_ERR_BLOBFILEALREADYEXISTS))
{
// freeeze the dbrecord memory-object and effectively make it read-only
// from this point forward. we give up our ownership of the HID.
SG_ASSERT( pszHidComputed );
_sg_dbrecord__freeze(pCtx, pRecord, pszHidComputed);
pszHidComputed = NULL;
SG_STRING_NULLFREE(pCtx, pString);
return; // return _OK or __BLOBFILEALREADYEXISTS
}
fail:
SG_STRING_NULLFREE(pCtx, pString);
SG_NULLFREE(pCtx, pszHidComputed);
}
void MyFn(create_blob_from_bytes)(SG_context * pCtx,
SG_repo * pRepo,
SG_uint32 lenBuf1,
const char * szSrc)
{
// create a large buffer containing some known data.
// use it to create a blob directly from the buffer.
// read it back from the repo and verify it.
char* pszidTidRandom1 = NULL;
char* pszidTidRandom2 = NULL;
SG_uint32 lenSrc;
char* pszidHidBlob1 = NULL;
char* pszidHidBlob1Dup = NULL;
char* pszidHidBlob2 = NULL;
char* pszidHidVerify1 = NULL;
char* pszidHidVerify2 = NULL;
SG_bool bEqual;
char * pbuf1 = NULL;
char * pbuf2 = NULL;
char * pbuf1End;
char * p1;
SG_uint64 lenBuf2;
SG_repo_tx_handle* pTx = NULL;
SG_repo_fetch_blob_handle* pFetchHandle = NULL;
SG_uint64 lenAbortedBlob;
SG_uint32 lenGotAbortedBlob;
SG_bool b_done = SG_FALSE;
//////////////////////////////////////////////////////////////////
SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom1, 32) );
SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom2, 32) );
if (lenBuf1 < 100)
lenBuf1 += 100;
pbuf1 = (char *)SG_calloc(1,lenBuf1+1);
pbuf1End = pbuf1+lenBuf1;
p1 = pbuf1;
// write random gid at the beginning of the buffer
// so that we won't get collisions if we are called
// multiple times.
memcpy(p1,pszidTidRandom1,strlen(pszidTidRandom1));
p1 += strlen(pszidTidRandom1);
*p1++ = '\n';
// generate lots of data in the file so that we'll cause the
// blob routines to exercise the chunking stuff.
lenSrc = (SG_uint32)strlen(szSrc);
while (p1+lenSrc < pbuf1End)
{
memcpy(p1,szSrc,lenSrc);
p1 += lenSrc;
}
// use the buffer to create a blob in the repo. we use lenBuf1 as the
// length rather than (p1-pbuf1) so we may have some nulls at the end.
// hope this is ok??? i guess we'll find out...
VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
INFOP("create_blob_from_bytes",("Created blob [%s]",(pszidHidBlob1)));
//////////////////////////////////////////////////////////////////
// try to create blob again and verify we get an duplicate-hid error.
// Ian TODO: Put this back when SG_ERR_BLOBFILEALREADYEXISTS is replaced.
// VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
// err = SG_repo__store_blob_from_memory(pRepo,pTx,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1Dup);
// VERIFYP_CTX_ERR_IS("create_blob_from_bytes(duplicate)", pCtx, SG_ERR_BLOBFILEALREADYEXISTS, ("Duplicate create failed [%s][%s]",pszidHidBlob1,pszidHidBlob1Dup));
// VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, SG_DAGNUM__NONE, NULL, &pTx) );
//////////////////////////////////////////////////////////////////
// abort a fetch, ensure it doesn't interfere with subsequent complete fetch
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__begin(pCtx, pRepo, pszidHidBlob1, SG_TRUE, NULL, NULL, NULL, NULL, &lenAbortedBlob, &pFetchHandle) );
// We want to abort mid-blob, so we set an arbitrary (but small) chunk size, and verify that the blob is
// bigger than it.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__chunk(pCtx, pRepo, pFetchHandle, 64, (SG_byte*)pbuf1, &lenGotAbortedBlob, &b_done) );
VERIFY_COND("create_blob_from_bytes(fetch abort sufficient length)", lenAbortedBlob > lenGotAbortedBlob);
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__abort(pCtx, pRepo, &pFetchHandle) );
VERIFY_COND("create_blob_from_bytes(fetch abort freed handle)", !pFetchHandle);
//////////////////////////////////////////////////////////////////
// fetch blob into a new buffer and verify that it matches.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) );
VERIFY_COND("create_blob_from_bytes(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1));
//////////////////////////////////////////////////////////////////
// verify that the contents of buf-2 is identical to the contents of buf-1.
// (we already know that the HIDs match and was verified during the fetch,
// but there are times when the HID is just being used as a key -- it
// doesn't mean that what we actually restored is correct.
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, lenBuf1, (SG_byte *)pbuf1, &pszidHidVerify1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, (SG_uint32)lenBuf2, (SG_byte *)pbuf2, &pszidHidVerify2) );
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidVerify2)));
VERIFY_COND("create_blob_from_bytes(verify v1==v2)",bEqual);
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidBlob1)));
VERIFY_COND("create_blob_from_bytes(verify v1==id)",bEqual);
VERIFY_COND("creata_blob_from_bytes(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0));
//////////////////////////////////////////////////////////////////
// cleanup
SG_NULLFREE(pCtx, pbuf1);
SG_NULLFREE(pCtx, pbuf2);
SG_NULLFREE(pCtx, pszidTidRandom1);
SG_NULLFREE(pCtx, pszidTidRandom2);
SG_NULLFREE(pCtx, pszidHidBlob1);
SG_NULLFREE(pCtx, pszidHidBlob1Dup);
SG_NULLFREE(pCtx, pszidHidBlob2);
SG_NULLFREE(pCtx, pszidHidVerify1);
SG_NULLFREE(pCtx, pszidHidVerify2);
}
