void MyFn(one_dagnode)(SG_context * pCtx, SG_repo* pRepo)
{
char* pId = NULL;
SG_dagnode* pdnCreated = NULL;
SG_dagnode* pdnFetched = NULL;
SG_repo_tx_handle* pTx = NULL;
char buf_tid[SG_TID_MAX_BUFFER_LENGTH];
VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) );
VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx,
pRepo,
sizeof(buf_tid),
(SG_byte *)buf_tid,
&pId) );
VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) );
VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) );
// Add dagnode.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, SG_DAGNUM__TESTING__NOTHING, pdnCreated) );
pdnCreated = NULL;
// Should fail: tx not committed.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched),
SG_ERR_NOT_FOUND ); // Dag node visible before repo tx committed.
// Abort repo tx.
VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__abort_tx should null/free the repo transaction.", !pTx);
// Should fail: tx aborted.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched),
SG_ERR_NOT_FOUND ); // Dag node exists after repo tx abort
// Write dagnode, commit tx.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) );
VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) );
VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, SG_DAGNUM__TESTING__NOTHING, pdnCreated) );
pdnCreated = NULL;
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__commit_tx should null/free the repo transaction.", !pTx);
// Read back the dagnode. It should exist now.
VERIFY_ERR_CHECK( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched) );
// Fall through to common cleanup.
fail:
SG_NULLFREE(pCtx, pId);
SG_DAGNODE_NULLFREE(pCtx, pdnCreated);
SG_DAGNODE_NULLFREE(pCtx, pdnFetched);
}
// Store a blob. Make sure it doesn't show up until/unless the repo tx is committed.
void MyFn(one_blob)(SG_context * pCtx, SG_repo* pRepo)
{
SG_byte* pBufIn = NULL;
SG_uint32 lenBufIn = 0;
SG_byte* pBufOut = NULL;
SG_uint64 lenBufOut = 0;
SG_repo_tx_handle* pTx;
char* pszHidReturned = NULL;
VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pBufIn, &lenBufIn) );
// Start writing blob.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) );
// Should fail: tx not committed.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut),
SG_ERR_BLOB_NOT_FOUND ); // Blob visible before repo tx committed
SG_NULLFREE(pCtx, pBufOut);
// Abort repo tx.
VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__abort_tx should null/free the repo transaction.", !pTx);
// Should fail: tx aborted.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut),
SG_ERR_BLOB_NOT_FOUND ); // Blob exists after repo tx abort
SG_NULLFREE(pCtx, pBufOut);
SG_NULLFREE(pCtx, pszHidReturned);
// Write blob, commit tx.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) );
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
VERIFY_COND("SG_repo__commit_tx should null/free the repo transaction.", !pTx);
// Read back the blob. It should exist now.
VERIFY_ERR_CHECK( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut) );
// Just verify the length. It's another test's job to roundtrip blobs and verify data.
VERIFY_COND( "blob length mismatch", lenBufOut == lenBufIn );
// Fall through to common cleanup.
fail:
SG_NULLFREE(pCtx, pszHidReturned);
SG_NULLFREE(pCtx, pBufIn);
SG_NULLFREE(pCtx, pBufOut);
}
void SG_repo__unpack(SG_context* pCtx, SG_repo * pRepo, SG_blob_encoding blob_encoding)
{
SG_vhash* pvh = NULL;
SG_uint32 count = 0;
SG_uint32 i = 0;
SG_repo_tx_handle* pTx;
SG_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, blob_encoding, SG_FALSE, SG_FALSE, 500, 0, &pvh) );
SG_ERR_CHECK( SG_vhash__count(pCtx, pvh, &count) );
for (i=0; i<count; i++)
{
const char* psz_hid = NULL;
const SG_variant* pv = NULL;
SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvh, i, &psz_hid, &pv) );
// Not a lot of thought went into doing each of these in its own repo tx. Consider alternatives.
SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid, SG_BLOBENCODING__FULL, NULL, NULL, NULL, NULL, NULL) );
SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
}
SG_VHASH_NULLFREE(pCtx, pvh);
return;
fail:
SG_VHASH_NULLFREE(pCtx, pvh);
}
void u0048_multidag__add_dagnode(SG_context * pCtx,
char** ppszid, const char* pszidParent, SG_uint32 iDagNum, SG_repo* pRepo)
{
char buf_tid[SG_TID_MAX_BUFFER_LENGTH];
SG_dagnode* pdn = NULL;
SG_repo_tx_handle* pTx;
// create a TID just to get some random data. use it to create a HID.
// use the HID as the HID of a hypothetical changeset so that we can create the dagnode.
VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) );
VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx,
pRepo,
sizeof(buf_tid),
(SG_byte *)buf_tid,
ppszid) );
VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdn,*ppszid, pszidParent ? 2: 1 ) );
if (pszidParent)
{
VERIFY_ERR_CHECK( SG_dagnode__add_parent(pCtx, pdn,pszidParent) );
}
VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdn) );
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, iDagNum, pdn) );
pdn = NULL;
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo,&pTx) );
return;
fail:
return;
}
void MyFn(create_zero_byte_blob)(SG_context* pCtx,
SG_repo* pRepo)
{
struct z
{
const char * pszHashMethod;
const char * pszTrivialHash;
};
struct z az[] = { { "SHA1/160", "da39a3ee5e6b4b0d3255bfef95601890afd80709" },
{ "SHA2/256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" },
{ "SHA2/384", "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b" },
{ "SHA2/512", "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" },
};
char* pszidHidBlob1 = NULL;
char * pbuf1 = NULL;
char * pbuf2 = NULL;
SG_uint64 lenBuf2;
SG_repo_tx_handle* pTx = NULL;
SG_uint32 lenBuf1 = 0;
char * pszHashMethod = NULL;
SG_uint32 k, kLimit;
pbuf1 = (char *)SG_calloc(1,lenBuf1+1);
VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
INFOP("create_zero_byte_blob",("Created blob [%s]",(pszidHidBlob1)));
//////////////////////////////////////////////////////////////////
// fetch blob into a new buffer and verify that it matches.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) );
VERIFY_COND("create_zero_byte_blob(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1));
VERIFY_COND("create_zero_byte_blob(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0));
VERIFY_ERR_CHECK_DISCARD( SG_repo__get_hash_method(pCtx, pRepo, &pszHashMethod) );
kLimit = SG_NrElements(az);
for (k=0; k<kLimit; k++)
{
if (strcmp(pszHashMethod,az[k].pszHashMethod) == 0)
{
// The empty blob should always have this hid
VERIFY_COND("zero byte blob hid mismatch",
strcmp(pszidHidBlob1, az[k].pszTrivialHash) == 0);
}
}
//////////////////////////////////////////////////////////////////
// cleanup
SG_NULLFREE(pCtx, pbuf1);
SG_NULLFREE(pCtx, pbuf2);
SG_NULLFREE(pCtx, pszidHidBlob1);
SG_NULLFREE(pCtx, pszHashMethod);
}
void SG_repo__pack__zlib(SG_context* pCtx, SG_repo * pRepo)
{
SG_vhash* pvh = NULL;
SG_uint32 count = 0;
SG_uint32 i = 0;
SG_repo_tx_handle* pTx;
SG_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, SG_BLOBENCODING__FULL, SG_TRUE, SG_TRUE, 500, 0, &pvh) );
SG_ERR_CHECK( SG_vhash__count(pCtx, pvh, &count) );
SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
for (i=0; i<count; i++)
{
const char* psz_hid = NULL;
const SG_variant* pv = NULL;
SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvh, i, &psz_hid, &pv) );
SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid, SG_BLOBENCODING__ZLIB, NULL, NULL, NULL, NULL, NULL);
if (SG_context__has_err(pCtx))
{
if (!SG_context__err_equals(pCtx,SG_ERR_REPO_BUSY))
{
SG_ERR_RETHROW;
}
else
{
SG_context__err_reset(pCtx);
}
}
}
SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
SG_VHASH_NULLFREE(pCtx, pvh);
return;
fail:
SG_VHASH_NULLFREE(pCtx, pvh);
}
void MyFn(empty_tx)(SG_context * pCtx, SG_repo* pRepo)
{
SG_blobset* pbs = NULL;
SG_uint32 count_blobs_before = 0;
SG_uint64 len_encoded_before = 0;
SG_uint64 len_full_before = 0;
SG_uint32 count_blobs_after = 0;
SG_uint64 len_encoded_after = 0;
SG_uint64 len_full_after = 0;
SG_repo_tx_handle* pTx = NULL;
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_before,
&len_encoded_before,
&len_full_before,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
// Commit empty tx.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_after,
&len_encoded_after,
&len_full_after,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after);
VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after);
VERIFY_COND("len_full mismatch", len_full_before == len_full_after);
// Abort empty tx.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_after,
&len_encoded_after,
&len_full_after,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after);
VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after);
VERIFY_COND("len_full mismatch", len_full_before == len_full_after);
return;
fail:
return;
}
// You can only store one blob at a time.
void MyFn(multiple_stores_fail)(SG_context * pCtx, SG_repo* pRepo)
{
SG_blobset* pbs = NULL;
SG_uint32 count_blobs_before = 0;
SG_uint64 len_encoded_before = 0;
SG_uint64 len_full_before = 0;
SG_uint32 count_blobs_after = 0;
SG_uint64 len_encoded_after = 0;
SG_uint64 len_full_after = 0;
SG_repo_tx_handle* pTx = NULL;
SG_repo_store_blob_handle* pbh;
SG_repo_store_blob_handle* pbh2;
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_before,
&len_encoded_before,
&len_full_before,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
// Start writing a blob.
VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, 500, 0, NULL, &pbh) );
// Start another blob. Should fail.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, 500, 0, NULL, &pbh2),
SG_ERR_INCOMPLETE_BLOB_IN_REPO_TX ); // SG_repo__store_blob__begin didn't fail when previous blob wasn't done.
VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_after,
&len_encoded_after,
&len_full_after,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after);
VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after);
VERIFY_COND("len_full mismatch", len_full_before == len_full_after);
return;
fail:
return;
}
// Create 8 blobs, commit those whose bit is set in blobMask, abort the rest. Verify results.
void MyFn(eight_blobs_commit_masked)(SG_context * pCtx, SG_repo* pRepo, SG_uint8 blobMask)
{
SG_blobset* pbs = NULL;
SG_uint32 count_blobs_before = 0;
SG_uint64 len_encoded_before = 0;
SG_uint64 len_full_before = 0;
SG_uint32 count_blobs_after = 0;
SG_uint64 len_encoded_after = 0;
SG_uint64 len_full_after = 0;
SG_byte* pRandomBuf = NULL;
SG_uint32 lenRandomBuf;
SG_uint64 lenTotal = 0;
SG_repo_tx_handle* pTx = NULL;
SG_repo_store_blob_handle* pbh;
SG_uint32 i,j;
SG_uint32 iLenWritten = 0;
char* apszHids[8];
SG_uint8 countBlobsToAdd;
SG_uint8 mask = blobMask;
// Count the number of bits set in blobMask.
for (countBlobsToAdd = 0; mask; mask >>= 1)
countBlobsToAdd += mask & 1;
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_before,
&len_encoded_before,
&len_full_before,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
for (i=0; i < 8; i++)
{
VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pRandomBuf, &lenRandomBuf) );
lenTotal = lenRandomBuf * 3;
VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenTotal, 0, NULL, &pbh) );
for (j=0; j < 3; j++)
{
VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenRandomBuf, pRandomBuf, &iLenWritten) );
// This chunk is much smaller than SG_STREAMING_BUFFER_SIZE, so the whole thing should be written.
VERIFY_COND("SG_repo__store_blob__chunk length mismatch.", iLenWritten == lenRandomBuf);
if ((1 == j) && ((1 << i & blobMask) == 0)) // we're mid-blob and blob is to be aborted (bit is unset)
break;
}
if ((1 << i & blobMask) != 0) // blob is to be commit (bit is set)
{
// Finish the blob.
VERIFY_ERR_CHECK( SG_repo__store_blob__end(pCtx, pRepo, pTx, &pbh, &(apszHids[i])) );
}
else
{
// Abort the blob.
VERIFY_ERR_CHECK( SG_repo__store_blob__abort(pCtx, pRepo, pTx, &pbh) );
apszHids[i] = NULL;
}
SG_NULLFREE(pCtx, pRandomBuf);
}
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_after,
&len_encoded_after,
&len_full_after,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_COND("blob count mismatch", (count_blobs_before + countBlobsToAdd)
== count_blobs_after);
// Verify HIDs we think we added, were.
for (i=0; i < 8; i++)
{
if (apszHids[i])
{
SG_uint64 len = 0;
VERIFY_ERR_CHECK( SG_repo__fetch_blob_into_memory(pCtx, pRepo, apszHids[i], &pRandomBuf, &len) );
SG_NULLFREE(pCtx, pRandomBuf);
}
}
// Fall through to common cleanup.
fail:
SG_NULLFREE(pCtx, pRandomBuf);
for (i=0; i < 8; i++)
SG_NULLFREE(pCtx, apszHids[i]);
}
// Commit a repo tx while a blob is being stored.
//
// We jump through some awkward hoops to clean up memory in this case. If it becomes more trouble than
// it's worth, this test might not be worth running.
void MyFn(commit_mid_blob)(SG_context * pCtx, SG_repo* pRepo)
{
SG_blobset* pbs = NULL;
SG_uint32 count_blobs_before = 0;
SG_uint64 len_encoded_before = 0;
SG_uint64 len_full_before = 0;
SG_uint32 count_blobs_after = 0;
SG_uint64 len_encoded_after = 0;
SG_uint64 len_full_after = 0;
SG_byte* pRandomBuf = NULL;
SG_uint32 lenRandomBuf;
SG_uint64 lenTotal = 0;
SG_repo_tx_handle* pTx = NULL;
SG_repo_store_blob_handle* pbh;
SG_uint32 i;
SG_uint32 iLenWritten = 0;
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_before,
&len_encoded_before,
&len_full_before,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pRandomBuf, &lenRandomBuf) );
lenTotal = lenRandomBuf * 5;
// Start writing blob.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenTotal, 0, NULL, &pbh) );
for (i=0; i < 3; i++)
{
VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenRandomBuf, pRandomBuf, &iLenWritten) );
// This chunk is much smaller than SG_STREAMING_BUFFER_SIZE, so the whole thing should be written.
VERIFY_COND("SG_repo__store_blob__chunk length mismatch.", iLenWritten == lenRandomBuf);
}
// We're not done yet, so this should fail.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx),
SG_ERR_INCOMPLETE_BLOB_IN_REPO_TX ); // SG_repo__commit_tx should return SG_ERR_INCOMPLETE_BLOB_IN_REPO_TX
VERIFY_COND("SG_repo__commit_tx should free the repo transaction.", !pTx);
SG_NULLFREE(pCtx, pRandomBuf);
VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) );
VERIFY_ERR_CHECK( SG_blobset__get_stats(
pCtx,
pbs,
0,
&count_blobs_after,
&len_encoded_after,
&len_full_after,
NULL,
NULL
) );
SG_BLOBSET_NULLFREE(pCtx, pbs);
VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after);
VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after);
VERIFY_COND("len_full mismatch", len_full_before == len_full_after);
return;
fail:
SG_NULLFREE(pCtx, pRandomBuf);
}
void MyFn(no_repo_tx)(SG_context * pCtx, SG_repo* pRepo)
{
SG_repo_tx_handle* pTx = NULL;
SG_repo_store_blob_handle* pbh;
SG_byte* pBufIn = NULL;
SG_uint32 lenBufIn = 0;
char* pszHidReturned = NULL;
char* pId = NULL;
SG_dagnode* pdnCreated = NULL;
char buf_tid[SG_TID_MAX_BUFFER_LENGTH];
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__commit_tx(pCtx, pRepo, NULL),
SG_ERR_INVALIDARG ); // commit_tx without repo tx didn't fail with INVALIDARG.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__abort_tx(pCtx, pRepo, NULL),
SG_ERR_INVALIDARG ); // abort_tx without repo tx didn't fail with INVALIDARG.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__begin(pCtx, pRepo, NULL, SG_BLOBENCODING__FULL, NULL, 10, 0, NULL, &pbh),
SG_ERR_INVALIDARG ); // store_blob__begin without repo tx didn't fail with INVALIDARG.
// Create a repo tx so we can test the other blob functions.
VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pBufIn, &lenBufIn) );
VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenBufIn, 0, NULL, &pbh) );
VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenBufIn, pBufIn, NULL) );
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__end(pCtx, pRepo, NULL, &pbh, &pszHidReturned),
SG_ERR_INVALIDARG ); // store_blob__end without repo tx didn't fail with INVALIDARG.
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__abort(pCtx, pRepo, NULL, &pbh),
SG_ERR_INVALIDARG ); // store_blob__abort without repo tx didn't fail with INVALIDARG.
VERIFY_ERR_CHECK( SG_repo__store_blob__end(pCtx, pRepo, pTx, &pbh, &pszHidReturned) );
VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
// create a TID just to get some random data. use it to create a HID.
// use the HID as the HID of a hypothetical changeset so that we can create the dagnode.
VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) );
VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx,
pRepo,
sizeof(buf_tid),
(SG_byte *)buf_tid,
&pId) );
VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) );
VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) );
VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_dagnode(pCtx, pRepo, NULL, SG_DAGNUM__TESTING__NOTHING, pdnCreated),
SG_ERR_INVALIDARG ); // store_dagnode without repo tx didn't fail with INVALIDARG.
// We're intentionally not testing the higher-level store_blob_from_memory and store_blob_from_file
// routines here because they're just wrappers for the begin/chunk/end routines we do test.
// Fall through to common cleanup.
fail:
SG_NULLFREE(pCtx, pszHidReturned);
SG_NULLFREE(pCtx, pBufIn);
SG_NULLFREE(pCtx, pId);
SG_DAGNODE_NULLFREE(pCtx, pdnCreated);
}
void MyFn(create_blob_from_file)(SG_context * pCtx,
SG_repo * pRepo,
const SG_pathname * pPathnameTempDir,
SG_uint64 lenFile,
const char * szSrc)
{
// create a file of length "lenFile" in the temp directory.
// use it to create a blob.
// try to create it a second time and verify that we get an duplicate-hid error.
char* pszidGidRandom1 = NULL;
char* pszidGidRandom2 = NULL;
SG_pathname * pPathnameTempFile1 = NULL;
SG_pathname * pPathnameTempFile2 = NULL;
SG_file * pFileTempFile1 = NULL;
SG_file * pFileTempFile2 = NULL;
SG_uint32 lenSrc;
SG_uint64 lenWritten;
char* pszidHidBlob1 = NULL;
char* pszidHidBlob1Dup = NULL;
char* pszidHidBlob2 = NULL;
char* pszidHidVerify1 = NULL;
char* pszidHidVerify2 = NULL;
SG_bool bEqual;
SG_repo_tx_handle* pTx = NULL;
SG_uint64 iBlobFullLength = 0;
//////////////////////////////////////////////////////////////////
// create temp-file-1 of length "lenFile" in the temp directory.
VERIFY_ERR_CHECK_DISCARD( SG_gid__alloc(pCtx, &pszidGidRandom1) );
VERIFY_ERR_CHECK_DISCARD( SG_gid__alloc(pCtx, &pszidGidRandom2) );
VERIFY_ERR_CHECK_DISCARD( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pPathnameTempFile1,pPathnameTempDir,(pszidGidRandom1)) );
VERIFY_ERR_CHECK_DISCARD( SG_file__open__pathname(pCtx, pPathnameTempFile1,SG_FILE_RDWR|SG_FILE_CREATE_NEW,0644,&pFileTempFile1) );
// write random gid at the beginning of the file
// so that we won't get collisions if we are called
// multiple times.
VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,(SG_uint32)strlen(pszidGidRandom1),(SG_byte *)pszidGidRandom1,NULL) );
VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,1,(SG_byte *)"\n",NULL) );
// generate lots of data in the file so that we'll cause the
// blob routines to exercise the chunking stuff.
lenSrc = (SG_uint32)strlen(szSrc);
lenWritten = 0;
while (lenWritten < lenFile)
{
VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,lenSrc,(SG_byte *)szSrc,NULL) );
lenWritten += lenSrc;
}
// the test file does NOT have a final LF. i'm not sure it matters one way or the
// other, but i'm just saying that we're not putting on a final LF.
SG_ERR_IGNORE( SG_file__seek(pCtx, pFileTempFile1,0) );
//////////////////////////////////////////////////////////////////
// use currently open temp file to create a blob.
// we get the HID back. (we need to free it later.)
VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_file(pCtx, pRepo,pTx,NULL,SG_FALSE,pFileTempFile1,&pszidHidBlob1,&iBlobFullLength) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
INFOP("create_blob_from_file",("Created blob [%s]",(pszidHidBlob1)));
//////////////////////////////////////////////////////////////////
// try to create blob again and verify we get an duplicate-hid error.
// Ian TODO: Put this back when SG_ERR_BLOBFILEALREADYEXISTS has been replaced.
// VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
// err = SG_repo__store_blob_from_file(pRepo,pTx,SG_FALSE,pFileTempFile1,&pszidHidBlob1Dup);
// VERIFYP_CTX_ERR_IS("create_blob_from_file(duplicate)", pCtx, SG_ERR_BLOBFILEALREADYEXISTS, ("Duplicate create failed [%s][%s]",pszidHidBlob1,pszidHidBlob1Dup));
// VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, SG_DAGNUM__NONE, NULL, &pTx) );
//////////////////////////////////////////////////////////////////
// create empty temp-file-2 and try to read the blob from the repo.
VERIFY_ERR_CHECK_DISCARD( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pPathnameTempFile2,pPathnameTempDir,(pszidGidRandom2)) );
VERIFY_ERR_CHECK_DISCARD( SG_file__open__pathname(pCtx, pPathnameTempFile2,SG_FILE_RDWR|SG_FILE_CREATE_NEW,0644,&pFileTempFile2) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_file(pCtx, pRepo,pszidHidBlob1,pFileTempFile2,NULL) );
//////////////////////////////////////////////////////////////////
// verify that the contents of temp-file-2 is identical to the
// contents of temp-file-1. (we already know that the HIDs match
// and was verified during the fetch, but there are times when the
// HID is just being used as a key -- it doesn't mean that what we
// actually restored is correct.
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_file(pCtx, pRepo, pFileTempFile1, &pszidHidVerify1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_file(pCtx, pRepo, pFileTempFile2, &pszidHidVerify2) );
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidVerify2)));
VERIFY_COND("create_blob_from_file(verify v1==v2)",bEqual);
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidBlob1)));
VERIFY_COND("create_blob_from_file(verify v1==id)",bEqual);
//////////////////////////////////////////////////////////////////
// TODO delete temp source file
SG_ERR_IGNORE( SG_file__close(pCtx, &pFileTempFile1) );
SG_ERR_IGNORE( SG_file__close(pCtx, &pFileTempFile2) );
//////////////////////////////////////////////////////////////////
// cleanup
SG_NULLFREE(pCtx, pszidGidRandom1);
SG_NULLFREE(pCtx, pszidGidRandom2);
SG_NULLFREE(pCtx, pszidHidBlob1);
SG_NULLFREE(pCtx, pszidHidBlob1Dup);
SG_NULLFREE(pCtx, pszidHidBlob2);
SG_NULLFREE(pCtx, pszidHidVerify1);
SG_NULLFREE(pCtx, pszidHidVerify2);
SG_PATHNAME_NULLFREE(pCtx, pPathnameTempFile1);
SG_PATHNAME_NULLFREE(pCtx, pPathnameTempFile2);
}
void MyFn(create_blob_from_bytes)(SG_context * pCtx,
SG_repo * pRepo,
SG_uint32 lenBuf1,
const char * szSrc)
{
// create a large buffer containing some known data.
// use it to create a blob directly from the buffer.
// read it back from the repo and verify it.
char* pszidTidRandom1 = NULL;
char* pszidTidRandom2 = NULL;
SG_uint32 lenSrc;
char* pszidHidBlob1 = NULL;
char* pszidHidBlob1Dup = NULL;
char* pszidHidBlob2 = NULL;
char* pszidHidVerify1 = NULL;
char* pszidHidVerify2 = NULL;
SG_bool bEqual;
char * pbuf1 = NULL;
char * pbuf2 = NULL;
char * pbuf1End;
char * p1;
SG_uint64 lenBuf2;
SG_repo_tx_handle* pTx = NULL;
SG_repo_fetch_blob_handle* pFetchHandle = NULL;
SG_uint64 lenAbortedBlob;
SG_uint32 lenGotAbortedBlob;
SG_bool b_done = SG_FALSE;
//////////////////////////////////////////////////////////////////
SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom1, 32) );
SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom2, 32) );
if (lenBuf1 < 100)
lenBuf1 += 100;
pbuf1 = (char *)SG_calloc(1,lenBuf1+1);
pbuf1End = pbuf1+lenBuf1;
p1 = pbuf1;
// write random gid at the beginning of the buffer
// so that we won't get collisions if we are called
// multiple times.
memcpy(p1,pszidTidRandom1,strlen(pszidTidRandom1));
p1 += strlen(pszidTidRandom1);
*p1++ = '\n';
// generate lots of data in the file so that we'll cause the
// blob routines to exercise the chunking stuff.
lenSrc = (SG_uint32)strlen(szSrc);
while (p1+lenSrc < pbuf1End)
{
memcpy(p1,szSrc,lenSrc);
p1 += lenSrc;
}
// use the buffer to create a blob in the repo. we use lenBuf1 as the
// length rather than (p1-pbuf1) so we may have some nulls at the end.
// hope this is ok??? i guess we'll find out...
VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
INFOP("create_blob_from_bytes",("Created blob [%s]",(pszidHidBlob1)));
//////////////////////////////////////////////////////////////////
// try to create blob again and verify we get an duplicate-hid error.
// Ian TODO: Put this back when SG_ERR_BLOBFILEALREADYEXISTS is replaced.
// VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
// err = SG_repo__store_blob_from_memory(pRepo,pTx,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1Dup);
// VERIFYP_CTX_ERR_IS("create_blob_from_bytes(duplicate)", pCtx, SG_ERR_BLOBFILEALREADYEXISTS, ("Duplicate create failed [%s][%s]",pszidHidBlob1,pszidHidBlob1Dup));
// VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, SG_DAGNUM__NONE, NULL, &pTx) );
//////////////////////////////////////////////////////////////////
// abort a fetch, ensure it doesn't interfere with subsequent complete fetch
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__begin(pCtx, pRepo, pszidHidBlob1, SG_TRUE, NULL, NULL, NULL, NULL, &lenAbortedBlob, &pFetchHandle) );
// We want to abort mid-blob, so we set an arbitrary (but small) chunk size, and verify that the blob is
// bigger than it.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__chunk(pCtx, pRepo, pFetchHandle, 64, (SG_byte*)pbuf1, &lenGotAbortedBlob, &b_done) );
VERIFY_COND("create_blob_from_bytes(fetch abort sufficient length)", lenAbortedBlob > lenGotAbortedBlob);
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__abort(pCtx, pRepo, &pFetchHandle) );
VERIFY_COND("create_blob_from_bytes(fetch abort freed handle)", !pFetchHandle);
//////////////////////////////////////////////////////////////////
// fetch blob into a new buffer and verify that it matches.
VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) );
VERIFY_COND("create_blob_from_bytes(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1));
//////////////////////////////////////////////////////////////////
// verify that the contents of buf-2 is identical to the contents of buf-1.
// (we already know that the HIDs match and was verified during the fetch,
// but there are times when the HID is just being used as a key -- it
// doesn't mean that what we actually restored is correct.
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, lenBuf1, (SG_byte *)pbuf1, &pszidHidVerify1) );
VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, (SG_uint32)lenBuf2, (SG_byte *)pbuf2, &pszidHidVerify2) );
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidVerify2)));
VERIFY_COND("create_blob_from_bytes(verify v1==v2)",bEqual);
bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidBlob1)));
VERIFY_COND("create_blob_from_bytes(verify v1==id)",bEqual);
VERIFY_COND("creata_blob_from_bytes(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0));
//////////////////////////////////////////////////////////////////
// cleanup
SG_NULLFREE(pCtx, pbuf1);
SG_NULLFREE(pCtx, pbuf2);
SG_NULLFREE(pCtx, pszidTidRandom1);
SG_NULLFREE(pCtx, pszidTidRandom2);
SG_NULLFREE(pCtx, pszidHidBlob1);
SG_NULLFREE(pCtx, pszidHidBlob1Dup);
SG_NULLFREE(pCtx, pszidHidBlob2);
SG_NULLFREE(pCtx, pszidHidVerify1);
SG_NULLFREE(pCtx, pszidHidVerify2);
}
void SG_repo__pack__vcdiff(SG_context* pCtx, SG_repo * pRepo)
{
SG_rbtree* prb_leaves = NULL;
SG_uint32 count_leaves = 0;
const char* psz_hid_cs = NULL;
SG_rbtree* prb_blobs = NULL;
SG_bool b;
SG_rbtree_iterator* pit = NULL;
SG_rbtree_iterator* pit_for_gid = NULL;
SG_bool b_for_gid;
const char* psz_hid_ref = NULL;
const char* psz_hid_blob = NULL;
const char* psz_gid = NULL;
SG_rbtree* prb = NULL;
const char* psz_gen = NULL;
SG_repo_tx_handle* pTx;
SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo,SG_DAGNUM__VERSION_CONTROL,&prb_leaves) );
SG_ERR_CHECK( SG_rbtree__count(pCtx, prb_leaves, &count_leaves) );
SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, NULL, prb_leaves, &b, &psz_hid_cs, NULL) );
SG_ERR_CHECK( SG_RBTREE__ALLOC(pCtx, &prb_blobs) );
SG_ERR_CHECK( sg_pack__do_changeset(pCtx, pRepo, psz_hid_cs, prb_blobs) );
SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prb_blobs, &b, &psz_gid, (void**) &prb) );
while (b)
{
SG_uint32 count_for_gid = 0;
SG_ERR_CHECK( SG_rbtree__count(pCtx, prb, &count_for_gid) );
if (count_for_gid > 1)
{
psz_hid_ref = NULL;
SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit_for_gid, prb, &b_for_gid, &psz_gen, (void**) &psz_hid_blob) );
while (b_for_gid)
{
// Not a lot of thought went into doing each of these in its own repo tx. Consider alternatives.
SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) );
if (psz_hid_ref)
{
SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid_blob, SG_BLOBENCODING__VCDIFF, psz_hid_ref, NULL, NULL, NULL, NULL) );
// TODO be tolerant here of SG_ERR_REPO_BUSY
}
else
{
psz_hid_ref = psz_hid_blob;
SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid_ref, SG_BLOBENCODING__FULL, NULL, NULL, NULL, NULL, NULL) );
// TODO be tolerant here of SG_ERR_REPO_BUSY
}
SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) );
SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit_for_gid, &b_for_gid, &psz_gen, (void**) &psz_hid_blob) );
}
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit_for_gid);
psz_hid_ref = NULL;
}
SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &b, &psz_gid, (void**) &prb) );
}
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
SG_RBTREE_NULLFREE_WITH_ASSOC(pCtx, prb_blobs, _sg_repo__free_rbtree);
SG_RBTREE_NULLFREE(pCtx, prb_leaves);
return;
fail:
return;
}
