void SG_repo__unpack(SG_context* pCtx, SG_repo * pRepo, SG_blob_encoding blob_encoding) { SG_vhash* pvh = NULL; SG_uint32 count = 0; SG_uint32 i = 0; SG_repo_tx_handle* pTx; SG_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, blob_encoding, SG_FALSE, SG_FALSE, 500, 0, &pvh) ); SG_ERR_CHECK( SG_vhash__count(pCtx, pvh, &count) ); for (i=0; i<count; i++) { const char* psz_hid = NULL; const SG_variant* pv = NULL; SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvh, i, &psz_hid, &pv) ); // Not a lot of thought went into doing each of these in its own repo tx. Consider alternatives. SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid, SG_BLOBENCODING__FULL, NULL, NULL, NULL, NULL, NULL) ); SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); } SG_VHASH_NULLFREE(pCtx, pvh); return; fail: SG_VHASH_NULLFREE(pCtx, pvh); }
void u0048_multidag__add_dagnode(SG_context * pCtx, char** ppszid, const char* pszidParent, SG_uint32 iDagNum, SG_repo* pRepo) { char buf_tid[SG_TID_MAX_BUFFER_LENGTH]; SG_dagnode* pdn = NULL; SG_repo_tx_handle* pTx; // create a TID just to get some random data. use it to create a HID. // use the HID as the HID of a hypothetical changeset so that we can create the dagnode. VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) ); VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, sizeof(buf_tid), (SG_byte *)buf_tid, ppszid) ); VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdn,*ppszid, pszidParent ? 2: 1 ) ); if (pszidParent) { VERIFY_ERR_CHECK( SG_dagnode__add_parent(pCtx, pdn,pszidParent) ); } VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdn) ); VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, iDagNum, pdn) ); pdn = NULL; VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo,&pTx) ); return; fail: return; }
void MyFn(create_zero_byte_blob)(SG_context* pCtx, SG_repo* pRepo) { struct z { const char * pszHashMethod; const char * pszTrivialHash; }; struct z az[] = { { "SHA1/160", "da39a3ee5e6b4b0d3255bfef95601890afd80709" }, { "SHA2/256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "SHA2/384", "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b" }, { "SHA2/512", "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" }, }; char* pszidHidBlob1 = NULL; char * pbuf1 = NULL; char * pbuf2 = NULL; SG_uint64 lenBuf2; SG_repo_tx_handle* pTx = NULL; SG_uint32 lenBuf1 = 0; char * pszHashMethod = NULL; SG_uint32 k, kLimit; pbuf1 = (char *)SG_calloc(1,lenBuf1+1); VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); INFOP("create_zero_byte_blob",("Created blob [%s]",(pszidHidBlob1))); ////////////////////////////////////////////////////////////////// // fetch blob into a new buffer and verify that it matches. VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) ); VERIFY_COND("create_zero_byte_blob(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1)); VERIFY_COND("create_zero_byte_blob(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0)); VERIFY_ERR_CHECK_DISCARD( SG_repo__get_hash_method(pCtx, pRepo, &pszHashMethod) ); kLimit = SG_NrElements(az); for (k=0; k<kLimit; k++) { if (strcmp(pszHashMethod,az[k].pszHashMethod) == 0) { // The empty blob should always have this hid VERIFY_COND("zero byte blob hid mismatch", strcmp(pszidHidBlob1, az[k].pszTrivialHash) == 0); } } ////////////////////////////////////////////////////////////////// // cleanup SG_NULLFREE(pCtx, pbuf1); SG_NULLFREE(pCtx, pbuf2); SG_NULLFREE(pCtx, pszidHidBlob1); SG_NULLFREE(pCtx, pszHashMethod); }
void MyFn(one_dagnode)(SG_context * pCtx, SG_repo* pRepo) { char* pId = NULL; SG_dagnode* pdnCreated = NULL; SG_dagnode* pdnFetched = NULL; SG_repo_tx_handle* pTx = NULL; char buf_tid[SG_TID_MAX_BUFFER_LENGTH]; VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) ); VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, sizeof(buf_tid), (SG_byte *)buf_tid, &pId) ); VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) ); VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) ); // Add dagnode. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, SG_DAGNUM__TESTING__NOTHING, pdnCreated) ); pdnCreated = NULL; // Should fail: tx not committed. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched), SG_ERR_NOT_FOUND ); // Dag node visible before repo tx committed. // Abort repo tx. VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) ); VERIFY_COND("SG_repo__abort_tx should null/free the repo transaction.", !pTx); // Should fail: tx aborted. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched), SG_ERR_NOT_FOUND ); // Dag node exists after repo tx abort // Write dagnode, commit tx. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) ); VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) ); VERIFY_ERR_CHECK( SG_repo__store_dagnode(pCtx, pRepo, pTx, SG_DAGNUM__TESTING__NOTHING, pdnCreated) ); pdnCreated = NULL; VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); VERIFY_COND("SG_repo__commit_tx should null/free the repo transaction.", !pTx); // Read back the dagnode. It should exist now. VERIFY_ERR_CHECK( SG_repo__fetch_dagnode(pCtx, pRepo, SG_DAGNUM__TESTING__NOTHING, pId, &pdnFetched) ); // Fall through to common cleanup. fail: SG_NULLFREE(pCtx, pId); SG_DAGNODE_NULLFREE(pCtx, pdnCreated); SG_DAGNODE_NULLFREE(pCtx, pdnFetched); }
// Store a blob. Make sure it doesn't show up until/unless the repo tx is committed. void MyFn(one_blob)(SG_context * pCtx, SG_repo* pRepo) { SG_byte* pBufIn = NULL; SG_uint32 lenBufIn = 0; SG_byte* pBufOut = NULL; SG_uint64 lenBufOut = 0; SG_repo_tx_handle* pTx; char* pszHidReturned = NULL; VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pBufIn, &lenBufIn) ); // Start writing blob. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) ); // Should fail: tx not committed. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut), SG_ERR_BLOB_NOT_FOUND ); // Blob visible before repo tx committed SG_NULLFREE(pCtx, pBufOut); // Abort repo tx. VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) ); VERIFY_COND("SG_repo__abort_tx should null/free the repo transaction.", !pTx); // Should fail: tx aborted. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut), SG_ERR_BLOB_NOT_FOUND ); // Blob exists after repo tx abort SG_NULLFREE(pCtx, pBufOut); SG_NULLFREE(pCtx, pszHidReturned); // Write blob, commit tx. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__store_blob_from_memory(pCtx, pRepo, pTx, SG_FALSE, pBufIn, lenBufIn, &pszHidReturned) ); VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); VERIFY_COND("SG_repo__commit_tx should null/free the repo transaction.", !pTx); // Read back the blob. It should exist now. VERIFY_ERR_CHECK( SG_repo__fetch_blob_into_memory(pCtx, pRepo, pszHidReturned, &pBufOut, &lenBufOut) ); // Just verify the length. It's another test's job to roundtrip blobs and verify data. VERIFY_COND( "blob length mismatch", lenBufOut == lenBufIn ); // Fall through to common cleanup. fail: SG_NULLFREE(pCtx, pszHidReturned); SG_NULLFREE(pCtx, pBufIn); SG_NULLFREE(pCtx, pBufOut); }
void SG_repo__pack__zlib(SG_context* pCtx, SG_repo * pRepo) { SG_vhash* pvh = NULL; SG_uint32 count = 0; SG_uint32 i = 0; SG_repo_tx_handle* pTx; SG_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, SG_BLOBENCODING__FULL, SG_TRUE, SG_TRUE, 500, 0, &pvh) ); SG_ERR_CHECK( SG_vhash__count(pCtx, pvh, &count) ); SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); for (i=0; i<count; i++) { const char* psz_hid = NULL; const SG_variant* pv = NULL; SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvh, i, &psz_hid, &pv) ); SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid, SG_BLOBENCODING__ZLIB, NULL, NULL, NULL, NULL, NULL); if (SG_context__has_err(pCtx)) { if (!SG_context__err_equals(pCtx,SG_ERR_REPO_BUSY)) { SG_ERR_RETHROW; } else { SG_context__err_reset(pCtx); } } } SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); SG_VHASH_NULLFREE(pCtx, pvh); return; fail: SG_VHASH_NULLFREE(pCtx, pvh); }
void MyFn(empty_tx)(SG_context * pCtx, SG_repo* pRepo) { SG_blobset* pbs = NULL; SG_uint32 count_blobs_before = 0; SG_uint64 len_encoded_before = 0; SG_uint64 len_full_before = 0; SG_uint32 count_blobs_after = 0; SG_uint64 len_encoded_after = 0; SG_uint64 len_full_after = 0; SG_repo_tx_handle* pTx = NULL; VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_before, &len_encoded_before, &len_full_before, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); // Commit empty tx. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_after, &len_encoded_after, &len_full_after, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after); VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after); VERIFY_COND("len_full mismatch", len_full_before == len_full_after); // Abort empty tx. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__abort_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_after, &len_encoded_after, &len_full_after, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after); VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after); VERIFY_COND("len_full mismatch", len_full_before == len_full_after); return; fail: return; }
// Create 8 blobs, commit those whose bit is set in blobMask, abort the rest. Verify results. void MyFn(eight_blobs_commit_masked)(SG_context * pCtx, SG_repo* pRepo, SG_uint8 blobMask) { SG_blobset* pbs = NULL; SG_uint32 count_blobs_before = 0; SG_uint64 len_encoded_before = 0; SG_uint64 len_full_before = 0; SG_uint32 count_blobs_after = 0; SG_uint64 len_encoded_after = 0; SG_uint64 len_full_after = 0; SG_byte* pRandomBuf = NULL; SG_uint32 lenRandomBuf; SG_uint64 lenTotal = 0; SG_repo_tx_handle* pTx = NULL; SG_repo_store_blob_handle* pbh; SG_uint32 i,j; SG_uint32 iLenWritten = 0; char* apszHids[8]; SG_uint8 countBlobsToAdd; SG_uint8 mask = blobMask; // Count the number of bits set in blobMask. for (countBlobsToAdd = 0; mask; mask >>= 1) countBlobsToAdd += mask & 1; VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_before, &len_encoded_before, &len_full_before, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); for (i=0; i < 8; i++) { VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pRandomBuf, &lenRandomBuf) ); lenTotal = lenRandomBuf * 3; VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenTotal, 0, NULL, &pbh) ); for (j=0; j < 3; j++) { VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenRandomBuf, pRandomBuf, &iLenWritten) ); // This chunk is much smaller than SG_STREAMING_BUFFER_SIZE, so the whole thing should be written. VERIFY_COND("SG_repo__store_blob__chunk length mismatch.", iLenWritten == lenRandomBuf); if ((1 == j) && ((1 << i & blobMask) == 0)) // we're mid-blob and blob is to be aborted (bit is unset) break; } if ((1 << i & blobMask) != 0) // blob is to be commit (bit is set) { // Finish the blob. VERIFY_ERR_CHECK( SG_repo__store_blob__end(pCtx, pRepo, pTx, &pbh, &(apszHids[i])) ); } else { // Abort the blob. VERIFY_ERR_CHECK( SG_repo__store_blob__abort(pCtx, pRepo, pTx, &pbh) ); apszHids[i] = NULL; } SG_NULLFREE(pCtx, pRandomBuf); } VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_after, &len_encoded_after, &len_full_after, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_COND("blob count mismatch", (count_blobs_before + countBlobsToAdd) == count_blobs_after); // Verify HIDs we think we added, were. for (i=0; i < 8; i++) { if (apszHids[i]) { SG_uint64 len = 0; VERIFY_ERR_CHECK( SG_repo__fetch_blob_into_memory(pCtx, pRepo, apszHids[i], &pRandomBuf, &len) ); SG_NULLFREE(pCtx, pRandomBuf); } } // Fall through to common cleanup. fail: SG_NULLFREE(pCtx, pRandomBuf); for (i=0; i < 8; i++) SG_NULLFREE(pCtx, apszHids[i]); }
// Commit a repo tx while a blob is being stored. // // We jump through some awkward hoops to clean up memory in this case. If it becomes more trouble than // it's worth, this test might not be worth running. void MyFn(commit_mid_blob)(SG_context * pCtx, SG_repo* pRepo) { SG_blobset* pbs = NULL; SG_uint32 count_blobs_before = 0; SG_uint64 len_encoded_before = 0; SG_uint64 len_full_before = 0; SG_uint32 count_blobs_after = 0; SG_uint64 len_encoded_after = 0; SG_uint64 len_full_after = 0; SG_byte* pRandomBuf = NULL; SG_uint32 lenRandomBuf; SG_uint64 lenTotal = 0; SG_repo_tx_handle* pTx = NULL; SG_repo_store_blob_handle* pbh; SG_uint32 i; SG_uint32 iLenWritten = 0; VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_before, &len_encoded_before, &len_full_before, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pRandomBuf, &lenRandomBuf) ); lenTotal = lenRandomBuf * 5; // Start writing blob. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenTotal, 0, NULL, &pbh) ); for (i=0; i < 3; i++) { VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenRandomBuf, pRandomBuf, &iLenWritten) ); // This chunk is much smaller than SG_STREAMING_BUFFER_SIZE, so the whole thing should be written. VERIFY_COND("SG_repo__store_blob__chunk length mismatch.", iLenWritten == lenRandomBuf); } // We're not done yet, so this should fail. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx), SG_ERR_INCOMPLETE_BLOB_IN_REPO_TX ); // SG_repo__commit_tx should return SG_ERR_INCOMPLETE_BLOB_IN_REPO_TX VERIFY_COND("SG_repo__commit_tx should free the repo transaction.", !pTx); SG_NULLFREE(pCtx, pRandomBuf); VERIFY_ERR_CHECK( SG_repo__list_blobs(pCtx, pRepo, 0, 0, 0, &pbs) ); VERIFY_ERR_CHECK( SG_blobset__get_stats( pCtx, pbs, 0, &count_blobs_after, &len_encoded_after, &len_full_after, NULL, NULL ) ); SG_BLOBSET_NULLFREE(pCtx, pbs); VERIFY_COND("blob count mismatch", count_blobs_before == count_blobs_after); VERIFY_COND("len_encoded mismatch", len_encoded_before == len_encoded_after); VERIFY_COND("len_full mismatch", len_full_before == len_full_after); return; fail: SG_NULLFREE(pCtx, pRandomBuf); }
void MyFn(no_repo_tx)(SG_context * pCtx, SG_repo* pRepo) { SG_repo_tx_handle* pTx = NULL; SG_repo_store_blob_handle* pbh; SG_byte* pBufIn = NULL; SG_uint32 lenBufIn = 0; char* pszHidReturned = NULL; char* pId = NULL; SG_dagnode* pdnCreated = NULL; char buf_tid[SG_TID_MAX_BUFFER_LENGTH]; VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__commit_tx(pCtx, pRepo, NULL), SG_ERR_INVALIDARG ); // commit_tx without repo tx didn't fail with INVALIDARG. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__abort_tx(pCtx, pRepo, NULL), SG_ERR_INVALIDARG ); // abort_tx without repo tx didn't fail with INVALIDARG. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__begin(pCtx, pRepo, NULL, SG_BLOBENCODING__FULL, NULL, 10, 0, NULL, &pbh), SG_ERR_INVALIDARG ); // store_blob__begin without repo tx didn't fail with INVALIDARG. // Create a repo tx so we can test the other blob functions. VERIFY_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK( MyFn(alloc_random_buffer)(pCtx, &pBufIn, &lenBufIn) ); VERIFY_ERR_CHECK( SG_repo__store_blob__begin(pCtx, pRepo, pTx, SG_BLOBENCODING__FULL, NULL, lenBufIn, 0, NULL, &pbh) ); VERIFY_ERR_CHECK( SG_repo__store_blob__chunk(pCtx, pRepo, pbh, lenBufIn, pBufIn, NULL) ); VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__end(pCtx, pRepo, NULL, &pbh, &pszHidReturned), SG_ERR_INVALIDARG ); // store_blob__end without repo tx didn't fail with INVALIDARG. VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_blob__abort(pCtx, pRepo, NULL, &pbh), SG_ERR_INVALIDARG ); // store_blob__abort without repo tx didn't fail with INVALIDARG. VERIFY_ERR_CHECK( SG_repo__store_blob__end(pCtx, pRepo, pTx, &pbh, &pszHidReturned) ); VERIFY_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); // create a TID just to get some random data. use it to create a HID. // use the HID as the HID of a hypothetical changeset so that we can create the dagnode. VERIFY_ERR_CHECK( SG_tid__generate(pCtx, buf_tid, sizeof(buf_tid)) ); VERIFY_ERR_CHECK( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, sizeof(buf_tid), (SG_byte *)buf_tid, &pId) ); VERIFY_ERR_CHECK( SG_dagnode__alloc(pCtx, &pdnCreated, pId, 1, 0) ); VERIFY_ERR_CHECK( SG_dagnode__freeze(pCtx, pdnCreated) ); VERIFY_ERR_CHECK_ERR_EQUALS_DISCARD( SG_repo__store_dagnode(pCtx, pRepo, NULL, SG_DAGNUM__TESTING__NOTHING, pdnCreated), SG_ERR_INVALIDARG ); // store_dagnode without repo tx didn't fail with INVALIDARG. // We're intentionally not testing the higher-level store_blob_from_memory and store_blob_from_file // routines here because they're just wrappers for the begin/chunk/end routines we do test. // Fall through to common cleanup. fail: SG_NULLFREE(pCtx, pszHidReturned); SG_NULLFREE(pCtx, pBufIn); SG_NULLFREE(pCtx, pId); SG_DAGNODE_NULLFREE(pCtx, pdnCreated); }
void MyFn(create_blob_from_file)(SG_context * pCtx, SG_repo * pRepo, const SG_pathname * pPathnameTempDir, SG_uint64 lenFile, const char * szSrc) { // create a file of length "lenFile" in the temp directory. // use it to create a blob. // try to create it a second time and verify that we get an duplicate-hid error. char* pszidGidRandom1 = NULL; char* pszidGidRandom2 = NULL; SG_pathname * pPathnameTempFile1 = NULL; SG_pathname * pPathnameTempFile2 = NULL; SG_file * pFileTempFile1 = NULL; SG_file * pFileTempFile2 = NULL; SG_uint32 lenSrc; SG_uint64 lenWritten; char* pszidHidBlob1 = NULL; char* pszidHidBlob1Dup = NULL; char* pszidHidBlob2 = NULL; char* pszidHidVerify1 = NULL; char* pszidHidVerify2 = NULL; SG_bool bEqual; SG_repo_tx_handle* pTx = NULL; SG_uint64 iBlobFullLength = 0; ////////////////////////////////////////////////////////////////// // create temp-file-1 of length "lenFile" in the temp directory. VERIFY_ERR_CHECK_DISCARD( SG_gid__alloc(pCtx, &pszidGidRandom1) ); VERIFY_ERR_CHECK_DISCARD( SG_gid__alloc(pCtx, &pszidGidRandom2) ); VERIFY_ERR_CHECK_DISCARD( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pPathnameTempFile1,pPathnameTempDir,(pszidGidRandom1)) ); VERIFY_ERR_CHECK_DISCARD( SG_file__open__pathname(pCtx, pPathnameTempFile1,SG_FILE_RDWR|SG_FILE_CREATE_NEW,0644,&pFileTempFile1) ); // write random gid at the beginning of the file // so that we won't get collisions if we are called // multiple times. VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,(SG_uint32)strlen(pszidGidRandom1),(SG_byte *)pszidGidRandom1,NULL) ); VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,1,(SG_byte *)"\n",NULL) ); // generate lots of data in the file so that we'll cause the // blob routines to exercise the chunking stuff. lenSrc = (SG_uint32)strlen(szSrc); lenWritten = 0; while (lenWritten < lenFile) { VERIFY_ERR_CHECK_DISCARD( SG_file__write(pCtx, pFileTempFile1,lenSrc,(SG_byte *)szSrc,NULL) ); lenWritten += lenSrc; } // the test file does NOT have a final LF. i'm not sure it matters one way or the // other, but i'm just saying that we're not putting on a final LF. SG_ERR_IGNORE( SG_file__seek(pCtx, pFileTempFile1,0) ); ////////////////////////////////////////////////////////////////// // use currently open temp file to create a blob. // we get the HID back. (we need to free it later.) VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_file(pCtx, pRepo,pTx,NULL,SG_FALSE,pFileTempFile1,&pszidHidBlob1,&iBlobFullLength) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); INFOP("create_blob_from_file",("Created blob [%s]",(pszidHidBlob1))); ////////////////////////////////////////////////////////////////// // try to create blob again and verify we get an duplicate-hid error. // Ian TODO: Put this back when SG_ERR_BLOBFILEALREADYEXISTS has been replaced. // VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); // err = SG_repo__store_blob_from_file(pRepo,pTx,SG_FALSE,pFileTempFile1,&pszidHidBlob1Dup); // VERIFYP_CTX_ERR_IS("create_blob_from_file(duplicate)", pCtx, SG_ERR_BLOBFILEALREADYEXISTS, ("Duplicate create failed [%s][%s]",pszidHidBlob1,pszidHidBlob1Dup)); // VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, SG_DAGNUM__NONE, NULL, &pTx) ); ////////////////////////////////////////////////////////////////// // create empty temp-file-2 and try to read the blob from the repo. VERIFY_ERR_CHECK_DISCARD( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pPathnameTempFile2,pPathnameTempDir,(pszidGidRandom2)) ); VERIFY_ERR_CHECK_DISCARD( SG_file__open__pathname(pCtx, pPathnameTempFile2,SG_FILE_RDWR|SG_FILE_CREATE_NEW,0644,&pFileTempFile2) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_file(pCtx, pRepo,pszidHidBlob1,pFileTempFile2,NULL) ); ////////////////////////////////////////////////////////////////// // verify that the contents of temp-file-2 is identical to the // contents of temp-file-1. (we already know that the HIDs match // and was verified during the fetch, but there are times when the // HID is just being used as a key -- it doesn't mean that what we // actually restored is correct. VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_file(pCtx, pRepo, pFileTempFile1, &pszidHidVerify1) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_file(pCtx, pRepo, pFileTempFile2, &pszidHidVerify2) ); bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidVerify2))); VERIFY_COND("create_blob_from_file(verify v1==v2)",bEqual); bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidBlob1))); VERIFY_COND("create_blob_from_file(verify v1==id)",bEqual); ////////////////////////////////////////////////////////////////// // TODO delete temp source file SG_ERR_IGNORE( SG_file__close(pCtx, &pFileTempFile1) ); SG_ERR_IGNORE( SG_file__close(pCtx, &pFileTempFile2) ); ////////////////////////////////////////////////////////////////// // cleanup SG_NULLFREE(pCtx, pszidGidRandom1); SG_NULLFREE(pCtx, pszidGidRandom2); SG_NULLFREE(pCtx, pszidHidBlob1); SG_NULLFREE(pCtx, pszidHidBlob1Dup); SG_NULLFREE(pCtx, pszidHidBlob2); SG_NULLFREE(pCtx, pszidHidVerify1); SG_NULLFREE(pCtx, pszidHidVerify2); SG_PATHNAME_NULLFREE(pCtx, pPathnameTempFile1); SG_PATHNAME_NULLFREE(pCtx, pPathnameTempFile2); }
void MyFn(create_blob_from_bytes)(SG_context * pCtx, SG_repo * pRepo, SG_uint32 lenBuf1, const char * szSrc) { // create a large buffer containing some known data. // use it to create a blob directly from the buffer. // read it back from the repo and verify it. char* pszidTidRandom1 = NULL; char* pszidTidRandom2 = NULL; SG_uint32 lenSrc; char* pszidHidBlob1 = NULL; char* pszidHidBlob1Dup = NULL; char* pszidHidBlob2 = NULL; char* pszidHidVerify1 = NULL; char* pszidHidVerify2 = NULL; SG_bool bEqual; char * pbuf1 = NULL; char * pbuf2 = NULL; char * pbuf1End; char * p1; SG_uint64 lenBuf2; SG_repo_tx_handle* pTx = NULL; SG_repo_fetch_blob_handle* pFetchHandle = NULL; SG_uint64 lenAbortedBlob; SG_uint32 lenGotAbortedBlob; SG_bool b_done = SG_FALSE; ////////////////////////////////////////////////////////////////// SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom1, 32) ); SG_ERR_IGNORE( SG_tid__alloc2(pCtx, &pszidTidRandom2, 32) ); if (lenBuf1 < 100) lenBuf1 += 100; pbuf1 = (char *)SG_calloc(1,lenBuf1+1); pbuf1End = pbuf1+lenBuf1; p1 = pbuf1; // write random gid at the beginning of the buffer // so that we won't get collisions if we are called // multiple times. memcpy(p1,pszidTidRandom1,strlen(pszidTidRandom1)); p1 += strlen(pszidTidRandom1); *p1++ = '\n'; // generate lots of data in the file so that we'll cause the // blob routines to exercise the chunking stuff. lenSrc = (SG_uint32)strlen(szSrc); while (p1+lenSrc < pbuf1End) { memcpy(p1,szSrc,lenSrc); p1 += lenSrc; } // use the buffer to create a blob in the repo. we use lenBuf1 as the // length rather than (p1-pbuf1) so we may have some nulls at the end. // hope this is ok??? i guess we'll find out... VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__store_blob_from_memory(pCtx, pRepo,pTx,NULL,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); INFOP("create_blob_from_bytes",("Created blob [%s]",(pszidHidBlob1))); ////////////////////////////////////////////////////////////////// // try to create blob again and verify we get an duplicate-hid error. // Ian TODO: Put this back when SG_ERR_BLOBFILEALREADYEXISTS is replaced. // VERIFY_ERR_CHECK_DISCARD( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); // err = SG_repo__store_blob_from_memory(pRepo,pTx,SG_FALSE,(SG_byte *)pbuf1,lenBuf1,&pszidHidBlob1Dup); // VERIFYP_CTX_ERR_IS("create_blob_from_bytes(duplicate)", pCtx, SG_ERR_BLOBFILEALREADYEXISTS, ("Duplicate create failed [%s][%s]",pszidHidBlob1,pszidHidBlob1Dup)); // VERIFY_ERR_CHECK_DISCARD( SG_repo__commit_tx(pCtx, pRepo, SG_DAGNUM__NONE, NULL, &pTx) ); ////////////////////////////////////////////////////////////////// // abort a fetch, ensure it doesn't interfere with subsequent complete fetch VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__begin(pCtx, pRepo, pszidHidBlob1, SG_TRUE, NULL, NULL, NULL, NULL, &lenAbortedBlob, &pFetchHandle) ); // We want to abort mid-blob, so we set an arbitrary (but small) chunk size, and verify that the blob is // bigger than it. VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__chunk(pCtx, pRepo, pFetchHandle, 64, (SG_byte*)pbuf1, &lenGotAbortedBlob, &b_done) ); VERIFY_COND("create_blob_from_bytes(fetch abort sufficient length)", lenAbortedBlob > lenGotAbortedBlob); VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob__abort(pCtx, pRepo, &pFetchHandle) ); VERIFY_COND("create_blob_from_bytes(fetch abort freed handle)", !pFetchHandle); ////////////////////////////////////////////////////////////////// // fetch blob into a new buffer and verify that it matches. VERIFY_ERR_CHECK_DISCARD( SG_repo__fetch_blob_into_memory(pCtx, pRepo,pszidHidBlob1,(SG_byte **)&pbuf2,&lenBuf2) ); VERIFY_COND("create_blob_from_bytes(fetch blob)",(lenBuf2 == (SG_uint64)lenBuf1)); ////////////////////////////////////////////////////////////////// // verify that the contents of buf-2 is identical to the contents of buf-1. // (we already know that the HIDs match and was verified during the fetch, // but there are times when the HID is just being used as a key -- it // doesn't mean that what we actually restored is correct. VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, lenBuf1, (SG_byte *)pbuf1, &pszidHidVerify1) ); VERIFY_ERR_CHECK_DISCARD( SG_repo__alloc_compute_hash__from_bytes(pCtx, pRepo, (SG_uint32)lenBuf2, (SG_byte *)pbuf2, &pszidHidVerify2) ); bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidVerify2))); VERIFY_COND("create_blob_from_bytes(verify v1==v2)",bEqual); bEqual = (0 == (strcmp(pszidHidVerify1,pszidHidBlob1))); VERIFY_COND("create_blob_from_bytes(verify v1==id)",bEqual); VERIFY_COND("creata_blob_from_bytes(memcmp)",(memcmp(pbuf1,pbuf2,lenBuf1)==0)); ////////////////////////////////////////////////////////////////// // cleanup SG_NULLFREE(pCtx, pbuf1); SG_NULLFREE(pCtx, pbuf2); SG_NULLFREE(pCtx, pszidTidRandom1); SG_NULLFREE(pCtx, pszidTidRandom2); SG_NULLFREE(pCtx, pszidHidBlob1); SG_NULLFREE(pCtx, pszidHidBlob1Dup); SG_NULLFREE(pCtx, pszidHidBlob2); SG_NULLFREE(pCtx, pszidHidVerify1); SG_NULLFREE(pCtx, pszidHidVerify2); }
void SG_repo__pack__vcdiff(SG_context* pCtx, SG_repo * pRepo) { SG_rbtree* prb_leaves = NULL; SG_uint32 count_leaves = 0; const char* psz_hid_cs = NULL; SG_rbtree* prb_blobs = NULL; SG_bool b; SG_rbtree_iterator* pit = NULL; SG_rbtree_iterator* pit_for_gid = NULL; SG_bool b_for_gid; const char* psz_hid_ref = NULL; const char* psz_hid_blob = NULL; const char* psz_gid = NULL; SG_rbtree* prb = NULL; const char* psz_gen = NULL; SG_repo_tx_handle* pTx; SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo,SG_DAGNUM__VERSION_CONTROL,&prb_leaves) ); SG_ERR_CHECK( SG_rbtree__count(pCtx, prb_leaves, &count_leaves) ); SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, NULL, prb_leaves, &b, &psz_hid_cs, NULL) ); SG_ERR_CHECK( SG_RBTREE__ALLOC(pCtx, &prb_blobs) ); SG_ERR_CHECK( sg_pack__do_changeset(pCtx, pRepo, psz_hid_cs, prb_blobs) ); SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prb_blobs, &b, &psz_gid, (void**) &prb) ); while (b) { SG_uint32 count_for_gid = 0; SG_ERR_CHECK( SG_rbtree__count(pCtx, prb, &count_for_gid) ); if (count_for_gid > 1) { psz_hid_ref = NULL; SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit_for_gid, prb, &b_for_gid, &psz_gen, (void**) &psz_hid_blob) ); while (b_for_gid) { // Not a lot of thought went into doing each of these in its own repo tx. Consider alternatives. SG_ERR_CHECK( SG_repo__begin_tx(pCtx, pRepo, &pTx) ); if (psz_hid_ref) { SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid_blob, SG_BLOBENCODING__VCDIFF, psz_hid_ref, NULL, NULL, NULL, NULL) ); // TODO be tolerant here of SG_ERR_REPO_BUSY } else { psz_hid_ref = psz_hid_blob; SG_ERR_CHECK( SG_repo__change_blob_encoding(pCtx, pRepo, pTx, psz_hid_ref, SG_BLOBENCODING__FULL, NULL, NULL, NULL, NULL, NULL) ); // TODO be tolerant here of SG_ERR_REPO_BUSY } SG_ERR_CHECK( SG_repo__commit_tx(pCtx, pRepo, &pTx) ); SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit_for_gid, &b_for_gid, &psz_gen, (void**) &psz_hid_blob) ); } SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit_for_gid); psz_hid_ref = NULL; } SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &b, &psz_gid, (void**) &prb) ); } SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit); SG_RBTREE_NULLFREE_WITH_ASSOC(pCtx, prb_blobs, _sg_repo__free_rbtree); SG_RBTREE_NULLFREE(pCtx, prb_leaves); return; fail: return; }