static void _add_dagnodes_until_connected(SG_context* pCtx, SG_vhash** ppvhStagingStatus, sg_pull_instance_data* pMe, SG_client* pClient) { SG_bool disconnected = SG_FALSE; SG_vhash* pvhFragballRequest = NULL; char* pszFragballName = NULL; SG_vhash* pvhRequestStatus = NULL; const SG_pathname* pStagingPathname; SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pMe->pStaging, &pStagingPathname) ); SG_ERR_CHECK( SG_vhash__has(pCtx, *ppvhStagingStatus, SG_SYNC_STATUS_KEY__DAGS, &disconnected) ); while (disconnected) { #if TRACE_PULL SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, *ppvhStagingStatus, "pull staging status") ); #endif // There's at least one dag with connection problems. // Convert the staging status vhash into a fragball request vhash. pvhFragballRequest = *ppvhStagingStatus; *ppvhStagingStatus = NULL; SG_ERR_CHECK( SG_vhash__add__int64(pCtx, pvhFragballRequest, SG_SYNC_STATUS_KEY__GENERATIONS, GENERATIONS_PER_ROUNDTRIP) ); SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pClient, pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhRequestStatus) ); /* Ian TODO: inspect pvhRequestStatus */ SG_VHASH_NULLFREE(pCtx, pvhRequestStatus); SG_VHASH_NULLFREE(pCtx, pvhFragballRequest); SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pMe->pStaging, (const char*)pszFragballName) ); SG_NULLFREE(pCtx, pszFragballName); SG_ERR_CHECK( SG_staging__check_status(pCtx, pMe->pStaging, SG_TRUE, SG_FALSE, SG_FALSE, SG_FALSE, SG_FALSE, ppvhStagingStatus) ); SG_ERR_CHECK( SG_vhash__has(pCtx, *ppvhStagingStatus, SG_SYNC_STATUS_KEY__DAGS, &disconnected) ); #if TRACE_PULL SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, *ppvhStagingStatus, "pull staging status") ); #endif } SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "done") ); /* fall through */ fail: SG_VHASH_NULLFREE(pCtx, *ppvhStagingStatus); SG_VHASH_NULLFREE(pCtx, pvhFragballRequest); SG_NULLFREE(pCtx, pszFragballName); SG_VHASH_NULLFREE(pCtx, pvhRequestStatus); SG_ERR_IGNORE( SG_context__msg__emit(pCtx, "\n") ); }
static void _add_blobs_until_done(SG_context* pCtx, SG_staging* pStaging, SG_client* pClient) { SG_bool need_blobs = SG_FALSE; SG_vhash* pvhFragballRequest = NULL; char* pszFragballName = NULL; SG_vhash* pvhRequestStatus = NULL; const SG_pathname* pStagingPathname = NULL; SG_vhash* pvhStagingStatus = NULL; SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "Retrieving blobs...") ); SG_ERR_CHECK( SG_staging__check_status(pCtx, pStaging, SG_FALSE, SG_FALSE, SG_FALSE, SG_TRUE, SG_TRUE, &pvhStagingStatus) ); SG_ERR_CHECK( SG_vhash__has(pCtx, pvhStagingStatus, SG_SYNC_STATUS_KEY__BLOBS, &need_blobs) ); if (need_blobs) SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pStaging, &pStagingPathname) ); while (need_blobs) { pvhFragballRequest = pvhStagingStatus; pvhStagingStatus = NULL; SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pClient, pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhRequestStatus) ); /* Ian TODO: inspect pvhRequestStatus */ SG_VHASH_NULLFREE(pCtx, pvhRequestStatus); SG_VHASH_NULLFREE(pCtx, pvhFragballRequest); SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pStaging, (const char*)pszFragballName) ); SG_NULLFREE(pCtx, pszFragballName); SG_ERR_CHECK( SG_staging__check_status(pCtx, pStaging, SG_FALSE, SG_FALSE, SG_FALSE, SG_TRUE, SG_TRUE, &pvhStagingStatus) ); #if TRACE_PULL SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhStagingStatus, "pull staging status") ); #endif SG_ERR_CHECK( SG_vhash__has(pCtx, pvhStagingStatus, SG_SYNC_STATUS_KEY__BLOBS, &need_blobs) ); } SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "done") ); /* fall through */ fail: SG_VHASH_NULLFREE(pCtx, pvhStagingStatus); SG_VHASH_NULLFREE(pCtx, pvhFragballRequest); SG_NULLFREE(pCtx, pszFragballName); SG_VHASH_NULLFREE(pCtx, pvhRequestStatus); SG_ERR_IGNORE( SG_context__msg__emit(pCtx, "\n") ); }
void SG_pull__commit( SG_context* pCtx, SG_pull** ppPull, SG_varray** ppvaZingErr, SG_varray** ppvaZingLog) { _sg_pull* pMyPull = NULL; sg_pull_instance_data* pMe = NULL; SG_vhash* pvhStatus = NULL; char* pszFragballName = NULL; const SG_pathname* pStagingPathname; SG_NULL_PP_CHECK_RETURN(ppPull); pMyPull = (_sg_pull*)*ppPull; pMe = pMyPull->pPullInstance; SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pMe->pStaging, &pStagingPathname) ); SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "Retrieving dagnodes...") ); SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pMyPull->pClient, pMyPull->pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhStatus) ); #if TRACE_PULL SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhStatus, "pull staging status") ); #endif /* Ian TODO: check status of pull request */ SG_ERR_CHECK( SG_VHASH_NULLFREE(pCtx, pvhStatus) ); SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pMe->pStaging, pszFragballName) ); SG_ERR_CHECK( SG_staging__check_status(pCtx, pMe->pStaging, SG_TRUE, SG_FALSE, SG_FALSE, SG_FALSE, SG_FALSE, &pvhStatus) ); /* Finish the pull using the status data we got back. */ SG_ERR_CHECK( _pull_common(pCtx, pMyPull->pClient, &pvhStatus, pMe, ppvaZingErr, ppvaZingLog) ); /* fall through */ fail: SG_ERR_IGNORE( _sg_pull__nullfree(pCtx, (_sg_pull**)ppPull) ); SG_VHASH_NULLFREE(pCtx, pvhStatus); SG_NULLFREE(pCtx, pszFragballName); }
void SG_server__pull_request_fragball(SG_context* pCtx, SG_repo* pRepo, SG_vhash* pvhRequest, const SG_pathname* pFragballDirPathname, char** ppszFragballName, SG_vhash** ppvhStatus) { SG_pathname* pFragballPathname = NULL; SG_uint32* paDagNums = NULL; SG_rbtree* prbDagnodes = NULL; SG_string* pstrFragballName = NULL; char* pszRevFullHid = NULL; SG_rbtree_iterator* pit = NULL; SG_uint32* repoDagnums = NULL; SG_NULLARGCHECK_RETURN(pRepo); SG_NULLARGCHECK_RETURN(pFragballDirPathname); SG_NULLARGCHECK_RETURN(ppvhStatus); #if TRACE_SERVER SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhRequest, "pull fragball request") ); #endif SG_ERR_CHECK( SG_fragball__create(pCtx, pFragballDirPathname, &pFragballPathname) ); if (!pvhRequest) { // Add leaves from every dag to the fragball. SG_uint32 count_dagnums; SG_uint32 i; SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_dagnums, &paDagNums) ); for (i=0; i<count_dagnums; i++) { SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, paDagNums[i], &prbDagnodes) ); SG_ERR_CHECK( SG_fragball__append__dagnodes(pCtx, pFragballPathname, pRepo, paDagNums[i], prbDagnodes) ); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); } SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) ); SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) ); } else { // Build the requested fragball. SG_bool found; SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__CLONE, &found) ); if (found) { // Full clone requested. SG_ERR_CHECK( SG_repo__fetch_repo__fragball(pCtx, pRepo, pFragballDirPathname, ppszFragballName) ); } else { // Not a full clone. SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &found) ); if (found) { // Dagnodes were requested. SG_uint32 generations = 0; SG_vhash* pvhDags; SG_uint32 count_requested_dagnums; SG_uint32 count_repo_dagnums = 0; SG_uint32 i; const char* pszDagNum = NULL; const SG_variant* pvRequestedNodes = NULL; SG_vhash* pvhRequestedNodes = NULL; const char* pszHidRequestedDagnode = NULL; // Were additional generations requested? SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__GENERATIONS, &found) ); if (found) SG_ERR_CHECK( SG_vhash__get__uint32(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__GENERATIONS, &generations) ); SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &pvhDags) ); SG_ERR_CHECK( SG_vhash__count(pCtx, pvhDags, &count_requested_dagnums) ); if (count_requested_dagnums) SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_repo_dagnums, &repoDagnums) ); // For each requested dag, get the requested nodes. for (i=0; i<count_requested_dagnums; i++) { SG_uint32 iMissingNodeCount; SG_uint32 iDagnum; SG_uint32 j; SG_bool isValidDagnum = SG_FALSE; SG_bool bSpecificNodesRequested = SG_FALSE; // Get the dag's missing node vhash. SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhDags, i, &pszDagNum, &pvRequestedNodes) ); SG_ERR_CHECK( SG_dagnum__from_sz__decimal(pCtx, pszDagNum, &iDagnum) ); // Verify that requested dagnum exists for (j = 0; j < count_repo_dagnums; j++) { if (repoDagnums[j] == iDagnum) { isValidDagnum = SG_TRUE; break; } } if (!isValidDagnum) { char buf[SG_DAGNUM__BUF_MAX__NAME]; SG_ERR_CHECK( SG_dagnum__to_name(pCtx, iDagnum, buf, sizeof(buf)) ); SG_ERR_THROW2(SG_ERR_NO_SUCH_DAG, (pCtx, "%s", buf)); } if (pvRequestedNodes) { SG_ERR_CHECK( SG_variant__get__vhash(pCtx, pvRequestedNodes, &pvhRequestedNodes) ); // Get each node listed for the dag SG_ERR_CHECK( SG_vhash__count(pCtx, pvhRequestedNodes, &iMissingNodeCount) ); if (iMissingNodeCount > 0) { SG_uint32 j; const SG_variant* pvVal; bSpecificNodesRequested = SG_TRUE; SG_ERR_CHECK( SG_RBTREE__ALLOC__PARAMS(pCtx, &prbDagnodes, iMissingNodeCount, NULL) ); for (j=0; j<iMissingNodeCount; j++) { SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhRequestedNodes, j, &pszHidRequestedDagnode, &pvVal) ); if (pvVal) { const char* pszVal; SG_ERR_CHECK( SG_variant__get__sz(pCtx, pvVal, &pszVal) ); if (pszVal) { if (0 == strcmp(pszVal, SG_SYNC_REQUEST_VALUE_HID_PREFIX)) { SG_ERR_CHECK( SG_repo__hidlookup__dagnode(pCtx, pRepo, iDagnum, pszHidRequestedDagnode, &pszRevFullHid) ); pszHidRequestedDagnode = pszRevFullHid; } else if (0 == strcmp(pszVal, SG_SYNC_REQUEST_VALUE_TAG)) { SG_ERR_CHECK( SG_vc_tags__lookup__tag(pCtx, pRepo, pszHidRequestedDagnode, &pszRevFullHid) ); if (!pszRevFullHid) SG_ERR_THROW(SG_ERR_TAG_NOT_FOUND); pszHidRequestedDagnode = pszRevFullHid; } else SG_ERR_THROW(SG_ERR_PULL_INVALID_FRAGBALL_REQUEST); } } SG_ERR_CHECK( SG_rbtree__update(pCtx, prbDagnodes, pszHidRequestedDagnode) ); // Get additional dagnode generations, if requested. SG_ERR_CHECK( SG_sync__add_n_generations(pCtx, pRepo, pszHidRequestedDagnode, prbDagnodes, generations) ); SG_NULLFREE(pCtx, pszRevFullHid); } } } if (!bSpecificNodesRequested) { // When no specific nodes are in the request, add all leaves. SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, iDagnum, &prbDagnodes) ); // Get additional dagnode generations, if requested. if (generations) { SG_bool found; const char* hid; SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prbDagnodes, &found, &hid, NULL) ); while (found) { SG_ERR_CHECK( SG_sync__add_n_generations(pCtx, pRepo, hid, prbDagnodes, generations) ); SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &found, &hid, NULL) ); } } } if (prbDagnodes) // can be null when leaves of an empty dag are requested { SG_ERR_CHECK( SG_fragball__append__dagnodes(pCtx, pFragballPathname, pRepo, iDagnum, prbDagnodes) ); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); } } // dagnum loop } // if "dags" exists /* Add requested blobs to the fragball */ SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &found) ); if (found) { // Blobs were requested. SG_vhash* pvhBlobs; SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &pvhBlobs) ); SG_ERR_CHECK( SG_sync__add_blobs_to_fragball(pCtx, pRepo, pFragballPathname, pvhBlobs) ); } SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) ); SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) ); } } /* fallthru */ fail: // If we had an error, delete the half-baked fragball. if (pFragballPathname && SG_context__has_err(pCtx)) SG_ERR_IGNORE( SG_fsobj__remove__pathname(pCtx, pFragballPathname) ); SG_PATHNAME_NULLFREE(pCtx, pFragballPathname); SG_NULLFREE(pCtx, paDagNums); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); SG_STRING_NULLFREE(pCtx, pstrFragballName); SG_NULLFREE(pCtx, pszRevFullHid); SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit); SG_NULLFREE(pCtx, repoDagnums); }
/** * Deal with the status for 1 item and accumulate * the result in pvaStatus. * */ static void _sg_wc_tx__status(SG_context * pCtx, SG_wc_tx * pWcTx, SG_varray * pvaStatus, const char * pszInput, SG_uint32 depth, SG_bool bListUnchanged, SG_bool bNoIgnores, SG_bool bNoTSC, SG_bool bListSparse, SG_bool bListReserved, SG_vhash ** ppvhLegend) { SG_string * pStringRepoPath = NULL; sg_wc_liveview_item * pLVI; // we do not own this SG_bool bKnown; char chDomain; SG_vhash * pvhCSets = NULL; SG_vhash * pvhLegend = NULL; const char * pszWasLabel_l = "Baseline (B)"; const char * pszWasLabel_r = "Working"; SG_NULLARGCHECK_RETURN( pWcTx ); SG_NULLARGCHECK_RETURN( pvaStatus ); // pszInput is optional -- if omitted, we assume "@/". // ppvhLegend is optional SG_ERR_CHECK( sg_wc_db__path__anything_to_repopath(pCtx, pWcTx->pDb, pszInput, SG_WC_DB__PATH__IMPORT_FLAGS__TREAT_NULL_AS_ROOT, &pStringRepoPath, &chDomain) ); #if TRACE_WC_TX_STATUS SG_ERR_IGNORE( SG_console(pCtx, SG_CS_STDERR, "SG_wc_tx__status: '%s' normalized to [domain %c] '%s'\n", pszInput, chDomain, SG_string__sz(pStringRepoPath)) ); #endif SG_ERR_CHECK( sg_wc_tx__liveview__fetch_item__domain(pCtx, pWcTx, pStringRepoPath, &bKnown, &pLVI) ); if (!bKnown) { // We only get this if the path is completely bogus and // took us off into the weeds (as opposed to reporting // something just not-controlled). SG_ERR_THROW2( SG_ERR_NOT_FOUND, (pCtx, "Unknown item '%s'.", SG_string__sz(pStringRepoPath)) ); } SG_ERR_CHECK( sg_wc_tx__rp__status__lvi(pCtx, pWcTx, pLVI, depth, bListUnchanged, bNoIgnores, bNoTSC, bListSparse, bListReserved, pszWasLabel_l, pszWasLabel_r, pvaStatus) ); #if TRACE_WC_TX_STATUS SG_ERR_IGNORE( SG_console(pCtx, SG_CS_STDERR, "SG_wc_tx__status: computed status [depth %d][bListUnchanged %d][bNoIgnores %d][bNoTSC %d][bListSparse %d][bListReserved %d] on '%s':\n", depth, bListUnchanged, bNoIgnores, bNoTSC, bListSparse, bListReserved, SG_string__sz(pStringRepoPath)) ); SG_ERR_IGNORE( SG_varray_debug__dump_varray_of_vhashes_to_console(pCtx, pvaStatus, "") ); #endif if (ppvhLegend) { const char * pszHid_x; SG_ERR_CHECK( SG_wc_tx__get_wc_csets__vhash(pCtx, pWcTx, &pvhCSets) ); SG_ERR_CHECK( SG_VHASH__ALLOC(pCtx, &pvhLegend) ); SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "A", &pszHid_x) ); if (pszHid_x) SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "A", pszHid_x) ); SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "L0", &pszHid_x) ); if (pszHid_x) SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "B", pszHid_x) ); SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "L1", &pszHid_x) ); if (pszHid_x) SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "C", pszHid_x) ); #if TRACE_WC_TX_STATUS SG_ERR_IGNORE( SG_vhash_debug__dump_to_console__named(pCtx, pvhLegend, "Legend") ); #endif *ppvhLegend = pvhLegend; pvhLegend = NULL; } fail: SG_STRING_NULLFREE(pCtx, pStringRepoPath); SG_VHASH_NULLFREE(pCtx, pvhLegend); SG_VHASH_NULLFREE(pCtx, pvhCSets); }
void SG_sync_remote__request_fragball( SG_context* pCtx, SG_repo* pRepo, const SG_pathname* pFragballDirPathname, SG_vhash* pvhRequest, char** ppszFragballName) { SG_pathname* pFragballPathname = NULL; SG_uint64* paDagNums = NULL; SG_string* pstrFragballName = NULL; SG_rbtree* prbDagnodes = NULL; SG_rbtree_iterator* pit = NULL; SG_rev_spec* pRevSpec = NULL; SG_stringarray* psaFullHids = NULL; SG_rbtree* prbDagnums = NULL; SG_dagfrag* pFrag = NULL; char* pszRepoId = NULL; char* pszAdminId = NULL; SG_fragball_writer* pfb = NULL; SG_NULLARGCHECK_RETURN(pRepo); SG_NULLARGCHECK_RETURN(pFragballDirPathname); { char buf_filename[SG_TID_MAX_BUFFER_LENGTH]; SG_ERR_CHECK( SG_tid__generate(pCtx, buf_filename, sizeof(buf_filename)) ); SG_ERR_CHECK( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pFragballPathname, pFragballDirPathname, buf_filename) ); } if (!pvhRequest) { // Add leaves from every dag to the fragball. SG_uint32 count_dagnums; SG_uint32 i; SG_ERR_CHECK( SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb) ); SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_dagnums, &paDagNums) ); for (i=0; i<count_dagnums; i++) { SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, paDagNums[i], &prbDagnodes) ); SG_ERR_CHECK( SG_fragball__write__dagnodes(pCtx, pfb, paDagNums[i], prbDagnodes) ); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); } SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) ); SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) ); SG_ERR_CHECK( SG_fragball_writer__close(pCtx, pfb) ); } else { // Specific dags/nodes were requested. Build that fragball. SG_bool found; #if TRACE_SYNC_REMOTE && 0 SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhRequest, "fragball request") ); #endif SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__CLONE, &found) ); if (found) { // SG_SYNC_STATUS_KEY__CLONE_REQUEST is currently ignored SG_ERR_CHECK( SG_repo__fetch_repo__fragball(pCtx, pRepo, 3, pFragballDirPathname, ppszFragballName) ); } else { // Not a full clone. SG_ERR_CHECK( SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb) ); SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &found) ); if (found) { SG_vhash* pvh_since = NULL; SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &pvh_since) ); SG_ERR_CHECK( _do_since(pCtx, pRepo, pvh_since, pfb) ); } SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &found) ); if (found) { // Specific Dagnodes were requested. Add just those nodes to our "start from" rbtree. SG_vhash* pvhDags; SG_uint32 count_requested_dagnums; SG_uint32 i; const SG_variant* pvRevSpecs = NULL; SG_vhash* pvhRevSpec = NULL; // For each requested dag, get rev spec request. SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &pvhDags) ); SG_ERR_CHECK( SG_vhash__count(pCtx, pvhDags, &count_requested_dagnums) ); if (count_requested_dagnums) SG_ERR_CHECK( SG_repo__list_dags__rbtree(pCtx, pRepo, &prbDagnums) ); for (i=0; i<count_requested_dagnums; i++) { SG_bool isValidDagnum = SG_FALSE; SG_bool bSpecificNodesRequested = SG_FALSE; const char* pszRefDagNum = NULL; SG_uint64 iDagnum; // Get the dag's missing node vhash. SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhDags, i, &pszRefDagNum, &pvRevSpecs) ); // Verify that requested dagnum exists SG_ERR_CHECK( SG_rbtree__find(pCtx, prbDagnums, pszRefDagNum, &isValidDagnum, NULL) ); if (!isValidDagnum) continue; SG_ERR_CHECK( SG_dagnum__from_sz__hex(pCtx, pszRefDagNum, &iDagnum) ); if (pvRevSpecs && pvRevSpecs->type != SG_VARIANT_TYPE_NULL) { SG_uint32 countRevSpecs = 0; SG_ERR_CHECK( SG_variant__get__vhash(pCtx, pvRevSpecs, &pvhRevSpec) ); SG_ERR_CHECK( SG_rev_spec__from_vash(pCtx, pvhRevSpec, &pRevSpec) ); // Process the rev spec for each dag SG_ERR_CHECK( SG_rev_spec__count(pCtx, pRevSpec, &countRevSpecs) ); if (countRevSpecs > 0) { bSpecificNodesRequested = SG_TRUE; SG_ERR_CHECK( SG_rev_spec__get_all__repo(pCtx, pRepo, pRevSpec, SG_TRUE, &psaFullHids, NULL) ); SG_ERR_CHECK( SG_stringarray__to_rbtree_keys(pCtx, psaFullHids, &prbDagnodes) ); SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids); } SG_REV_SPEC_NULLFREE(pCtx, pRevSpec); } if (!bSpecificNodesRequested) { // When no specific nodes are in the request, add all leaves. SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, iDagnum, &prbDagnodes) ); } if (prbDagnodes) // can be null when leaves of an empty dag are requested { // Get the leaves of the other repo, which we need to connect to. SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &found) ); if (found) { SG_vhash* pvhRefAllLeaves; SG_vhash* pvhRefDagLeaves; SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &pvhRefAllLeaves) ); SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, pszRefDagNum, &found) ); { SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRefAllLeaves, pszRefDagNum, &pvhRefDagLeaves) ); SG_ERR_CHECK( SG_sync__build_best_guess_dagfrag(pCtx, pRepo, iDagnum, prbDagnodes, pvhRefDagLeaves, &pFrag) ); } } else { // The other repo's leaves weren't provided: add just the requested nodes, make no attempt to connect. SG_ERR_CHECK( SG_repo__get_repo_id(pCtx, pRepo, &pszRepoId) ); SG_ERR_CHECK( SG_repo__get_admin_id(pCtx, pRepo, &pszAdminId) ); SG_ERR_CHECK( SG_dagfrag__alloc(pCtx, &pFrag, pszRepoId, pszAdminId, iDagnum) ); SG_ERR_CHECK( SG_dagfrag__load_from_repo__simple(pCtx, pFrag, pRepo, prbDagnodes) ); SG_NULLFREE(pCtx, pszRepoId); SG_NULLFREE(pCtx, pszAdminId); } SG_ERR_CHECK( SG_fragball__write__frag(pCtx, pfb, pFrag) ); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); SG_DAGFRAG_NULLFREE(pCtx, pFrag); } } // dagnum loop } // if "dags" exists /* Add requested blobs to the fragball */ SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &found) ); if (found) { // Blobs were requested. SG_vhash* pvhBlobs; SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &pvhBlobs) ); SG_ERR_CHECK( SG_sync__add_blobs_to_fragball(pCtx, pfb, pvhBlobs) ); } SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) ); SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) ); } SG_ERR_CHECK( SG_fragball_writer__close(pCtx, pfb) ); } /* fallthru */ fail: // If we had an error, delete the half-baked fragball. if (pFragballPathname && SG_context__has_err(pCtx)) { SG_ERR_IGNORE( SG_fsobj__remove__pathname(pCtx, pFragballPathname) ); } SG_PATHNAME_NULLFREE(pCtx, pFragballPathname); SG_NULLFREE(pCtx, paDagNums); SG_STRING_NULLFREE(pCtx, pstrFragballName); SG_RBTREE_NULLFREE(pCtx, prbDagnodes); SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit); SG_RBTREE_NULLFREE(pCtx, prbDagnums); SG_REV_SPEC_NULLFREE(pCtx, pRevSpec); SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids); SG_DAGFRAG_NULLFREE(pCtx, pFrag); SG_NULLFREE(pCtx, pszRepoId); SG_NULLFREE(pCtx, pszAdminId); SG_FRAGBALL_WRITER_NULLFREE(pCtx, pfb); }
void SG_sync__build_best_guess_dagfrag( SG_context* pCtx, SG_repo* pRepo, SG_uint64 iDagNum, SG_rbtree* prbStartFromHids, SG_vhash* pvhConnectToHidsAndGens, SG_dagfrag** ppFrag) { SG_uint32 i, countConnectTo; SG_rbtree_iterator* pit = NULL; SG_dagnode* pdn = NULL; SG_dagfrag* pFrag = NULL; SG_repo_fetch_dagnodes_handle* pdh = NULL; SG_int32 minGen = SG_INT32_MAX; SG_int32 maxGen = 0; SG_uint32 gensToFetch = 0; char* psz_repo_id = NULL; char* psz_admin_id = NULL; SG_bool bNextHid; const char* pszRefHid; SG_int32 gen; #if TRACE_SYNC SG_int64 startTime; SG_int64 endTime; #endif SG_NULLARGCHECK_RETURN(prbStartFromHids); /* Find the minimum generation in pertinent "connect to" nodes. */ if (pvhConnectToHidsAndGens) { SG_ERR_CHECK( SG_vhash__count(pCtx, pvhConnectToHidsAndGens, &countConnectTo) ); for (i = 0; i < countConnectTo; i++) { SG_int32 gen; SG_ERR_CHECK( SG_vhash__get_nth_pair__int32(pCtx, pvhConnectToHidsAndGens, i, &pszRefHid, &gen) ); if (gen < minGen) minGen = gen; } } /* Happens when pulling into an empty repo, or when an entire dag is specifically requested. */ if (minGen == SG_INT32_MAX) minGen = -1; SG_ERR_CHECK( SG_repo__fetch_dagnodes__begin(pCtx, pRepo, iDagNum, &pdh) ); /* Find the maximum generation in pertinent "start from" nodes. */ SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prbStartFromHids, &bNextHid, &pszRefHid, NULL) ); while (bNextHid) { SG_ERR_CHECK( SG_repo__fetch_dagnodes__one(pCtx, pRepo, pdh, pszRefHid, &pdn) ); SG_ERR_CHECK( SG_dagnode__get_generation(pCtx, pdn, &gen) ); if (gen > maxGen) maxGen = gen; SG_DAGNODE_NULLFREE(pCtx, pdn); SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &bNextHid, &pszRefHid, NULL) ); } SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit); if (maxGen <= minGen) gensToFetch = FALLBACK_GENS_PER_ROUNDTRIP; else gensToFetch = maxGen - minGen; #if TRACE_SYNC { char buf_dagnum[SG_DAGNUM__BUF_MAX__HEX]; SG_uint32 count; SG_ERR_CHECK( SG_dagnum__to_sz__hex(pCtx, iDagNum, buf_dagnum, sizeof(buf_dagnum)) ); SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "Building best guess dagfrag for dag %s...\n", buf_dagnum) ); SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "Starting from nodes:\n") ); SG_ERR_CHECK( SG_rbtree_debug__dump_keys_to_console(pCtx, prbStartFromHids) ); SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhConnectToHidsAndGens, "Connecting to nodes") ); SG_ERR_CHECK( SG_rbtree__count(pCtx, prbStartFromHids, &count) ); SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "result has %d generations from %u starting nodes.\n", gensToFetch, count) ); SG_ERR_CHECK( SG_console__flush(pCtx, SG_CS_STDERR) ); SG_ERR_CHECK( SG_time__get_milliseconds_since_1970_utc(pCtx, &startTime) ); } #endif /* Return a frag with the corresponding generations filled in. */ SG_ERR_CHECK( SG_repo__get_repo_id(pCtx, pRepo, &psz_repo_id) ); SG_ERR_CHECK( SG_repo__get_admin_id(pCtx, pRepo, &psz_admin_id) ); SG_ERR_CHECK( SG_dagfrag__alloc(pCtx, &pFrag, psz_repo_id, psz_admin_id, iDagNum) ); SG_ERR_CHECK( SG_dagfrag__load_from_repo__multi(pCtx, pFrag, pRepo, prbStartFromHids, gensToFetch) ); #if TRACE_SYNC SG_ERR_CHECK( SG_time__get_milliseconds_since_1970_utc(pCtx, &endTime) ); { SG_uint32 dagnodeCount; double seconds = ((double)endTime-(double)startTime)/1000; SG_ERR_CHECK( SG_dagfrag__dagnode_count(pCtx, pFrag, &dagnodeCount) ); SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, " - %u nodes in frag, built in %1.3f seconds\n", dagnodeCount, seconds) ); SG_ERR_CHECK( SG_dagfrag_debug__dump__console(pCtx, pFrag, "best-guess dagfrag", 0, SG_CS_STDERR) ); } #endif *ppFrag = pFrag; pFrag = NULL; /* Common cleanup */ fail: SG_NULLFREE(pCtx, psz_repo_id); SG_NULLFREE(pCtx, psz_admin_id); SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit); SG_DAGNODE_NULLFREE(pCtx, pdn); SG_DAGFRAG_NULLFREE(pCtx, pFrag); SG_ERR_IGNORE( SG_repo__fetch_dagnodes__end(pCtx, pRepo, &pdh) ); }