static void _add_dagnodes_until_connected(SG_context* pCtx,
SG_vhash** ppvhStagingStatus,
sg_pull_instance_data* pMe,
SG_client* pClient)
{
SG_bool disconnected = SG_FALSE;
SG_vhash* pvhFragballRequest = NULL;
char* pszFragballName = NULL;
SG_vhash* pvhRequestStatus = NULL;
const SG_pathname* pStagingPathname;
SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pMe->pStaging, &pStagingPathname) );
SG_ERR_CHECK( SG_vhash__has(pCtx, *ppvhStagingStatus, SG_SYNC_STATUS_KEY__DAGS, &disconnected) );
while (disconnected)
{
#if TRACE_PULL
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, *ppvhStagingStatus, "pull staging status") );
#endif
// There's at least one dag with connection problems.
// Convert the staging status vhash into a fragball request vhash.
pvhFragballRequest = *ppvhStagingStatus;
*ppvhStagingStatus = NULL;
SG_ERR_CHECK( SG_vhash__add__int64(pCtx, pvhFragballRequest, SG_SYNC_STATUS_KEY__GENERATIONS, GENERATIONS_PER_ROUNDTRIP) );
SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pClient, pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhRequestStatus) );
/* Ian TODO: inspect pvhRequestStatus */
SG_VHASH_NULLFREE(pCtx, pvhRequestStatus);
SG_VHASH_NULLFREE(pCtx, pvhFragballRequest);
SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pMe->pStaging, (const char*)pszFragballName) );
SG_NULLFREE(pCtx, pszFragballName);
SG_ERR_CHECK( SG_staging__check_status(pCtx, pMe->pStaging, SG_TRUE, SG_FALSE, SG_FALSE, SG_FALSE, SG_FALSE, ppvhStagingStatus) );
SG_ERR_CHECK( SG_vhash__has(pCtx, *ppvhStagingStatus, SG_SYNC_STATUS_KEY__DAGS, &disconnected) );
#if TRACE_PULL
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, *ppvhStagingStatus, "pull staging status") );
#endif
}
SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "done") );
/* fall through */
fail:
SG_VHASH_NULLFREE(pCtx, *ppvhStagingStatus);
SG_VHASH_NULLFREE(pCtx, pvhFragballRequest);
SG_NULLFREE(pCtx, pszFragballName);
SG_VHASH_NULLFREE(pCtx, pvhRequestStatus);
SG_ERR_IGNORE( SG_context__msg__emit(pCtx, "\n") );
}
static void _add_blobs_until_done(SG_context* pCtx,
SG_staging* pStaging,
SG_client* pClient)
{
SG_bool need_blobs = SG_FALSE;
SG_vhash* pvhFragballRequest = NULL;
char* pszFragballName = NULL;
SG_vhash* pvhRequestStatus = NULL;
const SG_pathname* pStagingPathname = NULL;
SG_vhash* pvhStagingStatus = NULL;
SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "Retrieving blobs...") );
SG_ERR_CHECK( SG_staging__check_status(pCtx, pStaging, SG_FALSE, SG_FALSE, SG_FALSE, SG_TRUE, SG_TRUE, &pvhStagingStatus) );
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhStagingStatus, SG_SYNC_STATUS_KEY__BLOBS, &need_blobs) );
if (need_blobs)
SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pStaging, &pStagingPathname) );
while (need_blobs)
{
pvhFragballRequest = pvhStagingStatus;
pvhStagingStatus = NULL;
SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pClient, pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhRequestStatus) );
/* Ian TODO: inspect pvhRequestStatus */
SG_VHASH_NULLFREE(pCtx, pvhRequestStatus);
SG_VHASH_NULLFREE(pCtx, pvhFragballRequest);
SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pStaging, (const char*)pszFragballName) );
SG_NULLFREE(pCtx, pszFragballName);
SG_ERR_CHECK( SG_staging__check_status(pCtx, pStaging, SG_FALSE, SG_FALSE, SG_FALSE, SG_TRUE, SG_TRUE, &pvhStagingStatus) );
#if TRACE_PULL
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhStagingStatus, "pull staging status") );
#endif
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhStagingStatus, SG_SYNC_STATUS_KEY__BLOBS, &need_blobs) );
}
SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "done") );
/* fall through */
fail:
SG_VHASH_NULLFREE(pCtx, pvhStagingStatus);
SG_VHASH_NULLFREE(pCtx, pvhFragballRequest);
SG_NULLFREE(pCtx, pszFragballName);
SG_VHASH_NULLFREE(pCtx, pvhRequestStatus);
SG_ERR_IGNORE( SG_context__msg__emit(pCtx, "\n") );
}
void SG_pull__commit(
SG_context* pCtx,
SG_pull** ppPull,
SG_varray** ppvaZingErr,
SG_varray** ppvaZingLog)
{
_sg_pull* pMyPull = NULL;
sg_pull_instance_data* pMe = NULL;
SG_vhash* pvhStatus = NULL;
char* pszFragballName = NULL;
const SG_pathname* pStagingPathname;
SG_NULL_PP_CHECK_RETURN(ppPull);
pMyPull = (_sg_pull*)*ppPull;
pMe = pMyPull->pPullInstance;
SG_ERR_CHECK( SG_staging__get_pathname(pCtx, pMe->pStaging, &pStagingPathname) );
SG_ERR_CHECK_RETURN( SG_context__msg__emit(pCtx, "Retrieving dagnodes...") );
SG_ERR_CHECK( SG_client__pull_request_fragball(pCtx, pMyPull->pClient, pMyPull->pvhFragballRequest, pStagingPathname, &pszFragballName, &pvhStatus) );
#if TRACE_PULL
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhStatus, "pull staging status") );
#endif
/* Ian TODO: check status of pull request */
SG_ERR_CHECK( SG_VHASH_NULLFREE(pCtx, pvhStatus) );
SG_ERR_CHECK( SG_staging__slurp_fragball(pCtx, pMe->pStaging, pszFragballName) );
SG_ERR_CHECK( SG_staging__check_status(pCtx, pMe->pStaging, SG_TRUE, SG_FALSE, SG_FALSE, SG_FALSE, SG_FALSE, &pvhStatus) );
/* Finish the pull using the status data we got back. */
SG_ERR_CHECK( _pull_common(pCtx, pMyPull->pClient, &pvhStatus, pMe, ppvaZingErr, ppvaZingLog) );
/* fall through */
fail:
SG_ERR_IGNORE( _sg_pull__nullfree(pCtx, (_sg_pull**)ppPull) );
SG_VHASH_NULLFREE(pCtx, pvhStatus);
SG_NULLFREE(pCtx, pszFragballName);
}
void SG_server__pull_request_fragball(SG_context* pCtx,
SG_repo* pRepo,
SG_vhash* pvhRequest,
const SG_pathname* pFragballDirPathname,
char** ppszFragballName,
SG_vhash** ppvhStatus)
{
SG_pathname* pFragballPathname = NULL;
SG_uint32* paDagNums = NULL;
SG_rbtree* prbDagnodes = NULL;
SG_string* pstrFragballName = NULL;
char* pszRevFullHid = NULL;
SG_rbtree_iterator* pit = NULL;
SG_uint32* repoDagnums = NULL;
SG_NULLARGCHECK_RETURN(pRepo);
SG_NULLARGCHECK_RETURN(pFragballDirPathname);
SG_NULLARGCHECK_RETURN(ppvhStatus);
#if TRACE_SERVER
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhRequest, "pull fragball request") );
#endif
SG_ERR_CHECK( SG_fragball__create(pCtx, pFragballDirPathname, &pFragballPathname) );
if (!pvhRequest)
{
// Add leaves from every dag to the fragball.
SG_uint32 count_dagnums;
SG_uint32 i;
SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_dagnums, &paDagNums) );
for (i=0; i<count_dagnums; i++)
{
SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, paDagNums[i], &prbDagnodes) );
SG_ERR_CHECK( SG_fragball__append__dagnodes(pCtx, pFragballPathname, pRepo, paDagNums[i], prbDagnodes) );
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
}
SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) );
SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) );
}
else
{
// Build the requested fragball.
SG_bool found;
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__CLONE, &found) );
if (found)
{
// Full clone requested.
SG_ERR_CHECK( SG_repo__fetch_repo__fragball(pCtx, pRepo, pFragballDirPathname, ppszFragballName) );
}
else
{
// Not a full clone.
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &found) );
if (found)
{
// Dagnodes were requested.
SG_uint32 generations = 0;
SG_vhash* pvhDags;
SG_uint32 count_requested_dagnums;
SG_uint32 count_repo_dagnums = 0;
SG_uint32 i;
const char* pszDagNum = NULL;
const SG_variant* pvRequestedNodes = NULL;
SG_vhash* pvhRequestedNodes = NULL;
const char* pszHidRequestedDagnode = NULL;
// Were additional generations requested?
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__GENERATIONS, &found) );
if (found)
SG_ERR_CHECK( SG_vhash__get__uint32(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__GENERATIONS, &generations) );
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &pvhDags) );
SG_ERR_CHECK( SG_vhash__count(pCtx, pvhDags, &count_requested_dagnums) );
if (count_requested_dagnums)
SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_repo_dagnums, &repoDagnums) );
// For each requested dag, get the requested nodes.
for (i=0; i<count_requested_dagnums; i++)
{
SG_uint32 iMissingNodeCount;
SG_uint32 iDagnum;
SG_uint32 j;
SG_bool isValidDagnum = SG_FALSE;
SG_bool bSpecificNodesRequested = SG_FALSE;
// Get the dag's missing node vhash.
SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhDags, i, &pszDagNum, &pvRequestedNodes) );
SG_ERR_CHECK( SG_dagnum__from_sz__decimal(pCtx, pszDagNum, &iDagnum) );
// Verify that requested dagnum exists
for (j = 0; j < count_repo_dagnums; j++)
{
if (repoDagnums[j] == iDagnum)
{
isValidDagnum = SG_TRUE;
break;
}
}
if (!isValidDagnum)
{
char buf[SG_DAGNUM__BUF_MAX__NAME];
SG_ERR_CHECK( SG_dagnum__to_name(pCtx, iDagnum, buf, sizeof(buf)) );
SG_ERR_THROW2(SG_ERR_NO_SUCH_DAG, (pCtx, "%s", buf));
}
if (pvRequestedNodes)
{
SG_ERR_CHECK( SG_variant__get__vhash(pCtx, pvRequestedNodes, &pvhRequestedNodes) );
// Get each node listed for the dag
SG_ERR_CHECK( SG_vhash__count(pCtx, pvhRequestedNodes, &iMissingNodeCount) );
if (iMissingNodeCount > 0)
{
SG_uint32 j;
const SG_variant* pvVal;
bSpecificNodesRequested = SG_TRUE;
SG_ERR_CHECK( SG_RBTREE__ALLOC__PARAMS(pCtx, &prbDagnodes, iMissingNodeCount, NULL) );
for (j=0; j<iMissingNodeCount; j++)
{
SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhRequestedNodes, j, &pszHidRequestedDagnode, &pvVal) );
if (pvVal)
{
const char* pszVal;
SG_ERR_CHECK( SG_variant__get__sz(pCtx, pvVal, &pszVal) );
if (pszVal)
{
if (0 == strcmp(pszVal, SG_SYNC_REQUEST_VALUE_HID_PREFIX))
{
SG_ERR_CHECK( SG_repo__hidlookup__dagnode(pCtx, pRepo, iDagnum, pszHidRequestedDagnode, &pszRevFullHid) );
pszHidRequestedDagnode = pszRevFullHid;
}
else if (0 == strcmp(pszVal, SG_SYNC_REQUEST_VALUE_TAG))
{
SG_ERR_CHECK( SG_vc_tags__lookup__tag(pCtx, pRepo, pszHidRequestedDagnode, &pszRevFullHid) );
if (!pszRevFullHid)
SG_ERR_THROW(SG_ERR_TAG_NOT_FOUND);
pszHidRequestedDagnode = pszRevFullHid;
}
else
SG_ERR_THROW(SG_ERR_PULL_INVALID_FRAGBALL_REQUEST);
}
}
SG_ERR_CHECK( SG_rbtree__update(pCtx, prbDagnodes, pszHidRequestedDagnode) );
// Get additional dagnode generations, if requested.
SG_ERR_CHECK( SG_sync__add_n_generations(pCtx, pRepo, pszHidRequestedDagnode, prbDagnodes, generations) );
SG_NULLFREE(pCtx, pszRevFullHid);
}
}
}
if (!bSpecificNodesRequested)
{
// When no specific nodes are in the request, add all leaves.
SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, iDagnum, &prbDagnodes) );
// Get additional dagnode generations, if requested.
if (generations)
{
SG_bool found;
const char* hid;
SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prbDagnodes, &found, &hid, NULL) );
while (found)
{
SG_ERR_CHECK( SG_sync__add_n_generations(pCtx, pRepo, hid, prbDagnodes, generations) );
SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &found, &hid, NULL) );
}
}
}
if (prbDagnodes) // can be null when leaves of an empty dag are requested
{
SG_ERR_CHECK( SG_fragball__append__dagnodes(pCtx, pFragballPathname, pRepo, iDagnum, prbDagnodes) );
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
}
} // dagnum loop
} // if "dags" exists
/* Add requested blobs to the fragball */
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &found) );
if (found)
{
// Blobs were requested.
SG_vhash* pvhBlobs;
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &pvhBlobs) );
SG_ERR_CHECK( SG_sync__add_blobs_to_fragball(pCtx, pRepo, pFragballPathname, pvhBlobs) );
}
SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) );
SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) );
}
}
/* fallthru */
fail:
// If we had an error, delete the half-baked fragball.
if (pFragballPathname && SG_context__has_err(pCtx))
SG_ERR_IGNORE( SG_fsobj__remove__pathname(pCtx, pFragballPathname) );
SG_PATHNAME_NULLFREE(pCtx, pFragballPathname);
SG_NULLFREE(pCtx, paDagNums);
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
SG_STRING_NULLFREE(pCtx, pstrFragballName);
SG_NULLFREE(pCtx, pszRevFullHid);
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
SG_NULLFREE(pCtx, repoDagnums);
}
/**
* Deal with the status for 1 item and accumulate
* the result in pvaStatus.
*
*/
static void _sg_wc_tx__status(SG_context * pCtx,
SG_wc_tx * pWcTx,
SG_varray * pvaStatus,
const char * pszInput,
SG_uint32 depth,
SG_bool bListUnchanged,
SG_bool bNoIgnores,
SG_bool bNoTSC,
SG_bool bListSparse,
SG_bool bListReserved,
SG_vhash ** ppvhLegend)
{
SG_string * pStringRepoPath = NULL;
sg_wc_liveview_item * pLVI; // we do not own this
SG_bool bKnown;
char chDomain;
SG_vhash * pvhCSets = NULL;
SG_vhash * pvhLegend = NULL;
const char * pszWasLabel_l = "Baseline (B)";
const char * pszWasLabel_r = "Working";
SG_NULLARGCHECK_RETURN( pWcTx );
SG_NULLARGCHECK_RETURN( pvaStatus );
// pszInput is optional -- if omitted, we assume "@/".
// ppvhLegend is optional
SG_ERR_CHECK( sg_wc_db__path__anything_to_repopath(pCtx, pWcTx->pDb, pszInput,
SG_WC_DB__PATH__IMPORT_FLAGS__TREAT_NULL_AS_ROOT,
&pStringRepoPath, &chDomain) );
#if TRACE_WC_TX_STATUS
SG_ERR_IGNORE( SG_console(pCtx, SG_CS_STDERR,
"SG_wc_tx__status: '%s' normalized to [domain %c] '%s'\n",
pszInput, chDomain, SG_string__sz(pStringRepoPath)) );
#endif
SG_ERR_CHECK( sg_wc_tx__liveview__fetch_item__domain(pCtx, pWcTx, pStringRepoPath,
&bKnown, &pLVI) );
if (!bKnown)
{
// We only get this if the path is completely bogus and
// took us off into the weeds (as opposed to reporting
// something just not-controlled).
SG_ERR_THROW2( SG_ERR_NOT_FOUND,
(pCtx, "Unknown item '%s'.", SG_string__sz(pStringRepoPath)) );
}
SG_ERR_CHECK( sg_wc_tx__rp__status__lvi(pCtx,
pWcTx,
pLVI,
depth,
bListUnchanged,
bNoIgnores,
bNoTSC,
bListSparse,
bListReserved,
pszWasLabel_l, pszWasLabel_r,
pvaStatus) );
#if TRACE_WC_TX_STATUS
SG_ERR_IGNORE( SG_console(pCtx, SG_CS_STDERR,
"SG_wc_tx__status: computed status [depth %d][bListUnchanged %d][bNoIgnores %d][bNoTSC %d][bListSparse %d][bListReserved %d] on '%s':\n",
depth,
bListUnchanged, bNoIgnores, bNoTSC, bListSparse, bListReserved,
SG_string__sz(pStringRepoPath)) );
SG_ERR_IGNORE( SG_varray_debug__dump_varray_of_vhashes_to_console(pCtx, pvaStatus, "") );
#endif
if (ppvhLegend)
{
const char * pszHid_x;
SG_ERR_CHECK( SG_wc_tx__get_wc_csets__vhash(pCtx, pWcTx, &pvhCSets) );
SG_ERR_CHECK( SG_VHASH__ALLOC(pCtx, &pvhLegend) );
SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "A", &pszHid_x) );
if (pszHid_x)
SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "A", pszHid_x) );
SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "L0", &pszHid_x) );
if (pszHid_x)
SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "B", pszHid_x) );
SG_ERR_CHECK( SG_vhash__check__sz(pCtx, pvhCSets, "L1", &pszHid_x) );
if (pszHid_x)
SG_ERR_CHECK( SG_vhash__add__string__sz(pCtx, pvhLegend, "C", pszHid_x) );
#if TRACE_WC_TX_STATUS
SG_ERR_IGNORE( SG_vhash_debug__dump_to_console__named(pCtx, pvhLegend, "Legend") );
#endif
*ppvhLegend = pvhLegend;
pvhLegend = NULL;
}
fail:
SG_STRING_NULLFREE(pCtx, pStringRepoPath);
SG_VHASH_NULLFREE(pCtx, pvhLegend);
SG_VHASH_NULLFREE(pCtx, pvhCSets);
}
void SG_sync_remote__request_fragball(
SG_context* pCtx,
SG_repo* pRepo,
const SG_pathname* pFragballDirPathname,
SG_vhash* pvhRequest,
char** ppszFragballName)
{
SG_pathname* pFragballPathname = NULL;
SG_uint64* paDagNums = NULL;
SG_string* pstrFragballName = NULL;
SG_rbtree* prbDagnodes = NULL;
SG_rbtree_iterator* pit = NULL;
SG_rev_spec* pRevSpec = NULL;
SG_stringarray* psaFullHids = NULL;
SG_rbtree* prbDagnums = NULL;
SG_dagfrag* pFrag = NULL;
char* pszRepoId = NULL;
char* pszAdminId = NULL;
SG_fragball_writer* pfb = NULL;
SG_NULLARGCHECK_RETURN(pRepo);
SG_NULLARGCHECK_RETURN(pFragballDirPathname);
{
char buf_filename[SG_TID_MAX_BUFFER_LENGTH];
SG_ERR_CHECK( SG_tid__generate(pCtx, buf_filename, sizeof(buf_filename)) );
SG_ERR_CHECK( SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pFragballPathname, pFragballDirPathname, buf_filename) );
}
if (!pvhRequest)
{
// Add leaves from every dag to the fragball.
SG_uint32 count_dagnums;
SG_uint32 i;
SG_ERR_CHECK( SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb) );
SG_ERR_CHECK( SG_repo__list_dags(pCtx, pRepo, &count_dagnums, &paDagNums) );
for (i=0; i<count_dagnums; i++)
{
SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, paDagNums[i], &prbDagnodes) );
SG_ERR_CHECK( SG_fragball__write__dagnodes(pCtx, pfb, paDagNums[i], prbDagnodes) );
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
}
SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) );
SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) );
SG_ERR_CHECK( SG_fragball_writer__close(pCtx, pfb) );
}
else
{
// Specific dags/nodes were requested. Build that fragball.
SG_bool found;
#if TRACE_SYNC_REMOTE && 0
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhRequest, "fragball request") );
#endif
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__CLONE, &found) );
if (found)
{
// SG_SYNC_STATUS_KEY__CLONE_REQUEST is currently ignored
SG_ERR_CHECK( SG_repo__fetch_repo__fragball(pCtx, pRepo, 3, pFragballDirPathname, ppszFragballName) );
}
else
{
// Not a full clone.
SG_ERR_CHECK( SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb) );
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &found) );
if (found)
{
SG_vhash* pvh_since = NULL;
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &pvh_since) );
SG_ERR_CHECK( _do_since(pCtx, pRepo, pvh_since, pfb) );
}
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &found) );
if (found)
{
// Specific Dagnodes were requested. Add just those nodes to our "start from" rbtree.
SG_vhash* pvhDags;
SG_uint32 count_requested_dagnums;
SG_uint32 i;
const SG_variant* pvRevSpecs = NULL;
SG_vhash* pvhRevSpec = NULL;
// For each requested dag, get rev spec request.
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &pvhDags) );
SG_ERR_CHECK( SG_vhash__count(pCtx, pvhDags, &count_requested_dagnums) );
if (count_requested_dagnums)
SG_ERR_CHECK( SG_repo__list_dags__rbtree(pCtx, pRepo, &prbDagnums) );
for (i=0; i<count_requested_dagnums; i++)
{
SG_bool isValidDagnum = SG_FALSE;
SG_bool bSpecificNodesRequested = SG_FALSE;
const char* pszRefDagNum = NULL;
SG_uint64 iDagnum;
// Get the dag's missing node vhash.
SG_ERR_CHECK( SG_vhash__get_nth_pair(pCtx, pvhDags, i, &pszRefDagNum, &pvRevSpecs) );
// Verify that requested dagnum exists
SG_ERR_CHECK( SG_rbtree__find(pCtx, prbDagnums, pszRefDagNum, &isValidDagnum, NULL) );
if (!isValidDagnum)
continue;
SG_ERR_CHECK( SG_dagnum__from_sz__hex(pCtx, pszRefDagNum, &iDagnum) );
if (pvRevSpecs && pvRevSpecs->type != SG_VARIANT_TYPE_NULL)
{
SG_uint32 countRevSpecs = 0;
SG_ERR_CHECK( SG_variant__get__vhash(pCtx, pvRevSpecs, &pvhRevSpec) );
SG_ERR_CHECK( SG_rev_spec__from_vash(pCtx, pvhRevSpec, &pRevSpec) );
// Process the rev spec for each dag
SG_ERR_CHECK( SG_rev_spec__count(pCtx, pRevSpec, &countRevSpecs) );
if (countRevSpecs > 0)
{
bSpecificNodesRequested = SG_TRUE;
SG_ERR_CHECK( SG_rev_spec__get_all__repo(pCtx, pRepo, pRevSpec, SG_TRUE, &psaFullHids, NULL) );
SG_ERR_CHECK( SG_stringarray__to_rbtree_keys(pCtx, psaFullHids, &prbDagnodes) );
SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids);
}
SG_REV_SPEC_NULLFREE(pCtx, pRevSpec);
}
if (!bSpecificNodesRequested)
{
// When no specific nodes are in the request, add all leaves.
SG_ERR_CHECK( SG_repo__fetch_dag_leaves(pCtx, pRepo, iDagnum, &prbDagnodes) );
}
if (prbDagnodes) // can be null when leaves of an empty dag are requested
{
// Get the leaves of the other repo, which we need to connect to.
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &found) );
if (found)
{
SG_vhash* pvhRefAllLeaves;
SG_vhash* pvhRefDagLeaves;
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &pvhRefAllLeaves) );
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, pszRefDagNum, &found) );
{
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRefAllLeaves, pszRefDagNum, &pvhRefDagLeaves) );
SG_ERR_CHECK( SG_sync__build_best_guess_dagfrag(pCtx, pRepo, iDagnum,
prbDagnodes, pvhRefDagLeaves, &pFrag) );
}
}
else
{
// The other repo's leaves weren't provided: add just the requested nodes, make no attempt to connect.
SG_ERR_CHECK( SG_repo__get_repo_id(pCtx, pRepo, &pszRepoId) );
SG_ERR_CHECK( SG_repo__get_admin_id(pCtx, pRepo, &pszAdminId) );
SG_ERR_CHECK( SG_dagfrag__alloc(pCtx, &pFrag, pszRepoId, pszAdminId, iDagnum) );
SG_ERR_CHECK( SG_dagfrag__load_from_repo__simple(pCtx, pFrag, pRepo, prbDagnodes) );
SG_NULLFREE(pCtx, pszRepoId);
SG_NULLFREE(pCtx, pszAdminId);
}
SG_ERR_CHECK( SG_fragball__write__frag(pCtx, pfb, pFrag) );
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
SG_DAGFRAG_NULLFREE(pCtx, pFrag);
}
} // dagnum loop
} // if "dags" exists
/* Add requested blobs to the fragball */
SG_ERR_CHECK( SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &found) );
if (found)
{
// Blobs were requested.
SG_vhash* pvhBlobs;
SG_ERR_CHECK( SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &pvhBlobs) );
SG_ERR_CHECK( SG_sync__add_blobs_to_fragball(pCtx, pfb, pvhBlobs) );
}
SG_ERR_CHECK( SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName) );
SG_ERR_CHECK( SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName) );
}
SG_ERR_CHECK( SG_fragball_writer__close(pCtx, pfb) );
}
/* fallthru */
fail:
// If we had an error, delete the half-baked fragball.
if (pFragballPathname && SG_context__has_err(pCtx))
{
SG_ERR_IGNORE( SG_fsobj__remove__pathname(pCtx, pFragballPathname) );
}
SG_PATHNAME_NULLFREE(pCtx, pFragballPathname);
SG_NULLFREE(pCtx, paDagNums);
SG_STRING_NULLFREE(pCtx, pstrFragballName);
SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
SG_RBTREE_NULLFREE(pCtx, prbDagnums);
SG_REV_SPEC_NULLFREE(pCtx, pRevSpec);
SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids);
SG_DAGFRAG_NULLFREE(pCtx, pFrag);
SG_NULLFREE(pCtx, pszRepoId);
SG_NULLFREE(pCtx, pszAdminId);
SG_FRAGBALL_WRITER_NULLFREE(pCtx, pfb);
}
void SG_sync__build_best_guess_dagfrag(
SG_context* pCtx,
SG_repo* pRepo,
SG_uint64 iDagNum,
SG_rbtree* prbStartFromHids,
SG_vhash* pvhConnectToHidsAndGens,
SG_dagfrag** ppFrag)
{
SG_uint32 i, countConnectTo;
SG_rbtree_iterator* pit = NULL;
SG_dagnode* pdn = NULL;
SG_dagfrag* pFrag = NULL;
SG_repo_fetch_dagnodes_handle* pdh = NULL;
SG_int32 minGen = SG_INT32_MAX;
SG_int32 maxGen = 0;
SG_uint32 gensToFetch = 0;
char* psz_repo_id = NULL;
char* psz_admin_id = NULL;
SG_bool bNextHid;
const char* pszRefHid;
SG_int32 gen;
#if TRACE_SYNC
SG_int64 startTime;
SG_int64 endTime;
#endif
SG_NULLARGCHECK_RETURN(prbStartFromHids);
/* Find the minimum generation in pertinent "connect to" nodes. */
if (pvhConnectToHidsAndGens)
{
SG_ERR_CHECK( SG_vhash__count(pCtx, pvhConnectToHidsAndGens, &countConnectTo) );
for (i = 0; i < countConnectTo; i++)
{
SG_int32 gen;
SG_ERR_CHECK( SG_vhash__get_nth_pair__int32(pCtx, pvhConnectToHidsAndGens, i, &pszRefHid, &gen) );
if (gen < minGen)
minGen = gen;
}
}
/* Happens when pulling into an empty repo, or when an entire dag is specifically requested. */
if (minGen == SG_INT32_MAX)
minGen = -1;
SG_ERR_CHECK( SG_repo__fetch_dagnodes__begin(pCtx, pRepo, iDagNum, &pdh) );
/* Find the maximum generation in pertinent "start from" nodes. */
SG_ERR_CHECK( SG_rbtree__iterator__first(pCtx, &pit, prbStartFromHids, &bNextHid, &pszRefHid, NULL) );
while (bNextHid)
{
SG_ERR_CHECK( SG_repo__fetch_dagnodes__one(pCtx, pRepo, pdh, pszRefHid, &pdn) );
SG_ERR_CHECK( SG_dagnode__get_generation(pCtx, pdn, &gen) );
if (gen > maxGen)
maxGen = gen;
SG_DAGNODE_NULLFREE(pCtx, pdn);
SG_ERR_CHECK( SG_rbtree__iterator__next(pCtx, pit, &bNextHid, &pszRefHid, NULL) );
}
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
if (maxGen <= minGen)
gensToFetch = FALLBACK_GENS_PER_ROUNDTRIP;
else
gensToFetch = maxGen - minGen;
#if TRACE_SYNC
{
char buf_dagnum[SG_DAGNUM__BUF_MAX__HEX];
SG_uint32 count;
SG_ERR_CHECK( SG_dagnum__to_sz__hex(pCtx, iDagNum, buf_dagnum, sizeof(buf_dagnum)) );
SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "Building best guess dagfrag for dag %s...\n", buf_dagnum) );
SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "Starting from nodes:\n") );
SG_ERR_CHECK( SG_rbtree_debug__dump_keys_to_console(pCtx, prbStartFromHids) );
SG_ERR_CHECK( SG_vhash_debug__dump_to_console__named(pCtx, pvhConnectToHidsAndGens, "Connecting to nodes") );
SG_ERR_CHECK( SG_rbtree__count(pCtx, prbStartFromHids, &count) );
SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, "result has %d generations from %u starting nodes.\n",
gensToFetch, count) );
SG_ERR_CHECK( SG_console__flush(pCtx, SG_CS_STDERR) );
SG_ERR_CHECK( SG_time__get_milliseconds_since_1970_utc(pCtx, &startTime) );
}
#endif
/* Return a frag with the corresponding generations filled in. */
SG_ERR_CHECK( SG_repo__get_repo_id(pCtx, pRepo, &psz_repo_id) );
SG_ERR_CHECK( SG_repo__get_admin_id(pCtx, pRepo, &psz_admin_id) );
SG_ERR_CHECK( SG_dagfrag__alloc(pCtx, &pFrag, psz_repo_id, psz_admin_id, iDagNum) );
SG_ERR_CHECK( SG_dagfrag__load_from_repo__multi(pCtx, pFrag, pRepo, prbStartFromHids, gensToFetch) );
#if TRACE_SYNC
SG_ERR_CHECK( SG_time__get_milliseconds_since_1970_utc(pCtx, &endTime) );
{
SG_uint32 dagnodeCount;
double seconds = ((double)endTime-(double)startTime)/1000;
SG_ERR_CHECK( SG_dagfrag__dagnode_count(pCtx, pFrag, &dagnodeCount) );
SG_ERR_CHECK( SG_console(pCtx, SG_CS_STDERR, " - %u nodes in frag, built in %1.3f seconds\n", dagnodeCount, seconds) );
SG_ERR_CHECK( SG_dagfrag_debug__dump__console(pCtx, pFrag, "best-guess dagfrag", 0, SG_CS_STDERR) );
}
#endif
*ppFrag = pFrag;
pFrag = NULL;
/* Common cleanup */
fail:
SG_NULLFREE(pCtx, psz_repo_id);
SG_NULLFREE(pCtx, psz_admin_id);
SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
SG_DAGNODE_NULLFREE(pCtx, pdn);
SG_DAGFRAG_NULLFREE(pCtx, pFrag);
SG_ERR_IGNORE( SG_repo__fetch_dagnodes__end(pCtx, pRepo, &pdh) );
}
