Пример #1
0
void SG_dagfrag__alloc__from_vhash(SG_context * pCtx,
								   SG_dagfrag ** ppNew,
								   const SG_vhash * pvhFrag)
{
	const char * szVersion;
	SG_dagfrag * pFrag = NULL;
	struct _deserialize_data deserialize_data;
    SG_int64 iDagNum = 0;

	SG_NULLARGCHECK_RETURN(ppNew);
	SG_NULLARGCHECK_RETURN(pvhFrag);

	SG_ERR_CHECK(  SG_vhash__get__sz(pCtx,pvhFrag,KEY_VERSION,&szVersion)  );
	if (strcmp(szVersion,"1") == 0)
	{
		// handle dagfrags that were serialized by software compiled with
		// VALUE_VERSION == 1.

		SG_varray * pvaMyData;
        const char* psz_repo_id = NULL;
        const char* psz_admin_id = NULL;

        SG_ERR_CHECK(  SG_vhash__get__sz(pCtx,pvhFrag,KEY_REPO_ID,&psz_repo_id)  );
        SG_ERR_CHECK(  SG_vhash__get__sz(pCtx,pvhFrag,KEY_ADMIN_ID,&psz_admin_id)  );
        SG_ERR_CHECK(  SG_vhash__get__int64(pCtx,pvhFrag,KEY_DAGNUM,&iDagNum)  );

        SG_ERR_CHECK(  SG_dagfrag__alloc(pCtx,&pFrag,psz_repo_id,psz_admin_id,(SG_uint32) iDagNum)  );

		SG_ERR_CHECK(  SG_vhash__get__varray(pCtx,pvhFrag,KEY_DATA,&pvaMyData)  );

		deserialize_data.pFrag = pFrag;
		SG_ERR_CHECK(  SG_varray__foreach(pCtx,
										  pvaMyData,
										  _deserialize_data_ver_1_cb,
										  &deserialize_data)  );

		*ppNew = pFrag;
		return;
	}
	else
	{
		SG_ERR_THROW(  SG_ERR_DAGFRAG_DESERIALIZATION_VERSION  );
	}

fail:
	SG_DAGFRAG_NULLFREE(pCtx, pFrag);
}
void SG_sync_remote__request_fragball(
	SG_context* pCtx,
	SG_repo* pRepo,
	const SG_pathname* pFragballDirPathname,
	SG_vhash* pvhRequest,
	char** ppszFragballName)
{
	SG_pathname* pFragballPathname = NULL;
	SG_uint64* paDagNums = NULL;
	SG_string* pstrFragballName = NULL;
	SG_rbtree* prbDagnodes = NULL;
	SG_rbtree_iterator* pit = NULL;
	SG_rev_spec* pRevSpec = NULL;
	SG_stringarray* psaFullHids = NULL;
	SG_rbtree* prbDagnums = NULL;
	SG_dagfrag* pFrag = NULL;
	char* pszRepoId = NULL;
	char* pszAdminId = NULL;
    SG_fragball_writer* pfb = NULL;

	SG_NULLARGCHECK_RETURN(pRepo);
	SG_NULLARGCHECK_RETURN(pFragballDirPathname);

    {
        char buf_filename[SG_TID_MAX_BUFFER_LENGTH];
        SG_ERR_CHECK(  SG_tid__generate(pCtx, buf_filename, sizeof(buf_filename))  );
        SG_ERR_CHECK(  SG_PATHNAME__ALLOC__PATHNAME_SZ(pCtx, &pFragballPathname, pFragballDirPathname, buf_filename)  );
    }

	if (!pvhRequest)
	{
		// Add leaves from every dag to the fragball.
		SG_uint32 count_dagnums;
		SG_uint32 i;

        SG_ERR_CHECK(  SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb)  );
		SG_ERR_CHECK(  SG_repo__list_dags(pCtx, pRepo, &count_dagnums, &paDagNums)  );

		for (i=0; i<count_dagnums; i++)
		{
			SG_ERR_CHECK(  SG_repo__fetch_dag_leaves(pCtx, pRepo, paDagNums[i], &prbDagnodes)  );
			SG_ERR_CHECK(  SG_fragball__write__dagnodes(pCtx, pfb, paDagNums[i], prbDagnodes)  );
			SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
		}

		SG_ERR_CHECK(  SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName)  );
		SG_ERR_CHECK(  SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName)  );
        SG_ERR_CHECK(  SG_fragball_writer__close(pCtx, pfb)  );
	}
	else
	{
		// Specific dags/nodes were requested. Build that fragball.
		SG_bool found;

#if TRACE_SYNC_REMOTE && 0
		SG_ERR_CHECK(  SG_vhash_debug__dump_to_console__named(pCtx, pvhRequest, "fragball request")  );
#endif

		SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__CLONE, &found)  );
		if (found)
		{
            // SG_SYNC_STATUS_KEY__CLONE_REQUEST is currently ignored
            SG_ERR_CHECK(  SG_repo__fetch_repo__fragball(pCtx, pRepo, 3, pFragballDirPathname, ppszFragballName) );
		}
		else
		{
			// Not a full clone.

            SG_ERR_CHECK(  SG_fragball_writer__alloc(pCtx, pRepo, pFragballPathname, SG_TRUE, 2, &pfb)  );
			SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &found)  );
			if (found)
			{
                SG_vhash* pvh_since = NULL;

                SG_ERR_CHECK(  SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__SINCE, &pvh_since)  );

                SG_ERR_CHECK(  _do_since(pCtx, pRepo, pvh_since, pfb)  );
            }

			SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &found)  );
			if (found)
			{
				// Specific Dagnodes were requested. Add just those nodes to our "start from" rbtree.

				SG_vhash* pvhDags;
				SG_uint32 count_requested_dagnums;
				SG_uint32 i;
				const SG_variant* pvRevSpecs = NULL;
				SG_vhash* pvhRevSpec = NULL;

				// For each requested dag, get rev spec request.
				SG_ERR_CHECK(  SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__DAGS, &pvhDags)  );
				SG_ERR_CHECK(  SG_vhash__count(pCtx, pvhDags, &count_requested_dagnums)  );
				if (count_requested_dagnums)
					SG_ERR_CHECK(  SG_repo__list_dags__rbtree(pCtx, pRepo, &prbDagnums)  );
				for (i=0; i<count_requested_dagnums; i++)
				{
					SG_bool isValidDagnum = SG_FALSE;
					SG_bool bSpecificNodesRequested = SG_FALSE;
					const char* pszRefDagNum = NULL;
					SG_uint64 iDagnum;

					// Get the dag's missing node vhash.
					SG_ERR_CHECK(  SG_vhash__get_nth_pair(pCtx, pvhDags, i, &pszRefDagNum, &pvRevSpecs)  );

					// Verify that requested dagnum exists
					SG_ERR_CHECK(  SG_rbtree__find(pCtx, prbDagnums, pszRefDagNum, &isValidDagnum, NULL)  );
					if (!isValidDagnum)
                        continue;

					SG_ERR_CHECK(  SG_dagnum__from_sz__hex(pCtx, pszRefDagNum, &iDagnum)  );
					
					if (pvRevSpecs && pvRevSpecs->type != SG_VARIANT_TYPE_NULL)
					{
						SG_uint32 countRevSpecs = 0;
						
						SG_ERR_CHECK(  SG_variant__get__vhash(pCtx, pvRevSpecs, &pvhRevSpec)  );
						SG_ERR_CHECK(  SG_rev_spec__from_vash(pCtx, pvhRevSpec, &pRevSpec)  );

						// Process the rev spec for each dag
						SG_ERR_CHECK(  SG_rev_spec__count(pCtx, pRevSpec, &countRevSpecs)  );
						if (countRevSpecs > 0)
						{
							bSpecificNodesRequested = SG_TRUE;

							SG_ERR_CHECK(  SG_rev_spec__get_all__repo(pCtx, pRepo, pRevSpec, SG_TRUE, &psaFullHids, NULL)  );
							SG_ERR_CHECK(  SG_stringarray__to_rbtree_keys(pCtx, psaFullHids, &prbDagnodes)  );
							SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids);
						}
						SG_REV_SPEC_NULLFREE(pCtx, pRevSpec);
					}

					if (!bSpecificNodesRequested)
					{
						// When no specific nodes are in the request, add all leaves.
						SG_ERR_CHECK(  SG_repo__fetch_dag_leaves(pCtx, pRepo, iDagnum, &prbDagnodes)  );
					}

					if (prbDagnodes) // can be null when leaves of an empty dag are requested
					{
						// Get the leaves of the other repo, which we need to connect to.
						SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &found)  );
						if (found)
						{
							SG_vhash* pvhRefAllLeaves;
							SG_vhash* pvhRefDagLeaves;
							SG_ERR_CHECK(  SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__LEAVES, &pvhRefAllLeaves)  );
							SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, pszRefDagNum, &found)  );
							{
								SG_ERR_CHECK(  SG_vhash__get__vhash(pCtx, pvhRefAllLeaves, pszRefDagNum, &pvhRefDagLeaves)  );
								SG_ERR_CHECK(  SG_sync__build_best_guess_dagfrag(pCtx, pRepo, iDagnum, 
									prbDagnodes, pvhRefDagLeaves, &pFrag)  );
							}
						}
						else
						{
							// The other repo's leaves weren't provided: add just the requested nodes, make no attempt to connect.
							SG_ERR_CHECK(  SG_repo__get_repo_id(pCtx, pRepo, &pszRepoId)  );
							SG_ERR_CHECK(  SG_repo__get_admin_id(pCtx, pRepo, &pszAdminId)  );
							SG_ERR_CHECK(  SG_dagfrag__alloc(pCtx, &pFrag, pszRepoId, pszAdminId, iDagnum)  );
							SG_ERR_CHECK(  SG_dagfrag__load_from_repo__simple(pCtx, pFrag, pRepo, prbDagnodes)  );
							SG_NULLFREE(pCtx, pszRepoId);
							SG_NULLFREE(pCtx, pszAdminId);
						}

						SG_ERR_CHECK(  SG_fragball__write__frag(pCtx, pfb, pFrag)  );
						
						SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
						SG_DAGFRAG_NULLFREE(pCtx, pFrag);
					}

				} // dagnum loop
			} // if "dags" exists

			/* Add requested blobs to the fragball */
			SG_ERR_CHECK(  SG_vhash__has(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &found)  );
			if (found)
			{
				// Blobs were requested.
				SG_vhash* pvhBlobs;
				SG_ERR_CHECK(  SG_vhash__get__vhash(pCtx, pvhRequest, SG_SYNC_STATUS_KEY__BLOBS, &pvhBlobs)  );
				SG_ERR_CHECK(  SG_sync__add_blobs_to_fragball(pCtx, pfb, pvhBlobs)  );
			}

			SG_ERR_CHECK(  SG_pathname__get_last(pCtx, pFragballPathname, &pstrFragballName)  );
			SG_ERR_CHECK(  SG_STRDUP(pCtx, SG_string__sz(pstrFragballName), ppszFragballName)  );
		}
        SG_ERR_CHECK(  SG_fragball_writer__close(pCtx, pfb)  );
	}

	/* fallthru */
fail:
	// If we had an error, delete the half-baked fragball.
	if (pFragballPathname && SG_context__has_err(pCtx))
    {
		SG_ERR_IGNORE(  SG_fsobj__remove__pathname(pCtx, pFragballPathname)  );
    }

	SG_PATHNAME_NULLFREE(pCtx, pFragballPathname);
	SG_NULLFREE(pCtx, paDagNums);
	SG_STRING_NULLFREE(pCtx, pstrFragballName);
	SG_RBTREE_NULLFREE(pCtx, prbDagnodes);
	SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
	SG_RBTREE_NULLFREE(pCtx, prbDagnums);
	SG_REV_SPEC_NULLFREE(pCtx, pRevSpec);
	SG_STRINGARRAY_NULLFREE(pCtx, psaFullHids);
	SG_DAGFRAG_NULLFREE(pCtx, pFrag);
	SG_NULLFREE(pCtx, pszRepoId);
	SG_NULLFREE(pCtx, pszAdminId);
    SG_FRAGBALL_WRITER_NULLFREE(pCtx, pfb);
}
Пример #3
0
void SG_sync__build_best_guess_dagfrag(
	SG_context* pCtx,
	SG_repo* pRepo,
	SG_uint64 iDagNum,
	SG_rbtree* prbStartFromHids,
	SG_vhash* pvhConnectToHidsAndGens,
	SG_dagfrag** ppFrag)
{
	SG_uint32 i, countConnectTo;
	SG_rbtree_iterator* pit = NULL;
	SG_dagnode* pdn = NULL;
	SG_dagfrag* pFrag = NULL;
	SG_repo_fetch_dagnodes_handle* pdh = NULL;

	SG_int32 minGen = SG_INT32_MAX;
	SG_int32 maxGen = 0;
	SG_uint32 gensToFetch = 0;

	char* psz_repo_id = NULL;
	char* psz_admin_id = NULL;

	SG_bool bNextHid;
	const char* pszRefHid;
	SG_int32 gen;

#if TRACE_SYNC
	SG_int64 startTime;
	SG_int64 endTime;
#endif

	SG_NULLARGCHECK_RETURN(prbStartFromHids);

	/* Find the minimum generation in pertinent "connect to" nodes. */
	if (pvhConnectToHidsAndGens)
	{
		SG_ERR_CHECK(  SG_vhash__count(pCtx, pvhConnectToHidsAndGens, &countConnectTo)  );
		for (i = 0; i < countConnectTo; i++)
		{
			SG_int32 gen;
			SG_ERR_CHECK(  SG_vhash__get_nth_pair__int32(pCtx, pvhConnectToHidsAndGens, i, &pszRefHid, &gen)  );
			if (gen < minGen)
				minGen = gen;
		}
	}

	/* Happens when pulling into an empty repo, or when an entire dag is specifically requested. */
	if (minGen == SG_INT32_MAX)
		minGen = -1;

	SG_ERR_CHECK(  SG_repo__fetch_dagnodes__begin(pCtx, pRepo, iDagNum, &pdh)  );

	/* Find the maximum generation in pertinent "start from" nodes. */
	SG_ERR_CHECK(  SG_rbtree__iterator__first(pCtx, &pit, prbStartFromHids, &bNextHid, &pszRefHid, NULL)  );
	while (bNextHid)
	{
		SG_ERR_CHECK(  SG_repo__fetch_dagnodes__one(pCtx, pRepo, pdh, pszRefHid, &pdn)  );
		SG_ERR_CHECK(  SG_dagnode__get_generation(pCtx, pdn, &gen)  );
		if (gen > maxGen)
			maxGen = gen;
		SG_DAGNODE_NULLFREE(pCtx, pdn);

		SG_ERR_CHECK(  SG_rbtree__iterator__next(pCtx, pit, &bNextHid, &pszRefHid, NULL)  );
	}
	SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);

	if (maxGen <= minGen)
		gensToFetch = FALLBACK_GENS_PER_ROUNDTRIP;
	else
		gensToFetch = maxGen - minGen;

#if TRACE_SYNC
	{
		char buf_dagnum[SG_DAGNUM__BUF_MAX__HEX];
		SG_uint32 count;

		SG_ERR_CHECK(  SG_dagnum__to_sz__hex(pCtx, iDagNum, buf_dagnum, sizeof(buf_dagnum))  );

		SG_ERR_CHECK(  SG_console(pCtx, SG_CS_STDERR, "Building best guess dagfrag for dag %s...\n", buf_dagnum)  );
		SG_ERR_CHECK(  SG_console(pCtx, SG_CS_STDERR, "Starting from nodes:\n")  );
		SG_ERR_CHECK(  SG_rbtree_debug__dump_keys_to_console(pCtx, prbStartFromHids)  );
		SG_ERR_CHECK(  SG_vhash_debug__dump_to_console__named(pCtx, pvhConnectToHidsAndGens, "Connecting to nodes")  );

		SG_ERR_CHECK(  SG_rbtree__count(pCtx, prbStartFromHids, &count)  );

		SG_ERR_CHECK(  SG_console(pCtx, SG_CS_STDERR, "result has %d generations from %u starting nodes.\n",
			gensToFetch, count)  );
		SG_ERR_CHECK(  SG_console__flush(pCtx, SG_CS_STDERR)  );

		SG_ERR_CHECK(  SG_time__get_milliseconds_since_1970_utc(pCtx, &startTime)  );
	}
#endif

	/* Return a frag with the corresponding generations filled in. */
	SG_ERR_CHECK(  SG_repo__get_repo_id(pCtx, pRepo, &psz_repo_id)  );
	SG_ERR_CHECK(  SG_repo__get_admin_id(pCtx, pRepo, &psz_admin_id)  );
	SG_ERR_CHECK(  SG_dagfrag__alloc(pCtx, &pFrag, psz_repo_id, psz_admin_id, iDagNum)  );

	SG_ERR_CHECK(  SG_dagfrag__load_from_repo__multi(pCtx, pFrag, pRepo, prbStartFromHids, gensToFetch)  );
#if TRACE_SYNC
	SG_ERR_CHECK(  SG_time__get_milliseconds_since_1970_utc(pCtx, &endTime)  );
	{
		SG_uint32 dagnodeCount;
		double seconds = ((double)endTime-(double)startTime)/1000;

		SG_ERR_CHECK(  SG_dagfrag__dagnode_count(pCtx, pFrag, &dagnodeCount)  );
		SG_ERR_CHECK(  SG_console(pCtx, SG_CS_STDERR, " - %u nodes in frag, built in %1.3f seconds\n", dagnodeCount, seconds)  );
		SG_ERR_CHECK(  SG_dagfrag_debug__dump__console(pCtx, pFrag, "best-guess dagfrag", 0, SG_CS_STDERR)  );
	}
#endif

	*ppFrag = pFrag;
	pFrag = NULL;

	/* Common cleanup */
fail:
	SG_NULLFREE(pCtx, psz_repo_id);
	SG_NULLFREE(pCtx, psz_admin_id);
	SG_RBTREE_ITERATOR_NULLFREE(pCtx, pit);
	SG_DAGNODE_NULLFREE(pCtx, pdn);
	SG_DAGFRAG_NULLFREE(pCtx, pFrag);
	SG_ERR_IGNORE(  SG_repo__fetch_dagnodes__end(pCtx, pRepo, &pdh)  );
}