void
FutureBucket::makeLive(Application& app)
{
checkState();
assert(!isLive());
assert(hasHashes());
auto& bm = app.getBucketManager();
if (hasOutputHash())
{
setLiveOutput(bm.getBucketByHash(hexToBin256(getOutputHash())));
}
else
{
assert(mState == FB_HASH_INPUTS);
mInputCurrBucket =
bm.getBucketByHash(hexToBin256(mInputCurrBucketHash));
mInputSnapBucket =
bm.getBucketByHash(hexToBin256(mInputSnapBucketHash));
assert(mInputShadowBuckets.empty());
for (auto const& h : mInputShadowBucketHashes)
{
auto b = bm.getBucketByHash(hexToBin256(h));
assert(b);
CLOG(DEBUG, "Bucket") << "Reconstituting shadow " << h;
mInputShadowBuckets.push_back(b);
}
mState = FB_LIVE_INPUTS;
startMerge(app);
assert(isLive());
}
}
std::shared_ptr<Bucket const>
ApplyBucketsWork::getBucket(std::string const& hash)
{
auto i = mBuckets.find(hash);
auto b = (i != mBuckets.end())
? i->second
: mApp.getBucketManager().getBucketByHash(hexToBin256(hash));
assert(b);
return b;
}
void
BucketManagerImpl::assumeState(HistoryArchiveState const& has)
{
for (uint32_t i = 0; i < BucketList::kNumLevels; ++i)
{
auto curr = getBucketByHash(hexToBin256(has.currentBuckets.at(i).curr));
auto snap = getBucketByHash(hexToBin256(has.currentBuckets.at(i).snap));
if (!(curr && snap))
{
throw std::runtime_error(
"Missing bucket files while assuming saved BucketList state");
}
mBucketList.getLevel(i).setCurr(curr);
mBucketList.getLevel(i).setSnap(snap);
mBucketList.getLevel(i).setNext(has.currentBuckets.at(i).next);
}
mBucketList.restartMerges(mApp);
}
std::vector<std::string>
BucketManagerImpl::checkForMissingBucketsFiles(HistoryArchiveState const& has)
{
std::vector<std::string> buckets = has.allBuckets();
std::vector<std::string> result;
std::copy_if(buckets.begin(), buckets.end(), std::back_inserter(result),
[&](std::string b) {
auto filename = bucketFilename(b);
return !isZero(hexToBin256(b)) && !fs::exists(filename);
});
return result;
}
static void
checkHashEq(std::shared_ptr<Bucket> b, std::string const& h)
{
assert(b->getHash() == hexToBin256(h));
}
void
BucketManagerImpl::forgetUnreferencedBuckets()
{
std::lock_guard<std::recursive_mutex> lock(mBucketMutex);
std::set<Hash> referenced;
for (uint32_t i = 0; i < BucketList::kNumLevels; ++i)
{
auto const& level = mBucketList.getLevel(i);
referenced.insert(level.getCurr()->getHash());
referenced.insert(level.getSnap()->getHash());
for (auto const& h : level.getNext().getHashes())
{
referenced.insert(hexToBin256(h));
}
}
// Implicitly retain any buckets that are referenced by a state in
// the publish queue.
auto pub = mApp.getHistoryManager().getBucketsReferencedByPublishQueue();
{
for (auto const& h : pub)
{
CLOG(DEBUG, "Bucket")
<< "BucketManager::forgetUnreferencedBuckets: " << h
<< " referenced by publish queue";
referenced.insert(hexToBin256(h));
}
}
for (auto i = mSharedBuckets.begin(); i != mSharedBuckets.end();)
{
// Standard says map iterators other than the one you're erasing remain
// valid.
auto j = i;
++i;
// Only drop buckets if the bucketlist has forgotten them _and_
// no other in-progress structures (worker threads, shadow lists)
// have references to them, just us. It's ok to retain a few too
// many buckets, a little longer than necessary.
//
// This conservatism is important because we want to enforce that only
// one bucket ever exists in memory with a given filename, and that
// we're the first and last to know about it. Otherwise buckets might
// race on deleting the underlying file from one another.
if (referenced.find(j->first) == referenced.end() &&
j->second.use_count() == 1)
{
auto filename = j->second->getFilename();
CLOG(TRACE, "Bucket")
<< "BucketManager::forgetUnreferencedBuckets dropping "
<< filename;
if (!filename.empty())
{
CLOG(TRACE, "Bucket") << "removing bucket file: " << filename;
std::remove(filename.c_str());
}
mSharedBuckets.erase(j);
}
}
mSharedBucketsSize.set_count(mSharedBuckets.size());
}
