// Build a ledger from consensus transactions
std::shared_ptr<Ledger>
buildLedger(
std::shared_ptr<Ledger const> const& parent,
NetClock::time_point closeTime,
const bool closeTimeCorrect,
NetClock::duration closeResolution,
SHAMap const& txs,
Application& app,
CanonicalTXSet& retriableTxs,
beast::Journal j)
{
JLOG(j.debug()) << "Report: TxSt = " << txs.getHash().as_uint256()
<< ", close " << closeTime.time_since_epoch().count()
<< (closeTimeCorrect ? "" : " (incorrect)");
return buildLedgerImpl(
parent,
closeTime,
closeTimeCorrect,
closeResolution,
app,
j,
[&](OpenView& accum, std::shared_ptr<Ledger> const& buildLCL) {
retriableTxs = applyTransactions(app, txs, accum, buildLCL, j);
});
}
bool confuseMap (SHAMap& map, int count)
{
// add a bunch of random states to a map, then remove them
// map should be the same
uint256 beforeHash = map.getHash ();
std::list<uint256> items;
for (int i = 0; i < count; ++i)
{
std::shared_ptr<SHAMapItem> item = makeRandomAS ();
items.push_back (item->getTag ());
if (!map.addItem (*item, false, false))
{
log <<
"Unable to add item to map";
assert (false);
return false;
}
}
for (std::list<uint256>::iterator it = items.begin (); it != items.end (); ++it)
{
if (!map.delItem (*it))
{
log <<
"Unable to remove item from map";
assert (false);
return false;
}
}
if (beforeHash != map.getHash ())
{
log <<
"Hashes do not match " << beforeHash << " " << map.getHash ();
assert (false);
return false;
}
return true;
}
void run ()
{
testcase ("add/traverse");
beast::Journal const j; // debug journal
tests::TestFamily f(j);
// h3 and h4 differ only in the leaf, same terminal node (level 19)
uint256 h1, h2, h3, h4, h5;
h1.SetHex ("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
h2.SetHex ("436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe");
h3.SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8");
h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7");
SHAMap sMap (SHAMapType::FREE, f);
SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5));
unexpected (!sMap.addItem (i2, true, false), "no add");
unexpected (!sMap.addItem (i1, true, false), "no add");
auto i = sMap.begin();
auto e = sMap.end();
unexpected (i == e || (*i != i1), "bad traverse");
++i;
unexpected (i == e || (*i != i2), "bad traverse");
++i;
unexpected (i != e, "bad traverse");
sMap.addItem (i4, true, false);
sMap.delItem (i2.key());
sMap.addItem (i3, true, false);
i = sMap.begin();
e = sMap.end();
unexpected (i == e || (*i != i1), "bad traverse");
++i;
unexpected (i == e || (*i != i3), "bad traverse");
++i;
unexpected (i == e || (*i != i4), "bad traverse");
++i;
unexpected (i != e, "bad traverse");
testcase ("snapshot");
SHAMapHash mapHash = sMap.getHash ();
std::shared_ptr<SHAMap> map2 = sMap.snapShot (false);
unexpected (sMap.getHash () != mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
unexpected (!sMap.delItem (sMap.begin()->key()), "bad mod");
unexpected (sMap.getHash () == mapHash, "bad snapshot");
unexpected (map2->getHash () != mapHash, "bad snapshot");
testcase ("build/tear");
{
std::vector<uint256> keys(8);
keys[0].SetHex ("b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[1].SetHex ("b92881fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[2].SetHex ("b92691fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[3].SetHex ("b92791fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[4].SetHex ("b91891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[5].SetHex ("b99891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[6].SetHex ("f22891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
keys[7].SetHex ("292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8");
std::vector<uint256> hashes(8);
hashes[0].SetHex ("B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F7F");
hashes[1].SetHex ("FBC195A9592A54AB44010274163CB6BA95F497EC5BA0A8831845467FB2ECE266");
hashes[2].SetHex ("4E7D2684B65DFD48937FFB775E20175C43AF0C94066F7D5679F51AE756795B75");
hashes[3].SetHex ("7A2F312EB203695FFD164E038E281839EEF06A1B99BFC263F3CECC6C74F93E07");
hashes[4].SetHex ("395A6691A372387A703FB0F2C6D2C405DAF307D0817F8F0E207596462B0E3A3E");
hashes[5].SetHex ("D044C0A696DE3169CC70AE216A1564D69DE96582865796142CE7D98A84D9DDE4");
hashes[6].SetHex ("76DCC77C4027309B5A91AD164083264D70B77B5E43E08AEDA5EBF94361143615");
hashes[7].SetHex ("DF4220E93ADC6F5569063A01B4DC79F8DB9553B6A3222ADE23DEA02BBE7230E5");
SHAMap map (SHAMapType::FREE, f);
expect (map.getHash() == zero, "bad initial empty map hash");
for (int i = 0; i < keys.size(); ++i)
{
SHAMapItem item (keys[i], IntToVUC (i));
map.addItem (item, true, false);
expect (map.getHash().as_uint256() == hashes[i], "bad buildup map hash");
}
for (int i = keys.size() - 1; i >= 0; --i)
{
expect (map.getHash().as_uint256() == hashes[i], "bad teardown hash");
map.delItem (keys[i]);
}
expect (map.getHash() == zero, "bad final empty map hash");
}
}
CanonicalTXSet
applyTransactions(
Application& app,
SHAMap const& txns,
OpenView& view,
std::shared_ptr<Ledger> const& buildLCL,
beast::Journal j)
{
CanonicalTXSet retriableTxs(txns.getHash().as_uint256());
for (auto const& item : txns)
{
if (buildLCL->txExists(item.key()))
continue;
// The transaction wasn't filtered
// Add it to the set to be tried in canonical order
JLOG(j.debug()) << "Processing candidate transaction: " << item.key();
try
{
retriableTxs.insert(
std::make_shared<STTx const>(SerialIter{item.slice()}));
}
catch (std::exception const&)
{
JLOG(j.warn()) << "Txn " << item.key() << " throws";
}
}
bool certainRetry = true;
// Attempt to apply all of the retriable transactions
for (int pass = 0; pass < LEDGER_TOTAL_PASSES; ++pass)
{
JLOG(j.debug()) << "Pass: "******" Txns: " << retriableTxs.size()
<< (certainRetry ? " retriable" : " final");
int changes = 0;
auto it = retriableTxs.begin();
while (it != retriableTxs.end())
{
try
{
switch (applyTransaction(
app, view, *it->second, certainRetry, tapNONE, j))
{
case ApplyResult::Success:
it = retriableTxs.erase(it);
++changes;
break;
case ApplyResult::Fail:
it = retriableTxs.erase(it);
break;
case ApplyResult::Retry:
++it;
}
}
catch (std::exception const&)
{
JLOG(j.warn()) << "Transaction throws";
it = retriableTxs.erase(it);
}
}
JLOG(j.debug()) << "Pass: "******" finished " << changes
<< " changes";
// A non-retry pass made no changes
if (!changes && !certainRetry)
return retriableTxs;
// Stop retriable passes
if (!changes || (pass >= LEDGER_RETRY_PASSES))
certainRetry = false;
}
// If there are any transactions left, we must have
// tried them in at least one final pass
assert(retriableTxs.empty() || !certainRetry);
return retriableTxs;
}
void run ()
{
unsigned int seed;
// VFALCO DEPRECATED Should use C++11
RAND_pseudo_bytes (reinterpret_cast<unsigned char*> (&seed), sizeof (seed));
srand (seed);
beast::Journal const j; // debug journal
TestFamily f(j);
SHAMap source (SHAMapType::FREE, f, j);
SHAMap destination (SHAMapType::FREE, f, j);
int items = 10000;
for (int i = 0; i < items; ++i)
source.addItem (*makeRandomAS (), false, false);
unexpected (!confuseMap (source, 500), "ConfuseMap");
source.setImmutable ();
std::vector<SHAMapNodeID> nodeIDs, gotNodeIDs;
std::vector< Blob > gotNodes;
std::vector<uint256> hashes;
std::vector<SHAMapNodeID>::iterator nodeIDIterator;
std::vector< Blob >::iterator rawNodeIterator;
int passes = 0;
int nodes = 0;
destination.setSynching ();
unexpected (!source.getNodeFat (SHAMapNodeID (), nodeIDs, gotNodes,
(rand () % 2) == 0, rand () % 3),
"GetNodeFat");
unexpected (gotNodes.size () < 1, "NodeSize");
unexpected (!destination.addRootNode (*gotNodes.begin (), snfWIRE, nullptr).isGood(), "AddRootNode");
nodeIDs.clear ();
gotNodes.clear ();
#ifdef SMS_DEBUG
int bytes = 0;
#endif
do
{
f.clock().advance(std::chrono::seconds(1));
++passes;
hashes.clear ();
// get the list of nodes we know we need
destination.getMissingNodes (nodeIDs, hashes, 2048, nullptr);
if (nodeIDs.empty ()) break;
// get as many nodes as possible based on this information
for (nodeIDIterator = nodeIDs.begin (); nodeIDIterator != nodeIDs.end (); ++nodeIDIterator)
{
if (!source.getNodeFat (*nodeIDIterator, gotNodeIDs, gotNodes,
(rand () % 2) == 0, rand () % 3))
{
fail ("GetNodeFat");
}
else
{
pass ();
}
}
assert (gotNodeIDs.size () == gotNodes.size ());
nodeIDs.clear ();
hashes.clear ();
if (gotNodeIDs.empty ())
{
fail ("Got Node ID");
}
else
{
pass ();
}
for (nodeIDIterator = gotNodeIDs.begin (), rawNodeIterator = gotNodes.begin ();
nodeIDIterator != gotNodeIDs.end (); ++nodeIDIterator, ++rawNodeIterator)
{
++nodes;
#ifdef SMS_DEBUG
bytes += rawNodeIterator->size ();
#endif
if (!destination.addKnownNode (*nodeIDIterator, *rawNodeIterator, nullptr).isGood ())
{
fail ("AddKnownNode");
}
else
{
pass ();
}
}
gotNodeIDs.clear ();
gotNodes.clear ();
}
while (true);
destination.clearSynching ();
#ifdef SMS_DEBUG
log << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes, " <<
bytes / 1024 << " KB";
#endif
if (!source.deepCompare (destination))
{
fail ("Deep Compare");
}
else
{
pass ();
}
#ifdef SMS_DEBUG
log << "SHAMapSync test passed: " << items << " items, " <<
passes << " passes, " << nodes << " nodes";
#endif
}
void runTest ()
{
unsigned int seed;
// VFALCO TODO Replace this with beast::Random
RAND_pseudo_bytes (reinterpret_cast<unsigned char*> (&seed), sizeof (seed));
srand (seed);
SHAMap source (smtFREE), destination (smtFREE);
int items = 10000;
for (int i = 0; i < items; ++i)
source.addItem (*makeRandomAS (), false, false);
beginTestCase ("add/remove");
unexpected (!confuseMap (source, 500), "ConfuseMap");
source.setImmutable ();
std::vector<SHAMapNode> nodeIDs, gotNodeIDs;
std::list< Blob > gotNodes;
std::vector<uint256> hashes;
std::vector<SHAMapNode>::iterator nodeIDIterator;
std::list< Blob >::iterator rawNodeIterator;
int passes = 0;
int nodes = 0;
destination.setSynching ();
unexpected (!source.getNodeFat (SHAMapNode (), nodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0),
"GetNodeFat");
unexpected (gotNodes.size () < 1, "NodeSize");
unexpected (!destination.addRootNode (*gotNodes.begin (), snfWIRE, NULL), "AddRootNode");
nodeIDs.clear ();
gotNodes.clear ();
#ifdef SMS_DEBUG
int bytes = 0;
#endif
do
{
++passes;
hashes.clear ();
// get the list of nodes we know we need
destination.getMissingNodes (nodeIDs, hashes, 2048, NULL);
if (nodeIDs.empty ()) break;
// get as many nodes as possible based on this information
for (nodeIDIterator = nodeIDs.begin (); nodeIDIterator != nodeIDs.end (); ++nodeIDIterator)
{
if (!source.getNodeFat (*nodeIDIterator, gotNodeIDs, gotNodes, (rand () % 2) == 0, (rand () % 2) == 0))
{
WriteLog (lsFATAL, SHAMap) << "GetNodeFat fails";
fail ("GetNodeFat");
}
else
{
pass ();
}
}
assert (gotNodeIDs.size () == gotNodes.size ());
nodeIDs.clear ();
hashes.clear ();
if (gotNodeIDs.empty ())
{
fail ("Got Node ID");
}
else
{
pass ();
}
for (nodeIDIterator = gotNodeIDs.begin (), rawNodeIterator = gotNodes.begin ();
nodeIDIterator != gotNodeIDs.end (); ++nodeIDIterator, ++rawNodeIterator)
{
++nodes;
#ifdef SMS_DEBUG
bytes += rawNodeIterator->size ();
#endif
if (!destination.addKnownNode (*nodeIDIterator, *rawNodeIterator, NULL))
{
WriteLog (lsTRACE, SHAMap) << "AddKnownNode fails";
fail ("AddKnownNode");
}
else
{
pass ();
}
}
gotNodeIDs.clear ();
gotNodes.clear ();
}
while (true);
destination.clearSynching ();
#ifdef SMS_DEBUG
WriteLog (lsINFO, SHAMap) << "SYNCHING COMPLETE " << items << " items, " << nodes << " nodes, " <<
bytes / 1024 << " KB";
#endif
if (!source.deepCompare (destination))
{
fail ("Deep Compare");
}
else
{
pass ();
}
#ifdef SMS_DEBUG
WriteLog (lsINFO, SHAMap) << "SHAMapSync test passed: " << items << " items, " <<
passes << " passes, " << nodes << " nodes";
#endif
}
bool SHAMap::deepCompare (SHAMap& other)
{
// Intended for debug/test only
std::stack<SHAMapTreeNode::pointer> stack;
ScopedLockType sl (mLock, __FILE__, __LINE__);
stack.push (root);
while (!stack.empty ())
{
SHAMapTreeNode::pointer node = stack.top ();
stack.pop ();
SHAMapTreeNode::pointer otherNode;
if (node->isRoot ()) otherNode = other.root;
else otherNode = other.getNode (*node, node->getNodeHash (), false);
if (!otherNode)
{
WriteLog (lsINFO, SHAMap) << "unable to fetch node";
return false;
}
else if (otherNode->getNodeHash () != node->getNodeHash ())
{
WriteLog (lsWARNING, SHAMap) << "node hash mismatch";
return false;
}
// WriteLog (lsTRACE) << "Comparing inner nodes " << *node;
if (node->getNodeHash () != otherNode->getNodeHash ())
return false;
if (node->isLeaf ())
{
if (!otherNode->isLeaf ()) return false;
if (node->peekItem ()->getTag () != otherNode->peekItem ()->getTag ()) return false;
if (node->peekItem ()->getData () != otherNode->peekItem ()->getData ()) return false;
}
else if (node->isInner ())
{
if (!otherNode->isInner ())
return false;
for (int i = 0; i < 16; ++i)
{
if (node->isEmptyBranch (i))
{
if (!otherNode->isEmptyBranch (i)) return false;
}
else
{
SHAMapTreeNode::pointer next = getNode (node->getChildNodeID (i), node->getChildHash (i), false);
if (!next)
{
WriteLog (lsWARNING, SHAMap) << "unable to fetch inner node";
return false;
}
stack.push (next);
}
}
}
}
return true;
}
