/** Process a single ledger
@param ledgerIndex The index of the ledger to process.
@param ledgerHash The known correct hash of the ledger.
@param doNodes Ensure all ledger nodes are in the node db.
@param doTxns Reprocess (account) transactions to SQL databases.
@return `true` if the ledger was cleaned.
*/
bool doLedger(
LedgerIndex const& ledgerIndex,
LedgerHash const& ledgerHash,
bool doNodes,
bool doTxns)
{
Ledger::pointer nodeLedger =
getApp().getInboundLedgers().acquire (
ledgerHash, ledgerIndex, InboundLedger::fcGENERIC);
if (!nodeLedger)
{
m_journal.debug << "Ledger " << ledgerIndex << " not available";
return false;
}
Ledger::pointer dbLedger = Ledger::loadByIndex(ledgerIndex);
if (! dbLedger ||
(dbLedger->getHash() != ledgerHash) ||
(dbLedger->getParentHash() != nodeLedger->getParentHash()))
{
// Ideally we'd also check for more than one ledger with that index
m_journal.debug <<
"Ledger " << ledgerIndex << " mismatches SQL DB";
doTxns = true;
}
if(! getApp().getLedgerMaster().fixIndex(ledgerIndex, ledgerHash))
{
m_journal.debug << "ledger " << ledgerIndex << " had wrong entry in history";
doTxns = true;
}
if (doNodes && !nodeLedger->walkLedger())
{
m_journal.debug << "Ledger " << ledgerIndex << " is missing nodes";
getApp().getInboundLedgers().acquire(
ledgerHash, ledgerIndex, InboundLedger::fcGENERIC);
return false;
}
if (doTxns && !nodeLedger->pendSaveValidated(true, false))
{
m_journal.debug << "Failed to save ledger " << ledgerIndex;
return false;
}
return true;
}
void LedgerMaster::asyncAccept(Ledger::pointer ledger)
{
uint32 seq = ledger->getLedgerSeq();
uint256 prevHash = ledger->getParentHash();
while (seq > 0)
{
{
boost::recursive_mutex::scoped_lock ml(mLock);
mCompleteLedgers.setValue(seq);
--seq;
if (mCompleteLedgers.hasValue(seq))
break;
}
uint256 tHash, pHash;
if (!Ledger::getHashesByIndex(seq, tHash, pHash) || (tHash != prevHash))
break;
prevHash = pHash;
}
resumeAcquiring();
}
/**
* Instantiate an application and replay a ledger history out
* of the dump file `filename`.
*/
void
LedgerDump::loadTransactions (std::string const& filename)
{
std::ifstream in (filename);
require ((bool)in, "opening file");
std::unique_ptr <Application> app (make_Application ());
app->setup ();
auto &lm = app->getLedgerMaster ();
WriteLog (lsINFO, LedgerDump) << "Loading ledgers from " << filename;
auto nTxs = 0;
// app->setup() when called with START_UP == Config::FRESH calls
// ApplicationImp::startNewLedger(). Unfortunately it starts the new
// ledger at the wrong timestamp, so we call it again once we've read
// the first ledger we're going to apply. However, it's worth
// understanding that startNewLedger() establishes 3 ledgers:
//
// Ledger 0 is the genesis ledger, it's not a real ledger, just a
// number.
//
// Ledger 1 is the root-account deposit ledger, with a single pile of
// currency owned by a single account generated by the seed
// "masterpassword".
//
// Ledger 2 is created and closed immediately on start, not sure why.
//
// Ledger 3 is a new ledger chained to #2 and left open in
// ledgermaster.
//
// Ledger 3 is where replay should be starting, so (once we call
// startNewLedger() again) we pull out ledger #2 and use it as the
// parent of the new ledger 3 we're replaying, and throw away the #3
// that was made by startNewLedger().
Ledger::pointer parentLedger;
while (in)
{
if ((gLedgerSeq & 0xfff) == 0) {
Job j;
app->doSweep (j);
}
Json::Value j = loadJsonRecord (in);
Ledger::pointer deserializedLedger;
SHAMap::pointer txSet;
std::vector<uint256> txOrder;
std::tie (deserializedLedger, txSet, txOrder) =
loadLedgerAndTransactionsFromJSON (*app, j);
if (!parentLedger)
{
if (getConfig ().START_LEDGER.empty ())
{
require (deserializedLedger->getLedgerSeq () == 3,
"Initial ledger isn't seq #3");
// On first iteration, restart the app's view of the ledger
// history at the same instant as the parent close time of the
// first ledger (ledger #3).
app->startNewLedger (deserializedLedger->getParentCloseTimeNC ());
parentLedger = lm.getClosedLedger ();
require (parentLedger->getLedgerSeq () == 2,
"Initial ledger parent isn't seq #2");
}
else
{
// We're being invoked with --ledger, which is where we
// will start replay from.
require (app->loadOldLedger (getConfig ().START_LEDGER, false),
"Reloading old ledger failed.");
parentLedger = lm.getClosedLedger ();
}
auto const parentSeq = parentLedger->getLedgerSeq ();
auto seq = j["seq"].asUInt ();
while (parentSeq + 1 > seq)
{
// Fast-scan JSON records until we hit the right one.
WriteLog (lsINFO, LedgerDump) << "scanning past ledger: "
<< seq;
j = loadJsonRecord (in);
seq = j["seq"].asUInt ();
if (parentSeq + 1 <= seq)
{
require (parentSeq + 1 == seq,
"Missing ledgers between loaded and replay-start");
std::tie (deserializedLedger, txSet, txOrder) =
loadLedgerAndTransactionsFromJSON (*app, j);
}
}
gLedgerSeq = parentSeq;
require(parentLedger->getLedgerSeq () + 1 ==
deserializedLedger->getLedgerSeq (),
"Mismatched ledger-sequence prefix.");
}
Ledger::pointer currentLedger =
boost::make_shared<Ledger> (true, *parentLedger);
currentLedger->setCloseTime (deserializedLedger->getCloseTimeNC ());
currentLedger->setCloseFlags (deserializedLedger->getCloseFlags ());
currentLedger->setParentHash (deserializedLedger->getParentHash ());
WriteLog (lsINFO, LedgerDump) << "loading ledger: "
<< currentLedger->getLedgerSeq();
if (ShouldLog (lsTRACE, LedgerDump))
{
WriteLog (lsTRACE, LedgerDump) << "expecting next ledger:";
WriteLog (lsTRACE, LedgerDump) << deserializedLedger->getJson (0);
WriteLog (lsTRACE, LedgerDump) << "synthetic next ledger:";
WriteLog (lsTRACE, LedgerDump) << currentLedger->getJson (0);
}
gLedgerSeq++;
// Apply transactions, transitioning from one ledger state to next.
WriteLog (lsDEBUG, LedgerDump)
<< "Applying set of " << txOrder.size() << " transactions";
CanonicalTXSet retriableTransactions (txSet->getHash ());
std::set<uint256> failedTransactions;
LedgerConsensus::applyTransactions (txSet, currentLedger, currentLedger,
retriableTransactions, failedTransactions,
false, txOrder);
require (retriableTransactions.empty (), "failed retriable tx set is not empty");
require (failedTransactions.empty (), "failed tx set is not empty");
currentLedger->updateSkipList ();
currentLedger->setClosed ();
currentLedger->setCloseTime (deserializedLedger->getCloseTimeNC ());
int asf = currentLedger->peekAccountStateMap ()->flushDirty (
hotACCOUNT_NODE, currentLedger->getLedgerSeq());
int tmf = currentLedger->peekTransactionMap ()->flushDirty (
hotTRANSACTION_NODE, currentLedger->getLedgerSeq());
nTxs += tmf;
WriteLog (lsDEBUG, LedgerDump) << "Flushed " << asf << " account "
<< "and " << tmf << "transaction nodes";
// Finalize with the LedgerMaster.
currentLedger->setAccepted ();
bool alreadyHadLedger = lm.storeLedger (currentLedger);
assert (! alreadyHadLedger);
lm.pushLedger (currentLedger);
WriteLog (lsTRACE, LedgerDump) << "parent ledger:";
traceLedgerContents (*parentLedger);
WriteLog (lsTRACE, LedgerDump) << "current ledger:";
traceLedgerContents (*currentLedger);
try
{
checkLedgersEqual (*deserializedLedger, *currentLedger);
}
catch (...)
{
WriteLog (lsINFO, LedgerDump) << "bad ledger:";
std::cerr << currentLedger->getJson (LEDGER_JSON_FULL);
throw;
}
parentLedger = currentLedger;
}
WriteLog (lsINFO, LedgerDump) << "Loaded "
<< gLedgerSeq << "ledgers, "
<< nTxs << " transactions from "
<< filename;
exit (0);
}
void LedgerHistory::handleMismatch (LedgerHash const& built, LedgerHash const& valid)
{
assert (built != valid);
++mismatch_counter_;
Ledger::pointer builtLedger = getLedgerByHash (built);
Ledger::pointer validLedger = getLedgerByHash (valid);
if (builtLedger && validLedger)
{
assert (builtLedger->getLedgerSeq() == validLedger->getLedgerSeq());
// Determine the mismatch reason
// Distinguish Byzantine failure from transaction processing difference
if (builtLedger->getParentHash() != validLedger->getParentHash())
{
// Disagreement over prior ledger indicates sync issue
WriteLog (lsERROR, LedgerMaster) << "MISMATCH on prior ledger";
}
else if (builtLedger->getCloseTimeNC() != validLedger->getCloseTimeNC())
{
// Disagreement over close time indicates Byzantine failure
WriteLog (lsERROR, LedgerMaster) << "MISMATCH on close time";
}
else
{
std::vector <uint256> builtTx, validTx;
builtLedger->peekTransactionMap()->visitLeaves(
std::bind (&addLeaf, std::ref (builtTx), std::placeholders::_1));
validLedger->peekTransactionMap()->visitLeaves(
std::bind (&addLeaf, std::ref (validTx), std::placeholders::_1));
std::sort (builtTx.begin(), builtTx.end());
std::sort (validTx.begin(), validTx.end());
if (builtTx == validTx)
{
// Disagreement with same prior ledger, close time, and transactions
// indicates a transaction processing difference
WriteLog (lsERROR, LedgerMaster) <<
"MISMATCH with same " << builtTx.size() << " tx";
}
else
{
std::vector <uint256> notBuilt, notValid;
std::set_difference (
validTx.begin(), validTx.end(),
builtTx.begin(), builtTx.end(),
std::inserter (notBuilt, notBuilt.begin()));
std::set_difference (
builtTx.begin(), builtTx.end(),
validTx.begin(), validTx.end(),
std::inserter (notValid, notValid.begin()));
// This can be either a disagreement over the consensus
// set or difference in which transactions were rejected
// as invalid
WriteLog (lsERROR, LedgerMaster) << "MISMATCH tx differ "
<< builtTx.size() << " built, " << validTx.size() << " valid";
for (auto const& t : notBuilt)
{
WriteLog (lsERROR, LedgerMaster) << "MISMATCH built without " << t;
}
for (auto const& t : notValid)
{
WriteLog (lsERROR, LedgerMaster) << "MISMATCH valid without " << t;
}
}
}
}
else
WriteLog (lsERROR, LedgerMaster) << "MISMATCH cannot be analyzed";
}