void BlockchainScanner::updateSSH()
{
//loop over all subssh entiers in SUBSSH db,
//compile balance, txio count and summary map for each address
{
StoredDBInfo sdbi;
db_->getStoredDBInfo(SSH, sdbi);
BlockHeader* sdbiblock = nullptr;
try
{
sdbiblock = &blockchain_->getHeaderByHash(sdbi.topScannedBlkHash_);
}
catch (...)
{
sdbiblock = &blockchain_->getHeaderByHeight(0);
}
if (sdbiblock->isMainBranch())
{
if (sdbi.topBlkHgt_ != 0 &&
sdbi.topBlkHgt_ >= blockchain_->top().getBlockHeight())
{
LOGINFO << "no SSH to scan";
return;
}
}
}
map<BinaryData, StoredScriptHistory> sshMap_;
{
StoredScriptHistory* sshPtr = nullptr;
LMDBEnv::Transaction historyTx, sshTx;
db_->beginDBTransaction(&historyTx, SSH, LMDB::ReadOnly);
db_->beginDBTransaction(&sshTx, SUBSSH, LMDB::ReadOnly);
auto sshIter = db_->getIterator(SUBSSH);
sshIter.seekToFirst();
while (sshIter.advanceAndRead())
{
while (sshIter.isValid())
{
if (sshPtr != nullptr &&
sshIter.getKeyRef().contains(sshPtr->uniqueKey_))
break;
//new address
auto&& subsshkey = sshIter.getKey();
auto sshKey = subsshkey.getSliceRef(1, subsshkey.getSize() - 5);
sshPtr = &sshMap_[sshKey];
if (!scrAddrFilter_->hasScrAddress(sshKey))
{
LOGWARN << "invalid scrAddr in SUBSSH db";
continue;
}
//get what's already in the db
db_->getStoredScriptHistorySummary(*sshPtr, sshKey);
if (sshPtr->isInitialized())
{
//set iterator at unscanned height
auto hgtx = sshIter.getKeyRef().getSliceRef(-4, 4);
auto height = DBUtils::hgtxToHeight(hgtx);
if (sshPtr->alreadyScannedUpToBlk_ >= height)
{
//this ssh has already been scanned beyond the height sshIter is at,
//let's set the iterator to the correct height (or the next key)
auto&& newKey = sshIter.getKey().getSliceCopy(0, subsshkey.getSize() - 4);
auto&& newHgtx = DBUtils::heightAndDupToHgtx(
sshPtr->alreadyScannedUpToBlk_ + 1, 0);
newKey.append(newHgtx);
sshIter.seekTo(newKey);
continue;
}
}
else
{
sshPtr->uniqueKey_ = sshKey;
break;
}
}
//sanity check
if (!sshIter.isValid())
break;
//deser subssh
StoredSubHistory subssh;
subssh.unserializeDBKey(sshIter.getKeyRef());
//check dupID
if (db_->getValidDupIDForHeight(subssh.height_) != subssh.dupID_)
continue;
subssh.unserializeDBValue(sshIter.getValueRef());
for (auto& txioPair : subssh.txioMap_)
{
if (!txioPair.second.isMultisig())
{
//add up balance
if (txioPair.second.hasTxIn())
{
//check for same block fund&spend
auto&& keyOfOutput = txioPair.second.getDBKeyOfOutput();
auto&& keyOfInput = txioPair.second.getDBKeyOfInput();
if (keyOfOutput.startsWith(keyOfInput.getSliceRef(0, 4)))
{
//both output and input are part of the same block, skip
continue;
}
sshPtr->totalUnspent_ -= txioPair.second.getValue();
}
else
{
sshPtr->totalUnspent_ += txioPair.second.getValue();
}
}
}
//txio count
sshPtr->totalTxioCount_ += subssh.txioCount_;
//build subssh summary
sshPtr->subsshSummary_[subssh.height_] = subssh.txioCount_;
}
}
//write it
auto& topheader = blockchain_->getHeaderByHash(topScannedBlockHash_);
auto topheight = topheader.getBlockHeight();
LMDBEnv::Transaction putsshtx;
db_->beginDBTransaction(&putsshtx, SSH, LMDB::ReadWrite);
auto& scrAddrMap = scrAddrFilter_->getScrAddrMap();
for (auto& scrAddr : scrAddrMap)
{
auto& ssh = sshMap_[scrAddr.first];
if (!ssh.isInitialized())
{
ssh.uniqueKey_ = scrAddr.first;
}
BinaryData&& sshKey = ssh.getDBKey();
ssh.alreadyScannedUpToBlk_ = topheight;
BinaryWriter bw;
ssh.serializeDBValue(bw, ARMORY_DB_BARE, DB_PRUNE_NONE);
db_->putValue(SSH, sshKey.getRef(), bw.getDataRef());
}
//update sdbi
StoredDBInfo sdbi;
db_->getStoredDBInfo(SSH, sdbi);
sdbi.topScannedBlkHash_ = topScannedBlockHash_;
sdbi.topBlkHgt_ = topheight;
db_->putStoredDBInfo(SSH, sdbi);
}
void BlockchainScanner::undo(Blockchain::ReorganizationState& reorgState)
{
//dont undo subssh, these are skipped by dupID when loading history
BlockHeader* blockPtr = reorgState.prevTopBlock;
map<uint32_t, BlockFileMapPointer> fileMaps_;
map<DB_SELECT, set<BinaryData>> keysToDelete;
map<BinaryData, StoredScriptHistory> sshMap;
set<BinaryData> undoSpentness; //TODO: add spentness DB
//TODO: sanity checks on header ptrs from reorgState
if (reorgState.prevTopBlock->getBlockHeight() <=
reorgState.reorgBranchPoint->getBlockHeight())
throw runtime_error("invalid reorg state");
while (blockPtr != reorgState.reorgBranchPoint)
{
auto currentHeight = blockPtr->getBlockHeight();
auto currentDupId = blockPtr->getDuplicateID();
//create tx to pull subssh data
LMDBEnv::Transaction sshTx;
db_->beginDBTransaction(&sshTx, SUBSSH, LMDB::ReadOnly);
//grab blocks from previous top until branch point
if (blockPtr == nullptr)
throw runtime_error("reorg failed while tracing back to "
"branch point");
auto filenum = blockPtr->getBlockFileNum();
auto fileIter = fileMaps_.find(filenum);
if (fileIter == fileMaps_.end())
{
fileIter = fileMaps_.insert(make_pair(
filenum,
move(blockDataLoader_.get(filenum, false)))).first;
}
auto& filemap = fileIter->second;
BlockData bdata;
bdata.deserialize(filemap.get()->getPtr() + blockPtr->getOffset(),
blockPtr->getBlockSize(), blockPtr);
auto& txns = bdata.getTxns();
for (unsigned i = 0; i < txns.size(); i++)
{
auto& txn = txns[i];
//undo tx outs added by this block
for (unsigned y = 0; y < txn->txouts_.size(); y++)
{
auto& txout = txn->txouts_[y];
BinaryRefReader brr(
txn->data_ + txout.first, txout.second);
brr.advance(8);
unsigned scriptSize = (unsigned)brr.get_var_int();
auto&& scrAddr = BtcUtils::getTxOutScrAddr(
brr.get_BinaryDataRef(scriptSize));
if (!scrAddrFilter_->hasScrAddress(scrAddr))
continue;
//update ssh value and txio count
auto& ssh = sshMap[scrAddr];
if (!ssh.isInitialized())
db_->getStoredScriptHistorySummary(ssh, scrAddr);
if (ssh.alreadyScannedUpToBlk_ < currentHeight)
continue;
brr.resetPosition();
uint64_t value = brr.get_uint64_t();
ssh.totalUnspent_ -= value;
ssh.totalTxioCount_--;
//mark stxo key for deletion
auto&& txoutKey = DBUtils::getBlkDataKey(
currentHeight, currentDupId,
i, y);
keysToDelete[STXO].insert(txoutKey);
//decrement summary count at height, remove entry if necessary
auto& sum = ssh.subsshSummary_[currentHeight];
sum--;
if (sum <= 0)
ssh.subsshSummary_.erase(currentHeight);
}
//undo spends from this block
for (unsigned y = 0; y < txn->txins_.size(); y++)
{
auto& txin = txn->txins_[y];
BinaryDataRef outHash(
txn->data_ + txin.first, 32);
auto&& txKey = db_->getDBKeyForHash(outHash, currentDupId);
if (txKey.getSize() != 6)
continue;
uint16_t txOutId = (uint16_t)READ_UINT32_LE(
txn->data_ + txin.first + 32);
txKey.append(WRITE_UINT16_BE(txOutId));
StoredTxOut stxo;
if (!db_->getStoredTxOut(stxo, txKey))
continue;
//update ssh value and txio count
auto& scrAddr = stxo.getScrAddress();
auto& ssh = sshMap[scrAddr];
if (!ssh.isInitialized())
db_->getStoredScriptHistorySummary(ssh, scrAddr);
if (ssh.alreadyScannedUpToBlk_ < currentHeight)
continue;
ssh.totalUnspent_ += stxo.getValue();
ssh.totalTxioCount_--;
//mark txout key for undoing spentness
undoSpentness.insert(txKey);
//decrement summary count at height, remove entry if necessary
auto& sum = ssh.subsshSummary_[currentHeight];
sum--;
if (sum <= 0)
ssh.subsshSummary_.erase(currentHeight);
}
}
//set blockPtr to prev block
blockPtr = &blockchain_->getHeaderByHash(blockPtr->getPrevHashRef());
}
//at this point we have a map of updated ssh, as well as a
//set of keys to delete from the DB and spentness to undo by stxo key
//stxo
{
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, STXO, LMDB::ReadWrite);
//grab stxos and revert spentness
map<BinaryData, StoredTxOut> stxos;
for (auto& stxoKey : undoSpentness)
{
auto& stxo = stxos[stxoKey];
if (!db_->getStoredTxOut(stxo, stxoKey))
continue;
stxo.spentByTxInKey_.clear();
stxo.spentness_ = TXOUT_UNSPENT;
}
//put updated stxos
for (auto& stxo : stxos)
{
if (stxo.second.isInitialized())
db_->putStoredTxOut(stxo.second);
}
//delete invalidated stxos
auto& stxoKeysToDelete = keysToDelete[STXO];
for (auto& key : stxoKeysToDelete)
db_->deleteValue(STXO, key);
}
auto branchPointHeight =
reorgState.reorgBranchPoint->getBlockHeight();
//ssh
{
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, SSH, LMDB::ReadWrite);
//go thourgh all ssh in scrAddrFilter
auto& scrAddrMap = scrAddrFilter_->getScrAddrMap();
for (auto& scrAddr : scrAddrMap)
{
auto& ssh = sshMap[scrAddr.first];
//if the ssh isn't in our map, pull it from DB
if (!ssh.isInitialized())
{
db_->getStoredScriptHistorySummary(ssh, scrAddr.first);
if (ssh.uniqueKey_.getSize() == 0)
{
sshMap.erase(scrAddr.first);
continue;
}
}
//update alreadyScannedUpToBlk_ to branch point height
if (ssh.alreadyScannedUpToBlk_ > branchPointHeight)
ssh.alreadyScannedUpToBlk_ = branchPointHeight;
}
//write it all up
for (auto& ssh : sshMap)
{
if (!scrAddrFilter_->hasScrAddress(ssh.second.uniqueKey_))
{
LOGWARN << "invalid scrAddr during undo";
continue;
}
BinaryWriter bw;
ssh.second.serializeDBValue(bw, ARMORY_DB_BARE, DB_PRUNE_NONE);
db_->putValue(SSH,
ssh.second.getDBKey().getRef(),
bw.getDataRef());
}
//update SSH sdbi
StoredDBInfo sdbi;
db_->getStoredDBInfo(SSH, sdbi);
sdbi.topScannedBlkHash_ = reorgState.reorgBranchPoint->getThisHash();
sdbi.topBlkHgt_ = branchPointHeight;
db_->putStoredDBInfo(SSH, sdbi);
}
}
void BlockchainScanner::writeBlockData(
shared_ptr<BatchLink> batchLinkPtr)
{
auto getGlobalOffsetForBlock = [&](unsigned height)->size_t
{
auto& header = blockchain_->getHeaderByHeight(height);
size_t val = header.getBlockFileNum();
val *= 128 * 1024 * 1024;
val += header.getOffset();
return val;
};
ProgressCalculator calc(getGlobalOffsetForBlock(
blockchain_->top().getBlockHeight()));
calc.advance(getGlobalOffsetForBlock(startAt_));
auto writeHintsLambda =
[&](const vector<shared_ptr<BlockDataBatch>>& batchVec)->void
{ processAndCommitTxHints(batchVec); };
while (1)
{
if (batchLinkPtr == nullptr)
break;
{
unique_lock<mutex> batchIsReady(batchLinkPtr->readyToWrite_);
}
if (batchLinkPtr->next_ == nullptr)
break;
//start txhint writer thread
thread writeHintsThreadId =
thread(writeHintsLambda, batchLinkPtr->batchVec_);
auto& topheader =
blockchain_->getHeaderByHash(batchLinkPtr->topScannedBlockHash_);
auto topHeight = topheader.getBlockHeight();
//serialize data
map<BinaryData, BinaryWriter> serializedSubSSH;
map<BinaryData, BinaryWriter> serializedStxo;
map<BinaryData, BinaryWriter> serializedTxHints;
map<BinaryData, StoredTxHints> txHints;
{
for (auto& batchPtr : batchLinkPtr->batchVec_)
{
for (auto& ssh : batchPtr->ssh_)
{
for (auto& subssh : ssh.second.subHistMap_)
{
//TODO: modify subssh serialization to fit our needs
BinaryWriter subsshkey;
subsshkey.put_uint8_t(DB_PREFIX_SCRIPT);
subsshkey.put_BinaryData(ssh.first);
subsshkey.put_BinaryData(subssh.first);
auto& bw = serializedSubSSH[subsshkey.getDataRef()];
subssh.second.serializeDBValue(
bw, db_, ARMORY_DB_BARE, DB_PRUNE_NONE);
}
}
for (auto& utxomap : batchPtr->utxos_)
{
auto&& txHashPrefix = utxomap.first.getSliceCopy(0, 4);
StoredTxHints& stxh = txHints[txHashPrefix];
if (stxh.txHashPrefix_.getSize() == 0)
stxh.txHashPrefix_ = txHashPrefix;
for (auto& utxo : utxomap.second)
{
stxh.dbKeyList_.push_back(utxo.second.getDBKeyOfParentTx());
auto& bw = serializedStxo[utxo.second.getDBKey()];
utxo.second.serializeDBValue(
bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true);
}
stxh.preferredDBKey_ = stxh.dbKeyList_.front();
}
}
}
//we've serialized utxos, now let's do another pass for spent txouts
//to make sure they overwrite utxos that were found and spent within
//the same batch
for (auto& batchPtr : batchLinkPtr->batchVec_)
{
for (auto& stxo : batchPtr->spentTxOuts_)
{
auto& bw = serializedStxo[stxo.getDBKey()];
if (bw.getSize() > 0)
bw.reset();
stxo.serializeDBValue(
bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true);
}
}
//write data
{
//txouts
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, STXO, LMDB::ReadWrite);
for (auto& stxo : serializedStxo)
{
//TODO: dont rewrite utxos, check if they are already in DB first
db_->putValue(STXO,
stxo.first.getRef(),
stxo.second.getDataRef());
}
}
{
//subssh
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, SUBSSH, LMDB::ReadWrite);
for (auto& subssh : serializedSubSSH)
{
db_->putValue(
SUBSSH,
subssh.first.getRef(),
subssh.second.getDataRef());
}
//update SUBSSH sdbi
StoredDBInfo sdbi;
db_->getStoredDBInfo(SUBSSH, sdbi);
sdbi.topBlkHgt_ = batchLinkPtr->batchVec_[0]->end_;
sdbi.topScannedBlkHash_ = batchLinkPtr->topScannedBlockHash_;
db_->putStoredDBInfo(SUBSSH, sdbi);
}
//wait on writeHintsThreadId
if (writeHintsThreadId.joinable())
writeHintsThreadId.join();
LOGINFO << "scanned from height #" << batchLinkPtr->batchVec_[0]->start_
<< " to #" << batchLinkPtr->batchVec_[0]->end_;
size_t progVal = getGlobalOffsetForBlock(batchLinkPtr->batchVec_[0]->end_);
calc.advance(progVal);
if (reportProgress_)
progress_(BDMPhase_Rescan,
calc.fractionCompleted(), calc.remainingSeconds(),
progVal);
batchLinkPtr = batchLinkPtr->next_;
}
}
