void BlockchainScanner::updateSSH() { //loop over all subssh entiers in SUBSSH db, //compile balance, txio count and summary map for each address { StoredDBInfo sdbi; db_->getStoredDBInfo(SSH, sdbi); BlockHeader* sdbiblock = nullptr; try { sdbiblock = &blockchain_->getHeaderByHash(sdbi.topScannedBlkHash_); } catch (...) { sdbiblock = &blockchain_->getHeaderByHeight(0); } if (sdbiblock->isMainBranch()) { if (sdbi.topBlkHgt_ != 0 && sdbi.topBlkHgt_ >= blockchain_->top().getBlockHeight()) { LOGINFO << "no SSH to scan"; return; } } } map<BinaryData, StoredScriptHistory> sshMap_; { StoredScriptHistory* sshPtr = nullptr; LMDBEnv::Transaction historyTx, sshTx; db_->beginDBTransaction(&historyTx, SSH, LMDB::ReadOnly); db_->beginDBTransaction(&sshTx, SUBSSH, LMDB::ReadOnly); auto sshIter = db_->getIterator(SUBSSH); sshIter.seekToFirst(); while (sshIter.advanceAndRead()) { while (sshIter.isValid()) { if (sshPtr != nullptr && sshIter.getKeyRef().contains(sshPtr->uniqueKey_)) break; //new address auto&& subsshkey = sshIter.getKey(); auto sshKey = subsshkey.getSliceRef(1, subsshkey.getSize() - 5); sshPtr = &sshMap_[sshKey]; if (!scrAddrFilter_->hasScrAddress(sshKey)) { LOGWARN << "invalid scrAddr in SUBSSH db"; continue; } //get what's already in the db db_->getStoredScriptHistorySummary(*sshPtr, sshKey); if (sshPtr->isInitialized()) { //set iterator at unscanned height auto hgtx = sshIter.getKeyRef().getSliceRef(-4, 4); auto height = DBUtils::hgtxToHeight(hgtx); if (sshPtr->alreadyScannedUpToBlk_ >= height) { //this ssh has already been scanned beyond the height sshIter is at, //let's set the iterator to the correct height (or the next key) auto&& newKey = sshIter.getKey().getSliceCopy(0, subsshkey.getSize() - 4); auto&& newHgtx = DBUtils::heightAndDupToHgtx( sshPtr->alreadyScannedUpToBlk_ + 1, 0); newKey.append(newHgtx); sshIter.seekTo(newKey); continue; } } else { sshPtr->uniqueKey_ = sshKey; break; } } //sanity check if (!sshIter.isValid()) break; //deser subssh StoredSubHistory subssh; subssh.unserializeDBKey(sshIter.getKeyRef()); //check dupID if (db_->getValidDupIDForHeight(subssh.height_) != subssh.dupID_) continue; subssh.unserializeDBValue(sshIter.getValueRef()); for (auto& txioPair : subssh.txioMap_) { if (!txioPair.second.isMultisig()) { //add up balance if (txioPair.second.hasTxIn()) { //check for same block fund&spend auto&& keyOfOutput = txioPair.second.getDBKeyOfOutput(); auto&& keyOfInput = txioPair.second.getDBKeyOfInput(); if (keyOfOutput.startsWith(keyOfInput.getSliceRef(0, 4))) { //both output and input are part of the same block, skip continue; } sshPtr->totalUnspent_ -= txioPair.second.getValue(); } else { sshPtr->totalUnspent_ += txioPair.second.getValue(); } } } //txio count sshPtr->totalTxioCount_ += subssh.txioCount_; //build subssh summary sshPtr->subsshSummary_[subssh.height_] = subssh.txioCount_; } } //write it auto& topheader = blockchain_->getHeaderByHash(topScannedBlockHash_); auto topheight = topheader.getBlockHeight(); LMDBEnv::Transaction putsshtx; db_->beginDBTransaction(&putsshtx, SSH, LMDB::ReadWrite); auto& scrAddrMap = scrAddrFilter_->getScrAddrMap(); for (auto& scrAddr : scrAddrMap) { auto& ssh = sshMap_[scrAddr.first]; if (!ssh.isInitialized()) { ssh.uniqueKey_ = scrAddr.first; } BinaryData&& sshKey = ssh.getDBKey(); ssh.alreadyScannedUpToBlk_ = topheight; BinaryWriter bw; ssh.serializeDBValue(bw, ARMORY_DB_BARE, DB_PRUNE_NONE); db_->putValue(SSH, sshKey.getRef(), bw.getDataRef()); } //update sdbi StoredDBInfo sdbi; db_->getStoredDBInfo(SSH, sdbi); sdbi.topScannedBlkHash_ = topScannedBlockHash_; sdbi.topBlkHgt_ = topheight; db_->putStoredDBInfo(SSH, sdbi); }
void BlockchainScanner::undo(Blockchain::ReorganizationState& reorgState) { //dont undo subssh, these are skipped by dupID when loading history BlockHeader* blockPtr = reorgState.prevTopBlock; map<uint32_t, BlockFileMapPointer> fileMaps_; map<DB_SELECT, set<BinaryData>> keysToDelete; map<BinaryData, StoredScriptHistory> sshMap; set<BinaryData> undoSpentness; //TODO: add spentness DB //TODO: sanity checks on header ptrs from reorgState if (reorgState.prevTopBlock->getBlockHeight() <= reorgState.reorgBranchPoint->getBlockHeight()) throw runtime_error("invalid reorg state"); while (blockPtr != reorgState.reorgBranchPoint) { auto currentHeight = blockPtr->getBlockHeight(); auto currentDupId = blockPtr->getDuplicateID(); //create tx to pull subssh data LMDBEnv::Transaction sshTx; db_->beginDBTransaction(&sshTx, SUBSSH, LMDB::ReadOnly); //grab blocks from previous top until branch point if (blockPtr == nullptr) throw runtime_error("reorg failed while tracing back to " "branch point"); auto filenum = blockPtr->getBlockFileNum(); auto fileIter = fileMaps_.find(filenum); if (fileIter == fileMaps_.end()) { fileIter = fileMaps_.insert(make_pair( filenum, move(blockDataLoader_.get(filenum, false)))).first; } auto& filemap = fileIter->second; BlockData bdata; bdata.deserialize(filemap.get()->getPtr() + blockPtr->getOffset(), blockPtr->getBlockSize(), blockPtr); auto& txns = bdata.getTxns(); for (unsigned i = 0; i < txns.size(); i++) { auto& txn = txns[i]; //undo tx outs added by this block for (unsigned y = 0; y < txn->txouts_.size(); y++) { auto& txout = txn->txouts_[y]; BinaryRefReader brr( txn->data_ + txout.first, txout.second); brr.advance(8); unsigned scriptSize = (unsigned)brr.get_var_int(); auto&& scrAddr = BtcUtils::getTxOutScrAddr( brr.get_BinaryDataRef(scriptSize)); if (!scrAddrFilter_->hasScrAddress(scrAddr)) continue; //update ssh value and txio count auto& ssh = sshMap[scrAddr]; if (!ssh.isInitialized()) db_->getStoredScriptHistorySummary(ssh, scrAddr); if (ssh.alreadyScannedUpToBlk_ < currentHeight) continue; brr.resetPosition(); uint64_t value = brr.get_uint64_t(); ssh.totalUnspent_ -= value; ssh.totalTxioCount_--; //mark stxo key for deletion auto&& txoutKey = DBUtils::getBlkDataKey( currentHeight, currentDupId, i, y); keysToDelete[STXO].insert(txoutKey); //decrement summary count at height, remove entry if necessary auto& sum = ssh.subsshSummary_[currentHeight]; sum--; if (sum <= 0) ssh.subsshSummary_.erase(currentHeight); } //undo spends from this block for (unsigned y = 0; y < txn->txins_.size(); y++) { auto& txin = txn->txins_[y]; BinaryDataRef outHash( txn->data_ + txin.first, 32); auto&& txKey = db_->getDBKeyForHash(outHash, currentDupId); if (txKey.getSize() != 6) continue; uint16_t txOutId = (uint16_t)READ_UINT32_LE( txn->data_ + txin.first + 32); txKey.append(WRITE_UINT16_BE(txOutId)); StoredTxOut stxo; if (!db_->getStoredTxOut(stxo, txKey)) continue; //update ssh value and txio count auto& scrAddr = stxo.getScrAddress(); auto& ssh = sshMap[scrAddr]; if (!ssh.isInitialized()) db_->getStoredScriptHistorySummary(ssh, scrAddr); if (ssh.alreadyScannedUpToBlk_ < currentHeight) continue; ssh.totalUnspent_ += stxo.getValue(); ssh.totalTxioCount_--; //mark txout key for undoing spentness undoSpentness.insert(txKey); //decrement summary count at height, remove entry if necessary auto& sum = ssh.subsshSummary_[currentHeight]; sum--; if (sum <= 0) ssh.subsshSummary_.erase(currentHeight); } } //set blockPtr to prev block blockPtr = &blockchain_->getHeaderByHash(blockPtr->getPrevHashRef()); } //at this point we have a map of updated ssh, as well as a //set of keys to delete from the DB and spentness to undo by stxo key //stxo { LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, STXO, LMDB::ReadWrite); //grab stxos and revert spentness map<BinaryData, StoredTxOut> stxos; for (auto& stxoKey : undoSpentness) { auto& stxo = stxos[stxoKey]; if (!db_->getStoredTxOut(stxo, stxoKey)) continue; stxo.spentByTxInKey_.clear(); stxo.spentness_ = TXOUT_UNSPENT; } //put updated stxos for (auto& stxo : stxos) { if (stxo.second.isInitialized()) db_->putStoredTxOut(stxo.second); } //delete invalidated stxos auto& stxoKeysToDelete = keysToDelete[STXO]; for (auto& key : stxoKeysToDelete) db_->deleteValue(STXO, key); } auto branchPointHeight = reorgState.reorgBranchPoint->getBlockHeight(); //ssh { LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, SSH, LMDB::ReadWrite); //go thourgh all ssh in scrAddrFilter auto& scrAddrMap = scrAddrFilter_->getScrAddrMap(); for (auto& scrAddr : scrAddrMap) { auto& ssh = sshMap[scrAddr.first]; //if the ssh isn't in our map, pull it from DB if (!ssh.isInitialized()) { db_->getStoredScriptHistorySummary(ssh, scrAddr.first); if (ssh.uniqueKey_.getSize() == 0) { sshMap.erase(scrAddr.first); continue; } } //update alreadyScannedUpToBlk_ to branch point height if (ssh.alreadyScannedUpToBlk_ > branchPointHeight) ssh.alreadyScannedUpToBlk_ = branchPointHeight; } //write it all up for (auto& ssh : sshMap) { if (!scrAddrFilter_->hasScrAddress(ssh.second.uniqueKey_)) { LOGWARN << "invalid scrAddr during undo"; continue; } BinaryWriter bw; ssh.second.serializeDBValue(bw, ARMORY_DB_BARE, DB_PRUNE_NONE); db_->putValue(SSH, ssh.second.getDBKey().getRef(), bw.getDataRef()); } //update SSH sdbi StoredDBInfo sdbi; db_->getStoredDBInfo(SSH, sdbi); sdbi.topScannedBlkHash_ = reorgState.reorgBranchPoint->getThisHash(); sdbi.topBlkHgt_ = branchPointHeight; db_->putStoredDBInfo(SSH, sdbi); } }
void BlockchainScanner::writeBlockData( shared_ptr<BatchLink> batchLinkPtr) { auto getGlobalOffsetForBlock = [&](unsigned height)->size_t { auto& header = blockchain_->getHeaderByHeight(height); size_t val = header.getBlockFileNum(); val *= 128 * 1024 * 1024; val += header.getOffset(); return val; }; ProgressCalculator calc(getGlobalOffsetForBlock( blockchain_->top().getBlockHeight())); calc.advance(getGlobalOffsetForBlock(startAt_)); auto writeHintsLambda = [&](const vector<shared_ptr<BlockDataBatch>>& batchVec)->void { processAndCommitTxHints(batchVec); }; while (1) { if (batchLinkPtr == nullptr) break; { unique_lock<mutex> batchIsReady(batchLinkPtr->readyToWrite_); } if (batchLinkPtr->next_ == nullptr) break; //start txhint writer thread thread writeHintsThreadId = thread(writeHintsLambda, batchLinkPtr->batchVec_); auto& topheader = blockchain_->getHeaderByHash(batchLinkPtr->topScannedBlockHash_); auto topHeight = topheader.getBlockHeight(); //serialize data map<BinaryData, BinaryWriter> serializedSubSSH; map<BinaryData, BinaryWriter> serializedStxo; map<BinaryData, BinaryWriter> serializedTxHints; map<BinaryData, StoredTxHints> txHints; { for (auto& batchPtr : batchLinkPtr->batchVec_) { for (auto& ssh : batchPtr->ssh_) { for (auto& subssh : ssh.second.subHistMap_) { //TODO: modify subssh serialization to fit our needs BinaryWriter subsshkey; subsshkey.put_uint8_t(DB_PREFIX_SCRIPT); subsshkey.put_BinaryData(ssh.first); subsshkey.put_BinaryData(subssh.first); auto& bw = serializedSubSSH[subsshkey.getDataRef()]; subssh.second.serializeDBValue( bw, db_, ARMORY_DB_BARE, DB_PRUNE_NONE); } } for (auto& utxomap : batchPtr->utxos_) { auto&& txHashPrefix = utxomap.first.getSliceCopy(0, 4); StoredTxHints& stxh = txHints[txHashPrefix]; if (stxh.txHashPrefix_.getSize() == 0) stxh.txHashPrefix_ = txHashPrefix; for (auto& utxo : utxomap.second) { stxh.dbKeyList_.push_back(utxo.second.getDBKeyOfParentTx()); auto& bw = serializedStxo[utxo.second.getDBKey()]; utxo.second.serializeDBValue( bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true); } stxh.preferredDBKey_ = stxh.dbKeyList_.front(); } } } //we've serialized utxos, now let's do another pass for spent txouts //to make sure they overwrite utxos that were found and spent within //the same batch for (auto& batchPtr : batchLinkPtr->batchVec_) { for (auto& stxo : batchPtr->spentTxOuts_) { auto& bw = serializedStxo[stxo.getDBKey()]; if (bw.getSize() > 0) bw.reset(); stxo.serializeDBValue( bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true); } } //write data { //txouts LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, STXO, LMDB::ReadWrite); for (auto& stxo : serializedStxo) { //TODO: dont rewrite utxos, check if they are already in DB first db_->putValue(STXO, stxo.first.getRef(), stxo.second.getDataRef()); } } { //subssh LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, SUBSSH, LMDB::ReadWrite); for (auto& subssh : serializedSubSSH) { db_->putValue( SUBSSH, subssh.first.getRef(), subssh.second.getDataRef()); } //update SUBSSH sdbi StoredDBInfo sdbi; db_->getStoredDBInfo(SUBSSH, sdbi); sdbi.topBlkHgt_ = batchLinkPtr->batchVec_[0]->end_; sdbi.topScannedBlkHash_ = batchLinkPtr->topScannedBlockHash_; db_->putStoredDBInfo(SUBSSH, sdbi); } //wait on writeHintsThreadId if (writeHintsThreadId.joinable()) writeHintsThreadId.join(); LOGINFO << "scanned from height #" << batchLinkPtr->batchVec_[0]->start_ << " to #" << batchLinkPtr->batchVec_[0]->end_; size_t progVal = getGlobalOffsetForBlock(batchLinkPtr->batchVec_[0]->end_); calc.advance(progVal); if (reportProgress_) progress_(BDMPhase_Rescan, calc.fractionCompleted(), calc.remainingSeconds(), progVal); batchLinkPtr = batchLinkPtr->next_; } }