map<BinaryData, vector<BinaryData>> ZeroConfContainer::purge( function<bool(const BinaryData&)> filter) { map<BinaryData, vector<BinaryData>> invalidatedKeys; if (!db_) return invalidatedKeys; /*** When a new block appears, it will have added some ZC, but it also may invalidate other ZC, by adding a transcation that consumes the TxOut of one of our ZC (maleability is a good example). This would break a ZC chain starting off that one invalidated ZC, taking away the whole chain. The simpliest way to track down all invalidated ZC is to reparse them all, and compare the new list to the old one. For ZC chains to be parsed properly, it is important ZC transactions are parsed in the order they appeared. ***/ SCOPED_TIMER("purgeZeroConfPool"); map<HashString, HashString> txHashToDBKey; map<BinaryData, Tx> txMap; map<HashString, map<BinaryData, TxIOPair> > txioMap; keyToSpentScrAddr_.clear(); txOutsSpentByZC_.clear(); LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, HISTORY, LMDB::ReadOnly); //parse ZCs anew for (auto ZCPair : txMap_) { const BinaryData& txHash = ZCPair.second.getThisHash(); { BinaryData ZCkey; auto keyIter = txHashToDBKey_.find(txHash); if (ITER_IN_MAP(keyIter, txHashToDBKey_)) ZCkey = keyIter->second; else ZCkey = getNewZCkey(); map<BinaryData, map<BinaryData, TxIOPair> > newTxIO = ZCisMineBulkFilter( ZCPair.second, ZCkey, ZCPair.second.getTxTime(), filter ); //if a relevant ZC was found, add it to our map if (!newTxIO.empty()) { txHashToDBKey[txHash] = ZCkey; txMap[ZCPair.first] = ZCPair.second; for (const auto& scrAddrTxio : newTxIO) { auto& txioPair = txioMap[scrAddrTxio.first]; txioPair.insert(scrAddrTxio.second.begin(), scrAddrTxio.second.end()); } } } } //build the set of invalidated zc dbKeys and delete them from db vector<BinaryData> keysToWrite, keysToDelete; for (auto& tx : txMap_) { if (txMap.find(tx.first) == txMap.end()) keysToDelete.push_back(tx.first); } auto delFromDB = [&, this](void)->void { this->updateZCinDB(keysToWrite, keysToDelete); }; //run in dedicated thread to make sure we can get a RW tx thread delFromDBthread(delFromDB); delFromDBthread.join(); //intersect with current container map for (const auto& saMapPair : txioMap_) { auto saTxioIter = txioMap.find(saMapPair.first); if (saTxioIter == txioMap.end()) { auto& txioVec = invalidatedKeys[saMapPair.first]; for (const auto & txioPair : saMapPair.second) txioVec.push_back(txioPair.first); continue; } for (const auto& txioPair : saMapPair.second) { if (saTxioIter->second.find(txioPair.first) == saTxioIter->second.end()) { auto& txioVec = invalidatedKeys[saMapPair.first]; txioVec.push_back(txioPair.first); } } } //copy new containers over txHashToDBKey_ = txHashToDBKey; txMap_ = txMap; txioMap_ = txioMap; //now purge newTxioMap_ for (auto& newSaTxioPair : newTxioMap_) { auto validTxioIter = txioMap_.find(newSaTxioPair.first); if (ITER_NOT_IN_MAP(validTxioIter, txioMap_)) { newSaTxioPair.second.clear(); continue; } auto& validSaTxioMap = validTxioIter->second; auto& newSaTxioMap = newSaTxioPair.second; auto newTxioIter = newSaTxioMap.begin(); while (newTxioIter != newSaTxioMap.end()) { if (KEY_NOT_IN_MAP(newTxioIter->first, validSaTxioMap)) newSaTxioMap.erase(newTxioIter++); else ++newTxioIter; } } return invalidatedKeys; /* // Rewrite the zero-conf pool file if (hashRmVec.size() > 0) rewriteZeroConfFile(); */ }
bool ZeroConfContainer::parseNewZC(function<bool(const BinaryData&)> filter, bool updateDb) { /*** ZC transcations are pushed to the BDM by another thread (usually the thread managing network connections). This is processed by addRawTx, which is meant to return fast. It grabs the container lock, inserts the new Tx object in the newZCMap_ and return, and sets the new ZC flag. The BDM main thread checks the ZC flag and calls this method. This method processes all new ZC and clears the newZCMap_. It checks how many ZC have been processed against the newZCMap_ size to make sure it can clear the map without deleting any new ZC that may have been added during the process. Note: there is no concurency interference with purging the container (for reorgs and new blocks), as they methods called by the BDM main thread. ***/ uint32_t nProcessed = 0; bool zcIsOurs = false; //grab ZC container lock while (lock_.fetch_or(1, memory_order_acquire)); //copy new ZC map map<BinaryData, Tx> zcMap = newZCMap_; //release lock lock_.store(0, memory_order_release); LMDBEnv::Transaction tx; db_->beginDBTransaction(&tx, HISTORY, LMDB::ReadOnly); while (1) { vector<BinaryData> keysToWrite, keysToDelete; for (const auto& newZCPair : zcMap) { nProcessed++; const BinaryData& txHash = newZCPair.second.getThisHash(); if (txHashToDBKey_.find(txHash) != txHashToDBKey_.end()) continue; //already have this ZC //LOGWARN << "new ZC transcation: " << txHash.toHexStr(); { map<BinaryData, map<BinaryData, TxIOPair> > newTxIO = ZCisMineBulkFilter(newZCPair.second, newZCPair.first, newZCPair.second.getTxTime(), filter ); if (!newTxIO.empty()) { txHashToDBKey_[txHash] = newZCPair.first; txMap_[newZCPair.first] = newZCPair.second; keysToWrite.push_back(newZCPair.first); for (const auto& saTxio : newTxIO) { auto& txioPair = txioMap_[saTxio.first]; txioPair.insert(saTxio.second.begin(), saTxio.second.end()); auto& newTxioPair = newTxioMap_[saTxio.first]; newTxioPair.insert(saTxio.second.begin(), saTxio.second.end()); } zcIsOurs = true; } } } if (updateDb) { //write ZC in the new thread to guaranty we can get a RW tx auto writeNewZC = [&, this](void)->void { this->updateZCinDB(keysToWrite, keysToDelete); }; thread writeNewZCthread(writeNewZC); writeNewZCthread.join(); } //grab ZC container lock while (lock_.fetch_or(1, memory_order_acquire)); //check if newZCMap_ doesnt have new Txn if (nProcessed >= newZCMap_.size()) { //clear map and release lock newZCMap_.clear(); lock_.store(0, memory_order_release); //break out of the loop break; } //else search the new ZC container for unseen ZC map<BinaryData, Tx>::const_iterator newZcIter = newZCMap_.begin(); while (newZcIter != newZCMap_.begin()) { if (ITER_IN_MAP(zcMap.find(newZcIter->first), zcMap)) newZCMap_.erase(newZcIter++); else ++newZcIter; } zcMap = newZCMap_; //reset counter and release lock lock_.store(0, memory_order_release); nProcessed = 0; } return zcIsOurs; }
set<BinaryData> ZeroConfContainer::purge( function<bool(const BinaryData&)> filter) { set<BinaryData> invalidatedKeys; if (!db_) return invalidatedKeys; /*** When a new block appears, it will have added some ZC, but it also may invalidate other ZC, by adding a transcation that consumes the TxOut of one of our ZC (maleability is a good example). This would break a ZC chain starting off that one invalidated ZC, taking away the whole chain. The simpliest way to track down all invalidated ZC is to reparse them all, and compare the new list to the old one. For ZC chains to be parsed properly, it is important ZC transactions are parsed in the order they appeared. ***/ SCOPED_TIMER("purgeZeroConfPool"); map<HashString, HashString> txHashToDBKey; keyToSpentScrAddr_.clear(); txOutsSpentByZC_.clear(); txioMap_.clear(); newTxioMap_.clear(); vector<BinaryData> keysToWrite, keysToDelete; LMDBEnv::Transaction tx; db_->beginDBTransaction(tx, SSH, LMDB::ReadOnly); auto zcIter = txMap_.begin(); //parse ZCs anew while (zcIter != txMap_.end()) { const BinaryData& txHash = zcIter->second.getThisHash(); BinaryData ZCkey; auto keyIter = txHashToDBKey_.find(txHash); if (ITER_IN_MAP(keyIter, txHashToDBKey_)) ZCkey = keyIter->second; else ZCkey = getNewZCkey(); map<BinaryData, map<BinaryData, TxIOPair> > newTxIO = ZCisMineBulkFilter( zcIter->second, ZCkey, zcIter->second.getTxTime(), filter ); //if a relevant ZC was found, add it to our map if (!newTxIO.empty()) { txHashToDBKey[txHash] = ZCkey; for (const auto& saTxio : newTxIO) { auto& txioPair = txioMap_[saTxio.first]; auto satxio = saTxio.second; satxio.insert(txioPair.begin(), txioPair.end()); txioPair = move(satxio); auto& newTxioPair = newTxioMap_[saTxio.first]; satxio = saTxio.second; satxio.insert(newTxioPair.begin(), newTxioPair.end()); newTxioPair = move(satxio); } ++zcIter; } else { invalidatedKeys.insert(ZCkey); keysToDelete.push_back(ZCkey); txMap_.erase(zcIter++); } } //build the set of invalidated zc dbKeys and delete them from db auto delFromDB = [&, this](void)->void { this->updateZCinDB(keysToWrite, keysToDelete); }; //run in dedicated thread to make sure we can get a RW tx thread delFromDBthread(delFromDB); delFromDBthread.join(); return invalidatedKeys; }