void ProgressData::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(PROGRESSDATA_CODE);
bw.put_uint8_t((uint8_t)phase_);
bw.put_double(progress_);
bw.put_uint32_t(time_);
bw.put_uint32_t(numericProgress_);
}
BinaryData StackItem_OpCode::serialize(void) const
{
BinaryWriter bw;
bw.put_uint32_t(id_);
bw.put_uint8_t(STACKITEM_OPCODE_PREFIX);
bw.put_uint8_t(opcode_);
return bw.getData();
}
void LedgerEntryVector::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(LEDGERENTRYVECTOR_CODE);
bw.put_var_int(leVec_.size());
for (auto& le : leVec_)
{
auto idSize = le.ID_.size();
size_t totalsize = idSize + 53;
bw.put_var_int(totalsize);
bw.put_var_int(idSize);
bw.put_BinaryData((uint8_t*)le.ID_.c_str(), idSize);
bw.put_uint64_t(le.value_);
bw.put_uint32_t(le.blockNum_);
bw.put_BinaryData(le.txHash_);
bw.put_uint32_t(le.index_);
bw.put_uint32_t(le.txTime_);
BitPacker<uint8_t> bp;
bp.putBit(le.isCoinbase_);
bp.putBit(le.isSentToSelf_);
bp.putBit(le.isChangeBack_);
bp.putBit(le.optInRBF_);
bw.put_BitPacker(bp);
}
}
void TxIOPair::serializeDbValue(BinaryWriter& bw) const
{
uint8_t sersize = 17; //bit pack + amount + txout key
if (hasTxIn())
sersize += 8;
bw.put_uint8_t(sersize);
BitPacker<uint8_t> bitpacker;
bitpacker.putBit(isTxOutFromSelf_);
bitpacker.putBit(isFromCoinbase_);
bitpacker.putBit(hasTxIn());
bitpacker.putBit(isMultisig_);
bitpacker.putBit(isUTXO_);
bitpacker.putBit(isFromSameBlock_);
bw.put_BitPacker(bitpacker);
bw.put_uint64_t(amount_);
bw.put_BinaryData(getDBKeyOfOutput());
if (hasTxIn())
{
bw.put_BinaryData(getDBKeyOfInput());
}
}
BinaryData StackItem_PushData::serialize(void) const
{
BinaryWriter bw;
bw.put_uint32_t(id_);
bw.put_uint8_t(STACKITEM_PUSHDATA_PREFIX);
bw.put_var_int(data_.getSize());
bw.put_BinaryData(data_);
return bw.getData();
}
BinaryData StackItem_SerializedScript::serialize(void) const
{
BinaryWriter bw;
bw.put_uint32_t(id_);
bw.put_uint8_t(STACKITEM_SERSCRIPT_PREFIX);
bw.put_var_int(data_.getSize());
bw.put_BinaryData(data_);
return bw.getData();
}
void BinaryDataVector::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(BINARYDATAVECTOR_CODE);
size_t size = 0;
for (auto& bdo : bdVec_)
size += bdo.getSize();
bw.put_var_int(size);
bw.put_var_int(bdVec_.size());
for (auto& bdo : bdVec_)
{
bw.put_var_int(bdo.getSize());
bw.put_BinaryData(bdo);
}
}
BinaryData SigHashDataSegWit::getDataForSigHashAll(const TransactionStub& stub,
BinaryDataRef subScript, unsigned inputIndex)
{
//grab subscript
auto lastCSoffset = stub.getLastCodeSeparatorOffset(inputIndex);
auto subScriptLen = subScript.getSize() - lastCSoffset;
auto&& subscript = subScript.getSliceRef(lastCSoffset, subScriptLen);
//pre state
computePreState(stub);
//serialize hashdata
BinaryWriter hashdata;
//version
hashdata.put_uint32_t(stub.getVersion());
//hashPrevouts
hashdata.put_BinaryData(hashPrevouts_);
//hashSequence
hashdata.put_BinaryData(hashSequence_);
//outpoint
hashdata.put_BinaryDataRef(stub.getOutpoint(inputIndex));
//script code
hashdata.put_uint8_t(subScriptLen);
hashdata.put_BinaryDataRef(subscript);
//value
hashdata.put_uint64_t(stub.getOutpointValue(inputIndex));
//sequence
hashdata.put_uint32_t(stub.getTxInSequence(inputIndex));
//hashOutputs
hashdata.put_BinaryData(hashOutputs_);
//nLocktime
hashdata.put_uint32_t(stub.getLockTime());
//sighash type
hashdata.put_uint32_t(1);
return hashdata.getData();
}
BinaryData StackItem_MultiSig::serialize(void) const
{
BinaryWriter bw;
bw.put_uint32_t(id_);
bw.put_uint8_t(STACKITEM_MULTISIG_PREFIX);
bw.put_uint16_t(m_);
bw.put_var_int(sigs_.size());
for (auto& sig_pair : sigs_)
{
bw.put_uint16_t(sig_pair.first);
bw.put_var_int(sig_pair.second.getSize());
bw.put_BinaryData(sig_pair.second);
}
return bw.getData();
}
void IntType::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(INTTYPE_CODE);
bw.put_var_int(val_);
}
void ErrorType::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(ERRTYPE_CODE);
bw.put_var_int(err_.size());
bw.put_BinaryData((uint8_t*)err_.c_str(), err_.size());
}
void BinaryDataObject::serialize(BinaryWriter& bw) const
{
bw.put_uint8_t(BINARYDATAOBJECT_CODE);
bw.put_var_int(bd_.getSize());
bw.put_BinaryData(bd_);
}
void BlockchainScanner::writeBlockData(
shared_ptr<BatchLink> batchLinkPtr)
{
auto getGlobalOffsetForBlock = [&](unsigned height)->size_t
{
auto& header = blockchain_->getHeaderByHeight(height);
size_t val = header.getBlockFileNum();
val *= 128 * 1024 * 1024;
val += header.getOffset();
return val;
};
ProgressCalculator calc(getGlobalOffsetForBlock(
blockchain_->top().getBlockHeight()));
calc.advance(getGlobalOffsetForBlock(startAt_));
auto writeHintsLambda =
[&](const vector<shared_ptr<BlockDataBatch>>& batchVec)->void
{ processAndCommitTxHints(batchVec); };
while (1)
{
if (batchLinkPtr == nullptr)
break;
{
unique_lock<mutex> batchIsReady(batchLinkPtr->readyToWrite_);
}
if (batchLinkPtr->next_ == nullptr)
break;
//start txhint writer thread
thread writeHintsThreadId =
thread(writeHintsLambda, batchLinkPtr->batchVec_);
auto& topheader =
blockchain_->getHeaderByHash(batchLinkPtr->topScannedBlockHash_);
auto topHeight = topheader.getBlockHeight();
//serialize data
map<BinaryData, BinaryWriter> serializedSubSSH;
map<BinaryData, BinaryWriter> serializedStxo;
map<BinaryData, BinaryWriter> serializedTxHints;
map<BinaryData, StoredTxHints> txHints;
{
for (auto& batchPtr : batchLinkPtr->batchVec_)
{
for (auto& ssh : batchPtr->ssh_)
{
for (auto& subssh : ssh.second.subHistMap_)
{
//TODO: modify subssh serialization to fit our needs
BinaryWriter subsshkey;
subsshkey.put_uint8_t(DB_PREFIX_SCRIPT);
subsshkey.put_BinaryData(ssh.first);
subsshkey.put_BinaryData(subssh.first);
auto& bw = serializedSubSSH[subsshkey.getDataRef()];
subssh.second.serializeDBValue(
bw, db_, ARMORY_DB_BARE, DB_PRUNE_NONE);
}
}
for (auto& utxomap : batchPtr->utxos_)
{
auto&& txHashPrefix = utxomap.first.getSliceCopy(0, 4);
StoredTxHints& stxh = txHints[txHashPrefix];
if (stxh.txHashPrefix_.getSize() == 0)
stxh.txHashPrefix_ = txHashPrefix;
for (auto& utxo : utxomap.second)
{
stxh.dbKeyList_.push_back(utxo.second.getDBKeyOfParentTx());
auto& bw = serializedStxo[utxo.second.getDBKey()];
utxo.second.serializeDBValue(
bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true);
}
stxh.preferredDBKey_ = stxh.dbKeyList_.front();
}
}
}
//we've serialized utxos, now let's do another pass for spent txouts
//to make sure they overwrite utxos that were found and spent within
//the same batch
for (auto& batchPtr : batchLinkPtr->batchVec_)
{
for (auto& stxo : batchPtr->spentTxOuts_)
{
auto& bw = serializedStxo[stxo.getDBKey()];
if (bw.getSize() > 0)
bw.reset();
stxo.serializeDBValue(
bw, ARMORY_DB_BARE, DB_PRUNE_NONE, true);
}
}
//write data
{
//txouts
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, STXO, LMDB::ReadWrite);
for (auto& stxo : serializedStxo)
{
//TODO: dont rewrite utxos, check if they are already in DB first
db_->putValue(STXO,
stxo.first.getRef(),
stxo.second.getDataRef());
}
}
{
//subssh
LMDBEnv::Transaction tx;
db_->beginDBTransaction(&tx, SUBSSH, LMDB::ReadWrite);
for (auto& subssh : serializedSubSSH)
{
db_->putValue(
SUBSSH,
subssh.first.getRef(),
subssh.second.getDataRef());
}
//update SUBSSH sdbi
StoredDBInfo sdbi;
db_->getStoredDBInfo(SUBSSH, sdbi);
sdbi.topBlkHgt_ = batchLinkPtr->batchVec_[0]->end_;
sdbi.topScannedBlkHash_ = batchLinkPtr->topScannedBlockHash_;
db_->putStoredDBInfo(SUBSSH, sdbi);
}
//wait on writeHintsThreadId
if (writeHintsThreadId.joinable())
writeHintsThreadId.join();
LOGINFO << "scanned from height #" << batchLinkPtr->batchVec_[0]->start_
<< " to #" << batchLinkPtr->batchVec_[0]->end_;
size_t progVal = getGlobalOffsetForBlock(batchLinkPtr->batchVec_[0]->end_);
calc.advance(progVal);
if (reportProgress_)
progress_(BDMPhase_Rescan,
calc.fractionCompleted(), calc.remainingSeconds(),
progVal);
batchLinkPtr = batchLinkPtr->next_;
}
}
