bool TransfersConsumer::onNewBlocks(const CompleteBlock* blocks, uint32_t startHeight, uint32_t count) {
assert(blocks);
struct Tx {
BlockInfo blockInfo;
const ITransactionReader* tx;
};
struct PreprocessedTx : Tx, PreprocessInfo {};
std::vector<PreprocessedTx> preprocessedTransactions;
std::mutex preprocessedTransactionsMutex;
size_t workers = std::thread::hardware_concurrency();
if (workers == 0) {
workers = 2;
}
BlockingQueue<Tx> inputQueue(workers * 2);
std::atomic<bool> stopProcessing(false);
auto pushingThread = std::async(std::launch::async, [&] {
for( uint32_t i = 0; i < count && !stopProcessing; ++i) {
const auto& block = blocks[i].block;
if (!block.is_initialized()) {
continue;
}
// filter by syncStartTimestamp
if (m_syncStart.timestamp && block->timestamp < m_syncStart.timestamp) {
continue;
}
BlockInfo blockInfo;
blockInfo.height = startHeight + i;
blockInfo.timestamp = block->timestamp;
blockInfo.transactionIndex = 0; // position in block
for (const auto& tx : blocks[i].transactions) {
auto pubKey = tx->getTransactionPublicKey();
if (pubKey == NULL_PUBLIC_KEY) {
++blockInfo.transactionIndex;
continue;
}
Tx item = { blockInfo, tx.get() };
inputQueue.push(item);
++blockInfo.transactionIndex;
}
}
inputQueue.close();
});
auto processingFunction = [&] {
Tx item;
std::error_code ec;
while (!stopProcessing && inputQueue.pop(item)) {
PreprocessedTx output;
static_cast<Tx&>(output) = item;
ec = preprocessOutputs(item.blockInfo, *item.tx, output);
if (ec) {
stopProcessing = true;
break;
}
std::lock_guard<std::mutex> lk(preprocessedTransactionsMutex);
preprocessedTransactions.push_back(std::move(output));
}
return ec;
};
std::vector<std::future<std::error_code>> processingThreads;
for (size_t i = 0; i < workers; ++i) {
processingThreads.push_back(std::async(std::launch::async, processingFunction));
}
std::error_code processingError;
for (auto& f : processingThreads) {
try {
std::error_code ec = f.get();
if (!processingError && ec) {
processingError = ec;
}
} catch (const std::system_error& e) {
processingError = e.code();
} catch (const std::exception&) {
processingError = std::make_error_code(std::errc::operation_canceled);
}
}
if (!processingError) {
// sort by block height and transaction index in block
std::sort(preprocessedTransactions.begin(), preprocessedTransactions.end(), [](const PreprocessedTx& a, const PreprocessedTx& b) {
return std::tie(a.blockInfo.height, a.blockInfo.transactionIndex) < std::tie(b.blockInfo.height, b.blockInfo.transactionIndex);
});
for (const auto& tx : preprocessedTransactions) {
processingError = processTransaction(tx.blockInfo, *tx.tx, tx);
if (processingError) {
break;
}
}
}
if (processingError) {
forEachSubscription([&](TransfersSubscription& sub) {
sub.onError(processingError, startHeight);
});
return false;
}
auto newHeight = startHeight + count;
forEachSubscription([newHeight](TransfersSubscription& sub) {
sub.advanceHeight(newHeight);
});
return true;
}
uint32_t TransfersConsumer::onNewBlocks(const CompleteBlock* blocks, uint32_t startHeight, uint32_t count) {
assert(blocks);
assert(count > 0);
struct Tx {
TransactionBlockInfo blockInfo;
const ITransactionReader* tx;
bool isLastTransactionInBlock;
};
struct PreprocessedTx : Tx, PreprocessInfo {};
std::vector<PreprocessedTx> preprocessedTransactions;
std::mutex preprocessedTransactionsMutex;
size_t workers = std::thread::hardware_concurrency();
if (workers == 0) {
workers = 2;
}
BlockingQueue<Tx> inputQueue(workers * 2);
std::atomic<bool> stopProcessing(false);
std::atomic<size_t> emptyBlockCount(0);
auto pushingThread = std::async(std::launch::async, [&] {
for( uint32_t i = 0; i < count && !stopProcessing; ++i) {
const auto& block = blocks[i].block;
if (!block.is_initialized()) {
++emptyBlockCount;
continue;
}
// filter by syncStartTimestamp
if (m_syncStart.timestamp && block->timestamp < m_syncStart.timestamp) {
++emptyBlockCount;
continue;
}
TransactionBlockInfo blockInfo;
blockInfo.height = startHeight + i;
blockInfo.timestamp = block->timestamp;
blockInfo.transactionIndex = 0; // position in block
for (const auto& tx : blocks[i].transactions) {
auto pubKey = tx->getTransactionPublicKey();
if (pubKey == NULL_PUBLIC_KEY) {
++blockInfo.transactionIndex;
continue;
}
bool isLastTransactionInBlock = blockInfo.transactionIndex + 1 == blocks[i].transactions.size();
Tx item = { blockInfo, tx.get(), isLastTransactionInBlock };
inputQueue.push(item);
++blockInfo.transactionIndex;
}
}
inputQueue.close();
});
auto processingFunction = [&] {
Tx item;
std::error_code ec;
while (!stopProcessing && inputQueue.pop(item)) {
PreprocessedTx output;
static_cast<Tx&>(output) = item;
ec = preprocessOutputs(item.blockInfo, *item.tx, output);
if (ec) {
stopProcessing = true;
break;
}
std::lock_guard<std::mutex> lk(preprocessedTransactionsMutex);
preprocessedTransactions.push_back(std::move(output));
}
return ec;
};
std::vector<std::future<std::error_code>> processingThreads;
for (size_t i = 0; i < workers; ++i) {
processingThreads.push_back(std::async(std::launch::async, processingFunction));
}
std::error_code processingError;
for (auto& f : processingThreads) {
try {
std::error_code ec = f.get();
if (!processingError && ec) {
processingError = ec;
}
} catch (const std::system_error& e) {
processingError = e.code();
} catch (const std::exception&) {
processingError = std::make_error_code(std::errc::operation_canceled);
}
}
if (processingError) {
forEachSubscription([&](TransfersSubscription& sub) {
sub.onError(processingError, startHeight);
});
return 0;
}
std::vector<Crypto::Hash> blockHashes = getBlockHashes(blocks, count);
m_observerManager.notify(&IBlockchainConsumerObserver::onBlocksAdded, this, blockHashes);
// sort by block height and transaction index in block
std::sort(preprocessedTransactions.begin(), preprocessedTransactions.end(), [](const PreprocessedTx& a, const PreprocessedTx& b) {
return std::tie(a.blockInfo.height, a.blockInfo.transactionIndex) < std::tie(b.blockInfo.height, b.blockInfo.transactionIndex);
});
uint32_t processedBlockCount = emptyBlockCount;
try {
for (const auto& tx : preprocessedTransactions) {
processTransaction(tx.blockInfo, *tx.tx, tx);
if (tx.isLastTransactionInBlock) {
++processedBlockCount;
m_logger(TRACE) << "Processed block " << processedBlockCount << " of " << count << ", last processed block index " << tx.blockInfo.height <<
", hash " << blocks[processedBlockCount - 1].blockHash;
auto newHeight = startHeight + processedBlockCount - 1;
forEachSubscription([newHeight](TransfersSubscription& sub) {
sub.advanceHeight(newHeight);
});
}
}
} catch (const MarkTransactionConfirmedException& e) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions: failed to confirm transaction " << e.getTxHash() <<
", remove this transaction from all containers and transaction pool";
forEachSubscription([&e](TransfersSubscription& sub) {
sub.deleteUnconfirmedTransaction(e.getTxHash());
});
m_poolTxs.erase(e.getTxHash());
} catch (std::exception& e) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions, exception: " << e.what();
} catch (...) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions, unknown exception";
}
if (processedBlockCount < count) {
uint32_t detachIndex = startHeight + processedBlockCount;
m_logger(ERROR, BRIGHT_RED) << "Not all block transactions are processed, fully processed block count: " << processedBlockCount << " of " << count <<
", last processed block hash " << (processedBlockCount > 0 ? blocks[processedBlockCount - 1].blockHash : NULL_HASH) <<
", detach block index " << detachIndex << " to remove partially processed block";
forEachSubscription([detachIndex](TransfersSubscription& sub) {
sub.onBlockchainDetach(detachIndex);
});
}
return processedBlockCount;
}