void SQLiteObjectStorage::ApplyTransactionWorker::operator() () {
mResponse = new Protocol::Response;
processTransaction();
(*mDestroyMinitransaction)(mt);
cb(mResponse);
delete this;
}
std::error_code TransfersConsumer::onPoolUpdated(const std::vector<std::unique_ptr<ITransactionReader>>& addedTransactions, const std::vector<Hash>& deletedTransactions) {
BlockInfo unconfirmedBlockInfo;
unconfirmedBlockInfo.timestamp = 0;
unconfirmedBlockInfo.height = WALLET_LEGACY_UNCONFIRMED_TRANSACTION_HEIGHT;
std::error_code processingError;
for (auto& cryptonoteTransaction : addedTransactions) {
processingError = processTransaction(unconfirmedBlockInfo, *cryptonoteTransaction.get());
if (processingError) {
break;
}
}
if (processingError) {
for (auto& sub : m_subscriptions) {
sub.second->onError(processingError, WALLET_LEGACY_UNCONFIRMED_TRANSACTION_HEIGHT);
}
return processingError;
}
for (auto& deletedTxHash : deletedTransactions) {
for (auto& sub: m_subscriptions) {
sub.second->deleteUnconfirmedTransaction(*reinterpret_cast<const Hash*>(&deletedTxHash));
}
}
return std::error_code();
}
std::error_code TransfersConsumer::onPoolUpdated(const std::vector<cryptonote::Transaction>& addedTransactions, const std::vector<crypto::hash>& deletedTransactions) {
BlockInfo unconfirmedBlockInfo;
unconfirmedBlockInfo.timestamp = 0;
unconfirmedBlockInfo.height = UNCONFIRMED_TRANSACTION_HEIGHT;
std::error_code processingError;
for (auto& cryptonoteTransaction : addedTransactions) {
auto transaction = CryptoNote::createTransaction(cryptonoteTransaction);
processingError = processTransaction(unconfirmedBlockInfo, *transaction.get());
if (processingError) {
break;
}
}
if (processingError) {
for (auto& sub : m_subscriptions) {
sub.second->onError(processingError, UNCONFIRMED_TRANSACTION_HEIGHT);
}
return processingError;
}
for (auto& deletedTxHash : deletedTransactions) {
for (auto& sub: m_subscriptions) {
sub.second->deleteUnconfirmedTransaction(*reinterpret_cast<const Hash*>(&deletedTxHash));
}
}
return std::error_code();
}
std::error_code TransfersConsumer::onPoolUpdated(const std::vector<std::unique_ptr<ITransactionReader>>& addedTransactions, const std::vector<Hash>& deletedTransactions) {
TransactionBlockInfo unconfirmedBlockInfo;
unconfirmedBlockInfo.timestamp = 0;
unconfirmedBlockInfo.height = WALLET_UNCONFIRMED_TRANSACTION_HEIGHT;
std::error_code processingError;
for (auto& cryptonoteTransaction : addedTransactions) {
m_poolTxs.emplace(cryptonoteTransaction->getTransactionHash());
processingError = processTransaction(unconfirmedBlockInfo, *cryptonoteTransaction.get());
if (processingError) {
for (auto& sub : m_subscriptions) {
sub.second->onError(processingError, WALLET_UNCONFIRMED_TRANSACTION_HEIGHT);
}
return processingError;
}
}
for (auto& deletedTxHash : deletedTransactions) {
m_poolTxs.erase(deletedTxHash);
m_observerManager.notify(&IBlockchainConsumerObserver::onTransactionDeleteBegin, this, deletedTxHash);
for (auto& sub : m_subscriptions) {
sub.second->deleteUnconfirmedTransaction(*reinterpret_cast<const Hash*>(&deletedTxHash));
}
m_observerManager.notify(&IBlockchainConsumerObserver::onTransactionDeleteEnd, this, deletedTxHash);
}
return std::error_code();
}
std::error_code TransfersConsumer::addUnconfirmedTransaction(const ITransactionReader& transaction) {
TransactionBlockInfo unconfirmedBlockInfo;
unconfirmedBlockInfo.height = WALLET_UNCONFIRMED_TRANSACTION_HEIGHT;
unconfirmedBlockInfo.timestamp = 0;
unconfirmedBlockInfo.transactionIndex = 0;
return processTransaction(unconfirmedBlockInfo, transaction);
}
std::error_code TransfersConsumer::processTransaction(const BlockInfo& blockInfo, const ITransactionReader& tx) {
PreprocessInfo info;
auto ec = preprocessOutputs(blockInfo, tx, info);
if (ec) {
return ec;
}
return processTransaction(blockInfo, tx, info);
}
void SQLiteObjectStorage::ApplyTransactionMessage::operator() () {
Protocol::Response response;
mResponse = &response;
processTransaction();
assert(mResponse!=NULL);
(*mDestroyMinitransaction)(mt);
hdr.swap_source_and_destination();
mParent->forward(hdr,response);
mResponse=NULL;
delete this;
}
void BudgetBalancer::processRecords()
{
for (int i = 0; i < m_numRecords; ++i)
{
Q_ASSERT(m_recordedAccounts.contains(i) ||
m_recordedTransactions.contains(i));
if (m_recordedAccounts.contains(i))
{
processAccount(*m_recordedAccounts.find(i));
}
else if (m_recordedTransactions.contains(i))
{
processTransaction(*m_recordedTransactions.find(i));
}
}
m_recordedAccounts.clear();
m_recordedTransactions.clear();
m_numRecords = 0;
}
int main(int argc, char **argv) {
usage(argc, argv); /* modes[0]: tid on/off, modes[1]: predicate on/off*/
MessageHead_t head;
void *body = NULL;
uint32_t len;
Journal_t** journal_array = NULL;
ValidationList_t* validation_list = validationListCreate();
int relation_count = 0;
while (1) {
if (read(0, &head, sizeof(head)) <= 0) {
exit(EXIT_FAILURE);
} // crude error handling, should never happen
if (head.messageLen > 0 ) {
body = malloc(head.messageLen * sizeof(char));
if (read(0, body, head.messageLen) <= 0) {
fprintf(stderr, "Error in Read Body");
exit(EXIT_FAILURE);
} // crude error handling, should never happen
len -= (sizeof(head) + head.messageLen);
}
// And interpret it
switch (head.type) {
case Done:
// validationListPrint(validation_list);
destroySchema(journal_array, relation_count);
validationListDestroy(validation_list);
free(modes);
return EXIT_SUCCESS;
case DefineSchema:
journal_array = processDefineSchema(body, &relation_count, modes);
if (body != NULL)
free(body);
break;
case Transaction:
processTransaction(body,journal_array);
if (body != NULL)
free(body);
break;
case ValidationQueries:
processValidationQueries(body,journal_array,validation_list);
break;
case Flush:
processFlush(body,journal_array,validation_list);
if(modes[3] != 1){
while(flushes != 0){
processFlush(last_flush,journal_array,validation_list);
}
}
if (body != NULL)
free(body);
break;
case Forget:
processForget(body,journal_array,relation_count);
if (body != NULL)
free(body);
break;
default:
exit(EXIT_FAILURE); // crude error handling, should never happen
}
}
return 0;
}
void MkStudioConnect::run()
{
switch (m_state)
{
case STATE_INIT:
m_index = 0;
m_extraIndex = 0;
m_state = STATE_RECEIVE_FIRST_BYTE;
break;
case STATE_RECEIVE_FIRST_BYTE:
if (m_stream->available() > 0) {
const char c = m_stream->read();
if (c == (char)MKSTUDIO_CONNECT_FIRST_BYTE) {
m_state = STATE_RECEIVE;
break;
}
}
break;
case STATE_RECEIVE:
while (m_stream->available() > 0) {
const char c = m_stream->read();
if (c == (char)MKSTUDIO_CONNECT_FIRST_BYTE) {
m_index = 0;
m_extraIndex = 0;
break;
}
if (c == (char)MKSTUDIO_CONNECT_LAST_BYTE) {
if (m_stream->available() > 0) {
m_state = STATE_INIT;
break;
}
m_state = STATE_PROCESS;
break;
}
if (m_extraIndex == 0) {
m_extra = c << 1;
m_extraIndex = 7;
break;
}
else
{
m_buffer[m_index] = c | (m_extra & 0x80);
m_extraIndex--;
m_extra <<= 1;
m_index++;
if (m_index >= sizeof(m_buffer)) {
m_state = STATE_INIT;
break;
}
}
}
break;
case STATE_PROCESS:
processTransaction();
m_state = STATE_TRANSMIT;
break;
case STATE_TRANSMIT:
m_extraIndex = 0;
m_stream->write(MKSTUDIO_CONNECT_FIRST_BYTE);
for (uint8_t i = 0; i < m_index; ++i) {
if (m_extraIndex == 0) {
m_extra = 0;
uint8_t mask = 0x40;
for (uint8_t j = 0; (j < 7) && (i + j) < m_index; ++j) {
if (m_buffer[i + j] & 0x80) {
m_extra |= mask;
}
mask >>= 1;
}
m_extraIndex = 7;
m_stream->write(m_extra);
}
m_stream->write(m_buffer[i] & 0x7F);
m_extraIndex--;
}
m_stream->write(MKSTUDIO_CONNECT_LAST_BYTE);
m_stream->flush();
m_state = STATE_INIT;
break;
}
/// Process data from bitcoind daemon (esp. RPC getrawmempool for live transactions)
void daemonThreadFunc(BlockChain* blockChain)
{
leveldb::WriteBatch batch;
while (true)
{
std::this_thread::sleep_for(std::chrono::seconds(1));
if (requestedQuit)
break;
if (!txCheck)
continue;
try
{
auto transactions = rpcClient->CallMethod("getrawmempool", Json::Value());
BlockInfo blockInfo;
Hash256 latestBlock;
{
boost::lock_guard<boost::mutex> guard(chainMutex);
latestBlock = currentChain.back();
auto blockInfoIt = blocks.find(latestBlock);
if (blockInfoIt == blocks.end())
{
continue;
}
blockInfo = blockInfoIt->second;
}
if (pendingPreviousBlock != latestBlock)
{
pendingPreviousBlock = latestBlock;
// Broadcast new block for live update
Json::Value blockResult;
blockResult["item"] = "block";
convertBlock(blockResult, pendingPreviousBlock, blockInfo);
websocket_server.broadcast_livetx(blockResult.toStyledString().c_str());
{
boost::lock_guard<boost::mutex> guard(pendingTransactionsMutex);
pendingTransactionIndices.clear();
pendingTransactions.clear();
pendingTransactionsByAddress.clear();
}
}
for (auto tx = transactions.begin(); tx != transactions.end(); ++tx)
{
Hash256 txHash;
encodeHexString((uint8_t*) &txHash, 32, (*tx).asString(), true);
batch.Clear();
{
// Already exists?
boost::unique_lock<boost::mutex> guard(pendingTransactionsMutex);
if (pendingTransactionIndices.find(txHash) != pendingTransactionIndices.end())
continue;
}
{
DbTransaction dbTx;
Json::Value parameters(Json::arrayValue);
parameters.append(*tx);
auto rawTx = rpcClient->CallMethod("getrawtransaction", parameters);
auto data = rawTx.asCString();
auto bufferLength = strlen(data) / 2;
llvm::SmallVector<uint8_t, 65536> buffer;
buffer.resize(bufferLength);
encodeHexString(buffer.begin(), bufferLength, data);
BlockChain::BlockTransaction tx2;
if (!blockChain->processSingleTransaction(buffer.begin(), bufferLength, tx2))
assert("could not read tx" && false);
assert(memcmp(tx2.transactionHash, &txHash, 32) == 0);
uint64_t totalOutput = 0;
bool needBroadcast = false;
auto txIndex = dbHelper.txLoad(txHash, dbTx, NULL, NULL);
if (txIndex == 0)
{
{
boost::unique_lock<boost::mutex> guard(pendingTransactionsMutex);
try
{
txIndex = processTransaction(batch, dbTx, tx2, time(NULL), false, &pendingTransactionIndices, &pendingTransactions);
}
catch (std::exception e)
{
batch.Clear();
continue;
}
}
needBroadcast = true;
dbHelper.txSave(batch, txHash, dbTx);
db->Write(leveldb::WriteOptions(), &batch);
}
{
boost::unique_lock<boost::mutex> guard(pendingTransactionsMutex);
pendingTransactionIndices[txHash] = pendingTransactions.size();
pendingTransactions.emplace_back(txHash, dbTx);
// lastPendingTransactions only holds last N items
if (lastPendingTransactions.size() >= lastPendingTransactionsSize)
lastPendingTransactions.pop_front();
lastPendingTransactions.emplace_back(txHash, dbTx);
for (auto output = dbTx.outputs.begin(); output != dbTx.outputs.end(); ++output)
{
if (output->address.IsNull())
continue;
totalOutput += output->value;
pendingTransactionsByAddress.emplace(output->address, pendingTransactions.size() - 1);
}
for (auto input = dbTx.inputs.begin(); input != dbTx.inputs.end(); ++input)
{
if (input->address.IsNull())
continue;
pendingTransactionsByAddress.emplace(input->address, pendingTransactions.size() - 1);
}
}
if (needBroadcast)
{
Json::Value txResult;
// Broadcast tx for live update
txResult["item"] = "tx";
txResult["output"] = (double)totalOutput;
char timeBuffer[65];
convertTime(dbTx.getBestTimeStamp(), timeBuffer, sizeof(timeBuffer));
txResult["time"] = timeBuffer;
txResult["coinage_destroyed"] = dbTx.coinAgeDestroyed / 86400.0;
txResult["hash"] = *tx;
websocket_server.broadcast_livetx(txResult.toStyledString().c_str());
}
}
}
}
catch (std::exception e)
{
boost::lock_guard<boost::mutex> guard(pendingTransactionsMutex);
pendingTransactionIndices.clear();
pendingTransactions.clear();
pendingTransactionsByAddress.clear();
printf("Error processing live transactions: %s!\n", e.what());
}
}
}
bool TransfersConsumer::onNewBlocks(const CompleteBlock* blocks, uint32_t startHeight, uint32_t count) {
assert(blocks);
struct Tx {
BlockInfo blockInfo;
const ITransactionReader* tx;
};
struct PreprocessedTx : Tx, PreprocessInfo {};
std::vector<PreprocessedTx> preprocessedTransactions;
std::mutex preprocessedTransactionsMutex;
size_t workers = std::thread::hardware_concurrency();
if (workers == 0) {
workers = 2;
}
BlockingQueue<Tx> inputQueue(workers * 2);
std::atomic<bool> stopProcessing(false);
auto pushingThread = std::async(std::launch::async, [&] {
for( uint32_t i = 0; i < count && !stopProcessing; ++i) {
const auto& block = blocks[i].block;
if (!block.is_initialized()) {
continue;
}
// filter by syncStartTimestamp
if (m_syncStart.timestamp && block->timestamp < m_syncStart.timestamp) {
continue;
}
BlockInfo blockInfo;
blockInfo.height = startHeight + i;
blockInfo.timestamp = block->timestamp;
blockInfo.transactionIndex = 0; // position in block
for (const auto& tx : blocks[i].transactions) {
auto pubKey = tx->getTransactionPublicKey();
if (pubKey == NULL_PUBLIC_KEY) {
++blockInfo.transactionIndex;
continue;
}
Tx item = { blockInfo, tx.get() };
inputQueue.push(item);
++blockInfo.transactionIndex;
}
}
inputQueue.close();
});
auto processingFunction = [&] {
Tx item;
std::error_code ec;
while (!stopProcessing && inputQueue.pop(item)) {
PreprocessedTx output;
static_cast<Tx&>(output) = item;
ec = preprocessOutputs(item.blockInfo, *item.tx, output);
if (ec) {
stopProcessing = true;
break;
}
std::lock_guard<std::mutex> lk(preprocessedTransactionsMutex);
preprocessedTransactions.push_back(std::move(output));
}
return ec;
};
std::vector<std::future<std::error_code>> processingThreads;
for (size_t i = 0; i < workers; ++i) {
processingThreads.push_back(std::async(std::launch::async, processingFunction));
}
std::error_code processingError;
for (auto& f : processingThreads) {
try {
std::error_code ec = f.get();
if (!processingError && ec) {
processingError = ec;
}
} catch (const std::system_error& e) {
processingError = e.code();
} catch (const std::exception&) {
processingError = std::make_error_code(std::errc::operation_canceled);
}
}
if (!processingError) {
// sort by block height and transaction index in block
std::sort(preprocessedTransactions.begin(), preprocessedTransactions.end(), [](const PreprocessedTx& a, const PreprocessedTx& b) {
return std::tie(a.blockInfo.height, a.blockInfo.transactionIndex) < std::tie(b.blockInfo.height, b.blockInfo.transactionIndex);
});
for (const auto& tx : preprocessedTransactions) {
processingError = processTransaction(tx.blockInfo, *tx.tx, tx);
if (processingError) {
break;
}
}
}
if (processingError) {
forEachSubscription([&](TransfersSubscription& sub) {
sub.onError(processingError, startHeight);
});
return false;
}
auto newHeight = startHeight + count;
forEachSubscription([newHeight](TransfersSubscription& sub) {
sub.advanceHeight(newHeight);
});
return true;
}
uint32_t TransfersConsumer::onNewBlocks(const CompleteBlock* blocks, uint32_t startHeight, uint32_t count) {
assert(blocks);
assert(count > 0);
struct Tx {
TransactionBlockInfo blockInfo;
const ITransactionReader* tx;
bool isLastTransactionInBlock;
};
struct PreprocessedTx : Tx, PreprocessInfo {};
std::vector<PreprocessedTx> preprocessedTransactions;
std::mutex preprocessedTransactionsMutex;
size_t workers = std::thread::hardware_concurrency();
if (workers == 0) {
workers = 2;
}
BlockingQueue<Tx> inputQueue(workers * 2);
std::atomic<bool> stopProcessing(false);
std::atomic<size_t> emptyBlockCount(0);
auto pushingThread = std::async(std::launch::async, [&] {
for( uint32_t i = 0; i < count && !stopProcessing; ++i) {
const auto& block = blocks[i].block;
if (!block.is_initialized()) {
++emptyBlockCount;
continue;
}
// filter by syncStartTimestamp
if (m_syncStart.timestamp && block->timestamp < m_syncStart.timestamp) {
++emptyBlockCount;
continue;
}
TransactionBlockInfo blockInfo;
blockInfo.height = startHeight + i;
blockInfo.timestamp = block->timestamp;
blockInfo.transactionIndex = 0; // position in block
for (const auto& tx : blocks[i].transactions) {
auto pubKey = tx->getTransactionPublicKey();
if (pubKey == NULL_PUBLIC_KEY) {
++blockInfo.transactionIndex;
continue;
}
bool isLastTransactionInBlock = blockInfo.transactionIndex + 1 == blocks[i].transactions.size();
Tx item = { blockInfo, tx.get(), isLastTransactionInBlock };
inputQueue.push(item);
++blockInfo.transactionIndex;
}
}
inputQueue.close();
});
auto processingFunction = [&] {
Tx item;
std::error_code ec;
while (!stopProcessing && inputQueue.pop(item)) {
PreprocessedTx output;
static_cast<Tx&>(output) = item;
ec = preprocessOutputs(item.blockInfo, *item.tx, output);
if (ec) {
stopProcessing = true;
break;
}
std::lock_guard<std::mutex> lk(preprocessedTransactionsMutex);
preprocessedTransactions.push_back(std::move(output));
}
return ec;
};
std::vector<std::future<std::error_code>> processingThreads;
for (size_t i = 0; i < workers; ++i) {
processingThreads.push_back(std::async(std::launch::async, processingFunction));
}
std::error_code processingError;
for (auto& f : processingThreads) {
try {
std::error_code ec = f.get();
if (!processingError && ec) {
processingError = ec;
}
} catch (const std::system_error& e) {
processingError = e.code();
} catch (const std::exception&) {
processingError = std::make_error_code(std::errc::operation_canceled);
}
}
if (processingError) {
forEachSubscription([&](TransfersSubscription& sub) {
sub.onError(processingError, startHeight);
});
return 0;
}
std::vector<Crypto::Hash> blockHashes = getBlockHashes(blocks, count);
m_observerManager.notify(&IBlockchainConsumerObserver::onBlocksAdded, this, blockHashes);
// sort by block height and transaction index in block
std::sort(preprocessedTransactions.begin(), preprocessedTransactions.end(), [](const PreprocessedTx& a, const PreprocessedTx& b) {
return std::tie(a.blockInfo.height, a.blockInfo.transactionIndex) < std::tie(b.blockInfo.height, b.blockInfo.transactionIndex);
});
uint32_t processedBlockCount = emptyBlockCount;
try {
for (const auto& tx : preprocessedTransactions) {
processTransaction(tx.blockInfo, *tx.tx, tx);
if (tx.isLastTransactionInBlock) {
++processedBlockCount;
m_logger(TRACE) << "Processed block " << processedBlockCount << " of " << count << ", last processed block index " << tx.blockInfo.height <<
", hash " << blocks[processedBlockCount - 1].blockHash;
auto newHeight = startHeight + processedBlockCount - 1;
forEachSubscription([newHeight](TransfersSubscription& sub) {
sub.advanceHeight(newHeight);
});
}
}
} catch (const MarkTransactionConfirmedException& e) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions: failed to confirm transaction " << e.getTxHash() <<
", remove this transaction from all containers and transaction pool";
forEachSubscription([&e](TransfersSubscription& sub) {
sub.deleteUnconfirmedTransaction(e.getTxHash());
});
m_poolTxs.erase(e.getTxHash());
} catch (std::exception& e) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions, exception: " << e.what();
} catch (...) {
m_logger(ERROR, BRIGHT_RED) << "Failed to process block transactions, unknown exception";
}
if (processedBlockCount < count) {
uint32_t detachIndex = startHeight + processedBlockCount;
m_logger(ERROR, BRIGHT_RED) << "Not all block transactions are processed, fully processed block count: " << processedBlockCount << " of " << count <<
", last processed block hash " << (processedBlockCount > 0 ? blocks[processedBlockCount - 1].blockHash : NULL_HASH) <<
", detach block index " << detachIndex << " to remove partially processed block";
forEachSubscription([detachIndex](TransfersSubscription& sub) {
sub.onBlockchainDetach(detachIndex);
});
}
return processedBlockCount;
}