// @returns the block that represents the difference between m_previousBlock and m_currentBlock.
// (i.e. all the transactions we executed).
void State::commitToMine(BlockChain const& _bc)
{
if (m_currentBlock.sha3Transactions != h256() || m_currentBlock.sha3Uncles != h256())
{
Addresses uncleAddresses;
for (auto i: RLP(m_currentUncles))
uncleAddresses.push_back(i[2].toHash<Address>());
unapplyRewards(uncleAddresses);
}
cnote << "Commiting to mine on" << m_previousBlock.hash;
RLPStream uncles;
Addresses uncleAddresses;
if (m_previousBlock != BlockInfo::genesis())
{
// Find uncles if we're not a direct child of the genesis.
// cout << "Checking " << m_previousBlock.hash << ", parent=" << m_previousBlock.parentHash << endl;
auto us = _bc.details(m_previousBlock.parentHash).children;
assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent!
uncles.appendList(us.size() - 1); // one fewer - uncles precludes our parent from the list of grandparent's children.
for (auto const& u: us)
if (u != m_previousBlock.hash) // ignore our own parent - it's not an uncle.
{
BlockInfo ubi(_bc.block(u));
ubi.fillStream(uncles, true);
uncleAddresses.push_back(ubi.coinbaseAddress);
}
}
else
uncles.appendList(0);
applyRewards(uncleAddresses);
RLPStream txs(m_transactions.size());
for (auto const& i: m_transactions)
i.fillStream(txs);
txs.swapOut(m_currentTxs);
uncles.swapOut(m_currentUncles);
m_currentBlock.sha3Transactions = sha3(m_currentTxs);
m_currentBlock.sha3Uncles = sha3(m_currentUncles);
// Commit any and all changes to the trie that are in the cache, then update the state root accordingly.
commit();
cnote << "stateRoot:" << m_state.root();
// cnote << m_state;
// cnote << *this;
m_currentBlock.stateRoot = m_state.root();
m_currentBlock.parentHash = m_previousBlock.hash;
}
Addresses ClientBase::addresses(BlockNumber _block) const
{
Addresses ret;
for (auto const& i: asOf(_block).addresses())
ret.push_back(i.first);
return ret;
}
Addresses KeyManager::accounts() const
{
Addresses ret;
ret.reserve(m_addrLookup.size());
for (auto const& i: m_addrLookup)
if (m_keyInfo.count(i.second) > 0)
ret.push_back(i.first);
return ret;
}
Addresses ClientManager::getClients()
{
IceUtil::Mutex::Lock lock(m_mutex);
Addresses addresses;
map_type::iterator it = m_clients.begin();
for(; it != m_clients.end(); ++it){
addresses.push_back(it->first);
}
return addresses;
}
})E";
dev::Addresses dev::eth::childDaos()
{
Addresses daos;
json_spirit::mValue val;
json_spirit::read_string(c_childDaos, val);
json_spirit::mObject obj = val.get_obj();
for (auto const &items: obj)
{
json_spirit::mArray array = items.second.get_array();
for (auto account: array)
{
daos.push_back(Address(account.get_obj()["address"].get_str()));
daos.push_back(Address(account.get_obj()["extraBalanceAccount"].get_str()));
}
}
return daos;
}
void ClientManager::removeExpired(Addresses &expiredClients)
{
IceUtil::Mutex::Lock dirLock(m_mutex);
expiredClients.clear();
time_t now = time((time_t*)NULL);
map_type::iterator it = m_clients.begin();
while(it != m_clients.end()){
if (now - it->second > m_expireTime){
expiredClients.push_back(it->first);
m_clients.erase(it);
it = m_clients.begin(); // rewind the iterator!!
}
else{
++it;
}
}
}
int32_t JobServiceImpl::create(const MapReduceJob& job, const Ice::Current&)
{
if (m_db->m_workers.isSafeMode()){
///@todo throw SafeModeException
LOG4CPLUS_DEBUG(m_logger, "JobServiceImpl::create() We are in safe mode");
return -1;
}
// we do not assign task when in safe mode
WorkerManager::ToggleSafeMode safeMode(m_db->m_workers);
// get chunks info of the input file(s)
LocatedChunks allLocatedChunks;
try{
vector<string> names;
FilesInfo files;
FileInfo fileinfo = m_db->m_tfs.getFileInfo(job.inputFile);
if (fileinfo.flag & FileModeTypeDir){
// the input is a dir
files = m_db->m_tfs.listDirectory(job.inputFile);
string dir = job.inputFile;
if (dir[dir.size() - 1] != '/')
{
dir += '/';
}
for (int i = 0; i < files.size(); i++){
names.push_back(dir + files[i].name);
}
}
else{
// the input is a file
names.push_back(job.inputFile);
}
for (int i = 0; i < names.size(); i++){
// for each file in the dir
LocatedChunks lchks = m_db->m_tfsFile.getChunksInfo(names[i]);
allLocatedChunks.insert(allLocatedChunks.begin(), lchks.begin(), lchks.end());
LOG4CPLUS_DEBUG(m_logger, "Input file " << i+1 << " " << names[i] << " has " << lchks.size() << " chunks");
}
LOG4CPLUS_DEBUG(m_logger, "Input file(s) " << job.inputFile << " totally has " << allLocatedChunks.size() << " chunks");
if (allLocatedChunks.empty()){
LOG4CPLUS_WARN(m_logger, "JobServiceImpl::create() TFS file or dir" << job.inputFile << " doesn't exist or has no chunks");
return -1;
}
}catch(std::exception &ex){
LOG4CPLUS_WARN(m_logger, "JobServiceImpl::create() catch " << ex.what());
return -1;
}
catch(...){
LOG4CPLUS_WARN(m_logger, "JobServiceImpl::create(): Unknown tfs exception!!!");
return -1;
}
// create output dir if necessary
try{
if (!m_db->m_tfs.existDirectory(job.outputFile)){
m_db->m_tfs.createDirectory(job.outputFile);
}
}
catch(std::exception &ex){
LOG4CPLUS_WARN(m_logger, "JobServiceImpl::create() catch " << ex.what());
return -1;
}
MapReduceJob jobSpec(job);
// we assign each chunk a map task
jobSpec.mapTaskNum = allLocatedChunks.size();
// the number of reduce tasks is less than the number of workers or too large
if (jobSpec.reduceTaskNum < MasterConfig::MinReduceTaskNumPerJob){
jobSpec.reduceTaskNum = MasterConfig::MinReduceTaskNumPerJob;
}
else if (jobSpec.reduceTaskNum > MasterConfig::MaxReduceTaskNumPerJob){
jobSpec.reduceTaskNum = MasterConfig::MaxReduceTaskNumPerJob;
}
int32_t jobId = m_db->m_jobs.add(jobSpec, allLocatedChunks);
LOG4CPLUS_INFO(m_logger, "Create job for " << jobSpec);
// add map tasks
vector<int32_t> mapTaskIds;
try{
for (int i = 0; i < allLocatedChunks.size(); i++){
Address address;
Addresses candidates;
for (int j = 0; j < allLocatedChunks[i].locations.size(); j++){
address.name = allLocatedChunks[i].locations[j].name;
address.port = allLocatedChunks[i].locations[j].port;
candidates.push_back(address);
}
Address worker = m_db->m_workers.selectWorker(candidates);
int32_t taskId = m_db->m_tasks.addMapTask(jobId, i, worker);
mapTaskIds.push_back(taskId);
}
}catch (SafeModeException &ex){
LOG4CPLUS_WARN(m_logger, "JobServiceImpl::create() catch " << ex.what());
m_db->cancelJob(jobId);
return -1;
}
// add reduce tasks
Addresses workers = m_db->m_workers.getAllWorkers();
for (int i = 0; i < jobSpec.reduceTaskNum; i++){
LOG4CPLUS_DEBUG(m_logger, " reduceTask worker " << workers[i%workers.size()] << " "<< i << " "<< workers.size() );
int32_t reduceTaskId = m_db->m_tasks.addReduceTask(jobId, i, workers[i%workers.size()]);
// add trans tasks
for (int j = 0; j < mapTaskIds.size(); j++){
m_db->m_tasks.addTransTask(jobId, mapTaskIds[j], reduceTaskId);
}
}
// @add by Chen Rishan
// add job schedule infomation in workermanager
// Tasks tasks = m_db->m_tasks.getAllTasks(jobId);
// Job waitingJob = m_db->m_jobs.getJob(jobId);
// JobPriority jobPrio = waitingJob.second.jobInitPriority;
// m_db->m_workers.setJobPriority(tasks, jobPrio);
LOG4CPLUS_INFO(m_logger, "Created job " << jobId);
return jobId;
}
void State::commitToMine(BlockChain const& _bc)
{
uncommitToMine();
// cnote << "Committing to mine on block" << m_previousBlock.hash.abridged();
#ifdef ETH_PARANOIA
commit();
cnote << "Pre-reward stateRoot:" << m_state.root();
#endif
m_lastTx = m_db;
Addresses uncleAddresses;
RLPStream unclesData;
unsigned unclesCount = 0;
if (m_previousBlock.number != 0)
{
// Find great-uncles (or second-cousins or whatever they are) - children of great-grandparents, great-great-grandparents... that were not already uncles in previous generations.
// cout << "Checking " << m_previousBlock.hash << ", parent=" << m_previousBlock.parentHash << endl;
set<h256> knownUncles = _bc.allUnclesFrom(m_currentBlock.parentHash);
auto p = m_previousBlock.parentHash;
for (unsigned gen = 0; gen < 6 && p != _bc.genesisHash(); ++gen, p = _bc.details(p).parent)
{
auto us = _bc.details(p).children;
assert(us.size() >= 1); // must be at least 1 child of our grandparent - it's our own parent!
for (auto const& u: us)
if (!knownUncles.count(u)) // ignore any uncles/mainline blocks that we know about.
{
BlockInfo ubi(_bc.block(u));
ubi.streamRLP(unclesData, WithNonce);
++unclesCount;
uncleAddresses.push_back(ubi.coinbaseAddress);
}
}
}
MemoryDB tm;
GenericTrieDB<MemoryDB> transactionsTrie(&tm);
transactionsTrie.init();
MemoryDB rm;
GenericTrieDB<MemoryDB> receiptsTrie(&rm);
receiptsTrie.init();
RLPStream txs;
txs.appendList(m_transactions.size());
for (unsigned i = 0; i < m_transactions.size(); ++i)
{
RLPStream k;
k << i;
RLPStream receiptrlp;
m_receipts[i].streamRLP(receiptrlp);
receiptsTrie.insert(&k.out(), &receiptrlp.out());
RLPStream txrlp;
m_transactions[i].streamRLP(txrlp);
transactionsTrie.insert(&k.out(), &txrlp.out());
txs.appendRaw(txrlp.out());
}
txs.swapOut(m_currentTxs);
RLPStream(unclesCount).appendRaw(unclesData.out(), unclesCount).swapOut(m_currentUncles);
m_currentBlock.transactionsRoot = transactionsTrie.root();
m_currentBlock.receiptsRoot = receiptsTrie.root();
m_currentBlock.logBloom = logBloom();
m_currentBlock.sha3Uncles = sha3(m_currentUncles);
// Apply rewards last of all.
applyRewards(uncleAddresses);
// Commit any and all changes to the trie that are in the cache, then update the state root accordingly.
commit();
// cnote << "Post-reward stateRoot:" << m_state.root().abridged();
// cnote << m_state;
// cnote << *this;
m_currentBlock.gasUsed = gasUsed();
m_currentBlock.stateRoot = m_state.root();
m_currentBlock.parentHash = m_previousBlock.hash;
}
u256 State::enact(bytesConstRef _block, BlockChain const& _bc, bool _checkNonce)
{
// m_currentBlock is assumed to be prepopulated and reset.
#if !ETH_RELEASE
BlockInfo bi(_block, _checkNonce);
assert(m_previousBlock.hash == bi.parentHash);
assert(m_currentBlock.parentHash == bi.parentHash);
assert(rootHash() == m_previousBlock.stateRoot);
#endif
if (m_currentBlock.parentHash != m_previousBlock.hash)
BOOST_THROW_EXCEPTION(InvalidParentHash());
// Populate m_currentBlock with the correct values.
m_currentBlock.populate(_block, _checkNonce);
m_currentBlock.verifyInternals(_block);
// cnote << "playback begins:" << m_state.root();
// cnote << m_state;
MemoryDB tm;
GenericTrieDB<MemoryDB> transactionsTrie(&tm);
transactionsTrie.init();
MemoryDB rm;
GenericTrieDB<MemoryDB> receiptsTrie(&rm);
receiptsTrie.init();
LastHashes lh = getLastHashes(_bc, (unsigned)m_previousBlock.number);
// All ok with the block generally. Play back the transactions now...
unsigned i = 0;
for (auto const& tr: RLP(_block)[1])
{
RLPStream k;
k << i;
transactionsTrie.insert(&k.out(), tr.data());
execute(lh, tr.data());
RLPStream receiptrlp;
m_receipts.back().streamRLP(receiptrlp);
receiptsTrie.insert(&k.out(), &receiptrlp.out());
++i;
}
if (transactionsTrie.root() != m_currentBlock.transactionsRoot)
{
cwarn << "Bad transactions state root!";
BOOST_THROW_EXCEPTION(InvalidTransactionsStateRoot());
}
if (receiptsTrie.root() != m_currentBlock.receiptsRoot)
{
cwarn << "Bad receipts state root.";
cwarn << "Block:" << toHex(_block);
cwarn << "Block RLP:" << RLP(_block);
cwarn << "Calculated: " << receiptsTrie.root();
for (unsigned j = 0; j < i; ++j)
{
RLPStream k;
k << j;
auto b = asBytes(receiptsTrie.at(&k.out()));
cwarn << j << ": ";
cwarn << "RLP: " << RLP(b);
cwarn << "Hex: " << toHex(b);
cwarn << TransactionReceipt(&b);
}
cwarn << "Recorded: " << m_currentBlock.receiptsRoot;
auto rs = _bc.receipts(m_currentBlock.hash);
for (unsigned j = 0; j < rs.receipts.size(); ++j)
{
auto b = rs.receipts[j].rlp();
cwarn << j << ": ";
cwarn << "RLP: " << RLP(b);
cwarn << "Hex: " << toHex(b);
cwarn << rs.receipts[j];
}
BOOST_THROW_EXCEPTION(InvalidReceiptsStateRoot());
}
if (m_currentBlock.logBloom != logBloom())
{
cwarn << "Bad log bloom!";
BOOST_THROW_EXCEPTION(InvalidLogBloom());
}
// Initialise total difficulty calculation.
u256 tdIncrease = m_currentBlock.difficulty;
// Check uncles & apply their rewards to state.
set<h256> nonces = { m_currentBlock.nonce };
Addresses rewarded;
set<h256> knownUncles = _bc.allUnclesFrom(m_currentBlock.parentHash);
for (auto const& i: RLP(_block)[2])
{
if (knownUncles.count(sha3(i.data())))
BOOST_THROW_EXCEPTION(UncleInChain(knownUncles, sha3(i.data()) ));
BlockInfo uncle = BlockInfo::fromHeader(i.data());
if (nonces.count(uncle.nonce))
BOOST_THROW_EXCEPTION(DuplicateUncleNonce());
BlockInfo uncleParent(_bc.block(uncle.parentHash));
if ((bigint)uncleParent.number < (bigint)m_currentBlock.number - 7)
BOOST_THROW_EXCEPTION(UncleTooOld());
uncle.verifyParent(uncleParent);
nonces.insert(uncle.nonce);
tdIncrease += uncle.difficulty;
rewarded.push_back(uncle.coinbaseAddress);
}
applyRewards(rewarded);
// Commit all cached state changes to the state trie.
commit();
// Hash the state trie and check against the state_root hash in m_currentBlock.
if (m_currentBlock.stateRoot != m_previousBlock.stateRoot && m_currentBlock.stateRoot != rootHash())
{
cwarn << "Bad state root!";
cnote << "Given to be:" << m_currentBlock.stateRoot;
cnote << TrieDB<Address, OverlayDB>(&m_db, m_currentBlock.stateRoot);
cnote << "Calculated to be:" << rootHash();
cnote << m_state;
cnote << *this;
// Rollback the trie.
m_db.rollback();
BOOST_THROW_EXCEPTION(InvalidStateRoot());
}
if (m_currentBlock.gasUsed != gasUsed())
{
// Rollback the trie.
m_db.rollback();
BOOST_THROW_EXCEPTION(InvalidGasUsed() << RequirementError(bigint(gasUsed()), bigint(m_currentBlock.gasUsed)));
}
return tdIncrease;
}
u256 State::playback(bytesConstRef _block, BlockInfo const& _grandParent, bool _fullCommit)
{
if (m_currentBlock.parentHash != m_previousBlock.hash)
throw InvalidParentHash();
// cnote << "playback begins:" << m_state.root();
// cnote << m_state;
// All ok with the block generally. Play back the transactions now...
for (auto const& i: RLP(_block)[1])
execute(i.data());
// Initialise total difficulty calculation.
u256 tdIncrease = m_currentBlock.difficulty;
// Check uncles & apply their rewards to state.
// TODO: Check for uniqueness of uncles.
Addresses rewarded;
for (auto const& i: RLP(_block)[2])
{
BlockInfo uncle = BlockInfo::fromHeader(i.data());
if (m_previousBlock.parentHash != uncle.parentHash)
throw InvalidUncle();
if (_grandParent)
uncle.verifyParent(_grandParent);
tdIncrease += uncle.difficulty;
rewarded.push_back(uncle.coinbaseAddress);
}
applyRewards(rewarded);
// Commit all cached state changes to the state trie.
commit();
// Hash the state trie and check against the state_root hash in m_currentBlock.
if (m_currentBlock.stateRoot != rootHash())
{
cwarn << "Bad state root!";
cnote << "Given to be:" << m_currentBlock.stateRoot;
cnote << TrieDB<Address, Overlay>(&m_db, m_currentBlock.stateRoot);
cnote << "Calculated to be:" << rootHash();
cnote << m_state;
cnote << *this;
// Rollback the trie.
m_db.rollback();
throw InvalidStateRoot();
}
if (_fullCommit)
{
// Commit the new trie to disk.
m_db.commit();
m_previousBlock = m_currentBlock;
resetCurrent();
}
else
{
m_db.rollback();
resetCurrent();
}
return tdIncrease;
}
