//--------------------------------------------------------------------------------
SFBool CSlurperApp::Slurp(COptions& options, SFString& message)
{
double start = vrNow();
// Do we have the data for this address cached?
SFString cacheFilename = cachePath(theAccount.addr+".bin");
SFBool needToRead = SFos::fileExists(cacheFilename);
if (options.rerun && theAccount.transactions.getCount())
needToRead=FALSE;
if (needToRead)
{
// Once a transaction is on the blockchain, it will never change
// therefore, we can store them in a binary cache. Here we read
// from a previously stored cache.
SFArchive archive(TRUE, NO_SCHEMA);
if (!archive.Lock(cacheFilename, binaryReadOnly, LOCK_NOWAIT))
{
message = "Could not open file: '" + cacheFilename + "'\n";
return options.cmdFile;
}
theAccount.Serialize(archive);
archive.Close();
}
SFTime now = Now();
SFTime fileTime = SFos::fileLastModifyDate(cacheFilename);
// If the user tells us he/she wants to update the cache, or the cache
// hasn't been updated in five minutes, then update it
SFInt32 nSeconds = MAX(60,config.GetProfileIntGH("SETTINGS", "update_freq", 300));
if (options.slurp || (now - fileTime) > SFTimeSpan(0,0,0,nSeconds))
{
// This is how many records we currently have
SFInt32 origCount = theAccount.transactions.getCount();
SFInt32 nNewBlocks = 0;
SFInt32 nextRecord = origCount;
outErr << "\tSlurping new transactions from blockchain...\n";
SFInt32 nRequests = 0, nRead = 0;
// We already have 'page' pages, so start there.
SFInt32 page = MAX(theAccount.lastPage,1);
// Keep reading until we get less than a full page
SFString contents;
SFBool done = FALSE;
while (!done)
{
SFString url = SFString("https://api.etherscan.io/api?module=account&action=txlist&sort=asc") +
"&address=" + theAccount.addr + "&page=" + asString(page) + "&offset=" + asString(options.pageSize) + "&apikey=" + api.getKey();
// Grab a page of data from the web api
SFString thisPage = urlToString(url);
// See if it's good data, if not, bail
message = nextTokenClear(thisPage, '[');
if (!message.Contains("{\"status\":\"1\",\"message\":\"OK\""))
{
if (message.Contains("{\"status\":\"0\",\"message\":\"No transactions found\",\"result\":"))
message = "No transactions were found for address '" + theAccount.addr + "'. Is it correct?";
return options.cmdFile;
}
contents += thisPage;
SFInt32 nRecords = countOf('}',thisPage)-1;
nRead += nRecords;
outErr << "\tDownloaded " << nRead << " potentially new transactions." << (isTesting?"\n":"\r");
// If we got a full page, there are more to come
done = (nRecords < options.pageSize);
if (!done)
page++;
// Etherscan.io doesn't want more than five pages per second, so sleep for a second
if (++nRequests==4)
{
SFos::sleep(1.0);
nRequests=0;
}
// Make sure we don't spin forever
if (nRead >= options.maxTransactions)
done=TRUE;
}
SFInt32 minBlock=0,maxBlock=0;
findBlockRange(contents, minBlock, maxBlock);
outErr << "\n\tDownload contains blocks from " << minBlock << " to " << maxBlock << "\n";
// Keep track of which last full page we've read
theAccount.lastPage = page;
theAccount.pageSize = options.pageSize;
// pre allocate the array
theAccount.transactions.Grow(nRead);
SFInt32 lastBlock=0;
char *p = cleanUpJson((char *)(const char*)contents);
while (p && *p)
{
CTransaction trans;SFInt32 nFields=0;
p = trans.parseJson(p,nFields);
if (nFields)
{
SFInt32 transBlock = trans.blockNumber;
if (transBlock > theAccount.lastBlock) // add the new transaction if it's in a new block
{
theAccount.transactions[nextRecord++] = trans;
lastBlock = transBlock;
if (!(++nNewBlocks%REP_FREQ))
{
outErr << "\tFound new transaction at block " << transBlock << ". Importing..." << (isTesting?"\n":"\r");
outErr.Flush();
}
}
}
}
if (!isTesting && nNewBlocks) { outErr << "\tFound new transaction at block " << lastBlock << ". Importing...\n"; outErr.Flush(); }
theAccount.lastBlock = lastBlock;
// Write the data if we got new data
SFInt32 newRecords = (theAccount.transactions.getCount() - origCount);
if (newRecords)
{
outErr << "\tWriting " << newRecords << " new records to cache\n";
SFArchive archive(FALSE, NO_SCHEMA);
if (archive.Lock(cacheFilename, binaryWriteCreate, LOCK_CREATE))
{
theAccount.transactions.Sort(sortTransactionsForWrite);
theAccount.Serialize(archive);
archive.Close();
} else
{
message = "Could not open file: '" + cacheFilename + "'\n";
return options.cmdFile;
}
}
}
if (!isTesting)
{
double stop = vrNow();
double timeSpent = stop-start;
fprintf(stderr, "\tLoaded %ld total records in %f seconds\n", theAccount.transactions.getCount(), timeSpent);
fflush(stderr);
}
return (options.cmdFile || theAccount.transactions.getCount()>0);
}
