void testDiskCache_exampleCom( void ) {
CacheLayer *testCache = createSimpleCache(true, true, true);
testCache->purgeFromCache(SHA256::convertFromHex(EXAMPLE_HASH));
doExampleComTest(testCache);
tearDownCache();
// Ensure that it is now in the disk cache.
doExampleComTest(createSimpleCache(false, true, false));
}
/** If you want to change the ordering of the cache layers.
* This may be safe, but it's probably not smart to use except when
* constructing or destroying a CacheLayer.
*/
void setNext(CacheLayer *newNext) {
if (mNext) {
mNext->setResponder(NULL);
}
mNext = newNext;
if (mNext) {
mNext->setResponder(this);
}
}
shared_ptr<Patch> CachingPyramidFeatureExtractor::extractOutputCopying(CacheLayer& layer, int x, int y) const {
CacheKey key(x, y);
unordered_map<CacheKey, shared_ptr<Patch>, CacheKey::hash>& layerCache = layer.getCache();
auto iterator = layerCache.find(key);
if (iterator == layerCache.end()) {
shared_ptr<Patch> patch = extractor->extract(layer.getIndex(), x, y);
layerCache.emplace(key, patch);
return patch;
}
return make_shared<Patch>(*(iterator->second));
}
shared_ptr<Patch> CachingPyramidFeatureExtractor::extractInputCopying(CacheLayer& layer, int x, int y) const {
CacheKey key(x, y);
unordered_map<CacheKey, shared_ptr<Patch>, CacheKey::hash>& layerCache = layer.getCache();
auto iterator = layerCache.find(key);
if (iterator == layerCache.end()) {
shared_ptr<Patch> patch = extractor->extract(layer.getIndex(), x, y);
if (patch) // store a copy of the patch only if it exists
layerCache.emplace(key, make_shared<Patch>(*patch));
return patch;
}
return iterator->second;
}
void testMemoryCache_exampleCom( void ) {
CacheLayer *disk = createDiskCache();
CacheLayer *memory = createMemoryCache(disk);
// test disk cache.
SILOG(transfer,debug,"Testing disk cache...");
doExampleComTest(memory);
// test memory cache.
memory->setNext(NULL);
// ensure it is not using the disk cache.
SILOG(transfer,debug,"Testing memory cache...");
doExampleComTest(memory);
}
void addToCache(const Fingerprint &fileId, const DenseDataPtr &data) {
if (mNext) {
mNext->addToCache(fileId, data);
} else {
populateCache(fileId, data);
}
}
/**
* Query this cache layer. If successful, call callback with the data and also
* call populateCache in order to populate the previous cache levels.
*
* @param fid A unique identifier corresponding to the file (contains a hash).
* @param requestedRange A Range object specifying a single range that you need.
* @param callback To be called with the data if successful, or NULL if failed.
* @return false, if the callback happened synchronously (i.e. in memory cache)
*/
virtual void getData(const RemoteFileId &fid, const Range &requestedRange,
const TransferCallback&callback) {
if (mNext) {
mNext->getData(fid, requestedRange, callback);
} else {
// need some way to signal error
callback(NULL);
}
}
void testRange( void ) {
Transfer::RemoteFileId exampleComUri (SHA256::convertFromHex(EXAMPLE_HASH), URI(URIContext(), SERVER_URI));
CacheLayer *http = createTransferLayer();
CacheLayer *disk = createDiskCache(http);
CacheLayer *memory = createMemoryCache(disk);
using std::tr1::placeholders::_1;
memory->purgeFromCache(exampleComUri.fingerprint());
{
Transfer::MutableDenseDataPtr expect(new Transfer::DenseData(Transfer::Range(2, 6, Transfer::LENGTH)));
memcpy(expect->writableData(), "TML>\r\n", (size_t)expect->length());
memory->getData(exampleComUri,
(Transfer::Range)*expect,
std::tr1::bind(&CacheLayerTestSuite::compareCallback, this, expect, _1));
}
{
Transfer::MutableDenseDataPtr expect(new Transfer::DenseData(Transfer::Range(8, 6, Transfer::LENGTH)));
memcpy(expect->writableData(), "<HEAD>", (size_t)expect->length());
memory->getData(exampleComUri,
(Transfer::Range)*expect,
std::tr1::bind(&CacheLayerTestSuite::compareCallback, this, expect, _1));
}
waitFor(2);
{
Transfer::MutableDenseDataPtr expect(new Transfer::DenseData(Transfer::Range(2, 12, Transfer::LENGTH)));
memcpy(expect->writableData(), "TML>\r\n<HEAD>", (size_t)expect->length());
memory->setNext(NULL);
memory->getData(exampleComUri,
(Transfer::Range)*expect,
std::tr1::bind(&CacheLayerTestSuite::compareCallback, this, expect, _1));
}
waitFor(3);
}
void testCleanup( void ) {
Transfer::RemoteFileId testUri (SHA256::computeDigest("01234"), URI(URIContext(), "http://www.google.com/"));
Transfer::RemoteFileId testUri2 (SHA256::computeDigest("56789"), URI(URIContext(), "http://www.google.com/intl/en_ALL/images/logo.gif"));
Transfer::RemoteFileId exampleComUri (SHA256::convertFromHex(EXAMPLE_HASH), URI(URIContext(), SERVER_URI));
using std::tr1::placeholders::_1;
Transfer::TransferCallback simpleCB = std::tr1::bind(&CacheLayerTestSuite::simpleCallback, this, _1);
Transfer::TransferCallback checkNullCB = std::tr1::bind(&CacheLayerTestSuite::checkNullCallback, this, _1);
CacheLayer *transfer = createSimpleCache(true, true, true);
transfer->purgeFromCache(testUri.fingerprint());
transfer->purgeFromCache(testUri2.fingerprint());
transfer->getData(testUri, Transfer::Range(true), checkNullCB);
transfer->getData(testUri2, Transfer::Range(true), checkNullCB);
// localhost should be in disk cache--make sure it waits for the request.
// disk cache is required to finish all pending requests before cleaning up.
transfer->getData(exampleComUri, Transfer::Range(true), simpleCB);
// do not wait--we want to clean up these requests.
}
/** Goes up the heirararchy of cache layers filling in data.
* Note that you must *NOT* call the callback until you have
* populated the cache.
*
* @TODO: Handle two requests for the same chunk at the same time.
* */
inline void populateParentCaches(const Fingerprint &fileId, const DenseDataPtr &data) {
if (mRespondTo) {
mRespondTo->populateCache(fileId, data);
}
}
/**
* Purges this hash from all subsequent caches. In general, it is not
* useful to do this manually, since the CachePolicy handles freeing
* extra data, and it is usually assumed that the cache is correct.
*
* Validity checking should probably be added elsewhere--though it is
* not really feasable for incomplete downloads if we used a range.
*/
virtual void purgeFromCache(const Fingerprint &fileId) {
if (mNext) {
mNext->purgeFromCache(fileId);
}
}
virtual ~CacheLayer() {
if (mNext) {
mNext->setResponder(NULL);
}
}
void testOverlappingRange( void ) {
using std::tr1::placeholders::_1;
Transfer::TransferCallback simpleCB = std::tr1::bind(&CacheLayerTestSuite::simpleCallback, this, _1);
int numtests = 0;
CacheLayer *http = createTransferLayer();
CacheLayer *disk = createDiskCache(http);
CacheLayer *memory = createMemoryCache(disk);
Transfer::RemoteFileId exampleComUri (SHA256::convertFromHex(EXAMPLE_HASH), URI(URIContext(), SERVER_URI));
memory->purgeFromCache(exampleComUri.fingerprint());
std::string diskFile = "diskCache/" + exampleComUri.fingerprint().convertToHexString() + ".part";
// First test: GetData something which will overlap with the next two.
printf("1\n");
http->getData(exampleComUri,
Transfer::Range(6, 10, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=1);
// Now getData two pieces (both of these should kick out the first one)
printf("2/3\n");
http->getData(exampleComUri,
Transfer::Range(2, 8, Transfer::BOUNDS),
simpleCB);
http->getData(exampleComUri,
Transfer::Range(8, 14, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=2);
// Now check that an overlapping range from before doesn't cause problems
printf("4\n");
http->getData(exampleComUri,
Transfer::Range(6, 13, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=1);
printf("5 -> THIS ONE IS FAILING\n");
// Everything here should be cached
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(5, 8, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=1);
printf("6 -> THIS ONE IS FAILING?\n");
// And the whole range we just got.
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(2, 14, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=1);
printf("7\n");
// getDatas from 2 to the end. -- should not be cached
// and should overwrite all previous ranges because it is bigger.
memory->setNext(disk);
memory->getData(exampleComUri,
Transfer::Range(2, true),
simpleCB);
waitFor(numtests+=1);
using std::tr1::placeholders::_1;
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(2, true),
std::tr1::bind(&CacheLayerTestSuite::checkOneDenseDataCallback, this, _1));
waitFor(numtests+=1);
// Whole file trumps anything else.
memory->setNext(disk);
memory->getData(exampleComUri,
Transfer::Range(true),
simpleCB);
waitFor(numtests+=1);
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(2, true),
std::tr1::bind(&CacheLayerTestSuite::checkOneDenseDataCallback, this, _1));
waitFor(numtests+=1);
// should be cached
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(2, 14, Transfer::BOUNDS),
simpleCB);
waitFor(numtests+=1);
// should be 1--end should be cached as well.
memory->setNext(NULL);
memory->getData(exampleComUri,
Transfer::Range(1, 10, Transfer::BOUNDS, true),
simpleCB);
waitFor(numtests+=1);
}
