/**
* Inserts a data block into the hashtable
*/
static void bufferDataBlock(Hashtable* ht, FieldData* data)
{
uint16_t hash = getHash(ht, data);
HashBucket* bucket = ht->bucket[hash];
if (bucket == 0) {
/* This slot is still free, place the bucket here */
DPRINTF("bufferDataBlock: creating bucket\n");
ht->bucket[hash] = createBucket(ht, data);
return;
}
/* This slot is already used, search spill chain for equal flow */
while(1) {
if (equalFlow(ht, bucket->data, data)) {
DPRINTF("appending to bucket\n");
aggregateFlow(ht, bucket->data, data);
bucket->expireTime = time(0) + ht->minBufferTime;
/* The flow's data block is no longer needed */
free(data);
break;
}
if (bucket->next == 0) {
DPRINTF("creating bucket\n");
bucket->next = createBucket(ht, data);
break;
}
bucket = (HashBucket*)bucket->next;
}
}
void
FileStorModifiedBucketsTest::fileStorRepliesToRecheckBucketCommands()
{
BucketCheckerInjector bcj(*_node, *this);
TestFileStorComponents c(*this, "fileStorRepliesToRecheckBucketCommands", bcj);
setClusterState("storage:1 distributor:1");
document::BucketId bucket(16, 0);
createBucket(makeSpiBucket(bucket));
c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000));
c.top.waitForMessages(1, MSG_WAIT_TIME);
c.top.reset();
modifyBuckets(0, 1);
c.top.waitForMessages(1, MSG_WAIT_TIME);
assertIsNotifyCommandWithActiveBucket(*c.top.getReply(0));
// If we don't reply to the recheck bucket commands, we won't trigger
// a new round of getModifiedBuckets and recheck commands.
c.top.reset();
createBucket(makeSpiBucket(document::BucketId(16, 1)));
modifyBuckets(1, 1);
c.top.waitForMessages(1, MSG_WAIT_TIME);
assertIsNotifyCommandWithActiveBucket(*c.top.getReply(0));
}
void
MergeBlockingTest::testRejectApplyDiffWhenBucketHasBecomeInconsistent()
{
TestFileStorComponents c(*this, "testRejectApplyDiffWhenBucketHasBecomeInconsistent");
createBucket(leafBucket);
createBucket(innerBucket);
std::shared_ptr<api::ApplyBucketDiffCommand> applyDiff(
createApplyDiff(innerBucket, getNodes()));
c.top.sendDown(applyDiff);
expectAbortedReply<api::ApplyBucketDiffReply>(c.top);
}
void
FileStorModifiedBucketsTest::modifiedBucketsSendNotifyBucketChange()
{
BucketCheckerInjector bcj(*_node, *this);
TestFileStorComponents c(*this, "modifiedBucketsSendNotifyBucketChange", bcj);
setClusterState("storage:1 distributor:1");
uint32_t numBuckets = 10;
for (uint32_t i = 0; i < numBuckets; ++i) {
document::BucketId bucket(16, i);
createBucket(makeSpiBucket(bucket));
c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000));
}
c.top.waitForMessages(numBuckets, MSG_WAIT_TIME);
c.top.reset();
modifyBuckets(0, numBuckets);
c.top.waitForMessages(numBuckets, MSG_WAIT_TIME);
for (uint32_t i = 0; i < 10; ++i) {
assertIsNotifyCommandWithActiveBucket(*c.top.getReply(i));
StorBucketDatabase::WrappedEntry entry(
_node->getStorageBucketDatabase().get(
document::BucketId(16, i), "foo"));
CPPUNIT_ASSERT(entry->info.isActive());
}
}
void HashMap<Key, T, Hasher, EqualKey, Alloc>::
rehashImpl(size_type n){
assert(n != bucket_count_);
if (createBucket(n) ){
refillBucket();
}
}
void
MergeBlockingTest::testRejectMergeForInconsistentLeafBucket()
{
TestFileStorComponents c(*this, "testRejectMergeForInconsistentInnerBucket");
createBucket(innerBucket);
std::shared_ptr<api::MergeBucketCommand> cmd(createMerge(leafBucket));
c.top.sendDown(cmd);
expectAbortedReply<api::MergeBucketReply>(c.top);
CPPUNIT_ASSERT(!bucketExistsInDb(leafBucket));
}
void
MergeBlockingTest::testRejectGetBucketDiffWithInconsistentBucket()
{
TestFileStorComponents c(*this, "testRejectGetBucketDiffWithInconsistentBucket");
CPPUNIT_ASSERT(innerBucket.contains(leafBucket));
createBucket(innerBucket);
std::shared_ptr<api::GetBucketDiffCommand> cmd(createGetDiff(leafBucket, getNodes()));
c.top.sendDown(cmd);
expectAbortedReply<api::GetBucketDiffReply>(c.top);
CPPUNIT_ASSERT(!bucketExistsInDb(leafBucket));
}
void
MergeBlockingTest::testRejectGetDiffReplyWhenBucketHasBecomeInconsistent()
{
TestFileStorComponents c(*this, "testRejectGetDiffReplyWhenBucketHasBecomeInconsistent");
createBucket(innerBucket);
std::shared_ptr<api::GetBucketDiffCommand> getDiff(
createGetDiff(innerBucket, getNodesWithForwarding()));
c.top.sendDown(getDiff);
c.top.waitForMessages(1, MSG_WAIT_TIME);
api::StorageMessage::SP fwdDiff(
c.top.getAndRemoveMessage(api::MessageType::GETBUCKETDIFF));
api::GetBucketDiffCommand& diffCmd(
dynamic_cast<api::GetBucketDiffCommand&>(*fwdDiff));
api::GetBucketDiffReply::SP diffReply(
new api::GetBucketDiffReply(diffCmd));
createBucket(innerBucket2);
c.top.sendDown(diffReply);
expectAbortedReply<api::GetBucketDiffReply>(c.top);
}
/**
* Test case for buckets in ticket 6389558, comment #4.
*/
void
MergeBlockingTest::testRejectMergeWhenLowUsedBitCount()
{
document::BucketId superBucket(1, 0x1);
document::BucketId subBucket(2, 0x1);
CPPUNIT_ASSERT(superBucket.contains(subBucket));
TestFileStorComponents c(*this, "testRejectMergeWithInconsistentBucket");
createBucket(superBucket);
std::shared_ptr<api::MergeBucketCommand> cmd(createMerge(subBucket));
c.top.sendDown(cmd);
expectAbortedReply<api::MergeBucketReply>(c.top);
CPPUNIT_ASSERT(!bucketExistsInDb(subBucket));
}
void
SingleBucketJoinTest::testPersistenceCanHandleSingleBucketJoin()
{
TestFileStorComponents c(*this, "testPersistenceCanHandleSingleBucketJoin");
document::BucketId targetBucket(16, 1);
document::BucketId sourceBucket(17, 1);
createBucket(sourceBucket);
// Make sure it's not empty
c.sendPut(sourceBucket, DocumentIndex(0), PutTimestamp(1000));
expectOkReply<api::PutReply>(c.top);
c.top.getRepliesOnce();
auto cmd = std::make_shared<api::JoinBucketsCommand>(makeDocumentBucket(targetBucket));
cmd->getSourceBuckets().push_back(sourceBucket);
cmd->getSourceBuckets().push_back(sourceBucket);
c.top.sendDown(cmd);
// If single bucket join locking is not working properly, this
// will hang forever.
expectOkReply<api::JoinBucketsReply>(c.top);
}
void TileParser::parseStyleLayers(util::ptr<StyleLayerGroup> group) {
if (!group) {
return;
}
for (const util::ptr<StyleLayer> &layer_desc : group->layers) {
// Cancel early when parsing.
if (obsolete()) {
return;
}
if (layer_desc->isBackground()) {
// background is a special, fake bucket
continue;
} else if (layer_desc->layers) {
// This is a layer group.
parseStyleLayers(layer_desc->layers);
}
if (layer_desc->bucket) {
// This is a singular layer. Check if this bucket already exists. If not,
// parse this bucket.
auto bucket_it = tile.buckets.find(layer_desc->bucket->name);
if (bucket_it == tile.buckets.end()) {
// We need to create this bucket since it doesn't exist yet.
std::unique_ptr<Bucket> bucket = createBucket(layer_desc->bucket);
if (bucket) {
// Bucket creation might fail because the data tile may not
// contain any data that falls into this bucket.
tile.buckets[layer_desc->bucket->name] = std::move(bucket);
}
}
} else {
fprintf(stderr, "[WARNING] layer '%s' does not have child layers or buckets\n", layer_desc->id.c_str());
}
}
}
int main(int argc, char **argv)
{
/*
* Initialize IPRT and create the test.
*/
RTTEST hTest;
int rc = RTTestInitAndCreate("tstRTS3", &hTest);
if (rc)
return rc;
RTTestBanner(hTest);
/*
* If no args, display usage.
*/
if (argc <= 2)
{
RTTestPrintf(hTest, RTTESTLVL_ALWAYS, "Syntax: %s [Access Key] [Secret Key]\n", argv[0]);
return RTTestSkipAndDestroy(hTest, "Missing required arguments\n");
}
RTTestSubF(hTest, "Create S3");
RTS3 hS3;
rc = RTS3Create(&hS3, argv[1], argv[2], "object.storage.network.com", "tstS3-agent/1.0");
if (RT_FAILURE(rc))
{
RTTestIFailed("RTS3Create -> %Rrc", rc);
return RTTestSummaryAndDestroy(hTest);
}
RTTestSub(hTest, "Fetch buckets");
fetchAllBuckets(hS3);
RTTestSub(hTest, "Fetch keys");
fetchAllKeys(hS3, "bla");
#ifdef TSTS3_CREATEBUCKET
RTTestSub(hTest, "Create bucket");
createBucket(hS3, TSTS3_CREATEBUCKET_BUCKETNAME);
fetchAllBuckets(hS3);
deleteBucket(hS3, TSTS3_CREATEBUCKET_BUCKETNAME);
fetchAllBuckets(hS3);
#endif /* TSTS3_CREATEBUCKET */
#ifdef TSTS3_PUTGETKEY
RTTestSub(hTest, "Put key");
createBucket(hS3, TSTS3_PUTGETKEY_BUCKETNAME);
putKey(hS3, TSTS3_PUTGETKEY_BUCKETNAME, TSTS3_PUTGETKEY_KEYNAME, TSTS3_PUTGETKEY_PUTFILE);
fetchAllKeys(hS3, TSTS3_PUTGETKEY_BUCKETNAME);
getKey(hS3, TSTS3_PUTGETKEY_BUCKETNAME, TSTS3_PUTGETKEY_KEYNAME, TSTS3_PUTGETKEY_GETFILE);
deleteKey(hS3, TSTS3_PUTGETKEY_BUCKETNAME, TSTS3_PUTGETKEY_KEYNAME);
fetchAllKeys(hS3, TSTS3_PUTGETKEY_BUCKETNAME);
deleteBucket(hS3, TSTS3_PUTGETKEY_BUCKETNAME);
#endif /* TSTS3_PUTGETKEY */
RTS3Destroy(hS3);
/*
* Summary
*/
return RTTestSummaryAndDestroy(hTest);
}
/**
* Inserts a data block into the hashtable
*/
void FlowHashtable::bufferDataBlock(boost::shared_array<IpfixRecord::Data> data)
{
statRecordsReceived++;
uint32_t nhash = getHash(data.get(), false);
DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash);
HashtableBucket* prevbucket;
HashtableBucket* bucket = lookupBucket(nhash, data.get(), false, &prevbucket);
if (bucket != NULL) {
DPRINTFL(MSG_VDEBUG, "aggregating flow");
aggregateFlow(bucket->data.get(), data.get(), false);
bucket->expireTime = time(0) + minBufferTime;
if (bucket->forceExpireTime>bucket->expireTime) {
exportList.remove(bucket->listNode);
exportList.push(bucket->listNode);
}
} else {
if (biflowAggregation) {
// try reverse flow
uint32_t rhash = getHash(data.get(), true);
DPRINTFL(MSG_VDEBUG, "rhash=%u", rhash);
bucket = lookupBucket(rhash, data.get(), true, &prevbucket);
if (bucket != NULL) {
DPRINTFL(MSG_VDEBUG, "aggregating reverse flow");
if (aggregateFlow(bucket->data.get(), data.get(), true)==1) {
DPRINTFL(MSG_VDEBUG, "reversing whole flow");
// reverse flow
reverseFlowBucket(bucket);
// delete reference from hash table
if (prevbucket==NULL)
buckets[rhash] = bucket->next;
else {
prevbucket->next = bucket->next;
}
if (bucket->next!=NULL)
bucket->next->prev = prevbucket;
// insert into hash table again
nhash = getHash(bucket->data.get(), false);
DPRINTFL(MSG_VDEBUG, "nhash=%u", nhash);
bucket->next = buckets[nhash];
bucket->hash = nhash;
buckets[nhash] = bucket;
bucket->prev = 0;
if (bucket->next != NULL) bucket->next->prev = bucket;
bucket->expireTime = time(0) + minBufferTime;
if (bucket->forceExpireTime>bucket->expireTime) {
exportList.remove(bucket->listNode);
exportList.push(bucket->listNode);
}
}
}
}
if (bucket == NULL) {
DPRINTFL(MSG_VDEBUG, "creating new bucket");
HashtableBucket* n = buckets[nhash];
buckets[nhash] = createBucket(data, 0, n, 0, nhash); // FIXME: insert observationDomainID!
if (n != NULL) n->prev = buckets[nhash];
BucketListElement* node = hbucketIM.getNewInstance();
node->reset();
buckets[nhash]->listNode = node;
node->bucket = buckets[nhash];
exportList.push(node);
}
}
atomic_release(&aggInProgress);
}
int main(int argc, char** argv)
{
String base = "=== [AWS API Init";
std::cout << base << "]: Start===\n";
Aws::SDKOptions options;
// set the options
options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;
// end of options
Aws::InitAPI(options);
// setup
String signVer = SIGN_VER, host = HOST, portStr = PORT,
bucketName = BUCKET;
if (argc == 5)
{
signVer = argv[1];
host = argv[2];
portStr = argv[3];
bucketName = argv[4];
}
auto client = init(host, portStr);
std::cout << base << "]: End ===\n\n";
// call tests here
createBucket(client, bucketName);
std::cout << '\n';
// put object
putObject(client, bucketName, "test.simple", SMALL_TEST_FILE);
putObject(client, bucketName, "test.medium", MED_TEST_FILE);
putObject(client, bucketName, "test.large", LARGE_TEST_FILE);
// put object with metadata
Map metadata;
metadata[METADATA_KEY] = METADATA_VAL;
putObject(client, bucketName, "test.simple.meta", SMALL_TEST_FILE, metadata);
putObject(client, bucketName, "test.medium.meta", MED_TEST_FILE, metadata);
putObject(client, bucketName, "test.large.meta", LARGE_TEST_FILE, metadata);
// put object in parts
putObjectMp(client, bucketName, "test.simple.mp", SMALL_TEST_FILE);
putObjectMp(client, bucketName, "test.medium.mp", MED_TEST_FILE);
putObjectMp(client, bucketName, "test.large.mp", LARGE_TEST_FILE);
// put object in parts with metadata
putObjectMp(client, bucketName, "test.simple.meta.mp", SMALL_TEST_FILE, metadata);
putObjectMp(client, bucketName, "test.medium.meta.mp", MED_TEST_FILE, metadata);
putObjectMp(client, bucketName, "test.large.meta.mp", LARGE_TEST_FILE, metadata);
// head is already tested
// get object
getObject(client, bucketName, "test.simple", SMALL_TEST_FILE);
getObject(client, bucketName, "test.medium", MED_TEST_FILE);
getObject(client, bucketName, "test.large", LARGE_TEST_FILE);
getObject(client, bucketName, "test.simple.mp", SMALL_TEST_FILE);
getObject(client, bucketName, "test.medium.mp", MED_TEST_FILE);
getObject(client, bucketName, "test.large.mp", LARGE_TEST_FILE);
getObject(client, bucketName, "test.simple.meta", SMALL_TEST_FILE, metadata);
getObject(client, bucketName, "test.medium.meta", MED_TEST_FILE, metadata);
getObject(client, bucketName, "test.large.meta", LARGE_TEST_FILE, metadata);
getObject(client, bucketName, "test.simple.meta.mp", SMALL_TEST_FILE, metadata);
getObject(client, bucketName, "test.medium.meta.mp", MED_TEST_FILE, metadata);
getObject(client, bucketName, "test.large.meta.mp", LARGE_TEST_FILE, metadata);
// get fake object
getFakeObject(client, bucketName, "test.noexist");
// range get object
rangeObject(client, bucketName, "test.simple", SMALL_TEST_FILE, 1, 4);
// rangeObject(client, bucketName, "test.simple.mp", SMALL_TEST_FILE, 1, 4);
rangeObject(client, bucketName, "test.large", LARGE_TEST_FILE, 1048576, 40485760);
// rangeObject(client, bucketName, "test.large.mp", LARGE_TEST_FILE, 1048576, 10485760);
// copy object
copyObject(client, bucketName, "test.simple", "test.simple.copy");
getObject(client, bucketName, "test.simple.copy", SMALL_TEST_FILE);
// list object
listObjects(client, bucketName, "", -1);
// delete all objects
deleteAllObjects(client, bucketName);
listObjects(client, bucketName, "", 0);
// put dummy objects
putObject(client, bucketName, "list/test.small.", SMALL_TEST_FILE, Map(), 35);
// multi-page list obj
listObjects(client, bucketName, "list/", 35, 10);
// multi-delete
deleteObjects(client, bucketName, "list/test.small.", 10);
listObjects(client, bucketName, "list/", 25);
// get-put acl
// delete bucket
deleteBucket(client, bucketName);
// end of tests
std::cout << "=== AWS API Shutdown: Start===\n";
Aws::ShutdownAPI(options);
std::cout << "=== AWS API Shutdown: End ===\n";
return 0;
}
