void testBackend (beast::String type, std::int64_t const seedValue,
int numObjectsToTest = 2000)
{
std::unique_ptr <Manager> manager (make_Manager ());
DummyScheduler scheduler;
testcase ((beast::String ("Backend type=") + type).toStdString());
beast::StringPairArray params;
beast::File const path (beast::File::createTempFile ("node_db"));
params.set ("type", type);
params.set ("path", path.getFullPathName ());
// Create a batch
Batch batch;
createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
beast::Journal j;
{
// Open the backend
std::unique_ptr <Backend> backend (manager->make_Backend (
params, scheduler, j));
// Write the batch
storeBatch (*backend, batch);
{
// Read it back in
Batch copy;
fetchCopyOfBatch (*backend, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
{
// Reorder and read the copy again
Batch copy;
beast::UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue);
fetchCopyOfBatch (*backend, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
{
// Re-open the backend
std::unique_ptr <Backend> backend (manager->make_Backend (
params, scheduler, j));
// Read it back in
Batch copy;
fetchCopyOfBatch (*backend, ©, batch);
// Canonicalize the source and destination batches
std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
void testBackend (String type, int64 const seedValue, int numObjectsToTest = 2000)
{
DummyScheduler scheduler;
beginTestCase (String ("Backend type=") + type);
StringPairArray params;
File const path (File::createTempFile ("node_db"));
params.set ("type", type);
params.set ("path", path.getFullPathName ());
// Create a batch
Batch batch;
createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
{
// Open the backend
ScopedPointer <Backend> backend (DatabaseImp::createBackend (params, scheduler));
// Write the batch
storeBatch (*backend, batch);
{
// Read it back in
Batch copy;
fetchCopyOfBatch (*backend, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
{
// Reorder and read the copy again
Batch copy;
UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue);
fetchCopyOfBatch (*backend, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
{
// Re-open the backend
ScopedPointer <Backend> backend (DatabaseImp::createBackend (params, scheduler));
// Read it back in
Batch copy;
fetchCopyOfBatch (*backend, ©, batch);
// Canonicalize the source and destination batches
std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
void testImport (String destBackendType, String srcBackendType, int64 seedValue)
{
DummyScheduler scheduler;
File const node_db (File::createTempFile ("node_db"));
StringPairArray srcParams;
srcParams.set ("type", srcBackendType);
srcParams.set ("path", node_db.getFullPathName ());
// Create a batch
Batch batch;
createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
// Write to source db
{
ScopedPointer <Database> src (Database::New ("test", scheduler, srcParams));
storeBatch (*src, batch);
}
Batch copy;
{
// Re-open the db
ScopedPointer <Database> src (Database::New ("test", scheduler, srcParams));
// Set up the destination database
File const dest_db (File::createTempFile ("dest_db"));
StringPairArray destParams;
destParams.set ("type", destBackendType);
destParams.set ("path", dest_db.getFullPathName ());
ScopedPointer <Database> dest (Database::New ("test", scheduler, destParams));
beginTestCase (String ("import into '") + destBackendType + "' from '" + srcBackendType + "'");
// Do the import
dest->import (*src);
// Get the results of the import
fetchCopyOfBatch (*dest, ©, batch);
}
// Canonicalize the source and destination batches
std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
expect (areBatchesEqual (batch, copy), "Should be equal");
}
nsresult sbMockDevice::ProcessBatch(Batch & aBatch)
{
std::insert_iterator<std::vector<nsRefPtr<sbRequestItem> > >
insertIter(mBatch, mBatch.end());
std::copy(aBatch.begin(), aBatch.end(), insertIter);
/* don't process, let the js deal with it */
return NS_OK;
}
void OplogBufferCollection::pushEvenIfFull(OperationContext* txn, const Value& value) {
// This oplog entry is a sentinel
if (value.isEmpty()) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
_sentinels.push(_lastPushedTimestamp);
_count++;
_cvNoLongerEmpty.notify_all();
return;
}
Batch valueBatch = {value};
pushAllNonBlocking(txn, valueBatch.begin(), valueBatch.end());
}
nsresult sbRequestThreadQueue::ClearRequestsNoLock(Batch & aBatch)
{
NS_ENSURE_STATE(mLock);
// Copy all the requests on the queue to aBatch
std::insert_iterator<Batch> insertIter(aBatch, aBatch.end());
std::copy(mRequestQueue.begin(), mRequestQueue.end(), insertIter);
// Release all of our objects
std::for_each(mRequestQueue.begin(), mRequestQueue.end(), ReleaseRequestItem);
// Now that we have copied the requests clear our request queue
mRequestQueue.clear();
return NS_OK;
}
void storeBatch (Batch const& batch)
{
EncodedBlob::Pool::ScopedItem item (m_blobPool);
for (Batch::const_iterator iter (batch.begin());
iter != batch.end(); ++iter)
{
EncodedBlob& encoded (item.getObject ());
encoded.prepare (*iter);
int rv (sp_set (m_db,
encoded.getKey(), m_keyBytes,
encoded.getData(), encoded.getSize()));
if (rv != 0)
{
String s;
s << "Sophia failed with error code " << rv;
Throw (std::runtime_error (s.toStdString()), __FILE__, __LINE__);
}
}
}
bool Test::test(const Batch& actual, const Batch& expected, const string& msg) {
bool anyfail = false;
Batch::batch_iter a, e;
a = actual.begin();
e = expected.begin();
test(actual.Size(), expected.Size(), msg + " batch sizes");
for (; a != actual.end(); ++a) {
if (!qtest(e != expected.end(), msg + " too many output items")) {
anyfail = true;
break;
}
const Batch::BatchItem& aitem = *a;
const Batch::BatchItem& eitem = *e;
if (qtest(aitem.op, eitem.op, msg + " batch item op")) {
if (!eitem.node) {
g_log.warning() << "Expected batch item node is missing";
} else if (qtest(aitem.node, " - batch node")) {
if (Batch::OP_DELETE == aitem.op) {
// We can't look at the deleted node's name/value, because it may
// have been *deleted*. It's just a handle that should not be
// dereferenced; a label for the node that was removed.
// We also can't directly compare the expected node with the
// observed, because they're not the same (the Test-user is not
// actually providing the IncParser's previously-output node, as that
// would be inconvenient).
// So for now, don't do a check here, we can improve the API later.
//anyfail |= !qtest(aitem.node, eitem.node, " - - batch deleted node");
} else {
anyfail |= !qtest(aitem.node->name, eitem.node->name, " - - batch node name");
anyfail |= !qtest(aitem.node->value, eitem.node->value, " - - batch node value");
}
} else {
fail(msg + " emit an output item with no node");
anyfail = true;
}
anyfail |= !qtest(aitem.pos, eitem.pos, " - batch item pos");
} else {
anyfail = true;
}
++e;
}
for (; a != actual.end(); ++a) {
const Batch::BatchItem& aitem = *a;
if (Batch::OP_DELETE == aitem.op) {
// Cannot display deleted item, as it may have been erased from memory.
// It is only a "token".
fail(msg + " extra deleted item in batch");
} else {
fail(msg + " extra item in batch: " + a->Print());
}
anyfail = true;
}
for (; e != expected.end(); ++e) {
fail(msg + " item not found in batch: " + e->Print());
anyfail = true;
}
if (anyfail) {
fail(msg + ": failures in batch");
}
return !anyfail;
}
WindowResponse ParserWindow::Input(const Batch& ibatch) {
g_log.info() << "Updating incParser " << m_incParser.Name() << " with batch: " << ibatch.Print();
m_incParser.ApplyBatch(ibatch);
const statik::STree& root = m_incParser.GetRoot();
WindowResponse response;
if (root.GetState().IsBad()) {
g_log.debug() << " - Not extracting output, because root is bad!";
return response;
}
// Find first item in incParser's output list, and draw everything
response.actions.push_back(WindowAction(WindowAction::MOVE, 0, 0, 0));
Batch batch;
g_log.debug() << "Extracting changes from IncParser";
m_incParser.ExtractChanges(batch);
g_log.info() << "Printing WindowResponse list for batch of size " << batch.Size() << ": " << batch;
if (m_nodes) {
g_log.debug() << "m_nodes is: " << *m_nodes;
}
if (!batch.IsEmpty()) {
for (Batch::batch_iter i = batch.begin(); i != batch.end(); ++i) {
switch (i->op) {
case Batch::OP_INSERT: {
g_log.debug() << "Insert node: " << i->node->name << ":" << i->node->value;
List* node = new List(i->node->name, i->node->value);
if (i->pos) {
node_iter pos_i = m_nodeMap.find(i->pos);
if (m_nodeMap.end() == pos_i) {
throw ISError("Received invalid pos for Insert");
}
List* pos = pos_i->second;
g_log.debug() << " - at pos: " << pos->name << ":" << pos->value;
node->right = pos->right;
node->left = pos;
pos->right = node;
if (node->right) {
node->right->left = node;
}
g_log.debug() << "Inserted " << *node;
if (node->left) {
g_log.debug() << " - with left: " << *node->left;
if (node->left->right) {
g_log.debug() << " - - which has right: " << *node->left->right;
}
}
if (node->right) {
g_log.debug() << " - with right: " << *node->right;
if (node->right->left) {
g_log.debug() << " - - which has left: " << *node->right->left;
}
}
} else {
g_log.debug() << "Setting this node as m_nodes";
node->right = m_nodes;
if (node->right) {
node->right->left = node;
}
m_nodes = node;
}
m_nodeMap.insert(std::make_pair(i->node, node));
g_log.debug() << "Done inserting node";
}
break;
case Batch::OP_DELETE: {
// Careful! i->node is a pointer to dead data, it's just a lookup
// handle for us.
node_mod_iter node_i = m_nodeMap.find(i->node);
if (m_nodeMap.end() == node_i) {
throw ISError("Received invalid node for Delete");
}
List* node = node_i->second;
g_log.debug() << "Delete node: " << node->name << ":" << node->value;
List* left = node->left;
List* right = node->right;
if (left && left->right == node) {
left->right = right;
}
if (right && right->left == node) {
right->left = left;
}
if (m_nodes == node) {
g_log.debug() << "Deleted node is m_nodes";
if (left) {
g_log.debug() << "going left";
g_log.debug() << "going left to " << *left << " from " << *node;
m_nodes = left;
} else if (right) {
g_log.debug() << "going right";
g_log.debug() << "going right to " << *right << " from " << *node;
m_nodes = right;
} else {
g_log.debug() << "nowhere to go; m_nodes is clear";
m_nodes = NULL;
}
}
g_log.debug() << "Deleted " << *node;
if (node->left) {
g_log.debug() << " - with left: " << *node->left;
}
if (node->right) {
g_log.debug() << " - with right: " << *node->right;
}
delete node;
m_nodeMap.erase(node_i);
g_log.debug() << "Done deleting node";
}
break;
case Batch::OP_UPDATE: {
g_log.debug() << "Update node: " << i->node->name << ":" << i->node->value;
/*
node_mod_iter node_i = m_nodeMap.find(i->node);
if (m_nodeMap.end() == node_i) {
throw ISError("Received invalid node for Update");
}
List* node = node_i->second;
g_log.debug() << "Update node: " << node->name << ":" << node->value << " -> " << i->node->name << ":" << i->node->value;
node->value = i->node->value;
*/
}
break;
default:
throw ISError("Unknown batch operation " + Batch::UnMapBatchOp(i->op));
}
}
string old_str = m_str;
m_str = "";
bool first = true;
List* node = m_nodes;
while (node) {
if (first) {
first = false;
} else {
m_str += " ";
}
m_str += node->name;
/*if (!node->value.empty()) {
m_str += "[" + node->value + "]";
}*/
node = node->right;
}
g_log.info() << "Printing str: '" << m_str << "'";
for (size_t i = 0; i < m_str.size(); ++i) {
response.actions.push_back(WindowAction(WindowAction::INSERT, 0, i, m_str[i]));
}
// Clear out the rest of the line
for (size_t i = m_str.size(); i < old_str.size(); ++i) {
response.actions.push_back(WindowAction(WindowAction::INSERT, 0, i, ' '));
}
response.batch.Accept(batch);
}
return response;
}
void testNodeStore (String type,
bool const useEphemeralDatabase,
bool const testPersistence,
int64 const seedValue,
int numObjectsToTest = 2000)
{
DummyScheduler scheduler;
String s;
s << String ("NodeStore backend '") + type + "'";
if (useEphemeralDatabase)
s << " (with ephemeral database)";
beginTestCase (s);
File const node_db (File::createTempFile ("node_db"));
StringPairArray nodeParams;
nodeParams.set ("type", type);
nodeParams.set ("path", node_db.getFullPathName ());
File const temp_db (File::createTempFile ("temp_db"));
StringPairArray tempParams;
if (useEphemeralDatabase)
{
tempParams.set ("type", type);
tempParams.set ("path", temp_db.getFullPathName ());
}
// Create a batch
Batch batch;
createPredictableBatch (batch, 0, numObjectsToTest, seedValue);
{
// Open the database
ScopedPointer <Database> db (Database::New ("test", scheduler, nodeParams, tempParams));
// Write the batch
storeBatch (*db, batch);
{
// Read it back in
Batch copy;
fetchCopyOfBatch (*db, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
{
// Reorder and read the copy again
Batch copy;
UnitTestUtilities::repeatableShuffle (batch.size (), batch, seedValue);
fetchCopyOfBatch (*db, ©, batch);
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
if (testPersistence)
{
{
// Re-open the database without the ephemeral DB
ScopedPointer <Database> db (Database::New ("test", scheduler, nodeParams));
// Read it back in
Batch copy;
fetchCopyOfBatch (*db, ©, batch);
// Canonicalize the source and destination batches
std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
expect (areBatchesEqual (batch, copy), "Should be equal");
}
if (useEphemeralDatabase)
{
// Verify the ephemeral db
ScopedPointer <Database> db (Database::New ("test",
scheduler, tempParams, StringPairArray ()));
// Read it back in
Batch copy;
fetchCopyOfBatch (*db, ©, batch);
// Canonicalize the source and destination batches
std::sort (batch.begin (), batch.end (), NodeObject::LessThan ());
std::sort (copy.begin (), copy.end (), NodeObject::LessThan ());
expect (areBatchesEqual (batch, copy), "Should be equal");
}
}
}
