void dbrm_clear()
{
DBRM dbrm;
VSS vss;
VBBM vbbm;
int err, vssShmid, vbbmShmid, txnID = 1, i;
struct shmid_ds vssShminfo[3], vbbmShminfo[3];
LBIDRange_v ranges;
LBIDRange range;
VBRange_v freelist;
err = dbrm.clear();
CPPUNIT_ASSERT(err == ERR_OK);
// grab the size of vss and vbbm shmsegs somehow
vss.lock(VSS::READ);
vbbm.lock(VBBM::READ);
vssShmid = vss.getShmid();
vbbmShmid = vbbm.getShmid();
err = shmctl(vssShmid, IPC_STAT, &vssShminfo[0]);
CPPUNIT_ASSERT(err == 0);
err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[0]);
CPPUNIT_ASSERT(err == 0);
vss.release(VSS::READ);
vbbm.release(VBBM::READ);
// do begin, write, end vbcopy for 150k blocks
cerr << endl << "Adding 150k block entries. ";
range.start = 1;
range.size = 150000;
ranges.push_back(range);
err = dbrm.beginVBCopy(txnID, ranges, freelist);
CPPUNIT_ASSERT(err == 0);
for (i = range.start; (uint32_t) i < range.size; i++) {
if (i % 50000 == 0)
cerr << " ... " << i;
err = dbrm.writeVBEntry(txnID, i, 1, i);
CPPUNIT_ASSERT(err == 0);
}
err = dbrm.endVBCopy(txnID, ranges);
CPPUNIT_ASSERT(err == 0);
cerr << " done." << endl;
// grab the sizes again
vss.lock(VSS::READ);
vbbm.lock(VBBM::READ);
vssShmid = vss.getShmid();
vbbmShmid = vbbm.getShmid();
err = shmctl(vssShmid, IPC_STAT, &vssShminfo[1]);
CPPUNIT_ASSERT(err == 0);
err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[1]);
CPPUNIT_ASSERT(err == 0);
vss.release(VSS::READ);
vbbm.release(VBBM::READ);
// make sure they grew
CPPUNIT_ASSERT(vssShminfo[0].shm_segsz < vssShminfo[1].shm_segsz);
CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz < vbbmShminfo[1].shm_segsz);
dbrm.clear();
// check that the new size is the same as the original
vss.lock(VSS::READ);
vbbm.lock(VBBM::READ);
vssShmid = vss.getShmid();
vbbmShmid = vbbm.getShmid();
err = shmctl(vssShmid, IPC_STAT, &vssShminfo[2]);
CPPUNIT_ASSERT(err == 0);
err = shmctl(vbbmShmid, IPC_STAT, &vbbmShminfo[2]);
CPPUNIT_ASSERT(err == 0);
vss.release(VSS::READ);
vbbm.release(VBBM::READ);
CPPUNIT_ASSERT(vssShminfo[0].shm_segsz == vssShminfo[2].shm_segsz);
CPPUNIT_ASSERT(vbbmShminfo[0].shm_segsz == vbbmShminfo[2].shm_segsz);
}
//assumes write lock
void VBBM::getBlocks(int num, OID_t vbOID, vector<VBRange>& freeRanges, VSS& vss, bool flushPMCache)
{
int blocksLeftInFile, blocksGathered = 0, i;
uint32_t fileIndex;
uint32_t firstFBO, lastFBO;
VBRange range;
vector<VBRange>::iterator it;
vector<LBID_t> flushList;
freeRanges.clear();
fileIndex = addVBFileIfNotExists(vbOID);
/*
for (i = 0; i < vbbm->nFiles; i++) {
cout << "file " << i << " vbOID=" << files[i].OID << " size=" << files[i].fileSize
<< endl;
}
*/
if ((uint32_t) num > files[fileIndex].fileSize/BLOCK_SIZE) {
cout << "num = " << num << " filesize = " << files[fileIndex].fileSize << endl;
log("VBBM::getBlocks(): num is larger than the size of the version buffer",
logging::LOG_TYPE_DEBUG);
throw logging::VBBMBufferOverFlowExcept
("VBBM::getBlocks(): num is larger than the size of the version buffer");
}
while ((vbbm->vbCurrentSize + num) > vbbm->vbCapacity) {
growVBBM();
//cout << " requested num = " << num << " and Growing vbbm ... " << endl;
}
while (blocksGathered < num) {
blocksLeftInFile = (files[fileIndex].fileSize - files[fileIndex].nextOffset)/BLOCK_SIZE;
int blocksLeft = num - blocksGathered;
range.vbOID = files[fileIndex].OID;
range.vbFBO = files[fileIndex].nextOffset/BLOCK_SIZE;
range.size = (blocksLeftInFile >= blocksLeft ? blocksLeft : blocksLeftInFile);
makeUndoRecord(&files[fileIndex], sizeof(VBFileMetadata));
if (range.size == (uint32_t) blocksLeftInFile)
files[fileIndex].nextOffset = 0;
else
files[fileIndex].nextOffset += range.size * BLOCK_SIZE;
blocksGathered += range.size;
freeRanges.push_back(range);
}
//age the returned blocks out of the VB
for (it = freeRanges.begin(); it != freeRanges.end(); it++) {
uint32_t firstChunk, lastChunk;
vbOID = it->vbOID;
firstFBO = it->vbFBO;
lastFBO = it->vbFBO + it->size - 1;
/* Age out at least 100 blocks at a time to reduce the # of times we have to do it.
* How to detect when it needs to be done and when it doesn't?
*
* Split VB space into 100-block chunks. When a chunk boundary is crossed,
* clear the whole chunk.
*/
firstChunk = firstFBO/VBBM_CHUNK_SIZE;
lastChunk = lastFBO/VBBM_CHUNK_SIZE;
// if the current range falls in the middle of a chunk and doesn't span chunks,
// there's nothing to do b/c the chunk is assumed to have been cleared already
if (((firstFBO % VBBM_CHUNK_SIZE) != 0) && (firstChunk == lastChunk))
continue;
// round up to the next chunk boundaries
if ((firstFBO % VBBM_CHUNK_SIZE) != 0) // this implies the range spans chunks
firstFBO = (firstChunk + 1) * VBBM_CHUNK_SIZE; // the first FBO of the next chunk
lastFBO = ((lastChunk + 1) * VBBM_CHUNK_SIZE - 1); // the last FBO of the last chunk
// don't go past the end of the file
if (lastFBO > files[fileIndex].fileSize/BLOCK_SIZE)
lastFBO = files[fileIndex].fileSize/BLOCK_SIZE;
// at this point [firstFBO, lastFBO] is the range to age out.
// ugh, walk the whole vbbm looking for matches.
for (i = 0; i < vbbm->vbCapacity; i++)
if (storage[i].lbid != -1 && storage[i].vbOID == vbOID &&
storage[i].vbFBO >= firstFBO && storage[i].vbFBO <= lastFBO) {
if (vss.isEntryLocked(storage[i].lbid, storage[i].verID)) {
ostringstream msg;
msg << "VBBM::getBlocks(): version buffer overflow. Increase VersionBufferFileSize. Overflow occured in aged blocks. Requested NumBlocks:VbOid:vbFBO:lastFBO = "
<< num << ":" << vbOID <<":" << firstFBO << ":" << lastFBO <<" lbid locked is " << storage[i].lbid << endl;
log(msg.str(), logging::LOG_TYPE_CRITICAL);
freeRanges.clear();
throw logging::VBBMBufferOverFlowExcept(msg.str());
}
vss.removeEntry(storage[i].lbid, storage[i].verID, &flushList);
removeEntry(storage[i].lbid, storage[i].verID);
}
}
if (flushPMCache && !flushList.empty())
cacheutils::flushPrimProcAllverBlocks(flushList);
}
// cut & pasted from brm_good_2(), but with rollback instead of commit.
void brm_good_3()
{
DBRM brm;
VBBM vbbm;
VSS vss;
CopyLocks cl;
int i, err, size;
vector<LBID_t> lbids;
vector<LBID_t>::iterator lbid;
vector<EMEntry> extents;
LBIDRange_v ranges;
LBIDRange_v::iterator lbidRangeIT;
VBRange_v vbRanges, vbRanges2;
VBRange_v::iterator vbRangesIT;
LBIDRange_v tmp;
EMEntry em;
OID_t oid;
uint32_t fbo;
LBIDRange range;
VBRange vbRange;
VER_t verID;
bool vbFlag;
bool caughtException;
// Buildbot times out on the getBlocks() call during leakcheck b/c it takes
// > 5 mins for some reason. Have to ping it before 300 seconds go by.
void (*oldsig)(int);
cerr << "brm_good_3" << endl;
oldsig = signal(SIGALRM, keepalive);
alarm(290);
err = brm.lookup(0, 0, false, oid, fbo);
CPPUNIT_ASSERT(err != 0);
err = brm.lookup(0, 0, true, oid, fbo);
CPPUNIT_ASSERT(err != 0);
err = brm.createExtent(8000, 1, lbids, size);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(size == brm.getExtentSize());
CPPUNIT_ASSERT(lbids.size() == 1);
CPPUNIT_ASSERT(*(lbids.begin()) == 0);
err = brm.getExtents(1, extents);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(extents.size() == 1);
em = *(extents.begin());
CPPUNIT_ASSERT(em.range.start == 0);
CPPUNIT_ASSERT(em.range.size*1024 == static_cast<uint32_t>(brm.getExtentSize()));
CPPUNIT_ASSERT(em.HWM == 0);
CPPUNIT_ASSERT(em.blockOffset == 0);
for (i = 0; i < 5; i++) {
range.start = i * 100;
range.size = 100;
ranges.push_back(range);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
err = brm.beginVBCopy(1, ranges, vbRanges);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
err = brm.beginVBCopy(1, ranges, vbRanges2);
CPPUNIT_ASSERT(err != 0);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
for (i = 0; i < 500; i++) {
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err != 0);
}
vbRange = *(vbRanges.begin());
// CPPUNIT_ASSERT(vbRange.vbFBO == 0);
for (i = 0; i < (int)vbRange.size; i++) {
err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i);
CPPUNIT_ASSERT(err == 0);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
for (i = 0; i < (int)vbRange.size; i++) {
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 1);
CPPUNIT_ASSERT(vbFlag == false);
verID = MAXINT;
err = brm.vssLookup(i, verID, 0, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 0);
CPPUNIT_ASSERT(vbFlag == true);
}
for (; i < 500; i++) {
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err != 0);
}
err = brm.endVBCopy(0, ranges);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
err = brm.getUncommittedLBIDs(1, lbids);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(lbids.size() == vbRange.size);
sort<vector<LBID_t>::iterator>(lbids.begin(), lbids.end());
lbid = lbids.begin();
for (i = 0; i < static_cast<int>(lbids.size()); i++, lbid++)
CPPUNIT_ASSERT((*lbid) == static_cast<LBID_t>(i));
range.start = 0;
range.size = i;
tmp.push_back(range);
err = brm.vbRollback(1, tmp);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
for (i = 0; i < (int)vbRange.size; i++) {
verID = MAXINT;
err = brm.vssLookup(i, verID, 0, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 0);
CPPUNIT_ASSERT(vbFlag == false);
err = brm.lookup(i, verID, vbFlag, oid, fbo);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == 1);
CPPUNIT_ASSERT(fbo == static_cast<uint32_t>(i));
vbbm.lock(VBBM::WRITE);
vss.lock(VSS::WRITE);
#ifdef BRM_DEBUG
caughtException = false;
try {
vbbm.removeEntry(i, verID);
vbbm.confirmChanges();
}
catch (logic_error &e) {
vbbm.undoChanges();
caughtException = true;
}
CPPUNIT_ASSERT(caughtException);
caughtException = false;
try {
vss.removeEntry(i, 1);
vss.confirmChanges();
}
catch (logic_error &e) {
vss.undoChanges();
caughtException = true;
}
CPPUNIT_ASSERT(caughtException);
#endif
vss.removeEntry(i, verID);
vss.confirmChanges();
vss.release(VSS::WRITE);
vbbm.release(VBBM::WRITE);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
brm.deleteOID(1);
vbbm.lock(VBBM::READ);
vss.lock(VSS::READ);
CPPUNIT_ASSERT(vbbm.size() == 0);
CPPUNIT_ASSERT(vbbm.hashEmpty());
CPPUNIT_ASSERT(vss.size() == 0);
CPPUNIT_ASSERT(vss.hashEmpty());
vss.release(VSS::READ);
vbbm.release(VBBM::READ);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
err = brm.saveState();
CPPUNIT_ASSERT(err==0);
}
void brm_good_2()
{
BlockResolutionManager brm;
VBBM vbbm;
VSS vss;
CopyLocks cl;
int i, err, size;
vector<LBID_t> lbids;
vector<EMEntry> extents;
LBIDRange_v ranges;
LBIDRange_v::iterator lbidRangeIT;
VBRange_v vbRanges, vbRanges2;
VBRange_v::iterator vbRangesIT;
EMEntry em;
OID_t oid;
uint32_t fbo;
LBIDRange range;
VBRange vbRange;
VER_t verID;
bool vbFlag;
// Buildbot times out on the getBlocks() call during leakcheck b/c it takes
// > 5 mins for some reason. Have to ping it before 300 seconds go by.
void (*oldsig)(int);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
oldsig = signal(SIGALRM, keepalive);
alarm(290);
err = brm.lookup(0, 0, false, oid, fbo);
CPPUNIT_ASSERT(err == -1);
err = brm.lookup(0, 0, true, oid, fbo);
CPPUNIT_ASSERT(err == -1);
err = brm.createExtent(10000, 0, lbids, size);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(size == brm.getExtentSize());
CPPUNIT_ASSERT(lbids.size() == 1);
// CPPUNIT_ASSERT(*(lbids.begin()) == 0);
err = brm.getExtents(0, extents);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(extents.size() == 1);
em = *(extents.begin());
// CPPUNIT_ASSERT(em.range.start == 0);
CPPUNIT_ASSERT(em.range.size * 1024 == static_cast<uint32_t>(brm.getExtentSize()));
CPPUNIT_ASSERT(em.HWM == 0);
CPPUNIT_ASSERT(em.blockOffset == 0);
for (i = 0; i < 50; i++)
{
range.start = i * 100;
range.size = 100;
ranges.push_back(range);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
err = brm.beginVBCopy(1, ranges, vbRanges);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
err = brm.beginVBCopy(1, ranges, vbRanges2);
CPPUNIT_ASSERT(err == -1);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
for (i = 0; i < 5000; i++)
{
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err == -1);
}
vbRange = *(vbRanges.begin());
// CPPUNIT_ASSERT(vbRange.vbFBO == 0);
for (i = 0; i < (int)vbRange.size; i++)
{
err = brm.writeVBEntry(1, i, vbRange.vbOID, vbRange.vbFBO + i);
CPPUNIT_ASSERT(err == 0);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
for (i = 0; i < (int)vbRange.size; i++)
{
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 1);
CPPUNIT_ASSERT(vbFlag == false);
verID = MAXINT;
err = brm.vssLookup(i, verID, 0, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 0);
CPPUNIT_ASSERT(vbFlag == true);
}
for (; i < 5000; i++)
{
verID = MAXINT;
err = brm.vssLookup(i, verID, 1, vbFlag);
CPPUNIT_ASSERT(err == -1);
}
err = brm.endVBCopy(0, ranges);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
cl.lock(CopyLocks::READ);
for (lbidRangeIT = ranges.begin(); lbidRangeIT != ranges.end();
lbidRangeIT++)
CPPUNIT_ASSERT(!cl.isLocked(*lbidRangeIT));
cl.release(CopyLocks::READ);
brm.vbCommit(1);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
for (i = 0; i < (int)vbRange.size; i++)
{
verID = MAXINT;
err = brm.vssLookup(i, verID, 0, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 1);
CPPUNIT_ASSERT(vbFlag == false);
verID = 0;
err = brm.vssLookup(i, verID, 0, vbFlag);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(verID == 0);
CPPUNIT_ASSERT(vbFlag == true);
err = brm.lookup(i, verID, vbFlag, oid, fbo);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == vbRange.vbOID);
CPPUNIT_ASSERT(fbo == static_cast<uint32_t>(i + vbRange.vbFBO));
vbbm.lock(VBBM::WRITE);
vss.lock(VSS::WRITE);
vbbm.removeEntry(i, verID);
// vss.removeEntry(i, 1);
vss.removeEntry(i, verID);
vss.release(VSS::WRITE);
vbbm.release(VBBM::WRITE);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
brm.deleteOID(0);
vss.lock(VSS::READ);
vbbm.lock(VBBM::READ);
CPPUNIT_ASSERT(vbbm.size() == 0);
CPPUNIT_ASSERT(vbbm.hashEmpty());
CPPUNIT_ASSERT(vss.size() == 0);
CPPUNIT_ASSERT(vss.hashEmpty());
vss.release(VSS::READ);
vbbm.release(VBBM::READ);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
}
