void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid=1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable=0;
int i=0;
fExtentSize=dbrm.getExtentSize();
while ( oid < 5000 )
{
int ret=0;
ret = dbrm.lookup(oid, r);
if (ret==0 && r.size()>0) {
dbrm.getHWM(oid, hwm);
maxBlocksAvailable+=(r.size()*extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm=0;
r.clear();
}
oid++;
}
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i=0;
} // setUp()
void brm_HWM()
{
DBRM brm;
int err;
HWM_t hwm;
vector<EMEntry> extents;
cerr << "brm_HWM" << endl;
err = brm.getExtents(1, extents);
#ifdef BRM_DEBUG
if (err == 0)
cerr << "Make sure OID 1 isn't allocated in the extent map" << endl;
#endif
CPPUNIT_ASSERT(extents.size() == 0);
err = brm.setHWM(1, 10);
CPPUNIT_ASSERT(err != 0);
err = brm.getHWM(1, hwm);
CPPUNIT_ASSERT(err != 0);
}
int main(int argc, char *argv[]) {
if (argc>=2) thr_cnt=atoi(argv[1]);
if (argc>=3) fLoops=atoi(argv[2]);
if (thr_cnt<=0) thr_cnt=1;
if (thr_cnt>1024) thr_cnt=1024;
if (fLoops<=0) fLoops=1;
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
cout << "Starting " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid=3000;
uint32_t totalExt=0;
do {
int ret = dbrm.lookup(oid, r);
if (ret==0 && r.size() > 0) {
lowlbid = (r[0].start/extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start=lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx=0; idx<r.size(); idx++)
totalExt+=r[idx].size;
ranges.push_back(r);
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
cout << ranges.size() << " ranges found" << endl;
gettimeofday(&tv, NULL);
uint32_t blksLoaded=0;
int rangesLoaded=0;
for (uint32_t i =0; i<ranges.size() && blksLoaded < cacheSize; i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
}
cout << endl;
gettimeofday(&tv2, NULL);
cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec <<endl;
while (ranges.size() > rangesLoaded) ranges.pop_back();
#ifdef BLAH
for (uint32_t i =0; i<ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i =0; i<ranges.size(); i++)
{
LBIDRange_v rv=ranges[i];
for(uint32_t j=0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for(uint32_t k=l.start; k<l.start+l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
#endif
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait= {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
//start threads running
cout << "Starting driver threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt*(sizeof(pthread_t)));
for(int i=0; i<thr_cnt; i++) {
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
while (thr_wait.predicate>0) {
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
// join threads back to main
for(int i=0; i<thr_cnt; i++) {
pthread_join(thr_id[i], NULL);
}
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
if (elTime>0) {
avgTot=(bfoundTot+rfoundTot)/elTime;
rangeAvg=(rfoundTot)/elTime;
blkAvg=(bfoundTot)/elTime;
} else {
avgTot=bfoundTot+rfoundTot;
rangeAvg=rfoundTot;
blkAvg=bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot/thr_cnt << endl << endl;
return 0;
} // end main
void brm_extentmap_good_1()
{
DBRM brm;
int i, err, oid, allocdSize,
iterations = 100; // (EM_INITIAL_SIZE + 3*EM_INCREMENT)
uint32_t fbo, hwm;
vector<LBID_t> lbids;
HWM_t hwm2;
VER_t txnID;
const uint32_t extentSize = brm.getExtentSize();
cerr << "brm_extentmap_good_1" << endl;
for (i = 1; i < iterations; i++) {
err = brm.createExtent(extentSize, i, lbids, allocdSize);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(lbids.back() == static_cast<LBID_t>((i-1)*extentSize));
err=brm.markExtentInvalid(lbids[0]);
CPPUNIT_ASSERT(err==0);
int64_t min;
int64_t max;
int32_t seq;
err = brm.getExtentMaxMin(lbids[0], max, min, seq);
#ifdef SAFE_CP
CPPUNIT_ASSERT(err == 0);
#else
CPPUNIT_ASSERT(err == 1);
#endif
err = brm.setExtentMaxMin(lbids[0], max, min, seq);
CPPUNIT_ASSERT(err == 0);
}
CPPUNIT_ASSERT(err==0);
for (i = 1; i < iterations; i++) {
err = brm.lookup(static_cast<LBID_t>((i-1)*extentSize), 0, false, oid, fbo);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == i);
CPPUNIT_ASSERT(fbo == 0);
if (i != 1) {
err = brm.lookup(static_cast<LBID_t>((i-1)*extentSize - 1), 0, false, oid, fbo);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == i-1);
CPPUNIT_ASSERT(fbo == extentSize - 1);
}
if (i != iterations) {
err = brm.lookup(static_cast<LBID_t>((i-1)*extentSize + 1), 0, false, oid, fbo);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == i);
CPPUNIT_ASSERT(fbo == 1);
}
err = brm.markExtentInvalid(oid);
CPPUNIT_ASSERT(err==0);
err = brm.markExtentInvalid(lbids[0]);
CPPUNIT_ASSERT(err==0);
int64_t min;
int64_t max;
int32_t seq;
err = brm.getExtentMaxMin(lbids[0], max, min, seq);
#ifdef SAFE_CP
CPPUNIT_ASSERT(err == 0);
#else
CPPUNIT_ASSERT(err == 1);
#endif
err = brm.setExtentMaxMin(lbids[0], max, min, seq);
CPPUNIT_ASSERT(err == 0);
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
err = brm.lookup(static_cast<LBID_t>((i-1)*extentSize), 0, false, oid, fbo);
CPPUNIT_ASSERT(err == -1);
for (i = 1; i < iterations; i++) {
err = brm.getBulkInsertVars(static_cast<LBID_t>((i-1)*extentSize),
hwm2, txnID);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(hwm2 == 0);
CPPUNIT_ASSERT(txnID == 0);
err = brm.setBulkInsertVars(static_cast<LBID_t>((i-1)*extentSize),
i, i + 1);
CPPUNIT_ASSERT(err == 0);
err = brm.getBulkInsertVars(static_cast<LBID_t>((i-1)*extentSize),
hwm2, txnID);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(hwm2 == static_cast<BRM::LBID_t>(i));
CPPUNIT_ASSERT(txnID == static_cast<BRM::VER_t>(i+1));
err = brm.getHWM(i, hwm);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(hwm == 0);
err = brm.setHWM(i, ((uint32_t)i > extentSize - 1 ? extentSize - 1 : i));
CPPUNIT_ASSERT(err == 0);
err = brm.getHWM(i, hwm);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(hwm == static_cast<uint32_t>((uint32_t)i > extentSize - 1 ? extentSize - 1 : i));
}
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
#ifdef BRM_DEBUG
err = brm.setHWM(i, hwm);
CPPUNIT_ASSERT(err != 0);
#endif
for (i = 1; i < iterations; i++) {
err = brm.deleteOID(i);
CPPUNIT_ASSERT(err == 0);
}
err = brm.deleteOID(i);
CPPUNIT_ASSERT(err != 0);
CPPUNIT_ASSERT(brm.checkConsistency() == 0);
err = brm.saveState();
CPPUNIT_ASSERT(err==0);
}
static void* BRMRunner_si(void *arg)
{
// keep track of LBID ranges allocated here and
// randomly allocate, lookup, delete, get/set HWM, and
// destroy the EM object.
struct EMEntries {
u_int64_t LBIDstart;
u_int32_t size;
int OID;
u_int32_t FBO;
u_int32_t HWM;
u_int32_t secondHWM;
u_int32_t txnID;
struct EMEntries *next;
EMEntries() { next = NULL; HWM = 0; secondHWM = 0; txnID = 0; }
};
#ifdef BRM_VERBOSE
int threadNum = reinterpret_cast<int>(arg);
#endif
int op, listSize = 0, i;
uint randstate;
struct EMEntries *head = NULL, *tmp;
struct timeval tv;
ExtentMap em;
vector<LBID_t> lbids;
#ifdef BRM_VERBOSE
cerr << "thread number " << threadNum << " started." << endl;
#endif
gettimeofday(&tv, NULL);
randstate = static_cast<uint>(tv.tv_usec);
while (!threadStop) {
op = rand_r(&randstate) % 10;
#ifdef BRM_VERBOSE
cerr << "next op is " << op << endl;
#endif
switch (op) {
case 0: //allocate space for a new file
{
struct EMEntries *newEm;
int size = rand_r(&randstate) % 102399 + 1;
int entries, OID, allocdSize, err;
pthread_mutex_lock(&mutex);
OID = oid++;
opCount++;
pthread_mutex_unlock(&mutex);
err = brm_si.createExtent(size, OID, lbids, allocdSize);
CPPUNIT_ASSERT(err == 0);
entries = size/brm_si.getExtentSize();
if ((size % brm_si.getExtentSize()) != 0)
entries++;
CPPUNIT_ASSERT((uint)entries == lbids.size());
for (i = 0 ; i < entries; i++) {
newEm = new EMEntries();
newEm->size = brm_si.getExtentSize();
newEm->OID = OID;
newEm->FBO = i * brm_si.getExtentSize();
newEm->LBIDstart = lbids[i];
newEm->next = head;
head = newEm;
listSize++;
}
#ifdef BRM_VERBOSE
cerr << "created new space for OID " << newEm->OID << endl;
#endif
em.checkConsistency();
break;
}
case 1: //allocate space for an existing file
{
if (listSize == 0)
break;
struct EMEntries *newEm, *tmp;
int size = rand_r(&randstate) % 102399 + 1;
int fileRand = rand_r(&randstate) % listSize;
int i, lastExtent, blockEnd, oid;
int tmpHWM, entries, allocdSize, err;
vector<LBID_t> lbids;
for (i = 0, tmp = head; i < fileRand; i++)
tmp = tmp->next;
oid = tmp->OID;
for (lastExtent = 0, tmp = head; tmp != NULL; tmp = tmp->next) {
if (tmp->OID != oid)
continue;
tmpHWM = tmp->HWM;
blockEnd = tmp->FBO + tmp->size;
if (lastExtent < blockEnd)
lastExtent = blockEnd;
}
err = brm_si.createExtent(size, oid, lbids, allocdSize);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
entries = size/brm_si.getExtentSize();
if ((size % brm_si.getExtentSize()) != 0)
entries++;
CPPUNIT_ASSERT((uint)entries == lbids.size());
for (i = 0; i < entries; i++) {
newEm = new EMEntries();
if (i != entries)
newEm->size = brm_si.getExtentSize();
else
newEm->size = size % brm_si.getExtentSize();
newEm->OID = oid;
newEm->FBO = lastExtent + (i*brm_si.getExtentSize());
newEm->LBIDstart = lbids[i];
newEm->HWM = tmpHWM;
newEm->next = head;
head = newEm;
listSize++;
}
#ifdef BRM_VERBOSE
cerr << "created another extent for OID " << newEm->OID << endl;
#endif
em.checkConsistency();
break;
}
case 2: //delete an OID
{
if (listSize == 0)
break;
struct EMEntries *tmp, *prev;
int fileRand = rand_r(&randstate) % listSize;
int i, oid, err;
for (i = 0, tmp = head; i < fileRand; i++)
tmp = tmp->next;
oid = tmp->OID;
err = brm_si.deleteOID(oid);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
for (tmp = head; tmp != NULL;) {
if (tmp->OID == oid) {
if (tmp == head) {
head = head->next;
delete tmp;
tmp = head;
}
else {
prev->next = tmp->next;
delete tmp;
tmp = prev->next;
}
listSize--;
}
else {
prev = tmp;
tmp = tmp->next;
}
}
#ifdef BRM_VERBOSE
cerr << "deleted OID " << oid << endl;
#endif
em.checkConsistency();
break;
}
case 3: //lookup by LBID
{
if (listSize == 0)
break;
int entryRand = rand_r(&randstate) % listSize;
int i, err, offset, oid;
struct EMEntries *tmp;
LBID_t target;
u_int32_t fbo;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
offset = rand_r(&randstate) % tmp->size;
target = tmp->LBIDstart + offset;
err = brm_si.lookup(target, 0, false, oid, fbo);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
#ifdef BRM_VERBOSE
cerr << "looked up LBID " << target << " got oid " << oid << " fbo " << fbo << endl;
cerr << " oid should be " << tmp->OID << " fbo should be " << offset+tmp->FBO << endl;
#endif
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(oid == tmp->OID);
CPPUNIT_ASSERT(fbo == offset + tmp->FBO);
em.checkConsistency();
break;
}
case 4: //lookup by OID, FBO
{
if (listSize == 0)
break;
int entryRand = rand_r(&randstate) % listSize;
int i, oid, err, offset;
struct EMEntries *tmp;
LBID_t lbid;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
offset = rand_r(&randstate) % tmp->size;
oid = tmp->OID;
err = brm_si.lookup(oid, offset + tmp->FBO, lbid);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
#ifdef BRM_VERBOSE
cerr << "looked up OID " << oid << " fbo " << offset + tmp->FBO <<
" got lbid " << lbid << endl;
cerr << " lbid should be " << tmp->LBIDstart + offset << endl;
#endif
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(lbid == static_cast<LBID_t>(tmp->LBIDstart + offset));
em.checkConsistency();
break;
}
case 5: //getHWM
{
if (listSize == 0)
break;
int entryRand = rand_r(&randstate) % listSize;
int i, err;
struct EMEntries *tmp;
u_int32_t hwm;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
err = brm_si.getHWM(tmp->OID, hwm);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
#ifdef BRM_VERBOSE
cerr << "stored HWM for OID " << tmp->OID << " is " << tmp->HWM
<< " BRM says it's " << hwm << endl;
#endif
CPPUNIT_ASSERT(hwm == tmp->HWM);
em.checkConsistency();
break;
}
case 6: //setHWM
{
if (listSize == 0)
break;
int entryRand = rand_r(&randstate) % listSize;
int i, hwm, oid, err;
struct EMEntries *tmp;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
oid = tmp->OID;
hwm = rand_r(&randstate) % (tmp->FBO + brm_si.getExtentSize());
err = brm_si.setHWM(oid, hwm);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
for (tmp = head; tmp != NULL; tmp = tmp->next)
if (tmp->OID == oid)
tmp->HWM = hwm;
#ifdef BRM_VERBOSE
cerr << "setHWM of OID " << oid << " to " << hwm << endl;
#endif
em.checkConsistency();
break;
}
case 7: //getBulkInsertVars
{
if (listSize == 0)
break;
HWM_t hwm;
VER_t txnID;
int entryRand = rand_r(&randstate) % listSize;
int i, err, offset;
EMEntries *tmp;
LBID_t lbid;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
offset = rand_r(&randstate) % tmp->size;
lbid = tmp->LBIDstart + offset;
err = brm_si.getBulkInsertVars(lbid, hwm, txnID);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
CPPUNIT_ASSERT(hwm == tmp->secondHWM);
CPPUNIT_ASSERT(txnID == tmp->txnID);
break;
}
case 8: //setBulkInsertVars
{
if (listSize == 0)
break;
int entryRand = rand_r(&randstate) % listSize;
int i, err, offset;
EMEntries *tmp;
for (i = 0, tmp = head; i < entryRand; i++)
tmp = tmp->next;
offset = rand_r(&randstate) % tmp->size;
tmp->secondHWM = rand_r(&randstate) % MAXINT;
tmp->txnID = rand_r(&randstate) % MAXINT;
err = brm_si.setBulkInsertVars(tmp->LBIDstart + offset,
tmp->secondHWM, tmp->txnID);
pthread_mutex_lock(&mutex);
opCount++;
pthread_mutex_unlock(&mutex);
CPPUNIT_ASSERT(err == 0);
break;
}
default:
break;
}
}
while (head != NULL) {
tmp = head->next;
delete head;
head = tmp;
}
#ifdef BRM_VERBOSE
cerr << "thread " << threadNum << " exiting" << endl;
#endif
return NULL;
}