void DBCursor::short_list(FILE *fp, int *qout, int ltype, void *out)
{
int c;
int ret;
c = fgetc(fp);
FRIENDLY_ASSERT(c == '#');
c = fgetc(fp);
FRIENDLY_ASSERT(c == '\n');
switch(ltype){
case STRING_CODE:
{
typedef const char** ccpp;
ccpp *cpout = (ccpp*)out;
assert(f_list == NULL); /* only one SHORT_LIST per record supported */
f_list = new StringList;
while((c = fgetc(fp)) != '#'){
char *item;
int ftype;
ungetc(c, fp);
ret = fscanf(fp, "%d\n", &ftype);
if(ret == 0) throw(PosixError(errno, "Unable to fscanf\n"));
FRIENDLY_ASSERT(ftype == STRING_CODE);
string_field(fp, &item, NULL);
f_list->append(item);
}
*cpout = f_list->array();
*qout = f_list->qty();
}
break;
default:
abort(); /* only strings supported */
}
c = fgetc(fp);
FRIENDLY_ASSERT(c == '\n');
}
void runThread(ConnectionString& hostConn, unsigned threadId, unsigned seed,
BSONObj& cmdObj, BSONObjBuilder& result) {
stringstream ss;
ss << "thread-" << threadId;
setThreadName(ss.str().c_str());
// Lock name
string lockName = string_field(cmdObj, "lockName", this->name + "_lock");
// Range of clock skew in diff threads
int skewRange = (int) number_field(cmdObj, "skewRange", 1);
// How long to wait with the lock
int threadWait = (int) number_field(cmdObj, "threadWait", 30);
if(threadWait <= 0) threadWait = 1;
// Max amount of time (ms) a thread waits before checking the lock again
int threadSleep = (int) number_field(cmdObj, "threadSleep", 30);
if(threadSleep <= 0) threadSleep = 1;
// How long until the lock is forced in ms, only compared locally
unsigned long long takeoverMS = (unsigned long long) number_field(cmdObj, "takeoverMS", 0);
// Whether or not we should hang some threads
int hangThreads = (int) number_field(cmdObj, "hangThreads", 0);
boost::mt19937 gen((boost::mt19937::result_type) seed);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSkew(gen, boost::uniform_int<>(0, skewRange));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomWait(gen, boost::uniform_int<>(1, threadWait));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSleep(gen, boost::uniform_int<>(1, threadSleep));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomNewLock(gen, boost::uniform_int<>(0, 3));
int skew = 0;
if (!lock.get()) {
// Pick a skew, but the first two threads skew the whole range
if(threadId == 0)
skew = -skewRange / 2;
else if(threadId == 1)
skew = skewRange / 2;
else skew = randomSkew() - (skewRange / 2);
// Skew this thread
jsTimeVirtualThreadSkew( skew );
log() << "Initializing lock with skew of " << skew << " for thread " << threadId << endl;
lock.reset(new DistributedLock(hostConn, lockName, takeoverMS, true ));
log() << "Skewed time " << jsTime() << " for thread " << threadId << endl
<< " max wait (with lock: " << threadWait << ", after lock: " << threadSleep << ")" << endl
<< " takeover in " << takeoverMS << "(ms remote)" << endl;
}
DistributedLock* myLock = lock.get();
bool errors = false;
BSONObj lockObj;
while (keepGoing) {
try {
if (myLock->lock_try("Testing distributed lock with skew.", false, &lockObj )) {
log() << "**** Locked for thread " << threadId << " with ts " << lockObj["ts"] << endl;
if( count.loadRelaxed() % 2 == 1 && ! myLock->lock_try( "Testing lock re-entry.", true ) ) {
errors = true;
log() << "**** !Could not re-enter lock already held" << endl;
break;
}
if( count.loadRelaxed() % 3 == 1 && myLock->lock_try( "Testing lock non-re-entry.", false ) ) {
errors = true;
log() << "**** !Invalid lock re-entry" << endl;
break;
}
int before = count.addAndFetch(1);
int sleep = randomWait();
sleepmillis(sleep);
int after = count.loadRelaxed();
if(after != before) {
errors = true;
log() << "**** !Bad increment while sleeping with lock for: " << sleep << "ms" << endl;
break;
}
// Unlock only half the time...
if(hangThreads == 0 || threadId % hangThreads != 0) {
log() << "**** Unlocking for thread " << threadId << " with ts " << lockObj["ts"] << endl;
myLock->unlock( &lockObj );
}
else {
log() << "**** Not unlocking for thread " << threadId << endl;
verify( DistributedLock::killPinger( *myLock ) );
// We're simulating a crashed process...
break;
}
}
}
catch( const DBException& ex ) {
log() << "*** !Could not try distributed lock." << causedBy( ex ) << endl;
break;
}
// Create a new lock 1/3 of the time
if( randomNewLock() > 1 ){
lock.reset(new DistributedLock( hostConn, lockName, takeoverMS, true ));
myLock = lock.get();
}
sleepmillis(randomSleep());
}
result << "errors" << errors
<< "skew" << skew
<< "takeover" << (long long) takeoverMS
<< "localTimeout" << (takeoverMS > 0);
}
int DBCursor::next(int typeCode, ...)
{
int ret = 1;
FILE *fp = f_table->file(DB::READ);
int io;
int recordClass;
if(f_start < 0){ /* on first call to next(), reset the file */
rewind(fp);
}
f_start = ftell(fp);
io = fscanf(fp, "%d\n", &recordClass); /* get record code */
if(io != EOF){ /* got any data? */
FRIENDLY_ASSERT(io == 1);
// clean up previous fields first if they exist
f_fields->reset();
delete f_list; f_list = NULL;
va_list ap;
va_start(ap, typeCode);
int fieldQty;
io = fscanf(fp, "%d\n", &fieldQty); /* get field count */
FRIENDLY_ASSERT(io == 1);
/* iterate over fields in the input stream... */
while(fieldQty--){
int fieldCode;
io = fscanf(fp, "%d\n", &fieldCode); /* get field type */
FRIENDLY_ASSERT(io == 1);
assert(typeCode); /* make sure caller didn't give too few args */
switch(fieldCode){
case STRING_CODE:
{
char **data = NULL;
int *len = NULL;
if (fieldCode == typeCode || (fieldCode + typeCode) == 0) {
data = va_arg(ap, char**);
if (fieldCode != typeCode) {
len = va_arg(ap, int*);
}
}
string_field(fp, data, len);
}
break;
case INTEGER_CODE:
int_field(fp, fieldCode == typeCode ? va_arg(ap, int*) : 0);
break;
case SHORT_LIST_CODE:
{
int *qout = va_arg(ap, int *);
int ltype = va_arg(ap, int);
void *out = va_arg(ap, void*);
short_list(fp, qout, ltype, out);
}
break;
default:
abort();
}
void runThread(ConnectionString& hostConn, unsigned threadId, unsigned seed,
BSONObj& cmdObj, BSONObjBuilder& result) {
stringstream ss;
ss << "thread-" << threadId;
setThreadName(ss.str().c_str());
// Lock name
string lockName = string_field(cmdObj, "lockName", this->name + "_lock");
// Range of clock skew in diff threads
int skewRange = (int) number_field(cmdObj, "skewRange", 1);
// How long to wait with the lock
int threadWait = (int) number_field(cmdObj, "threadWait", 30);
if(threadWait <= 0) threadWait = 1;
// Max amount of time (ms) a thread waits before checking the lock again
int threadSleep = (int) number_field(cmdObj, "threadSleep", 30);
if(threadSleep <= 0) threadSleep = 1;
// (Legacy) how long until the lock is forced in mins, measured locally
int takeoverMins = (int) number_field(cmdObj, "takeoverMins", 0);
// How long until the lock is forced in ms, only compared locally
unsigned long long takeoverMS = (unsigned long long) number_field(cmdObj, "takeoverMS", 0);
// Whether or not we should hang some threads
int hangThreads = (int) number_field(cmdObj, "hangThreads", 0);
boost::mt19937 gen((boost::mt19937::result_type) seed);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSkew(gen, boost::uniform_int<>(0, skewRange));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomWait(gen, boost::uniform_int<>(1, threadWait));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSleep(gen, boost::uniform_int<>(1, threadSleep));
int skew = 0;
bool legacy = (takeoverMins > 0);
if (!lock.get()) {
// Pick a skew, but the first two threads skew the whole range
if(threadId == 0)
skew = -skewRange / 2;
else if(threadId == 1)
skew = skewRange / 2;
else skew = randomSkew() - (skewRange / 2);
// Skew this thread
jsTimeVirtualThreadSkew( skew );
log() << "Initializing lock with skew of " << skew << " for thread " << threadId << endl;
lock.reset(new DistributedLock(hostConn, lockName, legacy ? (unsigned long long)takeoverMins : takeoverMS, true, legacy));
log() << "Skewed time " << jsTime() << " for thread " << threadId << endl
<< " max wait (with lock: " << threadWait << ", after lock: " << threadSleep << ")" << endl
<< " takeover in " << (legacy ? (unsigned long long)takeoverMins : takeoverMS) << (legacy ? " (mins local)" : "(ms remote)") << endl;
}
DistributedLock* myLock = lock.get();
bool errors = false;
while (keepGoing) {
try {
if (myLock->lock_try("Testing distributed lock with skew.")) {
log() << "**** Locked for thread " << threadId << endl;
count++;
int before = count;
int sleep = randomWait();
sleepmillis(sleep);
int after = count;
if(after != before) {
errors = true;
log() << "**** !Bad increment while sleeping with lock for: " << sleep << "ms" << endl;
break;
}
// Unlock only half the time...
if(hangThreads == 0 || threadId % hangThreads != 0) {
log() << "**** Unlocking for thread " << threadId << endl;
myLock->unlock();
}
else {
log() << "**** Not unlocking for thread " << threadId << endl;
DistributedLock::killPinger( *myLock );
// We're simulating a crashed process...
break;
}
}
}
catch( LockException& e ) {
log() << "*** !Could not try distributed lock." << m_caused_by(e) << endl;
break;
}
sleepmillis(randomSleep());
}
result << "errors" << errors
<< "skew" << skew
<< "takeover" << (long long) (legacy ? takeoverMS : (unsigned long long)takeoverMins)
<< "localTimeout" << (takeoverMS > 0);
}
