static void
testLocking(
const char* const pathname, /* pathname of file */
const size_t len, /* length of memory-mapped portion */
useconds_t maxInterval) /* maximum sleep interval */
{
const int fd = open(pathname, O_RDWR, 0);
log_assert(len >= 0);
if (fd == -1) {
(void)fprintf(stderr, "%ld: Couldn't open file \"%s\": %s\n",
(long)getpid(), pathname, strerror(errno));
}
else {
/*
* Memory-map the file.
*/
void* addr = mmap(0, len, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
if (addr == MAP_FAILED) {
(void)fprintf(stderr, "%ld: Couldn't memory-map file \"%s\": %s\n",
(long)getpid(), pathname, strerror(errno));
}
else {
log_assert(addr != NULL);
while (!done) {
randomSleep(maxInterval);
/*
* Lock the file.
*/
if (!lockFile(fd, len))
break;
lockCount++;
/*
* Change the file.
*/
(void)memset(addr, 1+*(char*)addr, len);
randomSleep(maxInterval);
/*
* Unlock the file.
*/
if (!unlockFile(fd, len))
break;
} /* while not done */
(void)munmap(addr, len);
} /* "addr" is mapped */
(void)close(fd);
} /* "fd" is open */
}
void *consumer(void *arg)
{
while(1) {
if (removeProduct == 0) {break;}
sem_wait(&full);
Node *n1 = list_remove(fifo);
sem_post(&empty);
printf("Consumer %d consumed Item_%d (out of %d)\n",(int)arg,(int)n1->elm,products_max);
randomSleep();
}
pthread_exit(0);
}
void *producer(void *arg)
{
while(1) {
int product = createProduct();
if (product == 0) {break;}
sem_wait(&empty);
list_add(fifo, node_new_int(product));
sem_post(&full);
printf("Producer %d produced Item_%d (out of %d)\n",(int)arg,product,products_max);
randomSleep();
}
pthread_exit(0);
}
void* thread2(void* arg)
{
puts ("thread2-1");
randomSleep();
pthread_mutex_lock(&mutex2);
randomSleep();
puts ("thread2-2");
sched_yield();
randomSleep();
pthread_mutex_unlock(&mutex2);
randomSleep();
puts ("thread2-3");
randomSleep();
pthread_mutex_lock(&mutex1);
randomSleep();
puts ("thread2-4");
randomSleep();
pthread_mutex_unlock(&mutex1);
randomSleep();
}
void runThread(ConnectionString& hostConn, unsigned threadId, unsigned seed,
BSONObj& cmdObj, BSONObjBuilder& result) {
stringstream ss;
ss << "thread-" << threadId;
setThreadName(ss.str().c_str());
// Lock name
string lockName = string_field(cmdObj, "lockName", this->name + "_lock");
// Range of clock skew in diff threads
int skewRange = (int) number_field(cmdObj, "skewRange", 1);
// How long to wait with the lock
int threadWait = (int) number_field(cmdObj, "threadWait", 30);
if(threadWait <= 0) threadWait = 1;
// Max amount of time (ms) a thread waits before checking the lock again
int threadSleep = (int) number_field(cmdObj, "threadSleep", 30);
if(threadSleep <= 0) threadSleep = 1;
// How long until the lock is forced in ms, only compared locally
unsigned long long takeoverMS = (unsigned long long) number_field(cmdObj, "takeoverMS", 0);
// Whether or not we should hang some threads
int hangThreads = (int) number_field(cmdObj, "hangThreads", 0);
boost::mt19937 gen((boost::mt19937::result_type) seed);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSkew(gen, boost::uniform_int<>(0, skewRange));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomWait(gen, boost::uniform_int<>(1, threadWait));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSleep(gen, boost::uniform_int<>(1, threadSleep));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomNewLock(gen, boost::uniform_int<>(0, 3));
int skew = 0;
if (!lock.get()) {
// Pick a skew, but the first two threads skew the whole range
if(threadId == 0)
skew = -skewRange / 2;
else if(threadId == 1)
skew = skewRange / 2;
else skew = randomSkew() - (skewRange / 2);
// Skew this thread
jsTimeVirtualThreadSkew( skew );
log() << "Initializing lock with skew of " << skew << " for thread " << threadId << endl;
lock.reset(new DistributedLock(hostConn, lockName, takeoverMS, true ));
log() << "Skewed time " << jsTime() << " for thread " << threadId << endl
<< " max wait (with lock: " << threadWait << ", after lock: " << threadSleep << ")" << endl
<< " takeover in " << takeoverMS << "(ms remote)" << endl;
}
DistributedLock* myLock = lock.get();
bool errors = false;
BSONObj lockObj;
while (keepGoing) {
try {
if (myLock->lock_try("Testing distributed lock with skew.", false, &lockObj )) {
log() << "**** Locked for thread " << threadId << " with ts " << lockObj["ts"] << endl;
if( count.loadRelaxed() % 2 == 1 && ! myLock->lock_try( "Testing lock re-entry.", true ) ) {
errors = true;
log() << "**** !Could not re-enter lock already held" << endl;
break;
}
if( count.loadRelaxed() % 3 == 1 && myLock->lock_try( "Testing lock non-re-entry.", false ) ) {
errors = true;
log() << "**** !Invalid lock re-entry" << endl;
break;
}
int before = count.addAndFetch(1);
int sleep = randomWait();
sleepmillis(sleep);
int after = count.loadRelaxed();
if(after != before) {
errors = true;
log() << "**** !Bad increment while sleeping with lock for: " << sleep << "ms" << endl;
break;
}
// Unlock only half the time...
if(hangThreads == 0 || threadId % hangThreads != 0) {
log() << "**** Unlocking for thread " << threadId << " with ts " << lockObj["ts"] << endl;
myLock->unlock( &lockObj );
}
else {
log() << "**** Not unlocking for thread " << threadId << endl;
verify( DistributedLock::killPinger( *myLock ) );
// We're simulating a crashed process...
break;
}
}
}
catch( const DBException& ex ) {
log() << "*** !Could not try distributed lock." << causedBy( ex ) << endl;
break;
}
// Create a new lock 1/3 of the time
if( randomNewLock() > 1 ){
lock.reset(new DistributedLock( hostConn, lockName, takeoverMS, true ));
myLock = lock.get();
}
sleepmillis(randomSleep());
}
result << "errors" << errors
<< "skew" << skew
<< "takeover" << (long long) takeoverMS
<< "localTimeout" << (takeoverMS > 0);
}
void execProc(char *managingHost, char *jobIdString, char *reserved,
char *user, char *dir, char *in, char *out, char *err, long long memLimit,
char *exe, char **params)
/* This routine is the child process of doExec.
* It spawns a grandchild that actually does the
* work and waits on it. It sends message to the
* main message loop here when done. */
{
if ((grandChildId = forkOrDie()) == 0)
{
char *homeDir = "";
/* Change to given user (if root) */
changeUid(user, &homeDir);
/* create output files just after becoming user so that errors in the rest
* of this proc will go to the err file and be available via para
* problems */
setupProcStdio(in, out, err);
if (chdir(dir) < 0)
errnoAbort("can't chdir to %s", dir);
setsid();
// setpgid(0,0);
umask(umaskVal);
struct rlimit rlim;
rlim.rlim_cur = rlim.rlim_max = memLimit;
if(setrlimit(RLIMIT_CORE, &rlim) < 0)
perror("setrlimit");
/* Update environment. */
{
struct hash *hash = environToHash(environ);
hashUpdate(hash, "JOB_ID", jobIdString);
hashUpdate(hash, "USER", user);
hashUpdate(hash, "HOME", homeDir);
hashUpdate(hash, "HOST", hostName);
hashUpdate(hash, "PARASOL", "7");
updatePath(hash, userPath, homeDir, sysPath);
addEnvExtras(hash);
environ = hashToEnviron(hash);
freeHashAndVals(&hash);
}
randomSleep(); /* Sleep a random bit before executing this thing
* to help spread out i/o when a big batch of jobs
* hit idle cluster */
execvp(exe, params);
errnoAbort("execvp'ing %s", exe);
}
else
{
/* Wait on executed job and send jobID and status back to
* main process. */
int status = -1;
int cid;
struct paraMessage pm;
struct rudp *ru = NULL;
struct tms tms;
unsigned long uTime = 0;
unsigned long sTime = 0;
if (grandChildId >= 0)
{
signal(SIGTERM, termHandler);
cid = waitpid(grandChildId, &status, 0);
if (cid < 0)
errnoAbort("wait on grandchild failed");
times(&tms);
uTime = ticksToHundreths*tms.tms_cutime;
sTime = ticksToHundreths*tms.tms_cstime;
}
ru = rudpOpen();
if (ru != NULL)
{
ru->maxRetries = 20;
pmInit(&pm, localIp, paraNodePort);
pmPrintf(&pm, "jobDone %s %s %d %lu %lu", managingHost,
jobIdString, status, uTime, sTime);
pmSend(&pm, ru);
rudpClose(&ru);
}
}
}
void runThread(ConnectionString& hostConn, unsigned threadId, unsigned seed,
BSONObj& cmdObj, BSONObjBuilder& result) {
stringstream ss;
ss << "thread-" << threadId;
setThreadName(ss.str().c_str());
// Lock name
string lockName = string_field(cmdObj, "lockName", this->name + "_lock");
// Range of clock skew in diff threads
int skewRange = (int) number_field(cmdObj, "skewRange", 1);
// How long to wait with the lock
int threadWait = (int) number_field(cmdObj, "threadWait", 30);
if(threadWait <= 0) threadWait = 1;
// Max amount of time (ms) a thread waits before checking the lock again
int threadSleep = (int) number_field(cmdObj, "threadSleep", 30);
if(threadSleep <= 0) threadSleep = 1;
// (Legacy) how long until the lock is forced in mins, measured locally
int takeoverMins = (int) number_field(cmdObj, "takeoverMins", 0);
// How long until the lock is forced in ms, only compared locally
unsigned long long takeoverMS = (unsigned long long) number_field(cmdObj, "takeoverMS", 0);
// Whether or not we should hang some threads
int hangThreads = (int) number_field(cmdObj, "hangThreads", 0);
boost::mt19937 gen((boost::mt19937::result_type) seed);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSkew(gen, boost::uniform_int<>(0, skewRange));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomWait(gen, boost::uniform_int<>(1, threadWait));
boost::variate_generator<boost::mt19937&, boost::uniform_int<> > randomSleep(gen, boost::uniform_int<>(1, threadSleep));
int skew = 0;
bool legacy = (takeoverMins > 0);
if (!lock.get()) {
// Pick a skew, but the first two threads skew the whole range
if(threadId == 0)
skew = -skewRange / 2;
else if(threadId == 1)
skew = skewRange / 2;
else skew = randomSkew() - (skewRange / 2);
// Skew this thread
jsTimeVirtualThreadSkew( skew );
log() << "Initializing lock with skew of " << skew << " for thread " << threadId << endl;
lock.reset(new DistributedLock(hostConn, lockName, legacy ? (unsigned long long)takeoverMins : takeoverMS, true, legacy));
log() << "Skewed time " << jsTime() << " for thread " << threadId << endl
<< " max wait (with lock: " << threadWait << ", after lock: " << threadSleep << ")" << endl
<< " takeover in " << (legacy ? (unsigned long long)takeoverMins : takeoverMS) << (legacy ? " (mins local)" : "(ms remote)") << endl;
}
DistributedLock* myLock = lock.get();
bool errors = false;
while (keepGoing) {
try {
if (myLock->lock_try("Testing distributed lock with skew.")) {
log() << "**** Locked for thread " << threadId << endl;
count++;
int before = count;
int sleep = randomWait();
sleepmillis(sleep);
int after = count;
if(after != before) {
errors = true;
log() << "**** !Bad increment while sleeping with lock for: " << sleep << "ms" << endl;
break;
}
// Unlock only half the time...
if(hangThreads == 0 || threadId % hangThreads != 0) {
log() << "**** Unlocking for thread " << threadId << endl;
myLock->unlock();
}
else {
log() << "**** Not unlocking for thread " << threadId << endl;
DistributedLock::killPinger( *myLock );
// We're simulating a crashed process...
break;
}
}
}
catch( LockException& e ) {
log() << "*** !Could not try distributed lock." << m_caused_by(e) << endl;
break;
}
sleepmillis(randomSleep());
}
result << "errors" << errors
<< "skew" << skew
<< "takeover" << (long long) (legacy ? takeoverMS : (unsigned long long)takeoverMins)
<< "localTimeout" << (takeoverMS > 0);
}
