void output(MotifSearch* ms) {
string tmpstr(outfile);
string outstr(outfile);
string lockstr(outfile);
tmpstr.append(".tmp.ms");
outstr.append(".ms");
lockstr.append(".lock");
ofstream tmp(tmpstr.c_str(), ios::trunc);
ms->full_output(tmp);
tmp.close();
struct flock fl;
int fd;
fl.l_type = F_WRLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
fl.l_pid = getpid();
fd = open(lockstr.c_str(), O_WRONLY | O_CREAT, 0644);
while(fcntl(fd, F_SETLK, &fl) == -1) {
cerr << "Waiting for lock release on archive file...\n";
sleep(10);
}
rename(tmpstr.c_str(), outstr.c_str());
cerr << "Archive output completed.\n";
fl.l_type = F_UNLCK;
fcntl(fd, F_SETLK, &fl);
close(fd);
}
// Remove our mail-spool-file lock (n.b. we should only try this if
// we're the ones who made the lock in the first place!)
PRBool YieldSpoolLock(const char *spoolnameStr)
{
LOG(("YieldSpoolLock(%s)", spoolnameStr));
nsCAutoString lockstr(spoolnameStr);
lockstr.Append(".lock");
nsresult rv;
// Create nsILocalFile for the spool.lock file
nsCOMPtr<nsILocalFile> locklocfile;
rv = NS_NewNativeLocalFile(lockstr, PR_TRUE, getter_AddRefs(locklocfile));
if (NS_FAILED(rv))
return PR_FALSE;
// Check if the lock file exists
PRBool exists;
rv = locklocfile->Exists(&exists);
if (NS_FAILED(rv))
return PR_FALSE;
// Delete the file if it exists
if (exists) {
rv = locklocfile->Remove(PR_FALSE /* non-recursive */);
if (NS_FAILED(rv))
return PR_FALSE;
}
LOG(("YieldSpoolLock was successful."));
// Success.
return PR_TRUE;
}
// Remove our mail-spool-file lock (n.b. we should only try this if
// we're the ones who made the lock in the first place!)
bool YieldSpoolLock(const char *aSpoolName, bool aUsingLockFile)
{
LOG(("YieldSpoolLock(%s)", aSpoolName));
if (!aUsingLockFile) {
nsCOMPtr<nsILocalFile> spoolFile;
nsresult rv = NS_NewNativeLocalFile(nsDependentCString(aSpoolName),
true,
getter_AddRefs(spoolFile));
NS_ENSURE_SUCCESS(rv, false);
PRFileDesc *fd;
rv = spoolFile->OpenNSPRFileDesc(PR_RDWR, 0, &fd);
NS_ENSURE_SUCCESS(rv, false);
bool unlockSucceeded = PR_UnlockFile(fd) == PR_SUCCESS;
PR_Close(fd);
if (unlockSucceeded)
LOG(("YieldSpoolLock was successful."));
return unlockSucceeded;
}
nsCAutoString lockstr(aSpoolName);
lockstr.Append(".lock");
nsresult rv;
// Create nsILocalFile for the spool.lock file
nsCOMPtr<nsILocalFile> locklocfile;
rv = NS_NewNativeLocalFile(lockstr, true, getter_AddRefs(locklocfile));
if (NS_FAILED(rv))
return false;
// Check if the lock file exists
bool exists;
rv = locklocfile->Exists(&exists);
if (NS_FAILED(rv))
return false;
// Delete the file if it exists
if (exists) {
rv = locklocfile->Remove(false /* non-recursive */);
if (NS_FAILED(rv))
return false;
}
LOG(("YieldSpoolLock was successful."));
// Success.
return true;
}
bool ObtainSpoolLock(const char *aSpoolName,
int aSeconds /* number of seconds to retry */,
bool *aUsingLockFile)
{
NS_ENSURE_TRUE(aUsingLockFile, false);
/*
* Locking procedures:
* If the directory is not writable, we want to use the appropriate system
* utilites to lock the file.
* If the directory is writable, we want to go through the create-and-link
* locking procedures to make it atomic for certain networked file systems.
* This involves creating a .mozlock file and attempting to hard-link it to
* the customary .lock file.
*/
nsCOMPtr<nsILocalFile> spoolFile;
nsresult rv = NS_NewNativeLocalFile(nsDependentCString(aSpoolName),
true,
getter_AddRefs(spoolFile));
NS_ENSURE_SUCCESS(rv, false);
nsCOMPtr<nsIFile> directory;
rv = spoolFile->GetParent(getter_AddRefs(directory));
NS_ENSURE_SUCCESS(rv, false);
rv = directory->IsWritable(aUsingLockFile);
NS_ENSURE_SUCCESS(rv, false);
if (!*aUsingLockFile) {
LOG(("Attempting to use kernel file lock"));
PRFileDesc *fd;
rv = spoolFile->OpenNSPRFileDesc(PR_RDWR, 0, &fd);
NS_ENSURE_SUCCESS(rv, false);
PRStatus lock_result;
int retry_count = 0;
do {
lock_result = PR_TLockFile(fd);
retry_count++;
LOG(("Attempt %d of %d to lock file", retry_count, aSeconds));
if (aSeconds > 0 && lock_result == PR_FAILURE) {
// pause 1sec, waiting for .lock to go away
PRIntervalTime sleepTime = 1000; // 1 second
PR_Sleep(sleepTime);
}
} while (lock_result == PR_FAILURE && retry_count < aSeconds);
LOG(("Lock result: %d", lock_result));
PR_Close(fd);
return lock_result == PR_SUCCESS;
}
// How to lock using files:
// step 1: create SPOOLNAME.mozlock
// 1a: can remove it if it already exists (probably crash-droppings)
// step 2: hard-link SPOOLNAME.mozlock to SPOOLNAME.lock for NFS atomicity
// 2a: if SPOOLNAME.lock is >60sec old then nuke it from orbit
// 2b: repeat step 2 until retry-count expired or hard-link succeeds
// step 3: remove SPOOLNAME.mozlock
// step 4: If step 2 hard-link failed, fail hard; we do not hold the lock
// DONE.
//
// (step 2a not yet implemented)
nsCAutoString mozlockstr(aSpoolName);
mozlockstr.Append(".mozlock");
nsCAutoString lockstr(aSpoolName);
lockstr.Append(".lock");
// Create nsILocalFile for the spool.mozlock file
nsCOMPtr<nsILocalFile> tmplocfile;
rv = NS_NewNativeLocalFile(mozlockstr, true, getter_AddRefs(tmplocfile));
if (NS_FAILED(rv))
return false;
// THOUGHT: hmm, perhaps use MakeUnique to generate us a unique mozlock?
// ... perhaps not, MakeUnique implementation looks racey -- use mktemp()?
// step 1: create SPOOLNAME.mozlock
rv = tmplocfile->Create(nsIFile::NORMAL_FILE_TYPE, 0666);
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) {
// can't create our .mozlock file... game over already
LOG(("Failed to create file %s\n", mozlockstr.get()));
return false;
}
// step 2: hard-link .mozlock file to .lock file (this wackiness
// is necessary for non-racey locking on NFS-mounted spool dirs)
// n.b. XPCOM utilities don't support hard-linking yet, so we
// skip out to <unistd.h> and the POSIX interface for link()
int link_result = 0;
int retry_count = 0;
do {
link_result = link(mozlockstr.get(),lockstr.get());
retry_count++;
LOG(("Attempt %d of %d to create lock file", retry_count, aSeconds));
if (aSeconds > 0 && link_result == -1) {
// pause 1sec, waiting for .lock to go away
PRIntervalTime sleepTime = 1000; // 1 second
PR_Sleep(sleepTime);
}
} while (link_result == -1 && retry_count < aSeconds);
LOG(("Link result: %d", link_result));
// step 3: remove .mozlock file, in any case
rv = tmplocfile->Remove(false /* non-recursive */);
if (NS_FAILED(rv)) {
// Could not delete our .mozlock file... very unusual, but
// not fatal.
LOG(("Unable to delete %s", mozlockstr.get()));
}
// step 4: now we know whether we succeeded or failed
return link_result == 0;
}
int main(int argc, char *argv[]) {
set_new_handler(alloc_error);
if(argc < 6) {
print_usage(cout);
exit(0);
}
string seqfile; // file with sequences
string exprfile; // file with expression data
string subsetfile; // file with subset of sequence names to search
string scorefile; // file with scores
if(! GetArg2(argc, argv, "-s", seqfile)) {
cerr << "Please specify sequence file\n\n";
print_usage(cout);
exit(0);
}
if(! GetArg2(argc, argv, "-o", outfile)) {
cerr << "Please specify output file\n\n";
print_usage(cout);
exit(0);
}
int search_type = UNDEFINED;
if(GetArg2(argc, argv, "-ex", exprfile)) {
search_type = EXPRESSION;
}
if(GetArg2(argc, argv, "-su", subsetfile)) {
search_type = SUBSET;
}
if(GetArg2(argc, argv, "-sc", scorefile)) {
search_type = SCORE;
}
if(search_type == UNDEFINED) {
cerr << "Please specify either an expression data file, a file with a subset of sequence names, or a file with sequence scores.\n\n";
print_usage(cout);
exit(0);
}
// Decide mode of running
archive = false;
if(! GetArg2(argc, argv, "-worker", worker)) {
worker = -1;
archive = true;
}
// Read parameters
vector<string> seqs;
cerr << "Reading sequence data from '" << seqfile << "'... ";
get_fasta_fast(seqfile.c_str(), seqs, seq_nameset);
cerr << "done.\n";
ngenes = seq_nameset.size();
npoints = 0;
if(search_type == EXPRESSION) {
cerr << "Reading expression data from '" << exprfile << "'... ";
get_expr(exprfile.c_str(), expr, data_nameset);
cerr << "done.\n";
npoints = expr[0].size();
nsubset = 0;
} else if(search_type == SUBSET) {
cerr << "Reading subset of sequences to search from '" << subsetfile << "'... ";
get_list(subsetfile.c_str(), subset);
cerr << "done.\n";
npoints = 0;
for(int i = 0; i < ngenes; i++) {
vector<float> row(0);
expr.push_back(row);
}
nsubset = subset.size();
sort(subset.begin(), subset.end());
} else if(search_type == SCORE) {
cerr << "Reading sequence scores from '" << scorefile << "'... ";
get_scores(scorefile.c_str(), scores, data_nameset);
cerr << "done.\n";
npoints = 1;
nsubset = 0;
}
vector<vector <float> > newexpr;
vector<float> newscores;
if(search_type == EXPRESSION) {
order_data_expr(newexpr);
}
if(search_type == SCORE) {
order_data_scores(newscores);
}
if(search_type == EXPRESSION) {
cerr << "Successfully read input files -- dataset size is " << ngenes << " sequences X " << npoints << " timepoints\n";
} else if(search_type == SUBSET) {
cerr << "Successfully read input files -- dataset size is " << ngenes << " sequences, with " << nsubset << " to be searched\n";
} else if(search_type == SCORE) {
cerr << "Successfully read input files -- dataset size is " << ngenes << " scored sequences\n";
}
cerr << "Setting up MotifSearch... ";
if(! GetArg2(argc, argv, "-numcols", ncol)) ncol = 10;
if(! GetArg2(argc, argv, "-order", order)) order = 0;
if(! GetArg2(argc, argv, "-simcut", simcut)) simcut = 0.8;
if(! GetArg2(argc, argv, "-maxm", maxm)) maxm = 20;
MotifSearch* ms;
if(search_type == EXPRESSION) {
ms = new MotifSearchExpr(seq_nameset, seqs, ncol, order, simcut, maxm, newexpr, npoints);
} else if(search_type == SCORE) {
ms = new MotifSearchScore(seq_nameset, seqs, ncol, order, simcut, maxm, newscores);
} else {
ms = new MotifSearchSubset(seq_nameset, seqs, ncol, order, simcut, maxm, subset);
}
ms->modify_params(argc, argv);
ms->set_final_params();
ms->ace_initialize();
cerr << "done.\n";
cerr << "Random seed: " << ms->get_params().seed << '\n';
if(archive) {
cerr << "Running in archive mode...\n";
string archinstr(outfile);
archinstr.append(".ms");
ifstream archin(archinstr.c_str());
if(archin) {
cerr << "Refreshing from existing archive file " << archinstr << "... ";
ms->get_archive().read(archin);
cerr << "done.\n";
}
while(true) {
int found = read_motifs(ms);
if(found > 0) output(ms);
if(found < 50) sleep(10);
}
} else {
cerr << "Running as worker " << worker << "...\n";
int nruns = ms->positions_in_search_space()/(ms->get_params().expect * ncol);
nruns *= ms->get_params().oversample;
nruns /= ms->get_params().undersample;
cerr << "Restarts planned: " << nruns << '\n';
string archinstr(outfile);
archinstr.append(".ms");
string lockstr(outfile);
lockstr.append(".lock");
for(int j = 1; j <= nruns; j++) {
if(j == 1 || j % 50 == 0 || search_type == SUBSET) {
struct flock fl;
int fd;
fl.l_type = F_RDLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
fl.l_pid = getpid();
fd = open(lockstr.c_str(), O_RDONLY);
if(fd == -1) {
if(errno != ENOENT)
cerr << "\t\tUnable to read lock file, error was " << strerror(errno) << "\n";
} else {
while(fcntl(fd, F_SETLK, &fl) == -1) {
cerr << "\t\tWaiting for lock release on archive file... \n";
sleep(10);
}
ifstream archin(archinstr.c_str());
if(archin) {
cerr << "\t\tRefreshing archive from " << archinstr << "...";
ms->get_archive().clear();
ms->get_archive().read(archin);
archin.close();
cerr << "done.\n";
}
fl.l_type = F_UNLCK;
fcntl(fd, F_SETLK, &fl);
close(fd);
cerr << "\t\tArchive now has " << ms->get_archive().nmots() << " motifs\n";
}
}
cerr << "\t\tSearch restart #" << j << "/" << nruns << "\n";
ms->search_for_motif(worker, j, outfile);
}
}
delete ms;
return 0;
}
PRBool ObtainSpoolLock(const char *spoolnameStr,
int seconds /* number of seconds to retry */)
{
// How to lock:
// step 1: create SPOOLNAME.mozlock
// 1a: can remove it if it already exists (probably crash-droppings)
// step 2: hard-link SPOOLNAME.mozlock to SPOOLNAME.lock for NFS atomicity
// 2a: if SPOOLNAME.lock is >60sec old then nuke it from orbit
// 2b: repeat step 2 until retry-count expired or hard-link succeeds
// step 3: remove SPOOLNAME.mozlock
// step 4: If step 2 hard-link failed, fail hard; we do not hold the lock
// DONE.
//
// (step 2a not yet implemented)
nsCAutoString mozlockstr(spoolnameStr);
mozlockstr.Append(".mozlock");
nsCAutoString lockstr(spoolnameStr);
lockstr.Append(".lock");
nsresult rv;
// Create nsILocalFile for the spool.mozlock file
nsCOMPtr<nsILocalFile> tmplocfile;
rv = NS_NewNativeLocalFile(mozlockstr, PR_TRUE, getter_AddRefs(tmplocfile));
if (NS_FAILED(rv))
return PR_FALSE;
// THOUGHT: hmm, perhaps use MakeUnique to generate us a unique mozlock?
// ... perhaps not, MakeUnique implementation looks racey -- use mktemp()?
// step 1: create SPOOLNAME.mozlock
rv = tmplocfile->Create(nsIFile::NORMAL_FILE_TYPE, 0666);
if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) {
// can't create our .mozlock file... game over already
LOG(("Failed to create file %s\n", mozlockstr.get()));
return PR_FALSE;
}
// step 2: hard-link .mozlock file to .lock file (this wackiness
// is necessary for non-racey locking on NFS-mounted spool dirs)
// n.b. XPCOM utilities don't support hard-linking yet, so we
// skip out to <unistd.h> and the POSIX interface for link()
int link_result = 0;
int retry_count = 0;
do {
link_result = link(mozlockstr.get(),lockstr.get());
retry_count++;
LOG(("Attempt %d of %d to create lock file", retry_count, seconds));
if (seconds > 0 && link_result == -1) {
// pause 1sec, waiting for .lock to go away
PRIntervalTime sleepTime = 1000; // 1 second
PR_Sleep(sleepTime);
}
} while (link_result == -1 && retry_count < seconds);
LOG(("Link result: %d", link_result));
// step 3: remove .mozlock file, in any case
rv = tmplocfile->Remove(PR_FALSE /* non-recursive */);
if (NS_FAILED(rv)) {
// Could not delete our .mozlock file... very unusual, but
// not fatal.
LOG(("Unable to delete %s", mozlockstr.get()));
}
// step 4: now we know whether we succeeded or failed
if (link_result == 0)
return PR_TRUE; // got the lock.
else
return PR_FALSE; // didn't. :(
}