struct dlNode *dlGetBeforeHead(struct dlList *list)
/* Get the node before the head of the list */
{
if (dlEmpty(list))
return list->head;
else
return list->head->prev;
}
struct dlNode *dlGetAfterTail(struct dlList *list)
/* Get the node after the tail of the list */
{
if (dlEmpty(list))
return list->tail;
else
return list->tail->next;
}
struct cachedSeqFile *openTwoBitFromCache(struct dlList *cache, char *fileName)
/* Return open file handle via cache. In this case it's just a cache of one. */
{
struct cachedSeqFile *cn;
if (dlEmpty(cache))
{
AllocVar(cn);
cn->fileName = cloneString(fileName);
cn->tbf = twoBitOpen(fileName);
dlAddValHead(cache, cn);
}
else
cn = cache->head->val;
return cn;
}
void *synQueueGet(struct synQueue *sq)
/* Get message off start of queue. Wait until there is
* a message if queue is empty. */
{
void *message;
struct dlNode *node;
pthreadMutexLock(&sq->mutex);
while (dlEmpty(sq->queue))
pthreadCondWait(&sq->cond, &sq->mutex);
node = dlPopHead(sq->queue);
pthreadMutexUnlock(&sq->mutex);
message = node->val;
freeMem(node);
return message;
}
struct dgNodeRef *dgConstrainedPriorityOrder(struct diGraph *dg)
/* Return traversal of graph in priority order subject to
* constraint that all parents must be output before
* their children regardless of node priority.
* Graph must be cycle free. */
{
struct dlList *sortedList = newDlList();
struct dgNode *graphNode;
struct dlNode *listNode;
struct dgNodeRef *refList = NULL, *ref;
if (dgHasCycles(dg))
errAbort("Call to dgConstrainedPriorityOrder on graph with cycles.");
/* Make up list sorted by priority. */
for (graphNode = dg->nodeList; graphNode != NULL; graphNode = graphNode->next)
{
dlAddValTail(sortedList, graphNode);
graphNode->visited = FALSE;
}
dlSort(sortedList, cmpPriority);
/* Loop taking first member of list with no untraversed parents. */
while (!dlEmpty(sortedList))
{
for (listNode = sortedList->head; listNode->next != NULL; listNode = listNode->next)
{
graphNode = listNode->val;
if (dgParentsAllVisited(graphNode))
{
dlRemove(listNode);
freeMem(listNode);
AllocVar(ref);
ref->node = graphNode;
slAddHead(&refList, ref);
graphNode->visited = TRUE;
break;
}
}
}
freeDlList(&sortedList);
slReverse(&refList);
return refList;
}
void sweepOutOldPacketsSeen(struct rudp *ru, long now)
/* Sweep out old packets seen, we can only remember so many. */
{
int period = 8; /* seconds */
struct dlNode *node;
struct packetSeen *p;
while (TRUE)
{
if (dlEmpty(ru->recvList))
break;
p = ru->recvList->head->val;
if (now - p->lastChecked < period)
break;
--ru->recvCount;
node = dlPopHead(ru->recvList);
freeMem(node);
hashRemove(ru->recvHash, p->recvHashKey);
freePacketSeen(&p);
}
}
void rudpFree(struct rudp **pRu)
/* Free up rudp. Note this does *not* close the associated socket. */
{
struct rudp *ru = *pRu;
if (ru->recvHash)
{
struct dlNode *node;
while (!dlEmpty(ru->recvList))
{
node = dlPopHead(ru->recvList);
struct packetSeen *p = node->val;
freeMem(node);
hashRemove(ru->recvHash, p->recvHashKey);
freePacketSeen(&p);
}
freeDlList(&ru->recvList);
hashFree(&ru->recvHash);
}
freez(pRu);
}
void ccCp(char *source, char *dest, char *hostList)
/* Copy source to dest on all files in hostList. */
{
time_t startTime = time(NULL);
time_t curTime, lastTime = 0;
struct machine *machineList = NULL;
struct netSwitch *nsList;
struct machine *m, *m2;
struct dlList *toDoList = newDlList(); /* We haven't done these. */
struct dlList *finishedList = newDlList(); /* All done here. */
struct dlList *sourceList = newDlList(); /* These are sources for copies. */
struct dlList *workingList = newDlList(); /* These are copying data to themselves. */
struct dlList *errList = newDlList(); /* These are messed up 3x or more. */
bool firstOk = FALSE;
struct dlNode *finNode, *node, *sourceNode, *destNode;
struct dyString *cmd = newDyString(256);
int machineCount;
int machinesFinished = 0;
char *thisHost = getenv("HOST");
off_t size;
int goodMachines;
double grandTotal;
/* Get host and switch info. */
readHosts(hostList, &machineList, &nsList);
machineCount = slCount(machineList);
/* Make sure file exists.... */
if (!fileExists(source))
errAbort("%s doesn't exist\n", source);
size = fileSize(source);
printf("Copying %s (%lld bytes) to %d machines\n", source, (unsigned long long)size, machineCount);
/* Add everything to the to-do list. */
for (m = machineList; m != NULL; m = m->next)
{
dlAddValTail(toDoList, m);
}
/* Loop through to-do list trying to do first copy. */
for (node = toDoList->head; node->next != NULL; node = node->next)
{
m = node->val;
dyStringClear(cmd);
m = node->val;
if (sameString(thisHost, m->name))
{
if (sameString(source, dest))
{
/* Hey, this is too easy. */
firstOk = TRUE;
++machinesFinished;
break;
}
else
{
dyStringPrintf(cmd, "cp %s %s", source, dest);
}
}
else
{
dyStringPrintf(cmd, "rcp %s %s:%s", source, m->name, dest);
}
if (system(cmd->string) == 0)
{
dlRemove(node);
dlAddTail(finishedList, node);
firstOk = TRUE;
++machinesFinished;
break;
}
else /* some error in rcp */
{
warn("Problem with %s\n", cmd->string);
m->errCount += 1;
}
}
/* Loop around launching child processes to copy and
* wait for them to finish. */
while (machinesFinished < machineCount)
{
int pid;
int status;
/* Start all possible copies. */
while (matchMaker(finishedList, toDoList, &sourceNode, &destNode))
{
dlAddTail(sourceList, sourceNode);
dlAddTail(workingList, destNode);
m = destNode->val;
m->sourceNode = sourceNode;
startCopy(sourceNode->val, destNode->val, dest, thisHost, cmd);
}
curTime = time(NULL);
if (curTime - lastTime >= 3)
{
printf("%d finished in %d seconds, %d in progress, %d to start, %d errors, %d total\n",
dlCount(finishedList) + dlCount(sourceList), (int)(curTime - startTime),
dlCount(workingList), dlCount(toDoList), dlCount(errList), machineCount);
lastTime = curTime;
}
/* Wait for a child to finish. Figure out which machine it is. */
pid = wait(&status);
finNode = NULL;
for (node = workingList->head; node->next != NULL; node = node->next)
{
m = node->val;
if (m->pid == pid)
{
finNode = node;
break;
}
}
if (finNode == NULL)
{
errAbort("Returned from wait on unknown child %d\n", pid);
continue;
}
m = finNode->val;
m->pid = 0;
dlRemove(finNode);
dlRemove(m->sourceNode);
m2 = m->sourceNode->val;
if (m->netSwitch != m2->netSwitch)
--crossSwitchCount;
dlAddTail(finishedList, m->sourceNode);
if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
{
/* Good return - move self and source node to finished list. */
++machinesFinished;
dlAddTail(finishedList, finNode);
}
else
{
/* Bad return. Increment error count, and maybe move it to
* error list. */
if (++m->errCount >= maxErrCount)
{
++machinesFinished;
dlAddTail(errList, finNode);
fprintf(stderr, "Gave up on %s\n", m->name);
}
else
{
dlAddMiddle(toDoList, finNode);
fprintf(stderr, "Retry %d on %s\n", m->errCount, m->name);
}
}
}
if (!dlEmpty(errList))
{
fprintf(stderr, "errors in:");
for (node = errList->head; node->next != NULL; node = node->next)
{
m = node->val;
fprintf(stderr, " %s", m->name);
}
fprintf(stderr, "\n");
}
goodMachines = dlCount(finishedList);
grandTotal = (double)goodMachines * (double)size;
printf("Copied to %d of %d machines (grand total %e bytes) in %d seconds\n",
goodMachines, machineCount, grandTotal, (int)(time(NULL) - startTime));
}
struct wordStore *wordStoreForChainsInFile(char *fileName, int chainSize)
/* Return a wordStore containing all words, and also all chains-of-words of length
* chainSize seen in file. */
{
/* Stuff for processing file a line at a time. */
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *line, *word;
/* We'll build up the tree starting with an empty root node. */
struct wordStore *store = wordStoreNew(chainSize);
struct wordTree *wt = store->markovChains = wordTreeNew(wordStoreAdd(store, ""));
/* Loop through each line of file, treating it as a separate read. There's
* special cases at the beginning and end of line, and for short lines. In the
* main case we'll be maintaining a chain (doubly linked list) of maxChainSize words,
* popping off one word from the start, and adding one word to the end for each
* new word we encounter. This list is added to the tree each iteration. */
while (lineFileNext(lf, &line, NULL))
{
/* We'll keep a chain of three or so words in a doubly linked list. */
struct dlNode *node;
struct dlList *chain = dlListNew();
int curSize = 0;
int wordCount = 0;
/* skipping the first word which is the read id */
word = nextWord(&line);
while ((word = nextWord(&line)) != NULL)
{
struct wordInfo *info = wordStoreAdd(store, word);
/* For the first few words in the file after ID, we'll just build up the chain,
* only adding it to the tree when we finally do get to the desired
* chain size. Once past the initial section of the file we'll be
* getting rid of the first link in the chain as well as adding a new
* last link in the chain with each new word we see. */
if (curSize < chainSize)
{
dlAddValTail(chain, info);
++curSize;
if (curSize == chainSize)
addChainToTree(wt, chain);
}
else
{
/* Reuse doubly-linked-list node, but give it a new value, as we move
* it from head to tail of list. */
node = dlPopHead(chain);
node->val = info;
dlAddTail(chain, node);
addChainToTree(wt, chain);
}
++wordCount;
}
/* Handle last few words in line, where can't make a chain of full size. Also handles
* lines that have fewer than chain size words. */
if (curSize < chainSize)
addChainToTree(wt, chain);
while ((node = dlPopHead(chain)) != NULL)
{
if (!dlEmpty(chain))
addChainToTree(wt, chain);
freeMem(node);
}
dlListFree(&chain);
}
lineFileClose(&lf);
wordTreeSort(wt); // Make output of chain file prettier
return store;
}