void letterChain(char *inFile, char *outFile, int maxSize)
/* letterChain - Make Markov chain of letters in text. */
{
struct dlList *ll = dlListNew();
int llSize = 0;
int c;
FILE *in = mustOpen(inFile, "r");
FILE *out;
struct dlNode *node;
UBYTE *s;
struct trie *trie;
AllocVar(trie);
while ((c = getc(in)) >= 0)
{
if (llSize < maxSize)
{
s = needMem(1);
*s = c;
dlAddValTail(ll, s);
++llSize;
if (llSize == maxSize)
addToTrie(trie, ll);
}
else
{
node = dlPopHead(ll);
s = node->val;
*s = c;
dlAddTail(ll, node);
addToTrie(trie, ll);
}
}
if (llSize < maxSize)
addToTrie(trie, ll);
while ((node = dlPopHead(ll)) != NULL)
{
addToTrie(trie, ll);
freeMem(node->val);
freeMem(node);
}
dlListFree(&ll);
carefulClose(&in);
out = mustOpen(outFile, "w");
rDumpTrie(0, trie->useCount, trie, out);
carefulClose(&out);
}
void dlCat(struct dlList *a, struct dlList *b)
/* Move items from b to end of a. */
{
struct dlNode *node;
while ((node = dlPopHead(b)) != NULL)
dlAddTail(a, node);
}
static void wordTreeMakeNonsense(struct wordTree *wt, int maxSize, char *firstWord,
int maxOutputWords, FILE *f)
/* Go spew out a bunch of words according to probabilities in tree. */
{
struct dlList *ll = dlListNew();
int listSize = 0;
int outputWords = 0;
for (;;)
{
if (++outputWords > maxOutputWords)
break;
struct dlNode *node;
char *word;
/* Get next predicted word. */
if (listSize == 0)
{
AllocVar(node);
++listSize;
word = firstWord;
}
else if (listSize >= maxSize)
{
node = dlPopHead(ll);
word = predictNext(wt, ll);
}
else
{
word = predictNext(wt, ll);
AllocVar(node);
++listSize;
}
node->val = word;
dlAddTail(ll, node);
if (word == NULL)
break;
/* Output last word in list. */
{
node = ll->tail;
word = node->val;
fprintf(f, "%s", word);
if (word[strlen(word)-1] == '.')
fprintf(f, "\n");
else
fprintf(f, " ");
}
}
dlListFree(&ll);
}
void *synQueueGet(struct synQueue *sq)
/* Get message off start of queue. Wait until there is
* a message if queue is empty. */
{
void *message;
struct dlNode *node;
pthreadMutexLock(&sq->mutex);
while (dlEmpty(sq->queue))
pthreadCondWait(&sq->cond, &sq->mutex);
node = dlPopHead(sq->queue);
pthreadMutexUnlock(&sq->mutex);
message = node->val;
freeMem(node);
return message;
}
void *synQueueGrab(struct synQueue *sq)
/* Get message off start of queue. Return NULL immediately
* if queue is empty. */
{
void *message = NULL;
struct dlNode *node;
pthreadMutexLock(&sq->mutex);
node = dlPopHead(sq->queue);
pthreadMutexUnlock(&sq->mutex);
if (node != NULL)
{
message = node->val;
freeMem(node);
}
return message;
}
void sweepOutOldPacketsSeen(struct rudp *ru, long now)
/* Sweep out old packets seen, we can only remember so many. */
{
int period = 8; /* seconds */
struct dlNode *node;
struct packetSeen *p;
while (TRUE)
{
if (dlEmpty(ru->recvList))
break;
p = ru->recvList->head->val;
if (now - p->lastChecked < period)
break;
--ru->recvCount;
node = dlPopHead(ru->recvList);
freeMem(node);
hashRemove(ru->recvHash, p->recvHashKey);
freePacketSeen(&p);
}
}
void rudpFree(struct rudp **pRu)
/* Free up rudp. Note this does *not* close the associated socket. */
{
struct rudp *ru = *pRu;
if (ru->recvHash)
{
struct dlNode *node;
while (!dlEmpty(ru->recvList))
{
node = dlPopHead(ru->recvList);
struct packetSeen *p = node->val;
freeMem(node);
hashRemove(ru->recvHash, p->recvHashKey);
freePacketSeen(&p);
}
freeDlList(&ru->recvList);
hashFree(&ru->recvHash);
}
freez(pRu);
}
struct dgNodeRef *dgFindPath(struct diGraph *dg, struct dgNode *a, struct dgNode *b)
/* Find shortest path from a to b. Return NULL if can't be found. */
{
struct dgNodeRef *refList = NULL, *ref;
struct dgConnection *con;
struct dgNode *node, *nNode;
struct dlList *fifo;
struct dlNode *ffNode;
struct dgNode endNode;
int fifoSize = 1;
/* Do some quick and easy tests first to return if have no way out
* of node A, or if B directly follows A. */
if (a->nextList == NULL)
return NULL;
if (a == b)
{
AllocVar(ref);
ref->node = a;
return ref;
}
if ((con = dgFindNodeInConList(a->nextList, b)) != NULL)
{
AllocVar(refList);
refList->node = a;
node = con->node;
AllocVar(ref);
ref->node = node;
slAddTail(&refList, ref);
return refList;
}
/* Set up for breadth first traversal. Will use a doubly linked
* list as a fifo. */
for (node = dg->nodeList; node != NULL; node = node->next)
node->tempEntry = NULL;
fifo = newDlList();
dlAddValTail(fifo, a);
a->tempEntry = &endNode;
while ((ffNode = dlPopHead(fifo)) != NULL)
{
--fifoSize;
node = ffNode->val;
freeMem(ffNode);
for (con = node->nextList; con != NULL; con = con->next)
{
nNode = con->node;
if (nNode->tempEntry == NULL)
{
nNode->tempEntry = node;
if (nNode == b)
{
while (nNode != &endNode && nNode != NULL)
{
AllocVar(ref);
ref->node = nNode;
slAddHead(&refList, ref);
nNode = nNode->tempEntry;
}
break;
}
else
{
dlAddValTail(fifo, nNode);
++fifoSize;
if (fifoSize > 100000)
errAbort("Internal error in dgFindPath");
}
}
}
}
freeDlList(&fifo);
return refList;
}
struct wordTree *wordTreeForChainsInFile(char *fileName, int chainSize, struct lm *lm)
/* Return a wordTree of all chains-of-words of length chainSize seen in file.
* Allocate the structure in local memory pool lm. */
{
/* Stuff for processing file a line at a time. */
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *line, *word;
/* We'll keep a chain of three or so words in a doubly linked list. */
struct dlNode *node;
struct dlList *chain = dlListNew();
int curSize = 0;
/* We'll build up the tree starting with an empty root node. */
struct wordTree *wt = wordTreeNew("");
int wordCount = 0;
/* Save time/space by sharing stack between all "following" rbTrees. */
struct rbTreeNode **stack;
lmAllocArray(lm, stack, 256);
/* Loop through each line of input file, lowercasing the whole line, and then
* looping through each word of line, stripping out special chars, and finally
* processing each word. */
while (lineFileNext(lf, &line, NULL))
{
if (lower)
tolowers(line);
while ((word = nextWord(&line)) != NULL)
{
if (unpunc)
{
stripChar(word, ',');
stripChar(word, '.');
stripChar(word, ';');
stripChar(word, '-');
stripChar(word, '"');
stripChar(word, '?');
stripChar(word, '!');
stripChar(word, '(');
stripChar(word, ')');
if (word[0] == 0)
continue;
}
verbose(2, "%s\n", word);
/* We come to this point in the code for each word in the file.
* Here we want to maintain a chain of sequential words up to
* chainSize long. We do this with a doubly-linked list structure.
* For the first few words in the file we'll just build up the list,
* only adding it to the tree when we finally do get to the desired
* chain size. Once past the initial section of the file we'll be
* getting rid of the first link in the chain as well as adding a new
* last link in the chain with each new word we see. */
if (curSize < chainSize)
{
dlAddValTail(chain, cloneString(word));
++curSize;
if (curSize == chainSize)
addChainToTree(wt, chain, lm, stack);
}
else
{
/* Reuse doubly-linked-list node, but give it a new value, as we move
* it from head to tail of list. */
node = dlPopHead(chain);
freeMem(node->val);
node->val = cloneString(word);
dlAddTail(chain, node);
addChainToTree(wt, chain, lm, stack);
}
++wordCount;
}
}
/* Handle last few words in file, where can't make a chain of full size. Need
* a special case for file that has fewer than chain size words too. */
if (curSize < chainSize)
addChainToTree(wt, chain, lm, stack);
while ((node = dlPopHead(chain)) != NULL)
{
addChainToTree(wt, chain, lm, stack);
freeMem(node->val);
freeMem(node);
}
dlListFree(&chain);
lineFileClose(&lf);
return wt;
}
struct wordStore *wordStoreForChainsInFile(char *fileName, int chainSize)
/* Return a wordStore containing all words, and also all chains-of-words of length
* chainSize seen in file. */
{
/* Stuff for processing file a line at a time. */
struct lineFile *lf = lineFileOpen(fileName, TRUE);
char *line, *word;
/* We'll build up the tree starting with an empty root node. */
struct wordStore *store = wordStoreNew(chainSize);
struct wordTree *wt = store->markovChains = wordTreeNew(wordStoreAdd(store, ""));
/* Loop through each line of file, treating it as a separate read. There's
* special cases at the beginning and end of line, and for short lines. In the
* main case we'll be maintaining a chain (doubly linked list) of maxChainSize words,
* popping off one word from the start, and adding one word to the end for each
* new word we encounter. This list is added to the tree each iteration. */
while (lineFileNext(lf, &line, NULL))
{
/* We'll keep a chain of three or so words in a doubly linked list. */
struct dlNode *node;
struct dlList *chain = dlListNew();
int curSize = 0;
int wordCount = 0;
/* skipping the first word which is the read id */
word = nextWord(&line);
while ((word = nextWord(&line)) != NULL)
{
struct wordInfo *info = wordStoreAdd(store, word);
/* For the first few words in the file after ID, we'll just build up the chain,
* only adding it to the tree when we finally do get to the desired
* chain size. Once past the initial section of the file we'll be
* getting rid of the first link in the chain as well as adding a new
* last link in the chain with each new word we see. */
if (curSize < chainSize)
{
dlAddValTail(chain, info);
++curSize;
if (curSize == chainSize)
addChainToTree(wt, chain);
}
else
{
/* Reuse doubly-linked-list node, but give it a new value, as we move
* it from head to tail of list. */
node = dlPopHead(chain);
node->val = info;
dlAddTail(chain, node);
addChainToTree(wt, chain);
}
++wordCount;
}
/* Handle last few words in line, where can't make a chain of full size. Also handles
* lines that have fewer than chain size words. */
if (curSize < chainSize)
addChainToTree(wt, chain);
while ((node = dlPopHead(chain)) != NULL)
{
if (!dlEmpty(chain))
addChainToTree(wt, chain);
freeMem(node);
}
dlListFree(&chain);
}
lineFileClose(&lf);
wordTreeSort(wt); // Make output of chain file prettier
return store;
}
