static struct bed *bedTable2(struct sqlConnection *conn,
struct region *region, char *table2)
/* get a bed list, possibly complement, for table2 */
{
/* This use of bedTable rather than a bitmap is not really working. The
* rest of the table browser does intersection at the exon level, while
* the wig code, which this is part of, does it at the gene level. I
* noticed it while working on the corresponding routines for bigWig,
* which I'm building to work with bitmaps at the exon level. I'm not
* sure it's worth fixing this code since nobody has complained, and we're
* probably going to be doing mostly bigWig rather than wig in the future.
* -JK */
boolean invTable2 = cartCgiUsualBoolean(cart, hgtaInvertTable2, FALSE);
char *op = cartString(cart, hgtaIntersectOp);
struct bed *bedList = NULL;
struct lm *lm1 = lmInit(64*1024);
/* fetch table 2 as a bed list */
bedList = getFilteredBeds(conn, table2, region, lm1, NULL);
/* If table 2 bed list needs to be complemented (!table2), then do so */
if (invTable2 || sameString("none", op))
{
unsigned chromStart = 0; /* start == end == 0 */
unsigned chromEnd = 0; /* means do full chrom */
unsigned chromSize = hChromSize(database, region->chrom);
struct lm *lm2 = lmInit(64*1024);
struct bed *inverseBedList = NULL; /* new list */
if ((region->start != 0) || (region->end != 0))
{
chromStart = region->start;
chromEnd = region->end;
}
if ((struct bed *)NULL == bedList)
{
if (0 == region->end)
chromEnd = chromSize;
addBedElement(&inverseBedList, region->chrom, chromStart, chromEnd,
1, lm2);
}
else
inverseBedList=invertBedList(bedList, lm2, region->chrom, chromStart,
chromEnd, chromSize);
lmCleanup(&lm1); /* == bedFreeList(&bedList) */
return inverseBedList;
}
else
return bedList;
}
void doRewrite(char *outDir, char *inDir, char *trackFile)
/* Do some sort of rewrite on entire system. */
{
/* Make list and hash of root dir */
struct lm *rootLm = lmInit(0);
char rootName[PATH_LEN];
safef(rootName, sizeof(rootName), "%s/%s", inDir, trackFile);
struct raLevel *rootLevel = raLevelRead(rootName, rootLm);
/* Make subdirectory list. */
struct fileInfo *org, *orgList = listDirX(inDir, "*", FALSE);
for (org = orgList; org != NULL; org = org->next)
{
if (org->isDir)
{
struct lm *orgLm = lmInit(0);
char inOrgDir[PATH_LEN], outOrgDir[PATH_LEN];
safef(inOrgDir, sizeof(inOrgDir), "%s/%s", inDir, org->name);
safef(outOrgDir, sizeof(outOrgDir), "%s/%s", outDir, org->name);
char inOrgFile[PATH_LEN];
safef(inOrgFile, sizeof(inOrgFile), "%s/%s", inOrgDir, trackFile);
struct raLevel *orgLevel = raLevelRead(inOrgFile, orgLm);
orgLevel->parent = rootLevel;
rewriteLevel(orgLevel, outOrgDir, orgLm);
struct fileInfo *db, *dbList = listDirX(inOrgDir, "*", FALSE);
for (db = dbList; db != NULL; db = db->next)
{
if (db->isDir)
{
struct lm *dbLm = lmInit(0);
char inDbDir[PATH_LEN], outDbDir[PATH_LEN];
safef(inDbDir, sizeof(inDbDir), "%s/%s", inOrgDir, db->name);
safef(outDbDir, sizeof(outDbDir), "%s/%s", outOrgDir, db->name);
char inDbFile[PATH_LEN];
safef(inDbFile, sizeof(inDbFile), "%s/%s", inDbDir, trackFile);
struct raLevel *dbLevel = raLevelRead(inDbFile, dbLm);
dbLevel->parent = orgLevel;
rewriteLevel(dbLevel, outDbDir, dbLm);
hashFree(&dbLevel->trackHash);
lmCleanup(&dbLm);
}
}
hashFree(&orgLevel->trackHash);
lmCleanup(&orgLm);
}
}
hashFree(&rootLevel->trackHash);
lmCleanup(&rootLm);
}
void bigBedAddLinkedFeaturesFrom(struct track *track,
char *chrom, int start, int end, int scoreMin, int scoreMax, boolean useItemRgb,
int fieldCount, struct linkedFeatures **pLfList)
/* Read in items in chrom:start-end from bigBed file named in track->bbiFileName, convert
* them to linkedFeatures, and add to head of list. */
{
struct lm *lm = lmInit(0);
struct trackDb *tdb = track->tdb;
struct bigBedInterval *bb, *bbList = bigBedSelectRange(track, chrom, start, end, lm);
char *bedRow[32];
char startBuf[16], endBuf[16];
char *scoreFilter = cartOrTdbString(cart, track->tdb, "scoreFilter", NULL);
char *mouseOverField = cartOrTdbString(cart, track->tdb, "mouseOverField", NULL);
int minScore = 0;
if (scoreFilter)
minScore = atoi(scoreFilter);
int mouseOverIdx = bbExtraFieldIndex(tdb, mouseOverField);
for (bb = bbList; bb != NULL; bb = bb->next)
{
char* mouseOver = restField(bb, mouseOverIdx);
bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow));
struct bed *bed = bedLoadN(bedRow, fieldCount);
struct linkedFeatures *lf = bedMungToLinkedFeatures(&bed, tdb, fieldCount,
scoreMin, scoreMax, useItemRgb);
if (scoreFilter == NULL || lf->score >= minScore)
slAddHead(pLfList, lf);
lf->mouseOver = mouseOver; // leaks some memory, cloneString handles NULL ifself
if (sameString(track->tdb->type, "bigGenePred"))
lf->original = genePredFromBigGenePred(chromName, bb);
}
lmCleanup(&lm);
}
struct perBaseWig* perBaseWigLoadContinue(struct metaBig* mb, char* chrom, int start, int end)
/* load a perBaseWig from a wig/bigWig that's already open */
{
if (mb->type != isaBigWig)
return NULL;
struct perBaseWig* list = NULL;
struct lm* lm = lmInit(0);
struct bbiInterval* intervals = bigWigIntervalQuery(mb->big.bbi, chrom, start, end, lm);
struct bbiInterval *bbStart = intervals, *bbEnd;
while (bbStart != NULL) {
struct perBaseWig* region;
struct bbiInterval* cur;
int i = 0;
bbEnd = bbStart;
/* loop until discontinuity detected */
while ((bbEnd->next != NULL) && (bbEnd->end == bbEnd->next->start))
bbEnd = bbEnd->next;
region = alloc_perBaseWig(chrom, bbStart->start, bbEnd->end);
for (cur = bbStart; cur != bbEnd->next; cur = cur->next) {
int j;
for (j = cur->start; j < cur->end; j++)
region->data[i++] = cur->val;
}
slAddHead(&list, region);
bbStart = bbEnd->next;
}
lmCleanup(&lm);
slReverse(&list);
return list;
}
struct slName *randomBigBedIds(char *table, struct sqlConnection *conn, int count)
/* Return some arbitrary IDs from a bigBed file. */
{
/* Figure out bigBed file name and open it. Get contents for first chromosome as an example. */
struct slName *idList = NULL;
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chromList = bbiChromList(bbi);
struct lm *lm = lmInit(0);
int orderedCount = count * 4;
if (orderedCount < 100)
orderedCount = 100;
struct bigBedInterval *iv, *ivList = getNElements(bbi, chromList, lm, orderedCount);
shuffleList(&ivList);
// Make a list of item names from intervals.
int outCount = 0;
for (iv = ivList; iv != NULL && outCount < count; iv = iv->next)
{
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
bigBedIntervalToRow(iv, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
if (idList == NULL || differentString(row[3], idList->name))
{
slAddHead(&idList, slNameNew(row[3]));
outCount++;
}
}
lmCleanup(&lm);
bbiFileClose(&bbi);
freeMem(fileName);
return idList;
}
void bigPslToPsl(char *bigPslName, char *outputName)
/* bigPslToPsl - convert bigPsl file to psle. */
{
struct bbiFile *bbi = bigBedFileOpen(bigPslName);
struct lm *lm = lmInit(0);
FILE *f = mustOpen(outputName, "w");
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
int start = 0, end = chrom->size;
int itemsLeft = 0;
char *chromName = chrom->name;
struct bigBedInterval *bbList = bigBedIntervalQuery(bbi, chromName,
start, end, itemsLeft, lm);
for(; bbList; bbList = bbList->next)
{
struct psl *psl, *pslList = pslFromBigPsl(chromName, bbList, NULL, NULL);
for(psl=pslList; psl; psl = psl->next)
{
if (collapseStrand && (psl->strand[1] == '+'))
psl->strand[1] = 0;
pslTabOut(psl, f);
}
}
}
}
void joinTwoInfo(char *spec1, char *spec2)
/* joinTwoInfo - Look at two columns in two tables in mySQL and see how joinable they look.. */
{
char *s1[4], *s2[4];
struct lm *lm = lmInit(0);
int partCount = chopByChar(lmCloneString(lm, spec1), '.', s1, ArraySize(s1));
if (partCount != 3)
usage();
partCount = chopByChar(lmCloneString(lm, spec2), '.', s2, ArraySize(s2));
if (partCount != 3)
usage();
struct slName *list1 = getColumn(s1[0], s1[1], s1[2], lm);
struct hash *uniq1 = uniqHash(list1);
struct slName *list2 = getColumn(s2[0], s2[1], s2[2], lm);
struct hash *uniq2 = uniqHash(list2);
int countOneInTwo = countInHash(list1, uniq2);
int countTwoInOne = countInHash(list2, uniq1);
int countUniqOneInTwo = countUniqInHash(list1, uniq2);
int countUniqTwoInOne = countUniqInHash(list2, uniq1);
printf("%s: %d items, %d unique items, %d items (%d unique) in %s\n",
spec1, slCount(list1), uniq1->elCount, countOneInTwo, countUniqOneInTwo, spec2);
printf("%s: %d items, %d unique items, %d items (%d unique) in %s\n",
spec2, slCount(list2), uniq2->elCount, countTwoInOne, countUniqTwoInOne, spec1);
lmCleanup(&lm);
}
static void addFilteredBedsOnRegion(char *fileName, struct region *region,
char *table, struct asFilter *filter, struct lm *bedLm, struct bed **pBedList,
struct hash *idHash, int *pMaxOut)
/* Add relevant beds in reverse order to pBedList */
{
struct lm *lm = lmInit(0);
struct samAlignment *sam, *samList = bamFetchSamAlignment(fileName, region->chrom,
region->start, region->end, lm);
char *row[SAMALIGNMENT_NUM_COLS];
char numBuf[BAM_NUM_BUF_SIZE];
for (sam = samList; sam != NULL; sam = sam->next)
{
samAlignmentToRow(sam, numBuf, row);
if (asFilterOnRow(filter, row))
{
if ((idHash != NULL) && (hashLookup(idHash, sam->qName) == NULL))
continue;
struct bed *bed;
lmAllocVar(bedLm, bed);
bed->chrom = lmCloneString(bedLm, sam->rName);
bed->chromStart = sam->pos - 1;
bed->chromEnd = bed->chromStart + cigarWidth(sam->cigar, strlen(sam->cigar));
bed->name = lmCloneString(bedLm, sam->qName);
slAddHead(pBedList, bed);
}
(*pMaxOut)--;
if (*pMaxOut <= 0)
break;
}
lmCleanup(&lm);
}
INLINE void agTrimToStart(struct annoGrator *self, char *chrom, uint start)
/* If queue contains items whose end is to the left of start, splice them out. */
{
struct annoRow *qRow, *prevQRow = NULL, *nextQRow;
for (qRow = self->qHead; qRow != NULL; qRow = nextQRow)
{
nextQRow = qRow->next;
int cDifRowP = strcmp(qRow->chrom, chrom);
if (cDifRowP > 0 || (cDifRowP == 0 && qRow->start >= start))
break;
else if (cDifRowP < 0 || qRow->end < start)
{
if (prevQRow == NULL)
self->qHead = qRow->next;
else
prevQRow->next = qRow->next;
if (self->qTail == qRow)
self->qTail = prevQRow;
}
else
prevQRow = qRow;
}
if (self->qHead == NULL)
{
// Queue is empty - clean up lm
lmCleanup(&(self->qLm));
self->qLm = lmInit(0);
self->qSkippedCount = 0;
}
}
void bigWigFillDataVector(char *table, struct region *region,
struct sqlConnection *conn, struct dataVector *vector)
/* Fill in data vector with bigWig info on region. Handles filters and intersections. */
{
/* Figure out filter values if any. */
double ll, ul;
enum wigCompare cmp;
getWigFilter(database, curTable, &cmp, &ll, &ul);
/* Get intervals that pass filter and intersection. */
struct lm *lm = lmInit(0);
char *fileName = bigWigFileName(table, conn);
struct bbiFile *bwf = bigWigFileOpen(fileName);
struct bbiInterval *iv, *ivList;
ivList = intersectedFilteredBbiIntervalsOnRegion(conn, bwf, region, cmp, ll, ul, lm);
int vIndex = 0;
for (iv = ivList; iv != NULL; iv = iv->next)
{
int start = max(iv->start, region->start);
int end = min(iv->end, region->end);
double val = iv->val;
int i;
for (i=start; i<end && vIndex < vector->maxCount; ++i)
{
vector->value[vIndex] = val;
vector->position[vIndex] = i;
++vIndex;
}
}
vector->count = vIndex;
bbiFileClose(&bwf);
freeMem(fileName);
lmCleanup(&lm);
}
SEXP BWGSectionList_add(SEXP r_sections, SEXP r_seq, SEXP r_ranges,
SEXP r_score, SEXP r_format)
{
struct bwgSection *sections = NULL;
const char *seq = CHAR(asChar(r_seq));
double *score = REAL(r_score);
const char *format = CHAR(asChar(r_format));
SEXP ans;
struct lm *lm;
enum bwgSectionType type = bwgTypeBedGraph;
if (sameString(format, "fixedStep"))
type = bwgTypeFixedStep;
else if (sameString(format, "variableStep"))
type = bwgTypeVariableStep;
if (r_sections != R_NilValue) {
sections = R_ExternalPtrAddr(r_sections);
lm = R_ExternalPtrAddr(R_ExternalPtrTag(r_sections));
} else lm = lmInit(0);
pushRHandlers();
if (r_ranges != R_NilValue) {
BWGSectionList_addRle(§ions, seq, r_ranges, score, type, lm);
} else {
BWGSectionList_addAtomic(§ions, seq, score, length(r_score), lm);
}
popRHandlers();
PROTECT(ans = R_MakeExternalPtr(sections, R_NilValue, R_NilValue));
R_SetExternalPtrTag(ans, R_MakeExternalPtr(lm, R_NilValue, R_NilValue));
UNPROTECT(1);
return ans;
}
void wordChain(char *inFile, int maxSize)
/* wordChain - Create Markov chain of words and optionally output chain in two formats. */
{
struct lm *lm = lmInit(0);
struct wordTree *wt = wordTreeForChainsInFile(inFile, maxSize, lm);
if (optionExists("chain"))
{
char *fileName = optionVal("chain", NULL);
FILE *f = mustOpen(fileName, "w");
wordTreeDump(0, wt, f);
carefulClose(&f);
}
if (optionExists("nonsense"))
{
char *fileName = optionVal("nonsense", NULL);
FILE *f = mustOpen(fileName, "w");
int maxSize = min(wt->useCount, maxNonsenseSize);
wordTreeMakeNonsense(wt, maxChainSize, pickRandomWord(wt->following), maxSize, f);
carefulClose(&f);
}
lmCleanup(&lm); // Not really needed since we're just going to exit.
}
static void snapSoftToCloseHard(struct rbTree *vertexTree, struct rbTree *edgeTree, int maxSnapSize,
int maxUncheckedSnapSize, struct nibTwoCache *seqCache, char *chromName)
/* Snap hard vertices to nearby soft vertices of same type. */
{
struct lm *lm = lmInit(0);
addWaysInAndOut(vertexTree, edgeTree, lm);
struct dlList *vList = sortedListFromTree(vertexTree);
struct dlNode *node;
int snapCount = 0;
for (node = vList->head; !dlEnd(node); node = node->next)
{
if (snapVertex(node, maxSnapSize, maxUncheckedSnapSize, seqCache, chromName))
{
rbTreeRemove(vertexTree, node->val);
++snapCount;
}
}
/* Clean up ways in and out since have removed some nodes. */
for (node = vList->head; !dlEnd(node); node = node->next)
{
struct vertex *v = node->val;
v->waysIn = v->waysOut = NULL;
}
if (snapCount > 0)
{
verbose(3, "Snapped %d close edges, now have %d vertices\n", snapCount, vertexTree->n);
updateForwardedEdges(edgeTree);
}
dlListFree(&vList);
lmCleanup(&lm);
}
void doGenePredPal(struct sqlConnection *conn)
/* Output genePred protein alignment. */
{
if (doGalaxy() && !cgiOptionalString(hgtaDoGalaxyQuery))
{
sendParamsToGalaxy(hgtaDoPalOut, "submit");
return;
}
/* get rid of pesky cookies that would bring us back here */
cartRemove(cart, hgtaDoPal);
cartRemove(cart, hgtaDoPalOut);
if (anyIntersection() && intersectionIsBpWise())
errAbort("Can't do CDS FASTA output when bit-wise intersection is on. "
"Please go back and select another output type, or clear the intersection.");
checkNoGenomeDisabled(database, curTable);
struct lm *lm = lmInit(64*1024);
int fieldCount;
struct bed *bedList = cookedBedsOnRegions(conn, curTable, getRegions(),
lm, &fieldCount);
//lmCleanup(&lm);
textOpen();
int outCount = palOutPredsInBeds(conn, cart, bedList, curTable);
/* Do some error diagnostics for user. */
if (outCount == 0)
explainWhyNoResults(NULL);
}
void tagStormQueryMain(char *query)
/* tagStormQuery - Find stanzas in tag storm based on SQL-like query.. */
{
/* Get parsed out query */
struct lineFile *lf = lineFileOnString("query", TRUE, cloneString(query));
struct rqlStatement *rql = rqlStatementParse(lf);
int stormCount = slCount(rql->tableList);
if (stormCount != 1)
errAbort("Can only handle one tag storm file in query, got %d", stormCount);
char *tagsFileName = rql->tableList->name;
/* Read in tags */
struct tagStorm *tags = tagStormFromFile(tagsFileName);
/* Expand any field names with wildcards. */
struct slName *allFieldList = tagStormFieldList(tags);
rql->fieldList = wildExpandList(allFieldList, rql->fieldList, TRUE);
/* Traverse tree applying query */
struct lm *lm = lmInit(0);
doSelect = sameWord(rql->command, "select");
traverse(tags, tags->forest, rql, lm);
tagStormFree(&tags);
if (sameWord(rql->command, "count"))
printf("%d\n", matchCount);
}
static void loadCytoBandsIdeo(struct track *tg)
/* Load up cytoBandIdeo from database table to track items. */
{
if (tg->isBigBed)
{
struct lm *lm = lmInit(0);
int start = 0;
int end = hChromSize(database, chromName);
struct bigBedInterval *bb, *bbList = bigBedSelectRange(tg, chromName, start, end, lm);
char *bedRow[32];
char startBuf[16], endBuf[16];
for (bb = bbList; bb != NULL; bb = bb->next)
{
bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow));
struct cytoBand *bed = cytoBandLoad(bedRow);
slAddHead(&tg->items, bed);
}
slReverse(&tg->items);
lmCleanup(&lm);
return;
}
char query[256];
sqlSafef(query, sizeof(query),
"select * from cytoBandIdeo where chrom like '%s'", chromName);
if(hTableExists(database, "cytoBandIdeo"))
bedLoadItemByQuery(tg, "cytoBandIdeo", query, (ItemLoader)cytoBandLoad);
if(slCount(tg->items) == 0)
{
tg->limitedVisSet = TRUE;
tg->limitedVis = tvHide;
}
}
void cirTreeFileBulkIndexToOpenFile(
void *itemArray, int itemSize, bits64 itemCount,
bits32 blockSize, bits32 itemsPerSlot,
void *context,
struct cirTreeRange (*fetchKey)(const void *va, void *context),
bits64 (*fetchOffset)(const void *va, void *context),
bits64 endFileOffset, FILE *f)
/* Create a r tree index from a sorted array, writing output starting at current position
* of an already open file. See rTreeFileCreate for explanation of parameters. */
{
int levelCount;
struct lm *lm = lmInit(0);
struct rTree *tree = rTreeFromChromRangeArray(lm, blockSize, itemsPerSlot,
itemArray, itemSize, itemCount, context, fetchKey, fetchOffset, endFileOffset,
&levelCount);
bits32 magic = cirTreeSig;
bits32 reserved = 0;
writeOne(f, magic);
writeOne(f, blockSize);
writeOne(f, itemCount);
writeOne(f, tree->startChromIx);
writeOne(f, tree->startBase);
writeOne(f, tree->endChromIx);
writeOne(f, tree->endBase);
writeOne(f, endFileOffset);
writeOne(f, itemsPerSlot);
writeOne(f, reserved);
writeTreeToOpenFile(tree, blockSize, levelCount, f);
lmCleanup(&lm);
}
void bigWigFileCreate(
char *inName, /* Input file in ascii wiggle format. */
char *chromSizes, /* Two column tab-separated file: <chromosome> <size>. */
int blockSize, /* Number of items to bundle in r-tree. 1024 is good. */
int itemsPerSlot, /* Number of items in lowest level of tree. 512 is good. */
boolean clipDontDie, /* If TRUE then clip items off end of chrom rather than dying. */
boolean compress, /* If TRUE then compress data. */
boolean keepAllChromosomes, /* If TRUE then store all chromosomes in chromosomal b-tree. */
boolean fixedSummaries, /* If TRUE then impose fixed summary levels. */
char *outName)
/* Convert ascii format wig file (in fixedStep, variableStep or bedGraph format)
* to binary big wig format. */
{
/* This code needs to agree with code in two other places currently - bigBedFileCreate,
* and bbiFileOpen. I'm thinking of refactoring to share at least between
* bigBedFileCreate and bigWigFileCreate. It'd be great so it could be structured
* so that it could send the input in one chromosome at a time, and send in the zoom
* stuff only after all the chromosomes are done. This'd potentially reduce the memory
* footprint by a factor of 2 or 4. Still, for now it works. -JK */
struct hash *chromSizeHash = bbiChromSizesFromFile(chromSizes);
struct lm *lm = lmInit(0);
struct bwgSection *sectionList = bwgParseWig(inName, clipDontDie, chromSizeHash, itemsPerSlot, lm);
if (sectionList == NULL)
errAbort("%s is empty of data", inName);
bwgCreate(sectionList, chromSizeHash, blockSize, itemsPerSlot, compress, keepAllChromosomes, fixedSummaries, outName);
lmCleanup(&lm);
}
struct slName *randomBamIds(char *table, struct sqlConnection *conn, int count)
/* Return some semi-random qName based IDs from a BAM file. */
{
/* Read 10000 items from bam file, or if they ask for a big list, then 4x what they ask for. */
char *fileName = bamFileName(table, conn, NULL);
samfile_t *fh = bamOpen(fileName, NULL);
struct lm *lm = lmInit(0);
int orderedCount = count * 4;
if (orderedCount < 10000)
orderedCount = 10000;
struct samAlignment *sam, *samList = bamReadNextSamAlignments(fh, orderedCount, lm);
/* Shuffle list and extract qNames from first count of them. */
shuffleList(&samList);
struct slName *randomIdList = NULL;
int i;
for (i=0, sam = samList; i<count && sam != NULL; ++i, sam = sam->next)
slNameAddHead(&randomIdList, sam->qName);
/* Clean up and go home. */
lmCleanup(&lm);
bamClose(&fh);
freez(&fileName);
return randomIdList;
}
Bits *bitsForIntersectingTable(struct sqlConnection *conn, struct region *region, int chromSize,
boolean isBpWise)
/* Get a bitmap that corresponds to the table we are intersecting with.
* Consult CGI vars to figure out what table it is. */
{
boolean invTable2 = cartCgiUsualBoolean(cart, hgtaInvertTable2, FALSE);
char *table2 = cartString(cart, hgtaIntersectTable);
struct hTableInfo *hti2 = getHti(database, table2, conn);
struct lm *lm2 = lmInit(64*1024);
Bits *bits2 = bitAlloc(chromSize+8);
struct bed *bedList2;
if (isBigWigTable(table2))
bedList2 = bigWigIntervalsToBed(conn, table2, region, lm2);
else
// We should go straight to raw beds here, not through the routines that
// do filter & intersections, because the secondary table has no filter
// and sure shouldn't be intersected. :)
bedList2 = getFilteredBeds(conn, table2, region, lm2, NULL);
if (!isBpWise)
expandZeroSize(bedList2, hti2->hasBlocks, chromSize);
bedOrBits(bits2, chromSize, bedList2, hti2->hasBlocks, 0);
if (invTable2)
bitNot(bits2, chromSize);
lmCleanup(&lm2);
return bits2;
}
long metaBigNumItems(struct metaBig* mb, boolean verbose)
/* return the total number of items in a bigBed or BAM */
/* used on a bigWig will return 0 */
/* unfortunately this is a loop through the entire file basically. */
/* nicer would be something that just glances at the index, but doing that */
/* might count items that would be filtered out upon fetching. */
{
long sum = 0;
struct bed* section;
struct bed* chroms = NULL;
if (mb->type == isaBigWig)
return 0;
else if (mb->type == isaBigBed)
return (long)bigBedItemCount(mb->big.bbi);
else
chroms = sectionsFromChromSizes(mb->chromSizeHash);
for (section = chroms; section != NULL; section = section->next) {
struct lm* lm = lmInit(0);
struct bed6* list = metaBigBed6Fetch(mb, section->chrom, section->chromStart, section->chromEnd, lm);
int num = slCount(list);
if (verbose)
printf("Number of items in %s of %s: %d\n", section->chrom, mb->fileName, num);
sum += num;
lmCleanup(&lm);
}
bedFreeList(&chroms);
return sum;
}
boolean gffOpen(struct gff *gff, char *fileName)
/* Initialize gff structure and open file for it. */
{
dnaUtilOpen();
/* Initialize structure and open file. */
zeroBytes(gff, sizeof(*gff));
gff->memPool = lmInit(16*1024);
gff->fileSize = fileSize(fileName);
if (gff->fileSize < 0 ||
(gff->file = fopen(fileName, "rb")) == NULL)
{
warn("Couldn't find the file named %s\n", fileName);
return FALSE;
}
strcpy(gff->fileName, fileName);
gff->bufSize = ArraySize(gff->buf);
/* Make sure it's a gff file. */
_gffSeekDoubleSharpLine(gff);
if (strncmp(gff->buf, _gffIdent, strlen(_gffIdent)) != 0)
{
warn("%s doesn't appear to be a .gff file\n", fileName);
return FALSE;
}
return TRUE;
}
void bitmapToMaskArray(struct hash *bitmapHash, struct hash *tbHash)
/* Translate each bitmap in bitmapHash into an array of mask coordinates
* in the corresponding twoBit in tbHash. Assume tbHash's mask array is
* empty at the start -- we allocate it here. Free bitmap when done. */
{
struct hashCookie cookie = hashFirst(tbHash);
struct hashEl *hel = NULL;
while ((hel = hashNext(&cookie)) != NULL)
{
char *seqName = hel->name;
struct twoBit *tb = (struct twoBit *)(hel->val);
struct hashEl *bHel = hashLookup(bitmapHash, seqName);
Bits *bits;
unsigned start=0, end=0;
assert(tb != NULL);
assert(tb->maskBlockCount == 0);
if (bHel == NULL)
errAbort("Missing bitmap for seq \"%s\"", seqName);
bits = (Bits *)bHel->val;
if (bits != NULL)
{
struct lm *lm = lmInit(0);
struct unsignedRange *rangeList = NULL, *range = NULL;
int i;
for (;;)
{
start = bitFindSet(bits, end, tb->size);
if (start >= tb->size)
break;
end = bitFindClear(bits, start, tb->size);
if (end > start)
{
lmAllocVar(lm, range);
range->start = start;
range->size = (end - start);
slAddHead(&rangeList, range);
}
}
slReverse(&rangeList);
tb->maskBlockCount = slCount(rangeList);
if (tb->maskBlockCount > 0)
{
AllocArray(tb->maskStarts, tb->maskBlockCount);
AllocArray(tb->maskSizes, tb->maskBlockCount);
for (i = 0, range = rangeList; range != NULL;
i++, range = range->next)
{
tb->maskStarts[i] = range->start;
tb->maskSizes[i] = range->size;
}
}
lmCleanup(&lm);
bitFree(&bits);
bHel->val = NULL;
}
}
}
static void rowBufInit(struct rowBuf *rowBuf, int size)
/* Clean up rowBuf and give it a new lm and buffer[size]. */
{
resetRowBuf(rowBuf);
rowBuf->lm = lmInit(0);
rowBuf->size = size;
lmAllocArray(rowBuf->lm, rowBuf->buf, size);
}
void vcfFileMakeReusePool(struct vcfFile *vcff, int initialSize)
// Creates a separate memory pool for records. Establishing this pool allows
// using vcfFileFlushRecords to abandon previously read records and free
// the associated memory. Very useful when reading an entire file in batches.
{
assert(vcff->reusePool == NULL); // don't duplicate this
vcff->reusePool = lmInit(initialSize);
}
void intersectOnChrom(char *db, struct sqlConnection *conn, char *chrom,
char *track1, char *track2)
/* Do intersection on one chromosome. */
{
int chromSize = hChromSize(chrom);
struct lm *lm = lmInit(0);
struct bed *bedList1, *bedList2, *andBed;
struct featureBits *fb1, *fb2;
Bits *bit1, *bit2;
int fieldCount1, fieldCount2;
struct binKeeper *bk2;
uglyTime(NULL);
scanChromTable(conn, chrom, track1);
scanChromTable(conn, chrom, track2);
uglyTime("Scan tracks");
bedList1 = getChromAsBed(conn, db, track1, chrom, lm, &fieldCount1);
bedList2 = getChromAsBed(conn, db, track2, chrom, lm, &fieldCount2);
uglyTime("Tracks as bed");
uglyf("%d items with %d fields in %s, ", slCount(bedList1), fieldCount1, track1);
uglyf("%d items with %d fields in %s\n", slCount(bedList2), fieldCount2, track2);
bit1 = bitAlloc(chromSize+8);
bit2 = bitAlloc(chromSize+8);
uglyTime("bitAlloc");
fb1 = fbList(db, chrom, track1, bedList1, chromSize);
fb2 = fbList(db, chrom, track1, bedList1, chromSize);
uglyTime("bed to featureBits list");
fbOrBits(bit1, chromSize, fb1, 0);
fbOrBits(bit2, chromSize, fb2, 0);
uglyTime("or into bits");
bitAnd(bit1, bit2, chromSize);
uglyTime("Anding bitfields");
andBed = bitsToBed4List(bit1, chromSize, chrom, 0, 0, chromSize, lm);
uglyTime("Converting bitfield to bed 4");
bitCountAllOverlaps(bedList1, bit2, fieldCount2);
uglyTime("Counting overlaps in track1 with bitfield of track2");
bk2 = fbToBinKeeper(fb2, chromSize);
uglyTime("Adding featureBits list from track 2 into binKeeper.");
bkCountAllOverlaps(bedList1, bk2, fieldCount2);
uglyTime("Count overlaps in track1 with binKeeper of track2");
featureBitsFreeList(&fb1);
featureBitsFreeList(&fb2);
uglyTime("free featureBits");
bitFree(&bit1);
bitFree(&bit2);
uglyTime("bitFree");
}
struct tagStorm *tagStormNew(char *name)
/* Create a new, empty, tagStorm. */
{
struct lm *lm = lmInit(0);
struct tagStorm *tagStorm = lmAlloc(lm, sizeof(*tagStorm));
tagStorm->lm = lm;
tagStorm->fileName = lmCloneString(lm, name);
return tagStorm;
}
static void addBbCorrelations(struct bbiChromInfo *chrom, struct genomeRangeTree *targetGrt,
struct bbiFile *aBbi, struct bbiFile *bBbi,
int numColIx, struct correlate *c, struct correlate *cInEnriched,
long long *aTotalSpan, long long *bTotalSpan, long long *overlapTotalSpan)
/* Find bits of a and b that overlap and also overlap with targetRanges. Try to extract
* some number from the bed (which number depends on format). Returns total number of
* overlapping bases between the two big-beds. */
{
struct lm *lm = lmInit(0);
struct rbTree *targetRanges = NULL;
if (targetGrt != NULL)
targetRanges = genomeRangeTreeFindRangeTree(targetGrt, chrom->name);
struct bigBedInterval *a, *aList = bigBedIntervalQuery(aBbi, chrom->name, 0, chrom->size, 0, lm);
struct bigBedInterval *b, *bList = bigBedIntervalQuery(bBbi, chrom->name, 0, chrom->size, 0, lm);
long long totalOverlap = 0;
/* This is a slightly complex but useful loop for two sorted lists that will get overlaps between
* the two in linear time. */
a = aList;
b = bList;
for (;;)
{
if (a == NULL || b == NULL)
break;
int s = max(a->start,b->start);
int e = min(a->end,b->end);
int overlap = e - s;
if (overlap > 0)
{
totalOverlap += overlap;
/* Do correlation over a/b overlap */
double aVal = getDoubleValAt(a->rest, numColIx);
double bVal = getDoubleValAt(b->rest, numColIx);
correlateNextMulti(c, aVal, bVal, overlap);
/* Got intersection of a and b - is it also in targetRange? */
if (targetRanges)
{
int targetOverlap = rangeTreeOverlapSize(targetRanges, s, e);
if (targetOverlap > 0)
{
correlateNextMulti(cInEnriched, aVal, bVal, targetOverlap);
}
}
}
if (a->end < b->end)
a = a->next;
else
b = b->next;
}
*overlapTotalSpan += totalOverlap;
*aTotalSpan += bbIntervalListTotalSpan(aList);
*bTotalSpan += bbIntervalListTotalSpan(bList);
lmCleanup(&lm);
}
void callGraph(char *fileName)
/* callGraph - Make C function call graph. */
{
struct lm *lm = lmInit(0);
struct token *tokenList;
tokenList = token = readAllTokens(fileName, lm);
printf("digraph g {\n");
parseFunctions();
printf("}\n");
}
void searchOneProt(aaSeq *seq, struct genoFind *gf, FILE *f)
/* Search for protein seq in index and write results to psl. */
{
int hitCount;
struct lm *lm = lmInit(0);
struct gfClump *clumpList = gfFindClumps(gf, seq, lm, &hitCount);
gfAlignAaClumps(gf, clumpList, seq, FALSE, minScore, gvo);
gfClumpFreeList(&clumpList);
lmCleanup(&lm);
}