struct slName *randomBigBedIds(char *table, struct sqlConnection *conn, int count)
/* Return some arbitrary IDs from a bigBed file. */
{
/* Figure out bigBed file name and open it. Get contents for first chromosome as an example. */
struct slName *idList = NULL;
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chromList = bbiChromList(bbi);
struct lm *lm = lmInit(0);
int orderedCount = count * 4;
if (orderedCount < 100)
orderedCount = 100;
struct bigBedInterval *iv, *ivList = getNElements(bbi, chromList, lm, orderedCount);
shuffleList(&ivList);
// Make a list of item names from intervals.
int outCount = 0;
for (iv = ivList; iv != NULL && outCount < count; iv = iv->next)
{
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
bigBedIntervalToRow(iv, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
if (idList == NULL || differentString(row[3], idList->name))
{
slAddHead(&idList, slNameNew(row[3]));
outCount++;
}
}
lmCleanup(&lm);
bbiFileClose(&bbi);
freeMem(fileName);
return idList;
}
static void loadCytoBandsIdeo(struct track *tg)
/* Load up cytoBandIdeo from database table to track items. */
{
if (tg->isBigBed)
{
struct lm *lm = lmInit(0);
int start = 0;
int end = hChromSize(database, chromName);
struct bigBedInterval *bb, *bbList = bigBedSelectRange(tg, chromName, start, end, lm);
char *bedRow[32];
char startBuf[16], endBuf[16];
for (bb = bbList; bb != NULL; bb = bb->next)
{
bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow));
struct cytoBand *bed = cytoBandLoad(bedRow);
slAddHead(&tg->items, bed);
}
slReverse(&tg->items);
lmCleanup(&lm);
return;
}
char query[256];
sqlSafef(query, sizeof(query),
"select * from cytoBandIdeo where chrom like '%s'", chromName);
if(hTableExists(database, "cytoBandIdeo"))
bedLoadItemByQuery(tg, "cytoBandIdeo", query, (ItemLoader)cytoBandLoad);
if(slCount(tg->items) == 0)
{
tg->limitedVisSet = TRUE;
tg->limitedVis = tvHide;
}
}
void bigBedAddLinkedFeaturesFrom(struct track *track,
char *chrom, int start, int end, int scoreMin, int scoreMax, boolean useItemRgb,
int fieldCount, struct linkedFeatures **pLfList)
/* Read in items in chrom:start-end from bigBed file named in track->bbiFileName, convert
* them to linkedFeatures, and add to head of list. */
{
struct lm *lm = lmInit(0);
struct trackDb *tdb = track->tdb;
struct bigBedInterval *bb, *bbList = bigBedSelectRange(track, chrom, start, end, lm);
char *bedRow[32];
char startBuf[16], endBuf[16];
char *scoreFilter = cartOrTdbString(cart, track->tdb, "scoreFilter", NULL);
char *mouseOverField = cartOrTdbString(cart, track->tdb, "mouseOverField", NULL);
int minScore = 0;
if (scoreFilter)
minScore = atoi(scoreFilter);
int mouseOverIdx = bbExtraFieldIndex(tdb, mouseOverField);
for (bb = bbList; bb != NULL; bb = bb->next)
{
char* mouseOver = restField(bb, mouseOverIdx);
bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow));
struct bed *bed = bedLoadN(bedRow, fieldCount);
struct linkedFeatures *lf = bedMungToLinkedFeatures(&bed, tdb, fieldCount,
scoreMin, scoreMax, useItemRgb);
if (scoreFilter == NULL || lf->score >= minScore)
slAddHead(pLfList, lf);
lf->mouseOver = mouseOver; // leaks some memory, cloneString handles NULL ifself
if (sameString(track->tdb->type, "bigGenePred"))
lf->original = genePredFromBigGenePred(chromName, bb);
}
lmCleanup(&lm);
}
static struct bed6* bed6FromBigBedInterval(struct bigBedInterval* bbi, struct metaBigBed6Helper helper)
/* convert a bigBedInterval into something a bit more userful */
{
struct bed6* bed;
struct lm* lm = helper.lm;
struct metaBig* mb = helper.mb;
int start = bbi->start;
int end = bbi->end;
char* words[32];
char sBuf[16], eBuf[16];
int numFields = bigBedIntervalToRow(bbi, helper.chrom, sBuf, eBuf, words, 32);
char strand = 0;
if (numFields < 6)
errAbort("Expecting six fields in the bigBed and there are %d", numFields);
/* decide whether to add this or not now, because if so, more lm memory is to be alloc'd */
strand = words[5][0];
if ((mb->strand != 0) && (mb->strand != strand))
return NULL;
if (strand == '+') {
start += mb->shift;
if (mb->length > 0)
end = start + mb->length;
start += mb->shift;
end += mb->shift;
} else {
if (mb->length > 0)
start = end - mb->length;
;
start -= mb->shift;
end -= mb->shift;
}
/* filter out ones that go out-of-bounds after shifting/extending */
if ((start >= end) || (start < 0) || (end > helper.chromSize))
return NULL;
lmAllocVar(lm, bed);
bed->chrom = helper.chrom;
bed->chromStart = start;
bed->chromEnd = end;
if (mb->nameType == basicName)
bed->name = lmCloneString(lm, words[3]);
else
bed->name = helper.dot;
bed->score = sqlUnsigned(words[4]);
bed->strand[0] = strand;
return bed;
}
int bigBedIntervalToRowLookupChrom(struct bigBedInterval *interval,
struct bigBedInterval *prevInterval, struct bbiFile *bbi,
char *chromBuf, int chromBufSize, char *startBuf, char *endBuf, char **row, int rowSize)
/* Convert bigBedInterval to array of chars equivalend to what you'd get by parsing the
* bed file. If you already know what chromosome the interval is on use the simpler
* bigBedIntervalToRow. This one will look up the chromosome based on the chromId field
* of the interval, which is relatively time consuming. To avoid doing this unnecessarily
* pass in a non-NULL prevInterval, and if the chromId is the same on prevInterval as this,
* it will avoid the lookup. The chromBufSize should be at greater or equal to
* bbi->chromBpt->keySize+1. The startBuf and endBuf are used to hold the ascii representation of
* start and end, and should be 16 bytes. Note that the interval->rest string will have zeroes
* inserted as a side effect. Returns number of fields in row. */
{
int lastChromId = (prevInterval == NULL ? -1 : prevInterval->chromId);
bbiCachedChromLookup(bbi, interval->chromId, lastChromId, chromBuf, chromBufSize);
return bigBedIntervalToRow(interval, chromBuf, startBuf, endBuf, row, rowSize);
}
static void lrgLoadItems(struct track *tg)
/* Load LRGs in range, translate to linkedFeatures and store as tg->items. */
{
struct lm *lm = lmInit(0);
struct bigBedInterval *bb, *bbList = bigBedSelectRange(tg, chromName, winStart, winEnd, lm);
for (bb = bbList; bb != NULL; bb = bb->next)
{
char *lrgRow[LRG_NUM_COLS];
char startBuf[16], endBuf[16];
int bbFieldCount = bigBedIntervalToRow(bb, chromName, startBuf, endBuf, lrgRow,
ArraySize(lrgRow));
if (bbFieldCount != LRG_NUM_COLS)
errAbort("lrgLoadItems: expected %d columns for row has %d", LRG_NUM_COLS, bbFieldCount);
struct lrg *lrg = lrgLoad(lrgRow);
slAddHead(&(tg->items), lrgToLf(lrg));
}
slReverse(&(tg->items));
lmCleanup(&lm);
}
static void addFilteredBedsOnRegion(struct bbiFile *bbi, struct region *region,
char *table, struct asFilter *filter, struct lm *bedLm, struct bed **pBedList)
/* Add relevant beds in reverse order to pBedList */
{
struct lm *bbLm = lmInit(0);
struct bigBedInterval *ivList = NULL, *iv;
ivList = bigBedIntervalQuery(bbi, region->chrom, region->start, region->end, 0, bbLm);
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
for (iv = ivList; iv != NULL; iv = iv->next)
{
bigBedIntervalToRow(iv, region->chrom, startBuf, endBuf, row, bbi->fieldCount);
if (asFilterOnRow(filter, row))
{
struct bed *bed = bedLoadN(row, bbi->definedFieldCount);
struct bed *lmBed = lmCloneBed(bed, bedLm);
slAddHead(pBedList, lmBed);
bedFree(&bed);
}
}
lmCleanup(&bbLm);
}
void showSchemaBigBed(char *table, struct trackDb *tdb)
/* Show schema on bigBed. */
{
/* Figure out bigBed file name and open it. Get contents for first chromosome as an example. */
struct sqlConnection *conn = NULL;
if (!trackHubDatabase(database))
conn = hAllocConn(database);
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chromList = bbiChromList(bbi);
struct lm *lm = lmInit(0);
struct bigBedInterval *ivList = getNElements(bbi, chromList, lm, 10);
/* Get description of columns, making it up from BED records if need be. */
struct asObject *as = bigBedAsOrDefault(bbi);
hPrintf("<B>Database:</B> %s", database);
hPrintf(" <B>Primary Table:</B> %s<br>", table);
hPrintf("<B>Big Bed File:</B> %s", fileName);
if (bbi->version >= 2)
{
hPrintf("<BR><B>Item Count:</B> ");
printLongWithCommas(stdout, bigBedItemCount(bbi));
}
hPrintf("<BR>\n");
hPrintf("<B>Format description:</B> %s<BR>", as->comment);
/* Put up table that describes fields. */
hTableStart();
hPrintf("<TR><TH>field</TH>");
if (ivList != NULL)
hPrintf("<TH>example</TH>");
hPrintf("<TH>description</TH> ");
puts("</TR>\n");
struct asColumn *col;
int colCount = 0;
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
if (ivList != NULL)
{
char *dupeRest = lmCloneString(lm, ivList->rest); /* Manage rest-stomping side-effect */
bigBedIntervalToRow(ivList, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
ivList->rest = dupeRest;
}
for (col = as->columnList; col != NULL; col = col->next)
{
hPrintf("<TR><TD><TT>%s</TT></TD>", col->name);
if (ivList != NULL)
hPrintf("<TD>%s</TD>", row[colCount]);
hPrintf("<TD>%s</TD></TR>", col->comment);
++colCount;
}
/* If more fields than descriptions put up minimally helpful info (at least has example). */
for ( ; colCount < bbi->fieldCount; ++colCount)
{
hPrintf("<TR><TD><TT>column%d</TT></TD>", colCount+1);
if (ivList != NULL)
hPrintf("<TD>%s</TD>", row[colCount]);
hPrintf("<TD>n/a</TD></TR>\n");
}
hTableEnd();
if (ivList != NULL)
{
/* Put up another section with sample rows. */
webNewSection("Sample Rows");
hTableStart();
/* Print field names as column headers for example */
hPrintf("<TR>");
int colIx = 0;
for (col = as->columnList; col != NULL; col = col->next)
{
hPrintf("<TH>%s</TH>", col->name);
++colIx;
}
for (; colIx < colCount; ++colIx)
hPrintf("<TH>column%d</TH>", colIx+1);
hPrintf("</TR>\n");
/* Print sample lines. */
struct bigBedInterval *iv;
for (iv=ivList; iv != NULL; iv = iv->next)
{
bigBedIntervalToRow(iv, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
hPrintf("<TR>");
for (colIx=0; colIx<colCount; ++colIx)
{
writeHtmlCell(row[colIx]);
}
hPrintf("</TR>\n");
}
hTableEnd();
}
printTrackHtml(tdb);
/* Clean up and go home. */
lmCleanup(&lm);
bbiFileClose(&bbi);
freeMem(fileName);
hFreeConn(&conn);
}
void bigBedTabOut(char *db, char *table, struct sqlConnection *conn, char *fields, FILE *f)
/* Print out selected fields from Big Bed. If fields is NULL, then print out all fields. */
{
if (f == NULL)
f = stdout;
/* Convert comma separated list of fields to array. */
int fieldCount = chopByChar(fields, ',', NULL, 0);
char **fieldArray;
AllocArray(fieldArray, fieldCount);
chopByChar(fields, ',', fieldArray, fieldCount);
/* Get list of all fields in big bed and turn it into a hash of column indexes keyed by
* column name. */
struct hash *fieldHash = hashNew(0);
struct slName *bb, *bbList = bigBedGetFields(table, conn);
int i;
for (bb = bbList, i=0; bb != NULL; bb = bb->next, ++i)
hashAddInt(fieldHash, bb->name, i);
// If bigBed has name column, look up pasted/uploaded identifiers if any:
struct hash *idHash = NULL;
if (slCount(bbList) >= 4)
idHash = identifierHash(db, table);
/* Create an array of column indexes corresponding to the selected field list. */
int *columnArray;
AllocArray(columnArray, fieldCount);
for (i=0; i<fieldCount; ++i)
{
columnArray[i] = hashIntVal(fieldHash, fieldArray[i]);
}
/* Output row of labels */
fprintf(f, "#%s", fieldArray[0]);
for (i=1; i<fieldCount; ++i)
fprintf(f, "\t%s", fieldArray[i]);
fprintf(f, "\n");
/* Open up bigBed file. */
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
struct asFilter *filter = NULL;
if (anyFilter())
{
filter = asFilterFromCart(cart, db, table, as);
if (filter)
{
fprintf(f, "# Filtering on %d columns\n", slCount(filter->columnList));
}
}
/* Loop through outputting each region */
struct region *region, *regionList = getRegions();
for (region = regionList; region != NULL; region = region->next)
{
struct lm *lm = lmInit(0);
struct bigBedInterval *iv, *ivList = bigBedIntervalQuery(bbi, region->chrom,
region->start, region->end, 0, lm);
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
for (iv = ivList; iv != NULL; iv = iv->next)
{
bigBedIntervalToRow(iv, region->chrom, startBuf, endBuf, row, bbi->fieldCount);
if (asFilterOnRow(filter, row))
{
if ((idHash != NULL) && (hashLookup(idHash, row[3]) == NULL))
continue;
int i;
fprintf(f, "%s", row[columnArray[0]]);
for (i=1; i<fieldCount; ++i)
fprintf(f, "\t%s", row[columnArray[i]]);
fprintf(f, "\n");
}
}
lmCleanup(&lm);
}
/* Clean up and exit. */
bbiFileClose(&bbi);
hashFree(&fieldHash);
freeMem(fieldArray);
freeMem(columnArray);
}
static void bigBedClick(char *fileName, struct trackDb *tdb,
char *item, int start, int end, int bedSize)
/* Handle click in generic bigBed track. */
{
boolean showUrl = FALSE;
char *chrom = cartString(cart, "c");
/* Open BigWig file and get interval list. */
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct lm *lm = lmInit(0);
int ivStart = start, ivEnd = end;
if (start == end)
{
// item is an insertion; expand the search range from 0 bases to 2 so we catch it:
ivStart = max(0, start-1);
ivEnd++;
}
struct bigBedInterval *bbList = bigBedIntervalQuery(bbi, chrom, ivStart, ivEnd, 0, lm);
/* Get bedSize if it's not already defined. */
if (bedSize == 0)
{
bedSize = bbi->definedFieldCount;
showUrl = TRUE;
}
char *scoreFilter = cartOrTdbString(cart, tdb, "scoreFilter", NULL);
int minScore = 0;
if (scoreFilter)
minScore = atoi(scoreFilter);
/* Find particular item in list - matching start, and item if possible. */
boolean found = FALSE;
boolean firstTime = TRUE;
struct bigBedInterval *bb;
for (bb = bbList; bb != NULL; bb = bb->next)
{
if (!(bb->start == start && bb->end == end))
continue;
if (bedSize > 3)
{
char *name = cloneFirstWordByTab(bb->rest);
boolean match = sameString(name, item);
freez(&name);
if (!match)
continue;
}
found = TRUE;
if (firstTime)
printf("<BR>\n");
int seq1Seq2Fields = 0;
// check for seq1 and seq2 in columns 7+8 (eg, pairedTagAlign)
boolean seq1Seq2 = sameOk(trackDbSetting(tdb, BASE_COLOR_USE_SEQUENCE), "seq1Seq2");
if (seq1Seq2 && bedSize == 6)
seq1Seq2Fields = 2;
char *fields[bedSize+seq1Seq2Fields];
char startBuf[16], endBuf[16];
char *rest = cloneString(bb->rest);
int bbFieldCount = bigBedIntervalToRow(bb, chrom, startBuf, endBuf, fields,
bedSize+seq1Seq2Fields);
if (bbFieldCount != bedSize+seq1Seq2Fields)
{
errAbort("Disagreement between trackDb field count (%d) and %s fieldCount (%d)",
bedSize, fileName, bbFieldCount);
}
struct bed *bed = bedLoadN(fields, bedSize);
if (bedSize >= 6 && scoreFilter && bed->score < minScore)
continue;
if (showUrl && (bedSize >= 4))
printCustomUrl(tdb, item, TRUE);
bedPrintPos(bed, bedSize, tdb);
// display seq1 and seq2
if (seq1Seq2 && bedSize+seq1Seq2Fields == 8)
printf("<table><tr><th>Sequence 1</th><th>Sequence 2</th></tr>"
"<tr><td> %s </td><td> %s </td></tr></table>", fields[6], fields[7]);
else if (isNotEmpty(rest))
{
char *restFields[256];
int restCount = chopTabs(rest, restFields);
int restBedFields = bedSize - 3;
if (restCount > restBedFields)
{
if (0 == extraFieldsPrint(tdb,NULL,restFields + restBedFields,restCount - restBedFields))
{
int i;
char label[20];
safef(label, sizeof(label), "nonBedFieldsLabel");
printf("<B>%s </B>",
trackDbSettingOrDefault(tdb, label, "Non-BED fields:"));
for (i = restBedFields; i < restCount; i++)
printf("%s%s", (i > 0 ? "\t" : ""), restFields[i]);
printf("<BR>\n");
}
}
}
if (isCustomTrack(tdb->track))
{
time_t timep = bbiUpdateTime(bbi);
printBbiUpdateTime(&timep);
}
}
if (!found)
{
printf("No item %s starting at %d\n", emptyForNull(item), start);
}
lmCleanup(&lm);
bbiFileClose(&bbi);
}
void loadSimpleBed(struct track *tg)
/* Load the items in one track - just move beds in
* window... */
{
struct bed *(*loader)(char **row);
struct bed *bed, *list = NULL;
char **row;
int rowOffset;
char *words[3];
int wordCt;
char query[128];
char *setting = NULL;
bool doScoreCtFilter = FALSE;
int scoreFilterCt = 0;
char *topTable = NULL;
if (tg->bedSize <= 3)
loader = bedLoad3;
else if (tg->bedSize == 4)
loader = bedLoad;
else if (tg->bedSize == 5)
loader = bedLoad5;
else
loader = bedLoad6;
// pairedTagAlign loader is required for base coloring using sequence from seq1 & seq2
// after removing optional bin column, this loader assumes seq1 and seq2 are in
// row[6] and row[7] respectively of the sql result.
if ((setting = trackDbSetting(tg->tdb, BASE_COLOR_USE_SEQUENCE))
&& sameString(setting, "seq1Seq2"))
loader = bedLoadPairedTagAlign;
/* limit to a specified count of top scoring items.
* If this is selected, it overrides selecting item by specified score */
if ((setting = trackDbSettingClosestToHome(tg->tdb, "filterTopScorers")) != NULL)
{
wordCt = chopLine(cloneString(setting), words);
if (wordCt >= 3)
{
doScoreCtFilter = cartUsualBooleanClosestToHome(cart, tg->tdb, FALSE,
"filterTopScorersOn",sameString(words[0], "on"));
scoreFilterCt = cartUsualIntClosestToHome(cart, tg->tdb, FALSE,
"filterTopScorersCt", atoi(words[1]));
topTable = words[2];
/* if there are not too many rows in the table then can define */
/* top table as the track or subtrack table */
if (sameWord(topTable, "self"))
topTable = cloneString(tg->table);
}
}
/* Get list of items */
if (tg->isBigBed)
{ // avoid opening an unneeded db connection for bigBed; required not to use mysql for parallel fetch tracks
char *scoreFilter = cartOrTdbString(cart, tg->tdb, "scoreFilter", NULL);
struct lm *lm = lmInit(0);
struct bigBedInterval *bb, *bbList = bigBedSelectRange(tg, chromName, winStart, winEnd, lm);
char *bedRow[32];
char startBuf[16], endBuf[16];
int minScore = 0;
if (scoreFilter)
minScore = atoi(scoreFilter);
for (bb = bbList; bb != NULL; bb = bb->next)
{
bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow));
bed = loader(bedRow);
if (scoreFilter == NULL || bed->score >= minScore)
slAddHead(&list, bed);
}
lmCleanup(&lm);
}
else
{
struct sqlConnection *conn = hAllocConnTrack(database, tg->tdb);
struct sqlResult *sr = NULL;
/* limit to items above a specified score */
char *scoreFilterClause = getScoreFilterClause(cart, tg->tdb,NULL);
if (doScoreCtFilter && (topTable != NULL) && hTableExists(database, topTable))
{
sqlSafef(query, sizeof(query),"select * from %s order by score desc limit %d",
topTable, scoreFilterCt);
sr = sqlGetResult(conn, query);
rowOffset = hOffsetPastBin(database, hDefaultChrom(database), topTable);
}
else if(scoreFilterClause != NULL && tg->bedSize >= 5)
{
sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, scoreFilterClause, &rowOffset);
}
else
{
sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, NULL, &rowOffset);
}
freeMem(scoreFilterClause);
while ((row = sqlNextRow(sr)) != NULL)
{
bed = loader(row+rowOffset);
slAddHead(&list, bed);
}
sqlFreeResult(&sr);
hFreeConn(&conn);
}
if (doScoreCtFilter)
{
/* filter out items not in this window */
struct bed *newList = bedFilterListInRange(list, NULL, chromName, winStart, winEnd);
list = newList;
}
slReverse(&list);
tg->items = list;
}
