char *printBigBedViewInfo(FILE *f, char *indent, struct view *view,
struct composite *comp, struct taggedFile *tfList)
/* Print out info for a bigBed view. */
{
/* Get defined fields and total fields, and make sure they are the same for everyone. */
int defFields = 0, fields = 0;
struct taggedFile *tf, *bigBedTf = NULL;
for (tf = tfList; tf != NULL; tf = tf->next)
{
if (sameString(view->name, tf->manifest->outputType))
{
struct bbiFile *bbi = bigBedFileOpen(tf->manifest->fileName);
if (defFields == 0)
{
fields = bbi->fieldCount;
defFields = bbi->definedFieldCount;
bigBedTf = tf;
}
else
{
if (fields != bbi->fieldCount || defFields != bbi->definedFieldCount)
errAbort("Different formats for bigBeds in %s vs %s", bigBedTf->manifest->fileName,
tf->manifest->fileName);
}
bigBedFileClose(&bbi);
}
}
char type[32];
safef(type, sizeof(type), "bigBed %d%s", defFields, (fields > defFields ? " +" : ""));
fprintf(f, "%stype %s\n", indent, type);
return cloneString(type);
}
struct bbiFile *fetchBbiForTrack(struct track *track)
/* Fetch bbiFile from track, opening it if it is not already open. */
{
struct bbiFile *bbi = track->bbiFile;
if (bbi == NULL)
{
char *fileName = NULL;
if (track->parallelLoading) // do not use mysql during parallel fetch
{
fileName = cloneString(trackDbSetting(track->tdb, "bigDataUrl"));
}
else
{
struct sqlConnection *conn = NULL;
if (!trackHubDatabase(database))
conn = hAllocConnTrack(database, track->tdb);
fileName = bbiNameFromSettingOrTable(track->tdb, conn, track->table);
hFreeConn(&conn);
}
#ifdef USE_GBIB_PWD
#include "gbib.c"
#endif
bbi = track->bbiFile = bigBedFileOpen(fileName);
}
return bbi;
}
void bigPslToPsl(char *bigPslName, char *outputName)
/* bigPslToPsl - convert bigPsl file to psle. */
{
struct bbiFile *bbi = bigBedFileOpen(bigPslName);
struct lm *lm = lmInit(0);
FILE *f = mustOpen(outputName, "w");
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
int start = 0, end = chrom->size;
int itemsLeft = 0;
char *chromName = chrom->name;
struct bigBedInterval *bbList = bigBedIntervalQuery(bbi, chromName,
start, end, itemsLeft, lm);
for(; bbList; bbList = bbList->next)
{
struct psl *psl, *pslList = pslFromBigPsl(chromName, bbList, NULL, NULL);
for(psl=pslList; psl; psl = psl->next)
{
if (collapseStrand && (psl->strand[1] == '+'))
psl->strand[1] = 0;
pslTabOut(psl, f);
}
}
}
}
struct bed *bigBedGetFilteredBedsOnRegions(struct sqlConnection *conn,
char *db, char *table, struct region *regionList, struct lm *lm,
int *retFieldCount)
/* Get list of beds from bigBed, in all regions, that pass filtering. */
{
/* Connect to big bed and get metadata and filter. */
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
struct asFilter *filter = asFilterFromCart(cart, db, table, as);
/* Get beds a region at a time. */
struct bed *bedList = NULL;
struct region *region;
for (region = regionList; region != NULL; region = region->next)
addFilteredBedsOnRegion(bbi, region, table, filter, lm, &bedList);
slReverse(&bedList);
/* Clean up and return. */
if (retFieldCount != NULL)
*retFieldCount = bbi->definedFieldCount;
bbiFileClose(&bbi);
freeMem(fileName);
return bedList;
}
struct slName *randomBigBedIds(char *table, struct sqlConnection *conn, int count)
/* Return some arbitrary IDs from a bigBed file. */
{
/* Figure out bigBed file name and open it. Get contents for first chromosome as an example. */
struct slName *idList = NULL;
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chromList = bbiChromList(bbi);
struct lm *lm = lmInit(0);
int orderedCount = count * 4;
if (orderedCount < 100)
orderedCount = 100;
struct bigBedInterval *iv, *ivList = getNElements(bbi, chromList, lm, orderedCount);
shuffleList(&ivList);
// Make a list of item names from intervals.
int outCount = 0;
for (iv = ivList; iv != NULL && outCount < count; iv = iv->next)
{
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
bigBedIntervalToRow(iv, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
if (idList == NULL || differentString(row[3], idList->name))
{
slAddHead(&idList, slNameNew(row[3]));
outCount++;
}
}
lmCleanup(&lm);
bbiFileClose(&bbi);
freeMem(fileName);
return idList;
}
struct metaBig* metaBigOpenWithTmpDir(char* fileOrUrlwSections, char* cacheDir, char* sectionsBed)
/* load a file or URL with or without sectioning */
/* if it's a bam, load the index. */
{
struct metaBig* mb;
char* fullFileName = NULL;
char* remoteDir = NULL;
char* baseFileName = NULL;
char* sections = NULL;
AllocVar(mb);
mb->originalFileName = cloneString(fileOrUrlwSections);
/* first deal with filename and separate URL/file/sections */
mb->isRemote = parseMetaBigFileName(fileOrUrlwSections, &remoteDir, &fullFileName, &baseFileName, §ions);
mb->fileName = fullFileName;
mb->baseFileName = baseFileName;
mb->remoteSiteAndDir = remoteDir;
/* sniff the file */
mb->type = sniffBigFile(mb->fileName);
/* depending on the type, open the files and get the chrom-size hash different ways */
if (mb->type == isaBigBed) {
mb->big.bbi = bigBedFileOpen(mb->fileName);
mb->chromSizeHash = bbiChromSizes(mb->big.bbi);
mb->numReads = bigBedItemCount(mb->big.bbi);
}
#ifdef USE_HTSLIB
else if (mb->type == isaBam) {
mb->chromSizeHash = bamChromSizes(mb->fileName);
mb->header = bamGetHeaderOnly(mb->fileName);
mb->big.bam = sam_open(mb->fileName, "r");
/* Also need to load the index since it's a bam */
mb->idx = bam_index_load(mb->fileName);
metaBigBamFlagCountsInit(mb);
}
#endif
else if (mb->type == isaBigWig) {
mb->big.bbi = bigWigFileOpenWithDir(mb->fileName, cacheDir);
mb->chromSizeHash = bbiChromSizes(mb->big.bbi);
} else {
/* maybe I should free some stuff up here */
if (fullFileName)
freeMem(fullFileName);
if (remoteDir)
freeMem(remoteDir);
if (baseFileName)
freeMem(baseFileName);
if (sections)
freeMem(sections);
freez(&mb);
return NULL;
}
if (sectionsBed && sections) {
struct bed* regions = (fileExists(sectionsBed)) ? regionsLoad(sectionsBed) : parseSectionString(sectionsBed, mb->chromSizeHash);
struct bed* subsets = subset_beds(sections, ®ions, mb->chromSizeHash);
mb->sections = subsets;
} else if (sectionsBed) {
mb->sections = (fileExists(sectionsBed)) ? regionsLoad(sectionsBed) : parseSectionString(sectionsBed, mb->chromSizeHash);
} else
mb->sections = parseSectionString(sections, mb->chromSizeHash);
return mb;
}
void bigBedSummary(char *fileName, char *chrom, int start, int end, int dataPoints)
/* bigBedSummary - Extract summary information from a bigBed file.. */
{
/* Make up values array initialized to not-a-number. */
double nan0 = strtod("NaN", NULL);
double summaryValues[dataPoints];
int i;
for (i=0; i<dataPoints; ++i)
summaryValues[i] = nan0;
struct bbiFile *bbi = bigBedFileOpen(fileName);
if (bigBedSummaryArray(bbi, chrom, start, end, bbiSummaryTypeFromString(summaryType),
dataPoints, summaryValues))
{
for (i=0; i<dataPoints; ++i)
{
double val = summaryValues[i];
if (i != 0)
printf("\t");
if (isnan(val))
printf("n/a");
else
printf("%g", val);
}
printf("\n");
}
else
{
errAbort("no data in region %s:%d-%d in %s\n", chrom, start, end, fileName);
}
bbiFileClose(&bbi);
}
void printBiggestGap(char *database, struct sqlConnection *conn,
struct slName *chromList, struct hash *chromHash, char *track)
/* Look up track in database, figure out which type it is, call
* appropriate biggest gap finder, and then print result. */
{
struct trackDb *tdb = hTrackInfo(conn, track);
struct hTableInfo *hti = hFindTableInfo(database, chromList->name, tdb->table);
char *typeWord = cloneFirstWord(tdb->type);
boolean isBig = FALSE, isBigBed = FALSE;
struct bbiFile *bbi = NULL;
if (sameString(typeWord, "bigBed"))
{
isBig = TRUE;
isBigBed = TRUE;
bbi = bigBedFileOpen( bbiNameFromSettingOrTable(tdb, conn, tdb->table) );
}
else if (sameString(typeWord, "bigWig"))
{
isBig = TRUE;
bbi = bigWigFileOpen( bbiNameFromSettingOrTable(tdb, conn, tdb->table) );
}
char *biggestChrom = NULL;
int biggestSize = 0, biggestStart = 0, biggestEnd = 0;
struct slName *chrom;
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
if (!allParts && strchr(chrom->name, '_')) // Generally skip weird chroms
continue;
if (female && sameString(chrom->name, "chrY"))
continue;
int chromSize = hashIntVal(chromHash, chrom->name);
struct rbTree *rt = rangeTreeNew();
int start = 0, end = 0, size = 0;
if (isBig)
bigCoverageIntoTree(tdb, bbi, chrom->name, chromSize, rt, isBigBed);
else
tableCoverageIntoTree(hti, tdb, conn, chrom->name, chromSize, rt);
if (rt->n > 0) // Want to keep completely uncovered chromosome uncovered
addGaps(conn, chrom->name, rt);
biggestGapFromRangeTree(rt, chromSize, &start, &end, &size);
if (size > biggestSize)
{
biggestSize = size;
biggestStart = start;
biggestEnd = end;
biggestChrom = chrom->name;
}
rangeTreeFree(&rt);
}
printf("%s\t%s:%d-%d\t", track, biggestChrom, biggestStart+1, biggestEnd);
if (noComma)
printf("%d", biggestSize);
else
printLongWithCommas(stdout, biggestSize);
putchar('\n');
freez(&typeWord);
bbiFileClose(&bbi);
}
struct asObject *bigBedAsFromFileName(char *fileName)
/* Look up bigBed filename in table and get its internally stored autoSql definition. */
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *asObj = bigBedAs(bbi);
bigBedFileClose(&bbi);
return asObj;
}
struct sqlFieldType *bigBedListFieldsAndTypes(char *table, struct sqlConnection *conn)
/* Get fields of bigBed as list of sqlFieldType. */
{
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
struct sqlFieldType *list = sqlFieldTypesFromAs(as);
freeMem(fileName);
bbiFileClose(&bbi);
return list;
}
struct slName *bigBedGetFields(char *table, struct sqlConnection *conn)
/* Get fields of bigBed as simple name list. */
{
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
struct slName *names = asColNames(as);
freeMem(fileName);
bbiFileClose(&bbi);
return names;
}
struct asObject *bigBedFileAsObjOrDefault(char *fileName)
// Get asObject associated with bigBed file, or the default.
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
if (bbi)
{
struct asObject *as = bigBedAsOrDefault(bbi);
bbiFileClose(&bbi);
return as;
}
return NULL;
}
void bigBedFields(char *fileName)
/* Print out info about fields in bed file. */
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
printf("%d bed definition fields, %d total fields\n", bbi->definedFieldCount, bbi->fieldCount);
struct asObject *as = bigBedAs(bbi);
if (as != NULL)
{
struct asColumn *col;
for (col = as->columnList; col != NULL; col = col->next)
{
printf("\t%s\t%s\n", col->name, col->comment);
}
}
else
{
printf("No additional field information included.\n");
}
}
void bigBedToBed(char *inFile, char *outFile)
/* bigBedToBed - Convert from bigBed to ascii bed format.. */
{
struct bbiFile *bbi = bigBedFileOpen(inFile);
FILE *f = mustOpen(outFile, "w");
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
int itemCount = 0;
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
if (clChrom != NULL && !sameString(clChrom, chrom->name))
continue;
char *chromName = chrom->name;
int start = 0, end = chrom->size;
if (clStart > 0)
start = clStart;
if (clEnd > 0)
end = clEnd;
int itemsLeft = 0; // Zero actually means no limit....
if (maxItems != 0)
{
itemsLeft = maxItems - itemCount;
if (itemsLeft <= 0)
break;
}
struct lm *lm = lmInit(0);
struct bigBedInterval *interval, *intervalList = bigBedIntervalQuery(bbi, chromName,
start, end, itemsLeft, lm);
for (interval = intervalList; interval != NULL; interval = interval->next)
{
fprintf(f, "%s\t%u\t%u", chromName, interval->start, interval->end);
char *rest = interval->rest;
if (rest != NULL)
fprintf(f, "\t%s\n", rest);
else
fprintf(f, "\n");
}
lmCleanup(&lm);
}
bbiChromInfoFreeList(&chromList);
carefulClose(&f);
bbiFileClose(&bbi);
}
struct genomeRangeTree *edwGrtFromBigBed(char *fileName)
/* Return genome range tree for simple (unblocked) bed */
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
struct genomeRangeTree *grt = genomeRangeTreeNew();
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
struct rbTree *tree = genomeRangeTreeFindOrAddRangeTree(grt, chrom->name);
struct lm *lm = lmInit(0);
struct bigBedInterval *iv, *ivList = NULL;
ivList = bigBedIntervalQuery(bbi, chrom->name, 0, chrom->size, 0, lm);
for (iv = ivList; iv != NULL; iv = iv->next)
rangeTreeAdd(tree, iv->start, iv->end);
lmCleanup(&lm);
}
bigBedFileClose(&bbi);
bbiChromInfoFreeList(&chromList);
return grt;
}
struct hTableInfo *bigBedToHti(char *table, struct sqlConnection *conn)
/* Get fields of bigBed into hti structure. */
{
/* Get columns in asObject format. */
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
/* Allocate hTableInfo structure and fill in info about bed fields. */
struct hash *colHash = asColumnHash(as);
struct hTableInfo *hti;
AllocVar(hti);
hti->rootName = cloneString(table);
hti->isPos= TRUE;
fillField(colHash, "chrom", hti->chromField);
fillField(colHash, "chromStart", hti->startField);
fillField(colHash, "chromEnd", hti->endField);
fillField(colHash, "name", hti->nameField);
fillField(colHash, "score", hti->scoreField);
fillField(colHash, "strand", hti->strandField);
fillField(colHash, "thickStart", hti->cdsStartField);
fillField(colHash, "thickEnd", hti->cdsEndField);
fillField(colHash, "blockCount", hti->countField);
fillField(colHash, "chromStarts", hti->startsField);
fillField(colHash, "blockSizes", hti->endsSizesField);
hti->hasCDS = (bbi->definedFieldCount >= 8);
hti->hasBlocks = (bbi->definedFieldCount >= 12);
char type[256];
safef(type, sizeof(type), "bed %d %c", bbi->definedFieldCount,
(bbi->definedFieldCount == bbi->fieldCount ? '.' : '+'));
hti->type = cloneString(type);
freeMem(fileName);
hashFree(&colHash);
bbiFileClose(&bbi);
return hti;
}
void showSchemaBigBed(char *table, struct trackDb *tdb)
/* Show schema on bigBed. */
{
/* Figure out bigBed file name and open it. Get contents for first chromosome as an example. */
struct sqlConnection *conn = NULL;
if (!trackHubDatabase(database))
conn = hAllocConn(database);
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chromList = bbiChromList(bbi);
struct lm *lm = lmInit(0);
struct bigBedInterval *ivList = getNElements(bbi, chromList, lm, 10);
/* Get description of columns, making it up from BED records if need be. */
struct asObject *as = bigBedAsOrDefault(bbi);
hPrintf("<B>Database:</B> %s", database);
hPrintf(" <B>Primary Table:</B> %s<br>", table);
hPrintf("<B>Big Bed File:</B> %s", fileName);
if (bbi->version >= 2)
{
hPrintf("<BR><B>Item Count:</B> ");
printLongWithCommas(stdout, bigBedItemCount(bbi));
}
hPrintf("<BR>\n");
hPrintf("<B>Format description:</B> %s<BR>", as->comment);
/* Put up table that describes fields. */
hTableStart();
hPrintf("<TR><TH>field</TH>");
if (ivList != NULL)
hPrintf("<TH>example</TH>");
hPrintf("<TH>description</TH> ");
puts("</TR>\n");
struct asColumn *col;
int colCount = 0;
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
if (ivList != NULL)
{
char *dupeRest = lmCloneString(lm, ivList->rest); /* Manage rest-stomping side-effect */
bigBedIntervalToRow(ivList, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
ivList->rest = dupeRest;
}
for (col = as->columnList; col != NULL; col = col->next)
{
hPrintf("<TR><TD><TT>%s</TT></TD>", col->name);
if (ivList != NULL)
hPrintf("<TD>%s</TD>", row[colCount]);
hPrintf("<TD>%s</TD></TR>", col->comment);
++colCount;
}
/* If more fields than descriptions put up minimally helpful info (at least has example). */
for ( ; colCount < bbi->fieldCount; ++colCount)
{
hPrintf("<TR><TD><TT>column%d</TT></TD>", colCount+1);
if (ivList != NULL)
hPrintf("<TD>%s</TD>", row[colCount]);
hPrintf("<TD>n/a</TD></TR>\n");
}
hTableEnd();
if (ivList != NULL)
{
/* Put up another section with sample rows. */
webNewSection("Sample Rows");
hTableStart();
/* Print field names as column headers for example */
hPrintf("<TR>");
int colIx = 0;
for (col = as->columnList; col != NULL; col = col->next)
{
hPrintf("<TH>%s</TH>", col->name);
++colIx;
}
for (; colIx < colCount; ++colIx)
hPrintf("<TH>column%d</TH>", colIx+1);
hPrintf("</TR>\n");
/* Print sample lines. */
struct bigBedInterval *iv;
for (iv=ivList; iv != NULL; iv = iv->next)
{
bigBedIntervalToRow(iv, chromList->name, startBuf, endBuf, row, bbi->fieldCount);
hPrintf("<TR>");
for (colIx=0; colIx<colCount; ++colIx)
{
writeHtmlCell(row[colIx]);
}
hPrintf("</TR>\n");
}
hTableEnd();
}
printTrackHtml(tdb);
/* Clean up and go home. */
lmCleanup(&lm);
bbiFileClose(&bbi);
freeMem(fileName);
hFreeConn(&conn);
}
static void mafOrAxtClick2(struct sqlConnection *conn, struct sqlConnection *conn2, struct trackDb *tdb, char *axtOtherDb, char *fileName)
/* Display details for MAF or AXT tracks. */
{
hgBotDelay();
if (winEnd - winStart > 30000)
{
printf("Zoom so that window is 30,000 bases or less to see alignments and conservation statistics\n");
}
else
{
struct mafAli *mafList = NULL, *maf, *subList = NULL;
int aliIx = 0, realCount = 0;
char dbChrom[64];
char option[128];
char *capTrack;
struct consWiggle *consWig, *consWiggles;
struct hash *speciesOffHash = NULL;
char *speciesOrder = NULL;
char *speciesTarget = trackDbSetting(tdb, SPECIES_TARGET_VAR);
char buffer[1024];
int useTarg = FALSE;
int useIrowChains = FALSE;
safef(option, sizeof(option), "%s.%s", tdb->track, MAF_CHAIN_VAR);
if (cartCgiUsualBoolean(cart, option, FALSE) &&
trackDbSetting(tdb, "irows") != NULL)
useIrowChains = TRUE;
safef(buffer, sizeof(buffer), "%s.vis",tdb->track);
if (useIrowChains)
{
if (!cartVarExists(cart, buffer) && (speciesTarget != NULL))
useTarg = TRUE;
else
{
char *val;
val = cartUsualString(cart, buffer, "useCheck");
useTarg = sameString("useTarg",val);
}
}
if (sameString(tdb->type, "bigMaf"))
{
char *fileName = trackDbSetting(tdb, "bigDataUrl");
struct bbiFile *bbi = bigBedFileOpen(fileName);
mafList = bigMafLoadInRegion(bbi, seqName, winStart, winEnd);
}
else
mafList = mafOrAxtLoadInRegion2(conn,conn2, tdb, seqName, winStart, winEnd,
axtOtherDb, fileName);
safef(dbChrom, sizeof(dbChrom), "%s.%s", hubConnectSkipHubPrefix(database), seqName);
safef(option, sizeof(option), "%s.speciesOrder", tdb->track);
speciesOrder = cartUsualString(cart, option, NULL);
if (speciesOrder == NULL)
speciesOrder = trackDbSetting(tdb, "speciesOrder");
for (maf = mafList; maf != NULL; maf = maf->next)
{
int mcCount = 0;
struct mafComp *mc;
struct mafAli *subset;
struct mafComp *nextMc;
/* remove empty components and configured off components
* from MAF, and ignore
* the entire MAF if all components are empty
* (solely for gap annotation) */
if (!useTarg)
{
for (mc = maf->components->next; mc != NULL; mc = nextMc)
{
char buf[64];
char *organism;
mafSrcDb(mc->src, buf, sizeof buf);
organism = hOrganism(buf);
if (!organism)
organism = buf;
nextMc = mc->next;
safef(option, sizeof(option), "%s.%s", tdb->track, buf);
if (!cartUsualBoolean(cart, option, TRUE))
{
if (speciesOffHash == NULL)
speciesOffHash = newHash(4);
hashStoreName(speciesOffHash, organism);
}
if (!cartUsualBoolean(cart, option, TRUE))
slRemoveEl(&maf->components, mc);
else
mcCount++;
}
}
if (mcCount == 0)
continue;
if (speciesOrder)
{
int speciesCt;
char *species[2048];
struct mafComp **newOrder, *mcThis;
int i;
mcCount = 0;
speciesCt = chopLine(cloneString(speciesOrder), species);
newOrder = needMem((speciesCt + 1) * sizeof (struct mafComp *));
newOrder[mcCount++] = maf->components;
for (i = 0; i < speciesCt; i++)
{
if ((mcThis = mafMayFindCompSpecies(maf, species[i], '.')) == NULL)
continue;
newOrder[mcCount++] = mcThis;
}
maf->components = NULL;
for (i = 0; i < mcCount; i++)
{
newOrder[i]->next = 0;
slAddHead(&maf->components, newOrder[i]);
}
slReverse(&maf->components);
}
subset = mafSubsetE(maf, dbChrom, winStart, winEnd, TRUE);
if (subset != NULL)
{
/* Reformat MAF if needed so that sequence from current
* database is the first component and on the
* plus strand. */
mafMoveComponentToTop(subset, dbChrom);
if (subset->components->strand == '-')
mafFlipStrand(subset);
subset->score = mafScoreMultiz(subset);
slAddHead(&subList, subset);
++realCount;
}
}
slReverse(&subList);
mafAliFreeList(&mafList);
if (subList != NULL)
{
char *showVarName = "hgc.showMultiBase";
char *showVarVal = cartUsualString(cart, showVarName, "all");
boolean onlyDiff = sameWord(showVarVal, "diff");
#ifdef ADDEXONCAPITAL
char *codeVarName = "hgc.multiCapCoding";
char *codeVarVal = cartUsualString(cart, codeVarName, "coding");
boolean onlyCds = sameWord(codeVarVal, "coding");
#endif
/* add links for conservation score statistics */
consWiggles = wigMafWiggles(database, tdb);
int wigCount = slCount(consWiggles);
if (wigCount == 1)
{
conservationStatsLink(tdb, "Conservation score statistics", consWiggles->table);
}
else if (wigCount > 1)
{
/* multiple wiggles. List all that have been turned on with
* checkboxes */
/* Scan for cart variables -- do any exist, are any turned on ? */
boolean wigSet = FALSE;
boolean wigOn = FALSE;
for (consWig = consWiggles; consWig != NULL;
consWig = consWig->next)
{
char *wigVarSuffix = NULL;
(void)wigMafWiggleVar(tdb->track, consWig, &wigVarSuffix);
if (cartVarExistsAnyLevel(cart, tdb, FALSE, wigVarSuffix))
{
wigSet = TRUE;
if (cartBooleanClosestToHome(cart, tdb, FALSE, wigVarSuffix))
wigOn = TRUE;
}
}
/* If there are no cart vars, turn on the first (default) wig */
if (!wigSet)
{
char *prefix = tdb->track; // use when setting things to the cart
if (tdbIsContainerChild(tdb))
prefix = tdbGetContainer(tdb)->track;
cartSetBoolean(cart, wigMafWiggleVar(prefix, consWiggles, NULL), TRUE);
wigOn = TRUE;
}
if (wigOn)
{
boolean first = TRUE;
for (consWig = consWiggles; consWig != NULL;
consWig = consWig->next)
{
if (first)
{
printf("Conservation score statistics:");
first = FALSE;
}
char *wigVarSuffix = NULL;
(void)wigMafWiggleVar(tdb->track, consWig, &wigVarSuffix);
if (cartUsualBooleanClosestToHome(cart, tdb, FALSE, wigVarSuffix,FALSE))
{
printf(" ");
subChar(consWig->uiLabel, '_', ' ');
conservationStatsLink(tdb,
consWig->uiLabel, consWig->table);
}
}
}
}
puts("</P>\n");
/* no alignment to display when in visibilities where only wiggle is shown */
char *vis = cartOptionalString(cart, tdb->track);
if (vis)
{
enum trackVisibility tv = hTvFromStringNoAbort(vis);
if (tv == tvSquish || tv == tvDense)
return;
}
#ifdef ADDEXONCAPITAL
puts("<FORM ACTION=\"../cgi-bin/hgc\" NAME=\"gpForm\" METHOD=\"GET\">");
cartSaveSession(cart);
cgiContinueHiddenVar("g");
cgiContinueHiddenVar("c");
cgiContinueHiddenVar("i");
printf("Capitalize ");
cgiMakeDropListFull(codeVarName, codeAll, codeAll,
ArraySize(codeAll), codeVarVal, autoSubmit);
printf("exons based on ");
capTrack = genePredDropDown(cart, trackHash,
"gpForm", "hgc.multiCapTrack");
#endif
printf("show ");
cgiMakeDropListFull(showVarName, showAll, showAll,
ArraySize(showAll), showVarVal, autoSubmit);
printf("bases");
printf("<BR>\n");
printf("</FORM>\n");
#ifdef REVERSESTRAND
/* notify if bases are complemented (hgTracks is on reverse strand) */
if (cartCgiUsualBoolean(cart, COMPLEMENT_BASES_VAR, FALSE))
puts("<EM>Alignment displayed on reverse strand</EM><BR>");
#endif
puts("Place cursor over species for alignment detail. Click on 'B' to link to browser ");
puts("for aligned species, click on 'D' to get DNA for aligned species.<BR>");
printf("<TT><PRE>");
/* notify if species removed from alignment */
if (speciesOffHash)
{
char *species;
struct hashCookie hc = hashFirst(speciesOffHash);
puts("<B>Components not displayed:</B> ");
while ((species = hashNextName(&hc)) != NULL)
printf("%s ", species);
puts("<BR>");
}
for (maf = subList; maf != NULL; maf = maf->next)
{
mafLowerCase(maf);
#ifdef ADDEXONCAPITAL
if (capTrack != NULL)
capMafOnTrack(maf, capTrack, onlyCds);
#endif
printf("<B>Alignment block %d of %d in window, %d - %d, %d bps </B>\n",
++aliIx,realCount,maf->components->start + 1,
maf->components->start + maf->components->size, maf->components->size);
mafPrettyOut(stdout, maf, 70,onlyDiff, aliIx);
}
mafAliFreeList(&subList);
}
else
{
printf("No multiple alignment in browser window");
}
printf("</PRE></TT>");
}
}
void bigBedTabOut(char *db, char *table, struct sqlConnection *conn, char *fields, FILE *f)
/* Print out selected fields from Big Bed. If fields is NULL, then print out all fields. */
{
if (f == NULL)
f = stdout;
/* Convert comma separated list of fields to array. */
int fieldCount = chopByChar(fields, ',', NULL, 0);
char **fieldArray;
AllocArray(fieldArray, fieldCount);
chopByChar(fields, ',', fieldArray, fieldCount);
/* Get list of all fields in big bed and turn it into a hash of column indexes keyed by
* column name. */
struct hash *fieldHash = hashNew(0);
struct slName *bb, *bbList = bigBedGetFields(table, conn);
int i;
for (bb = bbList, i=0; bb != NULL; bb = bb->next, ++i)
hashAddInt(fieldHash, bb->name, i);
// If bigBed has name column, look up pasted/uploaded identifiers if any:
struct hash *idHash = NULL;
if (slCount(bbList) >= 4)
idHash = identifierHash(db, table);
/* Create an array of column indexes corresponding to the selected field list. */
int *columnArray;
AllocArray(columnArray, fieldCount);
for (i=0; i<fieldCount; ++i)
{
columnArray[i] = hashIntVal(fieldHash, fieldArray[i]);
}
/* Output row of labels */
fprintf(f, "#%s", fieldArray[0]);
for (i=1; i<fieldCount; ++i)
fprintf(f, "\t%s", fieldArray[i]);
fprintf(f, "\n");
/* Open up bigBed file. */
char *fileName = bigBedFileName(table, conn);
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *as = bigBedAsOrDefault(bbi);
struct asFilter *filter = NULL;
if (anyFilter())
{
filter = asFilterFromCart(cart, db, table, as);
if (filter)
{
fprintf(f, "# Filtering on %d columns\n", slCount(filter->columnList));
}
}
/* Loop through outputting each region */
struct region *region, *regionList = getRegions();
for (region = regionList; region != NULL; region = region->next)
{
struct lm *lm = lmInit(0);
struct bigBedInterval *iv, *ivList = bigBedIntervalQuery(bbi, region->chrom,
region->start, region->end, 0, lm);
char *row[bbi->fieldCount];
char startBuf[16], endBuf[16];
for (iv = ivList; iv != NULL; iv = iv->next)
{
bigBedIntervalToRow(iv, region->chrom, startBuf, endBuf, row, bbi->fieldCount);
if (asFilterOnRow(filter, row))
{
if ((idHash != NULL) && (hashLookup(idHash, row[3]) == NULL))
continue;
int i;
fprintf(f, "%s", row[columnArray[0]]);
for (i=1; i<fieldCount; ++i)
fprintf(f, "\t%s", row[columnArray[i]]);
fprintf(f, "\n");
}
}
lmCleanup(&lm);
}
/* Clean up and exit. */
bbiFileClose(&bbi);
hashFree(&fieldHash);
freeMem(fieldArray);
freeMem(columnArray);
}
void doBigBedReplicate(struct sqlConnection *conn, char *format, struct edwAssembly *assembly,
struct edwFile *elderEf, struct edwValidFile *elderVf,
struct edwFile *youngerEf, struct edwValidFile *youngerVf)
/* Do correlation analysis between elder and younger and save result to
* a new edwQaPairCorrelation record. Do this for a format where we have a bigBed file. */
{
/* If got both pairs, work is done already */
if (pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairSampleOverlap")
&& pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairCorrelation"))
return;
int numColIx = 0;
if (sameString(format, "narrowPeak") || sameString(format, "broadPeak"))
numColIx = 6; // signalVal
else
numColIx = 4; // score
numColIx -= 3; // Subtract off chrom/start/end
char *enrichedIn = elderVf->enrichedIn;
struct genomeRangeTree *targetGrt = NULL;
if (!isEmpty(enrichedIn) && !sameString(enrichedIn, "unknown"))
targetGrt = genomeRangeTreeForTarget(conn, assembly, enrichedIn);
/* Get open big bed files for both younger and older. */
char *elderPath = edwPathForFileId(conn, elderEf->id);
char *youngerPath = edwPathForFileId(conn, youngerEf->id);
struct bbiFile *elderBbi = bigBedFileOpen(elderPath);
struct bbiFile *youngerBbi = bigBedFileOpen(youngerPath);
/* Loop through a chromosome at a time adding to correlation, and at the end save result in r.*/
struct correlate *c = correlateNew(), *cInEnriched = correlateNew();
struct bbiChromInfo *chrom, *chromList = bbiChromList(elderBbi);
long long elderTotalSpan = 0, youngerTotalSpan = 0, overlapTotalSpan = 0;
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
addBbCorrelations(chrom, targetGrt, elderBbi, youngerBbi, numColIx, c, cInEnriched,
&elderTotalSpan, &youngerTotalSpan, &overlapTotalSpan);
}
/* Make up correlation structure and save. */
if (!pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairCorrelation"))
{
struct edwQaPairCorrelation *cor;
AllocVar(cor);
cor->elderFileId = elderVf->fileId;
cor->youngerFileId = youngerVf->fileId;
cor->pearsonOverall = correlateResult(c);
cor->pearsonInEnriched = correlateResult(cInEnriched);
edwQaPairCorrelationSaveToDb(conn, cor, "edwQaPairCorrelation", 128);
freez(&cor);
}
/* Also make up sample structure and save. */
if (!pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairSampleOverlap"))
{
struct edwQaPairSampleOverlap *sam;
AllocVar(sam);
sam->elderFileId = elderVf->fileId;
sam->youngerFileId = youngerVf->fileId;
sam->elderSampleBases = elderTotalSpan;
sam->youngerSampleBases = youngerTotalSpan;
sam->sampleOverlapBases = overlapTotalSpan;
setSampleSampleEnrichment(sam, format, assembly, elderVf, youngerVf);
edwQaPairSampleOverlapSaveToDb(conn, sam, "edwQaPairSampleOverlap", 128);
freez(&sam);
}
genomeRangeTreeFree(&targetGrt);
correlateFree(&c);
bigBedFileClose(&youngerBbi);
bigBedFileClose(&elderBbi);
freez(&youngerPath);
freez(&elderPath);
}
static void bigBedClick(char *fileName, struct trackDb *tdb,
char *item, int start, int end, int bedSize)
/* Handle click in generic bigBed track. */
{
boolean showUrl = FALSE;
char *chrom = cartString(cart, "c");
/* Open BigWig file and get interval list. */
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct lm *lm = lmInit(0);
int ivStart = start, ivEnd = end;
if (start == end)
{
// item is an insertion; expand the search range from 0 bases to 2 so we catch it:
ivStart = max(0, start-1);
ivEnd++;
}
struct bigBedInterval *bbList = bigBedIntervalQuery(bbi, chrom, ivStart, ivEnd, 0, lm);
/* Get bedSize if it's not already defined. */
if (bedSize == 0)
{
bedSize = bbi->definedFieldCount;
showUrl = TRUE;
}
char *scoreFilter = cartOrTdbString(cart, tdb, "scoreFilter", NULL);
int minScore = 0;
if (scoreFilter)
minScore = atoi(scoreFilter);
/* Find particular item in list - matching start, and item if possible. */
boolean found = FALSE;
boolean firstTime = TRUE;
struct bigBedInterval *bb;
for (bb = bbList; bb != NULL; bb = bb->next)
{
if (!(bb->start == start && bb->end == end))
continue;
if (bedSize > 3)
{
char *name = cloneFirstWordByTab(bb->rest);
boolean match = sameString(name, item);
freez(&name);
if (!match)
continue;
}
found = TRUE;
if (firstTime)
printf("<BR>\n");
int seq1Seq2Fields = 0;
// check for seq1 and seq2 in columns 7+8 (eg, pairedTagAlign)
boolean seq1Seq2 = sameOk(trackDbSetting(tdb, BASE_COLOR_USE_SEQUENCE), "seq1Seq2");
if (seq1Seq2 && bedSize == 6)
seq1Seq2Fields = 2;
char *fields[bedSize+seq1Seq2Fields];
char startBuf[16], endBuf[16];
char *rest = cloneString(bb->rest);
int bbFieldCount = bigBedIntervalToRow(bb, chrom, startBuf, endBuf, fields,
bedSize+seq1Seq2Fields);
if (bbFieldCount != bedSize+seq1Seq2Fields)
{
errAbort("Disagreement between trackDb field count (%d) and %s fieldCount (%d)",
bedSize, fileName, bbFieldCount);
}
struct bed *bed = bedLoadN(fields, bedSize);
if (bedSize >= 6 && scoreFilter && bed->score < minScore)
continue;
if (showUrl && (bedSize >= 4))
printCustomUrl(tdb, item, TRUE);
bedPrintPos(bed, bedSize, tdb);
// display seq1 and seq2
if (seq1Seq2 && bedSize+seq1Seq2Fields == 8)
printf("<table><tr><th>Sequence 1</th><th>Sequence 2</th></tr>"
"<tr><td> %s </td><td> %s </td></tr></table>", fields[6], fields[7]);
else if (isNotEmpty(rest))
{
char *restFields[256];
int restCount = chopTabs(rest, restFields);
int restBedFields = bedSize - 3;
if (restCount > restBedFields)
{
if (0 == extraFieldsPrint(tdb,NULL,restFields + restBedFields,restCount - restBedFields))
{
int i;
char label[20];
safef(label, sizeof(label), "nonBedFieldsLabel");
printf("<B>%s </B>",
trackDbSettingOrDefault(tdb, label, "Non-BED fields:"));
for (i = restBedFields; i < restCount; i++)
printf("%s%s", (i > 0 ? "\t" : ""), restFields[i]);
printf("<BR>\n");
}
}
}
if (isCustomTrack(tdb->track))
{
time_t timep = bbiUpdateTime(bbi);
printBbiUpdateTime(&timep);
}
}
if (!found)
{
printf("No item %s starting at %d\n", emptyForNull(item), start);
}
lmCleanup(&lm);
bbiFileClose(&bbi);
}
void doEnrichmentsFromBigBed(struct sqlConnection *conn,
struct cdwFile *ef, struct cdwValidFile *vf,
struct cdwAssembly *assembly, struct target *targetList)
/* Figure out enrichments from a bigBed file. */
{
/* Get path to bigBed, open it, and read all chromosomes. */
char *bigBedPath = cdwPathForFileId(conn, ef->id);
struct bbiFile *bbi = bigBedFileOpen(bigBedPath);
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
/* Do a pretty complex loop that just aims to set target->overlapBases and ->uniqOverlapBases
* for all targets. This is complicated by just wanting to keep one chromosome worth of
* bigBed data in memory. */
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
/* Get list of intervals in bigBed for this chromosome, and feed it to a rangeTree. */
struct lm *lm = lmInit(0);
struct bigBedInterval *ivList = bigBedIntervalQuery(bbi, chrom->name, 0, chrom->size, 0, lm);
struct bigBedInterval *iv;
struct rbTree *bbTree = rangeTreeNew();
for (iv = ivList; iv != NULL; iv = iv->next)
rangeTreeAdd(bbTree, iv->start, iv->end);
struct range *bbRange, *bbRangeList = rangeTreeList(bbTree);
/* Loop through all targets adding overlaps from ivList and unique overlaps from bbRangeList */
struct target *target;
for (target = targetList; target != NULL; target = target->next)
{
if (target->skip)
continue;
struct genomeRangeTree *grt = target->grt;
struct rbTree *targetTree = genomeRangeTreeFindRangeTree(grt, chrom->name);
if (targetTree != NULL)
{
struct bigBedInterval *iv;
for (iv = ivList; iv != NULL; iv = iv->next)
{
int overlap = rangeTreeOverlapSize(targetTree, iv->start, iv->end);
target->overlapBases += overlap;
}
for (bbRange = bbRangeList; bbRange != NULL; bbRange = bbRange->next)
{
int overlap = rangeTreeOverlapSize(targetTree, bbRange->start, bbRange->end);
target->uniqOverlapBases += overlap;
}
}
}
rangeTreeFree(&bbTree);
lmCleanup(&lm);
}
/* Now loop through targets and save enrichment info to database */
struct target *target;
for (target = targetList; target != NULL; target = target->next)
{
if (target->skip)
continue;
struct cdwQaEnrich *enrich = enrichFromOverlaps(ef, vf, assembly, target,
target->overlapBases, target->uniqOverlapBases);
cdwQaEnrichSaveToDb(conn, enrich, "cdwQaEnrich", 128);
cdwQaEnrichFree(&enrich);
}
bbiChromInfoFreeList(&chromList);
bigBedFileClose(&bbi);
freez(&bigBedPath);
}