char *printBigBedViewInfo(FILE *f, char *indent, struct view *view,
struct composite *comp, struct taggedFile *tfList)
/* Print out info for a bigBed view. */
{
/* Get defined fields and total fields, and make sure they are the same for everyone. */
int defFields = 0, fields = 0;
struct taggedFile *tf, *bigBedTf = NULL;
for (tf = tfList; tf != NULL; tf = tf->next)
{
if (sameString(view->name, tf->manifest->outputType))
{
struct bbiFile *bbi = bigBedFileOpen(tf->manifest->fileName);
if (defFields == 0)
{
fields = bbi->fieldCount;
defFields = bbi->definedFieldCount;
bigBedTf = tf;
}
else
{
if (fields != bbi->fieldCount || defFields != bbi->definedFieldCount)
errAbort("Different formats for bigBeds in %s vs %s", bigBedTf->manifest->fileName,
tf->manifest->fileName);
}
bigBedFileClose(&bbi);
}
}
char type[32];
safef(type, sizeof(type), "bigBed %d%s", defFields, (fields > defFields ? " +" : ""));
fprintf(f, "%stype %s\n", indent, type);
return cloneString(type);
}
void metaBigClose(struct metaBig** pMb)
/* close the file and free up everything. */
{
struct metaBig* mb = *pMb;
hashFree(&mb->chromSizeHash);
if (mb->rgList)
hashFree(&mb->rgList);
if (mb->sections)
bedFreeList(&mb->sections);
if (mb->originalFileName)
freeMem(mb->originalFileName);
if (mb->fileName)
freeMem(mb->fileName);
if (mb->baseFileName)
freeMem(mb->baseFileName);
if (mb->remoteSiteAndDir)
freeMem(mb->remoteSiteAndDir);
#ifdef USE_HTSLIB
if (mb->idx)
hts_idx_destroy(mb->idx);
#endif
if (mb->type == isaBigBed)
bigBedFileClose(&mb->big.bbi);
#ifdef USE_HTSLIB
else if (mb->type == isaBam)
sam_close(mb->big.bam);
#endif
else
bigWigFileClose(&mb->big.bbi);
#ifdef USE_HTSLIB
if (mb->header)
bam_hdr_destroy(mb->header);
#endif
freez(pMb);
}
struct asObject *bigBedAsFromFileName(char *fileName)
/* Look up bigBed filename in table and get its internally stored autoSql definition. */
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct asObject *asObj = bigBedAs(bbi);
bigBedFileClose(&bbi);
return asObj;
}
struct genomeRangeTree *edwGrtFromBigBed(char *fileName)
/* Return genome range tree for simple (unblocked) bed */
{
struct bbiFile *bbi = bigBedFileOpen(fileName);
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
struct genomeRangeTree *grt = genomeRangeTreeNew();
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
struct rbTree *tree = genomeRangeTreeFindOrAddRangeTree(grt, chrom->name);
struct lm *lm = lmInit(0);
struct bigBedInterval *iv, *ivList = NULL;
ivList = bigBedIntervalQuery(bbi, chrom->name, 0, chrom->size, 0, lm);
for (iv = ivList; iv != NULL; iv = iv->next)
rangeTreeAdd(tree, iv->start, iv->end);
lmCleanup(&lm);
}
bigBedFileClose(&bbi);
bbiChromInfoFreeList(&chromList);
return grt;
}
void doBigBedReplicate(struct sqlConnection *conn, char *format, struct edwAssembly *assembly,
struct edwFile *elderEf, struct edwValidFile *elderVf,
struct edwFile *youngerEf, struct edwValidFile *youngerVf)
/* Do correlation analysis between elder and younger and save result to
* a new edwQaPairCorrelation record. Do this for a format where we have a bigBed file. */
{
/* If got both pairs, work is done already */
if (pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairSampleOverlap")
&& pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairCorrelation"))
return;
int numColIx = 0;
if (sameString(format, "narrowPeak") || sameString(format, "broadPeak"))
numColIx = 6; // signalVal
else
numColIx = 4; // score
numColIx -= 3; // Subtract off chrom/start/end
char *enrichedIn = elderVf->enrichedIn;
struct genomeRangeTree *targetGrt = NULL;
if (!isEmpty(enrichedIn) && !sameString(enrichedIn, "unknown"))
targetGrt = genomeRangeTreeForTarget(conn, assembly, enrichedIn);
/* Get open big bed files for both younger and older. */
char *elderPath = edwPathForFileId(conn, elderEf->id);
char *youngerPath = edwPathForFileId(conn, youngerEf->id);
struct bbiFile *elderBbi = bigBedFileOpen(elderPath);
struct bbiFile *youngerBbi = bigBedFileOpen(youngerPath);
/* Loop through a chromosome at a time adding to correlation, and at the end save result in r.*/
struct correlate *c = correlateNew(), *cInEnriched = correlateNew();
struct bbiChromInfo *chrom, *chromList = bbiChromList(elderBbi);
long long elderTotalSpan = 0, youngerTotalSpan = 0, overlapTotalSpan = 0;
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
addBbCorrelations(chrom, targetGrt, elderBbi, youngerBbi, numColIx, c, cInEnriched,
&elderTotalSpan, &youngerTotalSpan, &overlapTotalSpan);
}
/* Make up correlation structure and save. */
if (!pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairCorrelation"))
{
struct edwQaPairCorrelation *cor;
AllocVar(cor);
cor->elderFileId = elderVf->fileId;
cor->youngerFileId = youngerVf->fileId;
cor->pearsonOverall = correlateResult(c);
cor->pearsonInEnriched = correlateResult(cInEnriched);
edwQaPairCorrelationSaveToDb(conn, cor, "edwQaPairCorrelation", 128);
freez(&cor);
}
/* Also make up sample structure and save. */
if (!pairExists(conn, elderEf->id, youngerEf->id, "edwQaPairSampleOverlap"))
{
struct edwQaPairSampleOverlap *sam;
AllocVar(sam);
sam->elderFileId = elderVf->fileId;
sam->youngerFileId = youngerVf->fileId;
sam->elderSampleBases = elderTotalSpan;
sam->youngerSampleBases = youngerTotalSpan;
sam->sampleOverlapBases = overlapTotalSpan;
setSampleSampleEnrichment(sam, format, assembly, elderVf, youngerVf);
edwQaPairSampleOverlapSaveToDb(conn, sam, "edwQaPairSampleOverlap", 128);
freez(&sam);
}
genomeRangeTreeFree(&targetGrt);
correlateFree(&c);
bigBedFileClose(&youngerBbi);
bigBedFileClose(&elderBbi);
freez(&youngerPath);
freez(&elderPath);
}
void doEnrichmentsFromBigBed(struct sqlConnection *conn,
struct cdwFile *ef, struct cdwValidFile *vf,
struct cdwAssembly *assembly, struct target *targetList)
/* Figure out enrichments from a bigBed file. */
{
/* Get path to bigBed, open it, and read all chromosomes. */
char *bigBedPath = cdwPathForFileId(conn, ef->id);
struct bbiFile *bbi = bigBedFileOpen(bigBedPath);
struct bbiChromInfo *chrom, *chromList = bbiChromList(bbi);
/* Do a pretty complex loop that just aims to set target->overlapBases and ->uniqOverlapBases
* for all targets. This is complicated by just wanting to keep one chromosome worth of
* bigBed data in memory. */
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
/* Get list of intervals in bigBed for this chromosome, and feed it to a rangeTree. */
struct lm *lm = lmInit(0);
struct bigBedInterval *ivList = bigBedIntervalQuery(bbi, chrom->name, 0, chrom->size, 0, lm);
struct bigBedInterval *iv;
struct rbTree *bbTree = rangeTreeNew();
for (iv = ivList; iv != NULL; iv = iv->next)
rangeTreeAdd(bbTree, iv->start, iv->end);
struct range *bbRange, *bbRangeList = rangeTreeList(bbTree);
/* Loop through all targets adding overlaps from ivList and unique overlaps from bbRangeList */
struct target *target;
for (target = targetList; target != NULL; target = target->next)
{
if (target->skip)
continue;
struct genomeRangeTree *grt = target->grt;
struct rbTree *targetTree = genomeRangeTreeFindRangeTree(grt, chrom->name);
if (targetTree != NULL)
{
struct bigBedInterval *iv;
for (iv = ivList; iv != NULL; iv = iv->next)
{
int overlap = rangeTreeOverlapSize(targetTree, iv->start, iv->end);
target->overlapBases += overlap;
}
for (bbRange = bbRangeList; bbRange != NULL; bbRange = bbRange->next)
{
int overlap = rangeTreeOverlapSize(targetTree, bbRange->start, bbRange->end);
target->uniqOverlapBases += overlap;
}
}
}
rangeTreeFree(&bbTree);
lmCleanup(&lm);
}
/* Now loop through targets and save enrichment info to database */
struct target *target;
for (target = targetList; target != NULL; target = target->next)
{
if (target->skip)
continue;
struct cdwQaEnrich *enrich = enrichFromOverlaps(ef, vf, assembly, target,
target->overlapBases, target->uniqOverlapBases);
cdwQaEnrichSaveToDb(conn, enrich, "cdwQaEnrich", 128);
cdwQaEnrichFree(&enrich);
}
bbiChromInfoFreeList(&chromList);
bigBedFileClose(&bbi);
freez(&bigBedPath);
}