void bbFileCreate(
char *inName, /* Input file in a tabular bed format <chrom><start><end> + whatever. */
char *chromSizes, /* Two column tab-separated file: <chromosome> <size>. */
int blockSize, /* Number of items to bundle in r-tree. 1024 is good. */
int itemsPerSlot, /* Number of items in lowest level of tree. 64 is good. */
char *asText, /* Field definitions in a string */
struct asObject *as, /* Field definitions parsed out */
boolean doCompress, /* If TRUE then compress data. */
struct slName *extraIndexList, /* List of extra indexes to add */
char *outName) /* BigBed output file name. */
/* Convert tab-separated bed file to binary indexed, zoomed bigBed version. */
{
/* Set up timing measures. */
verboseTimeInit();
struct lineFile *lf = lineFileOpen(inName, TRUE);
bits16 fieldCount = slCount(as->columnList);
bits16 extraIndexCount = slCount(extraIndexList);
struct bbExIndexMaker *eim = NULL;
if (extraIndexList != NULL)
eim = bbExIndexMakerNew(extraIndexList, as);
/* Load in chromosome sizes. */
struct hash *chromSizesHash = NULL;
if (sizesIs2Bit)
chromSizesHash = twoBitChromHash(chromSizes);
else
chromSizesHash = bbiChromSizesFromFile(chromSizes);
verbose(2, "Read %d chromosomes and sizes from %s\n", chromSizesHash->elCount, chromSizes);
/* Do first pass, mostly just scanning file and counting hits per chromosome. */
int minDiff = 0;
double aveSize = 0;
bits64 bedCount = 0;
bits32 uncompressBufSize = 0;
struct bbiChromUsage *usageList = bbiChromUsageFromBedFile(lf, chromSizesHash, eim,
&minDiff, &aveSize, &bedCount, tabSep);
verboseTime(1, "pass1 - making usageList (%d chroms)", slCount(usageList));
verbose(2, "%d chroms in %s. Average span of beds %f\n", slCount(usageList), inName, aveSize);
/* Open output file and write dummy header. */
FILE *f = mustOpen(outName, "wb");
bbiWriteDummyHeader(f);
bbiWriteDummyZooms(f);
/* Write out autoSql string */
bits64 asOffset = ftell(f);
mustWrite(f, asText, strlen(asText) + 1);
verbose(2, "as definition has %d columns\n", fieldCount);
/* Write out dummy total summary. */
struct bbiSummaryElement totalSum;
ZeroVar(&totalSum);
bits64 totalSummaryOffset = ftell(f);
bbiSummaryElementWrite(f, &totalSum);
/* Write out dummy header extension */
bits64 extHeaderOffset = ftell(f);
bits16 extHeaderSize = 64;
repeatCharOut(f, 0, extHeaderSize);
/* Write out extra index stuff if need be. */
bits64 extraIndexListOffset = 0;
bits64 extraIndexListEndOffset = 0;
if (extraIndexList != NULL)
{
extraIndexListOffset = ftell(f);
int extraIndexSize = 16 + 4*1; // Fixed record size 16, plus 1 times field size of 4
repeatCharOut(f, 0, extraIndexSize*extraIndexCount);
extraIndexListEndOffset = ftell(f);
}
/* Write out chromosome/size database. */
bits64 chromTreeOffset = ftell(f);
bbiWriteChromInfo(usageList, blockSize, f);
/* Set up to keep track of possible initial reduction levels. */
int resScales[bbiMaxZoomLevels], resSizes[bbiMaxZoomLevels];
int resTryCount = bbiCalcResScalesAndSizes(aveSize, resScales, resSizes);
/* Write out primary full resolution data in sections, collect stats to use for reductions. */
bits64 dataOffset = ftell(f);
bits32 blockCount = 0;
bits32 maxBlockSize = 0;
struct bbiBoundsArray *boundsArray = NULL;
writeOne(f, bedCount);
if (bedCount > 0)
{
blockCount = bbiCountSectionsNeeded(usageList, itemsPerSlot);
AllocArray(boundsArray, blockCount);
lineFileRewind(lf);
if (eim)
bbExIndexMakerAllocChunkArrays(eim, bedCount);
writeBlocks(usageList, lf, as, itemsPerSlot, boundsArray, blockCount, doCompress,
f, resTryCount, resScales, resSizes, eim, bedCount, fieldCount, &maxBlockSize);
}
verboseTime(1, "pass2 - checking and writing primary data (%lld records, %d fields)",
(long long)bedCount, fieldCount);
/* Write out primary data index. */
bits64 indexOffset = ftell(f);
cirTreeFileBulkIndexToOpenFile(boundsArray, sizeof(boundsArray[0]), blockCount,
blockSize, 1, NULL, bbiBoundsArrayFetchKey, bbiBoundsArrayFetchOffset,
indexOffset, f);
freez(&boundsArray);
verboseTime(2, "index write");
/* Declare arrays and vars that track the zoom levels we actually output. */
bits32 zoomAmounts[bbiMaxZoomLevels];
bits64 zoomDataOffsets[bbiMaxZoomLevels];
bits64 zoomIndexOffsets[bbiMaxZoomLevels];
/* Call monster zoom maker library function that bedGraphToBigWig also uses. */
int zoomLevels = 0;
if (bedCount > 0)
{
zoomLevels = bbiWriteZoomLevels(lf, f, blockSize, itemsPerSlot,
bedWriteReducedOnceReturnReducedTwice, fieldCount,
doCompress, indexOffset - dataOffset,
usageList, resTryCount, resScales, resSizes,
zoomAmounts, zoomDataOffsets, zoomIndexOffsets, &totalSum);
}
/* Write out extra indexes if need be. */
if (eim)
{
int i;
for (i=0; i < eim->indexCount; ++i)
{
eim->fileOffsets[i] = ftell(f);
maxBedNameSize = eim->maxFieldSize[i];
qsort(eim->chunkArrayArray[i], bedCount,
sizeof(struct bbNamedFileChunk), bbNamedFileChunkCmpByName);
assert(sizeof(struct bbNamedFileChunk) == sizeof(eim->chunkArrayArray[i][0]));
bptFileBulkIndexToOpenFile(eim->chunkArrayArray[i], sizeof(eim->chunkArrayArray[i][0]),
bedCount, blockSize, bbNamedFileChunkKey, maxBedNameSize, bbNamedFileChunkVal,
sizeof(bits64) + sizeof(bits64), f);
verboseTime(1, "Sorting and writing extra index %d", i);
}
}
/* Figure out buffer size needed for uncompression if need be. */
if (doCompress)
{
int maxZoomUncompSize = itemsPerSlot * sizeof(struct bbiSummaryOnDisk);
uncompressBufSize = max(maxBlockSize, maxZoomUncompSize);
}
/* Go back and rewrite header. */
rewind(f);
bits32 sig = bigBedSig;
bits16 version = bbiCurrentVersion;
bits16 summaryCount = zoomLevels;
bits32 reserved32 = 0;
bits64 reserved64 = 0;
bits16 definedFieldCount = bedN;
/* Write fixed header */
writeOne(f, sig);
writeOne(f, version);
writeOne(f, summaryCount);
writeOne(f, chromTreeOffset);
writeOne(f, dataOffset);
writeOne(f, indexOffset);
writeOne(f, fieldCount);
writeOne(f, definedFieldCount);
writeOne(f, asOffset);
writeOne(f, totalSummaryOffset);
writeOne(f, uncompressBufSize);
writeOne(f, extHeaderOffset);
assert(ftell(f) == 64);
/* Write summary headers with data. */
int i;
verbose(2, "Writing %d levels of zoom\n", zoomLevels);
for (i=0; i<zoomLevels; ++i)
{
verbose(3, "zoomAmounts[%d] = %d\n", i, (int)zoomAmounts[i]);
writeOne(f, zoomAmounts[i]);
writeOne(f, reserved32);
writeOne(f, zoomDataOffsets[i]);
writeOne(f, zoomIndexOffsets[i]);
}
/* Write rest of summary headers with no data. */
for (i=zoomLevels; i<bbiMaxZoomLevels; ++i)
{
writeOne(f, reserved32);
writeOne(f, reserved32);
writeOne(f, reserved64);
writeOne(f, reserved64);
}
/* Write total summary. */
fseek(f, totalSummaryOffset, SEEK_SET);
bbiSummaryElementWrite(f, &totalSum);
/* Write extended header */
fseek(f, extHeaderOffset, SEEK_SET);
writeOne(f, extHeaderSize);
writeOne(f, extraIndexCount);
writeOne(f, extraIndexListOffset);
repeatCharOut(f, 0, 52); // reserved
assert(ftell(f) - extHeaderOffset == extHeaderSize);
/* Write extra index offsets if need be. */
if (extraIndexCount != 0)
{
fseek(f, extraIndexListOffset, SEEK_SET);
int i;
for (i=0; i<extraIndexCount; ++i)
{
// Write out fixed part of index info
bits16 type = 0; // bPlusTree type
bits16 indexFieldCount = 1;
writeOne(f, type);
writeOne(f, indexFieldCount);
writeOne(f, eim->fileOffsets[i]);
repeatCharOut(f, 0, 4); // reserved
// Write out field list - easy this time because for now always only one field.
bits16 fieldId = eim->indexFields[i];
writeOne(f, fieldId);
repeatCharOut(f, 0, 2); // reserved
}
assert(ftell(f) == extraIndexListEndOffset);
}
/* Write end signature. */
fseek(f, 0L, SEEK_END);
writeOne(f, sig);
/* Clean up. */
lineFileClose(&lf);
carefulClose(&f);
freeHash(&chromSizesHash);
bbiChromUsageFreeList(&usageList);
asObjectFreeList(&as);
}
void loadOneBed(struct lineFile *lf, int bedSize, struct bedStub **pList)
/* Load one bed file. Make sure all lines have the correct number of fields.
* Put results in *pList. */
{
char *words[64], *line, *dupe;
int wordCount;
struct bedStub *bed;
struct asObject *asObj = getAsObj(bedSize);
int fieldCount = getFieldCount(bedSize, asObj);
struct bed *validateBed;
AllocVar(validateBed);
verbose(1, "Reading %s\n", lf->fileName);
while (lineFileNextReal(lf, &line))
{
if (hasBin)
nextWord(&line);
dupe = cloneString(line);
if (strictTab)
wordCount = chopTabs(line, words);
else
wordCount = chopLine(line, words);
/* ignore empty lines */
if (0 == wordCount)
continue;
lineFileExpectWords(lf, fieldCount, wordCount);
if (type)
// TODO also, may need to add a flag to the validateBed() interface to support -allowNegativeScores when not isCt
// although can probably get away without it since usually -allowNegativeScores is used by ct which has already verified it.
// thus -allowNegativeScores is unlikely to be used with -type.
{
loadAndValidateBed(words, typeBedN, fieldCount, lf, validateBed, asObj, FALSE);
checkChromNameAndSize(lf, validateBed->chrom, validateBed->chromEnd);
}
AllocVar(bed);
bed->chrom = cloneString(words[0]);
bed->chromStart = lineFileNeedNum(lf, words, 1);
bed->chromEnd = lineFileNeedNum(lf, words, 2);
if (! noStrict)
{
if ((bed->chromEnd < 1) && !allowStartEqualEnd)
errAbort("ERROR: line %d:'%s'\nchromEnd is less than 1\n",
lf->lineIx, dupe);
if (bed->chromStart == bed->chromEnd && !allowStartEqualEnd)
errAbort("ERROR: line %d:'%s'\nchromStart == chromEnd (%d) (zero-length item)\n"
"Use -allowStartEqualEnd if that is legit (e.g. for insertion point).\n",
lf->lineIx, dupe, bed->chromStart);
if (bed->chromStart > bed->chromEnd)
errAbort("ERROR: line %d:'%s'\nchromStart after chromEnd (%d > %d)\n",
lf->lineIx, dupe, bed->chromStart, bed->chromEnd);
}
bed->line = dupe;
slAddHead(pList, bed);
}
if (asObj)
asObjectFreeList(&asObj);
freez(&validateBed);
}
