static void writeBlocks(struct bbiChromUsage *usageList, struct lineFile *lf, struct asObject *as,
int itemsPerSlot, struct bbiBoundsArray *bounds,
int sectionCount, boolean doCompress, FILE *f,
int resTryCount, int resScales[], int resSizes[],
struct bbExIndexMaker *eim, int bedCount,
bits16 fieldCount, bits32 *retMaxBlockSize)
/* Read through lf, writing it in f. Save starting points of blocks (every itemsPerSlot)
* to boundsArray */
{
int maxBlockSize = 0;
struct bbiChromUsage *usage = usageList;
char *line, *row[fieldCount+1];
int lastField = fieldCount-1;
int itemIx = 0, sectionIx = 0;
bits64 blockStartOffset = 0;
int startPos = 0, endPos = 0;
bits32 chromId = 0;
struct dyString *stream = dyStringNew(0);
/* Will keep track of some things that help us determine how much to reduce. */
bits32 resEnds[resTryCount];
int resTry;
for (resTry = 0; resTry < resTryCount; ++resTry)
resEnds[resTry] = 0;
boolean atEnd = FALSE, sameChrom = FALSE;
bits32 start = 0, end = 0;
char *chrom = NULL;
struct bed *bed;
AllocVar(bed);
/* Help keep track of which beds are in current chunk so as to write out
* namedChunks to eim if need be. */
long sectionStartIx = 0, sectionEndIx = 0;
for (;;)
{
/* Get next line of input if any. */
if (lineFileNextReal(lf, &line))
{
/* Chop up line and make sure the word count is right. */
int wordCount;
if (tabSep)
wordCount = chopTabs(line, row);
else
wordCount = chopLine(line, row);
lineFileExpectWords(lf, fieldCount, wordCount);
loadAndValidateBed(row, bedN, fieldCount, lf, bed, as, FALSE);
chrom = bed->chrom;
start = bed->chromStart;
end = bed->chromEnd;
sameChrom = sameString(chrom, usage->name);
}
else /* No next line */
{
atEnd = TRUE;
}
/* Check conditions that would end block and save block info and advance to next if need be. */
if (atEnd || !sameChrom || itemIx >= itemsPerSlot)
{
/* Save stream to file, compressing if need be. */
if (stream->stringSize > maxBlockSize)
maxBlockSize = stream->stringSize;
if (doCompress)
{
size_t maxCompSize = zCompBufSize(stream->stringSize);
// keep around an area of scratch memory
static int compBufSize = 0;
static char *compBuf = NULL;
// check to see if buffer needed for compression is big enough
if (compBufSize < maxCompSize)
{
// free up the old not-big-enough piece
freez(&compBuf); // freez knows bout NULL
// get new scratch area
compBufSize = maxCompSize;
compBuf = needLargeMem(compBufSize);
}
int compSize = zCompress(stream->string, stream->stringSize, compBuf, maxCompSize);
mustWrite(f, compBuf, compSize);
}
else
mustWrite(f, stream->string, stream->stringSize);
dyStringClear(stream);
/* Save block offset and size for all named chunks in this section. */
if (eim != NULL)
{
bits64 blockEndOffset = ftell(f);
bbExIndexMakerAddOffsetSize(eim, blockStartOffset, blockEndOffset-blockStartOffset,
sectionStartIx, sectionEndIx);
sectionStartIx = sectionEndIx;
}
/* Save info on existing block. */
struct bbiBoundsArray *b = &bounds[sectionIx];
b->offset = blockStartOffset;
b->range.chromIx = chromId;
b->range.start = startPos;
b->range.end = endPos;
++sectionIx;
itemIx = 0;
if (atEnd)
break;
}
/* Advance to next chromosome if need be and get chromosome id. */
if (!sameChrom)
{
usage = usage->next;
assert(usage != NULL);
assert(sameString(chrom, usage->name));
for (resTry = 0; resTry < resTryCount; ++resTry)
resEnds[resTry] = 0;
}
chromId = usage->id;
/* At start of block we save a lot of info. */
if (itemIx == 0)
{
blockStartOffset = ftell(f);
startPos = start;
endPos = end;
}
/* Otherwise just update end. */
{
if (endPos < end)
endPos = end;
/* No need to update startPos since list is sorted. */
}
/* Save name into namedOffset if need be. */
if (eim != NULL)
{
bbExIndexMakerAddKeysFromRow(eim, row, sectionEndIx);
sectionEndIx += 1;
}
/* Write out data. */
dyStringWriteOne(stream, chromId);
dyStringWriteOne(stream, start);
dyStringWriteOne(stream, end);
if (fieldCount > 3)
{
int i;
/* Write 3rd through next to last field and a tab separator. */
for (i=3; i<lastField; ++i)
{
char *s = row[i];
dyStringAppend(stream, s);
dyStringAppendC(stream, '\t');
}
/* Write last field and terminal zero */
char *s = row[lastField];
dyStringAppend(stream, s);
}
dyStringAppendC(stream, 0);
itemIx += 1;
/* Do zoom counting. */
for (resTry = 0; resTry < resTryCount; ++resTry)
{
bits32 resEnd = resEnds[resTry];
if (start >= resEnd)
{
resSizes[resTry] += 1;
resEnds[resTry] = resEnd = start + resScales[resTry];
}
while (end > resEnd)
{
resSizes[resTry] += 1;
resEnds[resTry] = resEnd = resEnd + resScales[resTry];
}
}
}
assert(sectionIx == sectionCount);
freez(&bed);
*retMaxBlockSize = maxBlockSize;
}
void writeSections(struct bbiChromUsage *usageList, struct lineFile *lf,
int itemsPerSlot, struct bbiBoundsArray *bounds, int sectionCount, FILE *f,
int resTryCount, int resScales[], int resSizes[],
boolean doCompress, bits32 *retMaxSectionSize)
/* Read through lf, chunking it into sections that get written to f. Save info
* about sections in bounds. */
{
int maxSectionSize = 0;
struct bbiChromUsage *usage = usageList;
int itemIx = 0, sectionIx = 0;
bits32 reserved32 = 0;
UBYTE reserved8 = 0;
struct sectionItem items[itemsPerSlot];
struct sectionItem *lastB = NULL;
bits32 resEnds[resTryCount];
int resTry;
for (resTry = 0; resTry < resTryCount; ++resTry)
resEnds[resTry] = 0;
struct dyString *stream = dyStringNew(0);
/* remove initial browser and track lines */
lineFileRemoveInitialCustomTrackLines(lf);
for (;;)
{
/* Get next line of input if any. */
char *row[5];
int rowSize = lineFileChopNext(lf, row, ArraySize(row));
/* Figure out whether need to output section. */
boolean sameChrom = FALSE;
if (rowSize > 0)
sameChrom = sameString(row[0], usage->name);
if (itemIx >= itemsPerSlot || rowSize == 0 || !sameChrom)
{
/* Figure out section position. */
bits32 chromId = usage->id;
bits32 sectionStart = items[0].start;
bits32 sectionEnd = items[itemIx-1].end;
/* Save section info for indexing. */
assert(sectionIx < sectionCount);
struct bbiBoundsArray *section = &bounds[sectionIx++];
section->offset = ftell(f);
section->range.chromIx = chromId;
section->range.start = sectionStart;
section->range.end = sectionEnd;
/* Output section header to stream. */
dyStringClear(stream);
UBYTE type = bwgTypeBedGraph;
bits16 itemCount = itemIx;
dyStringWriteOne(stream, chromId); // chromId
dyStringWriteOne(stream, sectionStart); // start
dyStringWriteOne(stream, sectionEnd); // end
dyStringWriteOne(stream, reserved32); // itemStep
dyStringWriteOne(stream, reserved32); // itemSpan
dyStringWriteOne(stream, type); // type
dyStringWriteOne(stream, reserved8); // reserved
dyStringWriteOne(stream, itemCount); // itemCount
/* Output each item in section to stream. */
int i;
for (i=0; i<itemIx; ++i)
{
struct sectionItem *item = &items[i];
dyStringWriteOne(stream, item->start);
dyStringWriteOne(stream, item->end);
dyStringWriteOne(stream, item->val);
}
/* Save stream to file, compressing if need be. */
if (stream->stringSize > maxSectionSize)
maxSectionSize = stream->stringSize;
if (doCompress)
{
size_t maxCompSize = zCompBufSize(stream->stringSize);
char compBuf[maxCompSize];
int compSize = zCompress(stream->string, stream->stringSize, compBuf, maxCompSize);
mustWrite(f, compBuf, compSize);
}
else
mustWrite(f, stream->string, stream->stringSize);
/* If at end of input we are done. */
if (rowSize == 0)
break;
/* Set up for next section. */
itemIx = 0;
if (!sameChrom)
{
usage = usage->next;
assert(usage != NULL);
if (!sameString(row[0], usage->name))
errAbort("read %s, expecting %s on line %d in file %s\n",
row[0], usage->name, lf->lineIx, lf->fileName);
assert(sameString(row[0], usage->name));
lastB = NULL;
for (resTry = 0; resTry < resTryCount; ++resTry)
resEnds[resTry] = 0;
}
}
/* Parse out input. */
lineFileExpectWords(lf, 4, rowSize);
bits32 start = lineFileNeedNum(lf, row, 1);
bits32 end = lineFileNeedNum(lf, row, 2);
float val = lineFileNeedDouble(lf, row, 3);
/* Verify that inputs meets our assumption - that it is a sorted bedGraph file. */
if (start > end)
errAbort("Start (%u) after end (%u) line %d of %s", start, end, lf->lineIx, lf->fileName);
if (lastB != NULL)
{
if (lastB->start > start)
errAbort("BedGraph not sorted on start line %d of %s", lf->lineIx, lf->fileName);
if (lastB->end > start)
errAbort("Overlapping regions in bedGraph line %d of %s", lf->lineIx, lf->fileName);
}
/* Do zoom counting. */
for (resTry = 0; resTry < resTryCount; ++resTry)
{
bits32 resEnd = resEnds[resTry];
if (start >= resEnd)
{
resSizes[resTry] += 1;
resEnds[resTry] = resEnd = start + resScales[resTry];
}
while (end > resEnd)
{
resSizes[resTry] += 1;
resEnds[resTry] = resEnd = resEnd + resScales[resTry];
}
}
/* Save values in output array. */
struct sectionItem *b = &items[itemIx];
b->start = start;
b->end = end;
b->val = val;
lastB = b;
itemIx += 1;
}
assert(sectionIx == sectionCount);
*retMaxSectionSize = maxSectionSize;
}
