static bits64 bbiWriteSummaryAndIndexUnc(struct bbiSummary *summaryList,
int blockSize, int itemsPerSlot, FILE *f)
/* Write out summary and index to summary compressed, returning start position of
* summary index. */
{
bits32 i, count = slCount(summaryList);
struct bbiSummary **summaryArray;
AllocArray(summaryArray, count);
writeOne(f, count);
struct bbiSummary *summary;
for (summary = summaryList, i=0; summary != NULL; summary = summary->next, ++i)
{
summaryArray[i] = summary;
summary->fileOffset = ftell(f);
writeOne(f, summary->chromId);
writeOne(f, summary->start);
writeOne(f, summary->end);
writeOne(f, summary->validCount);
bbiWriteFloat(f, summary->minVal);
bbiWriteFloat(f, summary->maxVal);
bbiWriteFloat(f, summary->sumData);
bbiWriteFloat(f, summary->sumSquares);
}
bits64 indexOffset = ftell(f);
cirTreeFileBulkIndexToOpenFile(summaryArray, sizeof(summaryArray[0]), count,
blockSize, itemsPerSlot, NULL, bbiSummaryFetchKey, bbiSummaryFetchOffset,
indexOffset, f);
freez(&summaryArray);
return indexOffset;
}
void cirTreeFileBulkIndexToOpenFile(
void *itemArray, int itemSize, bits64 itemCount,
bits32 blockSize, bits32 itemsPerSlot,
void *context,
struct cirTreeRange (*fetchKey)(const void *va, void *context),
bits64 (*fetchOffset)(const void *va, void *context),
bits64 endFileOffset, FILE *f)
/* Create a r tree index from a sorted array, writing output starting at current position
* of an already open file. See rTreeFileCreate for explanation of parameters. */
{
int levelCount;
struct lm *lm = lmInit(0);
struct rTree *tree = rTreeFromChromRangeArray(lm, blockSize, itemsPerSlot,
itemArray, itemSize, itemCount, context, fetchKey, fetchOffset, endFileOffset,
&levelCount);
bits32 magic = cirTreeSig;
bits32 reserved = 0;
writeOne(f, magic);
writeOne(f, blockSize);
writeOne(f, itemCount);
writeOne(f, tree->startChromIx);
writeOne(f, tree->startBase);
writeOne(f, tree->endChromIx);
writeOne(f, tree->endBase);
writeOne(f, endFileOffset);
writeOne(f, itemsPerSlot);
writeOne(f, reserved);
writeTreeToOpenFile(tree, blockSize, levelCount, f);
lmCleanup(&lm);
}
void saveNt4(char *fileName, DNA *dna, bits32 dnaSize)
/* Save dna in an NT4 file. */
{
FILE *f = mustOpen(fileName, "wb");
bits32 signature = nt4Signature;
bits32 bases;
char last[16];
writeOne(f, signature);
writeOne(f, dnaSize);
while (dnaSize >= 16)
{
bases = packDna16(dna);
writeOne(f, bases);
dna += 16;
dnaSize -= 16;
}
if (dnaSize > 0)
{
zeroBytes(last, sizeof(last));
memcpy(last, dna, dnaSize);
bases = packDna16(last);
writeOne(f, bases);
}
fclose(f);
}
void rangeWriteOne(struct range *r, FILE *f)
/* Write out one range structure to binary file f.
* This only writes start and size. */
{
bits32 start = r->start;
bits32 size = r->end-r->start;
writeOne(f, start);
writeOne(f, size);
}
void bbiSummaryElementWrite(FILE *f, struct bbiSummaryElement *sum)
/* Write out summary element to file. */
{
writeOne(f, sum->validCount);
writeOne(f, sum->minVal);
writeOne(f, sum->maxVal);
writeOne(f, sum->sumData);
writeOne(f, sum->sumSquares);
}
void mapWriteHead(FILE *f, bits32 width, bits32 height, char *bac, int trim, char *repeatMask)
/* Write start of map file. */
{
bits32 sig = gluMapSig;
writeOne(f, sig);
writeOne(f,width);
writeOne(f,height);
mapWriteString(f, bac);
writeOne(f, trim);
mapWriteString(f, repeatMask);
}
void qacWriteNext(FILE *f, struct qaSeq *qa)
/* Write next record to qac file. */
{
int cBufSize = qa->size + (qa->size>>1);
signed char *cBuf = needLargeMem(cBufSize);
bits32 cSize, origSize;
origSize = qa->size;
cSize = rleCompress(qa->qa, qa->size, cBuf);
writeString(f, qa->name);
writeOne(f, origSize);
writeOne(f, cSize);
mustWrite(f, cBuf, cSize);
freeMem(cBuf);
}
void mapWriteBox(FILE *f, bits16 mt, int x, int y, int width, int height,
char *bac, bits16 contig, int qStart, int qSize, int tStart, int tSize)
/* Write out one hit box. */
{
writeOne(f, mt);
writeOne(f, x);
writeOne(f, y);
writeOne(f, width);
writeOne(f, height);
mapWriteString(f, bac);
writeOne(f, contig);
writeOne(f, qStart);
writeOne(f, qSize);
writeOne(f, tStart);
writeOne(f, tSize);
}
static bits64 bbiWriteSummaryAndIndexComp(struct bbiSummary *summaryList,
int blockSize, int itemsPerSlot, FILE *f)
/* Write out summary and index to summary uncompressed, returning start position of
* summary index. */
{
bits32 i, count = slCount(summaryList);
struct bbiSummary **summaryArray;
AllocArray(summaryArray, count);
writeOne(f, count);
struct bbiSummary *summary = summaryList;
/* Figure out max size of uncompressed and compressed blocks. */
bits32 itemSize = sizeof(summary->chromId) + sizeof(summary->start) + sizeof(summary->end) + sizeof(summary->validCount) + 4*sizeof(float);
int uncBufSize = itemSize * itemsPerSlot;
char uncBuf[uncBufSize];
int compBufSize = zCompBufSize(uncBufSize);
char compBuf[compBufSize];
/* Loop through compressing and writing one slot at a time. */
bits32 itemsLeft = count;
int sumIx = 0;
while (itemsLeft > 0)
{
bits32 itemsInSlot = itemsLeft;
if (itemsInSlot > itemsPerSlot)
itemsInSlot = itemsPerSlot;
char *writePt = uncBuf;
bits64 filePos = ftell(f);
for (i=0; i<itemsInSlot; ++i)
{
summaryArray[sumIx++] = summary;
memWriteOne(&writePt, summary->chromId);
memWriteOne(&writePt, summary->start);
memWriteOne(&writePt, summary->end);
memWriteOne(&writePt, summary->validCount);
memWriteFloat(&writePt, summary->minVal);
memWriteFloat(&writePt, summary->maxVal);
memWriteFloat(&writePt, summary->sumData);
memWriteFloat(&writePt, summary->sumSquares);
summary->fileOffset = filePos;
summary = summary->next;
if (summary == NULL)
break;
}
bits32 uncSize = writePt - uncBuf;
int compSize = zCompress(uncBuf, uncSize, compBuf, compBufSize);
mustWrite(f, compBuf, compSize);
itemsLeft -= itemsInSlot;
}
bits64 indexOffset = ftell(f);
cirTreeFileBulkIndexToOpenFile(summaryArray, sizeof(summaryArray[0]), count,
blockSize, itemsPerSlot, NULL, bbiSummaryFetchKey, bbiSummaryFetchOffset,
indexOffset, f);
freez(&summaryArray);
return indexOffset;
}
void mapWriteString(FILE *f, char *s)
/* Write string to map file. */
{
short len;
if (s == NULL)
{
len = 0;
writeOne(f, len);
}
else
{
len = strlen(s);
writeOne(f, len);
mustWrite(f, s, len);
}
}
void bptFileBulkIndexToOpenFile(void *itemArray, int itemSize, bits64 itemCount, bits32 blockSize,
void (*fetchKey)(const void *va, char *keyBuf), bits32 keySize,
void* (*fetchVal)(const void *va), bits32 valSize, FILE *f)
/* Create a b+ tree index from a sorted array, writing output starting at current position
* of an already open file. See bptFileCreate for explanation of parameters. */
{
bits32 magic = bptSig;
bits32 reserved = 0;
writeOne(f, magic);
writeOne(f, blockSize);
writeOne(f, keySize);
writeOne(f, valSize);
writeOne(f, itemCount);
writeOne(f, reserved);
writeOne(f, reserved);
bits64 indexOffset = ftell(f);
/* Write non-leaf nodes. */
int levels = bptCountLevels(blockSize, itemCount);
int i;
for (i=levels-1; i > 0; --i)
{
bits32 endLevelOffset = writeIndexLevel(blockSize, itemArray, itemSize, itemCount, indexOffset,
i, fetchKey, keySize, valSize, f);
indexOffset = ftell(f);
if (endLevelOffset != indexOffset)
internalErr();
}
/* Write leaf nodes */
writeLeafLevel(blockSize, itemArray, itemSize, itemCount,
fetchKey, keySize, fetchVal, valSize, f);
}
void writeIndex(struct chromInfo *chromList, int chromCount, char *fileName)
/* Write index file - a b+ tree. */
{
/* Open file and write out header. */
FILE *f = mustOpen(fileName, "w");
bits32 magic = chromSizeIndexSig;
bits32 count = chromCount;
bits16 bSize = blockSize;
bits16 reserved16 = 0;
bits32 reserved32 = 0;
writeOne(f, magic);
writeOne(f, chromCount);
writeOne(f, bSize);
writeOne(f, reserved16);
writeOne(f, reserved32);
bits32 indexOffset = ftell(f);
/* Make array for all chromosomes. */
struct chromInfo *chrom, **chromArray;
AllocArray(chromArray, chromCount);
int i;
for (i=0, chrom=chromList; i<chromCount; ++i, chrom=chrom->next)
chromArray[i] = chrom;
/* Figure out how many levels in B tree, and number of chroms between items at highest level. */
int levels = countLevels(blockSize, chromCount);
verbose(1, "%d levels with blockSize %d covers %d items\n", levels, blockSize, chromCount);
/* Write non-leaf nodes. */
for (i=levels-1; i > 0; --i)
{
bits32 endLevelOffset = writeIndexLevel(chromArray, chromCount, indexOffset, i, f);
indexOffset = ftell(f);
if (endLevelOffset != indexOffset)
errAbort("internal err: mismatch endLevelOffset=%u vs indexOffset=%u", endLevelOffset, indexOffset);
}
/* Write leaf nodes */
writeLeafLevel(chromArray, chromCount, f);
/* Clean up and go home. */
freez(&chromArray);
carefulClose(&f);
}
static void writeLeafLevel(bits16 blockSize, void *itemArray, int itemSize, int itemCount,
void (*fetchKey)(const void *va, char *keyBuf), bits32 keySize,
void* (*fetchVal)(const void *va), bits32 valSize,
FILE *f)
/* Write out leaf level blocks. */
{
char *items = itemArray;
int i,j;
UBYTE isLeaf = TRUE;
UBYTE reserved = 0;
bits16 countOne;
int countLeft = itemCount;
char keyBuf[keySize+1];
keyBuf[keySize] = 0;
for (i=0; i<itemCount; i += countOne)
{
/* Write block header */
if (countLeft > blockSize)
countOne = blockSize;
else
countOne = countLeft;
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, countOne);
/* Write out position in genome and in file for each item. */
for (j=0; j<countOne; ++j)
{
assert(i+j < itemCount);
void *item = items + (i+j)*itemSize;
memset(keyBuf, 0, keySize);
(*fetchKey)(item, keyBuf);
mustWrite(f, keyBuf, keySize);
mustWrite(f, (*fetchVal)(item), valSize);
}
/* Pad out any unused bits of last block with zeroes. */
int slotSize = keySize + valSize;
for (j=countOne; j<blockSize; ++j)
repeatCharOut(f, 0, slotSize);
countLeft -= countOne;
}
}
void writeWord(uint8 word){
uint8 i;
for(i = 0; i < 8; i++){
if(word & 0x80)
writeOne();
else
writeZero();
word <<= 1;
}
}
void writeLED(uint8 R, uint8 G, uint8 B){
uint16 i;
uint8 word = 0x3A;
uint8 words[4];
words[0] = 0x3A;
words[1] = R;
words[2] = G;
words[3] = B;
for(i = 0; i < 8; i++){
if(word & 0x80)
writeOne();
else
writeZero();
word <<= 1;
}
for(i = 0; i < 8; i++){
if(R & 0x80)
writeOne();
else
writeZero();
R <<= 1;
}
for(i = 0; i < 8; i++){
if(G & 0x80)
writeOne();
else
writeZero();
G <<= 1;
}
for(i = 0; i < 8; i++){
if(B & 0x80)
writeOne();
else
writeZero();
B <<= 1;
}
waitEOS();
waitGSLAT();
}
void writeData(uint8 data) {
uint8 i;
for (i = 0; i < 8; i++) {
if (data & 0b10000000) {
writeOne();
} else {
writeZero();
}
data <<= 1;
}
}
void writePgo(struct pgo *pgoList, char *outDir, char *chrom, char *suffix)
/* Write out pgo file for one chromosome. */
{
char fileName[512];
bits32 sig = pgoSig;
bits32 count = countOnChrom(pgoList, chrom);
bits32 nameSize = longestName(pgoList, chrom);
char *nameBuf = needMem(nameSize+1);
struct pgo *pgo;
FILE *f;
sprintf(fileName, "%s%s%s", outDir, chrom, suffix);
f = mustOpen(fileName, "wb");
printf("Writing %d entries nameSize %d into %s\n", count, nameSize, fileName);
writeOne(f, sig);
writeOne(f, count);
writeOne(f, nameSize);
for (pgo = pgoList; pgo != NULL; pgo = pgo->next)
{
if (pgo->chrom == chrom)
{
writeOne(f, pgo->start);
writeOne(f, pgo->end);
writeOne(f, pgo->strand);
zeroBytes(nameBuf, nameSize);
strcpy(nameBuf, pgo->gene);
mustWrite(f, nameBuf, nameSize);
}
}
freeMem(nameBuf);
fclose(f);
}
bits32 writeIndexLevel(struct chromInfo **chromArray, int chromCount,
bits32 indexOffset, int level, FILE *f)
/* Write out a non-leaf level. */
{
/* Calculate number of nodes to write at this level. */
int slotSizePer = xToY(blockSize, level); // Number of chroms per slot in node
int nodeSizePer = slotSizePer * blockSize; // Number of chroms per node
int nodeCount = (chromCount + nodeSizePer - 1)/nodeSizePer;
/* Calculate sizes and offsets. */
int bytesInBlock = (2*sizeof(UBYTE) + sizeof(bits16) + blockSize * (2*sizeof(bits32)));
bits32 levelSize = nodeCount * bytesInBlock;
bits32 endLevel = indexOffset + levelSize;
bits32 nextChild = endLevel;
UBYTE isLeaf = FALSE;
UBYTE reserved = 0;
int i,j;
for (i=0; i<chromCount; i += nodeSizePer)
{
/* Calculate size of this block */
bits16 countOne = (chromCount - i + slotSizePer - 1)/slotSizePer;
if (countOne > blockSize)
countOne = blockSize;
/* Write block header. */
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, countOne);
int slotsUsed = 0;
int endIx = i + nodeSizePer;
if (endIx > chromCount)
endIx = chromCount;
for (j=i; j<endIx; j += slotSizePer)
{
struct chromInfo *chrom = chromArray[j];
writeOne(f, chrom->genomeOffset);
writeOne(f, nextChild);
nextChild += bytesInBlock;
++slotsUsed;
}
assert(slotsUsed == countOne);
for (j=countOne; j<blockSize; ++j)
{
bits32 genomeOffsetPad=0;
bits32 binFileOffsetPad=0;
writeOne(f, genomeOffsetPad);
writeOne(f, binFileOffsetPad);
}
}
return endLevel;
}
void writeBinSaveOffsets(struct chromInfo *chromList, int chromCount, char *fileName)
/* Save chromosome info as a binary file, and save offsets of each chromosome within file. */
{
bits32 magic = chromSizeBinSig;
bits32 count = chromCount;
bits32 reserved = 0;
FILE *f = mustOpen(fileName, "wb");
writeOne(f, magic);
writeOne(f, chromCount);
writeOne(f, reserved);
writeOne(f, reserved);
struct chromInfo *chrom;
for (chrom = chromList; chrom != NULL; chrom = chrom->next)
{
chrom->binFileOffset = ftell(f);
writeOne(f, chrom->genomeOffset);
mustWrite(f, chrom->name, strlen(chrom->name)+1);
}
carefulClose(&f);
}
void initDataCollection()
{
apds9130_Params.waitEnable = 1;
apds9130_Params.proxEnable = 1;
apds9130_Params.powerOn = 1;
// Set WEN, PEN, and PON values to initialize data collection.
uint8_t dataToSend = (apds9130_Params.waitEnable << WEN) |
(apds9130_Params.proxEnable << PEN) |
(apds9130_Params.powerOn << PON);
uint8_t cmdByte = (0x01 << 7) | (0x01 << 5) | ENABLE_;
writeOne(I2C, apds9130_Params.deviceAddress_, cmdByte, dataToSend);
}
static void rWriteLeaves(int itemsPerSlot, int lNodeSize, struct rTree *tree, int curLevel,
int leafLevel, FILE *f)
/* Write out leaf-level nodes. */
{
if (curLevel == leafLevel)
{
/* We've reached the right level, write out a node header. */
UBYTE reserved = 0;
UBYTE isLeaf = TRUE;
bits16 countOne = slCount(tree->children);
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, countOne);
/* Write out elements of this node. */
struct rTree *el;
for (el = tree->children; el != NULL; el = el->next)
{
writeOne(f, el->startChromIx);
writeOne(f, el->startBase);
writeOne(f, el->endChromIx);
writeOne(f, el->endBase);
writeOne(f, el->startFileOffset);
bits64 size = el->endFileOffset - el->startFileOffset;
writeOne(f, size);
}
/* Write out zeroes for empty slots in node. */
int i;
for (i=countOne; i<itemsPerSlot; ++i)
repeatCharOut(f, 0, indexSlotSize);
}
else
{
/* Otherwise recurse on children. */
struct rTree *el;
for (el = tree->children; el != NULL; el = el->next)
rWriteLeaves(itemsPerSlot, lNodeSize, el, curLevel+1, leafLevel, f);
}
}
static bits64 rWriteIndexLevel(bits16 blockSize, int childNodeSize,
struct rTree *tree, int curLevel, int destLevel,
bits64 offsetOfFirstChild, FILE *f)
/* Recursively write an index level, skipping levels below destLevel,
* writing out destLevel. */
{
// uglyf("rWriteIndexLevel blockSize=%d, childNodeSize=%d, offsetOfFirstChild=%llu, curLevel=%d, destLevel=%d slCount(tree)=%d\n", blockSize, childNodeSize, offsetOfFirstChild, curLevel, destLevel, slCount(tree->children));
struct rTree *el;
bits64 offset = offsetOfFirstChild;
if (curLevel == destLevel)
{
/* We've reached the right level, write out a node header */
UBYTE reserved = 0;
UBYTE isLeaf = FALSE;
bits16 countOne = slCount(tree->children);
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, countOne);
/* Write out elements of this node. */
for (el = tree->children; el != NULL; el = el->next)
{
writeOne(f, el->startChromIx);
writeOne(f, el->startBase);
writeOne(f, el->endChromIx);
writeOne(f, el->endBase);
writeOne(f, offset);
offset += childNodeSize;
}
/* Write out zeroes for empty slots in node. */
int i;
for (i=countOne; i<blockSize; ++i)
repeatCharOut(f, 0, indexSlotSize);
}
else
{
/* Otherwise recurse on children. */
for (el = tree->children; el != NULL; el = el->next)
offset = rWriteIndexLevel(blockSize, childNodeSize, el, curLevel+1, destLevel,
offset, f);
}
return offset;
}
void xs_i2c_writeBlock(xsMachine* the)
{
FskErr err;
xsI2C i2c = xsGetHostData(xsThis);
int argc = xsToInteger(xsArgc), i;
UInt8 buffer[32], *bufPtr = buffer;
xsThrowIfNULL(i2c);
DBG_I2C("xs_i2c_writeBlock\n");
for (i = 0; i < argc; i++)
bufPtr = writeOne(the, i2c, &xsArg(i), bufPtr, buffer + sizeof(buffer));
FskPinI2CSetAddress(i2c->pin, i2c->address);
err = FskPinI2CWriteBytes(i2c->pin, bufPtr - buffer, buffer);
xsThrowDiagnosticIfFskErr(err, "I2C FskI2CWriteBlock failed with error %s %s.", FskInstrumentationGetErrorString(err), i2c->diagnosticID);
}
void xs_i2c_writeBlockDataSMB(xsMachine* the)
{
FskErr err;
xsI2C i2c = xsGetHostData(xsThis);
int argc = xsToInteger(xsArgc), i;
UInt8 command = (UInt8)xsToInteger(xsArg(0));
unsigned char buffer[32], *bufPtr = buffer;
xsThrowIfNULL(i2c);
DBG_I2C("xs_i2c_writeBlockDataSMB\n");
FskPinI2CSetAddress(i2c->pin, i2c->address);
for (i = 1; i < argc; i++)
bufPtr = writeOne(the, i2c, &xsArg(i), bufPtr, buffer + sizeof(buffer));
err = FskPinI2CWriteDataBytes(i2c->pin, command, (SInt32)(bufPtr - buffer), buffer);
xsThrowDiagnosticIfFskErr(err, "I2C writeBlockDataSMB register %d failed with error %s %s.", (int)command, FskInstrumentationGetErrorString(err), i2c->diagnosticID);
}
void writeCommand() {
//write command in hex is 3AA, binary is 11 1010 1010
writeOne();
writeOne();
writeOne();
writeZero();
writeOne();
writeZero();
writeOne();
writeZero();
writeOne();
writeZero();
}
void initAPDS9130()
{
// initialize all settings to default values.
apds9130_Params.proxIntegrationTime = 0xff; // PTIME default value.
apds9130_Params.waitTime = 0xff; //
apds9130_Params.proxPulseCount = 0x01;
apds9130_Params.proxDriveCurrent = 0x0;
apds9130_Params.proxDiodeSelect = 0x02;
apds9130_Params.proxGain = 0x0;
// Set PDRIVE, PDIODE, and PGAIN values in CONTROL_ register.
uint8_t dataToSend = (apds9130_Params.proxDriveCurrent << PDRIVE_) |
(apds9130_Params.proxGain << PGAIN_) |
(apds9130_Params.proxDiodeSelect << PDIODE_);
uint8_t cmdByte = (0x01 << 7) | (0x01 << 5) | CONTROL_;
writeOne(I2C, apds9130_Params.deviceAddress_, cmdByte, dataToSend);
}
void writeLeafLevel(struct chromInfo **chromArray, int chromCount, FILE *f)
/* Write out leaf level blocks. */
{
int i,j;
UBYTE isLeaf = TRUE;
UBYTE reserved = 0;
bits16 countOne;
bits32 genomeOffsetPad=0;
bits32 binFileOffsetPad=0;
int countLeft = chromCount;
for (i=0; i<chromCount; i += countOne)
{
/* Write block header */
if (countLeft > blockSize)
countOne = blockSize;
else
countOne = countLeft;
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, countOne);
/* Write out position in genome and in file for each chrom. */
for (j=0; j<countOne; ++j)
{
assert(i+j < chromCount);
struct chromInfo *chrom = chromArray[i+j];
writeOne(f, chrom->genomeOffset);
writeOne(f, chrom->binFileOffset);
}
/* Pad out any unused bits of last block with zeroes. */
for (j=countOne; j<blockSize; ++j)
{
writeOne(f, genomeOffsetPad);
writeOne(f, binFileOffsetPad);
}
countLeft -= countOne;
}
}
void bwgCreate(struct bwgSection *sectionList, struct hash *chromSizeHash,
int blockSize, int itemsPerSlot, boolean doCompress, boolean keepAllChromosomes,
boolean fixedSummaries, char *fileName)
/* Create a bigWig file out of a sorted sectionList. */
{
bits64 sectionCount = slCount(sectionList);
FILE *f = mustOpen(fileName, "wb");
bits32 sig = bigWigSig;
bits16 version = bbiCurrentVersion;
bits16 summaryCount = 0;
bits16 reserved16 = 0;
bits32 reserved32 = 0;
bits64 reserved64 = 0;
bits64 dataOffset = 0, dataOffsetPos;
bits64 indexOffset = 0, indexOffsetPos;
bits64 chromTreeOffset = 0, chromTreeOffsetPos;
bits64 totalSummaryOffset = 0, totalSummaryOffsetPos;
bits32 uncompressBufSize = 0;
bits64 uncompressBufSizePos;
struct bbiSummary *reduceSummaries[10];
bits32 reductionAmounts[10];
bits64 reductionDataOffsetPos[10];
bits64 reductionDataOffsets[10];
bits64 reductionIndexOffsets[10];
int i;
/* Figure out chromosome ID's. */
struct bbiChromInfo *chromInfoArray;
int chromCount, maxChromNameSize;
if (keepAllChromosomes)
bwgMakeAllChromInfo(sectionList, chromSizeHash, &chromCount, &chromInfoArray, &maxChromNameSize);
else
bwgMakeChromInfo(sectionList, chromSizeHash, &chromCount, &chromInfoArray, &maxChromNameSize);
if (fixedSummaries)
bwgComputeFixedSummaries(sectionList, reduceSummaries, &summaryCount, chromInfoArray, reductionAmounts);
else
bwgComputeDynamicSummaries(sectionList, reduceSummaries, &summaryCount, chromInfoArray, chromCount, reductionAmounts, doCompress);
/* Write fixed header. */
writeOne(f, sig);
writeOne(f, version);
writeOne(f, summaryCount);
chromTreeOffsetPos = ftell(f);
writeOne(f, chromTreeOffset);
dataOffsetPos = ftell(f);
writeOne(f, dataOffset);
indexOffsetPos = ftell(f);
writeOne(f, indexOffset);
writeOne(f, reserved16); /* fieldCount */
writeOne(f, reserved16); /* definedFieldCount */
writeOne(f, reserved64); /* autoSqlOffset. */
totalSummaryOffsetPos = ftell(f);
writeOne(f, totalSummaryOffset);
uncompressBufSizePos = ftell(f);
writeOne(f, uncompressBufSize);
writeOne(f, reserved64); /* nameIndexOffset */
assert(ftell(f) == 64);
/* Write summary headers */
for (i=0; i<summaryCount; ++i)
{
writeOne(f, reductionAmounts[i]);
writeOne(f, reserved32);
reductionDataOffsetPos[i] = ftell(f);
writeOne(f, reserved64); // Fill in with data offset later
writeOne(f, reserved64); // Fill in with index offset later
}
/* Write dummy summary */
struct bbiSummaryElement totalSum;
ZeroVar(&totalSum);
totalSummaryOffset = ftell(f);
bbiSummaryElementWrite(f, &totalSum);
/* Write chromosome bPlusTree */
chromTreeOffset = ftell(f);
int chromBlockSize = min(blockSize, chromCount);
bptFileBulkIndexToOpenFile(chromInfoArray, sizeof(chromInfoArray[0]), chromCount, chromBlockSize,
bbiChromInfoKey, maxChromNameSize, bbiChromInfoVal,
sizeof(chromInfoArray[0].id) + sizeof(chromInfoArray[0].size),
f);
/* Write out data section count and sections themselves. */
dataOffset = ftell(f);
writeOne(f, sectionCount);
struct bwgSection *section;
for (section = sectionList; section != NULL; section = section->next)
{
bits32 uncSizeOne = bwgSectionWrite(section, doCompress, f);
if (uncSizeOne > uncompressBufSize)
uncompressBufSize = uncSizeOne;
}
/* Write out index - creating a temporary array rather than list representation of
* sections in the process. */
indexOffset = ftell(f);
struct bwgSection **sectionArray;
AllocArray(sectionArray, sectionCount);
for (section = sectionList, i=0; section != NULL; section = section->next, ++i)
sectionArray[i] = section;
cirTreeFileBulkIndexToOpenFile(sectionArray, sizeof(sectionArray[0]), sectionCount,
blockSize, 1, NULL, bwgSectionFetchKey, bwgSectionFetchOffset,
indexOffset, f);
freez(§ionArray);
/* Write out summary sections. */
verbose(2, "bwgCreate writing %d summaries\n", summaryCount);
for (i=0; i<summaryCount; ++i)
{
reductionDataOffsets[i] = ftell(f);
reductionIndexOffsets[i] = bbiWriteSummaryAndIndex(reduceSummaries[i], blockSize, itemsPerSlot, doCompress, f);
verbose(3, "wrote %d of data, %d of index on level %d\n", (int)(reductionIndexOffsets[i] - reductionDataOffsets[i]), (int)(ftell(f) - reductionIndexOffsets[i]), i);
}
/* Calculate summary */
struct bbiSummary *sum = reduceSummaries[0];
if (sum != NULL)
{
totalSum.validCount = sum->validCount;
totalSum.minVal = sum->minVal;
totalSum.maxVal = sum->maxVal;
totalSum.sumData = sum->sumData;
totalSum.sumSquares = sum->sumSquares;
for (sum = sum->next; sum != NULL; sum = sum->next)
{
totalSum.validCount += sum->validCount;
if (sum->minVal < totalSum.minVal) totalSum.minVal = sum->minVal;
if (sum->maxVal > totalSum.maxVal) totalSum.maxVal = sum->maxVal;
totalSum.sumData += sum->sumData;
totalSum.sumSquares += sum->sumSquares;
}
/* Write real summary */
fseek(f, totalSummaryOffset, SEEK_SET);
bbiSummaryElementWrite(f, &totalSum);
}
else
totalSummaryOffset = 0; /* Edge case, no summary. */
/* Go back and fill in offsets properly in header. */
fseek(f, dataOffsetPos, SEEK_SET);
writeOne(f, dataOffset);
fseek(f, indexOffsetPos, SEEK_SET);
writeOne(f, indexOffset);
fseek(f, chromTreeOffsetPos, SEEK_SET);
writeOne(f, chromTreeOffset);
fseek(f, totalSummaryOffsetPos, SEEK_SET);
writeOne(f, totalSummaryOffset);
if (doCompress)
{
int maxZoomUncompSize = itemsPerSlot * sizeof(struct bbiSummaryOnDisk);
if (maxZoomUncompSize > uncompressBufSize)
uncompressBufSize = maxZoomUncompSize;
fseek(f, uncompressBufSizePos, SEEK_SET);
writeOne(f, uncompressBufSize);
}
/* Also fill in offsets in zoom headers. */
for (i=0; i<summaryCount; ++i)
{
fseek(f, reductionDataOffsetPos[i], SEEK_SET);
writeOne(f, reductionDataOffsets[i]);
writeOne(f, reductionIndexOffsets[i]);
}
/* Write end signature. */
fseek(f, 0L, SEEK_END);
writeOne(f, sig);
/* Clean up */
freez(&chromInfoArray);
carefulClose(&f);
}
static bits32 writeIndexLevel(bits16 blockSize,
void *itemArray, int itemSize, long itemCount,
bits32 indexOffset, int level,
void (*fetchKey)(const void *va, char *keyBuf), bits32 keySize, bits32 valSize,
FILE *f)
/* Write out a non-leaf level. */
{
char *items = itemArray;
/* Calculate number of nodes to write at this level. */
long slotSizePer = xToY(blockSize, level); // Number of items per slot in node
long nodeSizePer = slotSizePer * blockSize; // Number of items per node
long nodeCount = (itemCount + nodeSizePer - 1)/nodeSizePer;
/* Calculate sizes and offsets. */
long bytesInIndexBlock = (bptBlockHeaderSize + blockSize * (keySize+sizeof(bits64)));
long bytesInLeafBlock = (bptBlockHeaderSize + blockSize * (keySize+valSize));
bits64 bytesInNextLevelBlock = (level == 1 ? bytesInLeafBlock : bytesInIndexBlock);
bits64 levelSize = nodeCount * bytesInIndexBlock;
bits64 endLevel = indexOffset + levelSize;
bits64 nextChild = endLevel;
UBYTE isLeaf = FALSE;
UBYTE reserved = 0;
long i,j;
char keyBuf[keySize+1];
keyBuf[keySize] = 0;
for (i=0; i<itemCount; i += nodeSizePer)
{
/* Calculate size of this block */
long countOne = (itemCount - i + slotSizePer - 1)/slotSizePer;
if (countOne > blockSize)
countOne = blockSize;
bits16 shortCountOne = countOne;
/* Write block header. */
writeOne(f, isLeaf);
writeOne(f, reserved);
writeOne(f, shortCountOne);
/* Write out the slots that are used one by one, and do sanity check. */
int slotsUsed = 0;
long endIx = i + nodeSizePer;
if (endIx > itemCount)
endIx = itemCount;
for (j=i; j<endIx; j += slotSizePer)
{
void *item = items + j*itemSize;
memset(keyBuf, 0, keySize);
(*fetchKey)(item, keyBuf);
mustWrite(f, keyBuf, keySize);
writeOne(f, nextChild);
nextChild += bytesInNextLevelBlock;
++slotsUsed;
}
assert(slotsUsed == shortCountOne);
/* Write out empty slots as all zero. */
int slotSize = keySize + sizeof(bits64);
for (j=countOne; j<blockSize; ++j)
repeatCharOut(f, 0, slotSize);
}
return endLevel;
}
static void crTreeFileCreateLow(
char **chromNames, /* All chromosome (or contig) names */
int chromCount, /* Number of chromosomes. */
void *itemArray, /* Sorted array of things to index. */
int itemSize, /* Size of each element in array. */
bits64 itemCount, /* Number of elements in array. */
bits32 blockSize, /* R tree block size - # of children for each node. */
bits32 itemsPerSlot, /* Number of items to put in each index slot at lowest level. */
struct crTreeRange (*fetchKey)(const void *va), /* Given item, return key. */
bits64 (*fetchOffset)(const void *va), /* Given item, return file offset */
bits64 initialDataOffset, /* Offset of 1st piece of data in file. */
bits64 totalDataSize, /* Total size of data we are indexing. */
char *fileName) /* Name of output file. */
/* Create a r tree index file from an array of chromosomes and an array of items with
* basic bed (chromosome,start,end) and file offset information. */
{
// uglyf("crTreeFileCreate %s itemCount=%llu, chromCount=%d\n", fileName, itemCount, chromCount);
/* Open file and write header. */
FILE *f = mustOpen(fileName, "wb");
bits32 magic = crTreeSig;
bits32 reserved32 = 0;
bits64 chromOffset = crHeaderSize;
bits64 cirOffset = 0;
bits64 reserved64 = 0;
writeOne(f, magic);
writeOne(f, reserved32);
writeOne(f, chromOffset);
writeOne(f, cirOffset); /* Will fill this back in later */
writeOne(f, reserved64);
writeOne(f, reserved64);
writeOne(f, reserved64);
writeOne(f, reserved64);
writeOne(f, reserved64);
/* Convert array of chromosomes to a sorted array of name32s. Also
* figure out maximum chromosome name size. */
struct name32 *name32Array;
AllocArray(name32Array, chromCount);
bits32 chromIx;
int maxChromNameSize = 0;
for (chromIx=0; chromIx<chromCount; ++chromIx)
{
struct name32 *name32 = &name32Array[chromIx];
char *name = chromNames[chromIx];
name32->name = name;
int nameSize = strlen(name);
if (nameSize > maxChromNameSize)
maxChromNameSize = nameSize;
}
qsort(name32Array, chromCount, sizeof(name32Array[0]), name32Cmp);
for (chromIx=0; chromIx<chromCount; ++chromIx)
{
struct name32 *name32 = &name32Array[chromIx];
name32->val = chromIx;
}
/* Write out bPlusTree index of chromosome IDs. */
int chromBlockSize = min(blockSize, chromCount);
bptFileBulkIndexToOpenFile(name32Array, sizeof(name32Array[0]), chromCount, chromBlockSize,
name32Key, maxChromNameSize, name32Val, sizeof(name32Array[0].val), f);
/* Convert itemArray to ciItemArray. This is mainly to avoid having to do the chromosome to
* chromosome index conversion for each item. The cost is some memory though.... */
struct ciItem *ciItemArray;
AllocArray(ciItemArray, itemCount);
bits64 itemIx;
char *itemPos = itemArray;
char *lastChrom = "";
bits32 lastChromIx = 0;
for (itemIx=0; itemIx < itemCount; ++itemIx)
{
struct ciItem *ciItem = &ciItemArray[itemIx];
ciItem->item = itemPos;
ciItem->key = (*fetchKey)(itemPos);
if (!sameString(lastChrom, ciItem->key.chrom))
{
lastChrom = ciItem->key.chrom;
lastChromIx = mustFindChromIx(lastChrom, name32Array, chromCount);
}
ciItem->chromIx = lastChromIx;
itemPos += itemSize;
}
/* Record starting position of r tree and write it out. */
cirOffset = ftell(f);
struct ciContext context;
ZeroVar(&context);
context.fetchKey = fetchKey;
context.fetchOffset = fetchOffset;
cirTreeFileBulkIndexToOpenFile(ciItemArray, sizeof(ciItemArray[0]), itemCount, blockSize,
itemsPerSlot, &context, ciItemFetchKey, ciItemFetchOffset, totalDataSize, f);
/* Seek back and write offset to r tree. */
fseek(f, cirOffsetPos, SEEK_SET);
writeOne(f, cirOffset);
/* Clean up */
freez(&name32Array);
carefulClose(&f);
}
