struct qaSeq *qacReadNext(FILE *f, boolean isSwapped)
/* Read in next record in .qac file. */
{
bits32 cSize, origSize;
struct qaSeq *qa;
signed char *buf;
char *s;
s = readString(f);
if (s == NULL)
return NULL;
AllocVar(qa);
qa->name = s;
mustReadOne(f, origSize);
if (isSwapped)
origSize = byteSwap32(origSize);
mustReadOne(f, cSize);
if (isSwapped)
cSize = byteSwap32(cSize);
qa->size = origSize;
qa->qa = needLargeMem(origSize);
buf = needLargeMem(cSize);
mustRead(f, buf, cSize);
rleUncompress(buf, cSize, qa->qa, origSize);
freeMem(buf);
return qa;
}
struct hash *qacReadToHash(char *fileName)
/* Read in a qac file into a hash of qacs keyed by name. */
{
boolean isSwapped;
FILE *f = qacOpenVerify(fileName, &isSwapped);
bits32 compSize, uncSize;
struct qac *qac;
char *name;
struct hash *hash = newHash(18);
int count = 0;
for (;;)
{
name = readString(f);
if (name == NULL)
break;
mustReadOne(f, uncSize);
if (isSwapped)
uncSize = byteSwap32(uncSize);
mustReadOne(f, compSize);
if (isSwapped)
compSize = byteSwap32(compSize);
qac = needHugeMem(sizeof(*qac) + compSize - 1);
qac->uncSize = uncSize;
qac->compSize = compSize;
mustRead(f, qac->data, compSize);
hashAdd(hash, name, qac);
++count;
}
carefulClose(&f);
printf("Read %d qacs from %s\n", count, fileName);
return hash;
}
void rangeReadArray(FILE *f, struct rangeStartSize *r, int n, boolean isSwapped)
/* Read 'n' elements of range array (start,size) from file 'f'. */
{
mustRead(f, r, n*sizeof(*r));
if (isSwapped)
{
while (n)
{
r->start = byteSwap32(r->start);
r->size = byteSwap32(r->size);
--n;
++r;
}
}
}
enum metaBigFileType isBigWigOrBed(char* filename)
/* Peak at a file to see if it's bigWig */
{
enum metaBigFileType ret = isNotBig;
bits32 magic;
struct udcFile* udc;
udc = udcFileOpen(filename, udcDefaultDir());
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
udcMustRead(udc, &magic, sizeof(magic));
if (magic == bigWigSig)
ret = isaBigWig;
else if (magic == bigBedSig)
ret = isaBigBed;
if (ret != isNotBig) {
udcFileClose(&udc);
return ret;
}
magic = byteSwap32(magic);
if (magic == bigWigSig)
ret = isaBigWig;
else if (magic == bigBedSig)
ret = isaBigBed;
udcFileClose(&udc);
return ret;
}
static void chromNameCallback(void *context, void *key, int keySize, void *val, int valSize)
/* Callback that captures chromInfo from bPlusTree. */
{
struct chromNameCallbackContext *c = context;
struct bbiChromInfo *info;
struct bbiChromIdSize *idSize = val;
assert(valSize == sizeof(*idSize));
AllocVar(info);
info->name = cloneStringZ(key, keySize);
info->id = idSize->chromId;
info->size = idSize->chromSize;
if (c->isSwapped)
{
info->id = byteSwap32(info->id);
info->size = byteSwap32(info->size);
}
slAddHead(&c->list, info);
}
boolean isBigWig(char *fileName)
/* Peak at a file to see if it's bigWig */
{
FILE *f = mustOpen(fileName, "rb");
bits32 sig;
mustReadOne(f, sig);
fclose(f);
if (sig == bigWigSig)
return TRUE;
sig = byteSwap32(sig);
return sig == bigWigSig;
}
struct fileOffsetSize *bbiOverlappingBlocks(struct bbiFile *bbi, struct cirTreeFile *ctf,
char *chrom, bits32 start, bits32 end, bits32 *retChromId)
/* Fetch list of file blocks that contain items overlapping chromosome range. */
{
struct bbiChromIdSize idSize;
if (!bptFileFind(bbi->chromBpt, chrom, strlen(chrom), &idSize, sizeof(idSize)))
return NULL;
if (bbi->isSwapped)
idSize.chromId = byteSwap32(idSize.chromId);
if (retChromId != NULL)
*retChromId = idSize.chromId;
return cirTreeFindOverlappingBlocks(ctf, idSize.chromId, start, end);
}
void oocMaskCounts(char *oocFile, bits32 *tileCounts, int tileSize, bits32 maxPat)
/* Set items of tileCounts to maxPat if they are in oocFile.
* Effectively masks this out of index.*/
{
if (oocFile != NULL)
{
bits32 sig, psz;
FILE *f = mustOpen(oocFile, "rb");
boolean mustSwap = FALSE;
mustReadOne(f, sig);
mustReadOne(f, psz);
if (sig == oocSig)
mustSwap = FALSE;
else if (sig == oocSigSwapped)
{
mustSwap = TRUE;
psz = byteSwap32(psz);
}
else
errAbort("Bad signature on %s\n", oocFile);
if (psz != tileSize)
errAbort("Oligo size mismatch in %s. Expecting %d got %d\n",
oocFile, tileSize, psz);
if (mustSwap)
{
union {bits32 whole; UBYTE bytes[4];} u,v;
while (readOne(f, u))
{
v.bytes[0] = u.bytes[3];
v.bytes[1] = u.bytes[2];
v.bytes[2] = u.bytes[1];
v.bytes[3] = u.bytes[0];
tileCounts[v.whole] = maxPat;
}
}
else
{
bits32 oli;
while (readOne(f, oli))
tileCounts[oli] = maxPat;
}
fclose(f);
}
}
struct cirTreeFile *cirTreeFileAttach(char *fileName, struct udcFile *udc)
/* Open up r-tree index file on previously open file, with cirTree
* header at current file position. */
{
/* Open file and allocate structure to hold info from header etc. */
struct cirTreeFile *crt = needMem(sizeof(*crt));
crt->fileName = fileName;
crt->udc = udc;
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
bits32 magic;
boolean isSwapped = FALSE;
udcMustReadOne(udc, magic);
if (magic != cirTreeSig)
{
magic = byteSwap32(magic);
isSwapped = crt->isSwapped = TRUE;
if (magic != cirTreeSig)
errAbort("%s is not a chromosome id r-tree index file", fileName);
}
/* Read rest of defined bits of header, byte swapping as needed. */
crt->blockSize = udcReadBits32(udc, isSwapped);
crt->itemCount = udcReadBits64(udc, isSwapped);
crt->startChromIx = udcReadBits32(udc, isSwapped);
crt->startBase = udcReadBits32(udc, isSwapped);
crt->endChromIx = udcReadBits32(udc, isSwapped);
crt->endBase = udcReadBits32(udc, isSwapped);
crt->fileSize = udcReadBits64(udc, isSwapped);
crt->itemsPerSlot = udcReadBits32(udc, isSwapped);
/* Skip over reserved bits of header. */
bits32 reserved32;
udcMustReadOne(udc, reserved32);
/* Save position of root block of r tree. */
crt->rootOffset = udcTell(udc);
return crt;
}
struct crTreeFile *crTreeFileOpen(char *fileName)
/* Open up r-tree index file - reading headers and verifying things. */
{
/* Open file and allocate structure to hold info from header etc. */
struct udcFile *udc = udcFileOpen(fileName, udcDefaultDir());
struct crTreeFile *crt = needMem(sizeof(*crt));
fileName = crt->fileName = cloneString(fileName);
crt->udc = udc;
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
bits32 magic;
boolean isSwapped = FALSE;
udcMustReadOne(udc, magic);
if (magic != crTreeSig)
{
magic = byteSwap32(magic);
isSwapped = crt->isSwapped = TRUE;
if (magic != crTreeSig)
errAbort("%s is not a chromosome r-tree index file", fileName);
}
/* Read rest of high level header including notably the offsets to the
* chromosome and range indexes. */
bits32 reserved32;
udcMustReadOne(udc, reserved32);
crt->chromOffset = udcReadBits64(udc, isSwapped);
crt->cirOffset = udcReadBits64(udc, isSwapped);
/* Read in the chromosome index header. */
udcSeek(udc, crt->chromOffset);
crt->chromBpt = bptFileAttach(fileName, udc);
/* Read in range index header. */
udcSeek(udc, crt->cirOffset);
crt->cir = cirTreeFileAttach(fileName, udc);
return crt;
}
struct bptFile *bptFileAttach(char *fileName, struct udcFile *udc)
/* Open up index file on previously open file, with header at current file position. */
{
/* Open file and allocate structure to hold info from header etc. */
struct bptFile *bpt = needMem(sizeof(*bpt));
bpt->fileName = fileName;
bpt->udc = udc;
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
bits32 magic;
boolean isSwapped = FALSE;
udcMustReadOne(udc, magic);
if (magic != bptSig)
{
magic = byteSwap32(magic);
isSwapped = bpt->isSwapped = TRUE;
if (magic != bptSig)
errAbort("%s is not a bpt b-plus tree index file", fileName);
}
/* Read rest of defined bits of header, byte swapping as needed. */
bpt->blockSize = udcReadBits32(udc, isSwapped);
bpt->keySize = udcReadBits32(udc, isSwapped);
bpt->valSize = udcReadBits32(udc, isSwapped);
bpt->itemCount = udcReadBits64(udc, isSwapped);
/* Skip over reserved bits of header. */
bits32 reserved32;
udcMustReadOne(udc, reserved32);
udcMustReadOne(udc, reserved32);
/* Save position of root block of b+ tree. */
bpt->rootOffset = udcTell(udc);
return bpt;
}
struct bbiFile *bbiFileOpenWithDir(char *fileName, bits32 sig, char *typeName, char *udcDir)
/* same (mostly) as bbiFileOpen in bbiFile.c, but allows setting the temporary dir */
{
struct bbiFile *bbi;
AllocVar(bbi);
bbi->fileName = cloneString(fileName);
struct udcFile *udc = bbi->udc = udcFileOpen(fileName, udcDir);
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
bits32 magic;
boolean isSwapped = FALSE;
udcMustRead(udc, &magic, sizeof(magic));
if (magic != sig)
{
magic = byteSwap32(magic);
isSwapped = TRUE;
if (magic != sig)
errAbort("%s is not a %s file", fileName, typeName);
}
bbi->typeSig = sig;
bbi->isSwapped = isSwapped;
/* Read rest of defined bits of header, byte swapping as needed. */
bbi->version = udcReadBits16(udc, isSwapped);
bbi->zoomLevels = udcReadBits16(udc, isSwapped);
bbi->chromTreeOffset = udcReadBits64(udc, isSwapped);
bbi->unzoomedDataOffset = udcReadBits64(udc, isSwapped);
bbi->unzoomedIndexOffset = udcReadBits64(udc, isSwapped);
bbi->fieldCount = udcReadBits16(udc, isSwapped);
bbi->definedFieldCount = udcReadBits16(udc, isSwapped);
bbi->asOffset = udcReadBits64(udc, isSwapped);
bbi->totalSummaryOffset = udcReadBits64(udc, isSwapped);
bbi->uncompressBufSize = udcReadBits32(udc, isSwapped);
bbi->extensionOffset = udcReadBits64(udc, isSwapped);
/* Read zoom headers. */
int i;
struct bbiZoomLevel *level, *levelList = NULL;
for (i=0; i<bbi->zoomLevels; ++i)
{
AllocVar(level);
level->reductionLevel = udcReadBits32(udc, isSwapped);
level->reserved = udcReadBits32(udc, isSwapped);
level->dataOffset = udcReadBits64(udc, isSwapped);
level->indexOffset = udcReadBits64(udc, isSwapped);
slAddHead(&levelList, level);
}
slReverse(&levelList);
bbi->levelList = levelList;
/* Deal with header extension if any. */
if (bbi->extensionOffset != 0)
{
udcSeek(udc, bbi->extensionOffset);
bbi->extensionSize = udcReadBits16(udc, isSwapped);
bbi->extraIndexCount = udcReadBits16(udc, isSwapped);
bbi->extraIndexListOffset = udcReadBits64(udc, isSwapped);
}
/* Attach B+ tree of chromosome names and ids. */
udcSeek(udc, bbi->chromTreeOffset);
bbi->chromBpt = bptFileAttach(fileName, udc);
return bbi;
}
struct bbiFile *bbiFileOpen(char *fileName, bits32 sig, char *typeName)
/* Open up big wig or big bed file. */
{
/* This code needs to agree with code in two other places currently - bigBedFileCreate,
* and bigWigFileCreate. I'm thinking of refactoring to share at least between
* bigBedFileCreate and bigWigFileCreate. It'd be great so it could be structured
* so that it could send the input in one chromosome at a time, and send in the zoom
* stuff only after all the chromosomes are done. This'd potentially reduce the memory
* footprint by a factor of 2 or 4. Still, for now it works. -JK */
struct bbiFile *bbi;
AllocVar(bbi);
bbi->fileName = cloneString(fileName);
struct udcFile *udc = bbi->udc = udcFileOpen(fileName, udcDefaultDir());
/* Read magic number at head of file and use it to see if we are proper file type, and
* see if we are byte-swapped. */
bits32 magic;
boolean isSwapped = FALSE;
udcMustRead(udc, &magic, sizeof(magic));
if (magic != sig)
{
magic = byteSwap32(magic);
isSwapped = TRUE;
if (magic != sig)
errAbort("%s is not a %s file", fileName, typeName);
}
bbi->typeSig = sig;
bbi->isSwapped = isSwapped;
/* Read rest of defined bits of header, byte swapping as needed. */
bbi->version = udcReadBits16(udc, isSwapped);
bbi->zoomLevels = udcReadBits16(udc, isSwapped);
bbi->chromTreeOffset = udcReadBits64(udc, isSwapped);
bbi->unzoomedDataOffset = udcReadBits64(udc, isSwapped);
bbi->unzoomedIndexOffset = udcReadBits64(udc, isSwapped);
bbi->fieldCount = udcReadBits16(udc, isSwapped);
bbi->definedFieldCount = udcReadBits16(udc, isSwapped);
bbi->asOffset = udcReadBits64(udc, isSwapped);
bbi->totalSummaryOffset = udcReadBits64(udc, isSwapped);
bbi->uncompressBufSize = udcReadBits32(udc, isSwapped);
/* Skip over reserved area. */
udcSeek(udc, 64);
/* Read zoom headers. */
int i;
struct bbiZoomLevel *level, *levelList = NULL;
for (i=0; i<bbi->zoomLevels; ++i)
{
AllocVar(level);
level->reductionLevel = udcReadBits32(udc, isSwapped);
level->reserved = udcReadBits32(udc, isSwapped);
level->dataOffset = udcReadBits64(udc, isSwapped);
level->indexOffset = udcReadBits64(udc, isSwapped);
slAddHead(&levelList, level);
}
slReverse(&levelList);
bbi->levelList = levelList;
/* Attach B+ tree of chromosome names and ids. */
udcSeek(udc, bbi->chromTreeOffset);
bbi->chromBpt = bptFileAttach(fileName, udc);
return bbi;
}
