rc_t CC copycat_log_writer (void * self, const char * buffer, size_t buffer_size,
size_t * num_writ)
{
if (self)
{
void * bf = malloc (sizeof (SLNode) + buffer_size);
if (bf)
{
const char * ps, * pc;
size_t z;
ps = strchr (buffer, ' ');
if (ps)
{
++ps;
pc = strchr (ps, '-');
if (pc == NULL)
pc = strchr (ps, ':');
if (pc)
pc = pc + 2;
else
pc = ps;
}
else
pc = buffer;
z = buffer_size - (pc - buffer);
memcpy ( (void*)(((SLNode*)bf)+1), pc, z);
((char*)(((SLNode*)bf)+1))[z-1] = '\0';
SLListPushTail (self, bf);
}
}
return (log_writer != NULL)
? log_writer (log_data, buffer, buffer_size, num_writ) : 0;
}
LIB_EXPORT rc_t CC KLoadProgressbar_Make(const KLoadProgressbar** cself, uint64_t size)
{
rc_t rc = 0;
KLoadProgressbar* obj = NULL;
if( cself == NULL ) {
rc = RC(rcApp, rcFunction, rcConstructing, rcSelf, rcNull);
} else if( (obj = calloc(1, sizeof(*obj))) == NULL ) {
*cself = NULL;
rc = RC(rcApp, rcFunction, rcConstructing, rcMemory, rcExhausted);
} else {
DBG(("%s: %p %lu\n", __func__, obj, size));
SLListPushTail(&g_jobs.jobs, &obj->node);
*cself = obj;
obj->active = true;
obj->total = size;
}
return rc;
}
static
rc_t IlluminaLoaderFmt_WriteData(IlluminaLoaderFmt* self, uint32_t argc, const SRALoaderFile* const argv[], int64_t* spots_bad_count)
{
rc_t rc = 0;
uint32_t t, i, k, ftype_q = sizeof(file_types) / sizeof(file_types[0]);
SLList files;
IlluminaFileInfo* file = NULL;
SLListInit(&files);
/* group files using spotname, for _prb. file name prefix is used,
files reviewed by type detected from name and ordered by file_type array */
for(t = 0; rc == 0 && t < ftype_q; t++) {
for(i = 0; rc == 0 && i < argc; i++) {
const char* fname, *blk_pfx;
int prefix_len = 0;
ERunFileType ftype;
EIlluminaNativeFileType type = eIlluminaNativeFileTypeNotSet;
FGroup_Find_data data;
if( (rc = SRALoaderFileName(argv[i], &fname)) != 0 ) {
SRALoaderFile_LOG(argv[i], klogErr, rc, "reading file name", NULL);
break;
}
if( (rc = SRALoaderFile_FileType(argv[i], &ftype)) != 0 ) {
SRALoaderFile_LOG(argv[i], klogErr, rc, "reading file type", NULL);
break;
}
if( (rc = SRALoaderFileBlockName(argv[i], &blk_pfx)) != 0 ) {
SRALoaderFile_LOG(argv[i], klogErr, rc, "reading DATA_BLOCK/@name", NULL);
break;
}
if( blk_pfx == NULL ) {
blk_pfx = "";
}
{{
/* skip path if present */
const char* p = strrchr(fname, '/');
fname = p ? p + 1 : fname;
p = NULL;
for(k = 0; type == eIlluminaNativeFileTypeNotSet && k < ftype_q; k++) {
const char* const* e = file_types[k].key;
while( *e != NULL ) {
p = strstr(fname, *e++);
if( p != NULL ) {
type = file_types[k].type;
break;
}
}
}
if( p != NULL ) {
prefix_len = p - fname;
}
}}
if( ftype == rft_IlluminaNativeSeq ) {
type = eIlluminaNativeFileTypeFasta;
} else if( ftype == rft_IlluminaNativePrb ) {
type = eIlluminaNativeFileTypeQuality4;
} else if( ftype == rft_IlluminaNativeInt ) {
type = eIlluminaNativeFileTypeIntensity;
} else if( ftype == rft_IlluminaNativeQseq ) {
type = eIlluminaNativeFileTypeQSeq;
}
if( type == eIlluminaNativeFileTypeNotSet ) {
rc = RC(rcSRA, rcFormatter, rcReading, rcData, rcUnrecognized);
SRALoaderFile_LOG(argv[i], klogErr, rc, "detecting file type by file name", NULL);
break;
}
if( type != file_types[t].type ) {
/* one type at a time */
continue;
}
DEBUG_MSG(3, ("file '%s' type set to %d\n", fname, type));
file = calloc(1, sizeof(*file));
if( file == NULL ) {
rc = RC(rcSRA, rcFormatter, rcReading, rcMemory, rcExhausted);
SRALoaderFile_LOG(argv[i], klogErr, rc, "allocating file object", NULL);
break;
}
IlluminaFileInfo_init(file);
file->file = argv[i];
file->type = type;
if( file->type == eIlluminaNativeFileTypeQuality4 ) {
/* in _prb there is no spotname inside so use file prefix */
rc = pstring_assign(&data.key, fname, prefix_len);
} else {
/* try to get 1st spot so group can be organized by spot name */
if( (rc = read_next_spot(blk_pfx, file)) != 0 || !file->ready ) {
rc = rc ? rc : RC(rcSRA, rcFormatter, rcReading, rcData, rcNotFound);
SRALoaderFile_LOG(argv[i], klogErr, rc, "reading 1st spot", NULL);
break;
}
rc = pstring_copy(&data.key, &file->name);
}
data.found = NULL;
if( SLListDoUntil(&files, FGroup_Find, &data) && data.found != NULL ) {
IlluminaFileInfo* ss = data.found->files;
while( rc == 0 && file != NULL ) {
if( ss->type != eIlluminaNativeFileTypeQSeq && ss->type == file->type ) {
rc = RC(rcSRA, rcFormatter, rcReading, rcFile, rcDuplicate);
SRALoaderFile_LOG(argv[i], klogErr, rc, "type of file for lane", NULL);
} else if( ss->next != NULL ) {
ss = ss->next;
} else {
ss->next = file;
file->prev = ss;
data.found->mask |= file->type;
file = NULL;
}
}
} else {
data.found = calloc(1, sizeof(*data.found));
if( data.found == NULL ) {
rc = RC(rcSRA, rcFormatter, rcReading, rcMemory, rcInsufficient);
SRALoaderFile_LOG(argv[i], klogErr, rc, "preparing file group", NULL);
break;
} else {
if( (rc = pstring_assign(&data.found->key, fname, prefix_len)) != 0 ) {
SRALoaderFile_LOG(argv[i], klogErr, rc, "setting file group key", NULL);
FGroup_Whack(&data.found->dad, NULL);
break;
} else {
FGroup* curr = (FGroup*)SLListHead(&files), *prev = NULL;
data.found->blk_pfx = blk_pfx;
data.found->files = file;
data.found->mask = file->type;
/* group inserted into list by coords in 1st spot */
while( curr != NULL ) {
if( curr->files[0].coord[0] > file->coord[0] ||
(curr->files[0].coord[0] == file->coord[0] &&
curr->files[0].coord[1] > file->coord[1]) ) {
data.found->dad.next = &curr->dad;
if( prev == NULL ) {
files.head = &data.found->dad;
} else {
prev->dad.next = &data.found->dad;
}
break;
}
prev = curr;
curr = (FGroup*)curr->dad.next;
}
if( curr == NULL ) {
SLListPushTail(&files, &data.found->dad);
}
file = NULL;
}
}
}
}
}
if( rc == 0 ) {
SLListForEach(&files, FGroup_Validate, &rc);
}
if( rc == 0 ) {
FGroup_Parse_data data;
data.self = self;
if( SLListDoUntil(&files, FGroup_Parse, &data) ) {
rc = data.rc;
}
} else {
free(file);
}
SLListWhack(&files, FGroup_Whack, NULL);
*spots_bad_count = self->spots_bad_count;
return rc;
}
static
rc_t FastqGzip_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz)
{
rc_t rc = 0;
const FastqReader* reader = NULL;
uint16_t zlib_ver = ZLIB_VERNUM;
uint8_t colorSpace = false;
char* colorSpaceKey = "\0";
uint8_t origFormat = false;
uint8_t printLabel = true;
uint8_t printReadId = true;
uint8_t clipQuality = true;
uint32_t minReadLen = 0;
uint16_t qualityOffset = 0;
{{
const SRAColumn* c = NULL;
const uint8_t *platform = SRA_PLATFORM_UNDEFINED;
bitsz_t o, z;
if( (rc = SRATableOpenColumnRead(sratbl, &c, "PLATFORM", sra_platform_id_t)) != 0 ) {
return rc;
}
if( (rc = SRAColumnRead(c, 1, (const void **)&platform, &o, &z)) != 0 ) {
return rc;
}
if( *platform == SRA_PLATFORM_ABSOLID ) {
colorSpace = true;
}
SRAColumnRelease(c);
}}
if( (rc = FastqReaderMake(&reader, sratbl, g_accession,
colorSpace, origFormat, false, printLabel, printReadId,
!clipQuality, minReadLen, qualityOffset, colorSpaceKey[0],
obj->minSpotId, obj->maxSpotId)) != 0 ) {
return rc;
} else {
size_t written = 0;
uint32_t blk = 0, spots_per_block = 0, proj_id_qty = 0;
SIndexNode* inode = NULL;
size_t z_blk = 0;
size_t spots_buf_sz = g_file_block_sz * 100;
size_t zbuf_sz = spots_buf_sz + 100;
char* zbuf = malloc(zbuf_sz);
char* spots_buf = malloc(spots_buf_sz);
bool eof = false;
if( zbuf == NULL || spots_buf == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
}
while( rc == 0 ) {
if( (rc = FastqReader_GetNextSpotSplitData(reader, buffer, buffer_sz, &written)) == 0 ) {
if( inode == NULL ) {
spotid_t spotid = 0;
if( (rc = FastqReaderCurrentSpot(reader, &spotid)) != 0 ) {
break;
}
inode = malloc(sizeof(SIndexNode));
if( inode == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
break;
}
inode->key = obj->file_size;
inode->key_size = 0;
inode->id = spotid;
inode->id_qty = 0;
DEBUG_MSG(5, ("%s open key: spot %ld, offset %lu\n", obj->index, inode->id, inode->key));
}
if( blk + written > spots_buf_sz ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient);
break;
}
inode->id_qty++;
memmove(&spots_buf[blk], buffer, written);
blk += written;
if( g_dump ) {
fwrite(buffer, written, 1, stderr);
}
}
if( (eof = (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted)) ) {
rc = 0;
if( inode == NULL ) {
break;
}
}
if( rc == 0 && (eof ||
(proj_id_qty == 0 && inode->id_qty > (spots_per_block * 0.95)) ||
(proj_id_qty > 0 && inode->id_qty >= proj_id_qty) ) ) {
rc = ZLib_DeflateBlock(spots_buf, blk, zbuf, zbuf_sz, &z_blk);
if( z_blk < g_file_block_sz ) {
/* project needed id_qty */
proj_id_qty = g_file_block_sz * inode->id_qty / z_blk * 1.05;
DEBUG_MSG(5, ("%s: project id qty %u\n", obj->index, proj_id_qty));
} else {
DEBUG_MSG(10, ("%s: no projection %u > %u\n", obj->index, z_blk, g_file_block_sz));
}
}
if( rc == 0 && (eof || z_blk >= g_file_block_sz) ) {
obj->file_size += z_blk;
MD5StateAppend(&obj->md5, zbuf, z_blk);
inode->key_size = z_blk;
SLListPushTail(&obj->li, &inode->n);
DEBUG_MSG(5, ("%s close key: spots %lu, size %lu, ratio %hu%%, raw %u\n",
obj->index, inode->id_qty, inode->key_size, (uint16_t)(((float)(blk - z_blk)/blk)*100), blk ));
spots_per_block = inode->id_qty;
inode = NULL;
if( blk > obj->buffer_sz ) {
obj->buffer_sz = blk;
}
blk = 0;
z_blk = 0;
proj_id_qty = 0;
}
if( eof ) {
break;
}
}
rc = rc ? rc : Quitting();
if( rc != 0 ) {
spotid_t spot = 0;
FastqReaderCurrentSpot(reader, &spot);
PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot));
}
free(zbuf);
free(spots_buf);
}
if( rc == 0 ) {
KMDataNode* opt = NULL, *nd = NULL;
if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) {
return rc;
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "ZlibVersion")) == 0 ) {
rc = KMDataNodeWriteB16(nd, &zlib_ver);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpace")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &colorSpace);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpaceKey")) == 0 ) {
rc = KMDataNodeWrite(nd, colorSpaceKey, 1);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "origFormat")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &origFormat);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printLabel")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &printLabel);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printReadId")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &printReadId);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "clipQuality")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &clipQuality);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "minReadLen")) == 0 ) {
rc = KMDataNodeWriteB32(nd, &minReadLen);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "qualityOffset")) == 0 ) {
rc = KMDataNodeWriteB16(nd, &qualityOffset);
KMDataNodeRelease(nd);
}
KMDataNodeRelease(opt);
}
FastqReaderWhack(reader);
return rc;
}
static
rc_t Fastq_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz)
{
rc_t rc = 0;
const FastqReader* reader = NULL;
uint8_t colorSpace = false;
char* colorSpaceKey = "\0";
uint8_t origFormat = false;
uint8_t printLabel = true;
uint8_t printReadId = true;
uint8_t clipQuality = true;
uint32_t minReadLen = 0;
uint16_t qualityOffset = 0;
{{
const SRAColumn* c = NULL;
const uint8_t *platform = SRA_PLATFORM_UNDEFINED;
bitsz_t o, z;
if( (rc = SRATableOpenColumnRead(sratbl, &c, "PLATFORM", sra_platform_id_t)) != 0 ) {
return rc;
}
if( (rc = SRAColumnRead(c, 1, (const void **)&platform, &o, &z)) != 0 ) {
return rc;
}
if( *platform == SRA_PLATFORM_ABSOLID ) {
colorSpace = true;
}
SRAColumnRelease(c);
}}
if( (rc = FastqReaderMake(&reader, sratbl, g_accession,
colorSpace, origFormat, false, printLabel, printReadId,
!clipQuality, minReadLen, qualityOffset, colorSpaceKey[0],
obj->minSpotId, obj->maxSpotId)) != 0 ) {
return rc;
} else {
KMDataNode* opt = NULL, *nd = NULL;
if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) {
return rc;
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpace")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &colorSpace);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpaceKey")) == 0 ) {
rc = KMDataNodeWrite(nd, colorSpaceKey, 1);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "origFormat")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &origFormat);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printLabel")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &printLabel);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printReadId")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &printReadId);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "clipQuality")) == 0 ) {
rc = KMDataNodeWriteB8(nd, &clipQuality);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "minReadLen")) == 0 ) {
rc = KMDataNodeWriteB32(nd, &minReadLen);
KMDataNodeRelease(nd);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "qualityOffset")) == 0 ) {
rc = KMDataNodeWriteB16(nd, &qualityOffset);
KMDataNodeRelease(nd);
}
KMDataNodeRelease(opt);
}
if( rc == 0 ) {
size_t written = 0;
uint32_t blk = 0;
SIndexNode* inode = NULL;
while( rc == 0 ) {
rc = FastqReader_GetNextSpotSplitData(reader, buffer, buffer_sz, &written);
if( blk >= g_file_block_sz || (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted) ) {
inode->key_size = blk;
SLListPushTail(&obj->li, &inode->n);
DEBUG_MSG(5, ("Fastq index closed spots %lu, offset %lu, block size %lu\n",
inode->id_qty, inode->key, inode->key_size));
inode = NULL;
if( blk > obj->buffer_sz ) {
obj->buffer_sz = blk;
}
blk = 0;
}
if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted ) {
rc = 0;
break;
}
if( inode == NULL ) {
spotid_t spotid = 0;
if( (rc = FastqReaderCurrentSpot(reader, &spotid)) != 0 ) {
break;
}
inode = malloc(sizeof(SIndexNode));
if( inode == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
break;
}
inode->key = obj->file_size;
inode->key_size = 0;
inode->id = spotid;
inode->id_qty = 0;
DEBUG_MSG(5, ("Fastq index opened spot %ld, offset %lu\n", inode->id, inode->key));
}
inode->id_qty++;
obj->file_size += written;
blk += written;
MD5StateAppend(&obj->md5, buffer, written);
if( g_dump ) {
fwrite(buffer, written, 1, stderr);
}
}
rc = rc ? rc : Quitting();
if( rc != 0 ) {
spotid_t spot = 0;
FastqReaderCurrentSpot(reader, &spot);
PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot));
}
}
FastqReaderWhack(reader);
return rc;
}
static
rc_t SFFGzip_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz)
{
rc_t rc = 0;
uint16_t zlib_ver = ZLIB_VERNUM;
const SFFReader* reader = NULL;
if( (rc = SFFReaderMake(&reader, sratbl, g_accession, obj->minSpotId, obj->maxSpotId)) != 0 ) {
return rc;
} else {
size_t written = 0;
uint32_t blk = 0, spots_per_block = 0, proj_id_qty = 0;
SIndexNode* inode = NULL;
size_t z_blk = 0;
size_t spots_buf_sz = g_file_block_sz * 100;
size_t zbuf_sz = spots_buf_sz + 100;
char* zbuf = malloc(zbuf_sz);
char* spots_buf = malloc(spots_buf_sz);
bool eof = false;
if( zbuf == NULL || spots_buf == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
}
while( rc == 0 ) {
if( (rc = SFFReader_GetNextSpotData(reader, buffer, buffer_sz, &written)) == 0 ) {
if( inode == NULL ) {
spotid_t spotid = 0;
if( (rc = SFFReaderCurrentSpot(reader, &spotid)) != 0 ) {
break;
}
inode = malloc(sizeof(SIndexNode));
if( inode == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
break;
}
inode->key = obj->file_size;
inode->key_size = 0;
inode->id = spotid;
inode->id_qty = 0;
DEBUG_MSG(5, ("%s open key: spot %ld, offset %lu\n", obj->index, inode->id, inode->key));
if( spotid == 1 ) {
char hd[10240];
size_t hd_sz = 0;
if( (rc = SFFReaderHeader(reader, 0, hd, sizeof(hd), &hd_sz)) == 0 ) {
if( hd_sz + written > spots_buf_sz ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient);
break;
}
memmove(&spots_buf[blk], hd, hd_sz);
blk += hd_sz;
if( g_dump ) {
fwrite(hd, hd_sz, 1, stderr);
}
}
}
}
if( blk + written > spots_buf_sz ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient);
break;
}
inode->id_qty++;
memmove(&spots_buf[blk], buffer, written);
blk += written;
if( g_dump ) {
fwrite(buffer, written, 1, stderr);
}
}
if( (eof = (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted)) ) {
rc = 0;
if( inode == NULL ) {
break;
}
}
if( rc == 0 && (eof ||
(proj_id_qty == 0 && inode->id_qty > (spots_per_block * 0.95)) ||
(proj_id_qty > 0 && inode->id_qty >= proj_id_qty) ) ) {
rc = ZLib_DeflateBlock(spots_buf, blk, zbuf, zbuf_sz, &z_blk);
if( z_blk < g_file_block_sz ) {
/* project needed id_qty */
proj_id_qty = g_file_block_sz * inode->id_qty / z_blk * 1.05;
DEBUG_MSG(5, ("%s: project id qty %lu\n", obj->index, proj_id_qty));
} else {
DEBUG_MSG(10, ("%s: no projection %lu > %lu\n", obj->index, z_blk, g_file_block_sz));
}
}
if( rc == 0 && (eof || z_blk >= g_file_block_sz) ) {
obj->file_size += z_blk;
MD5StateAppend(&obj->md5, zbuf, z_blk);
inode->key_size = z_blk;
SLListPushTail(&obj->li, &inode->n);
DEBUG_MSG(5, ("%s close key: spots %lu, size %lu, ratio %hu%%, raw %lu\n",
obj->index, inode->id_qty, inode->key_size, (uint16_t)(((float)(blk - z_blk)/blk)*100), blk));
spots_per_block = inode->id_qty;
inode = NULL;
if( blk > obj->buffer_sz ) {
obj->buffer_sz = blk;
}
blk = 0;
z_blk = 0;
proj_id_qty = 0;
}
if( eof ) {
break;
}
}
rc = rc ? rc : Quitting();
if( rc != 0 ) {
spotid_t spot = 0;
SFFReaderCurrentSpot(reader, &spot);
PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot));
}
free(zbuf);
free(spots_buf);
}
if( rc == 0 ) {
KMDataNode* opt = NULL, *nd = NULL;
if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) {
return rc;
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "ZlibVersion")) == 0 ) {
rc = KMDataNodeWriteB16(nd, &zlib_ver);
KMDataNodeRelease(nd);
}
KMDataNodeRelease(opt);
}
SFFReaderWhack(reader);
return rc;
}
static
rc_t SFF_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz)
{
rc_t rc = 0;
const SFFReader* reader = NULL;
if( (rc = SFFReaderMake(&reader, sratbl, g_accession, obj->minSpotId, obj->maxSpotId)) != 0 ) {
return rc;
} else {
size_t written = 0;
uint32_t blk = 0;
SIndexNode* inode = NULL;
while( rc == 0 ) {
rc = SFFReader_GetNextSpotData(reader, buffer, buffer_sz, &written);
if( blk >= g_file_block_sz || (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted) ) {
inode->key_size = blk;
SLListPushTail(&obj->li, &inode->n);
DEBUG_MSG(5, ("SFF index closed spots %lu, offset %lu, block size %lu\n", inode->id_qty, inode->key, inode->key_size));
inode = NULL;
if( blk > obj->buffer_sz ) {
obj->buffer_sz = blk;
}
blk = 0;
}
if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted ) {
rc = 0;
break;
}
if( inode == NULL ) {
spotid_t spotid = 0;
if( (rc = SFFReaderCurrentSpot(reader, &spotid)) != 0 ) {
break;
}
inode = malloc(sizeof(SIndexNode));
if( inode == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
break;
}
inode->key = obj->file_size;
inode->key_size = 0;
inode->id = spotid;
inode->id_qty = 0;
DEBUG_MSG(5, ("SFF index opened spot %ld, offset %lu\n", inode->id, inode->key));
if( spotid == 1 ) {
char hd[10240];
size_t hd_sz = 0;
if( (rc = SFFReaderHeader(reader, 0, hd, sizeof(hd), &hd_sz)) == 0 ) {
obj->file_size += hd_sz;
blk += hd_sz;
MD5StateAppend(&obj->md5, hd, hd_sz);
if( g_dump ) {
fwrite(hd, hd_sz, 1, stderr);
}
}
}
}
obj->file_size += written;
blk += written;
inode->id_qty++;
MD5StateAppend(&obj->md5, buffer, written);
if( g_dump ) {
fwrite(buffer, written, 1, stderr);
}
}
rc = rc ? rc : Quitting();
if( rc != 0 ) {
spotid_t spot = 0;
SFFReaderCurrentSpot(reader, &spot);
PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot));
}
}
SFFReaderWhack(reader);
return rc;
}
