static void CalcMD5 (void * buffer, size_t size, char hexstr [32]) { static const char to_hex[16] = "0123456789abcdef"; MD5State state; uint8_t digest [16]; uint8_t * bin; char * hex; int ix; MD5StateInit (&state); MD5StateAppend (&state, buffer, size); MD5StateFinish (&state, digest); bin = digest; hex = hexstr; for (ix = 0; ix < sizeof digest; ++ix) { uint8_t upper; uint8_t lower; upper = *bin++; lower = upper & 0xF; upper >>= 4; *hex++ = to_hex [upper]; *hex++ = to_hex[lower]; } }
static rc_t CC refseq_meta_stats( void *self, const VXformInfo *info, int64_t row_id, VRowResult *rslt, uint32_t argc, const VRowData argv [] ) { rc_t rc = 0; KMDataNode* node; refseq_meta_stats_data* data = self; uint64_t i, seq_len = argv[0].u.data.elem_count; const INSDC_4na_bin * seq = argv[0].u.data.base; seq += argv[0].u.data.first_elem; assert(data != NULL); if( data->buf_sz < seq_len ) { char* x = realloc(data->buf, seq_len); if( x == NULL ) { rc = RC(rcVDB, rcFunction, rcUpdating, rcMemory, rcExhausted); } else { data->buf = x; data->buf_sz = seq_len; } } for(i = 0; rc == 0 && i < seq_len; i++) { data->buf[i] = INSDC_4na_map_CHARSET[seq[i]]; } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "TOTAL_SEQ_LEN")) == 0 ) { if( data->total_seq_len + seq_len < data->total_seq_len ) { rc = RC(rcVDB, rcFunction, rcUpdating, rcMetadata, rcOutofrange); } else { data->total_seq_len += seq_len; rc = KMDataNodeWriteB64(node, &data->total_seq_len); } KMDataNodeRelease(node); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "CRC32")) == 0 ) { data->crc32 = CRC32(data->crc32, data->buf, seq_len); rc = KMDataNodeWriteB32(node, &data->crc32); KMDataNodeRelease(node); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "MD5")) == 0 ) { uint8_t digest[16]; MD5State md5; MD5StateAppend(&data->md5, data->buf, seq_len); memcpy(&md5, &data->md5, sizeof(md5)); MD5StateFinish(&md5, digest); rc = KMDataNodeWrite(node, digest, sizeof(digest)); KMDataNodeRelease(node); } return rc; }
static rc_t KColumnBlobValidateMD5 ( const KColumnBlob *self ) { rc_t rc; const KColumn *col = self -> col; uint8_t buffer [ 8 * 1024 ]; size_t to_read, num_read, total, size; MD5State md5; uint8_t digest [ 16 ]; MD5StateInit ( & md5 ); /* calculate checksum */ for ( size = self -> loc . u . blob . size, total = 0; total < size; total += num_read ) { to_read = size - total; if ( to_read > sizeof buffer ) to_read = sizeof buffer; rc = KColumnDataRead ( & col -> df, & self -> pmorig, total, buffer, to_read, & num_read ); if ( rc != 0 ) return rc; if ( num_read == 0 ) return RC ( rcDB, rcBlob, rcValidating, rcTransfer, rcIncomplete ); MD5StateAppend ( & md5, buffer, num_read ); } /* read stored checksum */ rc = KColumnDataRead ( & col -> df, & self -> pmorig, size, buffer, sizeof digest, & num_read ); if ( rc != 0 ) return rc; if ( num_read != sizeof digest ) return RC ( rcDB, rcBlob, rcValidating, rcTransfer, rcIncomplete ); /* finish MD5 digest */ MD5StateFinish ( & md5, digest ); if ( memcmp ( buffer, digest, sizeof digest ) != 0 ) return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); return 0; }
static rc_t read_loop( const KFile * f, const uint64_t a_pos, const uint64_t count, const size_t chunk_size, uint8_t digest [ 16 ] ) { rc_t rc = 0; uint64_t pos = a_pos; uint64_t n_bytes = 0; size_t num_read = 1; MD5State md5; char * buffer = malloc( chunk_size ); if ( buffer == NULL ) return RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted ); MD5StateInit ( &md5 ); while ( rc == 0 && num_read > 0 ) { size_t chunk = chunk_size; if ( ( count > 0 ) && ( ( n_bytes + chunk ) > count ) ) { chunk = ( count - n_bytes ); } OUTMSG(( "about to read from pos %lu\n", pos )); rc = KFileRead ( f, pos, buffer, chunk, &num_read ); OUTMSG(( "returned from KFileRead rc = %R, num_read = %zu\n\n", rc, num_read )); if ( rc == 0 && num_read > 0 ) { MD5StateAppend ( &md5, buffer, num_read ); pos += num_read; n_bytes += num_read; if ( ( count > 0 ) && ( n_bytes >= count ) ) { num_read = 0; } } } OUTMSG(( "%lu bytes read total\n", n_bytes )); free( buffer ); MD5StateFinish ( &md5, digest ); return rc; }
static rc_t KColumnBlobValidateBufferMD5 ( const void * buffer, size_t size, const uint8_t cs [ 16 ] ) { MD5State md5; uint8_t digest [ 16 ]; MD5StateInit ( & md5 ); /* calculate checksum */ MD5StateAppend ( & md5, buffer, size ); /* finish MD5 digest */ MD5StateFinish ( & md5, digest ); if ( memcmp ( cs, digest, sizeof digest ) != 0 ) return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt ); return 0; }
rc_t KRngSysEntropy (KRng * self, uint8_t * buffer, size_t buff_size) { MD5State state; KDirectory * dir; const KFile * file; uint64_t file_pos; rc_t rc; if (self == NULL) return RC (rcKrypto, rcRng, rcWriting, rcSelf, rcNull); if (buffer == NULL) return RC (rcKrypto, rcRng, rcWriting, rcParam, rcNull); rc = KDirectoryNativeDir (&dir); if (rc == 0) { rc = KDirectoryOpenFileRead (dir, &file, "/dev/random"); if (rc) file = NULL; } file_pos = 0; MD5StateInit (&state); while (buff_size > 0) { if (file) { size_t to_read; size_t num_read; to_read = (buff_size > 16) ? 16 : buff_size; rc = KFileRead (file, file_pos, buffer, to_read, &num_read); if (rc == 0) { buffer += num_read; buff_size -= num_read; file_pos += num_read; /* superfluous? */ } } if (buff_size > 0) { uint8_t digest [16]; char buff [1024]; size_t ii; string_printf (buff, sizeof (buff), &ii, "%p%zu%lu%lu%lu%lu%s%s%lu", buffer, buff_size, (uint64_t)clock(), (uint64_t)time(NULL), (uint64_t)getpid(), (uint64_t)getuid(), getlogin(), ttyname(0), (uint64_t)getgid()); MD5StateAppend (&state, buff, sizeof buff); MD5StateFinish (&state, digest); ii = (buff_size < sizeof (digest)) ? buff_size : sizeof (digest); memcpy (buffer, digest, ii); buff_size -= ii; buffer += ii; } } KFileRelease (file); KDirectoryRelease (dir); return 0; }
static rc_t FastqGzip_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz) { rc_t rc = 0; const FastqReader* reader = NULL; uint16_t zlib_ver = ZLIB_VERNUM; uint8_t colorSpace = false; char* colorSpaceKey = "\0"; uint8_t origFormat = false; uint8_t printLabel = true; uint8_t printReadId = true; uint8_t clipQuality = true; uint32_t minReadLen = 0; uint16_t qualityOffset = 0; {{ const SRAColumn* c = NULL; const uint8_t *platform = SRA_PLATFORM_UNDEFINED; bitsz_t o, z; if( (rc = SRATableOpenColumnRead(sratbl, &c, "PLATFORM", sra_platform_id_t)) != 0 ) { return rc; } if( (rc = SRAColumnRead(c, 1, (const void **)&platform, &o, &z)) != 0 ) { return rc; } if( *platform == SRA_PLATFORM_ABSOLID ) { colorSpace = true; } SRAColumnRelease(c); }} if( (rc = FastqReaderMake(&reader, sratbl, g_accession, colorSpace, origFormat, false, printLabel, printReadId, !clipQuality, minReadLen, qualityOffset, colorSpaceKey[0], obj->minSpotId, obj->maxSpotId)) != 0 ) { return rc; } else { size_t written = 0; uint32_t blk = 0, spots_per_block = 0, proj_id_qty = 0; SIndexNode* inode = NULL; size_t z_blk = 0; size_t spots_buf_sz = g_file_block_sz * 100; size_t zbuf_sz = spots_buf_sz + 100; char* zbuf = malloc(zbuf_sz); char* spots_buf = malloc(spots_buf_sz); bool eof = false; if( zbuf == NULL || spots_buf == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); } while( rc == 0 ) { if( (rc = FastqReader_GetNextSpotSplitData(reader, buffer, buffer_sz, &written)) == 0 ) { if( inode == NULL ) { spotid_t spotid = 0; if( (rc = FastqReaderCurrentSpot(reader, &spotid)) != 0 ) { break; } inode = malloc(sizeof(SIndexNode)); if( inode == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); break; } inode->key = obj->file_size; inode->key_size = 0; inode->id = spotid; inode->id_qty = 0; DEBUG_MSG(5, ("%s open key: spot %ld, offset %lu\n", obj->index, inode->id, inode->key)); } if( blk + written > spots_buf_sz ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient); break; } inode->id_qty++; memmove(&spots_buf[blk], buffer, written); blk += written; if( g_dump ) { fwrite(buffer, written, 1, stderr); } } if( (eof = (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted)) ) { rc = 0; if( inode == NULL ) { break; } } if( rc == 0 && (eof || (proj_id_qty == 0 && inode->id_qty > (spots_per_block * 0.95)) || (proj_id_qty > 0 && inode->id_qty >= proj_id_qty) ) ) { rc = ZLib_DeflateBlock(spots_buf, blk, zbuf, zbuf_sz, &z_blk); if( z_blk < g_file_block_sz ) { /* project needed id_qty */ proj_id_qty = g_file_block_sz * inode->id_qty / z_blk * 1.05; DEBUG_MSG(5, ("%s: project id qty %u\n", obj->index, proj_id_qty)); } else { DEBUG_MSG(10, ("%s: no projection %u > %u\n", obj->index, z_blk, g_file_block_sz)); } } if( rc == 0 && (eof || z_blk >= g_file_block_sz) ) { obj->file_size += z_blk; MD5StateAppend(&obj->md5, zbuf, z_blk); inode->key_size = z_blk; SLListPushTail(&obj->li, &inode->n); DEBUG_MSG(5, ("%s close key: spots %lu, size %lu, ratio %hu%%, raw %u\n", obj->index, inode->id_qty, inode->key_size, (uint16_t)(((float)(blk - z_blk)/blk)*100), blk )); spots_per_block = inode->id_qty; inode = NULL; if( blk > obj->buffer_sz ) { obj->buffer_sz = blk; } blk = 0; z_blk = 0; proj_id_qty = 0; } if( eof ) { break; } } rc = rc ? rc : Quitting(); if( rc != 0 ) { spotid_t spot = 0; FastqReaderCurrentSpot(reader, &spot); PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot)); } free(zbuf); free(spots_buf); } if( rc == 0 ) { KMDataNode* opt = NULL, *nd = NULL; if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) { return rc; } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "ZlibVersion")) == 0 ) { rc = KMDataNodeWriteB16(nd, &zlib_ver); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpace")) == 0 ) { rc = KMDataNodeWriteB8(nd, &colorSpace); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpaceKey")) == 0 ) { rc = KMDataNodeWrite(nd, colorSpaceKey, 1); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "origFormat")) == 0 ) { rc = KMDataNodeWriteB8(nd, &origFormat); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printLabel")) == 0 ) { rc = KMDataNodeWriteB8(nd, &printLabel); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printReadId")) == 0 ) { rc = KMDataNodeWriteB8(nd, &printReadId); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "clipQuality")) == 0 ) { rc = KMDataNodeWriteB8(nd, &clipQuality); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "minReadLen")) == 0 ) { rc = KMDataNodeWriteB32(nd, &minReadLen); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "qualityOffset")) == 0 ) { rc = KMDataNodeWriteB16(nd, &qualityOffset); KMDataNodeRelease(nd); } KMDataNodeRelease(opt); } FastqReaderWhack(reader); return rc; }
static rc_t Fastq_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz) { rc_t rc = 0; const FastqReader* reader = NULL; uint8_t colorSpace = false; char* colorSpaceKey = "\0"; uint8_t origFormat = false; uint8_t printLabel = true; uint8_t printReadId = true; uint8_t clipQuality = true; uint32_t minReadLen = 0; uint16_t qualityOffset = 0; {{ const SRAColumn* c = NULL; const uint8_t *platform = SRA_PLATFORM_UNDEFINED; bitsz_t o, z; if( (rc = SRATableOpenColumnRead(sratbl, &c, "PLATFORM", sra_platform_id_t)) != 0 ) { return rc; } if( (rc = SRAColumnRead(c, 1, (const void **)&platform, &o, &z)) != 0 ) { return rc; } if( *platform == SRA_PLATFORM_ABSOLID ) { colorSpace = true; } SRAColumnRelease(c); }} if( (rc = FastqReaderMake(&reader, sratbl, g_accession, colorSpace, origFormat, false, printLabel, printReadId, !clipQuality, minReadLen, qualityOffset, colorSpaceKey[0], obj->minSpotId, obj->maxSpotId)) != 0 ) { return rc; } else { KMDataNode* opt = NULL, *nd = NULL; if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) { return rc; } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpace")) == 0 ) { rc = KMDataNodeWriteB8(nd, &colorSpace); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "colorSpaceKey")) == 0 ) { rc = KMDataNodeWrite(nd, colorSpaceKey, 1); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "origFormat")) == 0 ) { rc = KMDataNodeWriteB8(nd, &origFormat); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printLabel")) == 0 ) { rc = KMDataNodeWriteB8(nd, &printLabel); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "printReadId")) == 0 ) { rc = KMDataNodeWriteB8(nd, &printReadId); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "clipQuality")) == 0 ) { rc = KMDataNodeWriteB8(nd, &clipQuality); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "minReadLen")) == 0 ) { rc = KMDataNodeWriteB32(nd, &minReadLen); KMDataNodeRelease(nd); } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "qualityOffset")) == 0 ) { rc = KMDataNodeWriteB16(nd, &qualityOffset); KMDataNodeRelease(nd); } KMDataNodeRelease(opt); } if( rc == 0 ) { size_t written = 0; uint32_t blk = 0; SIndexNode* inode = NULL; while( rc == 0 ) { rc = FastqReader_GetNextSpotSplitData(reader, buffer, buffer_sz, &written); if( blk >= g_file_block_sz || (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted) ) { inode->key_size = blk; SLListPushTail(&obj->li, &inode->n); DEBUG_MSG(5, ("Fastq index closed spots %lu, offset %lu, block size %lu\n", inode->id_qty, inode->key, inode->key_size)); inode = NULL; if( blk > obj->buffer_sz ) { obj->buffer_sz = blk; } blk = 0; } if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted ) { rc = 0; break; } if( inode == NULL ) { spotid_t spotid = 0; if( (rc = FastqReaderCurrentSpot(reader, &spotid)) != 0 ) { break; } inode = malloc(sizeof(SIndexNode)); if( inode == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); break; } inode->key = obj->file_size; inode->key_size = 0; inode->id = spotid; inode->id_qty = 0; DEBUG_MSG(5, ("Fastq index opened spot %ld, offset %lu\n", inode->id, inode->key)); } inode->id_qty++; obj->file_size += written; blk += written; MD5StateAppend(&obj->md5, buffer, written); if( g_dump ) { fwrite(buffer, written, 1, stderr); } } rc = rc ? rc : Quitting(); if( rc != 0 ) { spotid_t spot = 0; FastqReaderCurrentSpot(reader, &spot); PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot)); } } FastqReaderWhack(reader); return rc; }
static rc_t SFFGzip_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz) { rc_t rc = 0; uint16_t zlib_ver = ZLIB_VERNUM; const SFFReader* reader = NULL; if( (rc = SFFReaderMake(&reader, sratbl, g_accession, obj->minSpotId, obj->maxSpotId)) != 0 ) { return rc; } else { size_t written = 0; uint32_t blk = 0, spots_per_block = 0, proj_id_qty = 0; SIndexNode* inode = NULL; size_t z_blk = 0; size_t spots_buf_sz = g_file_block_sz * 100; size_t zbuf_sz = spots_buf_sz + 100; char* zbuf = malloc(zbuf_sz); char* spots_buf = malloc(spots_buf_sz); bool eof = false; if( zbuf == NULL || spots_buf == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); } while( rc == 0 ) { if( (rc = SFFReader_GetNextSpotData(reader, buffer, buffer_sz, &written)) == 0 ) { if( inode == NULL ) { spotid_t spotid = 0; if( (rc = SFFReaderCurrentSpot(reader, &spotid)) != 0 ) { break; } inode = malloc(sizeof(SIndexNode)); if( inode == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); break; } inode->key = obj->file_size; inode->key_size = 0; inode->id = spotid; inode->id_qty = 0; DEBUG_MSG(5, ("%s open key: spot %ld, offset %lu\n", obj->index, inode->id, inode->key)); if( spotid == 1 ) { char hd[10240]; size_t hd_sz = 0; if( (rc = SFFReaderHeader(reader, 0, hd, sizeof(hd), &hd_sz)) == 0 ) { if( hd_sz + written > spots_buf_sz ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient); break; } memmove(&spots_buf[blk], hd, hd_sz); blk += hd_sz; if( g_dump ) { fwrite(hd, hd_sz, 1, stderr); } } } } if( blk + written > spots_buf_sz ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcInsufficient); break; } inode->id_qty++; memmove(&spots_buf[blk], buffer, written); blk += written; if( g_dump ) { fwrite(buffer, written, 1, stderr); } } if( (eof = (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted)) ) { rc = 0; if( inode == NULL ) { break; } } if( rc == 0 && (eof || (proj_id_qty == 0 && inode->id_qty > (spots_per_block * 0.95)) || (proj_id_qty > 0 && inode->id_qty >= proj_id_qty) ) ) { rc = ZLib_DeflateBlock(spots_buf, blk, zbuf, zbuf_sz, &z_blk); if( z_blk < g_file_block_sz ) { /* project needed id_qty */ proj_id_qty = g_file_block_sz * inode->id_qty / z_blk * 1.05; DEBUG_MSG(5, ("%s: project id qty %lu\n", obj->index, proj_id_qty)); } else { DEBUG_MSG(10, ("%s: no projection %lu > %lu\n", obj->index, z_blk, g_file_block_sz)); } } if( rc == 0 && (eof || z_blk >= g_file_block_sz) ) { obj->file_size += z_blk; MD5StateAppend(&obj->md5, zbuf, z_blk); inode->key_size = z_blk; SLListPushTail(&obj->li, &inode->n); DEBUG_MSG(5, ("%s close key: spots %lu, size %lu, ratio %hu%%, raw %lu\n", obj->index, inode->id_qty, inode->key_size, (uint16_t)(((float)(blk - z_blk)/blk)*100), blk)); spots_per_block = inode->id_qty; inode = NULL; if( blk > obj->buffer_sz ) { obj->buffer_sz = blk; } blk = 0; z_blk = 0; proj_id_qty = 0; } if( eof ) { break; } } rc = rc ? rc : Quitting(); if( rc != 0 ) { spotid_t spot = 0; SFFReaderCurrentSpot(reader, &spot); PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot)); } free(zbuf); free(spots_buf); } if( rc == 0 ) { KMDataNode* opt = NULL, *nd = NULL; if( (rc = KMDataNodeOpenNodeUpdate(obj->meta, &opt, "Format/Options")) != 0 ) { return rc; } if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &nd, "ZlibVersion")) == 0 ) { rc = KMDataNodeWriteB16(nd, &zlib_ver); KMDataNodeRelease(nd); } KMDataNodeRelease(opt); } SFFReaderWhack(reader); return rc; }
static rc_t SFF_Idx(const SRATable* sratbl, SIndexObj* obj, char* buffer, const size_t buffer_sz) { rc_t rc = 0; const SFFReader* reader = NULL; if( (rc = SFFReaderMake(&reader, sratbl, g_accession, obj->minSpotId, obj->maxSpotId)) != 0 ) { return rc; } else { size_t written = 0; uint32_t blk = 0; SIndexNode* inode = NULL; while( rc == 0 ) { rc = SFFReader_GetNextSpotData(reader, buffer, buffer_sz, &written); if( blk >= g_file_block_sz || (GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted) ) { inode->key_size = blk; SLListPushTail(&obj->li, &inode->n); DEBUG_MSG(5, ("SFF index closed spots %lu, offset %lu, block size %lu\n", inode->id_qty, inode->key, inode->key_size)); inode = NULL; if( blk > obj->buffer_sz ) { obj->buffer_sz = blk; } blk = 0; } if( GetRCObject(rc) == rcRow && GetRCState(rc) == rcExhausted ) { rc = 0; break; } if( inode == NULL ) { spotid_t spotid = 0; if( (rc = SFFReaderCurrentSpot(reader, &spotid)) != 0 ) { break; } inode = malloc(sizeof(SIndexNode)); if( inode == NULL ) { rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted); break; } inode->key = obj->file_size; inode->key_size = 0; inode->id = spotid; inode->id_qty = 0; DEBUG_MSG(5, ("SFF index opened spot %ld, offset %lu\n", inode->id, inode->key)); if( spotid == 1 ) { char hd[10240]; size_t hd_sz = 0; if( (rc = SFFReaderHeader(reader, 0, hd, sizeof(hd), &hd_sz)) == 0 ) { obj->file_size += hd_sz; blk += hd_sz; MD5StateAppend(&obj->md5, hd, hd_sz); if( g_dump ) { fwrite(hd, hd_sz, 1, stderr); } } } } obj->file_size += written; blk += written; inode->id_qty++; MD5StateAppend(&obj->md5, buffer, written); if( g_dump ) { fwrite(buffer, written, 1, stderr); } } rc = rc ? rc : Quitting(); if( rc != 0 ) { spotid_t spot = 0; SFFReaderCurrentSpot(reader, &spot); PLOGERR(klogErr, (klogErr, rc, "spot $(s)", PLOG_U32(s), spot)); } } SFFReaderWhack(reader); return rc; }