static
void CalcMD5 (void * buffer, size_t size, char hexstr [32])
{
static const char to_hex[16] = "0123456789abcdef";
MD5State state;
uint8_t digest [16];
uint8_t * bin;
char * hex;
int ix;
MD5StateInit (&state);
MD5StateAppend (&state, buffer, size);
MD5StateFinish (&state, digest);
bin = digest;
hex = hexstr;
for (ix = 0; ix < sizeof digest; ++ix)
{
uint8_t upper;
uint8_t lower;
upper = *bin++;
lower = upper & 0xF;
upper >>= 4;
*hex++ = to_hex [upper];
*hex++ = to_hex[lower];
}
}
static
rc_t CC refseq_meta_stats( void *self, const VXformInfo *info, int64_t row_id,
VRowResult *rslt, uint32_t argc, const VRowData argv [] )
{
rc_t rc = 0;
KMDataNode* node;
refseq_meta_stats_data* data = self;
uint64_t i, seq_len = argv[0].u.data.elem_count;
const INSDC_4na_bin * seq = argv[0].u.data.base;
seq += argv[0].u.data.first_elem;
assert(data != NULL);
if( data->buf_sz < seq_len ) {
char* x = realloc(data->buf, seq_len);
if( x == NULL ) {
rc = RC(rcVDB, rcFunction, rcUpdating, rcMemory, rcExhausted);
} else {
data->buf = x;
data->buf_sz = seq_len;
}
}
for(i = 0; rc == 0 && i < seq_len; i++) {
data->buf[i] = INSDC_4na_map_CHARSET[seq[i]];
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "TOTAL_SEQ_LEN")) == 0 ) {
if( data->total_seq_len + seq_len < data->total_seq_len ) {
rc = RC(rcVDB, rcFunction, rcUpdating, rcMetadata, rcOutofrange);
} else {
data->total_seq_len += seq_len;
rc = KMDataNodeWriteB64(node, &data->total_seq_len);
}
KMDataNodeRelease(node);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "CRC32")) == 0 ) {
data->crc32 = CRC32(data->crc32, data->buf, seq_len);
rc = KMDataNodeWriteB32(node, &data->crc32);
KMDataNodeRelease(node);
}
if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(data->stats, &node, "MD5")) == 0 ) {
uint8_t digest[16];
MD5State md5;
MD5StateAppend(&data->md5, data->buf, seq_len);
memcpy(&md5, &data->md5, sizeof(md5));
MD5StateFinish(&md5, digest);
rc = KMDataNodeWrite(node, digest, sizeof(digest));
KMDataNodeRelease(node);
}
return rc;
}
static
rc_t KColumnBlobValidateMD5 ( const KColumnBlob *self )
{
rc_t rc;
const KColumn *col = self -> col;
uint8_t buffer [ 8 * 1024 ];
size_t to_read, num_read, total, size;
MD5State md5;
uint8_t digest [ 16 ];
MD5StateInit ( & md5 );
/* calculate checksum */
for ( size = self -> loc . u . blob . size, total = 0; total < size; total += num_read )
{
to_read = size - total;
if ( to_read > sizeof buffer )
to_read = sizeof buffer;
rc = KColumnDataRead ( & col -> df,
& self -> pmorig, total, buffer, to_read, & num_read );
if ( rc != 0 )
return rc;
if ( num_read == 0 )
return RC ( rcDB, rcBlob, rcValidating, rcTransfer, rcIncomplete );
MD5StateAppend ( & md5, buffer, num_read );
}
/* read stored checksum */
rc = KColumnDataRead ( & col -> df,
& self -> pmorig, size, buffer, sizeof digest, & num_read );
if ( rc != 0 )
return rc;
if ( num_read != sizeof digest )
return RC ( rcDB, rcBlob, rcValidating, rcTransfer, rcIncomplete );
/* finish MD5 digest */
MD5StateFinish ( & md5, digest );
if ( memcmp ( buffer, digest, sizeof digest ) != 0 )
return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt );
return 0;
}
static rc_t read_loop( const KFile * f, const uint64_t a_pos, const uint64_t count,
const size_t chunk_size, uint8_t digest [ 16 ] )
{
rc_t rc = 0;
uint64_t pos = a_pos;
uint64_t n_bytes = 0;
size_t num_read = 1;
MD5State md5;
char * buffer = malloc( chunk_size );
if ( buffer == NULL )
return RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted );
MD5StateInit ( &md5 );
while ( rc == 0 && num_read > 0 )
{
size_t chunk = chunk_size;
if ( ( count > 0 ) && ( ( n_bytes + chunk ) > count ) )
{
chunk = ( count - n_bytes );
}
OUTMSG(( "about to read from pos %lu\n", pos ));
rc = KFileRead ( f, pos, buffer, chunk, &num_read );
OUTMSG(( "returned from KFileRead rc = %R, num_read = %zu\n\n", rc, num_read ));
if ( rc == 0 && num_read > 0 )
{
MD5StateAppend ( &md5, buffer, num_read );
pos += num_read;
n_bytes += num_read;
if ( ( count > 0 ) && ( n_bytes >= count ) )
{
num_read = 0;
}
}
}
OUTMSG(( "%lu bytes read total\n", n_bytes ));
free( buffer );
MD5StateFinish ( &md5, digest );
return rc;
}
static
rc_t KColumnBlobValidateBufferMD5 ( const void * buffer, size_t size, const uint8_t cs [ 16 ] )
{
MD5State md5;
uint8_t digest [ 16 ];
MD5StateInit ( & md5 );
/* calculate checksum */
MD5StateAppend ( & md5, buffer, size );
/* finish MD5 digest */
MD5StateFinish ( & md5, digest );
if ( memcmp ( cs, digest, sizeof digest ) != 0 )
return RC ( rcDB, rcBlob, rcValidating, rcBlob, rcCorrupt );
return 0;
}
rc_t KRngSysEntropy (KRng * self, uint8_t * buffer, size_t buff_size)
{
MD5State state;
KDirectory * dir;
const KFile * file;
uint64_t file_pos;
rc_t rc;
if (self == NULL)
return RC (rcKrypto, rcRng, rcWriting, rcSelf, rcNull);
if (buffer == NULL)
return RC (rcKrypto, rcRng, rcWriting, rcParam, rcNull);
rc = KDirectoryNativeDir (&dir);
if (rc == 0)
{
rc = KDirectoryOpenFileRead (dir, &file, "/dev/random");
if (rc)
file = NULL;
}
file_pos = 0;
MD5StateInit (&state);
while (buff_size > 0)
{
if (file)
{
size_t to_read;
size_t num_read;
to_read = (buff_size > 16) ? 16 : buff_size;
rc = KFileRead (file, file_pos, buffer, to_read, &num_read);
if (rc == 0)
{
buffer += num_read;
buff_size -= num_read;
file_pos += num_read; /* superfluous? */
}
}
if (buff_size > 0)
{
uint8_t digest [16];
char buff [1024];
size_t ii;
string_printf (buff, sizeof (buff), &ii,
"%p%zu%lu%lu%lu%lu%s%s%lu",
buffer, buff_size, (uint64_t)clock(),
(uint64_t)time(NULL), (uint64_t)getpid(),
(uint64_t)getuid(), getlogin(), ttyname(0),
(uint64_t)getgid());
MD5StateAppend (&state, buff, sizeof buff);
MD5StateFinish (&state, digest);
ii = (buff_size < sizeof (digest)) ? buff_size : sizeof (digest);
memcpy (buffer, digest, ii);
buff_size -= ii;
buffer += ii;
}
}
KFileRelease (file);
KDirectoryRelease (dir);
return 0;
}
static
rc_t MakeIndexes(const SRATable* stbl, KTable* ktbl, KMetadata* meta)
{
rc_t rc = 0;
int i;
char* buffer = NULL;
size_t buffer_sz = g_file_block_sz * 100;
SIndexObj idx[] = {
/* meta, file, format, index, func, file_size, buffer_sz, minSpotId, maxSpotId */
{NULL, "fastq", "fastq", "fuse-fastq", Fastq_Idx, 0, 0, 0, 0},
{NULL, "sff", "SFF", "fuse-sff", SFF_Idx, 0, 0, 0, 0},
{NULL, "fastq.gz", "fastq-gzip", "fuse-fastq-gz", FastqGzip_Idx, 0, 0, 0, 0},
{NULL, "sff.gz", "SFF-gzip", "fuse-sff-gz", SFFGzip_Idx, 0, 0, 0, 0}
};
for(i = 0; rc == 0 && i < sizeof(idx) / sizeof(idx[0]); i++) {
KMDataNode* parent = NULL;
if( (rc = KMetadataOpenNodeUpdate(meta, &parent, "/FUSE")) == 0 ) {
KMDataNodeDropChild(parent, "root"); /* drop old stuff */
if( g_ungzip || strcmp(&idx[i].file[strlen(idx[i].file) - 3], ".gz") == 0 ) {
STSMSG(0, ("Preparing index %s", idx[i].index));
MD5StateInit(&idx[i].md5);
SLListInit(&idx[i].li);
KMDataNodeDropChild(parent, "%s.tmp", idx[i].file);
if( (rc = KMDataNodeOpenNodeUpdate(parent, &idx[i].meta, "%s.tmp", idx[i].file)) == 0 ) {
if( idx[i].func != NULL ) {
if( buffer == NULL ) {
if( (buffer = malloc(buffer_sz)) == NULL ) {
rc = RC(rcExe, rcIndex, rcConstructing, rcMemory, rcExhausted);
break;
}
}
rc = idx[i].func(stbl, &idx[i], buffer, buffer_sz);
if( rc == 0 ) {
MD5StateFinish(&idx[i].md5, idx[i].md5_digest);
rc = CommitIndex(ktbl, idx[i].index, &idx[i].li);
}
}
if( rc == 0 ) {
rc = WriteFileMeta(&idx[i]);
}
KMDataNodeRelease(idx[i].meta);
}
if( GetRCState(rc) == rcUnsupported ) {
KMDataNodeDropChild(parent, "%s", idx[i].file);
PLOGERR(klogWarn, (klogWarn, rc, "Index $(i) is not supported for this table", PLOG_S(i), idx[i].index));
rc = 0;
} else if( rc == 0 ) {
char f[4096];
strcpy(f, idx[i].file);
strcat(f, ".tmp");
KMDataNodeDropChild(parent, "%s", idx[i].file);
rc = KMDataNodeRenameChild(parent, f, idx[i].file);
}
} else if( !g_ungzip ) {
KTableDropIndex(ktbl, idx[i].index);
KMDataNodeDropChild(parent, "%s", idx[i].file);
}
KMDataNodeDropChild(parent, "%s.tmp", idx[i].file);
KMDataNodeRelease(parent);
}
SLListWhack(&idx[i].li, WhackIndexData, NULL);
}
free(buffer);
return rc;
}