static
rc_t stats_data_update_group(stats_data_t *self,
int64_t spot_id,
uint32_t spot_len,
uint32_t bio_spot_len,
uint32_t cmp_spot_len,
char const grp[],
uint64_t grp_len)
{
if (grp_len == 0 || grp == NULL || grp[0] == '\0' ||
(grp_len == 7 && strncasecmp("default", grp, 7) == 0))
{
group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len);
}
else
{
group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp);
if (fnd == NULL)
return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted);
if (fnd - get_group(self, 0) < MAX_GROUP_COUNT)
group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len);
else {
KDataBufferWhack(&self->group);
KDataBufferWhack(&self->names);
stats_data_init_funcs(self, false);
(void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count)));
}
}
return 0;
}
static
void CC stats_data_whack(void *const data)
{
stats_data_t *const self = data;
self->write_all(self);
KDataBufferWhack(&self->group);
KDataBufferWhack(&self->names);
KMetadataRelease(self->meta);
free(self);
}
static rc_t TableWriterSeq_WriteStatistics(TableWriterSeq const *cself, KMDataNode *node)
{
pb_t pb;
rc_t rc;
KDataBuffer buf;
rc = KDataBufferMake(&buf, 8 * sizeof(pb.stats[0]), cself->statsCount);
if (rc) return rc;
pb.stats = buf.base;
pb.i = 0;
rc = KVectorVisitU64(cself->stats, 0, stats_cb, &pb);
if (rc == 0) {
unsigned i;
unsigned const n = cself->statsCount < 126 ? cself->statsCount : 126;
uint64_t *const distance = buf.base;
ksort(pb.stats, cself->statsCount, sizeof(pb.stats[0]), stats_cmp_count, NULL);
ksort(pb.stats, n, sizeof(pb.stats[0]), stats_cmp_distance, NULL);
for (i = 0; i != n; ++i) {
distance[i] = pb.stats[i].distance;
}
rc = KMDataNodeWrite(node, distance, n * sizeof(distance[0]));
}
KDataBufferWhack(&buf);
return rc;
}
static rc_t KCurlRequestDestroy( struct KCurlRequest *self )
{
rc_t rc;
self->kns_mgr->curl_easy_cleanup_fkt( self->curl_handle );
rc = KNSManagerRelease( self->kns_mgr );
KDataBufferWhack( &self->fields );
free ( self );
return rc;
}
void destroy_fastq_sra_iter( struct fastq_sra_iter * self )
{
if ( self != NULL )
{
destroy_cmn_iter( self -> cmn );
if ( self -> qual_buffer . base != NULL )
KDataBufferWhack( &self -> qual_buffer );
free( ( void * ) self );
}
}
static
void Unsorted(Reference *self) {
(void)LOGMSG(klogWarn, "Alignments are unsorted");
self->out_of_order = true;
ReferenceMgr_SetCache(self->mgr, UNSORTED_CACHE_SIZE, UNSORTED_OPEN_TABLE_LIMIT);
KDataBufferWhack(&self->sec_align);
KDataBufferWhack(&self->pri_align);
KDataBufferWhack(&self->mismatches);
KDataBufferWhack(&self->indels);
KDataBufferWhack(&self->coverage);
KDataBufferWhack(&self->pri_overlap);
KDataBufferWhack(&self->sec_overlap);
}
/* Whack
*/
rc_t KColumnIdx2Whack ( KColumnIdx2 *self )
{
rc_t rc = KFileRelease ( self -> f );
if ( rc == 0 )
{
int i;
KColumnIdx2BlockCache * cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
self -> f = NULL;
for(i=0;i<self->cstorage.elem_count;i++){
free(cache[i].block);
}
KDataBufferWhack(&self->cstorage);
}
return rc;
}
static rc_t FastqRecordWhack( const FastqRecord* cself )
{
rc_t rc = 0;
FastqRecord* self = (FastqRecord*)cself;
assert(self);
rc = KDataBufferWhack( & self->source );
if (rc)
RejectedRelease(self->rej);
else
rc = RejectedRelease(self->rej);
free(self);
return rc;
}
static
rc_t CC f32zip_func(
void *Self,
const VXformInfo *info,
VBlobResult *dst,
const VBlobData *Src,
VBlobHeader *hdr
) {
rc_t rc;
const struct self_t *self = Self;
uint64_t dsize = (dst->elem_count * dst->elem_bits + 7) >> 3;
uint64_t element_count = (Src->elem_count * Src->elem_bits) >> 5;
uint64_t man_bytes;
KDataBuffer scratch;
assert(element_count >> 32 == 0);
assert(dsize >> 32 == 0);
rc = KDataBufferMakeBytes(&scratch, element_count << 2);
if (rc)
return rc;
VBlobHeaderSetVersion(hdr, 0);
VBlobHeaderOpPushTail(hdr, 0);
VBlobHeaderArgPushTail(hdr, self->mantissa);
man_bytes = split_and_pack_f32(
Src->data, (uint32_t)element_count,
self->mantissa,
scratch.base);
assert((element_count + man_bytes) >> 32 == 0);
rc = invoke_zlib(dst->data, &dsize, scratch.base, (uint32_t)(element_count + man_bytes), Z_RLE, Z_BEST_SPEED);
KDataBufferWhack(&scratch);
if (rc == 0) {
dst->elem_bits = 1;
dst->byte_order = vboNone;
if (dsize)
dst->elem_count = dsize << 3;
else
rc = RC(rcXF, rcFunction, rcExecuting, rcBuffer, rcInsufficient);
}
return rc;
}
/* InitDNSEndpoint
* initialize the endpoint with a DNS name and a port number
*
* "ep" [ OUT ] - address of endpoint block to be intialized
*
* "dns" [ IN ] - textual DNS address.
*
* "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
* if the special port number 0 is given, it represents any available port.
*/
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
else if ( dns == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else if ( dns -> size == 0 )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
KDataBuffer b;
char buffer [ 4096 ], * hostname = buffer;
size_t buff_size = sizeof buffer;
if ( dns -> size >= sizeof buffer )
{
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
hostname = b . base;
buff_size = ( size_t ) b . elem_count;
}
}
if ( rc == 0 )
{
size_t size;
rc = string_printf ( hostname, buff_size, & size, "%S", dns );
assert ( rc == 0 );
assert ( size < buff_size );
assert ( hostname [ size ] == 0 );
if ( rc == 0 )
{
struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
{
ep -> type = epIPV4;
memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
}
else switch ( h_errno )
{
case HOST_NOT_FOUND: /* The specified host is unknown */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
break;
case NO_ADDRESS: /* The requested names valid but does not have an IP address */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcInconsistent );
break;
#if ! defined NO_ADDRESS || ! defined NO_DATA || NO_ADDRESS != NO_DATA
case NO_DATA: /* The requested name s valid but does not have an IP address */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
break;
#endif
case NO_RECOVERY: /* A nonrecoverable name server error occured */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcDestroyed );
break;
case TRY_AGAIN: /* A temporary error occured on an authoritative name server. Try again later */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
break;
default :
rc = RC ( rcNS, rcNoTarg, rcValidating, rcError, rcUnknown );
}
}
}
if ( hostname != buffer )
KDataBufferWhack ( & b );
}
if ( rc != 0 )
memset ( ep, 0, sizeof * ep );
}
return rc;
}
static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self,
SraReleaseVersion *newVersion)
{
rc_t rc = 0;
KDataBuffer result;
KHttpRequest *req = NULL;
KHttpResult *rslt = NULL;
if (newVersion == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull);
}
memset(newVersion, 0, sizeof *newVersion);
if (self == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull);
}
memset(&result, 0, sizeof result);
if (rc == 0) {
rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL,
"https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
);
}
if (rc == 0) {
rc = KHttpRequestGET(req, &rslt);
}
if (rc == 0) {
uint32_t code = 0;
rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
if (rc == 0) {
if (code != 200) {
rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
}
}
}
if (rc == 0) {
size_t total = 0;
KStream *response = NULL;
rc = KHttpResultGetInputStream(rslt, &response);
if (rc == 0) {
rc = KDataBufferMakeBytes(&result, 1024);
}
while (rc == 0) {
size_t num_read = 0;
uint8_t *base = NULL;
uint64_t avail = result.elem_count - total;
if (avail < 256) {
rc = KDataBufferResize(&result, result.elem_count + 1024);
if (rc != 0) {
break;
}
}
base = result.base;
rc = KStreamRead(response, &base[total], result.elem_count - total,
&num_read);
if (num_read > 0 || rc != 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead"
"(sratoolkit.current.version, %zu) = %R\n", num_read, rc));
}
if (rc != 0) {
/* TBD - look more closely at rc */
if (num_read > 0) {
rc = 0;
}
else {
break;
}
}
if (num_read == 0) {
break;
}
total += num_read;
}
RELEASE(KStream, response);
if (rc == 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version (%zu)\n", total));
result.elem_count = total;
}
}
if (rc == 0) {
const char *start = (const void*)(result.base);
size_t size = KDataBufferBytes(&result);
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start));
rc = SraReleaseVersionInit(newVersion, start, size);
}
KDataBufferWhack(&result);
RELEASE(KHttpResult, rslt);
RELEASE(KHttpRequest, req);
return rc;
}
/* InitDNSEndpoint
* initialize the endpoint with a DNS name and a port number
*
* "ep" [ OUT ] - address of endpoint block to be intialized
*
* "dns" [ IN ] - textual DNS address.
*
* "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
* if the special port number 0 is given, it represents any available port.
*/
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
else if ( dns == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else if ( dns -> size == 0 )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
KDataBuffer b;
char buffer [ 4096 ], * hostname = buffer;
size_t buff_size = sizeof buffer;
if ( dns -> size >= sizeof buffer )
{
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
hostname = b . base;
buff_size = ( size_t ) b . elem_count;
}
}
if ( rc == 0 )
{
size_t size;
rc = string_printf ( hostname, buff_size, & size, "%S", dns );
assert ( rc == 0 );
assert ( size < buff_size );
assert ( hostname [ size ] == 0 );
if ( rc == 0 )
{
int lerrno;
struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
{
ep -> type = epIPV4;
memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
}
else switch ( lerrno = WSAGetLastError () )
{
case WSANOTINITIALISED: /* Must have WSAStartup call */
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcEnvironment, rcUndefined );
break;
case WSAENETDOWN:/* network subsystem failed */
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcNoObj, rcFailed );
break;
case WSAHOST_NOT_FOUND: /* Answer host not found */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
break;
case WSATRY_AGAIN: /* host not found or server failure */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
break;
case WSANO_RECOVERY: /* non-recoverable error */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
break;
case WSANO_DATA: /* name is valid but no data */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
break;
case WSAEINPROGRESS: /* call is in progress */
rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
break;
case WSAEFAULT: /* name paremeter is not valid part of addr space */
rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange );
break;
case WSAEINTR: /* socket call was calanceled */
rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
break;
default:
rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcError );
}
}
}
if ( hostname != buffer )
KDataBufferWhack ( & b );
}
if ( rc != 0 )
memset ( ep, 0, sizeof * ep );
}
return rc;
}
/* ReadAll
* read entire blob, plus any auxiliary checksum data
*
* "buffer" [ OUT ] - pointer to a KDataBuffer structure that will be initialized
* and resized to contain the entire blob. upon success, will contain the number of bytes
* in buffer->elem_count and buffer->elem_bits == 8.
*
* "opt_cs_data [ OUT, NULL OKAY ] - optional output parameter for checksum data
* associated with the blob in "buffer", if any exist.
*
* "cs_data_size" [ IN ] - sizeof of * opt_cs_data if not NULL, 0 otherwise
*/
LIB_EXPORT rc_t CC KColumnBlobReadAll ( const KColumnBlob * self, KDataBuffer * buffer,
KColumnBlobCSData * opt_cs_data, size_t cs_data_size )
{
rc_t rc = 0;
if ( opt_cs_data != NULL )
memset ( opt_cs_data, 0, cs_data_size );
if ( buffer == NULL )
rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcDB, rcBlob, rcReading, rcSelf, rcNull );
else
{
/* determine blob size */
size_t bsize = self -> loc . u . blob . size;
/* ignore blobs of size 0 */
if ( bsize == 0 )
rc = 0;
else
{
/* initialize the buffer */
rc = KDataBufferMakeBytes ( buffer, bsize );
if ( rc == 0 )
{
/* read the blob */
size_t num_read, remaining;
rc = KColumnBlobRead ( self, 0, buffer -> base, bsize, & num_read, & remaining );
if ( rc == 0 )
{
/* test that num_read is everything and we have no remaining */
if ( num_read != bsize || remaining != 0 )
rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );
else
{
/* set for MD5 - just due to switch ordering */
size_t cs_bytes = 16;
/* if not reading checksum data, then we're done */
if ( opt_cs_data == NULL )
return 0;
/* see what checksumming is in use */
switch ( self -> col -> checksum )
{
case kcsNone:
return 0;
case kcsCRC32:
/* reset for CRC32 */
cs_bytes = 4;
/* no break */
case kcsMD5:
if ( cs_data_size < cs_bytes )
{
rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcTooShort );
break;
}
/* read checksum information */
rc = KColumnDataRead ( & self -> col -> df,
& self -> pmorig, bsize, opt_cs_data, cs_bytes, & num_read );
if ( rc == 0 )
{
if ( num_read != cs_bytes )
rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );
else
{
/* success - read the blob AND the checksum data */
return 0;
}
}
break;
}
}
}
KDataBufferWhack ( buffer );
}
}
}
memset ( buffer, 0, sizeof * buffer );
}
return rc;
}