static rc_t stats_data_update_group(stats_data_t *self, int64_t spot_id, uint32_t spot_len, uint32_t bio_spot_len, uint32_t cmp_spot_len, char const grp[], uint64_t grp_len) { if (grp_len == 0 || grp == NULL || grp[0] == '\0' || (grp_len == 7 && strncasecmp("default", grp, 7) == 0)) { group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len); } else { group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp); if (fnd == NULL) return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted); if (fnd - get_group(self, 0) < MAX_GROUP_COUNT) group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len); else { KDataBufferWhack(&self->group); KDataBufferWhack(&self->names); stats_data_init_funcs(self, false); (void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count))); } } return 0; }
static void CC stats_data_whack(void *const data) { stats_data_t *const self = data; self->write_all(self); KDataBufferWhack(&self->group); KDataBufferWhack(&self->names); KMetadataRelease(self->meta); free(self); }
static rc_t TableWriterSeq_WriteStatistics(TableWriterSeq const *cself, KMDataNode *node) { pb_t pb; rc_t rc; KDataBuffer buf; rc = KDataBufferMake(&buf, 8 * sizeof(pb.stats[0]), cself->statsCount); if (rc) return rc; pb.stats = buf.base; pb.i = 0; rc = KVectorVisitU64(cself->stats, 0, stats_cb, &pb); if (rc == 0) { unsigned i; unsigned const n = cself->statsCount < 126 ? cself->statsCount : 126; uint64_t *const distance = buf.base; ksort(pb.stats, cself->statsCount, sizeof(pb.stats[0]), stats_cmp_count, NULL); ksort(pb.stats, n, sizeof(pb.stats[0]), stats_cmp_distance, NULL); for (i = 0; i != n; ++i) { distance[i] = pb.stats[i].distance; } rc = KMDataNodeWrite(node, distance, n * sizeof(distance[0])); } KDataBufferWhack(&buf); return rc; }
static rc_t KCurlRequestDestroy( struct KCurlRequest *self ) { rc_t rc; self->kns_mgr->curl_easy_cleanup_fkt( self->curl_handle ); rc = KNSManagerRelease( self->kns_mgr ); KDataBufferWhack( &self->fields ); free ( self ); return rc; }
void destroy_fastq_sra_iter( struct fastq_sra_iter * self ) { if ( self != NULL ) { destroy_cmn_iter( self -> cmn ); if ( self -> qual_buffer . base != NULL ) KDataBufferWhack( &self -> qual_buffer ); free( ( void * ) self ); } }
static void Unsorted(Reference *self) { (void)LOGMSG(klogWarn, "Alignments are unsorted"); self->out_of_order = true; ReferenceMgr_SetCache(self->mgr, UNSORTED_CACHE_SIZE, UNSORTED_OPEN_TABLE_LIMIT); KDataBufferWhack(&self->sec_align); KDataBufferWhack(&self->pri_align); KDataBufferWhack(&self->mismatches); KDataBufferWhack(&self->indels); KDataBufferWhack(&self->coverage); KDataBufferWhack(&self->pri_overlap); KDataBufferWhack(&self->sec_overlap); }
/* Whack */ rc_t KColumnIdx2Whack ( KColumnIdx2 *self ) { rc_t rc = KFileRelease ( self -> f ); if ( rc == 0 ) { int i; KColumnIdx2BlockCache * cache=(KColumnIdx2BlockCache *)self -> cstorage.base; self -> f = NULL; for(i=0;i<self->cstorage.elem_count;i++){ free(cache[i].block); } KDataBufferWhack(&self->cstorage); } return rc; }
static rc_t FastqRecordWhack( const FastqRecord* cself ) { rc_t rc = 0; FastqRecord* self = (FastqRecord*)cself; assert(self); rc = KDataBufferWhack( & self->source ); if (rc) RejectedRelease(self->rej); else rc = RejectedRelease(self->rej); free(self); return rc; }
static rc_t CC f32zip_func( void *Self, const VXformInfo *info, VBlobResult *dst, const VBlobData *Src, VBlobHeader *hdr ) { rc_t rc; const struct self_t *self = Self; uint64_t dsize = (dst->elem_count * dst->elem_bits + 7) >> 3; uint64_t element_count = (Src->elem_count * Src->elem_bits) >> 5; uint64_t man_bytes; KDataBuffer scratch; assert(element_count >> 32 == 0); assert(dsize >> 32 == 0); rc = KDataBufferMakeBytes(&scratch, element_count << 2); if (rc) return rc; VBlobHeaderSetVersion(hdr, 0); VBlobHeaderOpPushTail(hdr, 0); VBlobHeaderArgPushTail(hdr, self->mantissa); man_bytes = split_and_pack_f32( Src->data, (uint32_t)element_count, self->mantissa, scratch.base); assert((element_count + man_bytes) >> 32 == 0); rc = invoke_zlib(dst->data, &dsize, scratch.base, (uint32_t)(element_count + man_bytes), Z_RLE, Z_BEST_SPEED); KDataBufferWhack(&scratch); if (rc == 0) { dst->elem_bits = 1; dst->byte_order = vboNone; if (dsize) dst->elem_count = dsize << 3; else rc = RC(rcXF, rcFunction, rcExecuting, rcBuffer, rcInsufficient); } return rc; }
/* InitDNSEndpoint * initialize the endpoint with a DNS name and a port number * * "ep" [ OUT ] - address of endpoint block to be intialized * * "dns" [ IN ] - textual DNS address. * * "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order. * if the special port number 0 is given, it represents any available port. */ LIB_EXPORT rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self, KEndPoint *ep, struct String const *dns, uint16_t port ) { rc_t rc = 0; if ( ep == NULL ) rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull ); else { if ( self == NULL ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull ); else if ( dns == NULL ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull ); else if ( dns -> size == 0 ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient ); else { KDataBuffer b; char buffer [ 4096 ], * hostname = buffer; size_t buff_size = sizeof buffer; if ( dns -> size >= sizeof buffer ) { rc = KDataBufferMakeBytes ( & b, dns -> size + 1 ); if ( rc == 0 ) { hostname = b . base; buff_size = ( size_t ) b . elem_count; } } if ( rc == 0 ) { size_t size; rc = string_printf ( hostname, buff_size, & size, "%S", dns ); assert ( rc == 0 ); assert ( size < buff_size ); assert ( hostname [ size ] == 0 ); if ( rc == 0 ) { struct hostent *remote = gethostbyname ( hostname ); if ( remote != NULL ) { ep -> type = epIPV4; memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr ); ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr ); ep -> u . ipv4 . port = ( uint16_t ) port; } else switch ( h_errno ) { case HOST_NOT_FOUND: /* The specified host is unknown */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound ); break; case NO_ADDRESS: /* The requested names valid but does not have an IP address */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcInconsistent ); break; #if ! defined NO_ADDRESS || ! defined NO_DATA || NO_ADDRESS != NO_DATA case NO_DATA: /* The requested name s valid but does not have an IP address */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty ); break; #endif case NO_RECOVERY: /* A nonrecoverable name server error occured */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcDestroyed ); break; case TRY_AGAIN: /* A temporary error occured on an authoritative name server. Try again later */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy ); break; default : rc = RC ( rcNS, rcNoTarg, rcValidating, rcError, rcUnknown ); } } } if ( hostname != buffer ) KDataBufferWhack ( & b ); } if ( rc != 0 ) memset ( ep, 0, sizeof * ep ); } return rc; }
static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self, SraReleaseVersion *newVersion) { rc_t rc = 0; KDataBuffer result; KHttpRequest *req = NULL; KHttpResult *rslt = NULL; if (newVersion == NULL) { return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull); } memset(newVersion, 0, sizeof *newVersion); if (self == NULL) { return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull); } memset(&result, 0, sizeof result); if (rc == 0) { rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL, "https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version" ); } if (rc == 0) { rc = KHttpRequestGET(req, &rslt); } if (rc == 0) { uint32_t code = 0; rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL); if (rc == 0) { if (code != 200) { rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid); } } } if (rc == 0) { size_t total = 0; KStream *response = NULL; rc = KHttpResultGetInputStream(rslt, &response); if (rc == 0) { rc = KDataBufferMakeBytes(&result, 1024); } while (rc == 0) { size_t num_read = 0; uint8_t *base = NULL; uint64_t avail = result.elem_count - total; if (avail < 256) { rc = KDataBufferResize(&result, result.elem_count + 1024); if (rc != 0) { break; } } base = result.base; rc = KStreamRead(response, &base[total], result.elem_count - total, &num_read); if (num_read > 0 || rc != 0) { DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead" "(sratoolkit.current.version, %zu) = %R\n", num_read, rc)); } if (rc != 0) { /* TBD - look more closely at rc */ if (num_read > 0) { rc = 0; } else { break; } } if (num_read == 0) { break; } total += num_read; } RELEASE(KStream, response); if (rc == 0) { DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("sratoolkit.current.version (%zu)\n", total)); result.elem_count = total; } } if (rc == 0) { const char *start = (const void*)(result.base); size_t size = KDataBufferBytes(&result); DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start)); rc = SraReleaseVersionInit(newVersion, start, size); } KDataBufferWhack(&result); RELEASE(KHttpResult, rslt); RELEASE(KHttpRequest, req); return rc; }
/* InitDNSEndpoint * initialize the endpoint with a DNS name and a port number * * "ep" [ OUT ] - address of endpoint block to be intialized * * "dns" [ IN ] - textual DNS address. * * "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order. * if the special port number 0 is given, it represents any available port. */ LIB_EXPORT rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self, KEndPoint *ep, struct String const *dns, uint16_t port ) { rc_t rc = 0; if ( ep == NULL ) rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull ); else { if ( self == NULL ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull ); else if ( dns == NULL ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull ); else if ( dns -> size == 0 ) rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient ); else { KDataBuffer b; char buffer [ 4096 ], * hostname = buffer; size_t buff_size = sizeof buffer; if ( dns -> size >= sizeof buffer ) { rc = KDataBufferMakeBytes ( & b, dns -> size + 1 ); if ( rc == 0 ) { hostname = b . base; buff_size = ( size_t ) b . elem_count; } } if ( rc == 0 ) { size_t size; rc = string_printf ( hostname, buff_size, & size, "%S", dns ); assert ( rc == 0 ); assert ( size < buff_size ); assert ( hostname [ size ] == 0 ); if ( rc == 0 ) { int lerrno; struct hostent *remote = gethostbyname ( hostname ); if ( remote != NULL ) { ep -> type = epIPV4; memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr ); ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr ); ep -> u . ipv4 . port = ( uint16_t ) port; } else switch ( lerrno = WSAGetLastError () ) { case WSANOTINITIALISED: /* Must have WSAStartup call */ rc = RC ( rcNS, rcNoTarg, rcInitializing, rcEnvironment, rcUndefined ); break; case WSAENETDOWN:/* network subsystem failed */ rc = RC ( rcNS, rcNoTarg, rcInitializing, rcNoObj, rcFailed ); break; case WSAHOST_NOT_FOUND: /* Answer host not found */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound ); break; case WSATRY_AGAIN: /* host not found or server failure */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy ); break; case WSANO_RECOVERY: /* non-recoverable error */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError ); break; case WSANO_DATA: /* name is valid but no data */ rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty ); break; case WSAEINPROGRESS: /* call is in progress */ rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined ); break; case WSAEFAULT: /* name paremeter is not valid part of addr space */ rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange ); break; case WSAEINTR: /* socket call was calanceled */ rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled ); break; default: rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcError ); } } } if ( hostname != buffer ) KDataBufferWhack ( & b ); } if ( rc != 0 ) memset ( ep, 0, sizeof * ep ); } return rc; }
/* ReadAll * read entire blob, plus any auxiliary checksum data * * "buffer" [ OUT ] - pointer to a KDataBuffer structure that will be initialized * and resized to contain the entire blob. upon success, will contain the number of bytes * in buffer->elem_count and buffer->elem_bits == 8. * * "opt_cs_data [ OUT, NULL OKAY ] - optional output parameter for checksum data * associated with the blob in "buffer", if any exist. * * "cs_data_size" [ IN ] - sizeof of * opt_cs_data if not NULL, 0 otherwise */ LIB_EXPORT rc_t CC KColumnBlobReadAll ( const KColumnBlob * self, KDataBuffer * buffer, KColumnBlobCSData * opt_cs_data, size_t cs_data_size ) { rc_t rc = 0; if ( opt_cs_data != NULL ) memset ( opt_cs_data, 0, cs_data_size ); if ( buffer == NULL ) rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcNull ); else { if ( self == NULL ) rc = RC ( rcDB, rcBlob, rcReading, rcSelf, rcNull ); else { /* determine blob size */ size_t bsize = self -> loc . u . blob . size; /* ignore blobs of size 0 */ if ( bsize == 0 ) rc = 0; else { /* initialize the buffer */ rc = KDataBufferMakeBytes ( buffer, bsize ); if ( rc == 0 ) { /* read the blob */ size_t num_read, remaining; rc = KColumnBlobRead ( self, 0, buffer -> base, bsize, & num_read, & remaining ); if ( rc == 0 ) { /* test that num_read is everything and we have no remaining */ if ( num_read != bsize || remaining != 0 ) rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete ); else { /* set for MD5 - just due to switch ordering */ size_t cs_bytes = 16; /* if not reading checksum data, then we're done */ if ( opt_cs_data == NULL ) return 0; /* see what checksumming is in use */ switch ( self -> col -> checksum ) { case kcsNone: return 0; case kcsCRC32: /* reset for CRC32 */ cs_bytes = 4; /* no break */ case kcsMD5: if ( cs_data_size < cs_bytes ) { rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcTooShort ); break; } /* read checksum information */ rc = KColumnDataRead ( & self -> col -> df, & self -> pmorig, bsize, opt_cs_data, cs_bytes, & num_read ); if ( rc == 0 ) { if ( num_read != cs_bytes ) rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete ); else { /* success - read the blob AND the checksum data */ return 0; } } break; } } } KDataBufferWhack ( buffer ); } } } memset ( buffer, 0, sizeof * buffer ); } return rc; }