/* create_overflow
* create a buffer so large that it cannot overflow again
* copy in existing structures
*/
static
rc_t create_overflow ( ParseData *pd, uint32_t fmt_idx )
{
/* infer that we can never have more format items than characters in the string... */
/* our size will create a format for every byte.
this should be a total overkill. */
size_t buff_size = sizeof pd -> fmt [ 0 ] * pd -> fmt_size;
/* make the buffer */
rc_t rc = KDataBufferMakeBytes ( & pd -> overflow, buff_size );
if ( rc == 0 )
{
/* capture pointers to stack structures */
const PrintFmt *fmt = pd -> fmt;
/* destination pointer */
pd -> fmt = pd -> overflow . base;
/* copy existing data */
memcpy ( pd -> fmt, fmt, fmt_idx * sizeof pd -> fmt [ 0 ] );
}
pd -> fmt_idx = fmt_idx;
return rc;
}
static rc_t CC FastqRecordInit ( FastqRecord* self )
{
assert(self);
self->dad.vt.v1 = & FastqRecord_vt;
KDataBufferMakeBytes ( & self->source, 0 );
self->rej = 0;
return FastqSequenceInit(& self->seq);
}
LIB_EXPORT rc_t CC KCurlRequestPerform( struct KCurlRequest *self, KDataBuffer * buffer )
{
CURLcode rcc;
ReadContext ctx;
rc_t rc = 0;
if ( buffer == NULL )
return RC( rcNS, rcFile, rcReading, rcParam, rcNull );
memset ( buffer, 0, sizeof * buffer );
if ( self == NULL )
return RC( rcNS, rcFile, rcReading, rcSelf, rcNull );
rc = KDataBufferMakeBytes ( buffer, 0 );
if ( rc != 0 )
return rc;
ctx.buffer = buffer;
ctx.num_read = 0;
rcc = self->kns_mgr->curl_easy_setopt_fkt( self->curl_handle, CURLOPT_WRITEDATA, (void *)&ctx );
if ( rcc != CURLE_OK )
{
rc = RC( rcFS, rcFile, rcConstructing, rcFileDesc, rcInvalid );
LOGERR( klogErr, rc, "curl_easy_setopt( CURLOPT_WRITEDATA ) failed" );
}
if ( rc == 0 )
{
CURLcode rcc = self->kns_mgr->curl_easy_setopt_fkt( self->curl_handle, CURLOPT_WRITEFUNCTION, KCurlFileCallback );
if ( rcc != CURLE_OK )
{
rc = RC( rcFS, rcFile, rcConstructing, rcFileDesc, rcInvalid );
LOGERR( klogErr, rc, "curl_easy_setopt( CURLOPT_WRITEFUNCTION ) failed" );
}
}
if ( rc == 0 )
rc = set_curl_long_option( self, CURLOPT_FOLLOWLOCATION, 1, "CURLOPT_FOLLOWLOCATION" );
if ( rc == 0 )
rc = KCurlRequestSetFields( self );
if ( rc == 0 )
rc = perform( self->kns_mgr, self->curl_handle, "POST request" );
if ( rc == 0 )
rc = check_response_code( self->kns_mgr, self->curl_handle, "POST request" );
if ( rc == 0 )
{
buffer->elem_count = ctx.num_read;
( ( uint8_t * )buffer->base )[ ctx.num_read ] = 0;
}
return rc;
}
rc_t make_fastq_sra_iter( const cmn_params * params,
fastq_iter_opt opt,
const char * tbl_name,
struct fastq_sra_iter ** iter )
{
rc_t rc = 0;
fastq_sra_iter * self = calloc( 1, sizeof * self );
if ( self == NULL )
{
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
ErrMsg( "make_fastq_tbl_iter.calloc( %d ) -> %R", ( sizeof * self ), rc );
}
else
{
rc = KDataBufferMakeBytes( &self -> qual_buffer, 4096 );
if ( rc != 0 )
{
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
ErrMsg( "make_fastq_csra_iter.KDataBufferMakeBytes() -> %R", rc );
}
else
{
self -> opt = opt;
rc = make_cmn_iter( params, tbl_name, &( self -> cmn ) );
if ( rc == 0 && opt . with_name )
rc = cmn_iter_add_column( self -> cmn, "NAME", &( self -> name_id ) );
if ( rc == 0 )
rc = cmn_iter_add_column( self -> cmn, "READ", &( self -> read_id ) );
if ( rc == 0 )
rc = cmn_iter_add_column( self -> cmn, "QUALITY", &( self -> quality_id ) );
if ( rc == 0 && opt . with_read_len )
rc = cmn_iter_add_column( self -> cmn, "READ_LEN", &( self -> read_len_id ) );
if ( rc == 0 && opt . with_read_type )
rc = cmn_iter_add_column( self -> cmn, "READ_TYPE", &( self -> read_type_id ) );
if ( rc == 0 )
rc = cmn_iter_range( self -> cmn, self -> read_id );
if ( rc == 0 )
init_qual_to_ascii( &( self -> qual_2_ascii[ 0 ] ), sizeof( self -> qual_2_ascii ) );
if ( rc != 0 )
destroy_fastq_sra_iter( self );
else
*iter = self;
}
}
return rc;
}
static rc_t stats_data_init(stats_data_t *const self,
VTable *const tbl,
bool has_spot_group,
bool compressed)
{
stats_data_init_funcs(self, has_spot_group);
self->write = compressed ? group_stats_write_compressed : group_stats_write_no_compressed;
group_stats_init(&self->table, 0, 0, 0);
group_stats_init(&self->deflt, 0, 0, 0);
RC_THROW(VTableOpenMetadataUpdate(tbl, &self->meta));
if (has_spot_group) {
RC_THROW(KDataBufferMakeBytes(&self->names, 0));
RC_THROW(KDataBufferMake(&self->group, sizeof(group_stats_t) * 8, 0));
}
return 0;
}
static
rc_t CC f32zip_func(
void *Self,
const VXformInfo *info,
VBlobResult *dst,
const VBlobData *Src,
VBlobHeader *hdr
) {
rc_t rc;
const struct self_t *self = Self;
uint64_t dsize = (dst->elem_count * dst->elem_bits + 7) >> 3;
uint64_t element_count = (Src->elem_count * Src->elem_bits) >> 5;
uint64_t man_bytes;
KDataBuffer scratch;
assert(element_count >> 32 == 0);
assert(dsize >> 32 == 0);
rc = KDataBufferMakeBytes(&scratch, element_count << 2);
if (rc)
return rc;
VBlobHeaderSetVersion(hdr, 0);
VBlobHeaderOpPushTail(hdr, 0);
VBlobHeaderArgPushTail(hdr, self->mantissa);
man_bytes = split_and_pack_f32(
Src->data, (uint32_t)element_count,
self->mantissa,
scratch.base);
assert((element_count + man_bytes) >> 32 == 0);
rc = invoke_zlib(dst->data, &dsize, scratch.base, (uint32_t)(element_count + man_bytes), Z_RLE, Z_BEST_SPEED);
KDataBufferWhack(&scratch);
if (rc == 0) {
dst->elem_bits = 1;
dst->byte_order = vboNone;
if (dsize)
dst->elem_count = dsize << 3;
else
rc = RC(rcXF, rcFunction, rcExecuting, rcBuffer, rcInsufficient);
}
return rc;
}
/* InitDNSEndpoint
* initialize the endpoint with a DNS name and a port number
*
* "ep" [ OUT ] - address of endpoint block to be intialized
*
* "dns" [ IN ] - textual DNS address.
*
* "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
* if the special port number 0 is given, it represents any available port.
*/
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
else if ( dns == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else if ( dns -> size == 0 )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
KDataBuffer b;
char buffer [ 4096 ], * hostname = buffer;
size_t buff_size = sizeof buffer;
if ( dns -> size >= sizeof buffer )
{
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
hostname = b . base;
buff_size = ( size_t ) b . elem_count;
}
}
if ( rc == 0 )
{
size_t size;
rc = string_printf ( hostname, buff_size, & size, "%S", dns );
assert ( rc == 0 );
assert ( size < buff_size );
assert ( hostname [ size ] == 0 );
if ( rc == 0 )
{
struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
{
ep -> type = epIPV4;
memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
}
else switch ( h_errno )
{
case HOST_NOT_FOUND: /* The specified host is unknown */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
break;
case NO_ADDRESS: /* The requested names valid but does not have an IP address */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcInconsistent );
break;
#if ! defined NO_ADDRESS || ! defined NO_DATA || NO_ADDRESS != NO_DATA
case NO_DATA: /* The requested name s valid but does not have an IP address */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
break;
#endif
case NO_RECOVERY: /* A nonrecoverable name server error occured */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcDestroyed );
break;
case TRY_AGAIN: /* A temporary error occured on an authoritative name server. Try again later */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
break;
default :
rc = RC ( rcNS, rcNoTarg, rcValidating, rcError, rcUnknown );
}
}
}
if ( hostname != buffer )
KDataBufferWhack ( & b );
}
if ( rc != 0 )
memset ( ep, 0, sizeof * ep );
}
return rc;
}
static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self,
SraReleaseVersion *newVersion)
{
rc_t rc = 0;
KDataBuffer result;
KHttpRequest *req = NULL;
KHttpResult *rslt = NULL;
if (newVersion == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull);
}
memset(newVersion, 0, sizeof *newVersion);
if (self == NULL) {
return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull);
}
memset(&result, 0, sizeof result);
if (rc == 0) {
rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL,
"https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
);
}
if (rc == 0) {
rc = KHttpRequestGET(req, &rslt);
}
if (rc == 0) {
uint32_t code = 0;
rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
if (rc == 0) {
if (code != 200) {
rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
}
}
}
if (rc == 0) {
size_t total = 0;
KStream *response = NULL;
rc = KHttpResultGetInputStream(rslt, &response);
if (rc == 0) {
rc = KDataBufferMakeBytes(&result, 1024);
}
while (rc == 0) {
size_t num_read = 0;
uint8_t *base = NULL;
uint64_t avail = result.elem_count - total;
if (avail < 256) {
rc = KDataBufferResize(&result, result.elem_count + 1024);
if (rc != 0) {
break;
}
}
base = result.base;
rc = KStreamRead(response, &base[total], result.elem_count - total,
&num_read);
if (num_read > 0 || rc != 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead"
"(sratoolkit.current.version, %zu) = %R\n", num_read, rc));
}
if (rc != 0) {
/* TBD - look more closely at rc */
if (num_read > 0) {
rc = 0;
}
else {
break;
}
}
if (num_read == 0) {
break;
}
total += num_read;
}
RELEASE(KStream, response);
if (rc == 0) {
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version (%zu)\n", total));
result.elem_count = total;
}
}
if (rc == 0) {
const char *start = (const void*)(result.base);
size_t size = KDataBufferBytes(&result);
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start));
rc = SraReleaseVersionInit(newVersion, start, size);
}
KDataBufferWhack(&result);
RELEASE(KHttpResult, rslt);
RELEASE(KHttpRequest, req);
return rc;
}
/* InitDNSEndpoint
* initialize the endpoint with a DNS name and a port number
*
* "ep" [ OUT ] - address of endpoint block to be intialized
*
* "dns" [ IN ] - textual DNS address.
*
* "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
* if the special port number 0 is given, it represents any available port.
*/
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
KEndPoint *ep, struct String const *dns, uint16_t port )
{
rc_t rc = 0;
if ( ep == NULL )
rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
else if ( dns == NULL )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
else if ( dns -> size == 0 )
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
else
{
KDataBuffer b;
char buffer [ 4096 ], * hostname = buffer;
size_t buff_size = sizeof buffer;
if ( dns -> size >= sizeof buffer )
{
rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
if ( rc == 0 )
{
hostname = b . base;
buff_size = ( size_t ) b . elem_count;
}
}
if ( rc == 0 )
{
size_t size;
rc = string_printf ( hostname, buff_size, & size, "%S", dns );
assert ( rc == 0 );
assert ( size < buff_size );
assert ( hostname [ size ] == 0 );
if ( rc == 0 )
{
int lerrno;
struct hostent *remote = gethostbyname ( hostname );
if ( remote != NULL )
{
ep -> type = epIPV4;
memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
ep -> u . ipv4 . port = ( uint16_t ) port;
}
else switch ( lerrno = WSAGetLastError () )
{
case WSANOTINITIALISED: /* Must have WSAStartup call */
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcEnvironment, rcUndefined );
break;
case WSAENETDOWN:/* network subsystem failed */
rc = RC ( rcNS, rcNoTarg, rcInitializing, rcNoObj, rcFailed );
break;
case WSAHOST_NOT_FOUND: /* Answer host not found */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
break;
case WSATRY_AGAIN: /* host not found or server failure */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
break;
case WSANO_RECOVERY: /* non-recoverable error */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
break;
case WSANO_DATA: /* name is valid but no data */
rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
break;
case WSAEINPROGRESS: /* call is in progress */
rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
break;
case WSAEFAULT: /* name paremeter is not valid part of addr space */
rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange );
break;
case WSAEINTR: /* socket call was calanceled */
rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
break;
default:
rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcError );
}
}
}
if ( hostname != buffer )
KDataBufferWhack ( & b );
}
if ( rc != 0 )
memset ( ep, 0, sizeof * ep );
}
return rc;
}
/* ReadAll
* read entire blob, plus any auxiliary checksum data
*
* "buffer" [ OUT ] - pointer to a KDataBuffer structure that will be initialized
* and resized to contain the entire blob. upon success, will contain the number of bytes
* in buffer->elem_count and buffer->elem_bits == 8.
*
* "opt_cs_data [ OUT, NULL OKAY ] - optional output parameter for checksum data
* associated with the blob in "buffer", if any exist.
*
* "cs_data_size" [ IN ] - sizeof of * opt_cs_data if not NULL, 0 otherwise
*/
LIB_EXPORT rc_t CC KColumnBlobReadAll ( const KColumnBlob * self, KDataBuffer * buffer,
KColumnBlobCSData * opt_cs_data, size_t cs_data_size )
{
rc_t rc = 0;
if ( opt_cs_data != NULL )
memset ( opt_cs_data, 0, cs_data_size );
if ( buffer == NULL )
rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcDB, rcBlob, rcReading, rcSelf, rcNull );
else
{
/* determine blob size */
size_t bsize = self -> loc . u . blob . size;
/* ignore blobs of size 0 */
if ( bsize == 0 )
rc = 0;
else
{
/* initialize the buffer */
rc = KDataBufferMakeBytes ( buffer, bsize );
if ( rc == 0 )
{
/* read the blob */
size_t num_read, remaining;
rc = KColumnBlobRead ( self, 0, buffer -> base, bsize, & num_read, & remaining );
if ( rc == 0 )
{
/* test that num_read is everything and we have no remaining */
if ( num_read != bsize || remaining != 0 )
rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );
else
{
/* set for MD5 - just due to switch ordering */
size_t cs_bytes = 16;
/* if not reading checksum data, then we're done */
if ( opt_cs_data == NULL )
return 0;
/* see what checksumming is in use */
switch ( self -> col -> checksum )
{
case kcsNone:
return 0;
case kcsCRC32:
/* reset for CRC32 */
cs_bytes = 4;
/* no break */
case kcsMD5:
if ( cs_data_size < cs_bytes )
{
rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcTooShort );
break;
}
/* read checksum information */
rc = KColumnDataRead ( & self -> col -> df,
& self -> pmorig, bsize, opt_cs_data, cs_bytes, & num_read );
if ( rc == 0 )
{
if ( num_read != cs_bytes )
rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );
else
{
/* success - read the blob AND the checksum data */
return 0;
}
}
break;
}
}
}
KDataBufferWhack ( buffer );
}
}
}
memset ( buffer, 0, sizeof * buffer );
}
return rc;
}