Ejemplo n.º 1
0
static
rc_t stats_data_update_group(stats_data_t *self,
                             int64_t spot_id,
                             uint32_t spot_len,
                             uint32_t bio_spot_len,
                             uint32_t cmp_spot_len,
                             char const grp[],
                             uint64_t grp_len)
{
    if (grp_len == 0 || grp == NULL || grp[0] == '\0' ||
        (grp_len == 7 && strncasecmp("default", grp, 7) == 0))
    {
        group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len);
    }
    else
    {
        group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp);
        
        if (fnd == NULL)
            return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted);
        
        if (fnd - get_group(self, 0) < MAX_GROUP_COUNT)
            group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len);
        else {
            KDataBufferWhack(&self->group);
            KDataBufferWhack(&self->names);
            stats_data_init_funcs(self, false);
            (void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count)));
        }
    }
    return 0;
}
Ejemplo n.º 2
0
static
void CC stats_data_whack(void *const data)
{
    stats_data_t *const self = data;
    
    self->write_all(self);
    KDataBufferWhack(&self->group);
    KDataBufferWhack(&self->names);
    KMetadataRelease(self->meta);
    free(self);
}
Ejemplo n.º 3
0
static rc_t TableWriterSeq_WriteStatistics(TableWriterSeq const *cself, KMDataNode *node)
{
    pb_t pb;
    rc_t rc;
    KDataBuffer buf;
    
    rc = KDataBufferMake(&buf, 8 * sizeof(pb.stats[0]), cself->statsCount);
    if (rc) return rc;
    
    pb.stats = buf.base;
    pb.i = 0;
    rc = KVectorVisitU64(cself->stats, 0, stats_cb, &pb);
    if (rc == 0) {
        unsigned i;
        unsigned const n = cself->statsCount < 126 ? cself->statsCount : 126;
        uint64_t *const distance = buf.base;
        
        ksort(pb.stats, cself->statsCount, sizeof(pb.stats[0]), stats_cmp_count, NULL);
        ksort(pb.stats, n, sizeof(pb.stats[0]), stats_cmp_distance, NULL);
        for (i = 0; i != n; ++i) {
            distance[i] = pb.stats[i].distance;
        }
        rc = KMDataNodeWrite(node, distance, n * sizeof(distance[0]));
    }
    KDataBufferWhack(&buf);
    return rc;
}
Ejemplo n.º 4
0
static rc_t KCurlRequestDestroy( struct KCurlRequest *self )
{
    rc_t rc;
    self->kns_mgr->curl_easy_cleanup_fkt( self->curl_handle );
    rc = KNSManagerRelease( self->kns_mgr );
    KDataBufferWhack( &self->fields );
    free ( self );
    return rc;
}
Ejemplo n.º 5
0
void destroy_fastq_sra_iter( struct fastq_sra_iter * self )
{
    if ( self != NULL )
    {
        destroy_cmn_iter( self -> cmn );
        if ( self -> qual_buffer . base != NULL )
            KDataBufferWhack( &self -> qual_buffer );
        free( ( void * ) self );
    }
}
Ejemplo n.º 6
0
static
void Unsorted(Reference *self) {
    (void)LOGMSG(klogWarn, "Alignments are unsorted");
    self->out_of_order = true;
    ReferenceMgr_SetCache(self->mgr, UNSORTED_CACHE_SIZE, UNSORTED_OPEN_TABLE_LIMIT);
    KDataBufferWhack(&self->sec_align);
    KDataBufferWhack(&self->pri_align);
    KDataBufferWhack(&self->mismatches);
    KDataBufferWhack(&self->indels);
    KDataBufferWhack(&self->coverage);
    KDataBufferWhack(&self->pri_overlap);
    KDataBufferWhack(&self->sec_overlap);
}
Ejemplo n.º 7
0
/* Whack
 */
rc_t KColumnIdx2Whack ( KColumnIdx2 *self )
{
    rc_t rc = KFileRelease ( self -> f );
    if ( rc == 0 )
    {
        int i;
        KColumnIdx2BlockCache * cache=(KColumnIdx2BlockCache *)self -> cstorage.base;
        self -> f = NULL;
        for(i=0;i<self->cstorage.elem_count;i++){
            free(cache[i].block);
        }
        KDataBufferWhack(&self->cstorage);
    }
    return rc;
}
Ejemplo n.º 8
0
static rc_t FastqRecordWhack( const FastqRecord* cself )
{
    rc_t rc = 0;
    FastqRecord* self = (FastqRecord*)cself;
    assert(self);

    rc = KDataBufferWhack( & self->source );

    if (rc)
        RejectedRelease(self->rej);
    else
        rc = RejectedRelease(self->rej);

    free(self);

    return rc;
}
Ejemplo n.º 9
0
static
rc_t CC f32zip_func(
              void *Self,
              const VXformInfo *info,
              VBlobResult *dst,
              const VBlobData *Src,
              VBlobHeader *hdr
) {
    rc_t rc;
    const struct self_t *self = Self;
    uint64_t dsize = (dst->elem_count * dst->elem_bits + 7) >> 3;
    uint64_t element_count = (Src->elem_count * Src->elem_bits) >> 5;
    uint64_t man_bytes;
    KDataBuffer scratch;
    
    assert(element_count >> 32 == 0);
    assert(dsize >> 32 == 0);
    rc = KDataBufferMakeBytes(&scratch, element_count << 2);
    if (rc)
        return rc;
    
    VBlobHeaderSetVersion(hdr, 0);
    VBlobHeaderOpPushTail(hdr, 0);
    VBlobHeaderArgPushTail(hdr, self->mantissa);
    
    man_bytes = split_and_pack_f32(
                                   Src->data, (uint32_t)element_count,
                                   self->mantissa,
                                   scratch.base);
    assert((element_count + man_bytes) >> 32 == 0);
    rc = invoke_zlib(dst->data, &dsize, scratch.base, (uint32_t)(element_count + man_bytes), Z_RLE, Z_BEST_SPEED);
    KDataBufferWhack(&scratch);
    if (rc == 0) {
        dst->elem_bits = 1;
        dst->byte_order = vboNone;
        if (dsize)
            dst->elem_count = dsize << 3;
        else
            rc = RC(rcXF, rcFunction, rcExecuting, rcBuffer, rcInsufficient);
    }
    return rc;
}
Ejemplo n.º 10
0
/* InitDNSEndpoint
 *  initialize the endpoint with a DNS name and a port number
 *
 *  "ep" [ OUT ] - address of endpoint block to be intialized
 *
 *  "dns" [ IN ] - textual DNS address.
 *
 *  "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
 *   if the special port number 0 is given, it represents any available port.
 */
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
    KEndPoint *ep, struct String const *dns, uint16_t port )
{
    rc_t rc = 0;

    if ( ep == NULL )
        rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
    else
    {
        if ( self == NULL )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
        else if ( dns == NULL )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
        else if ( dns -> size == 0 )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
        else
        {
            KDataBuffer b;
            char buffer [ 4096 ], * hostname = buffer;
            size_t buff_size = sizeof buffer;

            if ( dns -> size >= sizeof buffer )
            {
                rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
                if ( rc == 0 )
                {
                    hostname = b . base;
                    buff_size = ( size_t ) b . elem_count;
                }
            }

            if ( rc == 0 )
            {
                size_t size;
                rc = string_printf ( hostname, buff_size, & size, "%S", dns );

                assert ( rc == 0 );
                assert ( size < buff_size );
                assert ( hostname [ size ] == 0 );

                if ( rc ==  0 )
                {
                    struct hostent *remote = gethostbyname ( hostname );
                    if ( remote != NULL )
                    { 
                        ep -> type = epIPV4;
                        memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
                        ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
                        ep -> u . ipv4 . port = ( uint16_t ) port;
                    }
                    else switch ( h_errno )
                    {
                    case HOST_NOT_FOUND: /* The specified host is unknown */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
                        break;
                    case NO_ADDRESS: /* The requested names valid but does not have an IP address */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcInconsistent );
                        break;
#if ! defined NO_ADDRESS || ! defined NO_DATA || NO_ADDRESS != NO_DATA
                    case NO_DATA: /* The requested name s valid but does not have an IP address */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
                        break;
#endif
                    case NO_RECOVERY: /* A nonrecoverable name server error occured */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcDestroyed );
                        break;
                    case TRY_AGAIN: /* A temporary error occured on an authoritative name server. Try again later */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
                        break;
                    default :
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcError, rcUnknown );
                    }
                }
            }

            if ( hostname != buffer )
                KDataBufferWhack ( & b );
        }

        if ( rc != 0 )
            memset ( ep, 0, sizeof * ep );        
    }

    return rc;
}
Ejemplo n.º 11
0
static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self,
    SraReleaseVersion *newVersion)
{
    rc_t rc = 0;
    KDataBuffer result;
    KHttpRequest *req = NULL;
    KHttpResult *rslt = NULL;
    if (newVersion == NULL) {
        return RC(rcNS, rcArgv, rcAccessing, rcParam, rcNull);
    }
    memset(newVersion, 0, sizeof *newVersion);
    if (self == NULL) {
        return RC(rcNS, rcArgv, rcAccessing, rcSelf, rcNull);
    }
    memset(&result, 0, sizeof result);
    if (rc == 0) {
        rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL,
  "https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
        );
    }
    if (rc == 0) {
        rc = KHttpRequestGET(req, &rslt);
    }
    if (rc == 0) {
        uint32_t code = 0;
        rc = KHttpResultStatus(rslt, &code, NULL, 0, NULL);
        if (rc == 0) {
            if (code != 200) {
                rc = RC(rcNS, rcFile, rcReading, rcFile, rcInvalid);
            }
        }
    }
    if (rc == 0) {
        size_t total = 0;
        KStream *response = NULL;
        rc = KHttpResultGetInputStream(rslt, &response);
        if (rc == 0) {
            rc = KDataBufferMakeBytes(&result, 1024);
        }
        while (rc == 0) {
            size_t num_read = 0;
            uint8_t *base = NULL;
            uint64_t avail = result.elem_count - total;
            if (avail < 256) {
                rc = KDataBufferResize(&result, result.elem_count + 1024);
                if (rc != 0) {
                    break;
                }
            }
            base = result.base;
            rc = KStreamRead(response, &base[total], result.elem_count - total,
                &num_read);
            if (num_read > 0 || rc != 0) {
                DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("KStreamRead"
                    "(sratoolkit.current.version, %zu) = %R\n", num_read, rc));
            }
            if (rc != 0) {
                /* TBD - look more closely at rc */
                if (num_read > 0) {
                    rc = 0;
                }
                else {
                    break;
                }
            }
            if (num_read == 0) {
                break;
            }
            total += num_read;
        }
        RELEASE(KStream, response);
        if (rc == 0) {
            DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
                ("sratoolkit.current.version (%zu)\n", total));
            result.elem_count = total;
        }
    }

    if (rc == 0) {
        const char *start = (const void*)(result.base);
        size_t size = KDataBufferBytes(&result);
        DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
            ("sratoolkit.current.version = '%.*s'\n", (uint32_t)size, start));
        rc = SraReleaseVersionInit(newVersion, start, size);
    }

    KDataBufferWhack(&result);
    RELEASE(KHttpResult, rslt);
    RELEASE(KHttpRequest, req);

    return rc;
}
Ejemplo n.º 12
0
/* InitDNSEndpoint
 *  initialize the endpoint with a DNS name and a port number
 *
 *  "ep" [ OUT ] - address of endpoint block to be intialized
 *
 *  "dns" [ IN ] - textual DNS address.
 *
 *  "port" [ IN, DEFAULT 0 ] - binary port number in native integer byte order.
 *   if the special port number 0 is given, it represents any available port.
 */
LIB_EXPORT
rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
    KEndPoint *ep, struct String const *dns, uint16_t port )
{
    rc_t rc = 0;

    if ( ep == NULL )
        rc = RC (rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
    else
    {
        if ( self == NULL )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcNull );
        else if ( dns == NULL )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcNull );
        else if ( dns -> size == 0 )
            rc = RC ( rcNS, rcNoTarg, rcInitializing, rcSelf, rcInsufficient );
        else
        {
            KDataBuffer b;
            char buffer [ 4096 ], * hostname = buffer;
            size_t buff_size = sizeof buffer;

            if ( dns -> size >= sizeof buffer )
            {
                rc = KDataBufferMakeBytes ( & b, dns -> size + 1 );
                if ( rc == 0 )
                {
                    hostname = b . base;
                    buff_size = ( size_t ) b . elem_count;
                }
            }

            if ( rc == 0 )
            {
                size_t size;
                rc = string_printf ( hostname, buff_size, & size, "%S", dns );

                assert ( rc == 0 );
                assert ( size < buff_size );
                assert ( hostname [ size ] == 0 );

                if ( rc ==  0 )
                {
                    int lerrno;
                    struct hostent *remote = gethostbyname ( hostname );
                    if ( remote != NULL )
                    { 
                        ep -> type = epIPV4;
                        memcpy ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
                        ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
                        ep -> u . ipv4 . port = ( uint16_t ) port;
                    }
                    else switch ( lerrno = WSAGetLastError () )
                    {
                    case WSANOTINITIALISED: /* Must have WSAStartup call */
                        rc = RC ( rcNS, rcNoTarg, rcInitializing, rcEnvironment, rcUndefined );
                        break;
                    case WSAENETDOWN:/* network subsystem failed */
                        rc = RC ( rcNS, rcNoTarg, rcInitializing, rcNoObj, rcFailed );
                        break;
                    case WSAHOST_NOT_FOUND: /* Answer host not found */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound );
                        break;
                    case WSATRY_AGAIN: /* host not found or server failure */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcBusy );
                        break;
                    case WSANO_RECOVERY: /* non-recoverable error */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcError );
                        break;
                    case WSANO_DATA: /* name is valid but no data */
                        rc = RC ( rcNS, rcNoTarg, rcValidating, rcConnection, rcEmpty );
                        break;
                    case WSAEINPROGRESS: /* call is in progress */
                        rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
                        break;
                    case WSAEFAULT: /* name paremeter is not valid part of addr space */
                        rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange );
                        break;
                    case WSAEINTR: /* socket call was calanceled */
                        rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
                        break;
                    default:
                        rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcError );
                    }
                }
            }

            if ( hostname != buffer )
                KDataBufferWhack ( & b );
        }

        if ( rc != 0 )
            memset ( ep, 0, sizeof * ep );        
    }

    return rc;
}
Ejemplo n.º 13
0
/* ReadAll
 *  read entire blob, plus any auxiliary checksum data
 *
 *  "buffer" [ OUT ] - pointer to a KDataBuffer structure that will be initialized
 *  and resized to contain the entire blob. upon success, will contain the number of bytes
 *  in buffer->elem_count and buffer->elem_bits == 8.
 *
 *  "opt_cs_data [ OUT, NULL OKAY ] - optional output parameter for checksum data
 *  associated with the blob in "buffer", if any exist.
 *
 *  "cs_data_size" [ IN ] - sizeof of * opt_cs_data if not NULL, 0 otherwise
 */
LIB_EXPORT rc_t CC KColumnBlobReadAll ( const KColumnBlob * self, KDataBuffer * buffer,
    KColumnBlobCSData * opt_cs_data, size_t cs_data_size )
{
    rc_t rc = 0;

    if ( opt_cs_data != NULL )
        memset ( opt_cs_data, 0, cs_data_size );

    if ( buffer == NULL )
        rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcNull );
    else
    {
        if ( self == NULL )
            rc = RC ( rcDB, rcBlob, rcReading, rcSelf, rcNull );
        else
        {
            /* determine blob size */
            size_t bsize = self -> loc . u . blob . size;

            /* ignore blobs of size 0 */
            if ( bsize == 0 )
                rc = 0;
            else
            {
                /* initialize the buffer */
                rc = KDataBufferMakeBytes ( buffer, bsize );
                if ( rc == 0 )
                {
                    /* read the blob */
                    size_t num_read, remaining;
                    rc = KColumnBlobRead ( self, 0, buffer -> base, bsize, & num_read, & remaining );
                    if ( rc == 0 )
                    {
                        /* test that num_read is everything and we have no remaining */
                        if ( num_read != bsize || remaining != 0 )
                            rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );

                        else
                        {
                            /* set for MD5 - just due to switch ordering */
                            size_t cs_bytes = 16;

                            /* if not reading checksum data, then we're done */
                            if ( opt_cs_data == NULL )
                                return 0;

                            /* see what checksumming is in use */
                            switch ( self -> col -> checksum )
                            {
                            case kcsNone:
                                return 0;

                            case kcsCRC32:
                                /* reset for CRC32 */
                                cs_bytes = 4;

                                /* no break */

                            case kcsMD5:
                                if ( cs_data_size < cs_bytes )
                                {
                                    rc = RC ( rcDB, rcBlob, rcReading, rcParam, rcTooShort );
                                    break;
                                }

                                /* read checksum information */
                                rc = KColumnDataRead ( & self -> col -> df,
                                    & self -> pmorig, bsize, opt_cs_data, cs_bytes, & num_read );
                                if ( rc == 0 )
                                {
                                    if ( num_read != cs_bytes )
                                        rc = RC ( rcDB, rcBlob, rcReading, rcTransfer, rcIncomplete );
                                    else
                                    {
                                        /* success - read the blob AND the checksum data */
                                        return 0;
                                    }
                                }
                                break;
                            }
                        }
                    }

                    KDataBufferWhack ( buffer );
                }
            }
        }

        memset ( buffer, 0, sizeof * buffer );
    }

    return rc;
}