Exemple #1
0
/* Make
 *  make a memory bank
 *
 *  "block_size" [ IN ] - ignored
 *
 *  "limit" [ IN, DFLT ZERO ] - ignored
 *
 *  "backing" [ IN ] - required.
 */
LIB_EXPORT rc_t CC KMemBankMake ( KMemBank **bankp,
    size_t block_size, uint64_t limit, KFile *backing )
{
    assert(backing != NULL);
    assert(bankp != NULL);
    if (bankp == NULL || backing == NULL) {
        PLOGMSG(klogFatal, (klogFatal, "PROGRAMMER ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__));
        abort();
        return RC(rcFS, rcMemory, rcConstructing, rcParam, rcNull);
    }
    else {
        KMemBank *const self = newMemBank();
        if (self == NULL) {
            PLOGMSG(klogFatal, (klogFatal, "OUT OF MEMORY - " __FILE__ ":$(line)", "line=%i", __LINE__));
            abort(); 
            return RC(rcFS, rcMemory, rcConstructing, rcMemory, rcExhausted);
        }
        if (backing) {
            self->pf = backing;
            KFileAddRef(self->pf);
        }
        *bankp = self;
        return 0;
    }
}
Exemple #2
0
static
rc_t XML_Open(const char* path, const FSNode** tree)
{
    rc_t rc = 0;
    char errmsg[4096] = "";
    KDirectory *dir = NULL;
    
    PLOGMSG(klogInfo, (klogInfo, "Reading XML file '$(x)'", PLOG_S(x), path));
    if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
        const KFile* file = NULL;
        if( (rc = KDirectoryOpenFileRead(dir, &file, "%s", path)) == 0 ) {
            if( (rc = FSNode_Make((FSNode**)tree, "ROOT", &RootNode_vtbl)) == 0 ) {
                const KXMLDoc* xmldoc = NULL;
                if( (rc = KXMLMgrMakeDocRead(g_xmlmgr, &xmldoc, file)) == 0 ) {
                    const KXMLNodeset* ns = NULL;
                    if( (rc = KXMLDocOpenNodesetRead(xmldoc, &ns, "/FUSE/*")) == 0 ) {
                        uint32_t count = 0;
                        if( (rc = KXMLNodesetCount(ns, &count)) == 0 ) {
                            if( count == 0 ) {
                                rc = RC(rcExe, rcDoc, rcValidating, rcData, rcEmpty);
                            } else {
                                uint32_t i = 0;
                                while(rc == 0 && i < count) {
                                    const KXMLNode* n = NULL;
                                    if( (rc = KXMLNodesetGetNodeRead(ns, &n, i++)) == 0 ) {
                                        SRAConfigFlags flags = ~0;
                                        errmsg[0] = '\0';
                                        rc = XML_ValidateNode((FSNode*)*tree, n, flags, errmsg);
                                        ReleaseComplain(KXMLNodeRelease, n);
                                    }
                                }
                                if( rc == 0 ) {
                                    rc = SRAList_NextVersion();
                                }
                            }
                        }
                        ReleaseComplain(KXMLNodesetRelease, ns);
                    }
                    ReleaseComplain(KXMLDocRelease, xmldoc);
                }
                if( rc != 0 ) {
                    FSNode_Release(*tree);
                    *tree = NULL;
                }
            }
            ReleaseComplain(KFileRelease, file);
        }
        ReleaseComplain(KDirectoryRelease, dir);
    }
    if( rc == 0 ) {
        PLOGMSG(klogInfo, (klogInfo, "XML file '$(x)' ok", PLOG_S(x), path));
    } else {
        if( strlen(errmsg) < 1 ) {
            strcpy(errmsg, path);
        }
        LOGERR(klogErr, rc, errmsg);
    }
    return rc;
}
Exemple #3
0
static rc_t MaxNReadsValidator_GetKey( const SRASplitter* cself,
                                       const char** key, spotid_t spot, readmask_t* readmask )
{
    rc_t rc = 0;
    MaxNReadsValidator* self = ( MaxNReadsValidator* )cself;

    if ( self == NULL || key == NULL )
    {
        rc = RC( rcSRA, rcNode, rcExecuting, rcParam, rcNull );
    }
    else
    {
        const void* nreads = NULL;
        bitsz_t o = 0, sz = 0;
        uint64_t nn = 0;

        *key = "";
        if ( self->col != NULL )
        {
            rc = SRAColumnRead( self->col, spot, &nreads, &o, &sz );
            if ( rc == 0 )
            {
                switch( sz )
                {
                case 8:
                    nn = *((const uint8_t*)nreads);
                    break;
                case 16:
                    nn = *((const uint16_t*)nreads);
                    break;
                case 32:
                    nn = *((const uint32_t*)nreads);
                    break;
                case 64:
                    nn = *((const uint64_t*)nreads);
                    break;
                default:
                    rc = RC( rcSRA, rcNode, rcExecuting, rcData, rcUnexpected );
                    break;
                }
                if ( nn > nreads_max )
                {
                    clear_readmask( readmask );
                    PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), maximum $(max) supported, skipped",
                                       PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn, spot, nreads_max));
                }
                else if ( nn == nreads_max - 1 )
                {
                    PLOGMSG(klogWarn, (klogWarn, "too many reads $(nreads) at spot id $(row), truncated to $(max)",
                                       PLOG_3(PLOG_U64(nreads),PLOG_I64(row),PLOG_U32(max)), nn + 1, spot, nreads_max));
                }
            }
        }
    }
    return rc;
}
rc_t runChecks(const TestCase& test_case, const VCursor * cursor, uint32_t name_idx, uint32_t name_range_idx)
{
    rc_t rc;
    int64_t first_id;
    uint64_t count_id;
    
    rc = VCursorIdRange( cursor, name_idx, &first_id, &count_id );
    if (rc != 0)
    {
        LOGERR( klogInt, rc, "VCursorIdRange() failed" );
        return rc;
    }
    
    for (uint64_t row_id = first_id; row_id < first_id + count_id; ++row_id)
    {
        const char * name = NULL;
        uint32_t name_len;
        RowRange *row_range;
        
        rc = VCursorCellDataDirect( cursor, row_id, name_idx, NULL, (void const **)&name, NULL, &name_len );
        if ( rc != 0 )
            return rc;
        
        rc = VCursorParamsSet( ( struct VCursorParams const * )cursor, "QUERY_NAME", "%.*s", name_len, name );
        if ( rc != 0 )
            return rc;
        
        rc = VCursorCellDataDirect( cursor, row_id, name_range_idx, NULL, (void const **)&row_range, NULL, NULL );
        if ( rc != 0 )
            return rc;
        
        std::string name_str(name, name_len);
        
        if (test_case.key_ranges.find(name_str) == test_case.key_ranges.end())
        {
            PLOGMSG( klogInt, (klogErr, "Unexpected name '$(NAME)' in test case '$(TC_NAME)'", "TC_NAME=%s,NAME=%s", test_case_name, name_str.c_str()) );
            return 1;
        }
        
        RowRange row_range_exp = test_case.key_ranges.find(name_str)->second;
        if (row_range->start_id != row_range_exp.start_id || row_range->stop_id != row_range_exp.stop_id)
        {
            PLOGMSG( klogInt, (klogErr, "Row range for name '$(NAME)' in test case '$(TC_NAME)' does not match. Expected: $(EXP_S)-$(EXP_F), actual: $(ACT_S)-$(ACT_F)",
                               "TC_NAME=%s,NAME=%s,EXP_S=%ld,EXP_F=%ld,ACT_S=%ld,ACT_F=%ld",
                               test_case_name, name_str.c_str(), row_range_exp.start_id, row_range_exp.stop_id, row_range->start_id, row_range->stop_id) );
            return 1;
        }
    }
    
    return rc;
}
Exemple #5
0
static
rc_t stats_data_update_group(stats_data_t *self,
                             int64_t spot_id,
                             uint32_t spot_len,
                             uint32_t bio_spot_len,
                             uint32_t cmp_spot_len,
                             char const grp[],
                             uint64_t grp_len)
{
    if (grp_len == 0 || grp == NULL || grp[0] == '\0' ||
        (grp_len == 7 && strncasecmp("default", grp, 7) == 0))
    {
        group_stats_update(&self->deflt, spot_id, spot_len, bio_spot_len, cmp_spot_len);
    }
    else
    {
        group_stats_t *const fnd = stats_data_get_group(self, (unsigned)grp_len, grp);
        
        if (fnd == NULL)
            return RC(rcXF, rcFunction, rcConstructing, rcMemory, rcExhausted);
        
        if (fnd - get_group(self, 0) < MAX_GROUP_COUNT)
            group_stats_update(fnd, spot_id, spot_len, bio_spot_len, cmp_spot_len);
        else {
            KDataBufferWhack(&self->group);
            KDataBufferWhack(&self->names);
            stats_data_init_funcs(self, false);
            (void)PLOGMSG(klogWarn, (klogWarn, "Too many spot groups ($(count)); dropping group stats", "count=%u", (unsigned)(self->count)));
        }
    }
    return 0;
}
Exemple #6
0
rc_t ProcessOneDo (ProcessOne * self)
{
    static const char F[] = PLOG_2(PLOG_S(p),PLOG_S(t));
    enum KPathType type;
    rc_t rc = 0;
    
    type = KDirectoryPathType (self->dir, self->path);

    switch (type)
    {
    case kptFile:
	rc = ProcessOneDoFile (self);
 	break;
/*     case kptDir: */
/* 	break; */
/*     case kptAlias|kptFile: */
/* 	break; */
/*     case kptAlias|kptDir: */
/* 	break; */
    default:
	PLOGMSG (klogInfo, "+ Skipping $(p) of type $(t)", F, self->path, typeToString(type));
	break;
    }
    return rc;
}
Exemple #7
0
rc_t ProcessOneDoFile (ProcessOne * self)
{
    rc_t rc = 0;
    KFile * mfile;
    
    PLOGMSG (klogInfo, "ProcessOneDoFile: $(F)", PLOG_S(F), self->path);


    rc = KFileMakeNewMD5Read (&mfile, self->file, self->md5, self->path);
    if (rc == 0)
    {
	const KFile * bfile;
	rc = KFileMakeBuf (&bfile, mfile, 64*1024);
	if (rc == 0)
	{
	    /* add more here */

	    KFileRelease (bfile);
	}
	else
	{
	    pLOGERR (klogErr, rc, "Failure to initiate buffer $(F)", PLOG_S(F), self->path);
	    KFileRelease (mfile);
	}
    }
    else
	pLOGERR (klogErr, rc, "Failure to initiate MD5 summing $(F)", PLOG_S(F), self->path);

    return rc;
}
Exemple #8
0
rc_t BufferQPopBuffer (BufferQ * self, const Buffer ** buff, timeout_t * tm)
{
    rc_t rc = 0;
    timeout_t t;
    void * p;
    LOGMSG (klogDebug10, "BufferQPopBuffer");
    assert (self != NULL);
    assert (buff != NULL);

    if (tm == NULL)
    {
	LOGMSG (klogDebug10, "BufferQPopBuffer tm was NULL");
	tm = &t;
	rc = TimeoutInit (tm, self->timeout);
    }

    if (rc == 0)
    {
	LOGMSG (klogDebug10, "BufferQPopBuffer call KQueuePop");
	rc = KQueuePop (self->q, &p, tm);
	PLOGMSG (klogDebug10, "BufferQPopBuffer back from KQueuePop $(rc)", PLOG_U32(rc), rc);
	if (rc == 0)
	    *buff = p;
	else
	{
	    *buff = NULL;
	}
    }
    LOGMSG (klogDebug10, "leave BufferQPopBuffer");
    return rc;
}
Exemple #9
0
rc_t load_consensus_src( con_ctx * sctx, KDirectory * hdf5_src )
{
    BaseCalls_cmn ConsensusTab;

    rc_t rc = 0;
    if ( sctx->lctx->check_src_obj )
        rc = check_src_objects( hdf5_src, consensus_groups_to_check, 
                                consensus_tables_to_check, false );
    if ( rc == 0 )
        rc = open_BaseCalls_cmn( hdf5_src, &ConsensusTab, true,
                                 "PulseData/ConsensusBaseCalls", sctx->lctx->cache_content, true );
    if ( rc == 0 )
    {
        uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
        uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];

        KLogLevel tmp_lvl = KLogLevelGet();
        KLogLevelSet( klogInfo );
        PLOGMSG( klogInfo, ( klogInfo,
                 "loading consensus-table ( $(bases) bases / $(spots) spots ):",
                 "bases=%lu,spots=%lu", total_bases, total_spots ));
        KLogLevelSet( tmp_lvl );

        if ( !check_Consensus_totalcount( &ConsensusTab, total_bases ) )
            rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
        else
            rc = zmw_for_each( &ConsensusTab.zmw, &sctx->lctx->xml_progress, sctx->cursor,
                               sctx->lctx->with_progress, sctx->col_idx, NULL,
                               true, consensus_load_spot, &ConsensusTab );
        close_BaseCalls_cmn( &ConsensusTab );
    }
    return rc;
}
Exemple #10
0
static
rc_t VProdResolveColExpr ( const VProdResolve *self, VProduction **out,
    VFormatdecl *fd, const SSymExpr *x, bool casting )
{
    rc_t rc;
    const SNameOverload *sname;
    const KSymbol *sym = x -> _sym;

    BSTree ordered;
    uint32_t i, count;
    SColumnBestFit buff [ 16 ], * nodes = buff;

    /* fail if "fd" has a format */
    if ( fd -> fmt != 0 )
    {
        PLOGMSG ( klogWarn, ( klogWarn, "illegal cast of column '$(name)'"
                   , "name=%.*s"
                   , ( int ) sym -> name . size
                   , sym -> name . addr ));
        return 0;
    }

    /* allocate nodes for indexing columns */
    sname = sym -> u . obj;
    count = VectorLength ( & sname -> items );
    if ( count > sizeof buff / sizeof buff [ 0 ] )
    {
        nodes = malloc ( sizeof * nodes * count );
        if ( nodes == NULL )
            return RC ( rcVDB, rcProduction, rcResolving, rcMemory, rcExhausted );
    }

    /* insert columns into ordered tree */
    BSTreeInit ( & ordered );
    for ( i = VectorStart ( & sname -> items ), count += i; i < count; ++ i )
    {
        /* get SColumn */
        nodes [ i ] . scol = ( const void* ) VectorGet ( & sname -> items, i );

        /* perform type cast and measure distance */
        if ( casting ?
             VTypedeclCommonAncestor ( & nodes [ i ] . scol -> td, self -> schema,
                 & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) :
             VTypedeclToTypedecl ( & nodes [ i ] . scol -> td, self -> schema,
                 & fd -> td, & nodes [ i ] . td, & nodes [ i ] . distance ) )
        {
            BSTreeInsert ( & ordered, & nodes [ i ] . n, order_column );
        }
    }

    /* try to resolve each in order */
    rc = VProdResolveBestColumn ( self, out, & ordered, x -> alt );

    if ( nodes != buff )
        free ( nodes );

    return rc;
}
Exemple #11
0
static
rc_t CCCopyDoDirectory (CCCopy * self)
{
    rc_t rc = 0;
    PLOGMSG (klogInfo, "CCCopyDoDirectory $(d)", PLOG_S(d), self->path);

    rc = RC (rcExe, rcDirectory, rcCopying, rcParam, rcUnsupported);
    return rc;
}
Exemple #12
0
static rc_t consensus_loader( ld_context *lctx, const char * table_name,
                              bool cache_content )
{
    uint32_t col_idx[ consensus_tab_count ];
    rc_t rc = add_columns( lctx->cursor, consensus_tab_count, -1,
                           col_idx, consensus_tab_names );
    if ( rc == 0 )
    {
        rc = VCursorOpen( lctx->cursor );
        if ( rc != 0 )
            LOGERR( klogErr, rc, "cannot open cursor on consensus-table" );

        else
        {
            BaseCalls_cmn ConsensusTab;
            const INSDC_SRA_platform_id platform = SRA_PLATFORM_PACBIO_SMRT;

            rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_PLATFORM ],
                                  sizeof platform * 8, &platform, 0, 1 );
            if ( rc != 0 )
                LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for platform-column" );
            else
            {
                const INSDC_SRA_read_filter filter = SRA_READ_FILTER_PASS;
                rc = VCursorDefault ( lctx->cursor, col_idx[ consensus_tab_READ_FILTER ],
                                  sizeof filter * 8, &filter, 0, 1 );
                if ( rc != 0 )
                    LOGERR( klogErr, rc, "cannot set cursor-default on consensus-table for read-filter-column" );
            }

            if ( rc == 0 )
                rc = open_BaseCalls_cmn( lctx->hdf5_dir, &ConsensusTab, true,
                                     "PulseData/ConsensusBaseCalls", cache_content, true );

            if ( rc == 0 )
            {
                uint64_t total_bases = zmw_total( &ConsensusTab.zmw );
                uint64_t total_spots = ConsensusTab.zmw.NumEvent.extents[ 0 ];

                KLogLevel tmp_lvl = KLogLevelGet();
                KLogLevelSet( klogInfo );
                PLOGMSG( klogInfo, ( klogInfo,
                         "loading consensus-table ( $(bases) bases / $(spots) spots ):",
                         "bases=%lu,spots=%lu", total_bases, total_spots ));
                KLogLevelSet( tmp_lvl );

                if ( check_Consensus_totalcount( &ConsensusTab, total_bases ) )
                    rc = zmw_for_each( &ConsensusTab.zmw, lctx, col_idx, NULL,
                                       true, consensus_load_spot, &ConsensusTab );
                else
                    rc = RC( rcExe, rcNoTarg, rcAllocating, rcParam, rcInvalid );
                close_BaseCalls_cmn( &ConsensusTab );
            }
        }
    }
    return rc;
}
Exemple #13
0
rc_t CopierDoOne (Copier * self)
{
    rc_t rc = 0;
    const Buffer * b;

    LOGMSG (klogDebug10, "CopierDoOne");
    rc = Quitting();
    if (rc == 0)
    {
	LOGMSG (klogDebug10, "call BufferQPopBuffer");
	rc = BufferQPopBuffer (self->q, &b, NULL);
	if (rc == 0)
	{
	    size_t w;
	    size_t z;
	    LOGMSG (klogDebug10, "call BufferContentGetSize");
	    z = BufferContentGetSize (b);
	    rc = KFileWrite (self->f, self->o, b, z, &w);
	    self->o += w;
	    if (w != z)
		rc = RC (rcExe, rcFile, rcWriting, rcTransfer, rcIncomplete);
	    else
		rc = BufferRelease (b);
	}
	/* ow this is ugly! */
	/* is the rc a "exhausted" on a timeout? */
	else if ((GetRCObject(rc) == rcTimeout) && (GetRCState(rc) == rcExhausted))
	{
	    rc = 0;
	    LOGMSG (klogDebug10, "CopierDoOne timeout");
	    /* if so is the queue also sealed? */
	    if (BufferQSealed (self->q) == true)
	    {
		LOGMSG (klogDebug10, "CopierDoOne sealed");
		/* if both then we are done and so signal */
		rc = KFileRelease (self->f);
		PLOGMSG (klogDebug10, "CopierDoOne back from KFileRelease $(rc)",PLOG_U32(rc),rc);
		if (rc == 0)
		{
		    self->f = NULL;
		    rc = BufferQRelease (self->q);
		    if (rc == 0)
		    {
			self->q = NULL;
			rc = RC (rcExe, rcNoTarg, rcCopying, rcNoTarg, rcDone );
		    }
		}
	    }
	}
	else
	    LOGMSG (klogDebug10, "CopierDoOne pop failure");

    }
    else
	LOGMSG (klogDebug10, "CopierDoOne: quitting");
    return rc;
}
Exemple #14
0
rc_t CGWriterAlgn_Whack(const CGWriterAlgn* cself, bool commit, uint64_t* rows_1st, uint64_t* rows_2nd)
{
    rc_t rc = 0;
    if( cself != NULL ) {
        CGWriterAlgn* self = (CGWriterAlgn*)cself;
        rc_t rc1 = TableWriterAlgn_Whack(cself->primary, commit, rows_1st);
        rc_t rc2 = TableWriterAlgn_Whack(cself->secondary, commit, rows_2nd);
        if( self->forced_pairs_cnt > 0 ) {
            PLOGMSG(klogInfo, (klogInfo, "$(forced_pairs_cnt) forced pairs to PRIMARY", "forced_pairs_cnt=%lu", self->forced_pairs_cnt));
        }
        if( self->dropped_mates_cnt > 0 ) {
            PLOGMSG(klogInfo, (klogInfo, "$(dropped_mates_cnt) dropped duplicate mates in SECONDARY", "dropped_mates_cnt=%lu", self->dropped_mates_cnt));
        }
        rc = rc1 ? rc1 : rc2;
        free(self);
    }
    return rc;
}
Exemple #15
0
static rc_t passes_load_loop( ld_context *lctx, VCursor * cursor, Passes_src *tab,
                              uint32_t *col_idx )
{
    rc_t rc = 0;
    KLogLevel tmp_lvl;
    pass_block block;
    pl_progress *progress;
    uint64_t pos = 0;
    uint64_t total_passes = tab->AdapterHitBefore.extents[0];

    pl_progress_make( &progress, total_passes );
    rc = progress_chunk( &lctx->xml_progress, total_passes );

    tmp_lvl = KLogLevelGet();
    KLogLevelSet( klogInfo );
    PLOGMSG( klogInfo, ( klogInfo,
                         "loading passes-table ( $(rows) rows ) :",
                         "rows=%lu", total_passes ));
    KLogLevelSet( tmp_lvl );

    while( pos < total_passes && rc == 0 )
    {
        rc = pass_block_read_from_src( tab, total_passes, pos, &block );
        if ( rc == 0 )
        {
            uint32_t i;
            for ( i = 0; i < block.n_read && rc == 0; ++i )
            {
                rc = Quitting();
                if ( rc == 0 )
                {
                    rc = passes_load_pass( cursor, &block, i, col_idx );
                    if ( rc == 0 )
                    {
                        rc = progress_step( lctx->xml_progress );
                        if ( lctx->with_progress )
                            pl_progress_increment( progress, 1 );
                    }
                }
                else
                    LOGERR( klogErr, rc, "...loading passes interrupted" );
            }
            pos += block.n_read;
        }
    }

    pl_progress_destroy( progress );

    if ( rc == 0 )
    {
        rc = VCursorCommit( cursor );
        if ( rc != 0 )
            LOGERR( klogErr, rc, "cannot commit cursor on PASSES-tab" );
    }
    return rc;
}
Exemple #16
0
rc_t WriteFileMeta(SIndexObj* obj)
{
    rc_t rc = 0;
    KMDataNode* nd = NULL;

    PLOGMSG(klogInfo, (klogInfo, "Meta $(f) on index $(i): file size $(s), buffer $(b)",
        PLOG_4(PLOG_S(f),PLOG_S(i),PLOG_U64(s),PLOG_U32(b)), obj->file, obj->index, obj->file_size, obj->buffer_sz));

    if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Format")) == 0 ) {
        KMDataNode* opt = NULL;
        rc = KMDataNodeWriteCString(nd, obj->format);
        if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(nd, &opt, "Options")) == 0 ) {
            KMDataNode* ond = NULL;
            if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "accession")) == 0 ) {
                rc = KMDataNodeWriteCString(ond, g_accession);
                KMDataNodeRelease(ond);
            }
            if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "minSpotId")) == 0 ) {
                rc = KMDataNodeWriteB64(ond, &obj->minSpotId);
                KMDataNodeRelease(ond);
            }
            if( rc == 0 && (rc = KMDataNodeOpenNodeUpdate(opt, &ond, "maxSpotId")) == 0 ) {
                rc = KMDataNodeWriteB64(ond, &obj->maxSpotId);
                KMDataNodeRelease(ond);
            }
            KMDataNodeRelease(opt);
        }
        KMDataNodeRelease(nd);
    }

    if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Size")) == 0 ) {
        rc = KMDataNodeWriteB64(nd, &obj->file_size);
        KMDataNodeRelease(nd);
    }

    if( rc == 0 && obj->buffer_sz > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Buffer")) == 0 ) {
        rc = KMDataNodeWriteB32(nd, &obj->buffer_sz);
        KMDataNodeRelease(nd);
    }

    if( rc == 0 && strlen(obj->index) > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "Index")) == 0 ) {
        rc = KMDataNodeWriteCString(nd, obj->index);
        KMDataNodeRelease(nd);
    }

    if( rc == 0 && obj->file_size > 0 && (rc = KMDataNodeOpenNodeUpdate(obj->meta, &nd, "md5")) == 0 ) {
        char x[5];
        int i;
        for( i = 0; rc == 0 && i < sizeof(obj->md5_digest); i++ ) {
            int l = snprintf(x, 4, "%02x", obj->md5_digest[i]);
            rc = KMDataNodeAppend(nd, x, l);
        }
        KMDataNodeRelease(nd);
    }
    return rc;
}
Exemple #17
0
static
rc_t XMLThread( const KThread *self, void *data )
{
    KDirectory *dir = NULL;

    PLOGMSG(klogInfo, (klogInfo, "XML sync thread started with $(s) sec", PLOG_U32(s), g_xml_sync));
    do {
        rc_t rc = 0;
        KTime_t dt = 0;

        DEBUG_MSG(8, ("XML sync thread checking %s\n", g_xml_path));
        if( (rc = KDirectoryNativeDir(&dir)) == 0 ) {
            rc = KDirectoryDate(dir, &dt, "%s", g_xml_path);
            ReleaseComplain(KDirectoryRelease, dir);
        }
        if( rc == 0 ) {
            if( dt != g_xml_mtime ) {
                const FSNode* new_root = NULL;
                PLOGMSG(klogInfo, (klogInfo, "File $(f) changed ($(m) <> $(d)), updating...",
                    PLOG_3(PLOG_S(f),PLOG_I64(m),PLOG_I64(d)), g_xml_path, g_xml_mtime, dt));
                if( XML_Open(g_xml_path, &new_root) == 0 ) {
                    if( (rc = XMLLock(true)) == 0 ) {
                        const FSNode* old_root = g_root;
                        g_root = new_root;
                        g_xml_mtime = dt;
                        XMLUnlock();
                        FSNode_Release(old_root);
                        PLOGMSG(klogInfo, (klogInfo, "Data from $(f) updated successfully", PLOG_S(f), g_xml_path));
                    }
                }
            } else {
                DEBUG_MSG(8, ("XML sync thread up-to-date %s\n", g_xml_path));
            }
        } else {
            LOGERR(klogErr, rc, g_xml_path);
        }
        SRAList_PostRefresh();
        sleep(g_xml_sync);
    } while( g_xml_sync > 0 );
    LOGMSG(klogInfo, "XML sync thread ended");
    return 0;
}
Exemple #18
0
    void get_ref_var_object (KSearch::CVRefVariation& obj, size_t ref_pos, size_t del_len,
        char const* allele, size_t allele_len, ngs::ReferenceSequence const& ref_seq,
        std::string & ref_allele)
    {
        size_t var_len = allele_len;

        size_t chunk_size = 5000; // TODO: add the method Reference[Sequence].getChunkSize() to the API
        size_t chunk_no = ref_pos / chunk_size;
        size_t ref_pos_in_slice = ref_pos % chunk_size;
        size_t bases_start = chunk_no * chunk_size;
        size_t chunk_no_last = ref_seq.getLength() / chunk_size;

        bool cont = false;
        size_t chunk_no_start = chunk_no, chunk_no_end = chunk_no;

        // optimization: first look into the current chunk only (using ngs::StringRef)
        {
            ngs::StringRef ref_chunk = ref_seq.getReferenceChunk ( bases_start );

            if ( ! check_ref_slice (ref_chunk.data() + ref_pos_in_slice, del_len) )
            {
                PLOGMSG ( klogWarn,
                    ( klogWarn,
                    "The selected reference region [$(REFSLICE)] does not contain valid bases, skipping...",
                    "REFSLICE=%.*s",
                    (int)del_len, ref_chunk.data() + ref_pos_in_slice ));
            }
            
            cont = Common::find_variation_core_step ( obj, g_Params.alg,
                ref_chunk.data(), ref_chunk.size(), ref_pos_in_slice,
                allele, var_len, del_len,
                chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );

            if ( !cont )
                ref_allele.assign (ref_chunk.data() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
        }

        // general case - expanding ref_slice to multiple chunks
        if ( cont )
        {
            ngs::String ref_slice;
            while ( cont )
            {
                ref_slice = ref_seq.getReferenceBases (
                    bases_start, (chunk_no_end - chunk_no_start + 1)*chunk_size );

                cont = Common::find_variation_core_step ( obj, g_Params.alg,
                    ref_slice.c_str(), ref_slice.size(), ref_pos_in_slice,
                    allele, var_len, del_len,
                    chunk_size, chunk_no_last, bases_start, chunk_no_start, chunk_no_end );
            }
            ref_allele.assign (ref_slice.c_str() + obj.GetAlleleStartRelative(), obj.GetAlleleLenOnRef());
        }
    }
static rc_t Run(const CmdLine* args)
{
    rc_t rc = 0;

    Db db;
    SpotIterator it;

    assert(args);

    if (!SpotIteratorFileExists(args->file)) {
        rc = RC(rcExe, rcFile, rcOpening, rcFile, rcNotFound);
        PLOGERR(klogErr,
            (klogErr, rc, "Cannot find '$(path)'", "path=%s", args->file));
    }
    else if (!SpotIteratorFileExists(args->table)) {
        rc = RC(rcExe, rcTable, rcOpening, rcTable, rcNotFound);
        PLOGERR(klogErr,
            (klogErr, rc, "Cannot find '$(path)'", "path=%s", args->table));
    }

    {
        rc_t rc2 = DbInit(rc, args, &db);
        if (rc == 0)
        {   rc = rc2; }
    }

    if (rc == 0) {
        rc = SpotIteratorInit(args->file, &db, &it);
    }

    if (rc == 0) {
        rc = Work(&db, &it);
    }

    if (rc == 0) {
        PLOGMSG(klogInfo, (klogInfo,
            "Success: redacted $(redacted) spots out of $(all)",
            "redacted=%d,all=%d", db.redactedSpots, db.nSpots));
    }

    {
        rc_t rc2 = SpotIteratorDestroy(&it);
        if (rc == 0)
        {   rc = rc2; }
    }

    {
        rc_t rc2 = DbDestroy(&db);
        if (rc == 0)
        {   rc = rc2; }
    }

    return rc;
}
Exemple #20
0
static void seq_load_info( regions_stat * stat )
{
    KLogLevel tmp_lvl = KLogLevelGet();
    KLogLevelSet( klogInfo );

    if ( stat->expands_a > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "adapter expanded: $(times) ( in $(spots) spots )",
             "times=%u,spots=%u",
             stat->expands_a, stat->expands_spots ));
    if ( stat->expands_i > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "insert expanded : $(times) ( in $(spots) spots )",
             "times=%u,spots=%u",
              stat->expands_i, stat->expands_spots ));
    if ( stat->inserts > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "regions inserted: $(times) ( in $(spots) spots )",
             "times=%u,spots=%u",
             stat->inserts, stat->inserts_spots ));
    if ( stat->end_gap > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "end-gap inserted: $(times)",
             "times=%u", stat->end_gap ));
    if ( stat->overlapps > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "overlapping rngs: $(times)",
             "times=%u", stat->overlapps ));
    if ( stat->removed > 0 )
        PLOGMSG( klogInfo, ( klogInfo,
             "removed rgns    : $(times)",
             "times=%u", stat->removed ));

    KLogLevelSet( tmp_lvl );
}
Exemple #21
0
    int expand_variations (int argc, char** argv)
    {
        int ret;
        try
        {
            KApp::CArgs args;
            args.MakeAndHandle (argc, argv, Options, countof (Options));

            if (args.GetOptionCount (OPTION_ALG) == 1)
            {
                char const* alg = args.GetOptionValue ( OPTION_ALG, 0 );
                if (!strcmp(alg, PARAM_ALG_SW))
                    g_Params.alg = ::refvarAlgSW;
                else if (!strcmp(alg, PARAM_ALG_RA))
                    g_Params.alg = ::refvarAlgRA;
                else
                {
                    PLOGMSG ( klogErr, ( klogErr,
                        "Error: Unknown algorithm specified: \"$(ALG)\"", "ALG=%s", alg ));
                    return 3;
                }
            }

            ret = expand_variations_impl ();
        }
        catch ( ngs::ErrorMsg const& e )
        {
            PLOGMSG ( klogErr,
                ( klogErr, "ngs::ErrorMsg: $(WHAT)", "WHAT=%s", e.what()
                ));
            ret = 3;
        }
        catch (...)
        {
            Utils::HandleException ();
            ret = 3;
        }

        return ret;
    }
Exemple #22
0
rc_t CCCopyDo (CCCopy * self)
{
    rc_t rc = 0;
    enum KPathType type;

    assert (self != NULL);
    assert (self->path != NULL);

    type = KDirectoryVPathType (self->in, self->path, NULL);
    switch (type & ~kptAlias)
    {
    case kptNotFound:
	rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcNotFound);
	break;
    default:
    case kptBadPath:
	rc = RC (rcExe, rcPath, rcAccessing, rcPath, rcInvalid);
	break;
    case kptFile:
	rc = CCCopyDoFile (self);
	break;
    case kptDir:
#if 0
	rc = CCCopyDoDirectory (self);
#else
	rc = PLOGMSG (klogInfo, "Ignoring directory $(p)",PLOG_S(p),self->path);
#endif
	break;
    case kptCharDev:
	rc = PLOGMSG (klogInfo, "Ignoring kptCharDev $(p)",PLOG_S(p),self->path);
	break;
    case kptBlockDev:
	rc = PLOGMSG (klogInfo, "Ignoring kptBlockDev $(p)",PLOG_S(p),self->path);
	break;
    case kptFIFO:
	rc = PLOGMSG (klogInfo, "Ignoring kptFIFO $(p)",PLOG_S(p),self->path);
	break;
    }
    return rc;
}
Exemple #23
0
/* OpenUpdate
 *  finish create operation
 */
static
rc_t VDatabaseOpenUpdate ( VDatabase *self, const char *decl )
{
    /* open metadata */
    rc_t rc = KDatabaseOpenMetadataUpdate ( self -> kdb, & self -> meta );
    if ( rc == 0 )
    {
        /* fetch stored schema */
        rc = VDatabaseLoadSchema ( self );
        if ( rc == 0 )
        {
            /* fetch requested schema */
            const SDatabase *sdb = self -> sdb;
            if ( decl != NULL && decl [ 0 ] != 0 )
            {
                uint32_t type;
                const SNameOverload *name;
                sdb = ( self -> dad != NULL ) ?
                    SDatabaseFind ( self -> dad -> sdb, self -> schema,
                        & name, & type, decl, "VDatabaseOpenUpdate" ):
                    VSchemaFind ( self -> schema,
                        & name, & type, decl, "VDatabaseOpenUpdate", true );
                if ( sdb != NULL && type != eDatabase )
                {
                    PLOGMSG ( klogWarn, ( klogWarn, "expression '$(expr)' is not a database",
                               "expr=%s", decl ));
                    sdb = NULL;
                }
            }

            /* error if the two definitions differ */
            if ( sdb != NULL && self -> sdb != NULL && sdb != self -> sdb )
                rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcIncorrect );
            else if ( sdb == NULL && self -> sdb == NULL )
                rc = RC ( rcVDB, rcDatabase, rcOpening, rcSchema, rcNotFound );
            else
            {
                if ( sdb != NULL )
                    self -> sdb = sdb;

                /* write schema to metadata */
                rc = VDatabaseStoreSchema ( self );
                if ( rc == 0 )
                    return 0;
            }
        }
    }

    DBGMSG(DBG_VDB, DBG_FLAG(DBG_VDB_VDB), ("VDatabaseOpenUpdate = %d\n", rc));

    return rc;
}
Exemple #24
0
/* AddRef
 * Release
 */
LIB_EXPORT rc_t CC KMemBankAddRef(const KMemBank *cself)
{
    KMemBank *const self = (KMemBank *)cself;
    if (self != NULL) {
        int const rslt = KRefcountAdd(&self->refcount, "KMemBank");
        assert(rslt == krefOkay);
        if (rslt != krefOkay) {
            PLOGMSG(klogFatal, (klogFatal, "LOGIC ERROR - " __FILE__ ":$(line)", "line=%i", __LINE__));
            abort();
            return RC(rcFS, rcMemory, rcAttaching, rcConstraint, rcViolated);
        }
    }
    return 0;
}
Exemple #25
0
static
rc_t VProdResolveProdExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym )
{
    const SProduction *sprod = sym -> u . obj;
    if ( ! sprod -> trigger )
        return VProdResolveSProduction ( self, out, sprod );

    PLOGMSG ( klogWarn, ( klogWarn, "trigger production '$(trig)' used in expression"
                          , "trig=%.*s"
                          , ( int ) sym -> name . size
                          , sym -> name . addr ));

    return 0;
}
Exemple #26
0
static rc_t remove_path( KDirectory * dir, const char * path, bool quiet )
{
    rc_t rc;

    if ( !quiet )
    {
        PLOGMSG( klogInfo, ( klogInfo, "removing '$(path)'", "path=%s", path ));
    }
    rc = KDirectoryRemove ( dir, true, path );
    if ( rc != 0 )
    {
        LOGERR( klogErr, rc, "remove_path:KDirectoryRemove() failed" );
    }
    return rc;
}
Exemple #27
0
/* ParamExpr
 *  resolve a simple parameter by name
 */
LIB_EXPORT rc_t CC VProdResolveParamExpr ( const VProdResolve *self, VProduction **out, const KSymbol *sym )
{
    const SProduction *sprod = sym -> u . obj;
    VProduction *vprod = VCursorCacheGet ( self -> cache, & sprod -> cid );
    if ( vprod != NULL )
    {
        * out = vprod;
        return 0;
    }

    PLOGMSG ( klogWarn, ( klogWarn, "unknown parameter '$(param)' used in expression"
                          , "param=%.*s"
                          , ( int ) sprod -> name -> name . size
                          , sprod -> name -> name . addr ));
    return 0;
}
Exemple #28
0
static
rc_t VProdResolvePhysExpr ( const VProdResolve *self,
    VProduction **out, const KSymbol *sym )
{
    if ( self -> chain == chainEncoding )
    {
        assert ( self -> curs -> read_only == false );
        PLOGMSG ( klogWarn, ( klogWarn, "illegal access of physical column '$(name)'"
                   , "name=%.*s"
                   , ( int ) sym -> name . size
                   , sym -> name . addr ));
        return 0;
    }

    return VProdResolveSPhysMember ( self, out, sym -> u . obj );
}
Exemple #29
0
static int string_buffer_add(KDataBuffer *const self,
                             unsigned const len, char const name[/* len */])
{
    size_t const rslt = self->elem_count;
    size_t const newsize = rslt + len;
    
    if (KDataBufferResize(self, newsize) != 0)
        return -1;
    
#if 1
    PLOGMSG(klogInfo, (klogInfo, "New spot group '$(NAME)'", "NAME=%.*s", (int)len, name));
#endif
    
    memcpy(((char *)self->base) + rslt, name, len);
    
    return (int)rslt;
}
Exemple #30
0
void seq_report_totals( ld_context *lctx )
{
    const char* accession;
    KLogLevel tmp_lvl = KLogLevelGet();
    KLogLevelSet( klogInfo );

    accession = strrchr( lctx->dst_path, '/' );
    if( accession == NULL )
        accession = lctx->dst_path;
    else
        accession++;

    PLOGMSG( klogInfo, ( klogInfo, "loaded",
            "severity=total,status=success,accession=%s,spot_count=%lu,base_count=%lu,bad_spots=0",
             accession, lctx->total_seq_spots, lctx->total_seq_bases ));

    KLogLevelSet( tmp_lvl );
}