Пример #1
0
/* Whack
 */
rc_t VCursorWhack ( VCursor *self )
{
#if VCURSOR_FLUSH_THREAD
    if ( self -> flush_thread != NULL )
    {
        rc_t rc = KLockAcquire ( self -> flush_lock );
        if ( rc == 0 )
        {
            while ( self -> flush_state == vfBusy )
            {
                MTCURSOR_DBG (( "VCursorWhack: waiting for thread to process\n" ));
                KConditionWait ( self -> flush_cond, self -> flush_lock );
            }
            self -> flush_state = vfExit;
            KConditionSignal ( self -> flush_cond );
            KLockUnlock ( self -> flush_lock );
        }

        MTCURSOR_DBG (( "VCursorWhack: waiting on thread to exit\n" ));
        KThreadWait ( self -> flush_thread, NULL );
    }

    MTCURSOR_DBG (( "VCursorWhack: finishing\n" ));
    KThreadRelease ( self -> flush_thread );
    KConditionRelease ( self -> flush_cond );
    KLockRelease ( self -> flush_lock );
#endif
    VCursorTerminatePagemapThread(self);
    return VCursorDestroy ( self );
}
Пример #2
0
/* Whack
 */
static
rc_t KQueueFileWhackRead ( KQueueFile *self )
{
    void *b;

    /* no more reading */
    QFMSG ( "%s: sealing queue\n", __func__ );
    KQueueSeal ( self -> q );

    /* flush the queue */
    QFMSG ( "%s: popping queue\n", __func__ );
    while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
    {
        QFMSG ( "%s: dousing a buffer\n", __func__ );
        free ( b );
    }

    /* wait for thread to exit */
    QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
    KThreadWait ( self -> t, NULL );

    /* tear it down */
    QFMSG ( "%s: freeing object\n", __func__ );
    free ( self -> b );
    KThreadRelease ( self -> t );
    KQueueRelease ( self -> q );
    KFileRelease ( self -> f );
    free ( self );

    return 0;
}
Пример #3
0
LIB_EXPORT rc_t CC VCursorFlushPage ( VCursor *self )
{
    rc_t rc = VCursorFlushPageInt ( self );
    if ( rc == 0 )
    {
#if VCURSOR_FLUSH_THREAD
        MTCURSOR_DBG (( "VCursorFlushPage: going to acquire lock\n" ));
        /* get lock */
        rc = KLockAcquire ( self -> flush_lock );
        if ( rc != 0 )
            return rc;

        MTCURSOR_DBG (( "VCursorFlushPage: have lock\n" ));

        /* wait until background thread has finished */
        while ( self -> flush_state == vfBusy )
        {
            MTCURSOR_DBG (( "VCursorFlushPage: waiting for background thread\n" ));
            rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
            if ( rc != 0 )
            {
                LOGERR ( klogSys, rc, "VCursorFlushPage: wait failed - exiting" );
                KLockUnlock ( self -> flush_lock );
                return rc;
            }
        }

        /* what was the proper rc */
        if ( self -> flush_state != vfReady )
        {
            if ( self -> flush_state != vfBgErr )
                rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
            else
            {
                rc_t rc2;
                MTCURSOR_DBG (( "VCursorFlushPage: waiting on thread to exit\n" ));
                rc = KThreadWait ( self -> flush_thread, & rc2 );
                if ( rc == 0 )
                {
                    rc = rc2;
                    MTCURSOR_DBG (( "VCursorFlushPage: releasing thread\n" ));
                    KThreadRelease ( self -> flush_thread );
                    self -> flush_thread = NULL;
                }
            }

            PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPage: not in ready state[$(state)] - exiting",
                                "state=%hu", self -> flush_state ));

            KLockUnlock ( self -> flush_lock );
            return rc;
        }
        KLockUnlock ( self -> flush_lock );
#endif
        assert ( self -> row_id == self -> start_id );
        self -> end_id = self -> row_id;
    }

    return rc;
}
Пример #4
0
static void BAMReaderWhack(BAMReader *const self)
{
    KThreadCancel(self->th);
    KThreadWait(self->th, NULL);
    BAMFileRelease(self->file);
    KConditionRelease(self->need_data);
    KConditionRelease(self->have_data);
    KLockRelease(self->lock);
    KThreadRelease(self->th);
}
Пример #5
0
void bg_update_release( bg_update * self )
{
    if ( self != NULL )
    {
        atomic_set( &self -> done, 1 );
        KThreadWait( self -> thread, NULL );
        KThreadRelease( self -> thread );
        free( ( void * ) self );
    }
}
Пример #6
0
static
rc_t KQueueFileWhackWrite ( KQueueFile *self )
{
    void *b;
    rc_t rc = 0;

    /* flush last buffer */
    if ( self -> b != NULL )
    {
        QFMSG ( "%s: have non-null buffer\n", __func__ );
        if ( self -> b -> bytes != 0 )
        {
            QFMSG ( "%s: buffer has %zu bytes\n", __func__, self -> b -> bytes );
            rc = KQueueFileFlush ( self, self -> b );
        }

        free ( self -> b );
        self -> b = NULL;
    }

    /* no more writing */
    QFMSG ( "%s: sealing queue\n", __func__ );
    KQueueSeal ( self -> q );

    /* wait for thread to exit */
    QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
    KThreadWait ( self -> t, NULL );

    /* the file should be written
       but flush the queue if bg thread didn't */
    QFMSG ( "%s: popping queue\n", __func__ );
    while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
    {
        QFMSG ( "%s: dousing a buffer\n", __func__ );
        free ( b );
    }

    /* tear it down */
    QFMSG ( "%s: freeing object\n", __func__ );
    KThreadRelease ( self -> t );
    KQueueRelease ( self -> q );
    KFileRelease ( self -> f );
    free ( self );

    return rc;
}
Пример #7
0
rc_t CC
_TaskStop ( const struct XTask * self )
{
    rc_t RCt;
    struct XTask * Task;

    RCt = 0;
    Task = ( struct XTask * ) self;

    if ( self == NULL ) {
        return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
    }

    KThreadCancel ( Task -> thread );
    KThreadWait ( Task -> thread, NULL );
    KThreadRelease ( Task -> thread );

    Task -> thread = NULL;

    return RCt;
}   /* TaskStop () */
Пример #8
0
rc_t execute_tbl_join( KDirectory * dir,
                    const char * accession_path,
                    const char * accession_short,
                    join_stats * stats,
                    const char * tbl_name,
                    const struct temp_dir * temp_dir,
                    struct temp_registry * registry,
                    size_t cur_cache,
                    size_t buf_size,
                    uint32_t num_threads,
                    bool show_progress,
                    format_t fmt,
                    const join_options * join_options )
{
    rc_t rc = 0;
    
    if ( show_progress )
        rc = KOutMsg( "join   :" );

    if ( rc == 0 )
    {
        uint64_t row_count = 0;
        rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */
        if ( rc == 0 && row_count > 0 )
        {
            bool name_column_present;

            if ( tbl_name == NULL )
                rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present );
            else
                rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present );
            
            if ( rc == 0 )
            {
                Vector threads;
                int64_t row = 1;
                uint32_t thread_id;
                uint64_t rows_per_thread;
                struct bg_progress * progress = NULL;
                struct join_options corrected_join_options; /* helper.h */
                
                VectorInit( &threads, 0, num_threads );
                
                corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true;
                corrected_join_options . skip_tech = join_options -> skip_tech;
                corrected_join_options . print_read_nr = join_options -> print_read_nr;
                corrected_join_options . print_name = name_column_present;
                corrected_join_options . min_read_len = join_options -> min_read_len;
                corrected_join_options . filter_bases = join_options -> filter_bases;
                corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid;
                
                if ( row_count < ( num_threads * 100 ) )
                {
                    num_threads = 1;
                    rows_per_thread = row_count;
                }
                else
                {
                    rows_per_thread = ( row_count / num_threads ) + 1;
                }
                
                if ( show_progress )
                    rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */
                
                for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id )
                {
                    join_thread_data * jtd = calloc( 1, sizeof * jtd );
                    if ( jtd != NULL )
                    {
                        jtd -> dir              = dir;
                        jtd -> accession_path   = accession_path;
                        jtd -> accession_short  = accession_short;
                        jtd -> tbl_name         = tbl_name;
                        jtd -> first_row        = row;
                        jtd -> row_count        = rows_per_thread;
                        jtd -> cur_cache        = cur_cache;
                        jtd -> buf_size         = buf_size;
                        jtd -> progress         = progress;
                        jtd -> registry         = registry;
                        jtd -> fmt              = fmt;
                        jtd -> join_options     = &corrected_join_options;

                        rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file,
                                    accession_short, thread_id ); /* temp_dir.c */

                        if ( rc == 0 )
                        {
                            rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd );
                            if ( rc != 0 )
                                ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc );
                            else
                            {
                                rc = VectorAppend( &threads, NULL, jtd );
                                if ( rc != 0 )
                                    ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc );
                            }
                            row += rows_per_thread;
                        }
                    }
                }
                
                {
                    /* collect the threads, and add the join_stats */
                    uint32_t i, n = VectorLength( &threads );
                    for ( i = VectorStart( &threads ); i < n; ++i )
                    {
                        join_thread_data * jtd = VectorGet( &threads, i );
                        if ( jtd != NULL )
                        {
                            rc_t rc_thread;
                            KThreadWait( jtd -> thread, &rc_thread );
                            if ( rc_thread != 0 )
                                rc = rc_thread;

                            KThreadRelease( jtd -> thread );
                            
                            add_join_stats( stats, &jtd -> stats );
                                
                            free( jtd );
                        }
                    }
                    VectorWhack ( &threads, NULL, NULL );
                }

                bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
            }
        }
    }
    return rc;
}
Пример #9
0
static 
rc_t 
WorkerThreadFn ( const KThread *self, void *data )  
{
    KStream* stream = (KStream*)data;
    char buf[256];
    size_t num;
    rc_t rc = KStreamReadAll(stream, buf, 1, &num);
    if (rc == 0)
    {
        if (num == 1) 
        {
            size_t toRead = (unsigned char)buf[0];
            pLogMsg(klogInfo, "KeyringServer: worker received length=$(l)\n", "l=%d", toRead);
            rc = KStreamReadAll(stream, buf, toRead, &num);
            if (rc == 0)
            {
                /*pLogMsg(klogInfo, "KeyringServer: worker received msg='$(buf)'\n", "buf=%.*s", num, buf);*/
                switch (buf[0])
                {
                case 'I':
                    if (buf[0] == 'I') /* init */
                    {
                        LogMsg ( klogInfo, "KeyringServer: received Init");

                        if (keyring == 0)
                        {
                            const KFile* std_in;
                            
                            rc = KFileMakeStdIn ( &std_in );
                            if (rc == 0)
                            {
                                KFile* std_out;
                                rc = KFileMakeStdOut ( &std_out );
                                if (rc == 0)
                                {
                                    rc = KeyRingOpen(&keyring, keyRingFileName, std_in, std_out);
                                    if (rc == 0)
                                    {
                                        LogMsg ( klogInfo, "KeyringServer: Init successful");
                                        pLogMsg(klogInfo, "KeyringServer: sending '$(buf)'\n", "buf=%.*s", string_size(initMsgSuccess), initMsgSuccess);
                                        rc = KStreamWrite(stream, initMsgSuccess, string_size(initMsgSuccess), NULL);
                                    }
                                    else
                                    {
                                        rc_t rc2;
                                        LogErr(klogErr, rc, "KeyringServer: Init failed");
                                        rc2 = KStreamWrite(stream, initMsgFailure, string_size(initMsgFailure), NULL);
                                        if (rc == 0)
                                            rc = rc2;
                                    }
                                    KFileRelease(std_out);
                                }
                                KFileRelease(std_in);
                            }
                        }
                        else
                        {   /* already running */
                            LogMsg ( klogInfo, "KeyringServer: Init successful");
                            rc = KStreamWrite(stream, initMsgSuccess, string_size(initMsgSuccess), NULL);
                        }
                    }
                    break;
                case 'X':
                    if (buf[0] == 'X') /* shutDown */
                    {
                        LogMsg ( klogInfo, "KeyringServer: received Shutdown");
                        shutDown = true;
                    }
                    break;
                case 'P': /* project */
                    if (toRead > 1 && buf[1] == 'A') /* Add */
                    {
                        String pkey;
                        String dlkey;
                        String enckey;

                        size_t idx = 2;
                        /*TODO: make sure idx is in range*/
                        StringInit(&pkey, buf + idx + 1, buf[idx], buf[idx]);
                        pkey.len = string_len(pkey.addr, pkey.size);

                        idx += pkey.size + 1;
                        /*TODO: make sure idx is in range*/
                        StringInit(&dlkey, buf + idx + 1, buf[idx], buf[idx]);
                        dlkey.len = string_len(dlkey.addr, dlkey.size);
                        
                        idx += dlkey.size + 1;
                        /*TODO: make sure idx is in range*/
                        StringInit(&enckey, buf + idx + 1, buf[idx], buf[idx]);
                        enckey.len = string_len(enckey.addr, enckey.size);
                        
                        pLogMsg(klogInfo,  
                                "KeyringServer: received Project Add(pkey='$(pkey)',dlkey='$(dlkey)',enckey='$(.....)')'\n",
                                "pkey=%.*s,dlkey=%.*s,enckey=%.*s", 
                                pkey.len, pkey.addr, dlkey.len, dlkey.addr, enckey.len, enckey.addr);
                        
                        rc = KeyRingAddProject(keyring, &pkey, &dlkey, &enckey);
                        if (rc != 0)
                            LogErr(klogErr, rc, "KeyringServer: KeyRingAddProject() failed");
                    }
                    break;
                default:
                    LogMsg ( klogInfo, "KeyringServer: unrecognised message received");
                    break;
                }
            }
            else
                LogErr(klogErr, rc, "KeyringServer: KStreamRead(body) failed");
        }
        else /* end of stream = the client closed the connection */
            LogMsg(klogInfo, "KeyringServer: worker received EOF\n");
    }
    else
        LogErr(klogErr, rc, "KeyringServer: KStreamRead(length) failed");
    LogMsg ( klogInfo, "KeyringServer: worker done");
    return KThreadRelease(self);
}
Пример #10
0
static
rc_t VCursorFlushPageInt ( VCursor *self )
{
    rc_t rc;

    if ( self == NULL )
        rc = RC ( rcVDB, rcCursor, rcFlushing, rcSelf, rcNull );
    else if ( self -> read_only )
        rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcReadonly );
    else
    {
        int64_t end_id;
#if ! VCURSOR_FLUSH_THREAD
        run_trigger_prod_data pb;
#endif
        switch ( self -> state )
        {
        case vcConstruct:
            rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcNotOpen );
            break;
        case vcFailed:
            rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInvalid );
            break;
        case vcRowOpen:
            rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcBusy );
            break;
        default:

            /* ignore request if there is no page to commit */
            if ( self -> start_id == self -> end_id )
            {
                /* the cursor should be in unwritten state,
                   where the row_id can be reset but drags
                   along the other markers. */
                assert ( self -> end_id == self -> row_id );
                return 0;
            }

#if VCURSOR_FLUSH_THREAD
            MTCURSOR_DBG (( "VCursorFlushPageInt: going to acquire lock\n" ));
            /* get lock */
            rc = KLockAcquire ( self -> flush_lock );
            if ( rc != 0 )
                return rc;

            MTCURSOR_DBG (( "VCursorFlushPageInt: have lock\n" ));

            /* make sure that background thread is ready */
            while ( self -> flush_state == vfBusy )
            {
                MTCURSOR_DBG (( "VCursorFlushPageInt: waiting for background thread\n" ));
                rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
                if ( rc != 0 )
                {
                    LOGERR ( klogSys, rc, "VCursorFlushPageInt: wait failed - exiting" );
                    KLockUnlock ( self -> flush_lock );
                    return rc;
                }
            }

            if ( self -> flush_state != vfReady )
            {
                if ( self -> flush_state != vfBgErr )
                    rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
                else
                {
                    rc_t rc2;
                    MTCURSOR_DBG (( "VCursorFlushPageInt: waiting on thread to exit\n" ));
                    rc = KThreadWait ( self -> flush_thread, & rc2 );
                    if ( rc == 0 )
                    {
                        rc = rc2;
                        MTCURSOR_DBG (( "VCursorFlushPageInt: releasing thread\n" ));
                        KThreadRelease ( self -> flush_thread );
                        self -> flush_thread = NULL;
                    }
                }

                PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPageInt: not in ready state[$(state)] - exiting","state=%hu",self -> flush_state ));
                KLockUnlock ( self -> flush_lock );
                return rc;
            }

            MTCURSOR_DBG (( "VCursorFlushPageInt: running buffer page\n" ));
#endif

            /* first, tell all columns to bundle up their pages into buffers */
            end_id = self -> end_id;
            rc = RC ( rcVDB, rcCursor, rcFlushing, rcMemory, rcExhausted );
            if ( VectorDoUntil ( & self -> row, false, WColumnBufferPage, & end_id ) )
            {
                VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
                self -> flush_state = vfFgErr;
            }
            else
            {
                /* supposed to be constant */
                assert ( end_id == self -> end_id );
#if VCURSOR_FLUSH_THREAD
                MTCURSOR_DBG (( "VCursorFlushPageInt: pages buffered - capturing id and count\n" ));
                self -> flush_id = self -> start_id;
                self -> flush_cnt = self -> end_id - self -> start_id;

                self -> start_id = self -> end_id;
                self -> end_id = self -> row_id + 1;
                self -> state = vcReady;

                MTCURSOR_DBG (( "VCursorFlushPageInt: state set to busy - signaling bg thread\n" ));
                self -> flush_state = vfBusy;
                rc = KConditionSignal ( self -> flush_cond );
                if ( rc != 0 )
                    LOGERR ( klogSys, rc, "VCursorFlushPageInt: condition returned error on signal" );
#else
                /* run all validation and trigger productions */
                pb . id = self -> start_id;
                pb . cnt = self -> end_id - self -> start_id;
                pb . rc = 0;
                if ( ! VectorDoUntil ( & self -> trig, false, run_trigger_prods, & pb ) )
                {
                    self -> start_id = self -> end_id;
                    self -> end_id = self -> row_id + 1;
                    self -> state = vcReady;
                }

                rc = pb . rc;

                /* drop page buffers */
                VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
#endif
            }

#if VCURSOR_FLUSH_THREAD
            MTCURSOR_DBG (( "VCursorFlushPageInt: unlocking\n" ));
            KLockUnlock ( self -> flush_lock );
#endif
        }
    }

    return rc;
}