/* Whack
*/
rc_t VCursorWhack ( VCursor *self )
{
#if VCURSOR_FLUSH_THREAD
if ( self -> flush_thread != NULL )
{
rc_t rc = KLockAcquire ( self -> flush_lock );
if ( rc == 0 )
{
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorWhack: waiting for thread to process\n" ));
KConditionWait ( self -> flush_cond, self -> flush_lock );
}
self -> flush_state = vfExit;
KConditionSignal ( self -> flush_cond );
KLockUnlock ( self -> flush_lock );
}
MTCURSOR_DBG (( "VCursorWhack: waiting on thread to exit\n" ));
KThreadWait ( self -> flush_thread, NULL );
}
MTCURSOR_DBG (( "VCursorWhack: finishing\n" ));
KThreadRelease ( self -> flush_thread );
KConditionRelease ( self -> flush_cond );
KLockRelease ( self -> flush_lock );
#endif
VCursorTerminatePagemapThread(self);
return VCursorDestroy ( self );
}
/* Whack
*/
static
rc_t KQueueFileWhackRead ( KQueueFile *self )
{
void *b;
/* no more reading */
QFMSG ( "%s: sealing queue\n", __func__ );
KQueueSeal ( self -> q );
/* flush the queue */
QFMSG ( "%s: popping queue\n", __func__ );
while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
{
QFMSG ( "%s: dousing a buffer\n", __func__ );
free ( b );
}
/* wait for thread to exit */
QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
KThreadWait ( self -> t, NULL );
/* tear it down */
QFMSG ( "%s: freeing object\n", __func__ );
free ( self -> b );
KThreadRelease ( self -> t );
KQueueRelease ( self -> q );
KFileRelease ( self -> f );
free ( self );
return 0;
}
LIB_EXPORT rc_t CC VCursorFlushPage ( VCursor *self )
{
rc_t rc = VCursorFlushPageInt ( self );
if ( rc == 0 )
{
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPage: going to acquire lock\n" ));
/* get lock */
rc = KLockAcquire ( self -> flush_lock );
if ( rc != 0 )
return rc;
MTCURSOR_DBG (( "VCursorFlushPage: have lock\n" ));
/* wait until background thread has finished */
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorFlushPage: waiting for background thread\n" ));
rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
if ( rc != 0 )
{
LOGERR ( klogSys, rc, "VCursorFlushPage: wait failed - exiting" );
KLockUnlock ( self -> flush_lock );
return rc;
}
}
/* what was the proper rc */
if ( self -> flush_state != vfReady )
{
if ( self -> flush_state != vfBgErr )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
else
{
rc_t rc2;
MTCURSOR_DBG (( "VCursorFlushPage: waiting on thread to exit\n" ));
rc = KThreadWait ( self -> flush_thread, & rc2 );
if ( rc == 0 )
{
rc = rc2;
MTCURSOR_DBG (( "VCursorFlushPage: releasing thread\n" ));
KThreadRelease ( self -> flush_thread );
self -> flush_thread = NULL;
}
}
PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPage: not in ready state[$(state)] - exiting",
"state=%hu", self -> flush_state ));
KLockUnlock ( self -> flush_lock );
return rc;
}
KLockUnlock ( self -> flush_lock );
#endif
assert ( self -> row_id == self -> start_id );
self -> end_id = self -> row_id;
}
return rc;
}
static
rc_t
Server( KNSManager* mgr )
{
rc_t rc = 0;
KEndPoint ep;
String name;
CONST_STRING(&name, KEYRING_IPC_NAME);
rc = KNSManagerInitIPCEndpoint(mgr, &ep, &name);
if (rc == 0)
{
KSocket* listener;
rc = KNSMakeListener ( &listener, &ep );
if (rc == 0)
{
shutDown = false;
while (!shutDown && rc == 0)
{
KStream* stream;
LogMsg ( klogInfo, "KeyringServer: listening");
rc = KNSListen ( listener, &stream ); /* may not return from here if no more incoming connections arrive */
if (rc == 0)
{
KThread* worker;
LogMsg ( klogInfo, "KeyringServer: detected connection");
rc = KThreadMake ( &worker, WorkerThreadFn, stream);
if (rc == 0 && worker != NULL)
{
KThreadWait(worker, NULL);
LogMsg ( klogInfo, "KeyringServer: out of worker");
}
else
LogErr(klogErr, rc, "KeyringServer: KThreadMake failed");
}
else
LogErr(klogErr, rc, "KeyringServer: KNSListen failed");
}
LogMsg ( klogInfo, "KeyringServer: shutting down");
/* TODO: make sure no incoming messages get dropped (stop accepting connections? wait for all active threads to exit?)
- lock the server */
if (keyring != NULL)
{
KeyRingRelease(keyring);
keyring = NULL;
}
KSocketRelease(listener);
}
else
LogErr(klogErr, rc, "KeyringServer: KNSMakeListener failed");
}
else
LogErr(klogErr, rc, "KeyringServer: KNSManagerInitIPCEndpoint failed");
LogMsg ( klogInfo, "KeyringServer: listener shut down");
return rc;
}
static void BAMReaderWhack(BAMReader *const self)
{
KThreadCancel(self->th);
KThreadWait(self->th, NULL);
BAMFileRelease(self->file);
KConditionRelease(self->need_data);
KConditionRelease(self->have_data);
KLockRelease(self->lock);
KThreadRelease(self->th);
}
void bg_update_release( bg_update * self )
{
if ( self != NULL )
{
atomic_set( &self -> done, 1 );
KThreadWait( self -> thread, NULL );
KThreadRelease( self -> thread );
free( ( void * ) self );
}
}
static
rc_t KQueueFileWhackWrite ( KQueueFile *self )
{
void *b;
rc_t rc = 0;
/* flush last buffer */
if ( self -> b != NULL )
{
QFMSG ( "%s: have non-null buffer\n", __func__ );
if ( self -> b -> bytes != 0 )
{
QFMSG ( "%s: buffer has %zu bytes\n", __func__, self -> b -> bytes );
rc = KQueueFileFlush ( self, self -> b );
}
free ( self -> b );
self -> b = NULL;
}
/* no more writing */
QFMSG ( "%s: sealing queue\n", __func__ );
KQueueSeal ( self -> q );
/* wait for thread to exit */
QFMSG ( "%s: waiting for bg thread to exit\n", __func__ );
KThreadWait ( self -> t, NULL );
/* the file should be written
but flush the queue if bg thread didn't */
QFMSG ( "%s: popping queue\n", __func__ );
while ( KQueuePop ( self -> q, & b, NULL ) == 0 )
{
QFMSG ( "%s: dousing a buffer\n", __func__ );
free ( b );
}
/* tear it down */
QFMSG ( "%s: freeing object\n", __func__ );
KThreadRelease ( self -> t );
KQueueRelease ( self -> q );
KFileRelease ( self -> f );
free ( self );
return rc;
}
rc_t CC
_TaskStop ( const struct XTask * self )
{
rc_t RCt;
struct XTask * Task;
RCt = 0;
Task = ( struct XTask * ) self;
if ( self == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
}
KThreadCancel ( Task -> thread );
KThreadWait ( Task -> thread, NULL );
KThreadRelease ( Task -> thread );
Task -> thread = NULL;
return RCt;
} /* TaskStop () */
rc_t execute_tbl_join( KDirectory * dir,
const char * accession_path,
const char * accession_short,
join_stats * stats,
const char * tbl_name,
const struct temp_dir * temp_dir,
struct temp_registry * registry,
size_t cur_cache,
size_t buf_size,
uint32_t num_threads,
bool show_progress,
format_t fmt,
const join_options * join_options )
{
rc_t rc = 0;
if ( show_progress )
rc = KOutMsg( "join :" );
if ( rc == 0 )
{
uint64_t row_count = 0;
rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */
if ( rc == 0 && row_count > 0 )
{
bool name_column_present;
if ( tbl_name == NULL )
rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present );
else
rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present );
if ( rc == 0 )
{
Vector threads;
int64_t row = 1;
uint32_t thread_id;
uint64_t rows_per_thread;
struct bg_progress * progress = NULL;
struct join_options corrected_join_options; /* helper.h */
VectorInit( &threads, 0, num_threads );
corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true;
corrected_join_options . skip_tech = join_options -> skip_tech;
corrected_join_options . print_read_nr = join_options -> print_read_nr;
corrected_join_options . print_name = name_column_present;
corrected_join_options . min_read_len = join_options -> min_read_len;
corrected_join_options . filter_bases = join_options -> filter_bases;
corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid;
if ( row_count < ( num_threads * 100 ) )
{
num_threads = 1;
rows_per_thread = row_count;
}
else
{
rows_per_thread = ( row_count / num_threads ) + 1;
}
if ( show_progress )
rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */
for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id )
{
join_thread_data * jtd = calloc( 1, sizeof * jtd );
if ( jtd != NULL )
{
jtd -> dir = dir;
jtd -> accession_path = accession_path;
jtd -> accession_short = accession_short;
jtd -> tbl_name = tbl_name;
jtd -> first_row = row;
jtd -> row_count = rows_per_thread;
jtd -> cur_cache = cur_cache;
jtd -> buf_size = buf_size;
jtd -> progress = progress;
jtd -> registry = registry;
jtd -> fmt = fmt;
jtd -> join_options = &corrected_join_options;
rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file,
accession_short, thread_id ); /* temp_dir.c */
if ( rc == 0 )
{
rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd );
if ( rc != 0 )
ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc );
else
{
rc = VectorAppend( &threads, NULL, jtd );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc );
}
row += rows_per_thread;
}
}
}
{
/* collect the threads, and add the join_stats */
uint32_t i, n = VectorLength( &threads );
for ( i = VectorStart( &threads ); i < n; ++i )
{
join_thread_data * jtd = VectorGet( &threads, i );
if ( jtd != NULL )
{
rc_t rc_thread;
KThreadWait( jtd -> thread, &rc_thread );
if ( rc_thread != 0 )
rc = rc_thread;
KThreadRelease( jtd -> thread );
add_join_stats( stats, &jtd -> stats );
free( jtd );
}
}
VectorWhack ( &threads, NULL, NULL );
}
bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
}
}
}
return rc;
}
static
rc_t VCursorFlushPageInt ( VCursor *self )
{
rc_t rc;
if ( self == NULL )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcSelf, rcNull );
else if ( self -> read_only )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcReadonly );
else
{
int64_t end_id;
#if ! VCURSOR_FLUSH_THREAD
run_trigger_prod_data pb;
#endif
switch ( self -> state )
{
case vcConstruct:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcNotOpen );
break;
case vcFailed:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInvalid );
break;
case vcRowOpen:
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcBusy );
break;
default:
/* ignore request if there is no page to commit */
if ( self -> start_id == self -> end_id )
{
/* the cursor should be in unwritten state,
where the row_id can be reset but drags
along the other markers. */
assert ( self -> end_id == self -> row_id );
return 0;
}
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: going to acquire lock\n" ));
/* get lock */
rc = KLockAcquire ( self -> flush_lock );
if ( rc != 0 )
return rc;
MTCURSOR_DBG (( "VCursorFlushPageInt: have lock\n" ));
/* make sure that background thread is ready */
while ( self -> flush_state == vfBusy )
{
MTCURSOR_DBG (( "VCursorFlushPageInt: waiting for background thread\n" ));
rc = KConditionWait ( self -> flush_cond, self -> flush_lock );
if ( rc != 0 )
{
LOGERR ( klogSys, rc, "VCursorFlushPageInt: wait failed - exiting" );
KLockUnlock ( self -> flush_lock );
return rc;
}
}
if ( self -> flush_state != vfReady )
{
if ( self -> flush_state != vfBgErr )
rc = RC ( rcVDB, rcCursor, rcFlushing, rcCursor, rcInconsistent );
else
{
rc_t rc2;
MTCURSOR_DBG (( "VCursorFlushPageInt: waiting on thread to exit\n" ));
rc = KThreadWait ( self -> flush_thread, & rc2 );
if ( rc == 0 )
{
rc = rc2;
MTCURSOR_DBG (( "VCursorFlushPageInt: releasing thread\n" ));
KThreadRelease ( self -> flush_thread );
self -> flush_thread = NULL;
}
}
PLOGERR ( klogInt, (klogInt, rc, "VCursorFlushPageInt: not in ready state[$(state)] - exiting","state=%hu",self -> flush_state ));
KLockUnlock ( self -> flush_lock );
return rc;
}
MTCURSOR_DBG (( "VCursorFlushPageInt: running buffer page\n" ));
#endif
/* first, tell all columns to bundle up their pages into buffers */
end_id = self -> end_id;
rc = RC ( rcVDB, rcCursor, rcFlushing, rcMemory, rcExhausted );
if ( VectorDoUntil ( & self -> row, false, WColumnBufferPage, & end_id ) )
{
VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
self -> flush_state = vfFgErr;
}
else
{
/* supposed to be constant */
assert ( end_id == self -> end_id );
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: pages buffered - capturing id and count\n" ));
self -> flush_id = self -> start_id;
self -> flush_cnt = self -> end_id - self -> start_id;
self -> start_id = self -> end_id;
self -> end_id = self -> row_id + 1;
self -> state = vcReady;
MTCURSOR_DBG (( "VCursorFlushPageInt: state set to busy - signaling bg thread\n" ));
self -> flush_state = vfBusy;
rc = KConditionSignal ( self -> flush_cond );
if ( rc != 0 )
LOGERR ( klogSys, rc, "VCursorFlushPageInt: condition returned error on signal" );
#else
/* run all validation and trigger productions */
pb . id = self -> start_id;
pb . cnt = self -> end_id - self -> start_id;
pb . rc = 0;
if ( ! VectorDoUntil ( & self -> trig, false, run_trigger_prods, & pb ) )
{
self -> start_id = self -> end_id;
self -> end_id = self -> row_id + 1;
self -> state = vcReady;
}
rc = pb . rc;
/* drop page buffers */
VectorForEach ( & self -> row, false, WColumnDropPage, NULL );
#endif
}
#if VCURSOR_FLUSH_THREAD
MTCURSOR_DBG (( "VCursorFlushPageInt: unlocking\n" ));
KLockUnlock ( self -> flush_lock );
#endif
}
}
return rc;
}
