static void CC on_merge( void *item, void *data )
{
on_merge_ctx * omc = data;
if ( item != NULL )
{
merge_data * md = calloc( 1, sizeof * md );
if ( md != NULL )
{
rc_t rc;
KThread * thread;
md -> cmn = omc -> cmn;
md -> files = item;
md -> idx = omc -> idx;
rc = KThreadMake( &thread, merge_thread_func, md );
if ( rc != 0 )
ErrMsg( "KThreadMake( on_merge #%d ) -> %R", omc -> idx, rc );
else
{
rc = VectorAppend( &omc -> threads, NULL, thread );
if ( rc != 0 )
ErrMsg( "VectorAppend( merge-thread #%d ) -> %R", omc -> idx, rc );
}
}
}
omc -> idx ++;
}
rc_t run_sorter_pool( const sorter_params * params )
{
rc_t rc = 0;
uint64_t row_count = find_out_row_count( params );
if ( row_count == 0 )
{
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
ErrMsg( "multi_threaded_make_lookup: row_count == 0!" );
}
else
{
cmn_params cp;
Vector threads;
KThread * progress_thread = NULL;
uint32_t prefix = 1;
multi_progress progress;
init_progress_data( &progress, row_count );
VectorInit( &threads, 0, params->num_threads );
init_cmn_params( &cp, params, row_count );
if ( params->show_progress )
rc = start_multi_progress( &progress_thread, &progress );
while ( rc == 0 && cp.first < row_count )
{
sorter_params * sp = calloc( 1, sizeof *sp );
if ( sp != NULL )
{
init_sorter_params( sp, params, prefix++ );
rc = make_raw_read_iter( &cp, &sp->src );
if ( rc == 0 )
{
KThread * thread;
if ( params->show_progress )
sp->sort_progress = &progress.progress_rows;
rc = KThreadMake( &thread, sort_thread_func, sp );
if ( rc != 0 )
ErrMsg( "KThreadMake( sort-thread #%d ) -> %R", prefix - 1, rc );
else
{
rc = VectorAppend( &threads, NULL, thread );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", prefix - 1, rc );
}
}
cp.first += cp.count;
}
}
join_and_release_threads( &threads );
/* all sorter-threads are done now, tell the progress-thread to terminate! */
join_multi_progress( progress_thread, &progress );
rc = merge_pool_files( params );
}
return rc;
}
static
rc_t
Server( KNSManager* mgr )
{
rc_t rc = 0;
KEndPoint ep;
String name;
CONST_STRING(&name, KEYRING_IPC_NAME);
rc = KNSManagerInitIPCEndpoint(mgr, &ep, &name);
if (rc == 0)
{
KSocket* listener;
rc = KNSMakeListener ( &listener, &ep );
if (rc == 0)
{
shutDown = false;
while (!shutDown && rc == 0)
{
KStream* stream;
LogMsg ( klogInfo, "KeyringServer: listening");
rc = KNSListen ( listener, &stream ); /* may not return from here if no more incoming connections arrive */
if (rc == 0)
{
KThread* worker;
LogMsg ( klogInfo, "KeyringServer: detected connection");
rc = KThreadMake ( &worker, WorkerThreadFn, stream);
if (rc == 0 && worker != NULL)
{
KThreadWait(worker, NULL);
LogMsg ( klogInfo, "KeyringServer: out of worker");
}
else
LogErr(klogErr, rc, "KeyringServer: KThreadMake failed");
}
else
LogErr(klogErr, rc, "KeyringServer: KNSListen failed");
}
LogMsg ( klogInfo, "KeyringServer: shutting down");
/* TODO: make sure no incoming messages get dropped (stop accepting connections? wait for all active threads to exit?)
- lock the server */
if (keyring != NULL)
{
KeyRingRelease(keyring);
keyring = NULL;
}
KSocketRelease(listener);
}
else
LogErr(klogErr, rc, "KeyringServer: KNSMakeListener failed");
}
else
LogErr(klogErr, rc, "KeyringServer: KNSManagerInitIPCEndpoint failed");
LogMsg ( klogInfo, "KeyringServer: listener shut down");
return rc;
}
rc_t VTableCreateCursorWriteInt ( VTable *self, VCursor **cursp, KCreateMode mode, bool create_thread )
{
rc_t rc;
if ( cursp == NULL )
rc = RC ( rcVDB, rcCursor, rcCreating, rcParam, rcNull );
else
{
if ( self == NULL )
rc = RC ( rcVDB, rcTable, rcAccessing, rcSelf, rcNull );
else if ( self -> read_only )
rc = RC ( rcVDB, rcCursor, rcCreating, rcTable, rcReadonly );
#if VCURSOR_WRITE_MODES_SUPPORTED
#error "expecting kcmInsert mode only"
#else
else if ( mode != kcmInsert )
rc = RC ( rcVDB, rcCursor, rcCreating, rcMode, rcUnsupported );
#endif
else
{
VCursor *curs;
#if LAZY_OPEN_COL_NODE
if ( self -> col_node == NULL )
KMetadataOpenNodeUpdate ( self -> meta, & self -> col_node, "col" );
#endif
rc = VCursorMake ( & curs, self );
if ( rc == 0 )
{
rc = VCursorSupplementSchema ( curs );
#if VCURSOR_FLUSH_THREAD
if ( rc == 0 && create_thread )
{
rc = KLockMake ( & curs -> flush_lock );
if ( rc == 0 )
rc = KConditionMake ( & curs -> flush_cond );
if ( rc == 0 )
rc = KThreadMake ( & curs -> flush_thread, run_flush_thread, curs );
}
if(rc == 0)
rc = VCursorLaunchPagemapThread(curs);
#endif
if ( rc == 0 )
{
* cursp = curs;
return 0;
}
VCursorRelease ( curs );
}
}
* cursp = NULL;
}
return rc;
}
void XML_Init(void)
{
rc_t rc = 0;
if( g_lock == NULL && (rc = KRWLockMake(&g_lock)) != 0 ) {
g_lock = NULL;
LOGERR(klogErr, rc, "XML lock");
}
if( (rc = KThreadMake(&g_xml_thread, XMLThread, NULL)) != 0 ) {
LOGERR(klogErr, rc, "XML sync thread");
}
}
rc_t BAMReaderMake( const BAMReader **result,
char const headerText[],
char const path[] )
{
rc_t rc;
BAMReader *self = malloc(sizeof(BAMReader));
if ( self == NULL )
{
*result = NULL;
return RC(rcAlign, rcFile, rcConstructing, rcMemory, rcExhausted);
}
else
{
atomic32_set( & self->refcount, 1 );
rc = BAMFileMakeWithHeader( & self->file, headerText, "%s", path);
if ( rc != 0 )
{
free(self);
*result = 0;
}
else
*result = self;
}
self->nque = 0;
self->rc = 0;
self->eof = false;
rc = KLockMake(&self->lock);
if (rc == 0)
{
rc = KConditionMake(&self->have_data);
if (rc == 0)
{
rc = KConditionMake(&self->need_data);
if (rc == 0)
{
rc = KThreadMake(&self->th, BAMReaderThreadMain, self);
if (rc == 0)
return 0;
KConditionRelease(self->need_data);
}
KConditionRelease(self->have_data);
}
KLockRelease(self->lock);
}
return rc;
}
rc_t bg_update_make( bg_update ** bga, uint32_t sleep_time )
{
rc_t rc = 0;
bg_update * p = calloc( 1, sizeof *p );
if ( p == NULL )
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
else
{
p -> sleep_time = sleep_time == 0 ? 200 : sleep_time;
rc = KThreadMake( & p -> thread, bg_update_thread_func, p );
if ( rc == 0 )
*bga = p;
else
free( p );
}
return rc;
}
rc_t bg_progress_make( bg_progress ** bgp, uint64_t max_value, uint32_t sleep_time, uint32_t digits )
{
rc_t rc = 0;
bg_progress * p = calloc( 1, sizeof *p );
if ( p == NULL )
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
else
{
atomic64_set( &p -> max_value, max_value );
p -> sleep_time = sleep_time == 0 ? 200 : sleep_time;
p -> digits = digits == 0 ? 2 : digits;
rc = KThreadMake( & p -> thread, bg_progress_thread_func, p );
if ( rc == 0 )
*bgp = p;
else
free( p );
}
return rc;
}
static
rc_t CC
_TaskRun ( const struct XTask * self )
{
struct XTask * Task = ( struct XTask * ) self;
if ( self == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcNull );
}
if ( self -> tasker == NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
if ( self -> thread != NULL ) {
return RC ( rcExe, rcNoTarg, rcProcessing, rcParam, rcInvalid );
}
atomic32_read_and_add (
( atomic32_t * ) & ( Task -> tasker -> is_run ), 1
);
return KThreadMake ( & ( Task -> thread ), _TaskThreadProc, Task );
} /* _TaskRun () */
/* MakeWrite
* make a queue file for writing-behind on background thread
*
* when the file is created, a background thread is started that
* waits for buffers to appear in the cross-thread queue. as the producer
* thread writes, data are accumulated into buffers which are pushed
* into the queue as they fill, and written in turn on the bg thread.
*
* the producer thread is throttled by queue capacity - determined by
* "queue_bytes" and "block_size", such that if the queue is full,
* the thread will sleep. the background thread is also throttled by
* the queue in that it will sleep if the queue is empty with pending
* data.
*
* the background thread will exit upon a permanent error, or if the
* queue is sealed by the producer thread.
*
* when the file is collected in response to a release message,
* the queue will be sealed against further inserts, pending buffers
* will be written, the background thread will be joined, and
* the source file will be released.
*
* the intended usage is serial writing of the file. random writes
* will be accepted, but may reduce the queue efficiency.
*
* "qf" [ OUT ] - return parameter for queue file
*
* "dst" [ IN ] - destination file for write-behind on background thread.
* must have write permissions.
*
* "queue_bytes" [ IN ] - the write-behind limit of the producer
* thread, in bytes. this is the amount of data that will be queued
* for the background thread before the producer thread sleeps.
*
* "block_size" [ IN, DEFAULT ZERO ] - optional parameter giving
* desired block size when writing to "dst". this may be used
* to tune writing for source data, e.g. 64K blocks for gzip.
*/
LIB_EXPORT rc_t CC KQueueFileMakeWrite ( KFile **qfp,
KFile *dst, size_t queue_bytes, size_t block_size, uint32_t timeout_ms )
{
rc_t rc;
if ( qfp == NULL )
rc = RC ( rcApp, rcFile, rcConstructing, rcParam, rcNull );
else
{
if ( dst == NULL )
rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcNull );
else if ( ! dst -> write_enabled )
{
if ( dst -> read_enabled )
rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcReadonly );
else
rc = RC ( rcApp, rcFile, rcConstructing, rcFile, rcNoPerm );
}
else
{
KQueueFile *qf = malloc ( sizeof * qf );
if ( qf == NULL )
rc = RC ( rcApp, rcFile, rcConstructing, rcMemory, rcExhausted );
else
{
rc = KFileInit ( & qf -> dad, ( const KFile_vt* ) & KQueueFileWrite_vt_v1, "KQueueFile", "no-name", false, true );
if ( rc == 0 )
{
qf -> f = dst;
rc = KFileAddRef ( dst );
if ( rc == 0 )
{
uint32_t capacity;
/* zero block size means default */
if ( block_size == 0 )
block_size = DEFAULT_BLOCK_SIZE;
/* queue capacity is expressed in bytes originally
translate to buffer count */
capacity = ( queue_bytes + block_size - 1 ) / block_size;
if ( capacity == 0 )
capacity = 1;
/* actual capacity will be a power of 2 */
rc = KQueueMake ( & qf -> q, capacity );
if ( rc == 0 )
{
/* capture current size if supported */
qf -> rc_from_size_on_open = KFileSize ( dst, & qf -> apparent_size );
/* starting position not used */
qf -> start_pos = 0;
/* requested buffer size */
qf -> b = NULL;
qf -> bsize = block_size;
/* timeout */
qf -> timeout_ms = timeout_ms == 0 ? DEFAULT_TIMEOUT_MS : timeout_ms;
/* finally, start the background thread */
rc = KThreadMake ( & qf -> t, KQueueFileRunWrite, qf );
if ( rc == 0 )
{
* qfp = & qf -> dad;
return 0;
}
KQueueRelease ( qf -> q );
}
KFileRelease ( qf -> f );
}
}
free ( qf );
}
}
* qfp = NULL;
}
return rc;
}
rc_t execute_tbl_join( KDirectory * dir,
const char * accession_path,
const char * accession_short,
join_stats * stats,
const char * tbl_name,
const struct temp_dir * temp_dir,
struct temp_registry * registry,
size_t cur_cache,
size_t buf_size,
uint32_t num_threads,
bool show_progress,
format_t fmt,
const join_options * join_options )
{
rc_t rc = 0;
if ( show_progress )
rc = KOutMsg( "join :" );
if ( rc == 0 )
{
uint64_t row_count = 0;
rc = extract_sra_row_count( dir, accession_path, tbl_name, cur_cache, &row_count ); /* above */
if ( rc == 0 && row_count > 0 )
{
bool name_column_present;
if ( tbl_name == NULL )
rc = cmn_check_tbl_column( dir, accession_path, "NAME", &name_column_present );
else
rc = cmn_check_db_column( dir, accession_path, tbl_name, "NAME", &name_column_present );
if ( rc == 0 )
{
Vector threads;
int64_t row = 1;
uint32_t thread_id;
uint64_t rows_per_thread;
struct bg_progress * progress = NULL;
struct join_options corrected_join_options; /* helper.h */
VectorInit( &threads, 0, num_threads );
corrected_join_options . rowid_as_name = name_column_present ? join_options -> rowid_as_name : true;
corrected_join_options . skip_tech = join_options -> skip_tech;
corrected_join_options . print_read_nr = join_options -> print_read_nr;
corrected_join_options . print_name = name_column_present;
corrected_join_options . min_read_len = join_options -> min_read_len;
corrected_join_options . filter_bases = join_options -> filter_bases;
corrected_join_options . terminate_on_invalid = join_options -> terminate_on_invalid;
if ( row_count < ( num_threads * 100 ) )
{
num_threads = 1;
rows_per_thread = row_count;
}
else
{
rows_per_thread = ( row_count / num_threads ) + 1;
}
if ( show_progress )
rc = bg_progress_make( &progress, row_count, 0, 0 ); /* progress_thread.c */
for ( thread_id = 0; rc == 0 && thread_id < num_threads; ++thread_id )
{
join_thread_data * jtd = calloc( 1, sizeof * jtd );
if ( jtd != NULL )
{
jtd -> dir = dir;
jtd -> accession_path = accession_path;
jtd -> accession_short = accession_short;
jtd -> tbl_name = tbl_name;
jtd -> first_row = row;
jtd -> row_count = rows_per_thread;
jtd -> cur_cache = cur_cache;
jtd -> buf_size = buf_size;
jtd -> progress = progress;
jtd -> registry = registry;
jtd -> fmt = fmt;
jtd -> join_options = &corrected_join_options;
rc = make_joined_filename( temp_dir, jtd -> part_file, sizeof jtd -> part_file,
accession_short, thread_id ); /* temp_dir.c */
if ( rc == 0 )
{
rc = KThreadMake( &jtd -> thread, cmn_thread_func, jtd );
if ( rc != 0 )
ErrMsg( "KThreadMake( fastq/special #%d ) -> %R", thread_id, rc );
else
{
rc = VectorAppend( &threads, NULL, jtd );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", thread_id, rc );
}
row += rows_per_thread;
}
}
}
{
/* collect the threads, and add the join_stats */
uint32_t i, n = VectorLength( &threads );
for ( i = VectorStart( &threads ); i < n; ++i )
{
join_thread_data * jtd = VectorGet( &threads, i );
if ( jtd != NULL )
{
rc_t rc_thread;
KThreadWait( jtd -> thread, &rc_thread );
if ( rc_thread != 0 )
rc = rc_thread;
KThreadRelease( jtd -> thread );
add_join_stats( stats, &jtd -> stats );
free( jtd );
}
}
VectorWhack ( &threads, NULL, NULL );
}
bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
}
}
}
return rc;
}
