rc_t run_sorter_pool( const sorter_params * params )
{
rc_t rc = 0;
uint64_t row_count = find_out_row_count( params );
if ( row_count == 0 )
{
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
ErrMsg( "multi_threaded_make_lookup: row_count == 0!" );
}
else
{
cmn_params cp;
Vector threads;
KThread * progress_thread = NULL;
uint32_t prefix = 1;
multi_progress progress;
init_progress_data( &progress, row_count );
VectorInit( &threads, 0, params->num_threads );
init_cmn_params( &cp, params, row_count );
if ( params->show_progress )
rc = start_multi_progress( &progress_thread, &progress );
while ( rc == 0 && cp.first < row_count )
{
sorter_params * sp = calloc( 1, sizeof *sp );
if ( sp != NULL )
{
init_sorter_params( sp, params, prefix++ );
rc = make_raw_read_iter( &cp, &sp->src );
if ( rc == 0 )
{
KThread * thread;
if ( params->show_progress )
sp->sort_progress = &progress.progress_rows;
rc = KThreadMake( &thread, sort_thread_func, sp );
if ( rc != 0 )
ErrMsg( "KThreadMake( sort-thread #%d ) -> %R", prefix - 1, rc );
else
{
rc = VectorAppend( &threads, NULL, thread );
if ( rc != 0 )
ErrMsg( "VectorAppend( sort-thread #%d ) -> %R", prefix - 1, rc );
}
}
cp.first += cp.count;
}
}
join_and_release_threads( &threads );
/* all sorter-threads are done now, tell the progress-thread to terminate! */
join_multi_progress( progress_thread, &progress );
rc = merge_pool_files( params );
}
return rc;
}
/* -------------------------------------------------------------------- */
rc_t temp_registry_merge( temp_registry * self,
KDirectory * dir,
const char * output_filename,
size_t buf_size,
bool show_progress,
bool force,
compress_t compress )
{
rc_t rc = 0;
if ( self == NULL )
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
else if ( output_filename == NULL )
rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcNull );
else
{
struct bg_progress * progress = NULL;
if ( show_progress )
{
rc = KOutMsg( "concat :" );
if ( rc == 0 )
{
uint64_t total = total_size( dir, &self -> lists );
rc = bg_progress_make( &progress, total, 0, 0 ); /* progress_thread.c */
}
}
if ( rc == 0 )
{
uint32_t first;
uint32_t count = count_valid_entries( &self -> lists, &first ); /* above */
if ( count == 1 )
{
/* we have only ONE set of files... */
VNamelist * l = VectorGet ( &self -> lists, first );
VNamelistReorder ( l, false );
rc = execute_concat( dir,
output_filename,
l,
buf_size,
progress,
force,
compress ); /* concatenator.c */
}
else if ( count > 1 )
{
/* we have MULTIPLE sets of files... */
cmn_merge cmn = { dir, output_filename, buf_size, progress, force, compress };
on_merge_ctx omc = { &cmn, 0 };
VectorInit( &omc . threads, 0, count );
VectorForEach ( &self -> lists, false, on_merge, &omc );
join_and_release_threads( &omc . threads ); /* helper.c */
}
bg_progress_release( progress ); /* progress_thread.c ( ignores NULL )*/
}
}
return rc;
}
