void
SocietyImpl::UpdatePairFormationRates( const IdmDateTime& rCurrentTime, float dt )
{
LOG_INFO_F( "%s: --------------------========== START society->UpdatePairFormationRates() ==========--------------------\n", __FUNCTION__ );
LOG_DEBUG_F("%s()\n", __FUNCTION__);
for( int irel = 0; irel < RelationshipType::COUNT; irel++ )
{
controller[ irel ]->UpdateEntryRates( rCurrentTime, dt );
}
LOG_INFO_F( "%s: --------------------========== society->UpdatePairFormationRates() FINISH ==========--------------------\n", __FUNCTION__ );
}
bool IncidenceCounterSurveillance::notifyOnEvent( IEventCoordinatorEventContext *pEntity, const EventTriggerCoordinator& trigger )
{
LOG_INFO_F(" notifyOnEvent received: %s, %s\n", pEntity->GetName().c_str(), trigger.ToString().c_str());
if( Find( m_PercentageEventsToCountCoordinator, trigger ) )
{
++m_PercentageEventsCounted;
}
if( Find( m_TriggerConditionListCoordinator, trigger ) )
{
++m_Count;
LOG_INFO_F( "notifyOnEvent received: %s m_Count: %d\n", trigger.ToString().c_str(), m_Count );
}
return true;
}
/**
* PRIVATE. Insert a region in a context.
*/
static inline int
bfwork_region_insert(bam_fwork_t *fwork, bam_region_t *region)
{
linked_list_t *list;
size_t list_l;
assert(fwork);
assert(region);
omp_set_lock(&fwork->regions_lock);
//List handle
list = fwork->regions_list;
list_l = linked_list_size(list);
if(list_l >= FWORK_REGIONS_MAX)
{
omp_unset_lock(&fwork->regions_lock);
//Wait for free slots
if(!omp_test_lock(&fwork->free_slots))
{
if(omp_get_num_threads() == 2)
{
#pragma omp taskwait //Force processing
}
omp_set_lock(&fwork->free_slots);
}
//LOG_FATAL_F("Not enough region slots, current: %d\n", list_l);
}
//This lock must be always locked until regions buffer have regions free
omp_test_lock(&fwork->free_slots);
//Add region to list
linked_list_insert_last(region, list);
omp_set_lock(®ion->lock);
LOG_INFO_F("Inserting region %d:%lu-%lu with %d reads\n",
region->chrom + 1, region->init_pos + 1,
region->end_pos + 1, region->size);
LOG_INFO_F("Regions to process %lu\n", linked_list_size(list));
omp_unset_lock(®ion->lock);
omp_unset_lock(&fwork->regions_lock);
return NO_ERROR;
}
void IncidenceCounterSurveillance::StartCounting()
{
m_CounterPeriod_current = m_CounterPeriod;
m_Count = 0;
m_PercentageEventsCounted = 0;
LOG_INFO_F( "StartCounting(), m_CounterPeriod_current = %f m_Count = %f\n", m_CounterPeriod_current, m_Count );
}
void FlowControllerImpl::UpdateDesiredFlow( const IdmDateTime& rCurrentTime, float dt )
{
LOG_DEBUG_F("%s()\n", __FUNCTION__);
// ----------------------------------------------------------
// --- Count the total number of people eligible for pairing
// ----------------------------------------------------------
float cumulative_base_flow = 0.0f;
for( int risk_group = 0; risk_group < RiskGroup::COUNT; risk_group ++)
{
auto& eligible_population = pair_formation_stats->GetEligible((RiskGroup::Enum)risk_group);
for (auto& entry : eligible_population)
{
for (float flow : entry.second)
{
cumulative_base_flow += flow;
}
}
if (LOG_LEVEL(INFO))
{
LOG_INFO_F( "%s: eligible population for %s risk group:\n", __FUNCTION__, RiskGroup::pairs::lookup_key(risk_group) );
for (auto& entry : eligible_population) {
cout << "{ " << entry.first << ", [ ";
for (int count : entry.second) {
cout << count << ' ';
}
cout << "] }" << endl;
}
}
}
// ---------------------------------------------------------------------
// --- Multiply the total number of people eligible times the base rate
// ---------------------------------------------------------------------
cumulative_base_flow *= parameters->FormationRate( rCurrentTime, dt );
if (cumulative_base_flow > 0.0f)
{
// -----------------------------------------------------------
// --- Determine the desired rate for each gender and age bin
// -----------------------------------------------------------
for (int sex = Gender::MALE; sex <= Gender::FEMALE; sex++)
{
auto& desired = desired_flow.at(sex); // important, use a reference here so we update desired_flow
auto& marginal = parameters->MarginalValues().at(sex); // use a reference here to avoid a copy
int bin_count = desired.size();
for (int bin_index = 0; bin_index < bin_count; bin_index++)
{
desired[bin_index] = 0.5f * cumulative_base_flow * marginal[bin_index];
}
}
}
else
{
memset(desired_flow[Gender::MALE].data(), 0, desired_flow[Gender::MALE].size() * sizeof(float));
memset(desired_flow[Gender::FEMALE].data(), 0, desired_flow[Gender::FEMALE].size() * sizeof(float));
}
}
void MultiInterventionEventCoordinator::DistributeInterventionsToNodes( INodeEventContext* event_context )
{
const json::Array & interventions_array = json::QuickInterpreter( intervention_config._json ).As<json::Array>();
LOG_DEBUG_F( "interventions array size = %d\n", interventions_array.Size() );
for( int idx = 0; idx < interventions_array.Size(); idx++ )
{
const json::Object& actualIntervention = json_cast<const json::Object&>(interventions_array[ idx ]);
Configuration * config = Configuration::CopyFromElement( actualIntervention );
assert( config );
LOG_DEBUG_F( "Attempting to instantiate intervention of class %s\n",
std::string( (*config)[ "class" ].As<json::String>() ).c_str() );
INodeDistributableIntervention *ndi = InterventionFactory::getInstance()->CreateNDIIntervention( config );
if( ndi == nullptr )
{
throw NullPointerException( __FILE__, __LINE__, __FUNCTION__, "Should have constructed a node-level intervention." );
}
if( ndi->Distribute( event_context, this ) )
{
LOG_INFO_F( "UpdateNodes() distributed '%s' intervention to node %d\n", ndi->GetName().c_str(), event_context->GetId().data );
}
ndi->Release();
delete config;
config = nullptr;
}
}
bool
NodeSetPolygon::Contains(
INodeEventContext *nec
)
{
LOG_DEBUG_F( "Contains node ID = %d ?\n", nec->GetId().data );
if( vertices_raw.length() > 0 ) // clear this after parse.
{
// haven't parsed raw list yet.
// Go through list and parse out.
if( polygon_format == PolygonFormatType::SHAPE )
{
parseEmodFormat();
}
// don't need else here because enum parser already catches such errors
}
if( polygon_format == PolygonFormatType::SHAPE && nec->IsInPolygon( points_array, int(num_points) ) )
{
LOG_INFO_F( "Node ID = %d is contained within intervention polygon\n", nec->GetId().data );
return true;
}
LOG_DEBUG("Polygon apparently does not contain this node.\n");
/*else if( polygon_format == PolygonFormatType::GEOJSON && nec->IsInPolygon( poly ) )
{
return true;
}*/
return false;
}
char* __cdecl
GetEModuleVersion(char* sVer, const Environment * pEnv)
{
Environment::setInstance(const_cast<Environment*>(pEnv));
ProgDllVersion pv;
LOG_INFO_F("GetEModuleVersion called with ver=%s\n", pv.getVersion());
if (sVer) strcpy(sVer, pv.getVersion());
return sVer;
}
void MultiInterventionEventCoordinator::DistributeInterventionsToIndividuals( INodeEventContext* event_context )
{
// For now, distribute evenly across nodes.
int limitPerNode = -1;
int totalIndivGivenIntervention = event_context->VisitIndividuals( this, limitPerNode );
LOG_INFO_F( "UpdateNodes() gave out %d interventions at node %d\n", totalIndivGivenIntervention, event_context->GetId().data );
}
bool IncidenceCounterSurveillance::notifyOnEvent( INodeEventContext *pEntity, const EventTriggerNode& trigger )
{
LOG_INFO_F( "notifyOnEvent received: %s\n", trigger.ToString().c_str() );
if ( m_NodePropertyRestrictions.Qualifies( pEntity->GetNodeContext()->GetNodeProperties() ) &&
!IsDoneCounting() )
{
if( Find( m_PercentageEventsToCountNode, trigger ) )
{
++m_PercentageEventsCounted;
}
if( Find( m_TriggerConditionListNode, trigger ) )
{
++m_Count;
LOG_INFO_F( "notifyOnEvent received: %s m_Count: %d\n", trigger.ToString().c_str(), m_Count );
}
}
return true;
}
HumanStateChange IndividualHumanPy::GetStateChange() const
{
HumanStateChange retVal = StateChange;
//auto parsed = IdmString(state_to_report).split();
if( state_to_report == "D" )
{
LOG_INFO_F( "[GetStateChange] Somebody died from their infection.\n" );
retVal = HumanStateChange::KilledByInfection;
}
return retVal;
}
// Obviously don't need this if it's not doing anything useful.
void ReferenceTrackingEventCoordinator::Update( float dt )
{
// Check if it's time for another distribution
if( parent->GetSimulationTime().Year() >= end_year )
{
LOG_INFO_F( "ReferenceTrackingEventCoordinator expired.\n" );
distribution_complete = true;
}
LOG_DEBUG_F( "Update...ing.\n" );
return StandardInterventionDistributionEventCoordinator::Update( dt );
}
void
SocietyImpl::UpdatePairFormationAgents( const IdmDateTime& rCurrentTime, float dt )
{
LOG_INFO_F( "%s: --------------------========== START society->UpdatePairFormationAgents() ==========--------------------\n", __FUNCTION__ );
LOG_DEBUG_F("%s()\n", __FUNCTION__);
for( int irel = 0; irel < RelationshipType::COUNT; irel++ )
{
pfa[ irel ]->Update( rCurrentTime, dt );
}
LOG_INFO_F( "%s: --------------------========== society->UpdatePairFormationAgents() FINISH ==========--------------------\n", __FUNCTION__ );
if (LOG_LEVEL(INFO))
{
for( int irel = 0; irel < RelationshipType::COUNT; irel++ )
{
pfa[ irel ]->Print( RelationshipType::pairs::lookup_key(irel) );
}
}
}
void IncidenceCounterSurveillance::UnregisterForEvents( ISimulationEventContext* context )
{
switch( m_CounterEventType )
{
case EventType::NODE:
{
for( EventTriggerNode& ect : m_TriggerConditionListNode )
{
context->GetNodeEventBroadcaster()->UnregisterObserver( this, ect );
LOG_INFO_F( "Unregister Trigger: %s\n", ect.c_str() );
}
for( EventTriggerNode& ect : m_PercentageEventsToCountNode )
{
if( !Find( m_TriggerConditionListNode, ect ) )
{
context->GetNodeEventBroadcaster()->UnregisterObserver( this, ect );
LOG_INFO_F( "Unregister Percentage_Events_To_Count: %s\n", ect.c_str() );
}
}
break;
}
case EventType::COORDINATOR:
{
for( EventTriggerCoordinator& ect : m_TriggerConditionListCoordinator )
{
context->GetCoordinatorEventBroadcaster()->UnregisterObserver( this, ect );
LOG_INFO_F( "Unregister Trigger: %s\n", ect.c_str() );
}
for( EventTriggerCoordinator& ect : m_PercentageEventsToCountCoordinator )
{
if( !Find( m_TriggerConditionListCoordinator, ect ) )
{
context->GetCoordinatorEventBroadcaster()->UnregisterObserver( this, ect );
LOG_INFO_F( "Unregister Percentage_Events_To_Count: %s\n", ect.c_str() );
}
}
break;
}
// no default here
}
}
void IncidenceCounterSurveillance::UnregisterForEvents( INodeEventContext* pNEC )
{
IncidenceCounter::UnregisterForEvents( pNEC );
for( EventTrigger& ect : m_PercentageEventsToCountIndividual )
{
if( !Find( m_TriggerConditionListIndividual, ect ) )
{
pNEC->GetIndividualEventBroadcaster()->UnregisterObserver( this, ect );
LOG_INFO_F( "Unregistered Percentage_Events_To_Count: %s\n", ect.c_str() );
}
}
}
void FlowControllerImpl::UpdateEntryRates( const IdmDateTime& rCurrentTime, float dt )
{
LOG_DEBUG_F("%s()\n", __FUNCTION__);
// -------------------------
// --- Update desired rates
// -------------------------
UpdateDesiredFlow( rCurrentTime, dt );
if (LOG_LEVEL(INFO))
{
LOG_INFO_F( "%s: desired flow:\n", __FUNCTION__ );
for (auto& entry : desired_flow) {
cout << "{ " << entry.first << ", [ ";
for (float flow : entry.second) {
cout << flow << ' ';
}
cout << "] }" << endl;
}
}
// --------------------------------------------------------------
// --- Get the desired rate(s) from the pfa,
// --- Normalize by the number of eligible from the stats object
// --- Update the desired rates in the rate table
// --------------------------------------------------------------
auto& eligible_population_LOW = pair_formation_stats->GetEligible(RiskGroup::LOW);
auto& eligible_population_HIGH = pair_formation_stats->GetEligible(RiskGroup::HIGH);
for (int sex = Gender::MALE; sex <= Gender::FEMALE; sex++)
{
auto& desired = desired_flow.at(sex);
auto& eligible_LOW = eligible_population_LOW.at(sex);
auto& eligible_HIGH = eligible_population_HIGH.at(sex);
for (int bin_index = 0; bin_index < desired_flow[sex].size(); bin_index++)
{
int eligible_count_LOW = eligible_LOW[bin_index];
int eligible_count_HIGH = eligible_HIGH[bin_index];
int effective_eligible_count = rate_ratio[sex] * eligible_count_HIGH + eligible_count_LOW;
float rate_LOW = (effective_eligible_count > 0) ? desired[bin_index] / effective_eligible_count : 0.0f;
float rate_HIGH = rate_ratio[sex] * rate_LOW;
rate_table->SetRateForBinAndSexAndRiskGroup(bin_index, sex, RiskGroup::LOW, rate_LOW);
rate_table->SetRateForBinAndSexAndRiskGroup(bin_index, sex, RiskGroup::HIGH, rate_HIGH);
}
}
if (LOG_LEVEL(INFO))
{
rate_table->DumpRates();
}
}
void NodeEventContextHost::PurgeExisting( const std::string& iv_name )
{
for (auto intervention : node_interventions)
{
std::string cur_iv_type_name = typeid( *intervention ).name();
if( cur_iv_type_name == iv_name )
{
LOG_INFO_F("Found an existing intervention by that name (%s) which we are purging\n", iv_name.c_str());
node_interventions.remove( intervention );
delete intervention;
break;
}
}
}
/**
* Configure BAM framework data structure for wandering.
*/
int
bfwork_configure(bam_fwork_t *fwork, const char *in_file, const char *out_file, const char *reference, bfwork_context_t *context)
{
assert(fwork);
assert(in_file);
//Set I/O
fwork->input_file_str = (char *)in_file;
omp_set_lock(&fwork->output_file_lock);
fwork->output_file_str = (char *)out_file;
omp_unset_lock(&fwork->output_file_lock);
omp_set_lock(&fwork->reference_lock);
fwork->reference_str = (char *)reference;
omp_unset_lock(&fwork->reference_lock);
//Initial context have been specified?
if(context)
{
//Set context
fwork->context = context;
//Add to context list
fwork->v_context[0] = context;
fwork->v_context_l = 1;
}
//Logging
LOG_INFO("Framework configured\n");
if(fwork->input_file_str)
LOG_INFO_F("Input file: %s\n", fwork->input_file_str);
if(fwork->output_file_str)
LOG_INFO_F("Output file: %s\n", fwork->output_file_str);
if(fwork->reference_str)
LOG_INFO_F("Reference file: %s\n", fwork->reference_str);
return NO_ERROR;
}
bool
SimulationSTI::Configure(
const Configuration * inputJson
)
{
initConfigTypeMap( "Base_Year", &base_year, Base_Year_DESC_TEXT, MIN_YEAR, MAX_YEAR, DEFAULT_BASE_YEAR );
bool ret = Simulation::Configure( inputJson );
if( ret )
{
LOG_INFO_F("Setting Base_Year to %f\n", base_year );
currentTime.setBaseYear( base_year );
}
return ret;
}
std::list<INodeDistributableIntervention*> NodeEventContextHost::GetInterventionsByType(const std::string& type_name)
{
std::list<INodeDistributableIntervention*> interventions_of_type;
LOG_INFO_F("Looking for intervention of type %s", type_name.c_str());
for (auto intervention : node_interventions)
{
std::string cur_iv_type_name = typeid( *intervention ).name();
if( cur_iv_type_name == type_name )
{
LOG_INFO( "Found one..." );
interventions_of_type.push_back( intervention );
}
}
return interventions_of_type;
}
void IndividualHumanPy::Update( float currenttime, float dt)
{
#ifdef ENABLE_PYTHON_FEVER
static auto pFunc = PythonSupportPtr->IdmPyInit( PythonSupport::SCRIPT_PYTHON_FEVER.c_str(), "update" );
if( pFunc )
{
// pass individual id AND dt
static PyObject * vars = PyTuple_New(2);
vars = Py_BuildValue( "ll", GetSuid().data, int(dt) );
auto pyVal = PyObject_CallObject( pFunc, vars );
if( pyVal != nullptr )
{
char * state = "UNSET";
PyArg_ParseTuple(pyVal,"si",&state, &state_changed ); //o-> pyobject |i-> int|s-> char*
state_to_report = state;
}
else
{
state_to_report = "D";
}
#if !defined(_WIN32) || !defined(_DEBUG)
Py_DECREF(pyVal);
#endif
PyErr_Print();
}
LOG_DEBUG_F( "state_to_report for individual %d = %s; Infected = %d, change = %d.\n", GetSuid().data, state_to_report.c_str(), IsInfected(), state_changed );
if( state_to_report == "S" && state_changed && GetInfections().size() > 0 )
{
LOG_DEBUG_F( "[Update] Somebody cleared their infection.\n" );
// ClearInfection
auto inf = GetInfections().front();
IInfectionPy * inf_pydemo = NULL;
if (s_OK != inf->QueryInterface(GET_IID(IInfectionPy ), (void**)&inf_pydemo) )
{
throw QueryInterfaceException( __FILE__, __LINE__, __FUNCTION__, "inf", "IInfectionPy ", "Infection" );
}
// get InfectionPy pointer
inf_pydemo->Clear();
}
else if( state_to_report == "D" && state_changed )
{
LOG_INFO_F( "[Update] Somebody died from their infection.\n" );
}
#endif
return IndividualHuman::Update( currenttime, dt);
}
void NodeEventCoordinator::UpdateNodes( float dt )
{
// Intervention class names for informative logging
std::ostringstream intervention_name;
intervention_name << std::string( json::QuickInterpreter(intervention_config._json)["class"].As<json::String>() );
// Simplest NDI distribution without repetition
INodeDistributableIntervention *ndi = nullptr;
for(auto *nec : cached_nodes)
{
auto tmp_config = Configuration::CopyFromElement( intervention_config._json );
ndi = InterventionFactory::getInstance()->CreateNDIIntervention( tmp_config );
delete tmp_config;
tmp_config = nullptr;
if(ndi)
{
if (!ndi->Distribute( nec, this ) )
{
ndi->Release(); // a bit wasteful for now, could cache it for the next fellow
}
else
{
LOG_INFO_F("UpdateNodes() distributed '%s' intervention to node %d\n", intervention_name.str().c_str(), nec->GetId().data );
}
}
else
{
// add NDI-only exception
std::string err = "Unable to create an instance of " + intervention_name.str() + " as an INodeDistributableIntervention.";
throw GeneralConfigurationException(__FILE__, __LINE__, __FUNCTION__, err.c_str());
}
}
// Dispose
if(ndi) ndi->Release();
distribution_complete = true;
}
// copy/paste to remove single-intervention-specific logging
// TODO: could be handled more gracefully
// also not robust in case of array containing NTI not just ITI
void MultiInterventionEventCoordinator::UpdateNodes( float dt )
{
// Only call VisitNodes on first call and if countdown == 0
if( tsteps_since_last != tsteps_between_reps )
{
return;
}
int grandTotal = 0;
int limitPerNode = -1;
LOG_DEBUG_F("[UpdateNodes] limitPerNode = %d\n", limitPerNode);
for (auto nec : cached_nodes)
{
try
{
// For now, distribute evenly across nodes.
int totalIndivGivenIntervention = nec->VisitIndividuals( this, limitPerNode );
grandTotal += totalIndivGivenIntervention;
LOG_INFO_F( "UpdateNodes() gave out %d interventions at node %d\n", totalIndivGivenIntervention, nec->GetId().data );
}
catch(json::Exception &e)
{
// ERROR: not ITI???
// ERROR: ::cerr << "exception casting intervention_config to array! " << e.what() << std::endl;
throw GeneralConfigurationException( __FILE__, __LINE__, __FUNCTION__, e.what() ); // ( "Intervention_Configs json problem: intervention_config is valid json but needs to be an array." );
}
}
tsteps_since_last = 0;
num_repetitions--;
if( num_repetitions == 0 )
{
distribution_complete = true; // we're done, signal disposal ok
}
}
int run_association_test(shared_options_data_t* shared_options_data, assoc_options_data_t* options_data) {
list_t *output_list = (list_t*) malloc (sizeof(list_t));
list_init("output", shared_options_data->num_threads, INT_MAX, output_list);
int ret_code = 0;
vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!vcf_file) {
LOG_FATAL("VCF file does not exist!\n");
}
ped_file_t *ped_file = ped_open(shared_options_data->ped_filename);
if (!ped_file) {
LOG_FATAL("PED file does not exist!\n");
}
LOG_INFO("About to read PED file...\n");
// Read PED file before doing any processing
ret_code = ped_read(ped_file);
if (ret_code != 0) {
LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename);
}
// Try to create the directory where the output files will be stored
ret_code = create_directory(shared_options_data->output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", shared_options_data->output_directory);
}
LOG_INFO("About to perform basic association test...\n");
#pragma omp parallel sections private(ret_code)
{
#pragma omp section
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 0, omp_get_num_threads());
double start = omp_get_wtime();
ret_code = vcf_read(vcf_file, 0,
(shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines,
shared_options_data->batch_bytes <= 0);
double stop = omp_get_wtime();
double total = stop - start;
if (ret_code) {
LOG_FATAL_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_reading(vcf_file);
}
#pragma omp section
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 10, omp_get_num_threads());
// Enable nested parallelism
omp_set_nested(1);
volatile int initialization_done = 0;
// Pedigree information
individual_t **individuals = NULL;
khash_t(ids) *sample_ids = NULL;
// Create chain of filters for the VCF file
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
double start = omp_get_wtime();
double *factorial_logarithms = NULL;
int i = 0;
#pragma omp parallel num_threads(shared_options_data->num_threads) shared(initialization_done, factorial_logarithms, filters, individuals)
{
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 11, omp_get_num_threads());
char *text_begin, *text_end;
vcf_reader_status *status;
while(text_begin = fetch_vcf_text_batch(vcf_file)) {
text_end = text_begin + strlen(text_begin);
if (text_begin == text_end) { // EOF
free(text_begin);
break;
}
# pragma omp critical
{
status = vcf_reader_status_new(shared_options_data->batch_lines, i);
i++;
}
if (shared_options_data->batch_bytes > 0) {
ret_code = run_vcf_parser(text_begin, text_end, 0, vcf_file, status);
} else if (shared_options_data->batch_lines > 0) {
ret_code = run_vcf_parser(text_begin, text_end, shared_options_data->batch_lines, vcf_file, status);
}
// Initialize structures needed for association tests and write headers of output files
if (!initialization_done && vcf_file->samples_names->size > 0) {
# pragma omp critical
{
// Guarantee that just one thread performs this operation
if (!initialization_done) {
// Create map to associate the position of individuals in the list of samples defined in the VCF file
sample_ids = associate_samples_and_positions(vcf_file);
// Sort individuals in PED as defined in the VCF file
individuals = sort_individuals(vcf_file, ped_file);
/*
printf("num samples = %zu\n", get_num_vcf_samples(file));
printf("pos = { ");
for (int j = 0; j < get_num_vcf_samples(file); j++) {
assert(individuals[j]);
printf("%s ", individuals[j]->id);
}
printf("}\n");
*/
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], vcf_file);
}
// Write file format, header entries and delimiter
if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); }
if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); }
LOG_DEBUG("VCF header written\n");
if (options_data->task == FISHER) {
factorial_logarithms = init_logarithm_array(get_num_vcf_samples(vcf_file) * 10);
}
initialization_done = 1;
}
}
}
// If it has not been initialized it means that header is not fully read
if (!initialization_done) {
continue;
}
vcf_batch_t *batch = fetch_vcf_batch(vcf_file);
if (i % 100 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
}
assert(batch);
// Launch association test over records that passed the filters
array_list_t *failed_records = NULL;
int num_variables = ped_file? get_num_variables(ped_file): 0;
array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records);
if (passed_records->size > 0) {
assoc_test(options_data->task, (vcf_record_t**) passed_records->items, passed_records->size,
individuals, get_num_vcf_samples(vcf_file), factorial_logarithms, output_list);
}
// Write records that passed and failed filters to separate files, and free them
write_filtering_output_files(passed_records, failed_records, passed_file, failed_file);
free_filtered_records(passed_records, failed_records, batch->records);
// Free batch and its contents
vcf_reader_status_free(status);
vcf_batch_free(batch);
}
notify_end_parsing(vcf_file);
}
double stop = omp_get_wtime();
double total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
for (int i = 0; i < num_filters; i++) {
filter_t *filter = filters[i];
filter->free_func(filter);
}
free(filters);
if (sample_ids) { kh_destroy(ids, sample_ids); }
if (individuals) { free(individuals); }
// Decrease list writers count
for (int i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
}
#pragma omp section
{
// Thread that writes the results to the output file
LOG_DEBUG_F("Level %d: number of threads in the team - %d\n", 20, omp_get_num_threads());
double start = omp_get_wtime();
// Get the file descriptor
char *path;
FILE *fd = get_assoc_output_file(options_data->task, shared_options_data, &path);
LOG_INFO_F("Association test output filename = %s\n", path);
// Write data: header + one line per variant
write_output_header(options_data->task, fd);
write_output_body(options_data->task, output_list, fd);
fclose(fd);
// Sort resulting file
char *cmd = calloc (40 + strlen(path) * 4, sizeof(char));
sprintf(cmd, "sort -k1,1h -k2,2n %s > %s.tmp && mv %s.tmp %s", path, path, path, path);
int sort_ret = system(cmd);
if (sort_ret) {
LOG_WARN("Association results could not be sorted by chromosome and position, will be shown unsorted\n");
}
double stop = omp_get_wtime();
double total = stop - start;
LOG_INFO_F("[%dW] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dW] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
}
}
free(output_list);
vcf_close(vcf_file);
ped_close(ped_file, 1,1);
return ret_code;
}
void SusceptibilityTyphoid::Update(float dt)
{
age += dt; // tracks age for immune purposes
// if cross N year age boundary
//end of maternal antibodies at 6 months
float age_boundary = 0.5f * DAYSPERYEAR;
if( ( age >= age_boundary ) && ( ( age - dt ) < age_boundary ) && ( mod_acquire == 0 ) )
{
if( parent->GetRng()->SmartDraw( IndividualHumanTyphoidConfig::typhoid_6month_susceptible_fraction ) )
{
LOG_DEBUG_F("id %lu 6-month old being made susceptible.\n", parent->GetSuid().data);
mod_acquire = 1.0f;
}
}
float age_boundary_2 = 3 * DAYSPERYEAR;
if( ( age >= age_boundary_2 ) && ( ( age - dt ) < age_boundary_2 ) && ( mod_acquire == 0 ) )
{
if( parent->GetRng()->SmartDraw( IndividualHumanTyphoidConfig::typhoid_3year_susceptible_fraction ) )
{
LOG_DEBUG_F("id %lu 3-yo being made susceptible.\n", parent->GetSuid().data);
mod_acquire = 1.0f;
}
}
float age_boundary_3 = 6 * DAYSPERYEAR;
if( ( age >= age_boundary_3 ) && ( ( age - dt ) < age_boundary_3 ) && ( mod_acquire == 0 ) )
{
if( parent->GetRng()->SmartDraw( IndividualHumanTyphoidConfig::typhoid_6year_susceptible_fraction ) )
{
LOG_DEBUG_F("id %lu 6-yo being made susceptible.\n", parent->GetSuid().data);
mod_acquire = 1.0f;
}
}
#define SUSPCEPT_INTRO_YEAR (20)
#define LAMBDA_THRESHOLD (1000)
if (IndividualHumanTyphoidConfig::typhoid_exposure_lambda < LAMBDA_THRESHOLD )
{
float twenty_years_old_days = SUSPCEPT_INTRO_YEAR*DAYSPERYEAR;
if( ( age < twenty_years_old_days ) && mod_acquire == 0 )
{
double perc = 0.0f;
double perc1 = 0.0f;
double perc2 = 0.0f;
double num = 0.0f;
double denom = 0.0f;
float lambda = IndividualHumanTyphoidConfig::typhoid_exposure_lambda;
perc2 = 1.0f - ((twenty_years_old_days - age) / (age*lambda + twenty_years_old_days ));
perc1 = 1.0f - ((twenty_years_old_days - (age-1)) / ((age-1)*lambda + twenty_years_old_days ));
perc = (perc2 - perc1) / (1 - perc1);
if( parent->GetRng()->SmartDraw( perc ) )
{
mod_acquire = 1.0f;
LOG_VALID_F("id %lu , age %f being made susceptible.\n", parent->GetSuid().data, age);
//LOG_INFO_F("SUSCEPTIBLE %f %f\n", perc, mod_acquire);
}
}
}
float intro_year = 10; // magic number?
if (IndividualHumanTyphoidConfig::typhoid_exposure_lambda < LAMBDA_THRESHOLD )
{
intro_year = SUSPCEPT_INTRO_YEAR;
}
float age_boundary_4 = intro_year * DAYSPERYEAR;
if (age >= age_boundary_4 && age - dt < age_boundary_4 && mod_acquire == 0)
{
LOG_DEBUG_F("id %lu Schoolkids being made susceptible.\n", parent->GetSuid().data);
mod_acquire = 1.0f;
}
if( ( age > age_boundary_4 + dt ) && ( mod_acquire == 0 ) )
{
LOG_INFO_F("SOMEONE WAS MISSED AGE %f CUTOFF DAY %f MODACQUIRE %f\n", age, age_boundary_4 + dt, mod_acquire);
}
}
int ped_ragel_read(list_t *batches_list, size_t batch_size, ped_file_t *file)
{
int cs;
char *p = file->data;
char *pe = p + file->data_len;
char *eof = pe;
char *ts;
int custom_field_count = 0;
current_batch = ped_batch_new(batch_size);
#line 41 "ped_reader.c"
{
cs = ped_start;
}
#line 46 "ped_reader.c"
{
if ( p == pe )
goto _test_eof;
switch ( cs )
{
case 21:
switch( (*p) ) {
case 10: goto st22;
case 35: goto st16;
}
if ( 33 <= (*p) && (*p) <= 126 )
goto tr36;
goto tr0;
tr0:
#line 53 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'family' field\n", lines + 1, file->filename);
}
goto st0;
tr3:
#line 65 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'individual' field\n", lines + 1, file->filename);
}
goto st0;
tr7:
#line 77 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'father' field\n", lines + 1, file->filename);
}
goto st0;
tr11:
#line 89 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'mother' field\n", lines + 1, file->filename);
}
goto st0;
tr15:
#line 109 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'sex' field\n", lines + 1, file->filename);
}
goto st0;
tr19:
#line 124 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'phenotype' field\n", lines + 1, file->filename);
}
goto st0;
tr26:
#line 141 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'header' field\n", lines + 1, file->filename);
}
goto st0;
tr44:
#line 161 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in '%s' field\n", lines + 1, file->filename, current_record->custom_field);
}
goto st0;
#line 108 "ped_reader.c"
st0:
cs = 0;
goto _out;
st22:
if ( ++p == pe )
goto _test_eof22;
case 22:
if ( (*p) == 10 )
goto st22;
goto st0;
tr36:
#line 22 "ped.ragel"
{
current_record = create_ped_record();
genotype = 0;
}
#line 45 "ped.ragel"
{
ts = p;
}
goto st1;
st1:
if ( ++p == pe )
goto _test_eof1;
case 1:
#line 134 "ped_reader.c"
if ( (*p) == 9 )
goto tr1;
if ( 33 <= (*p) && (*p) <= 126 )
goto st1;
goto tr0;
tr1:
#line 49 "ped.ragel"
{
set_ped_record_family_id(strndup(ts, p-ts), current_record);
}
goto st2;
st2:
if ( ++p == pe )
goto _test_eof2;
case 2:
#line 150 "ped_reader.c"
if ( (*p) == 95 )
goto tr4;
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr4;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr4;
} else
goto tr4;
goto tr3;
tr4:
#line 57 "ped.ragel"
{
ts = p;
}
goto st3;
st3:
if ( ++p == pe )
goto _test_eof3;
case 3:
#line 172 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr5;
case 95: goto st3;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto st3;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto st3;
} else
goto st3;
goto tr3;
tr5:
#line 61 "ped.ragel"
{
set_ped_record_individual_id(strndup(ts, p-ts), current_record);
}
goto st4;
st4:
if ( ++p == pe )
goto _test_eof4;
case 4:
#line 196 "ped_reader.c"
switch( (*p) ) {
case 46: goto tr8;
case 95: goto tr9;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr9;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr9;
} else
goto tr9;
goto tr7;
tr8:
#line 69 "ped.ragel"
{
ts = p;
}
goto st5;
st5:
if ( ++p == pe )
goto _test_eof5;
case 5:
#line 220 "ped_reader.c"
if ( (*p) == 9 )
goto tr10;
goto tr7;
tr10:
#line 73 "ped.ragel"
{
set_ped_record_father_id(strndup(ts, p-ts), current_record);
}
goto st6;
st6:
if ( ++p == pe )
goto _test_eof6;
case 6:
#line 234 "ped_reader.c"
switch( (*p) ) {
case 46: goto tr12;
case 95: goto tr13;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr13;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr13;
} else
goto tr13;
goto tr11;
tr12:
#line 81 "ped.ragel"
{
ts = p;
}
goto st7;
st7:
if ( ++p == pe )
goto _test_eof7;
case 7:
#line 258 "ped_reader.c"
if ( (*p) == 9 )
goto tr14;
goto tr11;
tr14:
#line 85 "ped.ragel"
{
set_ped_record_mother_id(strndup(ts, p-ts), current_record);
}
goto st8;
st8:
if ( ++p == pe )
goto _test_eof8;
case 8:
#line 272 "ped_reader.c"
if ( (*p) == 46 )
goto tr16;
if ( 48 <= (*p) && (*p) <= 57 )
goto tr17;
goto tr15;
tr16:
#line 93 "ped.ragel"
{
ts = p;
}
goto st9;
st9:
if ( ++p == pe )
goto _test_eof9;
case 9:
#line 288 "ped_reader.c"
if ( (*p) == 9 )
goto tr18;
goto tr15;
tr18:
#line 97 "ped.ragel"
{
char *field = strndup(ts, p-ts);
enum Sex sex = UNKNOWN_SEX;
if (atoi(field) == 1) {
sex = MALE;
} else if (atoi(field) == 2) {
sex = FEMALE;
}
set_ped_record_sex(sex, current_record);
free(field); // Not set as ped_record_t variable -> not freed later
}
goto st10;
st10:
if ( ++p == pe )
goto _test_eof10;
case 10:
#line 310 "ped_reader.c"
switch( (*p) ) {
case 32: goto tr20;
case 95: goto tr20;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr20;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr20;
} else
goto tr20;
goto tr19;
tr20:
#line 113 "ped.ragel"
{
ts = p;
}
goto st23;
tr42:
#line 117 "ped.ragel"
{
if (strncmp(".", ts, 1)) {
char *field = strndup(ts, p-ts);
set_ped_record_phenotype(field, current_record, file);
}
}
#line 145 "ped.ragel"
{
custom_field_count = 6;
}
goto st23;
st23:
if ( ++p == pe )
goto _test_eof23;
case 23:
#line 347 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr39;
case 10: goto tr40;
case 32: goto tr42;
case 95: goto st23;
}
if ( (*p) < 48 ) {
if ( 11 <= (*p) && (*p) <= 13 )
goto tr41;
} else if ( (*p) > 57 ) {
if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto st23;
} else if ( (*p) >= 65 )
goto st23;
} else
goto st23;
goto tr19;
tr39:
#line 117 "ped.ragel"
{
if (strncmp(".", ts, 1)) {
char *field = strndup(ts, p-ts);
set_ped_record_phenotype(field, current_record, file);
}
}
#line 145 "ped.ragel"
{
custom_field_count = 6;
}
goto st24;
tr49:
#line 153 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (custom_field_count == file->num_field) {
set_ped_record_custom_field(field_name, current_record, file);
}
}
goto st24;
st24:
if ( ++p == pe )
goto _test_eof24;
case 24:
#line 393 "ped_reader.c"
switch( (*p) ) {
case 9: goto st24;
case 10: goto tr46;
}
if ( (*p) > 13 ) {
if ( 32 <= (*p) && (*p) <= 126 )
goto tr48;
} else if ( (*p) >= 11 )
goto st26;
goto tr44;
tr40:
#line 117 "ped.ragel"
{
if (strncmp(".", ts, 1)) {
char *field = strndup(ts, p-ts);
set_ped_record_phenotype(field, current_record, file);
}
}
#line 145 "ped.ragel"
{
custom_field_count = 6;
}
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
#line 18 "ped.ragel"
{
lines++;
}
goto st25;
tr46:
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
#line 18 "ped.ragel"
{
lines++;
}
goto st25;
tr50:
#line 153 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (custom_field_count == file->num_field) {
set_ped_record_custom_field(field_name, current_record, file);
}
}
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
#line 18 "ped.ragel"
{
lines++;
}
goto st25;
st25:
if ( ++p == pe )
goto _test_eof25;
case 25:
#line 499 "ped_reader.c"
switch( (*p) ) {
case 10: goto tr46;
case 32: goto st26;
}
if ( (*p) < 33 ) {
if ( 9 <= (*p) && (*p) <= 13 )
goto st26;
} else if ( (*p) > 34 ) {
if ( 36 <= (*p) && (*p) <= 126 )
goto tr36;
} else
goto tr36;
goto tr0;
tr41:
#line 117 "ped.ragel"
{
if (strncmp(".", ts, 1)) {
char *field = strndup(ts, p-ts);
set_ped_record_phenotype(field, current_record, file);
}
}
#line 145 "ped.ragel"
{
custom_field_count = 6;
}
goto st26;
tr51:
#line 153 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (custom_field_count == file->num_field) {
set_ped_record_custom_field(field_name, current_record, file);
}
}
goto st26;
st26:
if ( ++p == pe )
goto _test_eof26;
case 26:
#line 540 "ped_reader.c"
switch( (*p) ) {
case 10: goto tr46;
case 32: goto st26;
}
if ( 9 <= (*p) && (*p) <= 13 )
goto st26;
goto st0;
tr48:
#line 149 "ped.ragel"
{
ts = p;
}
goto st27;
tr52:
#line 153 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (custom_field_count == file->num_field) {
set_ped_record_custom_field(field_name, current_record, file);
}
}
goto st27;
st27:
if ( ++p == pe )
goto _test_eof27;
case 27:
#line 568 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr49;
case 10: goto tr50;
case 32: goto tr52;
}
if ( (*p) > 13 ) {
if ( 33 <= (*p) && (*p) <= 126 )
goto st27;
} else if ( (*p) >= 11 )
goto tr51;
goto tr44;
tr17:
#line 93 "ped.ragel"
{
ts = p;
}
goto st11;
st11:
if ( ++p == pe )
goto _test_eof11;
case 11:
#line 590 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr18;
case 46: goto st12;
}
if ( 48 <= (*p) && (*p) <= 57 )
goto st11;
goto tr15;
st12:
if ( ++p == pe )
goto _test_eof12;
case 12:
if ( 48 <= (*p) && (*p) <= 57 )
goto st13;
goto tr15;
st13:
if ( ++p == pe )
goto _test_eof13;
case 13:
if ( (*p) == 9 )
goto tr18;
if ( 48 <= (*p) && (*p) <= 57 )
goto st13;
goto tr15;
tr13:
#line 81 "ped.ragel"
{
ts = p;
}
goto st14;
st14:
if ( ++p == pe )
goto _test_eof14;
case 14:
#line 624 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr14;
case 95: goto st14;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto st14;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto st14;
} else
goto st14;
goto tr11;
tr9:
#line 69 "ped.ragel"
{
ts = p;
}
goto st15;
st15:
if ( ++p == pe )
goto _test_eof15;
case 15:
#line 648 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr10;
case 95: goto st15;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto st15;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto st15;
} else
goto st15;
goto tr7;
st16:
if ( ++p == pe )
goto _test_eof16;
case 16:
switch( (*p) ) {
case 9: goto st17;
case 32: goto tr28;
case 95: goto tr29;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr29;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr29;
} else
goto tr29;
goto tr26;
st17:
if ( ++p == pe )
goto _test_eof17;
case 17:
switch( (*p) ) {
case 32: goto tr29;
case 95: goto tr29;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr29;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr29;
} else
goto tr29;
goto tr26;
tr29:
#line 128 "ped.ragel"
{
ts = p;
}
goto st18;
st18:
if ( ++p == pe )
goto _test_eof18;
case 18:
#line 707 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr30;
case 10: goto tr31;
case 32: goto st18;
case 95: goto st18;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto st18;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto st18;
} else
goto st18;
goto tr26;
tr30:
#line 132 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (file->variable_field && !strcmp(field_name, file->variable_field)) {
file->num_field = custom_field_count;
}
free(field_name);
}
goto st19;
st19:
if ( ++p == pe )
goto _test_eof19;
case 19:
#line 738 "ped_reader.c"
switch( (*p) ) {
case 9: goto st19;
case 10: goto st28;
case 32: goto tr29;
case 95: goto tr29;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr29;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr29;
} else
goto tr29;
goto tr26;
tr31:
#line 132 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (file->variable_field && !strcmp(field_name, file->variable_field)) {
file->num_field = custom_field_count;
}
free(field_name);
}
goto st28;
st28:
if ( ++p == pe )
goto _test_eof28;
case 28:
#line 769 "ped_reader.c"
if ( (*p) == 10 )
goto st22;
if ( (*p) > 34 ) {
if ( 36 <= (*p) && (*p) <= 126 )
goto tr36;
} else if ( (*p) >= 33 )
goto tr36;
goto tr0;
tr28:
#line 128 "ped.ragel"
{
ts = p;
}
goto st20;
st20:
if ( ++p == pe )
goto _test_eof20;
case 20:
#line 788 "ped_reader.c"
switch( (*p) ) {
case 9: goto tr30;
case 10: goto tr31;
case 32: goto tr29;
case 95: goto tr29;
}
if ( (*p) < 65 ) {
if ( 48 <= (*p) && (*p) <= 57 )
goto tr29;
} else if ( (*p) > 90 ) {
if ( 97 <= (*p) && (*p) <= 122 )
goto tr29;
} else
goto tr29;
goto tr26;
}
_test_eof22: cs = 22; goto _test_eof;
_test_eof1: cs = 1; goto _test_eof;
_test_eof2: cs = 2; goto _test_eof;
_test_eof3: cs = 3; goto _test_eof;
_test_eof4: cs = 4; goto _test_eof;
_test_eof5: cs = 5; goto _test_eof;
_test_eof6: cs = 6; goto _test_eof;
_test_eof7: cs = 7; goto _test_eof;
_test_eof8: cs = 8; goto _test_eof;
_test_eof9: cs = 9; goto _test_eof;
_test_eof10: cs = 10; goto _test_eof;
_test_eof23: cs = 23; goto _test_eof;
_test_eof24: cs = 24; goto _test_eof;
_test_eof25: cs = 25; goto _test_eof;
_test_eof26: cs = 26; goto _test_eof;
_test_eof27: cs = 27; goto _test_eof;
_test_eof11: cs = 11; goto _test_eof;
_test_eof12: cs = 12; goto _test_eof;
_test_eof13: cs = 13; goto _test_eof;
_test_eof14: cs = 14; goto _test_eof;
_test_eof15: cs = 15; goto _test_eof;
_test_eof16: cs = 16; goto _test_eof;
_test_eof17: cs = 17; goto _test_eof;
_test_eof18: cs = 18; goto _test_eof;
_test_eof19: cs = 19; goto _test_eof;
_test_eof28: cs = 28; goto _test_eof;
_test_eof20: cs = 20; goto _test_eof;
_test_eof: {}
if ( p == eof )
{
switch ( cs ) {
case 24:
case 25:
case 26:
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
break;
case 1:
#line 53 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'family' field\n", lines + 1, file->filename);
}
break;
case 2:
case 3:
#line 65 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'individual' field\n", lines + 1, file->filename);
}
break;
case 4:
case 5:
case 15:
#line 77 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'father' field\n", lines + 1, file->filename);
}
break;
case 6:
case 7:
case 14:
#line 89 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'mother' field\n", lines + 1, file->filename);
}
break;
case 8:
case 9:
case 11:
case 12:
case 13:
#line 109 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'sex' field\n", lines + 1, file->filename);
}
break;
case 10:
#line 124 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'phenotype' field\n", lines + 1, file->filename);
}
break;
case 16:
case 17:
case 18:
case 19:
case 20:
#line 141 "ped.ragel"
{
LOG_ERROR_F("Line %zu (%s): Error in 'header' field\n", lines + 1, file->filename);
}
break;
case 27:
#line 153 "ped.ragel"
{
char* field_name = strndup(ts, p-ts);
custom_field_count++;
if (custom_field_count == file->num_field) {
set_ped_record_custom_field(field_name, current_record, file);
}
}
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
break;
case 23:
#line 117 "ped.ragel"
{
if (strncmp(".", ts, 1)) {
char *field = strndup(ts, p-ts);
set_ped_record_phenotype(field, current_record, file);
}
}
#line 145 "ped.ragel"
{
custom_field_count = 6;
}
#line 27 "ped.ragel"
{
// If batch is full, add to the list of batches and create a new, empty one
if (ped_batch_is_full(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records\n", current_batch->length);
current_batch = ped_batch_new(batch_size);
}
// Add current record to current batch
if (current_record) {
add_record_to_ped_batch(current_record, current_batch);
num_records++;
}
current_record = NULL;
}
break;
#line 973 "ped_reader.c"
}
}
_out: {}
}
#line 221 "ped.ragel"
// Insert the last batch
if (!ped_batch_is_empty(current_batch))
{
list_item_t *item = list_item_new(num_records, 1, current_batch);
list_insert_item(item, batches_list);
LOG_DEBUG_F("Batch added - %zu records (last)\n", current_batch->length);
}
if ( cs <
#line 992 "ped_reader.c"
21
#line 231 "ped.ragel"
)
{
LOG_ERROR("The file was not successfully read\n");
LOG_INFO_F("Last state is %d, but %d was expected\n",
cs,
#line 1000 "ped_reader.c"
21
#line 235 "ped.ragel"
);
}
LOG_INFO_F("PED records read = %zu\n", num_records);
return cs <
#line 1009 "ped_reader.c"
21
#line 240 "ped.ragel"
;
}
static void parse_effect_response(int tid, char *output_directory, size_t output_directory_len, cp_hashtable *output_files,
list_t *output_list, cp_hashtable *summary_count, cp_hashtable *gene_list) {
int *SO_found = (int*) malloc (sizeof(int)); // Whether the SO code field has been found
int *count;
char tmp_consequence_type[128];
int num_lines;
char **split_batch = split(effect_line[tid], "\n", &num_lines);
for (int i = 0; i < num_lines; i++) {
int num_columns;
char *copy_buf = strdup(split_batch[i]);
char **split_result = split(copy_buf, "\t", &num_columns);
free(copy_buf);
// Find consequence type name (always after SO field)
*SO_found = 0;
if (num_columns == 25) {
// LOG_DEBUG_F("gene = %s\tSO = %d\tCT = %s\n", split_result[17], atoi(split_result[18] + 3), split_result[19]);
if (!cp_hashtable_contains(gene_list, split_result[17])) {
cp_hashtable_put(gene_list, strdup(split_result[17]), NULL);
}
*SO_found = atoi(split_result[18] + 3);
memset(tmp_consequence_type, 0, 128 * sizeof(char));
strncat(tmp_consequence_type, split_result[19], strlen(split_result[19]));
} else {
if (strlen(split_batch[i]) == 0) { // Last line in batch could be only a newline
for (int s = 0; s < num_columns; s++) {
free(split_result[s]);
}
free(split_result);
continue;
}
LOG_INFO_F("[%d] Non-valid line found (%d fields): '%s'\n", tid, num_columns, split_batch[i]);
for (int s = 0; s < num_columns; s++) {
free(split_result[s]);
}
free(split_result);
continue;
}
for (int s = 0; s < num_columns; s++) {
free(split_result[s]);
}
free(split_result);
if (!*SO_found) { // SO:000000 is not valid
LOG_INFO_F("[%d] Non-valid SO found (0)\n", tid);
continue;
}
// LOG_DEBUG_F("[%d] SO found = %d\n", tid, *SO_found);
size_t consequence_type_len = strlen(tmp_consequence_type);
// If file does not exist, create its descriptor and summary counter
FILE *aux_file = cp_hashtable_get(output_files, SO_found);
if (!aux_file) {
#pragma omp critical
{
// This construction avoids 2 threads trying to insert the same CT
aux_file = cp_hashtable_get(output_files, SO_found);
if (!aux_file) {
char filename[output_directory_len + consequence_type_len + 6];
memset(filename, 0, (output_directory_len + consequence_type_len + 6) * sizeof(char));
strncat(filename, output_directory, output_directory_len);
strncat(filename, "/", 1);
strncat(filename, tmp_consequence_type, consequence_type_len);
strncat(filename, ".txt", 4);
aux_file = fopen(filename, "a");
// Add to hashtables (file descriptors and summary counters)
int *SO_stored = (int*) malloc (sizeof(int));
*SO_stored = *SO_found;
cp_hashtable_put(output_files, SO_stored, aux_file);
LOG_INFO_F("[%d] New consequence type found = %s\n", tid, tmp_consequence_type);
}
}
}
// Write line[tid] to file corresponding to its consequence type
if (aux_file) {
#pragma omp critical
{
// TODO move critical one level below?
count = (int*) cp_hashtable_get(summary_count, tmp_consequence_type);
if (count == NULL) {
char *consequence_type = (char*) calloc (consequence_type_len+1, sizeof(char));
strncat(consequence_type, tmp_consequence_type, consequence_type_len);
assert(!strcmp(consequence_type, tmp_consequence_type));
count = (int*) malloc (sizeof(int));
*count = 0;
cp_hashtable_put(summary_count, consequence_type, count);
// LOG_DEBUG_F("[%d] Initialized summary count for %s\n", tmp_consequence_type);
}
// Increment counter for summary
(*count)++;
}
// LOG_DEBUG_F("[%d] before writing %s\n", tid, tmp_consequence_type);
list_item_t *output_item = list_item_new(tid, *SO_found, strdup(split_batch[i]));
list_insert_item(output_item, output_list);
// LOG_DEBUG_F("[%d] after writing %s\n", tid, tmp_consequence_type);
}
}
for (int i = 0; i < num_lines; i++) {
free(split_batch[i]);
}
free(split_batch);
}
int run_effect(char **urls, shared_options_data_t *shared_options_data, effect_options_data_t *options_data) {
int ret_code = 0;
double start, stop, total;
vcf_file_t *vcf_file = vcf_open(shared_options_data->vcf_filename, shared_options_data->max_batches);
if (!vcf_file) {
LOG_FATAL("VCF file does not exist!\n");
}
ped_file_t *ped_file = NULL;
if (shared_options_data->ped_filename) {
ped_file = ped_open(shared_options_data->ped_filename);
if (!ped_file) {
LOG_FATAL("PED file does not exist!\n");
}
LOG_INFO("About to read PED file...\n");
// Read PED file before doing any processing
ret_code = ped_read(ped_file);
if (ret_code != 0) {
LOG_FATAL_F("Can't read PED file: %s\n", ped_file->filename);
}
}
char *output_directory = shared_options_data->output_directory;
size_t output_directory_len = strlen(output_directory);
ret_code = create_directory(output_directory);
if (ret_code != 0 && errno != EEXIST) {
LOG_FATAL_F("Can't create output directory: %s\n", output_directory);
}
// Remove all .txt files in folder
ret_code = delete_files_by_extension(output_directory, "txt");
if (ret_code != 0) {
return ret_code;
}
// Initialize environment for connecting to the web service
ret_code = init_http_environment(0);
if (ret_code != 0) {
return ret_code;
}
// Output file descriptors
static cp_hashtable *output_files = NULL;
// Lines of the output data in the main .txt files
static list_t *output_list = NULL;
// Consequence type counters (for summary, must be kept between web service calls)
static cp_hashtable *summary_count = NULL;
// Gene list (for genes-with-variants, must be kept between web service calls)
static cp_hashtable *gene_list = NULL;
// Initialize collections of file descriptors and summary counters
ret_code = initialize_output_files(output_directory, output_directory_len, &output_files);
if (ret_code != 0) {
return ret_code;
}
initialize_output_data_structures(shared_options_data, &output_list, &summary_count, &gene_list);
initialize_ws_buffers(shared_options_data->num_threads);
// Create job.status file
char job_status_filename[output_directory_len + 10];
sprintf(job_status_filename, "%s/job.status", output_directory);
FILE *job_status = new_job_status_file(job_status_filename);
if (!job_status) {
LOG_FATAL("Can't create job status file\n");
} else {
update_job_status_file(0, job_status);
}
#pragma omp parallel sections private(start, stop, total)
{
#pragma omp section
{
LOG_DEBUG_F("Thread %d reads the VCF file\n", omp_get_thread_num());
start = omp_get_wtime();
ret_code = vcf_read(vcf_file, 1,
(shared_options_data->batch_bytes > 0) ? shared_options_data->batch_bytes : shared_options_data->batch_lines,
shared_options_data->batch_bytes <= 0);
stop = omp_get_wtime();
total = stop - start;
if (ret_code) {
LOG_ERROR_F("Error %d while reading the file %s\n", ret_code, vcf_file->filename);
}
LOG_INFO_F("[%dR] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%dR] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
notify_end_parsing(vcf_file);
}
#pragma omp section
{
// Enable nested parallelism and set the number of threads the user has chosen
omp_set_nested(1);
LOG_DEBUG_F("Thread %d processes data\n", omp_get_thread_num());
// Filters and files for filtering output
filter_t **filters = NULL;
int num_filters = 0;
if (shared_options_data->chain != NULL) {
filters = sort_filter_chain(shared_options_data->chain, &num_filters);
}
FILE *passed_file = NULL, *failed_file = NULL, *non_processed_file = NULL;
get_filtering_output_files(shared_options_data, &passed_file, &failed_file);
// Pedigree information (used in some filters)
individual_t **individuals = NULL;
khash_t(ids) *sample_ids = NULL;
// Filename structure outdir/vcfname.errors
char *prefix_filename = calloc(strlen(shared_options_data->vcf_filename), sizeof(char));
get_filename_from_path(shared_options_data->vcf_filename, prefix_filename);
char *non_processed_filename = malloc((strlen(shared_options_data->output_directory) + strlen(prefix_filename) + 9) * sizeof(char));
sprintf(non_processed_filename, "%s/%s.errors", shared_options_data->output_directory, prefix_filename);
non_processed_file = fopen(non_processed_filename, "w");
free(non_processed_filename);
// Maximum size processed by each thread (never allow more than 1000 variants per query)
if (shared_options_data->batch_lines > 0) {
shared_options_data->entries_per_thread = MIN(MAX_VARIANTS_PER_QUERY,
ceil((float) shared_options_data->batch_lines / shared_options_data->num_threads));
} else {
shared_options_data->entries_per_thread = MAX_VARIANTS_PER_QUERY;
}
LOG_DEBUG_F("entries-per-thread = %d\n", shared_options_data->entries_per_thread);
int i = 0;
vcf_batch_t *batch = NULL;
int ret_ws_0 = 0, ret_ws_1 = 0, ret_ws_2 = 0;
start = omp_get_wtime();
while (batch = fetch_vcf_batch(vcf_file)) {
if (i == 0) {
// Add headers associated to the defined filters
vcf_header_entry_t **filter_headers = get_filters_as_vcf_headers(filters, num_filters);
for (int j = 0; j < num_filters; j++) {
add_vcf_header_entry(filter_headers[j], vcf_file);
}
// Write file format, header entries and delimiter
if (passed_file != NULL) { write_vcf_header(vcf_file, passed_file); }
if (failed_file != NULL) { write_vcf_header(vcf_file, failed_file); }
if (non_processed_file != NULL) { write_vcf_header(vcf_file, non_processed_file); }
LOG_DEBUG("VCF header written\n");
if (ped_file) {
// Create map to associate the position of individuals in the list of samples defined in the VCF file
sample_ids = associate_samples_and_positions(vcf_file);
// Sort individuals in PED as defined in the VCF file
individuals = sort_individuals(vcf_file, ped_file);
}
}
// printf("batch loaded = '%.*s'\n", 50, batch->text);
// printf("batch text len = %zu\n", strlen(batch->text));
// if (i % 10 == 0) {
LOG_INFO_F("Batch %d reached by thread %d - %zu/%zu records \n",
i, omp_get_thread_num(),
batch->records->size, batch->records->capacity);
// }
int reconnections = 0;
int max_reconnections = 3; // TODO allow to configure?
// Write records that passed to a separate file, and query the WS with them as args
array_list_t *failed_records = NULL;
int num_variables = ped_file? get_num_variables(ped_file): 0;
array_list_t *passed_records = filter_records(filters, num_filters, individuals, sample_ids, num_variables, batch->records, &failed_records);
if (passed_records->size > 0) {
// Divide the list of passed records in ranges of size defined in config file
int num_chunks;
int *chunk_sizes;
int *chunk_starts = create_chunks(passed_records->size, shared_options_data->entries_per_thread, &num_chunks, &chunk_sizes);
do {
// OpenMP: Launch a thread for each range
#pragma omp parallel for num_threads(shared_options_data->num_threads)
for (int j = 0; j < num_chunks; j++) {
int tid = omp_get_thread_num();
LOG_DEBUG_F("[%d] WS invocation\n", tid);
LOG_DEBUG_F("[%d] -- effect WS\n", tid);
if (!reconnections || ret_ws_0) {
ret_ws_0 = invoke_effect_ws(urls[0], (vcf_record_t**) (passed_records->items + chunk_starts[j]),
chunk_sizes[j], options_data->excludes);
parse_effect_response(tid, output_directory, output_directory_len, output_files, output_list, summary_count, gene_list);
free(effect_line[tid]);
effect_line[tid] = (char*) calloc (max_line_size[tid], sizeof(char));
}
if (!options_data->no_phenotypes) {
if (!reconnections || ret_ws_1) {
LOG_DEBUG_F("[%d] -- snp WS\n", omp_get_thread_num());
ret_ws_1 = invoke_snp_phenotype_ws(urls[1], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_snp_phenotype_response(tid, output_list);
free(snp_line[tid]);
snp_line[tid] = (char*) calloc (snp_max_line_size[tid], sizeof(char));
}
if (!reconnections || ret_ws_2) {
LOG_DEBUG_F("[%d] -- mutation WS\n", omp_get_thread_num());
ret_ws_2 = invoke_mutation_phenotype_ws(urls[2], (vcf_record_t**) (passed_records->items + chunk_starts[j]), chunk_sizes[j]);
parse_mutation_phenotype_response(tid, output_list);
free(mutation_line[tid]);
mutation_line[tid] = (char*) calloc (mutation_max_line_size[tid], sizeof(char));
}
}
}
LOG_DEBUG_F("*** %dth web services invocation finished\n", i);
if (ret_ws_0 || ret_ws_1 || ret_ws_2) {
if (ret_ws_0) {
LOG_ERROR_F("Effect web service error: %s\n", get_last_http_error(ret_ws_0));
}
if (ret_ws_1) {
LOG_ERROR_F("SNP phenotype web service error: %s\n", get_last_http_error(ret_ws_1));
}
if (ret_ws_2) {
LOG_ERROR_F("Mutations phenotype web service error: %s\n", get_last_http_error(ret_ws_2));
}
// In presence of errors, wait 4 seconds before retrying
reconnections++;
LOG_ERROR_F("Some errors ocurred, reconnection #%d\n", reconnections);
sleep(4);
} else {
free(chunk_starts);
free(chunk_sizes);
}
} while (reconnections < max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2));
}
// If the maximum number of reconnections was reached still with errors,
// write the non-processed batch to the corresponding file
if (reconnections == max_reconnections && (ret_ws_0 || ret_ws_1 || ret_ws_2)) {
#pragma omp critical
{
write_vcf_batch(batch, non_processed_file);
}
}
// Write records that passed and failed filters to separate files, and free them
write_filtering_output_files(passed_records, failed_records, passed_file, failed_file);
free_filtered_records(passed_records, failed_records, batch->records);
// Free batch and its contents
vcf_batch_free(batch);
i++;
}
stop = omp_get_wtime();
total = stop - start;
LOG_INFO_F("[%d] Time elapsed = %f s\n", omp_get_thread_num(), total);
LOG_INFO_F("[%d] Time elapsed = %e ms\n", omp_get_thread_num(), total*1000);
// Free resources
if (passed_file) { fclose(passed_file); }
if (failed_file) { fclose(failed_file); }
if (non_processed_file) { fclose(non_processed_file); }
// Free filters
for (i = 0; i < num_filters; i++) {
filter_t *filter = filters[i];
filter->free_func(filter);
}
free(filters);
// Decrease list writers count
for (i = 0; i < shared_options_data->num_threads; i++) {
list_decr_writers(output_list);
}
}
#pragma omp section
{
// Thread which writes the results to all_variants, summary and one file per consequence type
int ret = 0;
char *line;
list_item_t* item = NULL;
FILE *fd = NULL;
FILE *all_variants_file = cp_hashtable_get(output_files, "all_variants");
FILE *snp_phenotype_file = cp_hashtable_get(output_files, "snp_phenotypes");
FILE *mutation_phenotype_file = cp_hashtable_get(output_files, "mutation_phenotypes");
while ((item = list_remove_item(output_list)) != NULL) {
line = item->data_p;
// Type greater than 0: consequence type identified by its SO code
// Type equals to -1: SNP phenotype
// Type equals to -2: mutation phenotype
if (item->type > 0) {
// Write entry in the consequence type file
fd = cp_hashtable_get(output_files, &(item->type));
int ret = fprintf(fd, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to file: '%s'\n", line);
}
// Write in all_variants
ret = fprintf(all_variants_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to all_variants: '%s'\n", line);
}
} else if (item->type == SNP_PHENOTYPE) {
ret = fprintf(snp_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to snp_phenotypes: '%s'\n", line);
}
} else if (item->type == MUTATION_PHENOTYPE) {
ret = fprintf(mutation_phenotype_file, "%s\n", line);
if (ret < 0) {
LOG_ERROR_F("Error writing to mutation_phenotypes: '%s'\n", line);
}
}
free(line);
list_item_free(item);
}
}
}
write_summary_file(summary_count, cp_hashtable_get(output_files, "summary"));
write_genes_with_variants_file(gene_list, output_directory);
write_result_file(shared_options_data, options_data, summary_count, output_directory);
free_output_data_structures(output_files, summary_count, gene_list);
free_ws_buffers(shared_options_data->num_threads);
free(output_list);
vcf_close(vcf_file);
update_job_status_file(100, job_status);
close_job_status_file(job_status);
return ret_code;
}
void StandardInterventionDistributionEventCoordinator::UpdateNodes( float dt )
{
// Only call VisitNodes on first call and if countdown == 0
if( tsteps_since_last != tsteps_between_reps )
{
return;
}
int limitPerNode = -1;
// intervention class names for informative logging
std::ostringstream intervention_name;
intervention_name << std::string( json::QuickInterpreter(intervention_config._json)["class"].As<json::String>() );
auto qi_as_config = Configuration::CopyFromElement( (intervention_config._json) );
_di = InterventionFactory::getInstance()->CreateIntervention(qi_as_config);
// including deeper information for "distributing" interventions (e.g. calendars)
formatInterventionClassNames( intervention_name, &json::QuickInterpreter(intervention_config._json) );
// Only visit individuals if this is NOT an NTI. Check...
// Check to see if intervention is an INodeDistributable...
INodeDistributableIntervention *ndi = InterventionFactory::getInstance()->CreateNDIIntervention(qi_as_config);
INodeDistributableIntervention *ndi2 = nullptr;
//LOG_DEBUG_F("[UpdateNodes] limitPerNode = %d\n", limitPerNode);
LOG_DEBUG_F("[UpdateNodes] visiting %d nodes per NodeSet\n", cached_nodes.size());
for (auto event_context : cached_nodes)
{
if( ndi )
{
ndi2 = InterventionFactory::getInstance()->CreateNDIIntervention(qi_as_config);
if(ndi2)
{
if( ndi2->Distribute( event_context, this ) )
{
LOG_INFO_F("UpdateNodes() distributed '%s' intervention to node %d\n", intervention_name.str().c_str(), event_context->GetId().data );
}
ndi2->Release();
}
}
else
{
preDistribute();
// For now, distribute evenly across nodes.
int totalIndivGivenIntervention = event_context->VisitIndividuals( this, limitPerNode );
// Create log message
std::stringstream ss;
ss << "UpdateNodes() gave out " << totalIndivGivenIntervention << " '" << intervention_name.str().c_str() << "' interventions ";
std::string restriction_str = demographic_restrictions.GetPropertyRestrictionsAsString();
if( !restriction_str.empty() )
{
ss << " with property restriction(s) " << restriction_str << " " ;
}
ss << "at node " << event_context->GetId().data << "\n" ;
LOG_INFO( ss.str().c_str() );
}
}
delete qi_as_config;
qi_as_config = nullptr;
if(ndi)
{
ndi->Release();
}
tsteps_since_last = 0;
num_repetitions--;
if( num_repetitions == 0 )
{
distribution_complete = true; // we're done, signal disposal ok
}
//distribution_complete = true; // we're done, signal disposal ok
// this signals each process individually that its ok to clean up, in general if the completion times might be different on different nodes
// we'd want to coordinate the cleanup signal in Update()
}
void CalendarEventCoordinator::UpdateNodes( float dt )
{
// Now issue events as they come up, including anything currently in the past or present
while( parent->GetSimulationTime().time >= times_and_coverages.begin()->first)
{
int grandTotal = 0;
int limitPerNode = -1;
// intervention class names for informative logging
std::ostringstream intervention_name;
intervention_name << std::string( json::QuickInterpreter(intervention_config._json)["class"].As<json::String>() );
auto qi_as_config = Configuration::CopyFromElement( (intervention_config._json), "campaign" );
_di = InterventionFactory::getInstance()->CreateIntervention(qi_as_config);
// including deeper information for "distributing" interventions (e.g. calendars)
formatInterventionClassNames( intervention_name, &json::QuickInterpreter(intervention_config._json) );
// Only visit individuals if this is NOT an NTI. Check...
// Check to see if intervention is an INodeDistributable...
INodeDistributableIntervention *ndi = InterventionFactory::getInstance()->CreateNDIIntervention(qi_as_config);
INodeDistributableIntervention *ndi2 = nullptr;
LOG_DEBUG_F("[UpdateNodes] limitPerNode = %d\n", limitPerNode);
for (auto nec : cached_nodes)
{
if (ndi)
{
throw NotYetImplementedException( __FILE__, __LINE__, __FUNCTION__ );
#if 0
ndi2 = InterventionFactory::getInstance()->CreateNDIIntervention( qi_as_config );
if(ndi2)
{
float duration = -1;
if (times_and_coverages.size() > 1)
{
auto iter = times_and_coverages.end();
//A node-targeted intervention issued through the calender coordinator lasts until the next NDI.
//Is there an overlap of one day here? Should there be a -1?
duration = (float)(prev(iter,2)->first - prev(iter, 1)->first);
}
INodeDistributableInterventionParameterSetterInterface* pNDIPSI = nullptr;
if (s_OK == ndi2->QueryInterface(GET_IID(INodeDistributableInterventionParameterSetterInterface), (void**)&pNDIPSI) )
{
pNDIPSI->SetDemographicCoverage(times_and_coverages.begin()->second);
pNDIPSI->SetMaxDuration(duration);
}
if (!ndi2->Distribute( nec, this ) )
{
LOG_INFO_F("UpdateNodes() distributed '%s' intervention to node %d\n", intervention_name.str().c_str(), nec->GetId().data );
}
ndi2->Release();
}
#endif
}
else
{
try
{
// For now, distribute evenly across nodes.
int totalIndivGivenIntervention = nec->VisitIndividuals( this, limitPerNode );
grandTotal += totalIndivGivenIntervention;
LOG_INFO_F( "UpdateNodes() gave out %d interventions at node %d\n", totalIndivGivenIntervention, nec->GetId().data );
}
catch( const json::Exception &e )
{
throw GeneralConfigurationException( __FILE__, __LINE__, __FUNCTION__, e.what() );
}
}
}
delete qi_as_config;
qi_as_config = nullptr;
times_and_coverages.erase(times_and_coverages.begin());
LOG_DEBUG_F("%d Distributions remaining from CalendarEventCoordinator\n", times_and_coverages.size());
if( times_and_coverages.empty() )
{
LOG_DEBUG_F("Signaling for disposal of CalendarEventCoordinator\n");
distribution_complete = true; // we're done, signal disposal ok
break;
}
}
return;
}
