void oskar_simulator_run(oskar_Simulator* h, int* status)
{
int i, num_threads = 1, num_vis_blocks;
if (*status) return;
/* Check the visibilities are going somewhere. */
if (!h->vis_name
#ifndef OSKAR_NO_MS
&& !h->ms_name
#endif
)
{
oskar_log_error(h->log, "No output file specified.");
#ifdef OSKAR_NO_MS
if (h->ms_name)
oskar_log_error(h->log,
"OSKAR was compiled without Measurement Set support.");
#endif
*status = OSKAR_ERR_FILE_IO;
return;
}
/* Initialise if required. */
oskar_simulator_check_init(h, status);
/* Get the number of visibility blocks to be processed. */
num_vis_blocks = oskar_simulator_num_vis_blocks(h);
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Initial memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
oskar_log_section(h->log, 'M', "Starting simulation...");
}
/* Start simulation timer. */
oskar_timer_start(h->tmr_sim);
/*-----------------------------------------------------------------------
*-- START OF MULTITHREADED SIMULATION CODE -----------------------------
*-----------------------------------------------------------------------*/
/* Loop over blocks of observation time, running simulation and file
* writing one block at a time. Simulation and file output are overlapped
* by using double buffering, and a dedicated thread is used for file
* output.
*
* Thread 0 is used for file writes.
* Threads 1 to n (mapped to compute devices) do the simulation.
*
* Note that no write is launched on the first loop counter (as no
* data are ready yet) and no simulation is performed for the last loop
* counter (which corresponds to the last block + 1) as this iteration
* simply writes the last block.
*/
#ifdef _OPENMP
num_threads = h->num_devices + 1;
omp_set_num_threads(num_threads);
omp_set_nested(0);
#else
oskar_log_warning(h->log, "OpenMP not found: Using one compute device.");
#endif
oskar_simulator_reset_work_unit_index(h);
#pragma omp parallel
{
int b, thread_id = 0, device_id = 0;
/* Get host thread ID and device ID. */
#ifdef _OPENMP
thread_id = omp_get_thread_num();
device_id = thread_id - 1;
#endif
/* Loop over simulation time blocks (+1, for the last write). */
for (b = 0; b < num_vis_blocks + 1; ++b)
{
if ((thread_id > 0 || num_threads == 1) && b < num_vis_blocks)
oskar_simulator_run_block(h, b, device_id, status);
if (thread_id == 0 && b > 0)
{
oskar_VisBlock* block;
block = oskar_simulator_finalise_block(h, b - 1, status);
oskar_simulator_write_block(h, block, b - 1, status);
}
/* Barrier 1: Reset work unit index. */
#pragma omp barrier
if (thread_id == 0)
oskar_simulator_reset_work_unit_index(h);
/* Barrier 2: Synchronise before moving to the next block. */
#pragma omp barrier
if (thread_id == 0 && b < num_vis_blocks && h->log && !*status)
oskar_log_message(h->log, 'S', 0, "Block %*i/%i (%3.0f%%) "
"complete. Simulation time elapsed: %.3f s",
disp_width(num_vis_blocks), b+1, num_vis_blocks,
100.0 * (b+1) / (double)num_vis_blocks,
oskar_timer_elapsed(h->tmr_sim));
}
}
/*-----------------------------------------------------------------------
*-- END OF MULTITHREADED SIMULATION CODE -------------------------------
*-----------------------------------------------------------------------*/
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Final memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
}
/* If there are sources in the simulation and the station beam is not
* normalised to 1.0 at the phase centre, the values of noise RMS
* may give a very unexpected S/N ratio!
* The alternative would be to scale the noise to match the station
* beam gain but that would require knowledge of the station beam
* amplitude at the phase centre for each time and channel. */
if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status)
{
int have_sources, amp_calibrated;
have_sources = (h->num_sky_chunks > 0 &&
oskar_sky_num_sources(h->sky_chunks[0]) > 0);
amp_calibrated = oskar_station_normalise_final_beam(
oskar_telescope_station_const(h->tel, 0));
if (have_sources && !amp_calibrated)
{
const char* a = "WARNING: System noise added to visibilities";
const char* b = "without station beam normalisation enabled.";
const char* c = "This will give an invalid signal to noise ratio.";
oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*');
oskar_log_message(h->log, 'W', -1, a);
oskar_log_message(h->log, 'W', -1, b);
oskar_log_message(h->log, 'W', -1, c);
oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' ');
}
}
/* Record times and summarise output files. */
if (h->log && !*status)
{
size_t log_size = 0;
char* log_data;
oskar_log_set_value_width(h->log, 25);
record_timing(h);
oskar_log_section(h->log, 'M', "Simulation complete");
oskar_log_message(h->log, 'M', 0, "Output(s):");
if (h->vis_name)
oskar_log_value(h->log, 'M', 1,
"OSKAR binary file", "%s", h->vis_name);
if (h->ms_name)
oskar_log_value(h->log, 'M', 1,
"Measurement Set", "%s", h->ms_name);
/* Write simulation log to the output files. */
log_data = oskar_log_file_data(h->log, &log_size);
#ifndef OSKAR_NO_MS
if (h->ms)
oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size);
#endif
if (h->vis)
oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN,
OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status);
free(log_data);
}
/* Finalise. */
oskar_simulator_finalise(h, status);
}
void oskar_interferometer_run(oskar_Interferometer* h, int* status)
{
int i, num_threads;
oskar_Thread** threads = 0;
ThreadArgs* args = 0;
if (*status || !h) return;
/* Check the visibilities are going somewhere. */
if (!h->vis_name
#ifndef OSKAR_NO_MS
&& !h->ms_name
#endif
)
{
oskar_log_error(h->log, "No output file specified.");
#ifdef OSKAR_NO_MS
if (h->ms_name)
oskar_log_error(h->log,
"OSKAR was compiled without Measurement Set support.");
#endif
*status = OSKAR_ERR_FILE_IO;
return;
}
/* Initialise if required. */
oskar_interferometer_check_init(h, status);
/* Set up worker threads. */
num_threads = h->num_devices + 1;
oskar_barrier_set_num_threads(h->barrier, num_threads);
threads = (oskar_Thread**) calloc(num_threads, sizeof(oskar_Thread*));
args = (ThreadArgs*) calloc(num_threads, sizeof(ThreadArgs));
for (i = 0; i < num_threads; ++i)
{
args[i].h = h;
args[i].num_threads = num_threads;
args[i].thread_id = i;
}
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Initial memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
oskar_log_section(h->log, 'M', "Starting simulation...");
}
/* Start simulation timer. */
oskar_timer_start(h->tmr_sim);
/* Set status code. */
h->status = *status;
/* Start the worker threads. */
oskar_interferometer_reset_work_unit_index(h);
for (i = 0; i < num_threads; ++i)
threads[i] = oskar_thread_create(run_blocks, (void*)&args[i], 0);
/* Wait for worker threads to finish. */
for (i = 0; i < num_threads; ++i)
{
oskar_thread_join(threads[i]);
oskar_thread_free(threads[i]);
}
free(threads);
free(args);
/* Get status code. */
*status = h->status;
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Final memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
}
/* If there are sources in the simulation and the station beam is not
* normalised to 1.0 at the phase centre, the values of noise RMS
* may give a very unexpected S/N ratio!
* The alternative would be to scale the noise to match the station
* beam gain but that would require knowledge of the station beam
* amplitude at the phase centre for each time and channel. */
if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status)
{
int have_sources, amp_calibrated;
have_sources = (h->num_sky_chunks > 0 &&
oskar_sky_num_sources(h->sky_chunks[0]) > 0);
amp_calibrated = oskar_station_normalise_final_beam(
oskar_telescope_station_const(h->tel, 0));
if (have_sources && !amp_calibrated)
{
const char* a = "WARNING: System noise added to visibilities";
const char* b = "without station beam normalisation enabled.";
const char* c = "This will give an invalid signal to noise ratio.";
oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*');
oskar_log_message(h->log, 'W', -1, a);
oskar_log_message(h->log, 'W', -1, b);
oskar_log_message(h->log, 'W', -1, c);
oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' ');
}
}
/* Record times and summarise output files. */
if (h->log && !*status)
{
size_t log_size = 0;
char* log_data;
oskar_log_set_value_width(h->log, 25);
record_timing(h);
oskar_log_section(h->log, 'M', "Simulation complete");
oskar_log_message(h->log, 'M', 0, "Output(s):");
if (h->vis_name)
oskar_log_value(h->log, 'M', 1,
"OSKAR binary file", "%s", h->vis_name);
if (h->ms_name)
oskar_log_value(h->log, 'M', 1,
"Measurement Set", "%s", h->ms_name);
/* Write simulation log to the output files. */
log_data = oskar_log_file_data(h->log, &log_size);
#ifndef OSKAR_NO_MS
if (h->ms)
oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size);
#endif
if (h->vis)
oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN,
OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status);
free(log_data);
}
/* Finalise. */
oskar_interferometer_finalise(h, status);
}
void oskar_binary_write_int(oskar_Binary* handle, unsigned char id_group,
unsigned char id_tag, int user_index, int value, int* status)
{
oskar_binary_write(handle, OSKAR_INT, id_group, id_tag,
user_index, sizeof(int), &value, status);
}
int main(int argc, char** argv)
{
int status = 0;
oskar::OptionParser opt("oskar_evaulate_pierce_points",
oskar_version_string());
opt.add_required("settings file");
if (!opt.check_options(argc, argv)) return EXIT_FAILURE;
const char* settings_file = opt.get_arg();
// Create the log.
oskar_Log* log = oskar_log_create(OSKAR_LOG_MESSAGE, OSKAR_LOG_STATUS);
oskar_log_message(log, 'M', 0, "Running binary %s", argv[0]);
// Enum values used in writing time-freq data binary files
enum OSKAR_TIME_FREQ_TAGS
{
TIME_IDX = 0,
FREQ_IDX = 1,
TIME_MJD_UTC = 2,
FREQ_HZ = 3,
NUM_FIELDS = 4,
NUM_FIELD_TAGS = 5,
HEADER_OFFSET = 10,
DATA = 0,
DIMS = 1,
LABEL = 2,
UNITS = 3,
GRP = OSKAR_TAG_GROUP_TIME_FREQ_DATA
};
oskar_Settings_old settings;
oskar_settings_old_load(&settings, log, settings_file, &status);
oskar_log_set_keep_file(log, settings.sim.keep_log_file);
if (status) return status;
oskar_Telescope* tel = oskar_settings_to_telescope(&settings, log, &status);
oskar_Sky* sky = oskar_settings_to_sky(&settings, log, &status);
// FIXME remove this restriction ... (see evaluate Z)
if (settings.ionosphere.num_TID_screens != 1)
return OSKAR_ERR_SETUP_FAIL;
int type = settings.sim.double_precision ? OSKAR_DOUBLE : OSKAR_SINGLE;
int loc = OSKAR_CPU;
int num_sources = oskar_sky_num_sources(sky);
oskar_Mem *hor_x, *hor_y, *hor_z;
hor_x = oskar_mem_create(type, loc, num_sources, &status);
hor_y = oskar_mem_create(type, loc, num_sources, &status);
hor_z = oskar_mem_create(type, loc, num_sources, &status);
oskar_Mem *pp_lon, *pp_lat, *pp_rel_path;
int num_stations = oskar_telescope_num_stations(tel);
int num_pp = num_stations * num_sources;
pp_lon = oskar_mem_create(type, loc, num_pp, &status);
pp_lat = oskar_mem_create(type, loc, num_pp, &status);
pp_rel_path = oskar_mem_create(type, loc, num_pp, &status);
// Pierce points for one station (non-owned oskar_Mem pointers)
oskar_Mem *pp_st_lon, *pp_st_lat, *pp_st_rel_path;
pp_st_lon = oskar_mem_create_alias(0, 0, 0, &status);
pp_st_lat = oskar_mem_create_alias(0, 0, 0, &status);
pp_st_rel_path = oskar_mem_create_alias(0, 0, 0, &status);
int num_times = settings.obs.num_time_steps;
double obs_start_mjd_utc = settings.obs.start_mjd_utc;
double dt_dump = settings.obs.dt_dump_days;
// Binary file meta-data
std::string label1 = "pp_lon";
std::string label2 = "pp_lat";
std::string label3 = "pp_path";
std::string units = "radians";
std::string units2 = "";
oskar_Mem *dims = oskar_mem_create(OSKAR_INT, loc, 2, &status);
/* FIXME is this the correct dimension order ?
* FIXME get the MATLAB reader to respect dimension ordering */
oskar_mem_int(dims, &status)[0] = num_sources;
oskar_mem_int(dims, &status)[1] = num_stations;
const char* filename = settings.ionosphere.pierce_points.filename;
oskar_Binary* h = oskar_binary_create(filename, 'w', &status);
double screen_height_m = settings.ionosphere.TID->height_km * 1000.0;
// printf("Number of times = %i\n", num_times);
// printf("Number of stations = %i\n", num_stations);
void *x_, *y_, *z_;
x_ = oskar_mem_void(oskar_telescope_station_true_x_offset_ecef_metres(tel));
y_ = oskar_mem_void(oskar_telescope_station_true_y_offset_ecef_metres(tel));
z_ = oskar_mem_void(oskar_telescope_station_true_z_offset_ecef_metres(tel));
for (int t = 0; t < num_times; ++t)
{
double t_dump = obs_start_mjd_utc + t * dt_dump; // MJD UTC
double gast = oskar_convert_mjd_to_gast_fast(t_dump + dt_dump / 2.0);
for (int i = 0; i < num_stations; ++i)
{
const oskar_Station* station =
oskar_telescope_station_const(tel, i);
double lon = oskar_station_lon_rad(station);
double lat = oskar_station_lat_rad(station);
double alt = oskar_station_alt_metres(station);
double x_ecef, y_ecef, z_ecef, x_offset, y_offset, z_offset;
if (type == OSKAR_DOUBLE)
{
x_offset = ((double*)x_)[i];
y_offset = ((double*)y_)[i];
z_offset = ((double*)z_)[i];
}
else
{
x_offset = (double)((float*)x_)[i];
y_offset = (double)((float*)y_)[i];
z_offset = (double)((float*)z_)[i];
}
oskar_convert_offset_ecef_to_ecef(1, &x_offset, &y_offset,
&z_offset, lon, lat, alt, &x_ecef, &y_ecef, &z_ecef);
double last = gast + lon;
if (type == OSKAR_DOUBLE)
{
oskar_convert_apparent_ra_dec_to_enu_directions_d(num_sources,
oskar_mem_double_const(oskar_sky_ra_rad_const(sky), &status),
oskar_mem_double_const(oskar_sky_dec_rad_const(sky), &status),
last, lat, oskar_mem_double(hor_x, &status),
oskar_mem_double(hor_y, &status),
oskar_mem_double(hor_z, &status));
}
else
{
oskar_convert_apparent_ra_dec_to_enu_directions_f(num_sources,
oskar_mem_float_const(oskar_sky_ra_rad_const(sky), &status),
oskar_mem_float_const(oskar_sky_dec_rad_const(sky), &status),
last, lat, oskar_mem_float(hor_x, &status),
oskar_mem_float(hor_y, &status),
oskar_mem_float(hor_z, &status));
}
int offset = i * num_sources;
oskar_mem_set_alias(pp_st_lon, pp_lon, offset, num_sources,
&status);
oskar_mem_set_alias(pp_st_lat, pp_lat, offset, num_sources,
&status);
oskar_mem_set_alias(pp_st_rel_path, pp_rel_path, offset,
num_sources, &status);
oskar_evaluate_pierce_points(pp_st_lon, pp_st_lat, pp_st_rel_path,
x_ecef, y_ecef, z_ecef, screen_height_m,
num_sources, hor_x, hor_y, hor_z, &status);
} // Loop over stations.
if (status != 0)
continue;
int index = t; // could be = (num_times * f) + t if we have frequency data
int num_fields = 3;
int num_field_tags = 4;
double freq_hz = 0.0;
int freq_idx = 0;
// Write the header TAGS
oskar_binary_write_int(h, GRP, TIME_IDX, index, t, &status);
oskar_binary_write_double(h, GRP, FREQ_IDX, index, freq_idx, &status);
oskar_binary_write_double(h, GRP, TIME_MJD_UTC, index, t_dump, &status);
oskar_binary_write_double(h, GRP, FREQ_HZ, index, freq_hz, &status);
oskar_binary_write_int(h, GRP, NUM_FIELDS, index, num_fields, &status);
oskar_binary_write_int(h, GRP, NUM_FIELD_TAGS, index, num_field_tags,
&status);
// Write data TAGS (fields)
int field, tagID;
field = 0;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_lon, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label1.size()+1, label1.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units.size()+1, units.c_str(), &status);
field = 1;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_lat, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label2.size()+1, label2.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units.size()+1, units.c_str(), &status);
field = 2;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_rel_path, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label3.size()+1, label3.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units2.size()+1, units2.c_str(), &status);
} // Loop over times
// Close the OSKAR binary file.
oskar_binary_free(h);
// clean up memory
oskar_mem_free(hor_x, &status);
oskar_mem_free(hor_y, &status);
oskar_mem_free(hor_z, &status);
oskar_mem_free(pp_lon, &status);
oskar_mem_free(pp_lat, &status);
oskar_mem_free(pp_rel_path, &status);
oskar_mem_free(pp_st_lon, &status);
oskar_mem_free(pp_st_lat, &status);
oskar_mem_free(pp_st_rel_path, &status);
oskar_mem_free(dims, &status);
oskar_telescope_free(tel, &status);
oskar_sky_free(sky, &status);
// Check for errors.
if (status)
oskar_log_error(log, "Run failed: %s.", oskar_get_error_string(status));
oskar_log_free(log);
return status;
}
void oskar_binary_write_double(oskar_Binary* handle, unsigned char id_group,
unsigned char id_tag, int user_index, double value, int* status)
{
oskar_binary_write(handle, OSKAR_DOUBLE, id_group, id_tag,
user_index, sizeof(double), &value, status);
}
