void oskar_evaluate_station_beam(oskar_Mem* beam_pattern, int num_points,
int coord_type, oskar_Mem* x, oskar_Mem* y, oskar_Mem* z,
double norm_ra_rad, double norm_dec_rad, const oskar_Station* station,
oskar_StationWork* work, int time_index, double frequency_hz,
double GAST, int* status)
{
int normalise_final_beam;
oskar_Mem* out;
/* Check if safe to proceed. */
if (*status) return;
/* Set default output beam array. */
out = beam_pattern;
/* Check that the arrays have enough space to add an extra source at the
* end (for normalisation). We don't want to reallocate here, since that
* will be slow to do each time: must simply ensure that we pass input
* arrays that are large enough.
* The normalisation doesn't need to happen if the station has an
* isotropic beam. */
normalise_final_beam = oskar_station_normalise_final_beam(station) &&
(oskar_station_type(station) != OSKAR_STATION_TYPE_ISOTROPIC);
if (normalise_final_beam)
{
double c_x = 0.0, c_y = 0.0, c_z = 1.0;
/* Increment number of points. */
num_points++;
/* Check the input arrays are big enough to hold the new source. */
if ((int)oskar_mem_length(x) < num_points ||
(int)oskar_mem_length(y) < num_points ||
(int)oskar_mem_length(z) < num_points)
{
*status = OSKAR_ERR_DIMENSION_MISMATCH;
return;
}
/* Set output beam array to work buffer. */
out = oskar_station_work_normalised_beam(work, beam_pattern, status);
/* Get the beam direction in the appropriate coordinate system. */
/* (Direction cosines are already set to the interferometer phase
* centre for relative directions.) */
if (coord_type == OSKAR_ENU_DIRECTIONS)
{
double t_x, t_y, t_z, ha0;
ha0 = (GAST + oskar_station_lon_rad(station)) - norm_ra_rad;
oskar_convert_relative_directions_to_enu_directions_d(
&t_x, &t_y, &t_z, 1, &c_x, &c_y, &c_z, ha0, norm_dec_rad,
oskar_station_lat_rad(station));
c_x = t_x;
c_y = t_y;
c_z = t_z;
}
/* Add the extra normalisation source to the end of the arrays. */
oskar_mem_set_element_real(x, num_points-1, c_x, status);
oskar_mem_set_element_real(y, num_points-1, c_y, status);
oskar_mem_set_element_real(z, num_points-1, c_z, status);
}
/* Evaluate the station beam for the given directions. */
if (coord_type == OSKAR_ENU_DIRECTIONS)
{
evaluate_station_beam_enu_directions(out, num_points, x, y, z,
station, work, time_index, frequency_hz, GAST, status);
}
else if (coord_type == OSKAR_RELATIVE_DIRECTIONS)
{
evaluate_station_beam_relative_directions(out, num_points, x, y, z,
station, work, time_index, frequency_hz, GAST, status);
}
else
{
*status = OSKAR_ERR_INVALID_ARGUMENT;
}
/* Scale beam pattern by value of the last source if required. */
if (normalise_final_beam)
{
double amp = 0.0;
/* Get the last element of the vector and convert to amplitude. */
if (oskar_mem_is_matrix(out))
{
double4c val;
val = oskar_mem_get_element_matrix(out, num_points-1, status);
/*
* Scale by square root of "Stokes I" autocorrelation:
* sqrt(0.5 * [sum of resultant diagonal]).
*
* We have
* [ Xa Xb ] [ Xa* Xc* ] = [ Xa Xa* + Xb Xb* (don't care) ]
* [ Xc Xd ] [ Xb* Xd* ] [ (don't care) Xc Xc* + Xd Xd* ]
*
* Stokes I is completely real, so need only evaluate the real
* part of all the multiplies. Because of the conjugate terms,
* these become re*re + im*im.
*
* Need the square root because we only want the normalised value
* for the beam itself (in isolation), not its actual
* autocorrelation!
*/
amp = val.a.x * val.a.x + val.a.y * val.a.y +
val.b.x * val.b.x + val.b.y * val.b.y +
val.c.x * val.c.x + val.c.y * val.c.y +
val.d.x * val.d.x + val.d.y * val.d.y;
amp = sqrt(0.5 * amp);
}
else
{
double2 val;
val = oskar_mem_get_element_complex(out, num_points-1, status);
/* Scale by voltage. */
amp = sqrt(val.x * val.x + val.y * val.y);
}
/* Scale beam array by normalisation value. */
oskar_mem_scale_real(out, 1.0/amp, status);
/* Copy output beam data. */
oskar_mem_copy_contents(beam_pattern, out, 0, 0, num_points-1, status);
}
}
void oskar_interferometer_run(oskar_Interferometer* h, int* status)
{
int i, num_threads;
oskar_Thread** threads = 0;
ThreadArgs* args = 0;
if (*status || !h) return;
/* Check the visibilities are going somewhere. */
if (!h->vis_name
#ifndef OSKAR_NO_MS
&& !h->ms_name
#endif
)
{
oskar_log_error(h->log, "No output file specified.");
#ifdef OSKAR_NO_MS
if (h->ms_name)
oskar_log_error(h->log,
"OSKAR was compiled without Measurement Set support.");
#endif
*status = OSKAR_ERR_FILE_IO;
return;
}
/* Initialise if required. */
oskar_interferometer_check_init(h, status);
/* Set up worker threads. */
num_threads = h->num_devices + 1;
oskar_barrier_set_num_threads(h->barrier, num_threads);
threads = (oskar_Thread**) calloc(num_threads, sizeof(oskar_Thread*));
args = (ThreadArgs*) calloc(num_threads, sizeof(ThreadArgs));
for (i = 0; i < num_threads; ++i)
{
args[i].h = h;
args[i].num_threads = num_threads;
args[i].thread_id = i;
}
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Initial memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
oskar_log_section(h->log, 'M', "Starting simulation...");
}
/* Start simulation timer. */
oskar_timer_start(h->tmr_sim);
/* Set status code. */
h->status = *status;
/* Start the worker threads. */
oskar_interferometer_reset_work_unit_index(h);
for (i = 0; i < num_threads; ++i)
threads[i] = oskar_thread_create(run_blocks, (void*)&args[i], 0);
/* Wait for worker threads to finish. */
for (i = 0; i < num_threads; ++i)
{
oskar_thread_join(threads[i]);
oskar_thread_free(threads[i]);
}
free(threads);
free(args);
/* Get status code. */
*status = h->status;
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Final memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
}
/* If there are sources in the simulation and the station beam is not
* normalised to 1.0 at the phase centre, the values of noise RMS
* may give a very unexpected S/N ratio!
* The alternative would be to scale the noise to match the station
* beam gain but that would require knowledge of the station beam
* amplitude at the phase centre for each time and channel. */
if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status)
{
int have_sources, amp_calibrated;
have_sources = (h->num_sky_chunks > 0 &&
oskar_sky_num_sources(h->sky_chunks[0]) > 0);
amp_calibrated = oskar_station_normalise_final_beam(
oskar_telescope_station_const(h->tel, 0));
if (have_sources && !amp_calibrated)
{
const char* a = "WARNING: System noise added to visibilities";
const char* b = "without station beam normalisation enabled.";
const char* c = "This will give an invalid signal to noise ratio.";
oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*');
oskar_log_message(h->log, 'W', -1, a);
oskar_log_message(h->log, 'W', -1, b);
oskar_log_message(h->log, 'W', -1, c);
oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' ');
}
}
/* Record times and summarise output files. */
if (h->log && !*status)
{
size_t log_size = 0;
char* log_data;
oskar_log_set_value_width(h->log, 25);
record_timing(h);
oskar_log_section(h->log, 'M', "Simulation complete");
oskar_log_message(h->log, 'M', 0, "Output(s):");
if (h->vis_name)
oskar_log_value(h->log, 'M', 1,
"OSKAR binary file", "%s", h->vis_name);
if (h->ms_name)
oskar_log_value(h->log, 'M', 1,
"Measurement Set", "%s", h->ms_name);
/* Write simulation log to the output files. */
log_data = oskar_log_file_data(h->log, &log_size);
#ifndef OSKAR_NO_MS
if (h->ms)
oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size);
#endif
if (h->vis)
oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN,
OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status);
free(log_data);
}
/* Finalise. */
oskar_interferometer_finalise(h, status);
}
void oskar_simulator_run(oskar_Simulator* h, int* status)
{
int i, num_threads = 1, num_vis_blocks;
if (*status) return;
/* Check the visibilities are going somewhere. */
if (!h->vis_name
#ifndef OSKAR_NO_MS
&& !h->ms_name
#endif
)
{
oskar_log_error(h->log, "No output file specified.");
#ifdef OSKAR_NO_MS
if (h->ms_name)
oskar_log_error(h->log,
"OSKAR was compiled without Measurement Set support.");
#endif
*status = OSKAR_ERR_FILE_IO;
return;
}
/* Initialise if required. */
oskar_simulator_check_init(h, status);
/* Get the number of visibility blocks to be processed. */
num_vis_blocks = oskar_simulator_num_vis_blocks(h);
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Initial memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
oskar_log_section(h->log, 'M', "Starting simulation...");
}
/* Start simulation timer. */
oskar_timer_start(h->tmr_sim);
/*-----------------------------------------------------------------------
*-- START OF MULTITHREADED SIMULATION CODE -----------------------------
*-----------------------------------------------------------------------*/
/* Loop over blocks of observation time, running simulation and file
* writing one block at a time. Simulation and file output are overlapped
* by using double buffering, and a dedicated thread is used for file
* output.
*
* Thread 0 is used for file writes.
* Threads 1 to n (mapped to compute devices) do the simulation.
*
* Note that no write is launched on the first loop counter (as no
* data are ready yet) and no simulation is performed for the last loop
* counter (which corresponds to the last block + 1) as this iteration
* simply writes the last block.
*/
#ifdef _OPENMP
num_threads = h->num_devices + 1;
omp_set_num_threads(num_threads);
omp_set_nested(0);
#else
oskar_log_warning(h->log, "OpenMP not found: Using one compute device.");
#endif
oskar_simulator_reset_work_unit_index(h);
#pragma omp parallel
{
int b, thread_id = 0, device_id = 0;
/* Get host thread ID and device ID. */
#ifdef _OPENMP
thread_id = omp_get_thread_num();
device_id = thread_id - 1;
#endif
/* Loop over simulation time blocks (+1, for the last write). */
for (b = 0; b < num_vis_blocks + 1; ++b)
{
if ((thread_id > 0 || num_threads == 1) && b < num_vis_blocks)
oskar_simulator_run_block(h, b, device_id, status);
if (thread_id == 0 && b > 0)
{
oskar_VisBlock* block;
block = oskar_simulator_finalise_block(h, b - 1, status);
oskar_simulator_write_block(h, block, b - 1, status);
}
/* Barrier 1: Reset work unit index. */
#pragma omp barrier
if (thread_id == 0)
oskar_simulator_reset_work_unit_index(h);
/* Barrier 2: Synchronise before moving to the next block. */
#pragma omp barrier
if (thread_id == 0 && b < num_vis_blocks && h->log && !*status)
oskar_log_message(h->log, 'S', 0, "Block %*i/%i (%3.0f%%) "
"complete. Simulation time elapsed: %.3f s",
disp_width(num_vis_blocks), b+1, num_vis_blocks,
100.0 * (b+1) / (double)num_vis_blocks,
oskar_timer_elapsed(h->tmr_sim));
}
}
/*-----------------------------------------------------------------------
*-- END OF MULTITHREADED SIMULATION CODE -------------------------------
*-----------------------------------------------------------------------*/
/* Record memory usage. */
if (h->log && !*status)
{
oskar_log_section(h->log, 'M', "Final memory usage");
#ifdef OSKAR_HAVE_CUDA
for (i = 0; i < h->num_gpus; ++i)
oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]);
#endif
system_mem_log(h->log);
}
/* If there are sources in the simulation and the station beam is not
* normalised to 1.0 at the phase centre, the values of noise RMS
* may give a very unexpected S/N ratio!
* The alternative would be to scale the noise to match the station
* beam gain but that would require knowledge of the station beam
* amplitude at the phase centre for each time and channel. */
if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status)
{
int have_sources, amp_calibrated;
have_sources = (h->num_sky_chunks > 0 &&
oskar_sky_num_sources(h->sky_chunks[0]) > 0);
amp_calibrated = oskar_station_normalise_final_beam(
oskar_telescope_station_const(h->tel, 0));
if (have_sources && !amp_calibrated)
{
const char* a = "WARNING: System noise added to visibilities";
const char* b = "without station beam normalisation enabled.";
const char* c = "This will give an invalid signal to noise ratio.";
oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*');
oskar_log_message(h->log, 'W', -1, a);
oskar_log_message(h->log, 'W', -1, b);
oskar_log_message(h->log, 'W', -1, c);
oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' ');
}
}
/* Record times and summarise output files. */
if (h->log && !*status)
{
size_t log_size = 0;
char* log_data;
oskar_log_set_value_width(h->log, 25);
record_timing(h);
oskar_log_section(h->log, 'M', "Simulation complete");
oskar_log_message(h->log, 'M', 0, "Output(s):");
if (h->vis_name)
oskar_log_value(h->log, 'M', 1,
"OSKAR binary file", "%s", h->vis_name);
if (h->ms_name)
oskar_log_value(h->log, 'M', 1,
"Measurement Set", "%s", h->ms_name);
/* Write simulation log to the output files. */
log_data = oskar_log_file_data(h->log, &log_size);
#ifndef OSKAR_NO_MS
if (h->ms)
oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size);
#endif
if (h->vis)
oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN,
OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status);
free(log_data);
}
/* Finalise. */
oskar_simulator_finalise(h, status);
}
void oskar_evaluate_station_beam(int num_points,
int coord_type, oskar_Mem* x, oskar_Mem* y, oskar_Mem* z,
double norm_ra_rad, double norm_dec_rad, const oskar_Station* station,
oskar_StationWork* work, int time_index, double frequency_hz,
double GAST, int offset_out, oskar_Mem* beam, int* status)
{
oskar_Mem* out;
const size_t num_points_orig = (size_t)num_points;
if (*status) return;
/* Set output beam array to work buffer. */
out = oskar_station_work_beam_out(work, beam, num_points_orig, status);
/* Check that the arrays have enough space to add an extra source at the
* end (for normalisation). We don't want to reallocate here, since that
* will be slow to do each time: must simply ensure that we pass input
* arrays that are large enough.
* The normalisation doesn't need to happen if the station has an
* isotropic beam. */
const int normalise = oskar_station_normalise_final_beam(station) &&
(oskar_station_type(station) != OSKAR_STATION_TYPE_ISOTROPIC);
if (normalise)
{
/* Increment number of points. */
num_points++;
/* Check the input arrays are big enough to hold the new source. */
if ((int)oskar_mem_length(x) < num_points ||
(int)oskar_mem_length(y) < num_points ||
(int)oskar_mem_length(z) < num_points)
{
*status = OSKAR_ERR_DIMENSION_MISMATCH;
return;
}
/* Get the beam direction in the appropriate coordinate system. */
const int bypass = (coord_type != OSKAR_ENU_DIRECTIONS);
const double ha0 = GAST + oskar_station_lon_rad(station) - norm_ra_rad;
const double lat = oskar_station_lat_rad(station);
oskar_convert_relative_directions_to_enu_directions(1, bypass, 0,
1, 0, 0, 0, ha0, norm_dec_rad, lat, num_points - 1, x, y, z,
status);
}
/* Evaluate the station beam for the given directions. */
if (coord_type == OSKAR_ENU_DIRECTIONS)
evaluate_station_beam_enu_directions(out, num_points, x, y, z,
station, work, time_index, frequency_hz, GAST, status);
else if (coord_type == OSKAR_RELATIVE_DIRECTIONS)
evaluate_station_beam_relative_directions(out, num_points, x, y, z,
station, work, time_index, frequency_hz, GAST, status);
else
*status = OSKAR_ERR_INVALID_ARGUMENT;
/* Scale beam pattern by amplitude at the last source if required. */
if (normalise)
oskar_mem_normalise(out, 0, oskar_mem_length(out),
num_points - 1, status);
/* Copy output beam data. */
oskar_mem_copy_contents(beam, out, offset_out, 0, num_points_orig, status);
}
