void oskar_simulator_run_block(oskar_Simulator* h, int block_index,
int device_id, int* status)
{
double obs_start_mjd, dt_dump_days;
int i_active, time_index_start, time_index_end;
int num_channels, num_times_block, total_chunks, total_times;
DeviceData* d;
if (*status) return;
/* Check that initialisation has happened. We can't initialise here,
* as we're already multi-threaded at this point. */
if (!h->header)
{
*status = OSKAR_ERR_MEMORY_NOT_ALLOCATED;
oskar_log_error(h->log, "Simulator not initalised. "
"Call oskar_simulator_check_init() first.");
return;
}
#ifdef _OPENMP
if (!h->coords_only)
{
/* Disable any nested parallelism. */
omp_set_num_threads(1);
omp_set_nested(0);
}
#endif
/* Set the GPU to use. (Supposed to be a very low-overhead call.) */
if (device_id >= 0 && device_id < h->num_gpus)
oskar_device_set(h->gpu_ids[device_id], status);
/* Clear the visibility block. */
i_active = block_index % 2; /* Index of the active buffer. */
d = &(h->d[device_id]);
oskar_timer_resume(d->tmr_compute);
oskar_vis_block_clear(d->vis_block, status);
/* Set the visibility block meta-data. */
total_chunks = h->num_sky_chunks;
num_channels = h->num_channels;
total_times = h->num_time_steps;
obs_start_mjd = h->time_start_mjd_utc;
dt_dump_days = h->time_inc_sec / 86400.0;
time_index_start = block_index * h->max_times_per_block;
time_index_end = time_index_start + h->max_times_per_block - 1;
if (time_index_end >= total_times)
time_index_end = total_times - 1;
num_times_block = 1 + time_index_end - time_index_start;
/* Set the number of active times in the block. */
oskar_vis_block_set_num_times(d->vis_block, num_times_block, status);
oskar_vis_block_set_start_time_index(d->vis_block, time_index_start);
/* Go though all possible work units in the block. A work unit is defined
* as the simulation for one time and one sky chunk. */
while (!h->coords_only)
{
oskar_Sky* sky;
int i_work_unit, i_chunk, i_time, i_channel, sim_time_idx;
oskar_mutex_lock(h->mutex);
i_work_unit = (h->work_unit_index)++;
oskar_mutex_unlock(h->mutex);
if ((i_work_unit >= num_times_block * total_chunks) || *status) break;
/* Convert slice index to chunk/time index. */
i_chunk = i_work_unit / num_times_block;
i_time = i_work_unit - i_chunk * num_times_block;
sim_time_idx = time_index_start + i_time;
/* Copy sky chunk to device only if different from the previous one. */
if (i_chunk != d->previous_chunk_index)
{
oskar_timer_resume(d->tmr_copy);
oskar_sky_copy(d->chunk, h->sky_chunks[i_chunk], status);
oskar_timer_pause(d->tmr_copy);
}
sky = h->apply_horizon_clip ? d->chunk_clip : d->chunk;
/* Apply horizon clip if required. */
if (h->apply_horizon_clip)
{
double gast, mjd;
mjd = obs_start_mjd + dt_dump_days * (sim_time_idx + 0.5);
gast = oskar_convert_mjd_to_gast_fast(mjd);
oskar_timer_resume(d->tmr_clip);
oskar_sky_horizon_clip(d->chunk_clip, d->chunk, d->tel, gast,
d->station_work, status);
oskar_timer_pause(d->tmr_clip);
}
/* Simulate all baselines for all channels for this time and chunk. */
for (i_channel = 0; i_channel < num_channels; ++i_channel)
{
if (*status) break;
if (h->log)
{
oskar_mutex_lock(h->mutex);
oskar_log_message(h->log, 'S', 1, "Time %*i/%i, "
"Chunk %*i/%i, Channel %*i/%i [Device %i, %i sources]",
disp_width(total_times), sim_time_idx + 1, total_times,
disp_width(total_chunks), i_chunk + 1, total_chunks,
disp_width(num_channels), i_channel + 1, num_channels,
device_id, oskar_sky_num_sources(sky));
oskar_mutex_unlock(h->mutex);
}
sim_baselines(h, d, sky, i_channel, i_time, sim_time_idx, status);
}
d->previous_chunk_index = i_chunk;
}
/* Copy the visibility block to host memory. */
oskar_timer_resume(d->tmr_copy);
oskar_vis_block_copy(d->vis_block_cpu[i_active], d->vis_block, status);
oskar_timer_pause(d->tmr_copy);
oskar_timer_pause(d->tmr_compute);
}
static void sim_baselines(oskar_Simulator* h, DeviceData* d, oskar_Sky* sky,
int channel_index_block, int time_index_block,
int time_index_simulation, int* status)
{
int num_baselines, num_stations, num_src, num_times_block, num_channels;
double dt_dump_days, t_start, t_dump, gast, frequency, ra0, dec0;
const oskar_Mem *x, *y, *z;
oskar_Mem* alias = 0;
/* Get dimensions. */
num_baselines = oskar_telescope_num_baselines(d->tel);
num_stations = oskar_telescope_num_stations(d->tel);
num_src = oskar_sky_num_sources(sky);
num_times_block = oskar_vis_block_num_times(d->vis_block);
num_channels = oskar_vis_block_num_channels(d->vis_block);
/* Return if there are no sources in the chunk,
* or if block time index requested is outside the valid range. */
if (num_src == 0 || time_index_block >= num_times_block) return;
/* Get the time and frequency of the visibility slice being simulated. */
dt_dump_days = h->time_inc_sec / 86400.0;
t_start = h->time_start_mjd_utc;
t_dump = t_start + dt_dump_days * (time_index_simulation + 0.5);
gast = oskar_convert_mjd_to_gast_fast(t_dump);
frequency = h->freq_start_hz + channel_index_block * h->freq_inc_hz;
/* Scale source fluxes with spectral index and rotation measure. */
oskar_sky_scale_flux_with_frequency(sky, frequency, status);
/* Evaluate station u,v,w coordinates. */
ra0 = oskar_telescope_phase_centre_ra_rad(d->tel);
dec0 = oskar_telescope_phase_centre_dec_rad(d->tel);
x = oskar_telescope_station_true_x_offset_ecef_metres_const(d->tel);
y = oskar_telescope_station_true_y_offset_ecef_metres_const(d->tel);
z = oskar_telescope_station_true_z_offset_ecef_metres_const(d->tel);
oskar_convert_ecef_to_station_uvw(num_stations, x, y, z, ra0, dec0, gast,
d->u, d->v, d->w, status);
/* Set dimensions of Jones matrices. */
if (d->R)
oskar_jones_set_size(d->R, num_stations, num_src, status);
if (d->Z)
oskar_jones_set_size(d->Z, num_stations, num_src, status);
oskar_jones_set_size(d->J, num_stations, num_src, status);
oskar_jones_set_size(d->E, num_stations, num_src, status);
oskar_jones_set_size(d->K, num_stations, num_src, status);
/* Evaluate station beam (Jones E: may be matrix). */
oskar_timer_resume(d->tmr_E);
oskar_evaluate_jones_E(d->E, num_src, OSKAR_RELATIVE_DIRECTIONS,
oskar_sky_l(sky), oskar_sky_m(sky), oskar_sky_n(sky), d->tel,
gast, frequency, d->station_work, time_index_simulation, status);
oskar_timer_pause(d->tmr_E);
#if 0
/* Evaluate ionospheric phase (Jones Z: scalar) and join with Jones E.
* NOTE this is currently only a CPU implementation. */
if (d->Z)
{
oskar_evaluate_jones_Z(d->Z, num_src, sky, d->tel,
&settings->ionosphere, gast, frequency, &(d->workJonesZ),
status);
oskar_timer_resume(d->tmr_join);
oskar_jones_join(d->E, d->Z, d->E, status);
oskar_timer_pause(d->tmr_join);
}
#endif
/* Evaluate parallactic angle (Jones R: matrix), and join with Jones Z*E.
* TODO Move this into station beam evaluation instead. */
if (d->R)
{
oskar_timer_resume(d->tmr_E);
oskar_evaluate_jones_R(d->R, num_src, oskar_sky_ra_rad_const(sky),
oskar_sky_dec_rad_const(sky), d->tel, gast, status);
oskar_timer_pause(d->tmr_E);
oskar_timer_resume(d->tmr_join);
oskar_jones_join(d->R, d->E, d->R, status);
oskar_timer_pause(d->tmr_join);
}
/* Evaluate interferometer phase (Jones K: scalar). */
oskar_timer_resume(d->tmr_K);
oskar_evaluate_jones_K(d->K, num_src, oskar_sky_l_const(sky),
oskar_sky_m_const(sky), oskar_sky_n_const(sky), d->u, d->v, d->w,
frequency, oskar_sky_I_const(sky),
h->source_min_jy, h->source_max_jy, status);
oskar_timer_pause(d->tmr_K);
/* Join Jones K with Jones Z*E. */
oskar_timer_resume(d->tmr_join);
oskar_jones_join(d->J, d->K, d->R ? d->R : d->E, status);
oskar_timer_pause(d->tmr_join);
/* Create alias for auto/cross-correlations. */
oskar_timer_resume(d->tmr_correlate);
alias = oskar_mem_create_alias(0, 0, 0, status);
/* Auto-correlate for this time and channel. */
if (oskar_vis_block_has_auto_correlations(d->vis_block))
{
oskar_mem_set_alias(alias,
oskar_vis_block_auto_correlations(d->vis_block),
num_stations *
(num_channels * time_index_block + channel_index_block),
num_stations, status);
oskar_auto_correlate(alias, num_src, d->J, sky, status);
}
/* Cross-correlate for this time and channel. */
if (oskar_vis_block_has_cross_correlations(d->vis_block))
{
oskar_mem_set_alias(alias,
oskar_vis_block_cross_correlations(d->vis_block),
num_baselines *
(num_channels * time_index_block + channel_index_block),
num_baselines, status);
oskar_cross_correlate(alias, num_src, d->J, sky, d->tel,
d->u, d->v, d->w, gast, frequency, status);
}
/* Free alias for auto/cross-correlations. */
oskar_mem_free(alias, status);
oskar_timer_pause(d->tmr_correlate);
}
int main(int argc, char** argv)
{
int status = 0;
oskar::OptionParser opt("oskar_evaulate_pierce_points",
oskar_version_string());
opt.add_required("settings file");
if (!opt.check_options(argc, argv)) return EXIT_FAILURE;
const char* settings_file = opt.get_arg();
// Create the log.
oskar_Log* log = oskar_log_create(OSKAR_LOG_MESSAGE, OSKAR_LOG_STATUS);
oskar_log_message(log, 'M', 0, "Running binary %s", argv[0]);
// Enum values used in writing time-freq data binary files
enum OSKAR_TIME_FREQ_TAGS
{
TIME_IDX = 0,
FREQ_IDX = 1,
TIME_MJD_UTC = 2,
FREQ_HZ = 3,
NUM_FIELDS = 4,
NUM_FIELD_TAGS = 5,
HEADER_OFFSET = 10,
DATA = 0,
DIMS = 1,
LABEL = 2,
UNITS = 3,
GRP = OSKAR_TAG_GROUP_TIME_FREQ_DATA
};
oskar_Settings_old settings;
oskar_settings_old_load(&settings, log, settings_file, &status);
oskar_log_set_keep_file(log, settings.sim.keep_log_file);
if (status) return status;
oskar_Telescope* tel = oskar_settings_to_telescope(&settings, log, &status);
oskar_Sky* sky = oskar_settings_to_sky(&settings, log, &status);
// FIXME remove this restriction ... (see evaluate Z)
if (settings.ionosphere.num_TID_screens != 1)
return OSKAR_ERR_SETUP_FAIL;
int type = settings.sim.double_precision ? OSKAR_DOUBLE : OSKAR_SINGLE;
int loc = OSKAR_CPU;
int num_sources = oskar_sky_num_sources(sky);
oskar_Mem *hor_x, *hor_y, *hor_z;
hor_x = oskar_mem_create(type, loc, num_sources, &status);
hor_y = oskar_mem_create(type, loc, num_sources, &status);
hor_z = oskar_mem_create(type, loc, num_sources, &status);
oskar_Mem *pp_lon, *pp_lat, *pp_rel_path;
int num_stations = oskar_telescope_num_stations(tel);
int num_pp = num_stations * num_sources;
pp_lon = oskar_mem_create(type, loc, num_pp, &status);
pp_lat = oskar_mem_create(type, loc, num_pp, &status);
pp_rel_path = oskar_mem_create(type, loc, num_pp, &status);
// Pierce points for one station (non-owned oskar_Mem pointers)
oskar_Mem *pp_st_lon, *pp_st_lat, *pp_st_rel_path;
pp_st_lon = oskar_mem_create_alias(0, 0, 0, &status);
pp_st_lat = oskar_mem_create_alias(0, 0, 0, &status);
pp_st_rel_path = oskar_mem_create_alias(0, 0, 0, &status);
int num_times = settings.obs.num_time_steps;
double obs_start_mjd_utc = settings.obs.start_mjd_utc;
double dt_dump = settings.obs.dt_dump_days;
// Binary file meta-data
std::string label1 = "pp_lon";
std::string label2 = "pp_lat";
std::string label3 = "pp_path";
std::string units = "radians";
std::string units2 = "";
oskar_Mem *dims = oskar_mem_create(OSKAR_INT, loc, 2, &status);
/* FIXME is this the correct dimension order ?
* FIXME get the MATLAB reader to respect dimension ordering */
oskar_mem_int(dims, &status)[0] = num_sources;
oskar_mem_int(dims, &status)[1] = num_stations;
const char* filename = settings.ionosphere.pierce_points.filename;
oskar_Binary* h = oskar_binary_create(filename, 'w', &status);
double screen_height_m = settings.ionosphere.TID->height_km * 1000.0;
// printf("Number of times = %i\n", num_times);
// printf("Number of stations = %i\n", num_stations);
void *x_, *y_, *z_;
x_ = oskar_mem_void(oskar_telescope_station_true_x_offset_ecef_metres(tel));
y_ = oskar_mem_void(oskar_telescope_station_true_y_offset_ecef_metres(tel));
z_ = oskar_mem_void(oskar_telescope_station_true_z_offset_ecef_metres(tel));
for (int t = 0; t < num_times; ++t)
{
double t_dump = obs_start_mjd_utc + t * dt_dump; // MJD UTC
double gast = oskar_convert_mjd_to_gast_fast(t_dump + dt_dump / 2.0);
for (int i = 0; i < num_stations; ++i)
{
const oskar_Station* station =
oskar_telescope_station_const(tel, i);
double lon = oskar_station_lon_rad(station);
double lat = oskar_station_lat_rad(station);
double alt = oskar_station_alt_metres(station);
double x_ecef, y_ecef, z_ecef, x_offset, y_offset, z_offset;
if (type == OSKAR_DOUBLE)
{
x_offset = ((double*)x_)[i];
y_offset = ((double*)y_)[i];
z_offset = ((double*)z_)[i];
}
else
{
x_offset = (double)((float*)x_)[i];
y_offset = (double)((float*)y_)[i];
z_offset = (double)((float*)z_)[i];
}
oskar_convert_offset_ecef_to_ecef(1, &x_offset, &y_offset,
&z_offset, lon, lat, alt, &x_ecef, &y_ecef, &z_ecef);
double last = gast + lon;
if (type == OSKAR_DOUBLE)
{
oskar_convert_apparent_ra_dec_to_enu_directions_d(num_sources,
oskar_mem_double_const(oskar_sky_ra_rad_const(sky), &status),
oskar_mem_double_const(oskar_sky_dec_rad_const(sky), &status),
last, lat, oskar_mem_double(hor_x, &status),
oskar_mem_double(hor_y, &status),
oskar_mem_double(hor_z, &status));
}
else
{
oskar_convert_apparent_ra_dec_to_enu_directions_f(num_sources,
oskar_mem_float_const(oskar_sky_ra_rad_const(sky), &status),
oskar_mem_float_const(oskar_sky_dec_rad_const(sky), &status),
last, lat, oskar_mem_float(hor_x, &status),
oskar_mem_float(hor_y, &status),
oskar_mem_float(hor_z, &status));
}
int offset = i * num_sources;
oskar_mem_set_alias(pp_st_lon, pp_lon, offset, num_sources,
&status);
oskar_mem_set_alias(pp_st_lat, pp_lat, offset, num_sources,
&status);
oskar_mem_set_alias(pp_st_rel_path, pp_rel_path, offset,
num_sources, &status);
oskar_evaluate_pierce_points(pp_st_lon, pp_st_lat, pp_st_rel_path,
x_ecef, y_ecef, z_ecef, screen_height_m,
num_sources, hor_x, hor_y, hor_z, &status);
} // Loop over stations.
if (status != 0)
continue;
int index = t; // could be = (num_times * f) + t if we have frequency data
int num_fields = 3;
int num_field_tags = 4;
double freq_hz = 0.0;
int freq_idx = 0;
// Write the header TAGS
oskar_binary_write_int(h, GRP, TIME_IDX, index, t, &status);
oskar_binary_write_double(h, GRP, FREQ_IDX, index, freq_idx, &status);
oskar_binary_write_double(h, GRP, TIME_MJD_UTC, index, t_dump, &status);
oskar_binary_write_double(h, GRP, FREQ_HZ, index, freq_hz, &status);
oskar_binary_write_int(h, GRP, NUM_FIELDS, index, num_fields, &status);
oskar_binary_write_int(h, GRP, NUM_FIELD_TAGS, index, num_field_tags,
&status);
// Write data TAGS (fields)
int field, tagID;
field = 0;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_lon, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label1.size()+1, label1.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units.size()+1, units.c_str(), &status);
field = 1;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_lat, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label2.size()+1, label2.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units.size()+1, units.c_str(), &status);
field = 2;
tagID = HEADER_OFFSET + (num_field_tags * field);
oskar_binary_write_mem(h, pp_rel_path, GRP, tagID + DATA,
index, 0, &status);
oskar_binary_write_mem(h, dims, GRP, tagID + DIMS,
index, 0, &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + LABEL,
index, label3.size()+1, label3.c_str(), &status);
oskar_binary_write(h, OSKAR_CHAR, GRP, tagID + UNITS,
index, units2.size()+1, units2.c_str(), &status);
} // Loop over times
// Close the OSKAR binary file.
oskar_binary_free(h);
// clean up memory
oskar_mem_free(hor_x, &status);
oskar_mem_free(hor_y, &status);
oskar_mem_free(hor_z, &status);
oskar_mem_free(pp_lon, &status);
oskar_mem_free(pp_lat, &status);
oskar_mem_free(pp_rel_path, &status);
oskar_mem_free(pp_st_lon, &status);
oskar_mem_free(pp_st_lat, &status);
oskar_mem_free(pp_st_rel_path, &status);
oskar_mem_free(dims, &status);
oskar_telescope_free(tel, &status);
oskar_sky_free(sky, &status);
// Check for errors.
if (status)
oskar_log_error(log, "Run failed: %s.", oskar_get_error_string(status));
oskar_log_free(log);
return status;
}
