oskar_VisBlock* oskar_simulator_finalise_block(oskar_Simulator* h,
int block_index, int* status)
{
int i, i_active;
oskar_VisBlock *b0 = 0, *b = 0;
if (*status) return 0;
/* The visibilities must be copied back
* at the end of the block simulation. */
/* Combine all vis blocks into the first one. */
i_active = (block_index + 1) % 2;
b0 = h->d[0].vis_block_cpu[!i_active];
if (!h->coords_only)
{
oskar_Mem *xc0 = 0, *ac0 = 0;
xc0 = oskar_vis_block_cross_correlations(b0);
ac0 = oskar_vis_block_auto_correlations(b0);
for (i = 1; i < h->num_devices; ++i)
{
b = h->d[i].vis_block_cpu[!i_active];
if (oskar_vis_block_has_cross_correlations(b))
oskar_mem_add(xc0, xc0, oskar_vis_block_cross_correlations(b),
oskar_mem_length(xc0), status);
if (oskar_vis_block_has_auto_correlations(b))
oskar_mem_add(ac0, ac0, oskar_vis_block_auto_correlations(b),
oskar_mem_length(ac0), status);
}
}
/* Calculate baseline uvw coordinates for the block. */
if (oskar_vis_block_has_cross_correlations(b0))
{
const oskar_Mem *x, *y, *z;
x = oskar_telescope_station_measured_x_offset_ecef_metres_const(h->tel);
y = oskar_telescope_station_measured_y_offset_ecef_metres_const(h->tel);
z = oskar_telescope_station_measured_z_offset_ecef_metres_const(h->tel);
oskar_convert_ecef_to_baseline_uvw(
oskar_telescope_num_stations(h->tel), x, y, z,
oskar_telescope_phase_centre_ra_rad(h->tel),
oskar_telescope_phase_centre_dec_rad(h->tel),
oskar_vis_block_num_times(b0),
oskar_vis_header_time_start_mjd_utc(h->header),
oskar_vis_header_time_inc_sec(h->header) / 86400.0,
oskar_vis_block_start_time_index(b0),
oskar_vis_block_baseline_uu_metres(b0),
oskar_vis_block_baseline_vv_metres(b0),
oskar_vis_block_baseline_ww_metres(b0), h->temp, status);
}
/* Add uncorrelated system noise to the combined visibilities. */
if (!h->coords_only)
{
oskar_vis_block_add_system_noise(b0, h->header, h->tel,
block_index, h->temp, status);
}
/* Return a pointer to the block. */
return b0;
}
/* Evaluate the TEC value for each pierce point - note: at the moment this is
* just the accumulation of one or more TID screens.
* TODO convert this to a stand-alone function.
*/
static void oskar_evaluate_TEC(oskar_WorkJonesZ* work, int num_pp,
const oskar_SettingsIonosphere* settings,
double gast, int* status)
{
int i;
/* FIXME(BM) For now limit number of screens to 1, this can be removed
* if a TEC model which is valid for multiple screens is implemented
*/
if (settings->num_TID_screens > 1)
*status = OSKAR_ERR_INVALID_ARGUMENT;
oskar_mem_set_value_real(work->total_TEC, 0.0, 0, 0, status);
/* Loop over TID screens to evaluate TEC values */
for (i = 0; i < settings->num_TID_screens; ++i)
{
oskar_mem_set_value_real(work->screen_TEC, 0.0, 0, 0, status);
/* Evaluate TEC values for the screen */
oskar_evaluate_tec_tid(work->screen_TEC, num_pp, work->pp_lon,
work->pp_lat, work->pp_rel_path, settings->TEC0,
&settings->TID[i], gast);
/* Accumulate into total TEC */
/* FIXME(BM) addition is not physical for more than one TEC screen in
* the way TIDs are currently evaluated as TEC0 is added into both
* screens.
*/
oskar_mem_add(work->total_TEC, work->total_TEC, work->screen_TEC,
oskar_mem_length(work->total_TEC), status);
}
}
static void* run_blocks(void* arg)
{
oskar_Imager* h;
oskar_Mem *plane, *uu, *vv, *ww = 0, *amp, *weight, *block, *l, *m, *n;
size_t max_size;
const size_t smallest = 1024, largest = 65536;
int dev_loc = OSKAR_CPU, *status;
/* Get thread function arguments. */
h = ((ThreadArgs*)arg)->h;
const int thread_id = ((ThreadArgs*)arg)->thread_id;
const int num_vis = ((ThreadArgs*)arg)->num_vis;
plane = ((ThreadArgs*)arg)->plane;
status = &(h->status);
/* Set the device used by the thread. */
if (thread_id < h->num_gpus)
{
dev_loc = h->dev_loc;
oskar_device_set(h->dev_loc, h->gpu_ids[thread_id], status);
}
/* Copy visibility data to device. */
uu = oskar_mem_create_copy(((ThreadArgs*)arg)->uu, dev_loc, status);
vv = oskar_mem_create_copy(((ThreadArgs*)arg)->vv, dev_loc, status);
amp = oskar_mem_create_copy(((ThreadArgs*)arg)->amp, dev_loc, status);
weight = oskar_mem_create_copy(((ThreadArgs*)arg)->weight, dev_loc, status);
if (h->algorithm == OSKAR_ALGORITHM_DFT_3D)
ww = oskar_mem_create_copy(((ThreadArgs*)arg)->ww, dev_loc, status);
#ifdef _OPENMP
/* Disable nested parallelism. */
omp_set_nested(0);
omp_set_num_threads(1);
#endif
/* Calculate the maximum pixel block size, and number of blocks. */
const size_t num_pixels = (size_t)h->image_size * (size_t)h->image_size;
max_size = num_pixels / h->num_devices;
max_size = ((max_size + smallest - 1) / smallest) * smallest;
if (max_size > largest) max_size = largest;
if (max_size < smallest) max_size = smallest;
const int num_blocks = (int) ((num_pixels + max_size - 1) / max_size);
/* Allocate device memory for pixel block data. */
block = oskar_mem_create(h->imager_prec, dev_loc, 0, status);
l = oskar_mem_create(h->imager_prec, dev_loc, max_size, status);
m = oskar_mem_create(h->imager_prec, dev_loc, max_size, status);
n = oskar_mem_create(h->imager_prec, dev_loc, max_size, status);
/* Loop until all blocks are done. */
for (;;)
{
size_t block_size;
/* Get a unique block index. */
oskar_mutex_lock(h->mutex);
const int i_block = (h->i_block)++;
oskar_mutex_unlock(h->mutex);
if ((i_block >= num_blocks) || *status) break;
/* Calculate the block size. */
const size_t block_start = i_block * max_size;
block_size = num_pixels - block_start;
if (block_size > max_size) block_size = max_size;
/* Copy the (l,m,n) positions for the block. */
oskar_mem_copy_contents(l, h->l, 0, block_start, block_size, status);
oskar_mem_copy_contents(m, h->m, 0, block_start, block_size, status);
if (h->algorithm == OSKAR_ALGORITHM_DFT_3D)
oskar_mem_copy_contents(n, h->n, 0, block_start,
block_size, status);
/* Run DFT for the block. */
oskar_dft_c2r(num_vis, 2.0 * M_PI, uu, vv, ww, amp, weight,
(int) block_size, l, m, n, block, status);
/* Add data to existing pixels. */
oskar_mem_add(plane, plane, block,
block_start, block_start, 0, block_size, status);
}
/* Free memory. */
oskar_mem_free(uu, status);
oskar_mem_free(vv, status);
oskar_mem_free(ww, status);
oskar_mem_free(amp, status);
oskar_mem_free(weight, status);
oskar_mem_free(block, status);
oskar_mem_free(l, status);
oskar_mem_free(m, status);
oskar_mem_free(n, status);
return 0;
}
