void oskar_evaluate_station_beam(oskar_Mem* beam_pattern, int num_points, int coord_type, oskar_Mem* x, oskar_Mem* y, oskar_Mem* z, double norm_ra_rad, double norm_dec_rad, const oskar_Station* station, oskar_StationWork* work, int time_index, double frequency_hz, double GAST, int* status) { int normalise_final_beam; oskar_Mem* out; /* Check if safe to proceed. */ if (*status) return; /* Set default output beam array. */ out = beam_pattern; /* Check that the arrays have enough space to add an extra source at the * end (for normalisation). We don't want to reallocate here, since that * will be slow to do each time: must simply ensure that we pass input * arrays that are large enough. * The normalisation doesn't need to happen if the station has an * isotropic beam. */ normalise_final_beam = oskar_station_normalise_final_beam(station) && (oskar_station_type(station) != OSKAR_STATION_TYPE_ISOTROPIC); if (normalise_final_beam) { double c_x = 0.0, c_y = 0.0, c_z = 1.0; /* Increment number of points. */ num_points++; /* Check the input arrays are big enough to hold the new source. */ if ((int)oskar_mem_length(x) < num_points || (int)oskar_mem_length(y) < num_points || (int)oskar_mem_length(z) < num_points) { *status = OSKAR_ERR_DIMENSION_MISMATCH; return; } /* Set output beam array to work buffer. */ out = oskar_station_work_normalised_beam(work, beam_pattern, status); /* Get the beam direction in the appropriate coordinate system. */ /* (Direction cosines are already set to the interferometer phase * centre for relative directions.) */ if (coord_type == OSKAR_ENU_DIRECTIONS) { double t_x, t_y, t_z, ha0; ha0 = (GAST + oskar_station_lon_rad(station)) - norm_ra_rad; oskar_convert_relative_directions_to_enu_directions_d( &t_x, &t_y, &t_z, 1, &c_x, &c_y, &c_z, ha0, norm_dec_rad, oskar_station_lat_rad(station)); c_x = t_x; c_y = t_y; c_z = t_z; } /* Add the extra normalisation source to the end of the arrays. */ oskar_mem_set_element_real(x, num_points-1, c_x, status); oskar_mem_set_element_real(y, num_points-1, c_y, status); oskar_mem_set_element_real(z, num_points-1, c_z, status); } /* Evaluate the station beam for the given directions. */ if (coord_type == OSKAR_ENU_DIRECTIONS) { evaluate_station_beam_enu_directions(out, num_points, x, y, z, station, work, time_index, frequency_hz, GAST, status); } else if (coord_type == OSKAR_RELATIVE_DIRECTIONS) { evaluate_station_beam_relative_directions(out, num_points, x, y, z, station, work, time_index, frequency_hz, GAST, status); } else { *status = OSKAR_ERR_INVALID_ARGUMENT; } /* Scale beam pattern by value of the last source if required. */ if (normalise_final_beam) { double amp = 0.0; /* Get the last element of the vector and convert to amplitude. */ if (oskar_mem_is_matrix(out)) { double4c val; val = oskar_mem_get_element_matrix(out, num_points-1, status); /* * Scale by square root of "Stokes I" autocorrelation: * sqrt(0.5 * [sum of resultant diagonal]). * * We have * [ Xa Xb ] [ Xa* Xc* ] = [ Xa Xa* + Xb Xb* (don't care) ] * [ Xc Xd ] [ Xb* Xd* ] [ (don't care) Xc Xc* + Xd Xd* ] * * Stokes I is completely real, so need only evaluate the real * part of all the multiplies. Because of the conjugate terms, * these become re*re + im*im. * * Need the square root because we only want the normalised value * for the beam itself (in isolation), not its actual * autocorrelation! */ amp = val.a.x * val.a.x + val.a.y * val.a.y + val.b.x * val.b.x + val.b.y * val.b.y + val.c.x * val.c.x + val.c.y * val.c.y + val.d.x * val.d.x + val.d.y * val.d.y; amp = sqrt(0.5 * amp); } else { double2 val; val = oskar_mem_get_element_complex(out, num_points-1, status); /* Scale by voltage. */ amp = sqrt(val.x * val.x + val.y * val.y); } /* Scale beam array by normalisation value. */ oskar_mem_scale_real(out, 1.0/amp, status); /* Copy output beam data. */ oskar_mem_copy_contents(beam_pattern, out, 0, 0, num_points-1, status); } }
void oskar_interferometer_run(oskar_Interferometer* h, int* status) { int i, num_threads; oskar_Thread** threads = 0; ThreadArgs* args = 0; if (*status || !h) return; /* Check the visibilities are going somewhere. */ if (!h->vis_name #ifndef OSKAR_NO_MS && !h->ms_name #endif ) { oskar_log_error(h->log, "No output file specified."); #ifdef OSKAR_NO_MS if (h->ms_name) oskar_log_error(h->log, "OSKAR was compiled without Measurement Set support."); #endif *status = OSKAR_ERR_FILE_IO; return; } /* Initialise if required. */ oskar_interferometer_check_init(h, status); /* Set up worker threads. */ num_threads = h->num_devices + 1; oskar_barrier_set_num_threads(h->barrier, num_threads); threads = (oskar_Thread**) calloc(num_threads, sizeof(oskar_Thread*)); args = (ThreadArgs*) calloc(num_threads, sizeof(ThreadArgs)); for (i = 0; i < num_threads; ++i) { args[i].h = h; args[i].num_threads = num_threads; args[i].thread_id = i; } /* Record memory usage. */ if (h->log && !*status) { oskar_log_section(h->log, 'M', "Initial memory usage"); #ifdef OSKAR_HAVE_CUDA for (i = 0; i < h->num_gpus; ++i) oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]); #endif system_mem_log(h->log); oskar_log_section(h->log, 'M', "Starting simulation..."); } /* Start simulation timer. */ oskar_timer_start(h->tmr_sim); /* Set status code. */ h->status = *status; /* Start the worker threads. */ oskar_interferometer_reset_work_unit_index(h); for (i = 0; i < num_threads; ++i) threads[i] = oskar_thread_create(run_blocks, (void*)&args[i], 0); /* Wait for worker threads to finish. */ for (i = 0; i < num_threads; ++i) { oskar_thread_join(threads[i]); oskar_thread_free(threads[i]); } free(threads); free(args); /* Get status code. */ *status = h->status; /* Record memory usage. */ if (h->log && !*status) { oskar_log_section(h->log, 'M', "Final memory usage"); #ifdef OSKAR_HAVE_CUDA for (i = 0; i < h->num_gpus; ++i) oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]); #endif system_mem_log(h->log); } /* If there are sources in the simulation and the station beam is not * normalised to 1.0 at the phase centre, the values of noise RMS * may give a very unexpected S/N ratio! * The alternative would be to scale the noise to match the station * beam gain but that would require knowledge of the station beam * amplitude at the phase centre for each time and channel. */ if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status) { int have_sources, amp_calibrated; have_sources = (h->num_sky_chunks > 0 && oskar_sky_num_sources(h->sky_chunks[0]) > 0); amp_calibrated = oskar_station_normalise_final_beam( oskar_telescope_station_const(h->tel, 0)); if (have_sources && !amp_calibrated) { const char* a = "WARNING: System noise added to visibilities"; const char* b = "without station beam normalisation enabled."; const char* c = "This will give an invalid signal to noise ratio."; oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*'); oskar_log_message(h->log, 'W', -1, a); oskar_log_message(h->log, 'W', -1, b); oskar_log_message(h->log, 'W', -1, c); oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' '); } } /* Record times and summarise output files. */ if (h->log && !*status) { size_t log_size = 0; char* log_data; oskar_log_set_value_width(h->log, 25); record_timing(h); oskar_log_section(h->log, 'M', "Simulation complete"); oskar_log_message(h->log, 'M', 0, "Output(s):"); if (h->vis_name) oskar_log_value(h->log, 'M', 1, "OSKAR binary file", "%s", h->vis_name); if (h->ms_name) oskar_log_value(h->log, 'M', 1, "Measurement Set", "%s", h->ms_name); /* Write simulation log to the output files. */ log_data = oskar_log_file_data(h->log, &log_size); #ifndef OSKAR_NO_MS if (h->ms) oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size); #endif if (h->vis) oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN, OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status); free(log_data); } /* Finalise. */ oskar_interferometer_finalise(h, status); }
void oskar_simulator_run(oskar_Simulator* h, int* status) { int i, num_threads = 1, num_vis_blocks; if (*status) return; /* Check the visibilities are going somewhere. */ if (!h->vis_name #ifndef OSKAR_NO_MS && !h->ms_name #endif ) { oskar_log_error(h->log, "No output file specified."); #ifdef OSKAR_NO_MS if (h->ms_name) oskar_log_error(h->log, "OSKAR was compiled without Measurement Set support."); #endif *status = OSKAR_ERR_FILE_IO; return; } /* Initialise if required. */ oskar_simulator_check_init(h, status); /* Get the number of visibility blocks to be processed. */ num_vis_blocks = oskar_simulator_num_vis_blocks(h); /* Record memory usage. */ if (h->log && !*status) { oskar_log_section(h->log, 'M', "Initial memory usage"); #ifdef OSKAR_HAVE_CUDA for (i = 0; i < h->num_gpus; ++i) oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]); #endif system_mem_log(h->log); oskar_log_section(h->log, 'M', "Starting simulation..."); } /* Start simulation timer. */ oskar_timer_start(h->tmr_sim); /*----------------------------------------------------------------------- *-- START OF MULTITHREADED SIMULATION CODE ----------------------------- *-----------------------------------------------------------------------*/ /* Loop over blocks of observation time, running simulation and file * writing one block at a time. Simulation and file output are overlapped * by using double buffering, and a dedicated thread is used for file * output. * * Thread 0 is used for file writes. * Threads 1 to n (mapped to compute devices) do the simulation. * * Note that no write is launched on the first loop counter (as no * data are ready yet) and no simulation is performed for the last loop * counter (which corresponds to the last block + 1) as this iteration * simply writes the last block. */ #ifdef _OPENMP num_threads = h->num_devices + 1; omp_set_num_threads(num_threads); omp_set_nested(0); #else oskar_log_warning(h->log, "OpenMP not found: Using one compute device."); #endif oskar_simulator_reset_work_unit_index(h); #pragma omp parallel { int b, thread_id = 0, device_id = 0; /* Get host thread ID and device ID. */ #ifdef _OPENMP thread_id = omp_get_thread_num(); device_id = thread_id - 1; #endif /* Loop over simulation time blocks (+1, for the last write). */ for (b = 0; b < num_vis_blocks + 1; ++b) { if ((thread_id > 0 || num_threads == 1) && b < num_vis_blocks) oskar_simulator_run_block(h, b, device_id, status); if (thread_id == 0 && b > 0) { oskar_VisBlock* block; block = oskar_simulator_finalise_block(h, b - 1, status); oskar_simulator_write_block(h, block, b - 1, status); } /* Barrier 1: Reset work unit index. */ #pragma omp barrier if (thread_id == 0) oskar_simulator_reset_work_unit_index(h); /* Barrier 2: Synchronise before moving to the next block. */ #pragma omp barrier if (thread_id == 0 && b < num_vis_blocks && h->log && !*status) oskar_log_message(h->log, 'S', 0, "Block %*i/%i (%3.0f%%) " "complete. Simulation time elapsed: %.3f s", disp_width(num_vis_blocks), b+1, num_vis_blocks, 100.0 * (b+1) / (double)num_vis_blocks, oskar_timer_elapsed(h->tmr_sim)); } } /*----------------------------------------------------------------------- *-- END OF MULTITHREADED SIMULATION CODE ------------------------------- *-----------------------------------------------------------------------*/ /* Record memory usage. */ if (h->log && !*status) { oskar_log_section(h->log, 'M', "Final memory usage"); #ifdef OSKAR_HAVE_CUDA for (i = 0; i < h->num_gpus; ++i) oskar_cuda_mem_log(h->log, 0, h->gpu_ids[i]); #endif system_mem_log(h->log); } /* If there are sources in the simulation and the station beam is not * normalised to 1.0 at the phase centre, the values of noise RMS * may give a very unexpected S/N ratio! * The alternative would be to scale the noise to match the station * beam gain but that would require knowledge of the station beam * amplitude at the phase centre for each time and channel. */ if (h->log && oskar_telescope_noise_enabled(h->tel) && !*status) { int have_sources, amp_calibrated; have_sources = (h->num_sky_chunks > 0 && oskar_sky_num_sources(h->sky_chunks[0]) > 0); amp_calibrated = oskar_station_normalise_final_beam( oskar_telescope_station_const(h->tel, 0)); if (have_sources && !amp_calibrated) { const char* a = "WARNING: System noise added to visibilities"; const char* b = "without station beam normalisation enabled."; const char* c = "This will give an invalid signal to noise ratio."; oskar_log_line(h->log, 'W', ' '); oskar_log_line(h->log, 'W', '*'); oskar_log_message(h->log, 'W', -1, a); oskar_log_message(h->log, 'W', -1, b); oskar_log_message(h->log, 'W', -1, c); oskar_log_line(h->log, 'W', '*'); oskar_log_line(h->log, 'W', ' '); } } /* Record times and summarise output files. */ if (h->log && !*status) { size_t log_size = 0; char* log_data; oskar_log_set_value_width(h->log, 25); record_timing(h); oskar_log_section(h->log, 'M', "Simulation complete"); oskar_log_message(h->log, 'M', 0, "Output(s):"); if (h->vis_name) oskar_log_value(h->log, 'M', 1, "OSKAR binary file", "%s", h->vis_name); if (h->ms_name) oskar_log_value(h->log, 'M', 1, "Measurement Set", "%s", h->ms_name); /* Write simulation log to the output files. */ log_data = oskar_log_file_data(h->log, &log_size); #ifndef OSKAR_NO_MS if (h->ms) oskar_ms_add_history(h->ms, "OSKAR_LOG", log_data, log_size); #endif if (h->vis) oskar_binary_write(h->vis, OSKAR_CHAR, OSKAR_TAG_GROUP_RUN, OSKAR_TAG_RUN_LOG, 0, log_size, log_data, status); free(log_data); } /* Finalise. */ oskar_simulator_finalise(h, status); }
void oskar_evaluate_station_beam(int num_points, int coord_type, oskar_Mem* x, oskar_Mem* y, oskar_Mem* z, double norm_ra_rad, double norm_dec_rad, const oskar_Station* station, oskar_StationWork* work, int time_index, double frequency_hz, double GAST, int offset_out, oskar_Mem* beam, int* status) { oskar_Mem* out; const size_t num_points_orig = (size_t)num_points; if (*status) return; /* Set output beam array to work buffer. */ out = oskar_station_work_beam_out(work, beam, num_points_orig, status); /* Check that the arrays have enough space to add an extra source at the * end (for normalisation). We don't want to reallocate here, since that * will be slow to do each time: must simply ensure that we pass input * arrays that are large enough. * The normalisation doesn't need to happen if the station has an * isotropic beam. */ const int normalise = oskar_station_normalise_final_beam(station) && (oskar_station_type(station) != OSKAR_STATION_TYPE_ISOTROPIC); if (normalise) { /* Increment number of points. */ num_points++; /* Check the input arrays are big enough to hold the new source. */ if ((int)oskar_mem_length(x) < num_points || (int)oskar_mem_length(y) < num_points || (int)oskar_mem_length(z) < num_points) { *status = OSKAR_ERR_DIMENSION_MISMATCH; return; } /* Get the beam direction in the appropriate coordinate system. */ const int bypass = (coord_type != OSKAR_ENU_DIRECTIONS); const double ha0 = GAST + oskar_station_lon_rad(station) - norm_ra_rad; const double lat = oskar_station_lat_rad(station); oskar_convert_relative_directions_to_enu_directions(1, bypass, 0, 1, 0, 0, 0, ha0, norm_dec_rad, lat, num_points - 1, x, y, z, status); } /* Evaluate the station beam for the given directions. */ if (coord_type == OSKAR_ENU_DIRECTIONS) evaluate_station_beam_enu_directions(out, num_points, x, y, z, station, work, time_index, frequency_hz, GAST, status); else if (coord_type == OSKAR_RELATIVE_DIRECTIONS) evaluate_station_beam_relative_directions(out, num_points, x, y, z, station, work, time_index, frequency_hz, GAST, status); else *status = OSKAR_ERR_INVALID_ARGUMENT; /* Scale beam pattern by amplitude at the last source if required. */ if (normalise) oskar_mem_normalise(out, 0, oskar_mem_length(out), num_points - 1, status); /* Copy output beam data. */ oskar_mem_copy_contents(beam, out, offset_out, 0, num_points_orig, status); }