void finalize_histogram(hist_info *hist){
if(!hist->flag_finalized){
SID_Allreduce(SID_IN_PLACE, (hist->bin_count), hist->n_bins,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
SID_Allreduce(SID_IN_PLACE,&(hist->count_hist),1, SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
SID_Allreduce(SID_IN_PLACE,&(hist->count_all), 1, SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
}
hist->flag_finalized=TRUE;
}
void calc_min_global(void *data_local,
void *result,
size_t n_data_local,
SID_Datatype type,
int mode,
SID_Comm *comm){
calc_min(data_local,result,n_data_local,type,mode);
if(check_mode_for_flag(mode,CALC_MODE_RETURN_DOUBLE))
SID_Allreduce(SID_IN_PLACE,result,1,SID_DOUBLE,SID_MIN,comm);
else
SID_Allreduce(SID_IN_PLACE,result,1,type, SID_MIN,comm);
}
void init_field(int n_d, int *n, double *L, field_info *FFT) {
ptrdiff_t n_x_local;
ptrdiff_t i_x_start_local;
ptrdiff_t n_y_transpose_local;
ptrdiff_t i_y_start_transpose_local;
ptrdiff_t *n_x_rank;
int flag_active;
int n_active;
int min_size, max_size;
SID_log("Initializing ", SID_LOG_OPEN);
for(ptrdiff_t i_d = 0; i_d < n_d; i_d++) {
if(i_d < (n_d - 1))
SID_log("%dx", SID_LOG_CONTINUE, n[i_d]);
else
SID_log("%d element %d-d FFT ", SID_LOG_CONTINUE, n[i_d], n_d);
}
SID_log("(%d byte precision)...", SID_LOG_CONTINUE, (int)sizeof(GBPREAL));
// Initialize FFT sizes
FFT->n_d = n_d;
FFT->n = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->L = (double *)SID_calloc(sizeof(double) * FFT->n_d);
FFT->n_k_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->n_R_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->i_R_start_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->i_k_start_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->i_R_stop_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
FFT->i_k_stop_local = (ptrdiff_t *)SID_calloc(sizeof(ptrdiff_t) * FFT->n_d);
for(ptrdiff_t i_d = 0; i_d < FFT->n_d; i_d++) {
FFT->n[i_d] = n[i_d];
FFT->L[i_d] = L[i_d];
FFT->i_R_start_local[i_d] = 0;
FFT->i_k_start_local[i_d] = 0;
FFT->n_R_local[i_d] = FFT->n[i_d];
FFT->n_k_local[i_d] = FFT->n[i_d];
}
FFT->n_k_local[FFT->n_d - 1] = FFT->n[FFT->n_d - 1] / 2 + 1;
// Initialize FFTW
// Create an integer version of FFT->n[] to pass to ..._create_plan
int *n_int=(int *)SID_malloc(sizeof(int)*FFT->n_d);
for(int i_d=0;i_d<FFT->n_d;i_d++)
n_int[i_d]=(int)FFT->n[i_d];
#if FFTW_V2
#if USE_MPI
int total_local_size_int;
int n_x_local_int;
int i_x_start_local_int;
int n_y_transpose_local_int;
int i_y_start_transpose_local_int;
FFT->plan = rfftwnd_mpi_create_plan(SID.COMM_WORLD->comm, FFT->n_d, n_int, FFTW_REAL_TO_COMPLEX, FFTW_ESTIMATE);
FFT->iplan = rfftwnd_mpi_create_plan(SID.COMM_WORLD->comm, FFT->n_d, n_int, FFTW_COMPLEX_TO_REAL, FFTW_ESTIMATE);
rfftwnd_mpi_local_sizes(FFT->plan,
&(n_x_local_int),
&(i_x_start_local_int),
&(n_y_transpose_local_int),
&(i_y_start_transpose_local_int),
&total_local_size_int);
n_x_local = (ptrdiff_t)n_x_local_int;
i_x_start_local = (ptrdiff_t)i_x_start_local_int;
n_y_transpose_local = (ptrdiff_t)n_y_transpose_local_int;
i_y_start_transpose_local = (ptrdiff_t)i_y_start_transpose_local_int;
FFT->total_local_size = (size_t)total_local_size_int;
#else
FFT->total_local_size = 1;
for(ptrdiff_t i_d = 0; i_d < FFT->n_d; i_d++) {
if(i_d < FFT->n_d - 1)
FFT->total_local_size *= FFT->n[i_d];
else
FFT->total_local_size *= 2 * (FFT->n[i_d] / 2 + 1);
}
#if USE_DOUBLE
FFT->plan = fftwnd_create_plan(FFT->n_d, n_int, FFTW_REAL_TO_COMPLEX, FFTW_ESTIMATE | FFTW_IN_PLACE);
FFT->iplan = fftwnd_create_plan(FFT->n_d, n_int, FFTW_COMPLEX_TO_REAL, FFTW_ESTIMATE | FFTW_IN_PLACE);
#else
FFT->plan = rfftwnd_create_plan(FFT->n_d, n_int, FFTW_REAL_TO_COMPLEX, FFTW_ESTIMATE | FFTW_IN_PLACE);
FFT->iplan = rfftwnd_create_plan(FFT->n_d, n_int, FFTW_COMPLEX_TO_REAL, FFTW_ESTIMATE | FFTW_IN_PLACE);
#endif
#endif
#else
#if USE_MPI
#if USE_DOUBLE
fftw_mpi_init();
FFT->total_local_size = fftw_mpi_local_size_many_transposed(FFT->n_d,
FFT->n,
1,
FFTW_MPI_DEFAULT_BLOCK,
FFTW_MPI_DEFAULT_BLOCK,
SID_COMM_WORLD->comm,
&(n_x_local),
&(i_x_start_local),
&(n_y_transpose_local),
&(i_y_start_transpose_local));
FFT->plan = fftw_mpi_plan_dft_r2c(FFT->n_d, FFT->n, FFT->field_local, FFT->cfield_local, SID_COMM_WORLD->comm, FFTW_ESTIMATE);
FFT->iplan = fftw_mpi_plan_dft_c2r(FFT->n_d, FFT->n, FFT->cfield_local, FFT->field_local, SID_COMM_WORLD->comm, FFTW_ESTIMATE);
#else
fftwf_mpi_init();
FFT->total_local_size = fftwf_mpi_local_size_many_transposed(FFT->n_d,
FFT->n,
1,
FFTW_MPI_DEFAULT_BLOCK,
FFTW_MPI_DEFAULT_BLOCK,
SID_COMM_WORLD->comm,
&(n_x_local),
&(i_x_start_local),
&(n_y_transpose_local),
&(i_y_start_transpose_local));
FFT->plan = fftwf_mpi_plan_dft_r2c(FFT->n_d, FFT->n, FFT->field_local, FFT->cfield_local, SID_COMM_WORLD->comm, FFTW_ESTIMATE);
FFT->iplan = fftwf_mpi_plan_dft_c2r(FFT->n_d, FFT->n, FFT->cfield_local, FFT->field_local, SID_COMM_WORLD->comm, FFTW_ESTIMATE);
#endif
#else
FFT->total_local_size = 1;
for(ptrdiff_t i_d=0; i_d < FFT->n_d; i_d++) {
if(i_d < FFT->n_d - 1)
FFT->total_local_size *= FFT->n[i_d];
else
FFT->total_local_size *= 2 * (FFT->n[i_d] / 2 + 1);
}
#if USE_DOUBLE
FFT->plan = fftw_plan_dft_r2c(FFT->n_d, FFT->n, FFT->field_local, FFT->cfield_local, FFTW_ESTIMATE);
FFT->iplan = fftw_plan_dft_c2r(FFT->n_d, FFT->n, FFT->cfield_local, FFT->field_local, FFTW_ESTIMATE);
#else
FFT->plan = fftwf_plan_dft_r2c(FFT->n_d, FFT->n, FFT->field_local, FFT->cfield_local, FFTW_ESTIMATE);
FFT->iplan = fftwf_plan_dft_c2r(FFT->n_d, FFT->n, FFT->cfield_local, FFT->field_local, FFTW_ESTIMATE);
#endif
#endif
#endif
SID_free(SID_FARG n_int);
// Set empty slabs to start at 0 to make ignoring them simple.
if(n_x_local == 0)
i_x_start_local = 0;
if(n_y_transpose_local == 0)
i_y_start_transpose_local = 0;
// Modify the local slab dimensions according to what FFTW chose.
FFT->i_R_start_local[0] = i_x_start_local;
FFT->n_R_local[0] = n_x_local;
if(FFT->n_d > 1) {
FFT->i_k_start_local[1] = i_y_start_transpose_local;
FFT->n_k_local[1] = n_y_transpose_local;
}
// Allocate field
#if USE_FFTW3
FFT->field_local = (gbpFFT_real *)fftwf_alloc_real(FFT->total_local_size);
#else
FFT->field_local = (gbpFFT_real *)SID_malloc(sizeof(gbpFFT_real)*FFT->total_local_size);
#endif
FFT->cfield_local = (gbpFFT_complex *)FFT->field_local;
// Upper limits of slab decomposition
for(ptrdiff_t i_d = 0; i_d < FFT->n_d; i_d++) {
FFT->i_R_stop_local[i_d] = FFT->i_R_start_local[i_d] + FFT->n_R_local[i_d] - 1;
FFT->i_k_stop_local[i_d] = FFT->i_k_start_local[i_d] + FFT->n_k_local[i_d] - 1;
}
// FFTW padding sizes
if(FFT->n_d > 1) {
FFT->pad_size_R = 2 * (FFT->n_R_local[FFT->n_d - 1] / 2 + 1) - FFT->n_R_local[FFT->n_d - 1];
FFT->pad_size_k = 0;
} else {
FFT->pad_size_R = 0;
FFT->pad_size_k = 0;
}
// Number of elements (global and local) in the FFT
ptrdiff_t i_d = 0;
for(FFT->n_field = 1, FFT->n_field_R_local = 1, FFT->n_field_k_local = 1; i_d < FFT->n_d; i_d++) {
FFT->n_field *= (size_t)FFT->n[i_d];
FFT->n_field_R_local *= (size_t)FFT->n_R_local[i_d];
FFT->n_field_k_local *= (size_t)FFT->n_k_local[i_d];
}
// Clear the field
clear_field(FFT);
// Initialize the FFT's real-space grid
FFT->R_field = (double **)SID_malloc(sizeof(double *) * FFT->n_d);
FFT->dR = (double *)SID_malloc(sizeof(double *) * FFT->n_d);
for(ptrdiff_t i_d = 0; i_d < FFT->n_d; i_d++) {
FFT->R_field[i_d] = (double *)SID_malloc(sizeof(double) * (FFT->n[i_d] + 1));
FFT->dR[i_d] = FFT->L[i_d] / (double)(FFT->n[i_d]);
for(ptrdiff_t i_i = 0; i_i < FFT->n[i_d]; i_i++)
FFT->R_field[i_d][i_i] = FFT->L[i_d] * ((double)i_i / (double)(FFT->n[i_d]));
FFT->R_field[i_d][FFT->n[i_d]] = FFT->L[i_d];
}
// Initialize the FFT's k-space grid
FFT->k_field = (double **)SID_malloc(sizeof(double *) * FFT->n_d);
FFT->dk = (double *)SID_malloc(sizeof(double *) * FFT->n_d);
FFT->k_Nyquist = (double *)SID_malloc(sizeof(double *) * FFT->n_d);
for(ptrdiff_t i_d = 0; i_d < FFT->n_d; i_d++) {
FFT->k_field[i_d] = (double *)SID_malloc(sizeof(double) * FFT->n[i_d]);
FFT->dk[i_d] = TWO_PI / FFT->L[i_d];
FFT->k_Nyquist[i_d] = TWO_PI * (double)(FFT->n[i_d]) / FFT->L[i_d] / 2.;
for(ptrdiff_t i_i = 0; i_i < FFT->n[i_d]; i_i++) {
if(i_i >= FFT->n[i_d] / 2)
FFT->k_field[i_d][i_i] = TWO_PI * (double)(i_i - FFT->n[i_d]) / FFT->L[i_d];
else
FFT->k_field[i_d][i_i] = TWO_PI * (double)(i_i) / FFT->L[i_d];
}
}
// Flags
FFT->flag_padded = GBP_FALSE;
// Slab info
FFT->slab.n_x_local = FFT->n_R_local[0];
FFT->slab.i_x_start_local = FFT->i_R_start_local[0];
FFT->slab.i_x_stop_local = FFT->i_R_stop_local[0];
FFT->slab.x_min_local = FFT->R_field[0][FFT->i_R_start_local[0]];
if(FFT->slab.n_x_local > 0)
FFT->slab.x_max_local = FFT->R_field[0][FFT->i_R_stop_local[0] + 1];
else
FFT->slab.x_max_local = FFT->slab.x_min_local;
SID_Allreduce(&(FFT->slab.x_max_local), &(FFT->slab.x_max), 1, SID_DOUBLE, SID_MAX, SID_COMM_WORLD);
#if USE_MPI
// All ranks are not necessarily assigned any slices, so
// we need to figure out what ranks are to the right and the left for
// buffer exchanges
n_x_rank = (ptrdiff_t *)SID_malloc(sizeof(ptrdiff_t) * SID.n_proc);
n_x_rank[SID.My_rank] = (ptrdiff_t)FFT->slab.n_x_local;
if(n_x_rank[SID.My_rank] > 0)
flag_active = GBP_TRUE;
else
flag_active = GBP_FALSE;
SID_Allreduce(&flag_active, &n_active, 1, SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(&n_x_rank[SID.My_rank], &min_size, 1, SID_INT, SID_MIN, SID_COMM_WORLD);
SID_Allreduce(&n_x_rank[SID.My_rank], &max_size, 1, SID_INT, SID_MAX, SID_COMM_WORLD);
for(int i_rank = 0; i_rank < SID.n_proc; i_rank++)
SID_Bcast(&(n_x_rank[i_rank]), 1, SID_INT, i_rank, SID_COMM_WORLD);
FFT->slab.rank_to_right = -1;
for(int i_rank = SID.My_rank + 1; i_rank < SID.My_rank + SID.n_proc && FFT->slab.rank_to_right < 0; i_rank++) {
int j_rank = i_rank % SID.n_proc;
if(n_x_rank[j_rank] > 0)
FFT->slab.rank_to_right = j_rank;
}
if(FFT->slab.rank_to_right < 0)
FFT->slab.rank_to_right = SID.My_rank;
FFT->slab.rank_to_left = -1;
for(int i_rank = SID.My_rank - 1; i_rank > SID.My_rank - SID.n_proc && FFT->slab.rank_to_left < 0; i_rank--) {
int j_rank = i_rank;
if(i_rank < 0)
j_rank = i_rank + SID.n_proc;
if(n_x_rank[j_rank] > 0)
FFT->slab.rank_to_left = j_rank;
}
if(FFT->slab.rank_to_left < 0)
FFT->slab.rank_to_left = SID.My_rank;
free(n_x_rank);
SID_log("(%d cores unused, min/max slab size=%d/%d)...", SID_LOG_CONTINUE, SID.n_proc - n_active, min_size, max_size);
#else
FFT->slab.rank_to_right = SID.My_rank;
FFT->slab.rank_to_left = SID.My_rank;
if(FFT->slab.n_x_local > 0) {
flag_active = GBP_TRUE;
n_active = 1;
min_size = FFT->slab.n_x_local;
max_size = FFT->slab.n_x_local;
} else {
flag_active = GBP_FALSE;
n_active = 0;
min_size = 0;
max_size = 0;
}
#endif
SID_log("Done.", SID_LOG_CLOSE);
}
void read_mark_file(plist_info *plist,
const char *mark_name,
const char *filename_in,
int mode){
int i_species;
size_t i_particle;
size_t j_particle;
size_t k_particle;
int i_rank;
size_t i_mark;
size_t n_particles_local;
size_t *mark_list_buffer;
int *mark_list;
size_t *ids_local;
size_t *mark_list_local;
size_t n_mark_total;
size_t n_mark_total_check;
size_t n_mark_type_local[N_GADGET_TYPE];
size_t n_mark_local;
size_t n_particle_local;
SID_fp fp_mark_file;
size_t i_start_local[N_GADGET_TYPE];
size_t n_mark_bcast;
size_t *ids_local_index;
size_t n_buffer;
int flag_allocate;
int flag_read_mode;
int flag_mark_mode;
int flag_op_mode;
markfile_header_info header={N_GADGET_TYPE};
SID_log("Reading mark file...",SID_LOG_OPEN);
// Interpret run mode
if(check_mode_for_flag(mode,MARK_READ_ALL))
flag_read_mode=MARK_READ_ALL;
else
flag_read_mode=MARK_DEFAULT;
if(check_mode_for_flag(mode,MARK_LIST_ONLY))
flag_mark_mode=MARK_LIST_ONLY;
else
flag_mark_mode=MARK_DEFAULT;
if(check_mode_for_flag(mode,MARK_INIT) || check_mode_for_flag(mode,MARK_OR))
flag_op_mode=MARK_DEFAULT;
else
flag_op_mode=MARK_AND;
// Open mark list and read header
SID_fopen_chunked(filename_in,
"r",
&fp_mark_file,
&header);
if(header.n_type!=N_GADGET_TYPE)
SID_trap_error("Inconsistant number of species in mark file (ie. %d!=%d)!",ERROR_LOGIC,header.n_type,N_GADGET_TYPE);
// List numbers of particles in the log output
size_t n_particles_all;
int n_non_zero;
for(i_species=0,n_particles_all=0,n_non_zero=0;i_species<header.n_type;i_species++){
if(header.n_mark_species[i_species]>0){
n_particles_all+=header.n_mark_species[i_species];
n_non_zero++;
}
}
SID_log("%lld",SID_LOG_CONTINUE,n_particles_all);
if(n_non_zero>0)
SID_log(" (",SID_LOG_CONTINUE,n_particles_all);
for(i_species=0;i_species<N_GADGET_TYPE;i_species++){
if(header.n_mark_species[i_species]>0){
if(i_species==n_non_zero-1){
if(n_non_zero>1)
SID_log("and %lld %s",SID_LOG_CONTINUE,header.n_mark_species[i_species],plist->species[i_species]);
else
SID_log("%lld %s",SID_LOG_CONTINUE,header.n_mark_species[i_species],plist->species[i_species]);
}
else{
if(n_non_zero>1)
SID_log("%lld %s, ",SID_LOG_CONTINUE,header.n_mark_species[i_species],plist->species[i_species]);
else
SID_log("%lld %s",SID_LOG_CONTINUE,header.n_mark_species[i_species],plist->species[i_species]);
}
}
}
if(n_non_zero>0)
SID_log(") particles...",SID_LOG_CONTINUE);
else
SID_log(" particles...",SID_LOG_CONTINUE);
// Set list sizes and prep offsets for reading
for(i_species=0,n_mark_local=0,n_mark_total_check=0;i_species<header.n_type;i_species++){
if(header.n_mark_species[i_species]>0){
ADaPS_store(&(plist->data),(void *)(&(header.n_mark_species[i_species])),"n_%s_%s",ADaPS_SCALAR_SIZE_T,mark_name,plist->species[i_species]);
switch(flag_read_mode){
case MARK_READ_ALL:
n_mark_type_local[i_species]=header.n_mark_species[i_species];
i_start_local[i_species] =0;
break;
default:
n_mark_type_local[i_species]=header.n_mark_species[i_species]/SID.n_proc;
i_start_local[i_species] =(SID.My_rank)*n_mark_type_local[i_species];
if(SID.I_am_last_rank)
n_mark_type_local[i_species]=header.n_mark_species[i_species]-i_start_local[i_species];
break;
}
ADaPS_store(&(plist->data),(void *)(&(n_mark_type_local[i_species])),"n_local_%s_%s",ADaPS_SCALAR_SIZE_T,mark_name,plist->species[i_species]);
n_mark_local +=n_mark_type_local[i_species];
n_mark_total_check+=header.n_mark_species[i_species];
}
}
// Sanity check
SID_Allreduce(&n_mark_local,&n_mark_total,1,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
if(n_mark_total!=n_mark_total_check)
SID_trap_error("Particle numbers don't add-up right in read_mark_file!",ERROR_LOGIC);
// Read file and create/store mark arrays
switch(flag_mark_mode){
case MARK_LIST_ONLY:
for(i_species=0;i_species<header.n_type;i_species++){
if(header.n_mark_species[i_species]>0){
// Allocate array
if(n_mark_type_local[i_species]>0)
mark_list_local=(size_t *)SID_malloc(sizeof(size_t)*n_mark_type_local[i_species]);
else
mark_list_local=NULL;
// Perform read
SID_fread_chunked(mark_list_local,
n_mark_type_local[i_species],
i_start_local[i_species],
&fp_mark_file);
// Sort marked particles
if(n_mark_type_local[i_species]>0){
merge_sort(mark_list_local,n_mark_type_local[i_species],NULL,SID_SIZE_T,SORT_INPLACE_ONLY,SORT_COMPUTE_INPLACE);
ADaPS_store(&(plist->data),(void *)(mark_list_local),"%s_%s",ADaPS_DEFAULT,mark_name,plist->species[i_species]);
}
}
}
break;
default:
mark_list_buffer=(size_t *)SID_malloc(sizeof(size_t)*MAX_MARK_BUFFER_SIZE);
for(i_species=0;i_species<header.n_type;i_species++){
if(header.n_mark_species[i_species]>0){
n_particles_local=((size_t *)ADaPS_fetch(plist->data,"n_%s",plist->species[i_species]))[0];
// Initialize arrays
ids_local=(size_t *)ADaPS_fetch(plist->data,"id_%s",plist->species[i_species]);
if(ADaPS_exist(plist->data,"%s_%s",mark_name,plist->species[i_species])){
mark_list=(int *)ADaPS_fetch(plist->data,"%s_%s",mark_name,plist->species[i_species]);
flag_allocate=FALSE;
}
else{
mark_list=(int *)SID_malloc(sizeof(int)*n_particles_local);
for(i_particle=0;i_particle<n_particles_local;i_particle++)
mark_list[i_particle]=FALSE;
flag_allocate=TRUE;
}
merge_sort(ids_local,n_particles_local,&ids_local_index,SID_SIZE_T,SORT_COMPUTE_INDEX,SORT_COMPUTE_NOT_INPLACE);
// Use a buffer to increase speed
for(i_particle=0;i_particle<header.n_mark_species[i_species];){
n_buffer=MIN(header.n_mark_species[i_species]-i_particle,MAX_MARK_BUFFER_SIZE);
SID_fread_chunked_all(mark_list_local,
n_buffer,
&fp_mark_file);
merge_sort(mark_list_local,n_buffer,NULL,SID_SIZE_T,SORT_INPLACE_ONLY,SORT_COMPUTE_INPLACE);
for(j_particle=0,k_particle=find_index(ids_local,mark_list_buffer[0],n_particles_local,ids_local_index);
j_particle<n_buffer;
j_particle++,i_particle++){
while(ids_local[ids_local_index[k_particle]]<mark_list_local[j_particle] && k_particle<n_buffer-1) k_particle++;
if(ids_local[ids_local_index[k_particle]]==mark_list_local[j_particle]){
switch(flag_op_mode){
case MARK_INIT:
case MARK_AND:
case MARK_OR:
mark_list[i_particle]=TRUE;
break;
}
}
}
}
SID_free((void **)&ids_local_index);
ADaPS_store(&(plist->data),(void *)mark_list,"%s_%s",ADaPS_DEFAULT,mark_name,plist->species[i_species]);
}
}
SID_free((void **)&mark_list_buffer);
break;
}
SID_fclose_chunked(&fp_mark_file);
SID_log("Done.",SID_LOG_CLOSE);
}
void SID_exit(int status){
int i_args=0;
int n_days;
int n_hrs;
int n_mins;
int n_secs;
size_t max_RAM;
size_t SID_max_RAM_local;
int i_rank;
char spacer[10];
char time_string[48];
// Deal with I/O channels
fflush(SID.fp_log);
SID_Barrier(SID.COMM_WORLD);
if(SID.fp_in!=stdin && SID.fp_in!=NULL)
fclose(SID.fp_in);
// Clean-up argument stuff
if(SID.args!=NULL){
while((SID_arg *)(SID.args[i_args])!=NULL){
if(SID.arg_alloc[i_args])
free(((SID_arg *)(SID.args[i_args]))->val);
i_args++;
}
if(i_args>0){
free(SID.arg_alloc);
free(SID.arg_set);
}
}
// Clean-up any memory still allocated
// Report execution statistics
if(status!=ERROR_SYNTAX){
if(SID.I_am_Master){
fprintf(SID.fp_log,"\n");
fprintf(SID.fp_log,"Run statistics:\n");
fprintf(SID.fp_log,"--------------\n");
(void)time(&(SID.time_stop));
n_secs =(int)(SID.time_stop-SID.time_start);
seconds2ascii(n_secs,time_string);
fprintf(SID.fp_log,"Time elapsed=%s.\n",time_string);
}
// Report memory usage
if(SID.I_am_Master){
fprintf(SID.fp_log,"\nMemory usage:\n");
fprintf(SID.fp_log,"------------\n");
}
#if USE_MPI
SID_Barrier(SID.COMM_WORLD);
SID_Allreduce(&(SID.max_RAM_local),&max_RAM,1,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
if(SID.n_proc>1){
for(i_rank=0;i_rank<SID.n_proc;i_rank++){
SID_max_RAM_local=SID.max_RAM_local;
if(i_rank!=MASTER_RANK)
SID_Bcast(&SID_max_RAM_local,sizeof(size_t),i_rank,SID.COMM_WORLD);
if(SID.I_am_Master){
if(SID.n_proc>1000){
if((float)SID.max_RAM_local/(float)SIZE_OF_MEGABYTE>1.)
fprintf(SID.fp_log,"Peak for rank %4d=%4.2lf Gb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_GIGIBYTE);
else if((float)SID.max_RAM_local/(float)SIZE_OF_KILOBYTE>1.)
fprintf(SID.fp_log,"Peak for rank %4d=%4.2lf Mb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_MEGABYTE);
else
fprintf(SID.fp_log,"Peak for rank %4d=%4.2lf kb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_KILOBYTE);
strcpy(spacer," ");
}
else if(SID.n_proc>100){
if((float)SID.max_RAM_local/(float)SIZE_OF_MEGABYTE>1.)
fprintf(SID.fp_log,"Peak for rank %3d=%4.2lf Gb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_GIGIBYTE);
else if((float)SID.max_RAM_local/(float)SIZE_OF_KILOBYTE>1.)
fprintf(SID.fp_log,"Peak for rank %3d=%4.2lf Mb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_MEGABYTE);
else
fprintf(SID.fp_log,"Peak for rank %3d=%4.2lf kb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_KILOBYTE);
strcpy(spacer," ");
}
else if(SID.n_proc>10){
if((float)SID.max_RAM_local/(float)SIZE_OF_MEGABYTE>1.)
fprintf(SID.fp_log,"Peak for rank %2d=%4.2lf Gb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_GIGIBYTE);
else if((float)SID.max_RAM_local/(float)SIZE_OF_KILOBYTE>1.)
fprintf(SID.fp_log,"Peak for rank %2d=%4.2lf Mb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_MEGABYTE);
else
fprintf(SID.fp_log,"Peak for rank %2d=%4.2lf kb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_KILOBYTE);
strcpy(spacer," ");
}
else{
if((float)SID.max_RAM_local/(float)SIZE_OF_MEGABYTE>1.)
fprintf(SID.fp_log,"Peak for rank %d=%4.2lf Gb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_GIGIBYTE);
else if((float)SID.max_RAM_local/(float)SIZE_OF_KILOBYTE>1.)
fprintf(SID.fp_log,"Peak for rank %d=%4.2lf Mb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_MEGABYTE);
else
fprintf(SID.fp_log,"Peak for rank %d=%4.2lf kb\n",i_rank,(float)SID_max_RAM_local/(float)SIZE_OF_KILOBYTE);
strcpy(spacer," ");
}
}
SID_Barrier(SID.COMM_WORLD);
}
}
#else
strcpy(spacer,"\0");
max_RAM=SID.max_RAM_local;
#endif
if(SID.I_am_Master){
if((float)max_RAM/(float)SIZE_OF_MEGABYTE>1.)
fprintf(SID.fp_log,"Peak total %s=%4.2lf Gb\n",spacer,(float)max_RAM/(float)SIZE_OF_GIGIBYTE);
else if((float)max_RAM/(float)SIZE_OF_KILOBYTE>1.)
fprintf(SID.fp_log,"Peak total %s=%4.2lf Mb\n",spacer,(float)max_RAM/(float)SIZE_OF_MEGABYTE);
else
fprintf(SID.fp_log,"Peak total %s=%4.2lf kb\n",spacer,(float)max_RAM/(float)SIZE_OF_KILOBYTE);
}
}
// Free some arrays
SID_free(SID_FARG SID.time_start_level);
SID_free(SID_FARG SID.time_stop_level);
SID_free(SID_FARG SID.time_total_level);
SID_free(SID_FARG SID.flag_use_timer);
SID_free(SID_FARG SID.IO_size);
SID_free(SID_FARG SID.My_node);
// Finalize MPI
SID_Comm_free(&(SID.COMM_WORLD));
#if USE_MPI_IO
MPI_Info_free(&(SID.file_info));
#endif
#if USE_MPI
MPI_Finalize();
#endif
exit(status);
}
int main(int argc, char *argv[]){
plist_info plist;
char filename_in_root[256];
char filename_out_root[256];
char filename_in[256];
char filename_out[256];
int subsample_factor;
int i_snap,i_file,j_file;
int n_files;
unsigned int n_local[N_GADGET_TYPE];
int i_type;
size_t i_particle,j_particle;
FILE *fp;
FILE *fp_out;
int record_length_in;
int record_length_out;
GBPREAL position[3];
GBPREAL velocity[3];
int ID_int;
RNG_info RNG;
int seed_init=10463743;
int seed;
GBPREAL RNG_max;
long long n_all_keep[N_GADGET_TYPE];
long long n_all_init[N_GADGET_TYPE];
unsigned int n_all_high_word[N_GADGET_TYPE];
unsigned int n_all[N_GADGET_TYPE];
double mass_array[N_GADGET_TYPE];
int n_keep_total;
int n_init_total;
int record_length;
int flag_long_ids;
int id_in_int;
long long id_in_long;
int n_groups;
SID_init(&argc,&argv,NULL,NULL);
// Parse command line
if(argc!=4){
fprintf(stderr,"\n syntax: %s filename_in_root snapshot_number subsample_factor\n",argv[0]);
fprintf(stderr," ------\n\n");
return(ERROR_SYNTAX);
}
else{
strcpy(filename_in_root, argv[1]);
i_snap =atoi(argv[2]);
subsample_factor=atoi(argv[3]);
}
sprintf(filename_out_root,"%s_subsample",filename_in_root);
RNG_max=1./(GBPREAL)(subsample_factor);
SID_log("Subsampling GADGET file by a factor of %d...",SID_LOG_OPEN,subsample_factor);
// Grab info from first header
// Read the header and determine the input file-format
gadget_read_info fp_gadget;
int flag_filefound=init_gadget_read(filename_in_root,i_snap,&fp_gadget);
int flag_multifile=fp_gadget.flag_multifile;
int flag_file_type=fp_gadget.flag_file_type;
gadget_header_info header =fp_gadget.header;
if(!flag_filefound)
SID_trap_error("File not found.",ERROR_LOGIC);
if(flag_filefound)
n_files =header.n_files;
else
n_files=0;
if(n_files>0){
init_seed_from_clock(&seed_init);
SID_log("Seed=%d",SID_LOG_COMMENT,seed_init);
SID_log("Counting particles in %d files...",SID_LOG_OPEN|SID_LOG_TIMER,n_files);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++)
n_all_keep[i_type]=0;
for(i_file=SID.My_rank,j_file=0;j_file<n_files;i_file+=SID.n_proc,j_file+=SID.n_proc){
if(n_files>1)
SID_log("Processing file(s) %d->%d...",SID_LOG_OPEN,j_file,MIN(j_file+SID.n_proc-1,n_files-1));
if(i_file<n_files){
set_gadget_filename(&fp_gadget,i_file,filename_in);
fp=fopen(filename_in,"r");
fread_verify(&record_length_in,sizeof(int),1,fp);
fread_verify(&header,sizeof(gadget_header_info),1,fp);
fread_verify(&record_length_out,sizeof(int),1,fp);
seed=seed_init+1387*i_file;
init_RNG(&seed,&RNG,RNG_DEFAULT);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<header.n_file[i_type];j_particle++){
if(random_number(&RNG)<=RNG_max)
n_all_keep[i_type]++;
}
}
free_RNG(&RNG);
fclose(fp);
}
if(n_files>1)
SID_log("Done.",SID_LOG_CLOSE);
}
SID_Allreduce(SID_IN_PLACE,n_all_keep,N_GADGET_TYPE,SID_LONG_LONG,SID_SUM,SID.COMM_WORLD);
SID_log("Done.",SID_LOG_CLOSE);
SID_log("Header properties will be...",SID_LOG_OPEN);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
n_all_high_word[i_type]=(unsigned int)(((long long)n_all_keep[i_type])>>32);
n_all[i_type] =(unsigned int)(((long long)n_all_keep[i_type])-(((long long)(n_all_high_word[i_type]))<<32));
n_all_init[i_type] =((long long)(header.n_all_lo_word[i_type]))+(((long long)(header.n_all_hi_word[i_type]))<<32);
if(n_all_keep[i_type]>0)
mass_array[i_type]=header.mass_array[i_type]*((double)n_all_init[i_type]/(double)n_all_keep[i_type]);
else
mass_array[i_type]=0.;
}
for(i_type=0;i_type<N_GADGET_TYPE;i_type++)
SID_log("mass_array[%d]=%le (was %le)", SID_LOG_COMMENT,i_type,mass_array[i_type],header.mass_array[i_type]);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++)
SID_log("n_all[%d] =%d (was %d)",SID_LOG_COMMENT,i_type,n_all[i_type],header.n_all_lo_word[i_type]);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++)
SID_log("high_word[%d] =%d (was %d)",SID_LOG_COMMENT,i_type,n_all_high_word[i_type],header.n_all_hi_word[i_type]);
SID_log("Done.",SID_LOG_CLOSE);
SID_log("Writing subsampled snapshot...",SID_LOG_OPEN|SID_LOG_TIMER);
for(i_file=SID.My_rank,j_file=0;j_file<n_files;i_file+=SID.n_proc,j_file+=SID.n_proc){
if(n_files>1)
SID_log("Processing file(s) %d->%d...",SID_LOG_OPEN,j_file,MIN(j_file+SID.n_proc-1,n_files-1));
if(i_file<n_files){
set_gadget_filename(&fp_gadget,i_file,filename_in);
change_gadget_filename(filename_in_root,"snapshot_subsample",i_snap,i_file,flag_multifile,flag_file_type,filename_out);
if(i_file==0){
FILE *fp_test;
char filename_out_directory[MAX_FILENAME_LENGTH];
strcpy(filename_out_directory,filename_out);
strip_file_root(filename_out_directory);
if((fp_test=fopen(filename_out_directory,"r"))==NULL)
mkdir(filename_out_directory,02755);
else
fclose(fp_test);
}
fp =fopen(filename_in, "r");
fp_out=fopen(filename_out,"w");
// Header
fread_verify(&record_length_in,sizeof(int),1,fp);
fread_verify(&header,sizeof(gadget_header_info),1,fp);
fread_verify(&record_length_out,sizeof(int),1,fp);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
n_local[i_type] =header.n_file[i_type];
header.n_file[i_type] =0;
header.mass_array[i_type] =mass_array[i_type];
header.n_all_lo_word[i_type] =n_all[i_type];
header.n_all_hi_word[i_type] =n_all_high_word[i_type];
}
seed=seed_init+1387*i_file;
init_RNG(&seed,&RNG,RNG_DEFAULT);
for(i_type=0,n_keep_total=0,n_init_total=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<n_local[i_type];j_particle++,i_particle++){
if(random_number(&RNG)<=RNG_max){
header.n_file[i_type]++;
n_keep_total++;
}
n_init_total++;
}
}
free_RNG(&RNG);
record_length=record_length_in;
fwrite(&record_length,sizeof(int),1,fp_out);
fwrite(&header,sizeof(gadget_header_info),1,fp_out);
fwrite(&record_length,sizeof(int),1,fp_out);
// Positions
record_length=3*sizeof(GBPREAL)*n_keep_total;
seed=seed_init+1387*i_file;
init_RNG(&seed,&RNG,RNG_DEFAULT);
fread_verify(&record_length_in,sizeof(int),1,fp);
fwrite(&record_length,sizeof(int),1,fp_out);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<n_local[i_type];j_particle++){
fread_verify(position,sizeof(GBPREAL),3,fp);
if(random_number(&RNG)<=RNG_max)
fwrite(position,sizeof(GBPREAL),3,fp_out);
}
}
fread_verify(&record_length_out,sizeof(int),1,fp);
fwrite(&record_length,sizeof(int),1,fp_out);
free_RNG(&RNG);
// Velocities
record_length=3*sizeof(GBPREAL)*n_keep_total;
seed=seed_init+1387*i_file;
init_RNG(&seed,&RNG,RNG_DEFAULT);
fread_verify(&record_length_in,sizeof(int),1,fp);
fwrite(&record_length,sizeof(int),1,fp_out);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<n_local[i_type];j_particle++){
fread_verify(velocity,sizeof(GBPREAL),3,fp);
if(random_number(&RNG)<=RNG_max)
fwrite(velocity,sizeof(GBPREAL),3,fp_out);
}
}
fread_verify(&record_length_out,sizeof(int),1,fp);
fwrite(&record_length,sizeof(int),1,fp_out);
free_RNG(&RNG);
// IDs
seed=seed_init+1387*i_file;
init_RNG(&seed,&RNG,RNG_DEFAULT);
fread_verify(&record_length_in,sizeof(int),1,fp);
if(record_length_in/sizeof(int)==n_init_total){
flag_long_ids=FALSE;
record_length=n_keep_total*sizeof(int);
}
else{
flag_long_ids=TRUE;
record_length=n_keep_total*sizeof(long long);
}
fwrite(&record_length,sizeof(int),1,fp_out);
if(flag_long_ids){
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<n_local[i_type];j_particle++){
fread_verify(&id_in_long,sizeof(long long),1,fp);
if(random_number(&RNG)<=RNG_max)
fwrite(&id_in_long,sizeof(long long),1,fp_out);
}
}
}
else{
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(j_particle=0;j_particle<n_local[i_type];j_particle++){
fread_verify(&id_in_int,sizeof(int),1,fp);
if(random_number(&RNG)<=RNG_max)
fwrite(&id_in_int,sizeof(int),1,fp_out);
}
}
}
fread_verify(&record_length_out,sizeof(int),1,fp);
fwrite(&record_length,sizeof(int),1,fp_out);
free_RNG(&RNG);
fclose(fp);
fclose(fp_out);
}
if(n_files>1)
SID_log("Done.",SID_LOG_CLOSE);
}
SID_log("Done.",SID_LOG_CLOSE);
}
void compute_treenode_list_marker_stats(tree_info *trees,treenode_list_info *list,tree_markers_info **markers_all,tree_markers_stats_info *stats,int **n_hist_count,int *n_hist){
int n_stats=3;
int n_M =2;
(*n_hist) =n_stats+n_M;
// Allocate histogram arrays -- stats
int i_hist=0;
int **hist =(int **)SID_calloc(sizeof(int *)*(*n_hist));
int *n_bins=(int *)SID_calloc(sizeof(int)*(*n_hist));
for(int i_stat=0;i_stat<n_stats;i_stat++){
n_bins[i_hist]=trees->n_snaps;
hist[i_hist++]=(int *)SID_calloc(sizeof(int)*n_bins[i_hist]);
}
// Allocate histogram arrays -- mass
double logM_min=7.;
int n_M_bins=90;
double dlogM =0.1;
for(int i_M=0;i_M<n_M;i_M++){
n_bins[i_hist]=n_M_bins;
hist[i_hist++]=(int *)SID_calloc(sizeof(int)*n_bins[i_hist]);
}
// Allocate histogram counts
if((*n_hist_count)==NULL)
(*n_hist_count)=(int *)SID_calloc(sizeof(int)*(*n_hist));
else{
for(i_hist=0;i_hist<(*n_hist);i_hist++)
(*n_hist_count)[i_hist]=0;
}
// Work with a precompiled markers array if it's been given
tree_markers_info *markers;
if(markers_all==NULL)
markers=(tree_markers_info *)SID_malloc(sizeof(tree_markers_info));
// Create histograms
for(int i_list=0;i_list<list->n_list_local;i_list++){
markers=fetch_treenode_precomputed_markers(trees,list->list[i_list]);
// Build histogram
int i_bin;
for(int i_hist=0;i_hist<(*n_hist);i_hist++){
i_bin=-1;
if(i_hist<n_stats){
tree_node_info *marker=NULL;
if(i_hist==0)
marker=markers->half_peak_mass;
else if(i_hist==1)
marker=markers->merger_33pc_remnant;
else if(i_hist==2)
marker=markers->merger_10pc_remnant;
else
SID_trap_error("Behaviour undefined in compute_treenode_list_marker_stats()",ERROR_LOGIC);
if(marker!=NULL)
i_bin=marker->snap_tree;
else
i_bin=-1;
}
else{
if(i_hist==(n_stats+0)){
halo_properties_info *properties=fetch_treenode_properties(trees,list->list[i_list]);
i_bin=(take_log10(properties->M_vir)-logM_min)/dlogM;
}
else if(i_hist==(n_stats+1))
i_bin=(take_log10(markers->M_peak)-logM_min)/dlogM;
else
SID_trap_error("Behaviour undefined in compute_treenode_list_marker_stats()",ERROR_LOGIC);
}
if(i_bin>=0 && i_bin<n_bins[i_hist]){
hist[i_hist][i_bin]++;
(*n_hist_count)[i_hist]++;
}
}
}
if(markers_all==NULL)
SID_free(SID_FARG markers);
for(int i_hist=0;i_hist<(*n_hist);i_hist++){
SID_Allreduce(SID_IN_PLACE,hist[i_hist],n_bins[i_hist],SID_INT,SID_SUM,SID.COMM_WORLD);
SID_Allreduce(SID_IN_PLACE,&((*n_hist_count)[i_hist]),1,SID_INT,SID_SUM,SID.COMM_WORLD);
}
// Find 68 and 95 percent confidence ranges
for(int i_hist=0;i_hist<(*n_hist);i_hist++){
int sum=0;
for(int i_bin=0;i_bin<n_bins[i_hist];i_bin++)
sum+=hist[i_hist][i_bin];
size_t *hist_index=NULL;
merge_sort(hist[i_hist],(size_t)(n_bins[i_hist]),&hist_index,SID_INT,SORT_COMPUTE_INDEX,FALSE);
int i_peak =hist_index[n_bins[i_hist]-1];
int i_68_lo=hist_index[n_bins[i_hist]-1];
int i_68_hi=hist_index[n_bins[i_hist]-1];
int target =(int)(0.68*(double)sum);
int accum =0;
int i_bin =0;
while(accum<=target && i_bin<n_bins[i_hist]){
size_t idx_i=hist_index[n_bins[i_hist]-i_bin-1];
if(idx_i<i_68_lo) i_68_lo=idx_i;
if(idx_i>i_68_hi) i_68_hi=idx_i;
accum+=hist[i_hist][idx_i];
i_bin++;
}
int i_95_lo=i_68_lo;
int i_95_hi=i_68_hi;
target=(int)(0.95*(double)sum);
while(accum<=target && i_bin<n_bins[i_hist]){
size_t idx_i=hist_index[n_bins[i_hist]-i_bin-1];
if(idx_i<i_95_lo) i_95_lo=idx_i;
if(idx_i>i_95_hi) i_95_hi=idx_i;
accum+=hist[i_hist][idx_i];
i_bin++;
}
SID_free(SID_FARG hist_index);
if(i_hist==0){
stats->t_half_peak_mass_ranges[0][0]=trees->t_list[i_68_lo];
stats->t_half_peak_mass_ranges[0][1]=trees->t_list[i_68_hi];
stats->t_half_peak_mass_ranges[1][0]=trees->t_list[i_95_lo];
stats->t_half_peak_mass_ranges[1][1]=trees->t_list[i_95_hi];
stats->t_half_peak_mass_peak =trees->t_list[i_peak];
}
else if(i_hist==1){
stats->t_merger_33pc_ranges[0][0]=trees->t_list[i_68_lo];
stats->t_merger_33pc_ranges[0][1]=trees->t_list[i_68_hi];
stats->t_merger_33pc_ranges[1][0]=trees->t_list[i_95_lo];
stats->t_merger_33pc_ranges[1][1]=trees->t_list[i_95_hi];
stats->t_merger_33pc_peak =trees->t_list[i_peak];
}
else if(i_hist==2){
stats->t_merger_10pc_ranges[0][0]=trees->t_list[i_68_lo];
stats->t_merger_10pc_ranges[0][1]=trees->t_list[i_68_hi];
stats->t_merger_10pc_ranges[1][0]=trees->t_list[i_95_lo];
stats->t_merger_10pc_ranges[1][1]=trees->t_list[i_95_hi];
stats->t_merger_10pc_peak =trees->t_list[i_peak];
}
else if(i_hist==3){
stats->M_peak_ranges[0][0]=logM_min+(double)(i_68_lo)*dlogM;
stats->M_peak_ranges[0][1]=logM_min+(double)(i_68_hi)*dlogM;
stats->M_peak_ranges[1][0]=logM_min+(double)(i_95_lo)*dlogM;
stats->M_peak_ranges[1][1]=logM_min+(double)(i_95_hi)*dlogM;
stats->M_peak_peak =logM_min+(double)( i_peak)*dlogM;
}
else if(i_hist==4){
stats->M_vir_ranges[0][0]=logM_min+(double)(i_68_lo)*dlogM;
stats->M_vir_ranges[0][1]=logM_min+(double)(i_68_hi)*dlogM;
stats->M_vir_ranges[1][0]=logM_min+(double)(i_95_lo)*dlogM;
stats->M_vir_ranges[1][1]=logM_min+(double)(i_95_hi)*dlogM;
stats->M_vir_peak =logM_min+(double)( i_peak)*dlogM;
}
else
SID_trap_error("Behaviour undefined in compute_treenode_list_marker_stats()",ERROR_LOGIC);
}
// Clean-up
for(int i_hist=0;i_hist<(*n_hist);i_hist++)
SID_free(SID_FARG hist[i_hist]);
SID_free(SID_FARG hist);
SID_free(SID_FARG n_bins);
}
void write_match_results(char * filename_out_dir,
char * filename_out_root,
int i_read,
int j_read,
const char *filename_cat1,
const char *filename_cat2,
plist_info *plist1,
plist_info *plist2,
int k_match,
float match_weight_rank_index,
int mode) {
char filename_out[256];
char filename_out_dir_snap[256];
FILE * fp_out;
int k_read, l_read;
int flag_go;
int i_read_start_file;
int i_read_stop_file;
int i_read_step_file;
int n_search_file;
int n_search_total;
int n_k_match;
int flag_match_subgroups;
char group_text_prefix[5];
int n_matches;
int n_files;
int n_groups_1;
int n_groups_1_local;
int n_groups_2;
int n_groups_2_local;
int i_group;
int buffered_count;
int buffered_count_local;
int j_group;
int index_test;
int i_rank;
int * n_particles;
int * n_sub_group;
int * file_index_1;
int * match_id = NULL;
float * match_score = NULL;
int * match_count = NULL;
char cat_name_1[20];
char cat_name_2[20];
size_t *match_rank = NULL;
size_t *match_index = NULL;
size_t offset;
int * n_return;
void * buffer;
int * buffer_int;
size_t *buffer_size_t;
float * buffer_float;
int n_buffer_max = 131072; // 32*32k=1MB for 8-byte values
int n_buffer;
int i_buffer;
int j_buffer;
switch(k_match) {
case 0:
flag_match_subgroups = MATCH_SUBGROUPS;
sprintf(group_text_prefix, "sub");
break;
case 1:
flag_match_subgroups = MATCH_GROUPS;
sprintf(group_text_prefix, "");
break;
}
// Intialize filenames
if(SID_CHECK_BITFIELD_SWITCH(mode, WRITE_MATCHES_MODE_TREES)) {
sprintf(filename_out_dir_snap, "%s/%s", filename_out_dir, filename_cat1);
// Create output directory if need-be
if(filename_out_dir != NULL)
mkdir(filename_out_dir, 02755);
} else if(SID_CHECK_BITFIELD_SWITCH(mode, WRITE_MATCHES_MODE_SINGLE))
sprintf(filename_out_dir_snap, "%s/", filename_out_dir);
else
SID_exit_error("Invalid write mode flag (%d).", SID_ERROR_LOGIC, mode);
if(filename_out_dir != NULL)
sprintf(filename_out, "%s/%sgroup_matches_%s_%s.dat", filename_out_dir_snap, group_text_prefix, filename_cat1, filename_cat2);
else
sprintf(filename_out, "%s_%sgroup_matches_%s_%s.dat", filename_out_root, group_text_prefix, filename_cat1, filename_cat2);
SID_log("Writing match results to {%s}...", SID_LOG_OPEN | SID_LOG_TIMER, filename_out);
// Fetch halo counts ...
n_groups_1 = ((int *)ADaPS_fetch(plist1->data, "n_%sgroups_all_%s", group_text_prefix, filename_cat1))[0];
n_groups_2 = ((int *)ADaPS_fetch(plist2->data, "n_%sgroups_all_%s", group_text_prefix, filename_cat2))[0];
// Write header.
if(SID.I_am_Master) {
if(filename_out_dir != NULL)
mkdir(filename_out_dir_snap, 02755);
if((fp_out = fopen(filename_out, "w")) == NULL)
SID_exit_error("Could not open {%s} for writing.", SID_ERROR_IO_OPEN, filename_out);
fwrite(&i_read, sizeof(int), 1, fp_out);
fwrite(&j_read, sizeof(int), 1, fp_out);
fwrite(&n_groups_1, sizeof(int), 1, fp_out);
fwrite(&n_groups_2, sizeof(int), 1, fp_out);
fwrite(&match_weight_rank_index, sizeof(float), 1, fp_out);
}
// Everything else only needs to be written if there are halos to match with
if(n_groups_1 > 0) {
// Fetch catalog and matching info ...
n_groups_1_local = ((int *)ADaPS_fetch(plist1->data, "n_%sgroups_%s", group_text_prefix, filename_cat1))[0];
file_index_1 = (int *)ADaPS_fetch(plist1->data, "file_index_%sgroups_%s", group_text_prefix, filename_cat1);
n_groups_2_local = ((int *)ADaPS_fetch(plist2->data, "n_%sgroups_%s", group_text_prefix, filename_cat2))[0];
match_id = (int *)ADaPS_fetch(plist1->data, "match_match");
match_score = (float *)ADaPS_fetch(plist1->data, "match_score_match");
match_count = (int *)ADaPS_fetch(plist1->data, "match_count_match");
// Generate ranking of matches
sort(match_id, (size_t)n_groups_1_local, &match_index, SID_INT, SORT_GLOBAL, SORT_COMPUTE_INDEX, SORT_COMPUTE_NOT_INPLACE);
sort(match_index, (size_t)n_groups_1_local, &match_rank, SID_SIZE_T, SORT_GLOBAL, SORT_COMPUTE_INDEX, SORT_COMPUTE_NOT_INPLACE);
SID_free(SID_FARG match_index);
// Now we write the matching results. We need to write back to the file in the
// order that it was read from the halo catalogs, not necessarily the PH order
// that it is stored in RAM. This requires some buffer trickery.
buffer = SID_malloc(n_buffer_max * sizeof(size_t));
buffer_int = (int *)buffer;
buffer_size_t = (size_t *)buffer;
buffer_float = (float *)buffer;
// Write match_ids ...
// ... loop over all the groups in buffer-sized batches
SID_log("Writing match IDs...", SID_LOG_OPEN | SID_LOG_TIMER);
for(i_group = 0, buffered_count_local = 0; i_group < n_groups_1; i_group += n_buffer) {
// Decide this buffer iteration's size
n_buffer = GBP_MIN(n_buffer_max, n_groups_1 - i_group);
// Set the buffer to a default value smaller than the smallest possible data size
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++)
buffer_int[i_buffer] = -2; // Min value of match_id is -1
// Determine if any of the local data is being used for this buffer
for(j_group = 0; j_group < n_groups_1_local; j_group++) {
index_test = file_index_1[j_group] - i_group;
// ... if so, set the appropriate buffer value
if(index_test >= 0 && index_test < n_buffer) {
buffer_int[index_test] = match_id[j_group];
buffered_count_local++;
}
}
// Doing a global max on the buffer yields the needed buffer on all ranks
SID_Allreduce(SID_IN_PLACE, buffer_int, n_buffer, SID_INT, SID_MAX, SID_COMM_WORLD);
if(SID.I_am_Master) {
// Sanity check
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++) {
if(buffer_int[i_buffer] < -1 || buffer_int[i_buffer] >= n_groups_2)
SID_exit_error(
"Illegal match_id result (%d) for group No. %d. There are %d groups in the target catalog.",
SID_ERROR_LOGIC, buffer_int[i_buffer], i_group + i_buffer, n_groups_2);
}
// Write the buffer
fwrite(buffer_int, sizeof(int), (size_t)n_buffer, fp_out);
}
}
// Sanity check
calc_sum_global(&buffered_count_local, &buffered_count, 1, SID_INT, CALC_MODE_DEFAULT, SID_COMM_WORLD);
if(buffered_count != n_groups_1)
SID_exit_error("Buffer counts don't make sense (ie %d!=%d) after writing match IDs.", SID_ERROR_LOGIC,
buffered_count, n_groups_1);
SID_log("Done.", SID_LOG_CLOSE);
// Write match_score ...
// ... loop over all the groups in buffer-sized batches
SID_log("Writing match scores...", SID_LOG_OPEN | SID_LOG_TIMER);
for(i_group = 0, buffered_count_local = 0; i_group < n_groups_1; i_group += n_buffer) {
// Decide this buffer iteration's size
n_buffer = GBP_MIN(n_buffer_max, n_groups_1 - i_group);
// Set the buffer to a default value smaller than the smallest possible data size
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++)
buffer_float[i_buffer] = -1.; // Min value of match_score is 0.
// Determine if any of the local data is being used for this buffer
for(j_group = 0; j_group < n_groups_1_local; j_group++) {
index_test = file_index_1[j_group] - i_group;
// ... if so, set the appropriate buffer value
if(index_test >= 0 && index_test < n_buffer) {
buffer_float[index_test] = match_score[j_group];
buffered_count_local++;
}
}
// Doing a global max on the buffer yields the needed buffer on all ranks
SID_Allreduce(SID_IN_PLACE, buffer_float, n_buffer, SID_FLOAT, SID_MAX, SID_COMM_WORLD);
if(SID.I_am_Master) {
// Sanity check
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++) {
if(buffer_float[i_buffer] < 0.)
SID_exit_error("Illegal match_score result (%f) for group No. %d.", SID_ERROR_LOGIC,
buffer_float[i_buffer], i_group + i_buffer);
}
// Write the buffer
fwrite(buffer, sizeof(float), (size_t)n_buffer, fp_out);
}
}
// Sanity check
calc_sum_global(&buffered_count_local, &buffered_count, 1, SID_INT, CALC_MODE_DEFAULT, SID_COMM_WORLD);
if(buffered_count != n_groups_1)
SID_exit_error("Buffer counts don't make sense (ie %d!=%d) after writing match scores.", SID_ERROR_LOGIC,
buffered_count, n_groups_1);
SID_log("Done.", SID_LOG_CLOSE);
// Write match_count ...
// ... loop over all the groups in buffer-sized batches
SID_log("Writing match counts...", SID_LOG_OPEN | SID_LOG_TIMER);
for(i_group = 0, buffered_count_local = 0; i_group < n_groups_1; i_group += n_buffer) {
// Decide this buffer iteration's size
n_buffer = GBP_MIN(n_buffer_max, n_groups_1 - i_group);
// Set the buffer to a default value smaller than the smallest possible data size
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++)
buffer_int[i_buffer] = -1; // Min value of match_count is 0.
// Determine if any of the local data is being used for this buffer
for(j_group = 0; j_group < n_groups_1_local; j_group++) {
index_test = file_index_1[j_group] - i_group;
// ... if so, set the appropriate buffer value
if(index_test >= 0 && index_test < n_buffer) {
buffer_int[index_test] = match_count[j_group];
buffered_count_local++;
}
}
// Doing a global max on the buffer yields the needed buffer on all ranks
SID_Allreduce(SID_IN_PLACE, buffer_int, n_buffer, SID_INT, SID_MAX, SID_COMM_WORLD);
if(SID.I_am_Master) {
// Sanity check
for(i_buffer = 0; i_buffer < n_buffer; i_buffer++) {
if(buffer_int[i_buffer] < 0.)
SID_exit_error("Illegal match_count result (%f) for group No. %d.", SID_ERROR_LOGIC,
buffer_int[i_buffer], i_group + i_buffer);
}
// Write the buffer
fwrite(buffer, sizeof(int), (size_t)n_buffer, fp_out);
}
}
// Sanity check
calc_sum_global(&buffered_count_local, &buffered_count, 1, SID_INT, CALC_MODE_DEFAULT, SID_COMM_WORLD);
if(buffered_count != n_groups_1)
SID_exit_error("Buffer counts don't make sense (ie %d!=%d) after writing match scores.", SID_ERROR_LOGIC,
buffered_count, n_groups_1);
SID_log("Done.", SID_LOG_CLOSE);
// Clean-up
SID_log("Cleaning-up...", SID_LOG_OPEN);
SID_free(SID_FARG buffer);
SID_free(SID_FARG match_rank);
SID_log("Done.", SID_LOG_CLOSE);
}
// Close the file
if(SID.I_am_Master)
fclose(fp_out);
SID_log("Done.", SID_LOG_CLOSE);
}
void write_treenode_all_hist(tree_info *trees, const char *filename_out_root_in, int i_type, double logM_min, double dlogM, int n_logM) {
SID_log("Writing treenode histograms...", SID_LOG_OPEN | SID_LOG_TIMER);
int n_z_bin = GBP_MAX(1, trees->n_snaps / 25);
// double logM_min = 9.00;
// double logM_min = 6.50;
// double dlogM = 0.25;
// int n_logM = 24;
double logN_min = 1.50;
double dlogN = 0.25;
int n_logN = 20;
char group_prefix[8];
if(i_type == 0)
sprintf(group_prefix, "");
else
sprintf(group_prefix, "sub");
treenode_hist_info *hist_all_N;
treenode_hist_info *hist_all_Npeak;
treenode_hist_info *hist_all_M;
treenode_hist_info *hist_all_Mpeak;
treenode_hist_info *hist_centrals_N;
treenode_hist_info *hist_centrals_Npeak;
treenode_hist_info *hist_centrals_M;
treenode_hist_info *hist_centrals_Mpeak;
treenode_hist_info *hist_substructure_N;
treenode_hist_info *hist_substructure_Npeak;
treenode_hist_info *hist_substructure_M;
treenode_hist_info *hist_substructure_Mpeak;
char filename_out_root[SID_MAX_FILENAME_LENGTH];
sprintf(filename_out_root, "%s_%sgroup", filename_out_root_in, group_prefix);
init_treenode_hist(trees, "all", "z", "N", TREENODE_HIST_LOG_Y, &hist_all_N, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "all", "z", "N_peak", TREENODE_HIST_LOG_Y, &hist_all_Npeak, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "all", "z", "M", TREENODE_HIST_LOG_Y, &hist_all_M, n_z_bin, logM_min, dlogM, n_logM);
init_treenode_hist(trees, "all", "z", "M_peak", TREENODE_HIST_LOG_Y, &hist_all_Mpeak, n_z_bin, logM_min, dlogM, n_logM);
if(i_type == 1) {
init_treenode_hist(trees, "centrals", "z", "N", TREENODE_HIST_LOG_Y, &hist_centrals_N, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "centrals", "z", "N_peak", TREENODE_HIST_LOG_Y, &hist_centrals_Npeak, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "centrals", "z", "M", TREENODE_HIST_LOG_Y, &hist_centrals_M, n_z_bin, logM_min, dlogM, n_logM);
init_treenode_hist(trees, "centrals", "z", "M_peak", TREENODE_HIST_LOG_Y, &hist_centrals_Mpeak, n_z_bin, logM_min, dlogM, n_logM);
init_treenode_hist(trees, "substructure", "z", "N", TREENODE_HIST_LOG_Y, &hist_substructure_N, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "substructure", "z", "N_peak", TREENODE_HIST_LOG_Y, &hist_substructure_Npeak, n_z_bin, logN_min, dlogN, n_logN);
init_treenode_hist(trees, "substructure", "z", "M", TREENODE_HIST_LOG_Y, &hist_substructure_M, n_z_bin, logM_min, dlogM, n_logM);
init_treenode_hist(trees, "substructure", "z", "M_peak", TREENODE_HIST_LOG_Y, &hist_substructure_Mpeak, n_z_bin, logM_min, dlogM, n_logM);
}
for(int i_snap = 0; i_snap < trees->n_snaps; i_snap++) {
tree_node_info *current_halo;
if(i_type == 0)
current_halo = trees->first_neighbour_groups[i_snap];
else
current_halo = trees->first_neighbour_subgroups[i_snap];
while(current_halo != NULL) {
if(check_treenode_if_central(current_halo)) {
add_to_treenode_hist(trees, hist_centrals_N, current_halo);
add_to_treenode_hist(trees, hist_centrals_Npeak, current_halo);
add_to_treenode_hist(trees, hist_centrals_M, current_halo);
add_to_treenode_hist(trees, hist_centrals_Mpeak, current_halo);
} else if(check_treenode_if_satellite(current_halo)) {
add_to_treenode_hist(trees, hist_substructure_N, current_halo);
add_to_treenode_hist(trees, hist_substructure_Npeak, current_halo);
add_to_treenode_hist(trees, hist_substructure_M, current_halo);
add_to_treenode_hist(trees, hist_substructure_Mpeak, current_halo);
}
add_to_treenode_hist(trees, hist_all_N, current_halo);
add_to_treenode_hist(trees, hist_all_Npeak, current_halo);
add_to_treenode_hist(trees, hist_all_M, current_halo);
add_to_treenode_hist(trees, hist_all_Mpeak, current_halo);
current_halo = current_halo->next_neighbour;
}
}
SID_Allreduce(SID_IN_PLACE, (hist_all_N->array), (hist_all_N->n_x * hist_all_N->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE, (hist_all_Npeak->array), (hist_all_Npeak->n_x * hist_all_Npeak->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE, (hist_all_M->array), (hist_all_M->n_x * hist_all_M->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE, (hist_all_Mpeak->array), (hist_all_Mpeak->n_x * hist_all_Mpeak->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
if(i_type == 1) {
SID_Allreduce(SID_IN_PLACE, (hist_centrals_N->array), (hist_centrals_N->n_x * hist_centrals_N->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(
SID_IN_PLACE, (hist_centrals_Npeak->array), (hist_centrals_Npeak->n_x * hist_centrals_Npeak->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE, (hist_centrals_M->array), (hist_centrals_M->n_x * hist_centrals_M->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(
SID_IN_PLACE, (hist_centrals_Mpeak->array), (hist_centrals_Mpeak->n_x * hist_centrals_Mpeak->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(
SID_IN_PLACE, (hist_substructure_N->array), (hist_substructure_N->n_x * hist_substructure_N->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE,
(hist_substructure_Npeak->array),
(hist_substructure_Npeak->n_x * hist_substructure_Npeak->n_y),
SID_INT,
SID_SUM,
SID_COMM_WORLD);
SID_Allreduce(
SID_IN_PLACE, (hist_substructure_M->array), (hist_substructure_M->n_x * hist_substructure_M->n_y), SID_INT, SID_SUM, SID_COMM_WORLD);
SID_Allreduce(SID_IN_PLACE,
(hist_substructure_Mpeak->array),
(hist_substructure_Mpeak->n_x * hist_substructure_Mpeak->n_y),
SID_INT,
SID_SUM,
SID_COMM_WORLD);
}
write_treenode_hist(trees, filename_out_root, hist_all_N);
write_treenode_hist(trees, filename_out_root, hist_all_Npeak);
write_treenode_hist(trees, filename_out_root, hist_all_M);
write_treenode_hist(trees, filename_out_root, hist_all_Mpeak);
if(i_type == 1) {
write_treenode_hist(trees, filename_out_root, hist_centrals_N);
write_treenode_hist(trees, filename_out_root, hist_centrals_Npeak);
write_treenode_hist(trees, filename_out_root, hist_centrals_M);
write_treenode_hist(trees, filename_out_root, hist_centrals_Mpeak);
write_treenode_hist(trees, filename_out_root, hist_substructure_N);
write_treenode_hist(trees, filename_out_root, hist_substructure_Npeak);
write_treenode_hist(trees, filename_out_root, hist_substructure_M);
write_treenode_hist(trees, filename_out_root, hist_substructure_Mpeak);
}
free_treenode_hist(&hist_all_N);
free_treenode_hist(&hist_all_Npeak);
free_treenode_hist(&hist_all_M);
free_treenode_hist(&hist_all_Mpeak);
if(i_type == 1) {
free_treenode_hist(&hist_centrals_N);
free_treenode_hist(&hist_centrals_Npeak);
free_treenode_hist(&hist_centrals_M);
free_treenode_hist(&hist_centrals_Mpeak);
free_treenode_hist(&hist_substructure_N);
free_treenode_hist(&hist_substructure_Npeak);
free_treenode_hist(&hist_substructure_M);
free_treenode_hist(&hist_substructure_Mpeak);
}
SID_log("Done.", SID_LOG_CLOSE);
}
size_t SID_fread_chunked_ordered(void *buffer,
size_t n_x_read_local,
SID_fp *fp){
int i_chunk;
int i_group;
int i_rank;
size_t i_x_offset_local;
size_t n_x_read_local_bcast;
size_t i_x_read_chunk;
size_t i_x_offset_global;
size_t i_x_chunk;
size_t n_skip;
size_t n_x_chunk;
size_t n_x_chunk_max;
size_t header_size;
size_t r_val=0;
char filename_chunk[256];
// Operates the same way as SID_fread_chunked() except i_x_offset_local is
// computed here under the assumption that reads are done in order by rank
i_x_offset_local=0;
#if USE_MPI
for(i_rank=0;i_rank<SID.n_proc;i_rank++){
n_x_read_local_bcast=n_x_read_local;
SID_Bcast(&n_x_read_local_bcast,sizeof(size_t),i_rank,SID.COMM_WORLD);
if(i_rank<SID.My_rank)
i_x_offset_local+=n_x_read_local_bcast;
}
#endif
i_x_offset_global=fp->last_item;
for(i_chunk=0,i_x_read_chunk=0,i_x_chunk=i_x_offset_local+i_x_offset_global;
i_chunk<fp->chunked_header.n_chunk;
i_chunk++){
if(fp->i_x_start_chunk[i_chunk]<=i_x_chunk && fp->i_x_last_chunk[i_chunk]>=i_x_chunk){
n_skip =i_x_chunk-fp->i_x_start_chunk[i_chunk];
n_x_chunk=MIN(n_x_read_local-i_x_read_chunk,fp->i_x_step_chunk[i_chunk]-n_skip);
}
else{
n_x_chunk=0;
n_skip =0;
}
SID_Allreduce(&n_x_chunk,&n_x_chunk_max,1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD);
if(n_x_chunk_max>0){
sprintf(filename_chunk,"%s.%d",fp->filename_root,i_chunk);
#if !USE_MPI_IO
for(i_group=0;i_group<SID.n_proc;i_group++){
if(SID.My_group==i_group && n_x_chunk>0){
#endif
SID_fopen(filename_chunk,"r",fp);
if(n_x_chunk>0){
SID_fseek(fp,
1,
fp->header_offset[i_chunk]+n_skip*fp->chunked_header.item_size,
SID_SEEK_SET);
SID_fread((char *)buffer+i_x_read_chunk*fp->chunked_header.item_size,
fp->chunked_header.item_size,
n_x_chunk,
fp);
i_x_chunk +=n_x_chunk;
i_x_read_chunk+=n_x_chunk;
}
SID_fclose(fp);
#if !USE_MPI_IO
}
SID_Barrier(SID.COMM_WORLD);
}
#endif
}
}
SID_Allreduce(&i_x_chunk,&(fp->last_item),1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD);
//fp->last_item--;
return(r_val);
}
void read_gadget_binary_local(char *filename_root_in,
int snapshot_number,
int i_coord,
int i_load,
int n_load,
GBPREAL mass_array[N_GADGET_TYPE],
slab_info *slab,
cosmo_info *cosmo,
plist_info *plist){
size_t n_of_type_local[N_GADGET_TYPE];
size_t n_of_type[N_GADGET_TYPE];
size_t type_counter[N_GADGET_TYPE];
GBPREAL *x_array[N_GADGET_TYPE];
GBPREAL *y_array[N_GADGET_TYPE];
GBPREAL *z_array[N_GADGET_TYPE];
GBPREAL *vx_array[N_GADGET_TYPE];
GBPREAL *vy_array[N_GADGET_TYPE];
GBPREAL *vz_array[N_GADGET_TYPE];
int i_type;
// Determine file format and read the header
gadget_read_info fp_gadget;
int flag_filefound=init_gadget_read(filename_root_in,snapshot_number,&fp_gadget);
int flag_multifile=fp_gadget.flag_multifile;
int flag_file_type=fp_gadget.flag_file_type;
gadget_header_info header =fp_gadget.header;
// A file was found ...
if(flag_filefound){
char **pname;
SID_log("Reading GADGET binary file...",SID_LOG_OPEN|SID_LOG_TIMER);
pname=plist->species;
// Expansion factor (or time)
ADaPS_store(&(plist->data),(void *)(&(header.time)),"expansion_factor",ADaPS_SCALAR_DOUBLE);
ADaPS_store(&(plist->data),(void *)(&(header.time)),"time", ADaPS_SCALAR_DOUBLE);
// Redshift
double d_value;
d_value=(double)header.redshift;
ADaPS_store(&(plist->data),(void *)(&d_value),"redshift",ADaPS_SCALAR_DOUBLE);
// Number of particles and masses for each species in all files
size_t n_all[N_GADGET_TYPE];
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
n_all[i_type] =(size_t)header.n_all_lo_word[i_type]+((size_t)header.n_all_hi_word[i_type])<<32;
mass_array[i_type]=(GBPREAL)header.mass_array[i_type];
}
// Number of files in this snapshot
int n_files;
ADaPS_store(&(plist->data),(void *)(&(header.n_files)),"n_files",ADaPS_SCALAR_INT);
n_files=header.n_files;
// Cosmology
// Omega_o
d_value=(double)header.Omega_M;
ADaPS_store(&(plist->data),(void *)(&d_value),"Omega_M",ADaPS_SCALAR_DOUBLE);
// Omega_Lambda
d_value=(double)header.Omega_Lambda;
ADaPS_store(&(plist->data),(void *)(&d_value),"Omega_Lambda",ADaPS_SCALAR_DOUBLE);
// Hubble parameter
double h_Hubble;
double redshift;
h_Hubble=(double)header.h_Hubble;
if(h_Hubble<1e-10) h_Hubble=1.;
ADaPS_store(&(plist->data),(void *)(&h_Hubble),"h_Hubble",ADaPS_SCALAR_DOUBLE);
redshift=header.redshift;
ADaPS_store(&(plist->data),(void *)(&redshift),"redshift",ADaPS_SCALAR_DOUBLE);
// Count and report the total number of particles
size_t n_particles_all;
int n_non_zero;
n_particles_all=0;
for(i_type=0,n_non_zero=0;i_type<N_GADGET_TYPE;i_type++){
if(n_all[i_type]>0){
n_particles_all+=n_all[i_type];
n_non_zero++;
}
}
SID_log("%zd",SID_LOG_CONTINUE,n_particles_all);
if(n_non_zero>0)
SID_log(" (",SID_LOG_CONTINUE,n_particles_all);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
if(n_all[i_type]>0){
if(i_type==n_non_zero-1){
if(n_non_zero>1)
SID_log("and %lld %s",SID_LOG_CONTINUE,n_all[i_type],pname[i_type]);
else
SID_log("%lld %s",SID_LOG_CONTINUE,n_all[i_type],pname[i_type]);
}
else{
if(n_non_zero>1)
SID_log("%lld %s, ",SID_LOG_CONTINUE,n_all[i_type],pname[i_type]);
else
SID_log("%lld %s",SID_LOG_CONTINUE,n_all[i_type],pname[i_type]);
}
}
}
if(n_non_zero>0)
SID_log(") particles...",SID_LOG_CONTINUE);
else
SID_log(" particles...",SID_LOG_CONTINUE);
// Count the number of particles that will be scattered to each rank
char filename[MAX_FILENAME_LENGTH];
size_t k_particle;
int i_file;
int record_length_open;
int record_length_close;
size_t i_particle;
size_t i_buffer;
size_t i_step;
int i_type;
size_t index;
GBPREAL *pos_buffer;
GBPREAL *vel_buffer;
double pos_test;
// Initialize some arrays
pos_buffer=(GBPREAL *)SID_malloc(sizeof(GBPREAL)*READ_BUFFER_ALLOC_LOCAL);
vel_buffer=(GBPREAL *)SID_malloc(sizeof(GBPREAL)*READ_BUFFER_ALLOC_LOCAL);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
n_of_type_local[i_type]=0;
n_of_type[i_type] =0;
type_counter[i_type] =0;
}
// Determine how many particles of each type will end-up on each core
SID_log("Performing domain decomposition...",SID_LOG_OPEN|SID_LOG_TIMER);
int n_read;
if(n_load<n_files)
n_read=n_files;
else
n_read=1;
for(i_file=i_load;i_file<(i_load+n_read);i_file++){
set_gadget_filename(&fp_gadget,i_file,filename);
// Read header and move to the positions
FILE *fp_pos;
FILE *fp_vel;
fp_pos=fopen(filename,"r");
fread_verify(&record_length_open,4,1,fp_pos);
fread_verify(&header,sizeof(gadget_header_info),1,fp_pos);
fread_verify(&record_length_close,4,1,fp_pos);
if(record_length_open!=record_length_close)
SID_log_warning("Problem with GADGET record size (close of header)",ERROR_LOGIC);
fread_verify(&record_length_open,4,1,fp_pos);
// Create a file pointer to the velocities
fp_vel=fopen(filename,"r");
fread_verify(&record_length_open,4,1,fp_vel);
fseeko(fp_vel,(off_t)(record_length_open),SEEK_CUR);
fread_verify(&record_length_close,4,1,fp_vel);
fread_verify(&record_length_open,4,1,fp_vel);
fseeko(fp_vel,(off_t)(record_length_open),SEEK_CUR);
fread_verify(&record_length_close,4,1,fp_vel);
if(record_length_open!=record_length_close)
SID_log_warning("Problem with GADGET record size (close of positons)",ERROR_LOGIC);
fread_verify(&record_length_open,4,1,fp_vel);
// We only have to worry about z-space effects for domain decomposition in this one case.
if(i_coord==1){
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(i_particle=0;i_particle<header.n_file[i_type];i_particle+=i_step){
i_step=MIN(READ_BUFFER_SIZE_LOCAL,header.n_file[i_type]-i_particle);
if(SID.I_am_Master){
fread_verify(pos_buffer,sizeof(GBPREAL),3*i_step,fp_pos);
fread_verify(vel_buffer,sizeof(GBPREAL),3*i_step,fp_vel);
}
SID_Bcast(pos_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
SID_Bcast(vel_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
for(i_buffer=0;i_buffer<i_step;i_buffer++){
index=3*i_buffer;
pos_test =(double)(pos_buffer[index]);
pos_test+=(double)(1e3*h_Hubble*((double)vel_buffer[index])/(a_of_z(redshift)*M_PER_MPC*H_convert(H_z(redshift,cosmo))));
if(pos_test<0) pos_test+=header.box_size;
if(pos_test>=header.box_size) pos_test-=header.box_size;
if(pos_test>=slab->x_min_local && pos_test<slab->x_max_local)
n_of_type_local[i_type]++;
}
}
}
}
else{
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(i_particle=0;i_particle<header.n_file[i_type];i_particle+=i_step){
i_step=MIN(READ_BUFFER_SIZE_LOCAL,header.n_file[i_type]-i_particle);
if(SID.I_am_Master){
fread_verify(pos_buffer,sizeof(GBPREAL),3*i_step,fp_pos);
fread_verify(vel_buffer,sizeof(GBPREAL),3*i_step,fp_vel);
}
SID_Bcast(pos_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
SID_Bcast(vel_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
for(i_buffer=0;i_buffer<i_step;i_buffer++){
pos_test=pos_buffer[3*i_buffer];
if(pos_test>=slab->x_min_local && pos_test<slab->x_max_local)
n_of_type_local[i_type]++;
}
}
i_step=MIN(READ_BUFFER_SIZE_LOCAL,header.n_file[i_type]-i_particle);
}
}
fclose(fp_pos);
fclose(fp_vel);
}
SID_log("Done.",SID_LOG_CLOSE);
// Allocate arrays
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
if(n_all[i_type]>0){
x_array[i_type] =(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
y_array[i_type] =(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
z_array[i_type] =(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
vx_array[i_type]=(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
vy_array[i_type]=(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
vz_array[i_type]=(GBPREAL *)SID_malloc(sizeof(GBPREAL)*n_of_type_local[i_type]);
}
}
// Perform read
SID_log("Performing read...",SID_LOG_OPEN|SID_LOG_TIMER);
for(i_file=i_load;i_file<(i_load+n_read);i_file++){
set_gadget_filename(&fp_gadget,i_file,filename);
// Read header and move to the positions
FILE *fp_pos;
FILE *fp_vel;
fp_pos=fopen(filename,"r");
fread_verify(&record_length_open,4,1,fp_pos);
fread_verify(&header,sizeof(gadget_header_info),1,fp_pos);
fread_verify(&record_length_close,4,1,fp_pos);
if(record_length_open!=record_length_close)
SID_log_warning("Problem with GADGET record size (close of header)",ERROR_LOGIC);
fread_verify(&record_length_open,4,1,fp_pos);
// Create a file pointer to the velocities
fp_vel=fopen(filename,"r");
fread_verify(&record_length_open,4,1,fp_vel);
fseeko(fp_vel,(off_t)(record_length_open),SEEK_CUR);
fread_verify(&record_length_close,4,1,fp_vel);
fread_verify(&record_length_open,4,1,fp_vel);
fseeko(fp_vel,(off_t)(record_length_open),SEEK_CUR);
fread_verify(&record_length_close,4,1,fp_vel);
if(record_length_open!=record_length_close)
SID_log_warning("Problem with GADGET record size (close of positions)",ERROR_LOGIC);
fread_verify(&record_length_open,4,1,fp_vel);
// Perform the read and populate the local position arrays
size_t i_particle;
size_t i_step;
int i_type;
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
for(i_particle=0;i_particle<header.n_file[i_type];i_particle+=i_step){
i_step=MIN(READ_BUFFER_SIZE_LOCAL,header.n_file[i_type]-i_particle);
if(SID.I_am_Master){
fread_verify(pos_buffer,sizeof(GBPREAL),3*i_step,fp_pos);
fread_verify(vel_buffer,sizeof(GBPREAL),3*i_step,fp_vel);
}
SID_Bcast(pos_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
SID_Bcast(vel_buffer,sizeof(GBPREAL)*3*i_step,MASTER_RANK,SID.COMM_WORLD);
for(i_buffer=0;i_buffer<i_step;i_buffer++){
double x_test;
double y_test;
double z_test;
double vx_test;
double vy_test;
double vz_test;
index=3*i_buffer;
x_test =(double)pos_buffer[index+0];
y_test =(double)pos_buffer[index+1];
z_test =(double)pos_buffer[index+2];
vx_test=(double)vel_buffer[index+0];
vy_test=(double)vel_buffer[index+1];
vz_test=(double)vel_buffer[index+2];
switch(i_coord){
case 1:
x_test+=(1e3*h_Hubble*vx_test/(a_of_z(redshift)*M_PER_MPC*H_convert(H_z(redshift,cosmo))));
if(x_test<0) x_test+=header.box_size;
if(x_test>=header.box_size) x_test-=header.box_size;
break;
case 2:
y_test+=(1e3*h_Hubble*vy_test/(a_of_z(redshift)*M_PER_MPC*H_convert(H_z(redshift,cosmo))));
if(y_test<0) y_test+=header.box_size;
if(y_test>=header.box_size) y_test-=header.box_size;
break;
case 3:
z_test+=(1e3*h_Hubble*vz_test/(a_of_z(redshift)*M_PER_MPC*H_convert(H_z(redshift,cosmo))));
if(z_test<0) z_test+=header.box_size;
if(z_test>=header.box_size) z_test-=header.box_size;
break;
}
if(x_test>=slab->x_min_local && x_test<slab->x_max_local){
x_array[i_type][type_counter[i_type]] =x_test;
y_array[i_type][type_counter[i_type]] =y_test;
z_array[i_type][type_counter[i_type]] =z_test;
vx_array[i_type][type_counter[i_type]]=vx_test;
vy_array[i_type][type_counter[i_type]]=vy_test;
vz_array[i_type][type_counter[i_type]]=vz_test;
type_counter[i_type]++;
}
}
}
}
// Close file pointers
fclose(fp_pos);
fclose(fp_vel);
}
SID_free(SID_FARG pos_buffer);
SID_free(SID_FARG vel_buffer);
SID_log("Done.",SID_LOG_CLOSE);
// Sanity checks
size_t n_particles_local;
size_t n_particles_read;
size_t n_particles_test;
for(i_type=0,n_particles_local=0,n_particles_test=0;i_type<N_GADGET_TYPE;i_type++){
n_particles_local+=n_of_type_local[i_type];
n_particles_test +=n_all[i_type];
}
SID_Allreduce(&n_particles_local,&n_particles_read,1,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
if(n_particles_read!=n_particles_test && n_load==1)
SID_trap_error("Total particle counts don't make sense after read_gadget (ie. %zd!=%zd).",ERROR_LOGIC,n_particles_read,n_particles_test);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
SID_Allreduce(&(n_of_type_local[i_type]),&(n_of_type[i_type]),1,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
if(n_of_type[i_type]!=n_all[i_type] && n_load==1)
SID_trap_error("Particle counts don't make sense after read_gadget (ie. %zd!=%zd).",ERROR_LOGIC,n_of_type[i_type],n_all[i_type]);
}
// Store results
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
if(n_of_type[i_type]>0){
ADaPS_store(&(plist->data),(void *)(&(n_of_type_local[i_type])),"n_%s", ADaPS_SCALAR_SIZE_T,pname[i_type]);
ADaPS_store(&(plist->data),(void *)(&(n_of_type[i_type])), "n_all_%s", ADaPS_SCALAR_SIZE_T,pname[i_type]);
}
}
ADaPS_store(&(plist->data),(void *)(&n_particles_all),"n_particles_all",ADaPS_SCALAR_SIZE_T);
for(i_type=0;i_type<N_GADGET_TYPE;i_type++){
if(n_of_type_local[i_type]>0){
ADaPS_store(&(plist->data),(void *)x_array[i_type], "x_%s", ADaPS_DEFAULT, pname[i_type]);
ADaPS_store(&(plist->data),(void *)y_array[i_type], "y_%s", ADaPS_DEFAULT, pname[i_type]);
ADaPS_store(&(plist->data),(void *)z_array[i_type], "z_%s", ADaPS_DEFAULT, pname[i_type]);
ADaPS_store(&(plist->data),(void *)vx_array[i_type],"vx_%s",ADaPS_DEFAULT,pname[i_type]);
ADaPS_store(&(plist->data),(void *)vy_array[i_type],"vy_%s",ADaPS_DEFAULT,pname[i_type]);
ADaPS_store(&(plist->data),(void *)vz_array[i_type],"vz_%s",ADaPS_DEFAULT,pname[i_type]);
}
}
SID_log("Done.",SID_LOG_CLOSE);
}
}
void read_smooth(plist_info *plist,
char *filename_root_in,
int snapshot_number,
int mode){
char filename[256];
size_t n_particles_all_mem;
size_t n_particles_local;
size_t n_particles_total;
int i_quantity;
int n_quantities=3;
int *used;
char *species_name;
char var_name[256];
char unit_name[256];
double unit_factor;
int i_file;
size_t *ids;
size_t *ids_index;
int n_particles_file;
int offset;
int n_files;
void *id_buf;
size_t *id_buf_index=NULL;
int *id_buf_i;
long long *id_buf_L;
long long *mark;
size_t i_particle;
size_t j_particle;
void *buffer;
float *local_array;
int n_mark;
float *r_smooth_array;
float *rho_array;
float *sigma_v_array;
char *read_array;
double expansion_factor;
double h_Hubble;
int flag_filefound=FALSE;
int flag_multifile=FALSE;
int flag_file_type;
int flag_LONGIDs;
int read_rank=MASTER_RANK;
int flag_log_sigma=FALSE;
int flag_log_rho=FALSE;
FILE *fp;
SID_log("Reading smooth file {%s}...",SID_LOG_OPEN|SID_LOG_TIMER,filename_root_in);
// Read header info
SID_log("Reading header information...",SID_LOG_OPEN);
smooth_header_info header;
flag_filefound =init_smooth_read(filename_root_in,snapshot_number,&flag_multifile,&flag_file_type,&header);
SID_log("n_files =%d", SID_LOG_COMMENT,header.n_files);
SID_log("n_particles=%lld",SID_LOG_COMMENT,header.n_particles_total);
SID_log("Done.",SID_LOG_CLOSE);
// Interpret the mode passed to this function
int flag_logs_used =FALSE;
int flag_log_quantity=FALSE;
if(check_mode_for_flag(mode,READ_SMOOTH_LOG_SIGMA)){
flag_log_sigma=TRUE;
flag_logs_used=TRUE;
}
else
flag_log_sigma=FALSE;
if(check_mode_for_flag(mode,READ_SMOOTH_LOG_RHO)){
flag_log_rho =TRUE;
flag_logs_used=TRUE;
}
else
flag_log_rho=FALSE;
// A file was found ...
if(flag_filefound){
// Communicate the header
n_particles_file =header.n_particles_file;
offset =header.offset;
n_particles_total=header.n_particles_total;
n_files =MAX(1,header.n_files);
SID_Bcast(&n_particles_file, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&offset, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&n_particles_total,(int)sizeof(long long),read_rank,SID.COMM_WORLD);
SID_Bcast(&n_files, (int)sizeof(int), read_rank,SID.COMM_WORLD);
// Fetch the number of particles and their ids
species_name =plist->species[GADGET_TYPE_DARK];
n_particles_local=((size_t *)ADaPS_fetch(plist->data,"n_%s", species_name))[0];
n_particles_all_mem =((size_t *)ADaPS_fetch(plist->data,"n_all_%s",species_name))[0];
ids = (size_t *)ADaPS_fetch(plist->data,"id_%s", species_name);
expansion_factor =((double *)ADaPS_fetch(plist->data,"expansion_factor"))[0];
h_Hubble =((double *)ADaPS_fetch(plist->data,"h_Hubble"))[0];
// Check if we are dealing with LONG IDs or not
if(ADaPS_exist(plist->data,"flag_LONGIDs"))
flag_LONGIDs=TRUE;
else
flag_LONGIDs=FALSE;
// Sort particle IDs
SID_log("Sorting particle IDs...",SID_LOG_OPEN|SID_LOG_TIMER);
merge_sort(ids,(size_t)n_particles_local,&ids_index,SID_SIZE_T,SORT_COMPUTE_INDEX,FALSE);
SID_log("Done.",SID_LOG_CLOSE);
// Allocate arrays
SID_log("Allocating arrays...",SID_LOG_OPEN);
read_array=(char *)SID_calloc(sizeof(char)*n_particles_local);
for(i_quantity=0;i_quantity<n_quantities;i_quantity++){
switch(i_quantity){
case 0:
r_smooth_array=(float *)SID_malloc(sizeof(float)*n_particles_local);
ADaPS_store(&(plist->data),r_smooth_array,"r_smooth_%s",ADaPS_DEFAULT,species_name);
break;
case 1:
rho_array=(float *)SID_malloc(sizeof(float)*n_particles_local);
ADaPS_store(&(plist->data),rho_array,"rho_%s",ADaPS_DEFAULT,species_name);
break;
case 2:
sigma_v_array=(float *)SID_malloc(sizeof(float)*n_particles_local);
ADaPS_store(&(plist->data),sigma_v_array,"sigma_v_%s",ADaPS_DEFAULT,species_name);
break;
}
}
SID_log("Done.",SID_LOG_CLOSE);
// Read each file in turn
pcounter_info pcounter;
size_t n_particles_read=0;
SID_log("Performing read...",SID_LOG_OPEN|SID_LOG_TIMER);
SID_init_pcounter(&pcounter,n_particles_total,10);
for(i_file=0;i_file<n_files;i_file++){
set_smooth_filename(filename_root_in,snapshot_number,i_file,flag_multifile,flag_file_type,filename);
if((fp=fopen(filename,"r"))!=NULL){
fread_verify(&(header.n_particles_file), sizeof(int), 1,fp);
fread_verify(&(header.offset), sizeof(int), 1,fp);
fread_verify(&(header.n_particles_total),sizeof(long long),1,fp);
fread_verify(&(header.n_files), sizeof(int), 1,fp);
}
else
SID_trap_error("Could not open smooth file {%s}",ERROR_IO_OPEN,filename);
n_particles_file =header.n_particles_file;
offset =header.offset;
n_particles_total=header.n_particles_total;
n_files =MAX(1,header.n_files);
SID_Bcast(&n_particles_file, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&offset, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&n_particles_total,(int)sizeof(long long),read_rank,SID.COMM_WORLD);
SID_Bcast(&n_files, (int)sizeof(int), read_rank,SID.COMM_WORLD);
// Read IDs
if(flag_LONGIDs){
if(i_file==0) SID_log("(long long) IDs...",SID_LOG_CONTINUE);
id_buf =SID_malloc(sizeof(long long)*n_particles_file);
id_buf_L=(long long *)id_buf;
if(SID.My_rank==read_rank){
fseeko(fp,(size_t)(3*n_particles_file*sizeof(float)),SEEK_CUR);
fread_verify(id_buf,sizeof(long long),n_particles_file,fp);
}
SID_Barrier(SID.COMM_WORLD);
SID_Bcast(id_buf_L,(int)(n_particles_file*sizeof(long long)),read_rank,SID.COMM_WORLD);
merge_sort(id_buf_L,(size_t)n_particles_file,&id_buf_index,SID_SIZE_T,SORT_COMPUTE_INDEX,FALSE);
}
else{
if(i_file==0) SID_log("(int) IDs...",SID_LOG_CONTINUE);
id_buf =SID_malloc(sizeof(int)*n_particles_file);
id_buf_i=(int *)id_buf;
if(SID.My_rank==read_rank){
fseeko(fp,(size_t)(3*n_particles_file*sizeof(float)),SEEK_CUR);
fread_verify(id_buf,sizeof(int),n_particles_file,fp);
}
SID_Barrier(SID.COMM_WORLD);
SID_Bcast(id_buf_i,(int)(n_particles_file*sizeof(int)),read_rank,SID.COMM_WORLD);
merge_sort(id_buf_i,(size_t)n_particles_file,&id_buf_index,SID_INT,SORT_COMPUTE_INDEX,FALSE);
}
// Create local particle mapping
int n_mark_i=0;
mark=(long long *)SID_malloc(sizeof(long long)*n_particles_file);
for(i_particle=0;i_particle<n_particles_file;i_particle++)
mark[i_particle]=-1;
if(flag_LONGIDs){
for(i_particle=0,j_particle=0,n_mark=0;i_particle<n_particles_file && j_particle<n_particles_local;i_particle++){
while(j_particle<n_particles_local-1 && ids[ids_index[j_particle]]<id_buf_L[id_buf_index[i_particle]]) j_particle++;
if(ids[ids_index[j_particle]]==id_buf_L[id_buf_index[i_particle]]){
mark[id_buf_index[i_particle]]=(long long)ids_index[j_particle];
n_mark++;
n_mark_i++;
}
}
}
else{
for(i_particle=0,j_particle=0,n_mark=0;i_particle<n_particles_file && j_particle<n_particles_local;i_particle++){
while(j_particle<n_particles_local-1 && ids[ids_index[j_particle]]<id_buf_i[id_buf_index[i_particle]]) j_particle++;
if(ids[ids_index[j_particle]]==id_buf_i[id_buf_index[i_particle]]){
mark[id_buf_index[i_particle]]=(long long)ids_index[j_particle];
n_mark++;
n_mark_i++;
}
}
}
SID_free(SID_FARG id_buf);
SID_free(SID_FARG id_buf_index);
// Move to the start of the particle quantities
if(SID.My_rank==read_rank){
rewind(fp);
fread_verify(&n_particles_file, sizeof(int), 1,fp);
fread_verify(&offset, sizeof(int), 1,fp);
fread_verify(&n_particles_total,sizeof(long long),1,fp);
fread_verify(&n_files, sizeof(int), 1,fp);
n_files=MAX(1,n_files);
}
SID_Bcast(&n_particles_file, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&offset, (int)sizeof(int), read_rank,SID.COMM_WORLD);
SID_Bcast(&n_particles_total,(int)sizeof(long long),read_rank,SID.COMM_WORLD);
SID_Bcast(&n_files, (int)sizeof(int), read_rank,SID.COMM_WORLD);
buffer=SID_malloc(sizeof(float)*n_particles_file);
for(i_quantity=0;i_quantity<n_quantities;i_quantity++){
int flag_log_quantity;
switch(i_quantity){
case 0:
sprintf(var_name,"r_smooth_%s",species_name);
sprintf(unit_name,"Mpc");
unit_factor=plist->length_unit/h_Hubble;
local_array=r_smooth_array;
break;
case 1:
sprintf(var_name,"rho_%s",species_name);
sprintf(unit_name,"Msol/Mpc^3");
unit_factor=h_Hubble*h_Hubble*plist->mass_unit/pow(plist->length_unit,3.);
local_array=rho_array;
break;
case 2:
sprintf(var_name,"sigma_v_%s",species_name);
sprintf(unit_name,"km/s");
unit_factor=sqrt(expansion_factor)*plist->velocity_unit;
local_array=sigma_v_array;
break;
}
// Read next quantity
if(SID.My_rank==read_rank)
fread_verify(buffer,sizeof(float),n_particles_file,fp);
SID_Barrier(SID.COMM_WORLD);
SID_Bcast(buffer,(int)(n_particles_file*sizeof(float)),read_rank,SID.COMM_WORLD);
// Place in final array
if(n_mark_i>0){
if(i_quantity==0){
for(i_particle=0;i_particle<n_particles_file;i_particle++)
if(mark[i_particle]>=0) read_array[mark[i_particle]]=TRUE;
}
for(i_particle=0;i_particle<n_particles_file;i_particle++)
if(mark[i_particle]>=0) local_array[mark[i_particle]]=((float *)buffer)[i_particle]*unit_factor;
}
}
SID_free(SID_FARG mark);
SID_free(SID_FARG buffer);
if(SID.My_rank==read_rank)
fclose(fp);
n_particles_read+=n_particles_file;
if(n_files>1)
SID_check_pcounter(&pcounter,n_particles_read);
} // i_file
SID_free(SID_FARG ids_index);
SID_Barrier(SID.COMM_WORLD);
SID_log("Done.",SID_LOG_CLOSE);
// Check that all particles have been treated
size_t sum_check=0;
for(i_particle=0;i_particle<n_particles_local;i_particle++)
sum_check+=(size_t)read_array[i_particle];
SID_Allreduce(SID_IN_PLACE,&sum_check,1,SID_SIZE_T,SID_SUM,SID.COMM_WORLD);
if(sum_check!=n_particles_all_mem)
SID_trap_error("Only %lld of %lld particles were set.",ERROR_LOGIC,sum_check,n_particles_all_mem);
SID_free(SID_FARG read_array);
SID_log("Summary...",SID_LOG_OPEN);
for(i_quantity=0;i_quantity<n_quantities;i_quantity++){
char var_name[32];
switch(i_quantity){
case 0:
sprintf(var_name,"Lengths: ");
sprintf(unit_name,"Mpc");
unit_factor=1./M_PER_MPC;
local_array=r_smooth_array;
break;
case 1:
sprintf(var_name,"Densities: ");
sprintf(unit_name,"Msol/Mpc^3");
unit_factor=M_PER_MPC*M_PER_MPC*M_PER_MPC/M_SOL;
local_array=rho_array;
break;
case 2:
sprintf(var_name,"sigmas_v's:");
sprintf(unit_name,"km/s");
unit_factor=1e-3;
local_array=sigma_v_array;
break;
}
// Report some stats
float var_min,var_max,var_mean;
calc_min_global(local_array,
&var_min,
n_particles_local,
SID_FLOAT,
CALC_MODE_DEFAULT,
SID.COMM_WORLD);
calc_max_global(local_array,
&var_max,
n_particles_local,
SID_FLOAT,
CALC_MODE_DEFAULT,
SID.COMM_WORLD);
calc_mean_global(local_array,
&var_mean,
n_particles_local,
SID_FLOAT,
CALC_MODE_DEFAULT,
SID.COMM_WORLD);
// Remove NaN's from sigma_v's if needed
size_t n_NaN=0;
for(i_particle=0;i_particle<n_particles_local;i_particle++){
if(isnan(local_array[i_particle])){
local_array[i_particle]=1000.*0.5*(var_min+var_max);
n_NaN++;
}
}
if(n_NaN>0)
SID_log("%s min=%e max=%e mean=%e [%s] (%lld are NaN)",
SID_LOG_COMMENT,
var_name,
var_min*unit_factor,
var_max*unit_factor,
var_mean*unit_factor,
unit_name,
n_NaN);
else
SID_log("%s min=%e max=%e mean=%e [%s]",
SID_LOG_COMMENT,
var_name,
var_min*unit_factor,
var_max*unit_factor,
var_mean*unit_factor,
unit_name);
}
SID_log("",SID_LOG_CLOSE|SID_LOG_NOPRINT);
if(flag_logs_used){
SID_log("Taking needed logs of quantities...",SID_LOG_OPEN|SID_LOG_TIMER);
for(i_quantity=0;i_quantity<n_quantities;i_quantity++){
switch(i_quantity){
case 0:
unit_factor=1./M_PER_MPC;
local_array=r_smooth_array;
flag_log_quantity=FALSE;
break;
case 1:
unit_factor=M_PER_MPC*M_PER_MPC*M_PER_MPC/M_SOL;
local_array=rho_array;
flag_log_quantity=flag_log_rho;
break;
case 2:
unit_factor=1e-3;
local_array=sigma_v_array;
flag_log_quantity=flag_log_sigma;
break;
}
if(flag_log_quantity){
for(i_particle=0;i_particle<n_particles_local;i_particle++)
local_array[i_particle]=take_log10(local_array[i_particle]);
}
}
SID_log("Done.",SID_LOG_CLOSE);
}
SID_Barrier(SID.COMM_WORLD);
SID_log("Done.",SID_LOG_CLOSE);
}
else
SID_trap_error("Could not find file with root {%s}",ERROR_IO_OPEN,filename_root_in);
}
