Exemplo n.º 1
0
int main(int argc, char *argv[]) {
    int    n_search;
    int    i_halo;
    char   filename_in[SID_MAX_FILENAME_LENGTH];
    char   group_text_prefix[4];
    int    n_files;
    int    k_read;
    int    max_n_groups;
    int    l_read;
    int    n_groups;
    int *  n_particles_i;
    int *  n_particles_j;
    int    j_read;
    int    mode;
    int    n_groups_i;
    int    n_groups_j;
    int    j_halo;
    int    i_read;
    int    i_read_start;
    int    i_read_stop;
    SID_fp fp_in;

    SID_Init(&argc, &argv, NULL);

    // Fetch user inputs
    char filename_root_in[SID_MAX_FILENAME_LENGTH];
    char filename_catalog_root[SID_MAX_FILENAME_LENGTH];
    char filename_halo_version[SID_MAX_FILENAME_LENGTH];
    strcpy(filename_root_in, argv[1]);
    strcpy(filename_halo_version, argv[2]);
    if(!strcmp(argv[3], "groups") || !strcmp(argv[3], "group"))
        mode = MATCH_GROUPS;
    else if(!strcmp(argv[3], "subgroups") || !strcmp(argv[3], "subgroup"))
        mode = MATCH_SUBGROUPS;
    else {
        SID_exit_error("Invalid mode selection {%s}.  Should be 'group' or 'subgroup'.", SID_ERROR_SYNTAX, argv[3]);
    }
    i_read               = atoi(argv[4]);
    j_read               = atoi(argv[5]);
    int flag_SSimPL_base = GBP_TRUE;
    if(argc == 7) {
        flag_SSimPL_base = GBP_FALSE;
        strcpy(filename_catalog_root, argv[6]);
        sprintf(filename_catalog_root, "%s/halos/%s", argv[6], filename_halo_version);
    } else
        sprintf(filename_catalog_root, "%s/halos/%s", filename_root_in, filename_halo_version);
    SID_log("Searching match information for halo #%d in file #%d of {%s}...", SID_LOG_OPEN | SID_LOG_TIMER, i_halo, i_read, filename_root_in);

    // Convert filename_root to filename
    switch(mode) {
        case MATCH_SUBGROUPS:
            sprintf(group_text_prefix, "sub");
            break;
        case MATCH_GROUPS:
            sprintf(group_text_prefix, "");
            break;
    }

    // Set the standard SSiMPL match file path
    char filename_root[SID_MAX_FILENAME_LENGTH];
    if(flag_SSimPL_base)
        sprintf(filename_root, "%s/trees/matches/%03d/", filename_root_in, i_read);
    else
        sprintf(filename_root, "%s_", filename_root_in);

    // Read header information
    int   i_read_in;
    int   j_read_in;
    int   n_groups_1;
    int   n_groups_2;
    float score_rank_index;
    sprintf(filename_in, "%s%sgroup_matches_%03d_%03d.dat", filename_root, group_text_prefix, i_read, j_read);
    SID_fopen(filename_in, "r", &fp_in);
    SID_fread(&i_read_in, sizeof(int), 1, &fp_in);
    SID_log("i_read    =%d", SID_LOG_COMMENT, i_read_in);
    SID_fread(&j_read_in, sizeof(int), 1, &fp_in);
    SID_log("j_read    =%d", SID_LOG_COMMENT, j_read_in);
    SID_fread(&n_groups_i, sizeof(int), 1, &fp_in);
    SID_log("n_groups_i=%d", SID_LOG_COMMENT, n_groups_i);
    SID_fread(&n_groups_j, sizeof(int), 1, &fp_in);
    SID_log("n_groups_j=%d", SID_LOG_COMMENT, n_groups_j);
    SID_fread(&score_rank_index, sizeof(int), 1, &fp_in);
    SID_log("score_idx =%f", SID_LOG_COMMENT, score_rank_index);

    // Allocate RAM
    int *  match = (int *)SID_malloc(sizeof(int) * n_groups_i);
    float *score = (float *)SID_malloc(sizeof(float) * n_groups_i);
    int *  count = (int *)SID_malloc(sizeof(int) * n_groups_i);

    // Read arrays
    SID_fread(match, sizeof(int), n_groups_i, &fp_in);
    SID_fread(score, sizeof(float), n_groups_i, &fp_in);
    SID_fread(count, sizeof(int), n_groups_i, &fp_in);

    // Close file
    SID_fclose(&fp_in);

    // Read halo sizes from header file
    SID_log("Reading halo sizes...", SID_LOG_OPEN);
    int *n_p_i = (int *)SID_malloc(sizeof(int) * n_groups_i);
    sprintf(filename_in, "%s_%03d.catalog_%sgroups", filename_catalog_root, i_read, group_text_prefix);
    SID_fopen(filename_in, "r", &fp_in);
    int n_cat_i;
    int offset_size_i;
    SID_fread(&n_cat_i, sizeof(int), 1, &fp_in);
    SID_fread(&offset_size_i, sizeof(int), 1, &fp_in);
    if(n_cat_i != n_groups_i)
        SID_exit_error("Catalog 'i' halo counts don't match (ie %d!=%d)", SID_ERROR_LOGIC, n_cat_i, n_groups_i);
    SID_fread(n_p_i, sizeof(int), n_cat_i, &fp_in);
    SID_fclose(&fp_in);

    int *n_p_j = (int *)SID_malloc(sizeof(int) * n_groups_j);
    sprintf(filename_in, "%s_%03d.catalog_%sgroups", filename_catalog_root, j_read, group_text_prefix);
    SID_fopen(filename_in, "r", &fp_in);
    int n_cat_j;
    int offset_size_j;
    SID_fread(&n_cat_j, sizeof(int), 1, &fp_in);
    SID_fread(&offset_size_j, sizeof(int), 1, &fp_in);
    if(n_cat_j != n_groups_j)
        SID_exit_error("Catalog 'i' halo counts don't match (ie %d!=%d)", SID_ERROR_LOGIC, n_cat_j, n_groups_j);
    SID_fread(n_p_j, sizeof(int), n_cat_j, &fp_in);
    SID_fclose(&fp_in);
    SID_log("Done.", SID_LOG_CLOSE);

    // Print results
    for(k_read = 0; k_read < n_groups_i; k_read++) {
        if(match[k_read] >= 0)
            printf("%7d %7d %7d %7d %7d %le %le %le\n",
                   k_read,
                   match[k_read],
                   n_p_i[k_read],
                   n_p_j[match[k_read]],
                   count[k_read],
                   score[k_read],
                   maximum_match_score(n_p_i[k_read]),
                   match_score_f_goodness(score[k_read], n_p_i[k_read]));
    }

    // Clean-up
    SID_free(SID_FARG match);
    SID_free(SID_FARG score);
    SID_free(SID_FARG count);
    SID_free(SID_FARG n_p_i);
    SID_free(SID_FARG n_p_j);

    SID_log("Done.", SID_LOG_CLOSE);
    SID_Finalize();
}
Exemplo n.º 2
0
void SID_init(int       *argc,
              char     **argv[],
              SID_args   args[],
              void      *mpi_comm_as_void){
  int  status;
  int  i_level;
  int  i_char;
  int  flag_continue;
  int  flag_passed_comm;

  // MPI-specific things
#if USE_MPI
  int      n_keys;
  int      i_key;
  char     key[256];
  char     key_value[256];
  int      key_exists;
  char     nodes_string[256];
  SID_fp   fp_tmp;
  FILE    *fp_hack;
  int      node_name_length;
  MPI_Comm mpi_comm;
#if USE_MPI_IO
  MPI_Info info_disp;
#endif

  if (mpi_comm_as_void == NULL)
  {
    flag_passed_comm = 0;
    MPI_Init(argc,argv);
    MPI_Comm_dup(MPI_COMM_WORLD, &mpi_comm);
  }
  else
  {
    mpi_comm = *((MPI_Comm *) mpi_comm_as_void);
    flag_passed_comm = 1;
  }

  MPI_Comm_size(mpi_comm, &(SID.n_proc));
  MPI_Comm_rank(mpi_comm, &(SID.My_rank));

  SID.My_node =(char *)SID_malloc(SID_MAXLENGTH_PROCESSOR_NAME * sizeof(char));
#if USE_MPI
  MPI_Get_processor_name(SID.My_node, &node_name_length);
#else
  sprintf(SID.My_node,"localhost");
  node_name_length=strlen(SID.My_node);
#endif
  if (node_name_length >= SID_MAXLENGTH_PROCESSOR_NAME-1)
    SID_trap_error("SID_MAXLENGTH_PROCESSOR_NAME needs to be increased",ERROR_LOGIC);

  // Make my_rank=MASTER_RANK the master
  if(SID.My_rank==MASTER_RANK)
    SID.I_am_Master=TRUE;
  else
    SID.I_am_Master=FALSE;

  // Identify the last rank
  if(SID.My_rank==SID.n_proc-1)
    SID.I_am_last_rank=TRUE;
  else
    SID.I_am_last_rank=FALSE;

  #if USE_MPI_IO
  // Fetch collective buffering defaults
  MPI_Info_create(&(SID.file_info));
  if(SID.I_am_Master){
    fp_hack=fopen(".tmp.SID","w+");    
    fclose(fp_hack);
  }
  MPI_Barrier(mpi_comm);
  MPI_File_open(mpi_comm,
                ".tmp.SID",
                MPI_MODE_WRONLY,
                MPI_INFO_NULL,
                &(fp_tmp.fp));
  MPI_File_get_info(fp_tmp.fp,&info_disp);
  MPI_Info_get_nkeys(info_disp,&n_keys);
  for(i_key=0;i_key<n_keys;i_key++){
    MPI_Info_get_nthkey(info_disp,i_key,key);
    MPI_Info_get(info_disp,key,MPI_MAX_INFO_VAL,key_value,&key_exists);
    if(key_exists)
      MPI_Info_set((SID.file_info),key,key_value);
  }
  MPI_File_close(&(fp_tmp.fp));
  if(SID.I_am_Master)
    remove(".tmp.SID");

  // Set user-defined colective buffering optimizations
  sprintf(nodes_string,"%d",MIN(SID.n_proc,N_IO_FILES_MAX));
  MPI_Info_set((SID.file_info),"cb_nodes",            nodes_string);
  MPI_Info_set((SID.file_info),"cb_config_list",      "*:1");
  #endif
#else
  SID.My_rank=MASTER_RANK;
  SID.n_proc =1;
#endif

/*
#if !USE_MPI_IO
    SID.n_groups=SID.n_proc/N_IO_FILES_MAX;
    if(SID.n_proc%N_IO_FILES_MAX) SID.n_groups++;
    SID.My_group=SID.My_rank/N_IO_FILES_MAX;
#endif
*/

  // Set ranks to the left and right
  SID.rank_to_right  =(SID.My_rank+1)%SID.n_proc;
  SID.rank_to_left   = SID.My_rank-1;
  if(SID.rank_to_left<0)
    SID.rank_to_left = SID.n_proc-1;

  // Intitialize log timing information
  SID.time_start_level=(time_t *)SID_malloc(sizeof(time_t)*SID_LOG_MAX_LEVELS);
  SID.time_stop_level =(time_t *)SID_malloc(sizeof(time_t)*SID_LOG_MAX_LEVELS);
  SID.time_total_level=(int    *)SID_malloc(sizeof(int)   *SID_LOG_MAX_LEVELS);
  SID.IO_size         =(double *)SID_malloc(sizeof(double)*SID_LOG_MAX_LEVELS);
  SID.flag_use_timer  =(int    *)SID_malloc(sizeof(int)   *SID_LOG_MAX_LEVELS);
  for(i_level=0;i_level<SID_LOG_MAX_LEVELS;i_level++){
    SID.time_start_level[i_level]=0;
    SID.time_stop_level[i_level] =0;
    SID.time_total_level[i_level]=0;
    SID.IO_size[i_level]         =0.;
    SID.flag_use_timer[i_level]  =FALSE;
  }

  // Initialize other log information
#if USE_MPI
  if(*argc>1)
    SID.fp_in        =fopen((*argv)[1],"r");
  else
    SID.fp_in        =NULL;
#else
  SID.fp_in          =stdin;
#endif
  if (flag_passed_comm)
    SID.fp_log       = NULL;
  else
    SID.fp_log       = stderr;
  SID.level          =0;
  SID.indent         =TRUE;
  SID.awake          =TRUE;
  SID.flag_results_on=FALSE;
  SID.verbosity      =SID_LOG_MAX_LEVELS;

  // Store the name of the binary executable that brought us here
  strcpy(SID.My_binary,(*argv)[0]);
  strip_path(SID.My_binary);

  // Initialize argument information
  if(args!=NULL){
    if((status=SID_parse_args(*argc,*argv,args))>0){
      SID_print_syntax(*argc,*argv,args);
      SID_exit(status);
    }
  }
  else
    SID.args=NULL;

#if USE_MPI_IO
  if(SID.I_am_Master){
    fp_hack=fopen(".tmp.SID","w+");
    fclose(fp_hack);
  }
  MPI_Barrier(mpi_comm);
  SID_fopen(".tmp.SID","w",&fp_tmp);
  MPI_File_get_info(fp_tmp.fp,&info_disp);
  if(SID.I_am_Master){
    fprintf(stdout,"\n");
    fprintf(stdout,"MPI-I/O Configuration:\n");
    fprintf(stdout,"---------------------\n");
    MPI_Info_get_nkeys(info_disp,&n_keys);
    for(i_key=0;i_key<n_keys;i_key++){
      MPI_Info_get_nthkey(info_disp,i_key,key);
      MPI_Info_get(info_disp,key,MPI_MAX_INFO_VAL,key_value,&key_exists);
      if(key_exists)
        fprintf(stdout,"key %2d of %d: {%s}={%s}\n",i_key+1,n_keys,key,key_value);
    }
    fprintf(stdout,"\n");
  }
  SID_fclose(&fp_tmp);
  if(SID.I_am_Master)
    remove(".tmp.SID");
#else
  #if USE_MPI
  if(SID.I_am_Master)
    fprintf(stdout,"MPI-I/O switched off.\n\n");
  #endif
#endif

  // Create private COMM_WORLD
  SID_Comm_init(&(SID.COMM_WORLD));
#if USE_MPI
  MPI_Comm_dup(mpi_comm,                &((SID.COMM_WORLD)->comm));
  MPI_Comm_group((SID.COMM_WORLD)->comm,&((SID.COMM_WORLD)->group));
  MPI_Comm_size(SID.COMM_WORLD->comm,   &((SID.COMM_WORLD)->n_proc));
  MPI_Comm_rank(SID.COMM_WORLD->comm,   &((SID.COMM_WORLD)->My_rank));

  // We have duplicated our duplicate mpi communicator - now we can free the
  // original duplicate
  MPI_Comm_free(&mpi_comm);
#else
  SID.COMM_WORLD->comm   =NULL;
  SID.COMM_WORLD->group  =NULL;
  SID.COMM_WORLD->n_proc =1;
  SID.COMM_WORLD->My_rank=MASTER_RANK;
#endif

  // Start total-run-ime timer
  (void)time(&(SID.time_start));

  // Default max wallclock
  SID.max_wallclock=DEFAULT_MAX_WALLCLOCK_TIME;
}
Exemplo n.º 3
0
int main(int argc, char *argv[]) {
    int    n_search;
    int    i_halo;
    char   filename_in[SID_MAX_FILENAME_LENGTH];
    char   group_text_prefix[4];
    int    n_files;
    int    k_read;
    int    l_read;
    int *  n_particles_i;
    int *  n_particles_j;
    int    j_read;
    int    mode;
    int    j_halo;
    int    i_read;
    int    i_read_start;
    int    i_read_stop;
    SID_fp fp_in;

    SID_Init(&argc, &argv, NULL);

    // Fetch user inputs
    char filename_SSimPL_root[SID_MAX_FILENAME_LENGTH];
    strcpy(filename_SSimPL_root, argv[1]);
    SID_log("Checking the integrity of the match files for {%s}...", SID_LOG_OPEN | SID_LOG_TIMER, filename_SSimPL_root);
    int *n_groups    = NULL;
    int *n_subgroups = NULL;
    for(int i_type = 0; i_type < 2; i_type++) {
        // Convert filename_root to filename
        switch(i_type) {
            case 0:
                mode = MATCH_SUBGROUPS;
                sprintf(group_text_prefix, "sub");
                break;
            case 1:
                mode = MATCH_GROUPS;
                sprintf(group_text_prefix, "");
                break;
        }
        SID_log("Processing %sgroups...", SID_LOG_OPEN | SID_LOG_TIMER, group_text_prefix);

        // Set the standard SSiMPL match file path
        char filename_root_in[SID_MAX_FILENAME_LENGTH];
        sprintf(filename_root_in, "%s/trees/matches/", filename_SSimPL_root);

        // Read halo sizes from header file
        SID_log("Processing header file...", SID_LOG_OPEN | SID_LOG_TIMER);
        sprintf(filename_in, "%s/%sgroup_matches_header.dat", filename_root_in, group_text_prefix);
        SID_fopen(filename_in, "r", &fp_in);
        SID_fread(&i_read_start, sizeof(int), 1, &fp_in);
        SID_fread(&i_read_stop, sizeof(int), 1, &fp_in);
        SID_fread(&n_search, sizeof(int), 1, &fp_in);
        SID_fread(&n_files, sizeof(int), 1, &fp_in);
        int *n_halos = NULL;
        switch(mode) {
            case MATCH_SUBGROUPS:
                n_subgroups = (int *)SID_malloc(sizeof(int) * n_files);
                n_halos     = n_subgroups;
                break;
            case MATCH_GROUPS:
                n_groups = (int *)SID_malloc(sizeof(int) * n_files);
                n_halos  = n_groups;
                break;
        }
        if(mode == MATCH_GROUPS)
            SID_log("Halo counts (snap/No. groups/No. subgroups):", SID_LOG_OPEN);
        for(k_read = 0; k_read < n_files; k_read++) {
            SID_fread(&l_read, sizeof(int), 1, &fp_in);
            SID_fread(&(n_halos[k_read]), sizeof(int), 1, &fp_in);
            SID_fskip(sizeof(int), n_halos[k_read], &fp_in);
            if(mode == MATCH_GROUPS) {
                int *n_subgroups_group = (int *)SID_malloc(sizeof(int) * n_halos[k_read]);
                SID_fread(n_subgroups_group, sizeof(int), n_halos[k_read], &fp_in);
                int n_subgroups_test = 0;
                for(int i_test = 0; i_test < n_halos[k_read]; i_test++)
                    n_subgroups_test += n_subgroups_group[i_test];
                if(n_subgroups[k_read] != n_subgroups_test)
                    SID_log("Error in %s header: l_read=%3d k_read=%3d n_subgroups: %d!=%d\n",
                            SID_LOG_COMMENT,
                            l_read,
                            k_read,
                            n_subgroups[k_read],
                            n_subgroups_test);
                SID_free(SID_FARG n_subgroups_group);
            }
            if(mode == MATCH_GROUPS)
                SID_log("%03d %d %d", SID_LOG_COMMENT, k_read, n_groups[k_read], n_subgroups[k_read]);
        }
        if(mode == MATCH_GROUPS)
            SID_log("", SID_LOG_CLOSE | SID_LOG_NOPRINT);
        SID_fclose(&fp_in);
        SID_log("Done.", SID_LOG_CLOSE);

        SID_log("Processing match files...", SID_LOG_OPEN | SID_LOG_TIMER);
        for(int i_read = i_read_start; i_read < i_read_stop; i_read++) {
            for(int j_read = GBP_MAX(0, i_read - n_search); j_read < GBP_MIN(i_read_stop, i_read + n_search); j_read++) {
                if(i_read != j_read) {
                    sprintf(filename_in, "%s/%03d/%sgroup_matches_%03d_%03d.dat", filename_root_in, i_read, group_text_prefix, i_read, j_read);
                    SID_log("Processing {%s}...", SID_LOG_OPEN, filename_in);

                    // Read header information
                    int i_read_in;
                    int j_read_in;
                    int n_groups_i;
                    int n_groups_j;
                    SID_fopen(filename_in, "r", &fp_in);
                    SID_fread(&i_read_in, sizeof(int), 1, &fp_in);
                    SID_fread(&j_read_in, sizeof(int), 1, &fp_in);
                    SID_fread(&n_groups_i, sizeof(int), 1, &fp_in);
                    SID_fread(&n_groups_j, sizeof(int), 1, &fp_in);

                    if(i_read_in != i_read || j_read_in != j_read || n_groups_i != n_halos[n_files - i_read_in - 1] ||
                       n_groups_j != n_halos[n_files - j_read_in - 1])
                        SID_log("Error in matching file: i_read=%3d j_read=%3d n_i_in=%d n_i=%d n_j_in=%d n_j=%d\n",
                                SID_LOG_COMMENT,
                                i_read,
                                j_read,
                                n_groups_i,
                                n_halos[n_files - i_read_in - 1],
                                n_groups_j,
                                n_halos[n_files - j_read_in - 1]);

                    // Read matches
                    int match;
                    for(k_read = 0; k_read < n_groups_i; k_read++)
                        SID_fread(&match, sizeof(int), 1, &fp_in);

                    // Read indices
                    size_t indices;
                    for(k_read = 0; k_read < n_groups_i; k_read++)
                        SID_fread(&indices, sizeof(size_t), 1, &fp_in);

                    // Read scores
                    float score;
                    for(k_read = 0; k_read < n_groups_i; k_read++)
                        SID_fread(&score, sizeof(float), 1, &fp_in);

                    // Close file
                    SID_fclose(&fp_in);

                    SID_log("Done.", SID_LOG_CLOSE);
                }
            }
        }
        SID_log("Done.", SID_LOG_CLOSE);

        SID_log("Done.", SID_LOG_CLOSE);
    }
    SID_free(SID_FARG n_groups);
    SID_free(SID_FARG n_subgroups);

    SID_log("Done.", SID_LOG_CLOSE);
    SID_Finalize();
}
Exemplo n.º 4
0
int main(int argc, char *argv[]){
  int     n_search;
  int     i_halo;
  char    filename_SSimPL_root[MAX_FILENAME_LENGTH];
  char    filename_in[MAX_FILENAME_LENGTH];
  char    group_text_prefix[4];
  int     n_files;
  int     k_read;
  int     max_n_groups;
  int     l_read;
  int     n_groups;
  int     j_read;
  int     mode;
  int     n_groups_i;
  int     n_groups_j;
  int     j_halo;
  int     match;
  int     i_read;
  int     i_read_start;
  int     i_read_stop;
  SID_fp  fp_in;

  SID_init(&argc,&argv,NULL,NULL);

  // Fetch user inputs
  strcpy(filename_SSimPL_root,argv[1]);
  if(!strcmp(argv[2],"groups") || !strcmp(argv[2],"group"))
     mode=MATCH_GROUPS;
  else if(!strcmp(argv[2],"subgroups") || !strcmp(argv[2],"subgroup"))
     mode=MATCH_SUBGROUPS;
  else{
     SID_log("Invalid mode selection {%s}.  Should be 'group' or 'subgroup'.",SID_LOG_COMMENT,argv[2]);
     SID_exit(ERROR_SYNTAX);
  }
  i_read=atoi(argv[3]);
  j_read=atoi(argv[4]);
  SID_log("Searching match information for halo #%d in file #%d of {%s}...",SID_LOG_OPEN|SID_LOG_TIMER,i_halo,i_read,filename_SSimPL_root);

  // Convert filename_root to filename
  switch(mode){
     case MATCH_SUBGROUPS:
     sprintf(group_text_prefix,"sub");
     break;
     case MATCH_GROUPS:
     sprintf(group_text_prefix,"");
     break;
  }

  // Set the standard SSiMPL match file path
  char filename_root_in[MAX_FILENAME_LENGTH];
  sprintf(filename_root_in,"%s/trees/matches/",filename_SSimPL_root);

  // Set the output file
  char filename_base[MAX_FILENAME_LENGTH];
  char filename_out[MAX_FILENAME_LENGTH];
  sprintf(filename_base,filename_SSimPL_root);
  if(!strcmp(&(filename_base[strlen(filename_base)-1]),"/"))
     strcpy(&(filename_base[strlen(filename_base)-1]),"\0");
  strip_path(filename_base);
  sprintf(filename_out,"%s_%d_%d_2way_matches.txt",filename_base,i_read,j_read);

  // Read header information
  SID_log("Reading header information...",SID_LOG_OPEN);
  sprintf(filename_in,"%s/%sgroup_matches_header.dat",filename_root_in,group_text_prefix);
  SID_fopen(filename_in,"r",&fp_in);
  SID_fread(&i_read_start,sizeof(int),1,&fp_in);SID_log("snap start  =%d",SID_LOG_COMMENT,i_read_start);
  SID_fread(&i_read_stop, sizeof(int),1,&fp_in);SID_log("snap stop   =%d",SID_LOG_COMMENT,i_read_stop);
  SID_fread(&n_search,    sizeof(int),1,&fp_in);SID_log("search range=%d",SID_LOG_COMMENT,n_search);
  SID_fread(&n_files,     sizeof(int),1,&fp_in);SID_log("# of files  =%d",SID_LOG_COMMENT,n_files);
  for(k_read=0,max_n_groups=0;k_read<n_files;k_read++){
     SID_fread(&l_read,  sizeof(int),1,&fp_in);
     SID_fread(&n_groups,sizeof(int),1,&fp_in);
     SID_fseek(&fp_in,   sizeof(int),n_groups,SID_SEEK_CUR);
     if(mode==MATCH_GROUPS)
        SID_fseek(&fp_in,   sizeof(int),n_groups,SID_SEEK_CUR);
     max_n_groups=MAX(max_n_groups,n_groups);
  }
  SID_log("Max # groups=%d",SID_LOG_COMMENT,max_n_groups);
  SID_fclose(&fp_in);
  SID_log("Done.",SID_LOG_CLOSE);

  // Initialize some arrays
  int    *n_particles_i       =(int    *)SID_malloc(sizeof(int)   *max_n_groups);
  int    *n_particles_j       =(int    *)SID_malloc(sizeof(int)   *max_n_groups);
  int    *match_forward_ids   =(int    *)SID_malloc(sizeof(int)   *max_n_groups);
  size_t *match_forward_index =(size_t *)SID_malloc(sizeof(size_t)*max_n_groups);
  float  *match_forward_score =(float  *)SID_malloc(sizeof(float) *max_n_groups);
  char   *match_forward_2way  =(char   *)SID_malloc(sizeof(char)  *max_n_groups);
  int    *match_backward_ids  =(int    *)SID_malloc(sizeof(int)   *max_n_groups);
  size_t *match_backward_index=(size_t *)SID_malloc(sizeof(size_t)*max_n_groups);
  float  *match_backward_score=(float  *)SID_malloc(sizeof(float) *max_n_groups);
  char   *match_backward_2way =(char   *)SID_malloc(sizeof(char)  *max_n_groups);

  // Loop over all matching combinations
  SID_log("Reading forward matches...",SID_LOG_OPEN|SID_LOG_TIMER);
  SID_set_verbosity(SID_SET_VERBOSITY_DEFAULT);
  read_matches(filename_root_in,
               i_read,
               j_read,
               max_n_groups,
               mode,
               &n_groups_i,
               &n_groups_j,
               n_particles_i,
               n_particles_j,
               NULL,
               NULL,
               match_forward_ids,
               match_forward_score,
               match_forward_index,
               match_forward_2way,
               FALSE);
  SID_log("Done.",SID_LOG_CLOSE);
  SID_log("Processing backwards matches...",SID_LOG_OPEN|SID_LOG_TIMER);
  read_matches(filename_root_in,
               j_read,
               i_read,
               max_n_groups,
               mode,
               &n_groups_j,
               &n_groups_i,
               n_particles_j,
               n_particles_i,
               NULL,
               NULL,
               match_backward_ids,
               match_backward_score,
               match_backward_index,
               match_backward_2way,
               FALSE);
  SID_log("Done.",SID_LOG_CLOSE);

  // Open output file
  FILE *fp_out;
  fp_out=fopen(filename_out,"w");
  int i_column=1;
  fprintf(fp_out,"# Column (%02d): Halo index for snapshot %d\n",          i_column++,i_read);
  fprintf(fp_out,"#        (%02d): Halo index for snapshot %d\n",          i_column++,j_read);
  fprintf(fp_out,"#        (%02d): No. particles in snapshot %d\n",        i_column++,i_read);
  fprintf(fp_out,"#        (%02d): No. particles in snapshot %d\n",        i_column++,j_read);
  fprintf(fp_out,"#        (%02d): Forward  match score\n",                i_column++);
  fprintf(fp_out,"#        (%02d): Forward  match score/min match score\n",i_column++);
  fprintf(fp_out,"#        (%02d): Backward match score\n",                i_column++);
  fprintf(fp_out,"#        (%02d): Backward match score/min match score\n",i_column++);
  for(int i_halo=0;i_halo<n_groups_i;i_halo++){
     int j_halo=match_forward_ids[i_halo];
     if(match_forward_2way[i_halo]){
        if(j_halo<0 || j_halo>n_groups_j)
           SID_trap_error("There's an invalid match id (ie. %d<0 || %d>%d)  attached to a 2-way match!",ERROR_LOGIC,j_halo,j_halo,n_groups_j);
        fprintf(fp_out,"%7d %7d %6d %6d %10.3le %10.3le %10.3le %10.3le\n",
                i_halo,
                j_halo,
                n_particles_i[i_halo],
                n_particles_j[j_halo],
                match_forward_score[i_halo],
                match_forward_score[i_halo]/minimum_match_score((double)n_particles_i[i_halo]),
                match_backward_score[j_halo],
                match_backward_score[j_halo]/minimum_match_score((double)n_particles_j[j_halo]));
     }                
  }
  fclose(fp_out);

  // Clean-up
  SID_free(SID_FARG n_particles_i);
  SID_free(SID_FARG n_particles_j);
  SID_free(SID_FARG match_forward_ids);
  SID_free(SID_FARG match_forward_index);
  SID_free(SID_FARG match_forward_score);
  SID_free(SID_FARG match_forward_2way);
  SID_free(SID_FARG match_backward_ids);
  SID_free(SID_FARG match_backward_index);
  SID_free(SID_FARG match_backward_score);
  SID_free(SID_FARG match_backward_2way);
  
  SID_log("Done.",SID_LOG_CLOSE);
  SID_exit(ERROR_NONE);
}
Exemplo n.º 5
0
void read_matches(char    *filename_in_dir,
                  int      i_read_in,
                  int      j_read_in,
                  int      n_halos_max,
                  int      mode,
                  int     *n_groups_i,
                  int     *n_groups_j,
                  int     *n_particles_i_in,
                  int     *n_particles_j_in,
                  int     *n_sub_group_i_in,
                  int     *n_sub_group_j_in,
                  int     *match_ids,
                  float   *match_score,
                  size_t  *match_index,
                  char    *match_flag_two_way,
                  int      flag_reject_bad_matches){
   char   group_text_prefix[5];
   char   filename_in[MAX_FILENAME_LENGTH];
   SID_fp fp_in;
   int    k_read;
   int    l_read;
   int    n_search;
   int    n_matches;
   size_t offset;
   int    flag_continue;
   int    i_read_file;
   int    j_read_file;
   int    n_groups_file;
   int    n_groups_file_1;
   int    n_groups_file_2;
   int    n_groups;
   int    n_groups_i_file;
   int    n_groups_j_file;
   int   *n_sub_group_i;
   int   *n_sub_group_j;
   int    flag_alloc_n_sub_i=FALSE;
   int    flag_alloc_n_sub_j=FALSE;
   
   if(i_read_in==j_read_in)
     SID_trap_error("i_read=j_read in read_matches",ERROR_LOGIC);

   switch(mode){
      case MATCH_SUBGROUPS:
      sprintf(group_text_prefix,"sub");
      break;
      case MATCH_GROUPS:
      sprintf(group_text_prefix,"");
      // We need n_subgroups arrays for removal of bad groups
      //    if they have not been passed to us.
      if(n_sub_group_i_in==NULL){
         flag_alloc_n_sub_i=TRUE;
         n_sub_group_i     =(int *)SID_malloc(sizeof(int)*n_halos_max);
      }
      else
         n_sub_group_i=n_sub_group_i_in;
      if(n_sub_group_j_in==NULL){
         flag_alloc_n_sub_j=TRUE;
         n_sub_group_j     =(int *)SID_malloc(sizeof(int)*n_halos_max);
      }
      else
         n_sub_group_j=n_sub_group_j_in;
      break;
   }

   // Since we need the particle counts for the goodness of match criterion,
   //   create temporary arrays in case we weren't passed an array for them.
   int *n_particles_i=n_particles_i_in;
   int *n_particles_j=n_particles_j_in;
   // 1) We always need n_particles_i
   int  flag_alloc_n_particles_i=FALSE;
   int  flag_alloc_n_particles_j=FALSE;
   if(n_particles_i==NULL)
      flag_alloc_n_particles_i=TRUE;
   if(n_particles_j==NULL)
      flag_alloc_n_particles_j=TRUE;
   // 2) We need n_particles_j if doing 2-way checks
   if(match_flag_two_way!=NULL && n_particles_j==NULL)
      flag_alloc_n_particles_j=TRUE;

   // Read the needed info from the header file
   int  i_read;
   int  i_read_start;
   int  i_read_stop;
   int  n_search_total;
   int  n_files;
   int  n_groups_in;
   int  counter=0;
   char filename_in_name[256];
   strcpy(filename_in_name,filename_in_dir);
   strip_path(filename_in_name);
   sprintf(filename_in,"%s/%sgroup_matches_header.dat",filename_in_dir,group_text_prefix);
   SID_fopen(filename_in,"r",&fp_in);
   SID_fread(&i_read_start,  sizeof(int),1,&fp_in);
   SID_fread(&i_read_stop,   sizeof(int),1,&fp_in);
   SID_fread(&n_search_total,sizeof(int),1,&fp_in);
   SID_fread(&n_files,       sizeof(int),1,&fp_in);
   for(i_read=i_read_stop;i_read>=i_read_start && counter<2;i_read--){
      SID_fread(&i_read_file, sizeof(int),1,&fp_in);
      if(i_read_file==i_read_in){
         SID_fread(n_groups_i,sizeof(int),1,&fp_in);
         if((*n_groups_i)>0){
            // Create a temporary array for n_particles_i if we were not passed one
            if(flag_alloc_n_particles_i)
               n_particles_i=(int *)SID_malloc(sizeof(int)*(*n_groups_i));
            if(n_particles_i!=NULL)
               SID_fread_ordered(n_particles_i,sizeof(int),(size_t)(*n_groups_i),&fp_in);
            else
               SID_fskip(sizeof(int),(*n_groups_i),&fp_in);
            if(mode==MATCH_GROUPS){
               if(n_sub_group_i!=NULL)
                  SID_fread_ordered(n_sub_group_i,sizeof(int),(size_t)(*n_groups_i),&fp_in);
               else
                  SID_fskip(sizeof(int),(*n_groups_i),&fp_in);
            }
         }
         counter++;
      }
      else if(i_read_file==j_read_in){
         SID_fread(n_groups_j,sizeof(int),1,&fp_in);
         if((*n_groups_j)>0){
            if(flag_alloc_n_particles_j)
               n_particles_j=(int *)SID_malloc(sizeof(int)*(*n_groups_j));
            if(n_particles_j!=NULL)
               SID_fread_ordered(n_particles_j,sizeof(int),(size_t)(*n_groups_j),&fp_in);
            else
               SID_fskip(sizeof(int),(*n_groups_j),&fp_in);
            if(mode==MATCH_GROUPS){
               if(n_sub_group_j!=NULL)
                  SID_fread_ordered(n_sub_group_j,sizeof(int),(size_t)(*n_groups_j),&fp_in);
               else
                  SID_fskip(sizeof(int),(*n_groups_j),&fp_in);
            }
         }
         counter++;
      }
      else{
         SID_fread(&n_groups_in,sizeof(int),1,&fp_in);
         if(n_groups_in>0){
            SID_fskip(sizeof(int),n_groups_in,&fp_in);
            if(mode==MATCH_GROUPS)
               SID_fskip(sizeof(int),n_groups_in,&fp_in);
         }
      }
   }
   SID_fclose(&fp_in);

   // Read the matching file
   char filename_cat1[256];
   char filename_cat2[256];
   char filename_in_dir_snap[256];
   sprintf(filename_cat1,"%03d",i_read_in);
   sprintf(filename_cat2,"%03d",j_read_in);
   sprintf(filename_in_dir_snap,"%s/%s",filename_in_dir,filename_cat1);
   if(filename_in_dir!=NULL)
      sprintf(filename_in,"%s/%sgroup_matches_%s_%s.dat",filename_in_dir_snap,group_text_prefix,filename_cat1,filename_cat2);
   else
      sprintf(filename_in,"%s_%sgroup_matches_%s_%s.dat",filename_in_name,    group_text_prefix,filename_cat1,filename_cat2);

   SID_fopen(filename_in,"r",&fp_in);
   SID_fread(&i_read_file,sizeof(int),1,&fp_in);
   SID_fread(&j_read_file,sizeof(int),1,&fp_in);
   SID_fread(n_groups_i,  sizeof(int),1,&fp_in);
   SID_fread(n_groups_j,  sizeof(int),1,&fp_in);

   // Read matching data
   SID_fread(match_ids,  sizeof(int),   (*n_groups_i),&fp_in);
   SID_fread(match_index,sizeof(size_t),(*n_groups_i),&fp_in);
   SID_fread(match_score,sizeof(float), (*n_groups_i),&fp_in);
   SID_fclose(&fp_in);

   // If one of the catalogs is empty, set to no-match defaults
   if((*n_groups_i)<=0 || (*n_groups_j)<=0){
      for(int i_halo=0;i_halo<(*n_groups_i);i_halo++){
         match_ids[i_halo]  =-1;
         match_score[i_halo]= 0.;
      }
      if(match_flag_two_way!=NULL){
         for(int i_halo=0;i_halo<(*n_groups_i);i_halo++)
            match_flag_two_way[i_halo]=FALSE;
      }
   }
   else{
      // If we are reading groups, nullify all matches
      //    between halos with no substructures.
      int i_halo;
      if(mode==MATCH_GROUPS){
         size_t *match_index_temp;
         for(i_halo=0;i_halo<(*n_groups_i);i_halo++){
            if(n_sub_group_i[i_halo]<=0){
               match_ids[i_halo]  =-1;
               match_score[i_halo]= 0.;
            }
            else if(match_ids[i_halo]>=0 && (*n_groups_j)>0){
               if(n_sub_group_j[match_ids[i_halo]]<=0){
                  match_ids[i_halo]  =-1;
                  match_score[i_halo]= 0.;
               }
            }
         }
         merge_sort(match_ids,(size_t)(*n_groups_i),&match_index_temp,SID_INT,SORT_COMPUTE_INDEX,SORT_COMPUTE_NOT_INPLACE);
         memcpy(match_index,match_index_temp,(*n_groups_i)*sizeof(size_t));
         SID_free(SID_FARG match_index_temp);
      }

      // Apply a goodness-of-fit criterion and check that the maximum allowed score has not been exceeded
      for(i_halo=0;i_halo<(*n_groups_i);i_halo++){
         if(match_ids[i_halo]>=0){
            if(flag_reject_bad_matches && !check_validity_of_match(n_particles_i[i_halo],match_score[i_halo]))
               match_ids[i_halo]=-1;
         }
      }

      // Since we may have changed some matches with the goodness 
      //    of fit criterion, we need to re-perform the sort
      size_t *match_index_temp=NULL;
      merge_sort(match_ids,(size_t)(*n_groups_i),&match_index_temp,SID_INT,SORT_COMPUTE_INDEX,SORT_COMPUTE_NOT_INPLACE);
      memcpy(match_index,match_index_temp,(*n_groups_i)*sizeof(size_t));
      SID_free(SID_FARG match_index_temp);

      // Determine if the matches are two-way if we have been asked to check this
      if(match_flag_two_way!=NULL && (*n_groups_i)>0){

         // We're going to need the particle counts in the target catalog for checking the goodness of return matches.  Make sure we have them.
         if(n_particles_j==NULL)
            SID_trap_error("Target catalog halo sizes are not defined in read_matches() but are needed for checking two-way match flags.",ERROR_LOGIC);

         // Flip the file names for reading the return matches
         sprintf(filename_in_dir_snap,"%s/%s",filename_in_dir,filename_cat2);
         if(filename_in_dir!=NULL)
            sprintf(filename_in,"%s/%sgroup_matches_%s_%s.dat",filename_in_dir_snap,group_text_prefix,filename_cat2,filename_cat1);
         else
            sprintf(filename_in,"%s_%sgroup_matches_%s_%s.dat",filename_in_name,    group_text_prefix,filename_cat2,filename_cat1);

         // Open two files, one for reading the IDs and one for matching the scores of the matching file
         int i_read_file_check;
         int j_read_file_check;
         int n_groups_i_check;
         int n_groups_j_check;
         SID_fp fp_check_ids;
         SID_fp fp_check_score;
         SID_fopen(filename_in,"r",&fp_check_ids);
         SID_fopen(filename_in,"r",&fp_check_score);
         SID_fread(&j_read_file_check,sizeof(int),1,&fp_check_ids);
         SID_fread(&i_read_file_check,sizeof(int),1,&fp_check_ids);
         SID_fread(&n_groups_j_check, sizeof(int),1,&fp_check_ids);
         SID_fread(&n_groups_i_check, sizeof(int),1,&fp_check_ids);

         // Check that we have the right files
         if(n_groups_i_check!=(*n_groups_i))
            SID_trap_error("Source halo counts don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,n_groups_i_check,(*n_groups_i));
         if(n_groups_j_check!=(*n_groups_j))
            SID_trap_error("Target halo counts don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,n_groups_j_check,(*n_groups_j));
         if(i_read_file_check!=i_read_file)
            SID_trap_error("Source file numbers don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,i_read_file_check,i_read_file);
         if(j_read_file_check!=j_read_file)
            SID_trap_error("Target file numbers don't match (ie. %d!=%d) in two-way check in read_matches().",ERROR_LOGIC,j_read_file_check,j_read_file);

         // Skip to the beginning of the relevant block for the score-reading file pointer
         SID_fskip(sizeof(int),   4,            &fp_check_score); // header
         SID_fskip(sizeof(int),   (*n_groups_j),&fp_check_score); // ids
         SID_fskip(sizeof(size_t),(*n_groups_j),&fp_check_score); // indices

         // Set everything to being a one-way match unless subsequently changed
         for(i_halo=0;i_halo<(*n_groups_i);i_halo++) 
            match_flag_two_way[i_halo]=FALSE;

         // Read matching data in buffered chunks 
         int     n_good      =0;
         int     n_2way      =0;
         int     n_buffer    =1024*1024;
         int     n_chunk     =0;
         int     n_remaining =(*n_groups_j);
         int    *buffer_ids  =(int   *)SID_malloc(sizeof(int)  *n_buffer);
         float  *buffer_score=(float *)SID_malloc(sizeof(float)*n_buffer);
         for(int j_halo=0;n_remaining>0;n_remaining-=n_chunk){
            n_chunk=MIN(n_remaining,n_buffer);
            SID_fread(buffer_ids,  sizeof(int),  n_chunk,&fp_check_ids);
            SID_fread(buffer_score,sizeof(float),n_chunk,&fp_check_score);
            for(int k_halo=0;k_halo<n_chunk;k_halo++,j_halo++){
               int id_i=buffer_ids[k_halo];
               if(id_i>=0){
                  if(id_i>=(*n_groups_i))
                     SID_trap_error("Allowed matching index has been exceeded i(ie %d>=%d) while determining two-way match flags.",
                                    ERROR_LOGIC,id_i,(*n_groups_i));
                  // Do this check first to avoid having to check if both needed n_particles_* references are defined
                  int id_j=match_ids[id_i];
                  if(id_j==j_halo){
                     if(!flag_reject_bad_matches || check_validity_of_match(n_particles_j[j_halo],buffer_score[k_halo])){
                        match_flag_two_way[id_i]=TRUE; 
                        n_2way++;
                     }
                     n_good++;
                  }
               }
            }
         }
         //SID_log("n_good=%d n_2way=%d",SID_LOG_COMMENT,n_good,n_2way);
         SID_fclose(&fp_check_ids);
         SID_fclose(&fp_check_score);
         SID_free(SID_FARG buffer_ids);
         SID_free(SID_FARG buffer_score);
      }
   }

   // If any of these arrays are temporary, free them.
   if(flag_alloc_n_sub_i)
      SID_free(SID_FARG n_sub_group_i);
   if(flag_alloc_n_sub_j)
      SID_free(SID_FARG n_sub_group_j);
   if(flag_alloc_n_particles_i)
      SID_free(SID_FARG n_particles_i);
   if(flag_alloc_n_particles_j)
      SID_free(SID_FARG n_particles_j);
}
Exemplo n.º 6
0
int main(int argc, char *argv[]) {
    int     n_search;
    int     n_files;
    int     k_read;
    int     max_n_groups;
    int     l_read;
    int *   n_particles_i;
    int *   n_particles_j;
    int     n_groups_i;
    int     n_groups_j;
    int *   match_ids;
    float * match_score;
    size_t *match_index;
    int     j_halo;
    int     match;
    int     i_read;
    int     i_read_start;
    int     i_read_stop;
    SID_fp  fp_in;

    SID_Init(&argc, &argv, NULL);

    // Fetch user inputs
    if(argc != 12)
        SID_exit_error("Invalid Syntax.", SID_ERROR_SYNTAX);
    char filename_SSimPL_root[SID_MAX_FILENAME_LENGTH];
    char filename_halos_root[SID_MAX_FILENAME_LENGTH];
    char filename_trees_root[SID_MAX_FILENAME_LENGTH];
    char filename_out_root[SID_MAX_FILENAME_LENGTH];
    strcpy(filename_SSimPL_root, argv[1]);
    strcpy(filename_halos_root, argv[2]);
    strcpy(filename_trees_root, argv[3]);
    double x_cen    = (double)atof(argv[4]);
    double y_cen    = (double)atof(argv[5]);
    double z_cen    = (double)atof(argv[6]);
    double radius   = (double)atof(argv[7]);
    double z_min_in = (double)atof(argv[8]);
    double z_max_in = (double)atof(argv[9]);
    double M_min    = (double)atof(argv[10]);
    strcpy(filename_out_root, argv[11]);
    double radius2 = radius * radius;

    SID_log("Query trees for sphere (x,y,z,r)=(%.2lf,%.2lf,%.2lf,%.2lf) between z=%.2lf and z=%.2lf...",
            SID_LOG_OPEN,
            x_cen,
            y_cen,
            z_cen,
            radius,
            z_min_in,
            z_max_in);

    char filename_catalog_root[SID_MAX_FILENAME_LENGTH];
    sprintf(filename_catalog_root, "%s/catalogs/%s", filename_SSimPL_root, filename_halos_root);

    // Read tree header information
    tree_info *trees;
    char       filename_file_root[SID_MAX_FILENAME_LENGTH];
    sprintf(filename_file_root, "%s/trees/%s", filename_SSimPL_root, filename_trees_root);
    SID_set_verbosity(SID_SET_VERBOSITY_RELATIVE, 0);
    init_trees_read(filename_SSimPL_root, filename_halos_root, filename_trees_root, TREE_READ_HEADER_ONLY, &trees);
    SID_set_verbosity(SID_SET_VERBOSITY_DEFAULT);

    // Turn given redshift range into snapshot range
    int i_snap_min_z = find_treesnap_z(trees, z_max_in);
    int i_snap_max_z = find_treesnap_z(trees, z_min_in);
    SID_log("z=%.2lf -> snapshot=%d", SID_LOG_COMMENT, z_min_in, trees->snap_list[i_snap_max_z]);
    SID_log("z=%.2lf -> snapshot=%d", SID_LOG_COMMENT, z_max_in, trees->snap_list[i_snap_min_z]);

    // Perform query
    int  n_groups_list    = 0;
    int  n_subgroups_list = 0;
    int *group_list       = NULL;
    int *subgroup_list    = NULL;
    for(int i_pass = 0; i_pass < 3; i_pass++) {
        if(i_pass == 0)
            SID_log("Counting halos to be queried...", SID_LOG_OPEN | SID_LOG_TIMER);
        else if(i_pass == 1)
            SID_log("Identifying halos to be queried...", SID_LOG_OPEN | SID_LOG_TIMER);
        else if(i_pass == 2)
            SID_log("Performing query...", SID_LOG_OPEN | SID_LOG_TIMER);
        // Write headers
        if(i_pass == 2) {
            for(int i_type = 0; i_type < 2; i_type++) {
                int  n_list    = 0;
                int *halo_list = NULL;
                if(i_type == 0) {
                    n_list    = n_groups_list;
                    halo_list = group_list;
                } else {
                    n_list    = n_subgroups_list;
                    halo_list = subgroup_list;
                }
                for(int i_list = 0; i_list < n_list; i_list++) {
                    int  i_column = 1;
                    char filename_out[SID_MAX_FILENAME_LENGTH];
                    if(i_type == 0)
                        sprintf(filename_out, "%s_group_%09d.txt", filename_out_root, halo_list[i_list]);
                    else
                        sprintf(filename_out, "%s_subgroup_%09d.txt", filename_out_root, halo_list[i_list]);
                    FILE *fp_out = fopen(filename_out, "w");
                    fprintf(fp_out, "# Column (%02d): Halo expansion factor\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo redshift\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo snapshot\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo index\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo ID\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo log10(M_vir [M_sol/h])\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo n_particles\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo n_particles_peak\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo x [Mpc/h])\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo y [Mpc/h])\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo z [Mpc/h])\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo radius [Mpc/h])\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo tree ID\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Descendant file offset\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Descendant snapshot\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Descendant index\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Descendant ID\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Bridge forematch snapshot\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Bridge forematch index\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Bridge backmatch snapshot\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Bridge backmatch index\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo type\n", i_column++);
                    fprintf(fp_out, "#        (%02d): Halo type string\n", i_column++);
                    if(i_type == 1) {
                        fprintf(fp_out, "#        (%02d): Group index\n", i_column++);
                        fprintf(fp_out, "#        (%02d): Group ID\n", i_column++);
                        fprintf(fp_out, "#        (%02d): Subgroup index\n", i_column++);
                        fprintf(fp_out, "#        (%02d): FoF Centre dx [Mpc/h])\n", i_column++);
                        fprintf(fp_out, "#        (%02d): FoF Centre dy [Mpc/h])\n", i_column++);
                        fprintf(fp_out, "#        (%02d): FoF Centre dz [Mpc/h])\n", i_column++);
                    }
                    fclose(fp_out);
                }
            }
        }

        // Set the snapshot range we need to scan
        int i_snap_start;
        int i_snap_stop;
        if(i_pass < 2) {
            i_snap_start = i_snap_min_z;
            i_snap_stop  = i_snap_max_z;
        } else {
            i_snap_start = 0;
            i_snap_stop  = trees->n_snaps - 1;
        }

        // Loop over the range of snapshots
        int i_list = 0;
        for(int i_snap = i_snap_start; i_snap <= i_snap_stop; i_snap++) {
            // Get the snapshot
            int snapshot = trees->snap_list[i_snap];
            if(i_pass == 2)
                SID_log("Processing snapshot %03d...", SID_LOG_OPEN, snapshot);

            // Open properties for this snapshot
            fp_catalog_info      fp_properties_groups;
            fp_catalog_info      fp_properties_subgroups;
            halo_properties_info properties_groups;
            halo_properties_info properties_subgroups;
            fopen_catalog(filename_catalog_root, snapshot, READ_CATALOG_GROUPS | READ_CATALOG_PROPERTIES, &fp_properties_groups);
            fopen_catalog(filename_catalog_root, snapshot, READ_CATALOG_SUBGROUPS | READ_CATALOG_PROPERTIES, &fp_properties_subgroups);

            // Open horizontal trees for this snapshot
            SID_fp fp_in_trees;
            SID_fp fp_in_bridge_forematch;
            SID_fp fp_in_bridge_backmatch;
            char   filename_in[SID_MAX_FILENAME_LENGTH];
            sprintf(filename_in, "%s/trees/%s/horizontal/trees/horizontal_trees_%03d.dat", filename_SSimPL_root, filename_trees_root, snapshot);
            SID_fopen(filename_in, "r", &fp_in_trees);
            sprintf(filename_in,
                    "%s/trees/%s/horizontal/trees/horizontal_trees_forematch_pointers_%03d.dat",
                    filename_SSimPL_root,
                    filename_trees_root,
                    snapshot);
            SID_fopen(filename_in, "r", &fp_in_bridge_forematch);
            sprintf(filename_in,
                    "%s/trees/%s/horizontal/trees/horizontal_trees_backmatch_pointers_%03d.dat",
                    filename_SSimPL_root,
                    filename_trees_root,
                    snapshot);
            SID_fopen(filename_in, "r", &fp_in_bridge_backmatch);

            // Read trees header
            int n_step_in;
            int n_search_in;
            int n_groups;
            int n_subgroups;
            int n_groups_max_in;
            int n_subgroups_max_in;
            int n_trees_subgroup_in;
            int n_trees_group_in;
            SID_fread_all(&n_step_in, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_search_in, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_groups, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_subgroups, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_groups_max_in, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_subgroups_max_in, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_trees_subgroup_in, sizeof(int), 1, &fp_in_trees);
            SID_fread_all(&n_trees_group_in, sizeof(int), 1, &fp_in_trees);
            SID_fskip(sizeof(int), 8, &fp_in_bridge_forematch);
            SID_fskip(sizeof(int), 8, &fp_in_bridge_backmatch);

            // Scan through the trees
            int i_subgroup = 0;
            for(int i_group = 0; i_group < n_groups; i_group++) {
                // Read group
                int   group_id;
                int   group_type;
                int   group_descendant_id;
                int   group_tree_id;
                int   group_file_offset;
                int   group_index;
                int   group_n_particles_peak;
                int   n_subgroups_group;
                int   group_forematch_id;
                int   group_forematch_first_file;
                int   group_forematch_first_index;
                float group_forematch_first_score;
                int   group_forematch_default_file;
                int   group_forematch_default_index;
                float group_forematch_default_score;
                int   group_forematch_best_file;
                int   group_forematch_best_index;
                float group_forematch_best_score;
                float group_forematch_score_prog;
                int   group_backmatch_id;
                int   group_backmatch_file;
                int   group_backmatch_index;
                float group_backmatch_score;
                float group_backmatch_score_prog;
                SID_fread_all(&group_id, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_type, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_descendant_id, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_tree_id, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_file_offset, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_index, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_n_particles_peak, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&n_subgroups_group, sizeof(int), 1, &fp_in_trees);
                SID_fread_all(&group_forematch_id, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_first_file, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_first_index, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_first_score, sizeof(float), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_default_file, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_default_index, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_default_score, sizeof(float), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_best_file, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_best_index, sizeof(int), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_best_score, sizeof(float), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_forematch_score_prog, sizeof(float), 1, &fp_in_bridge_forematch);
                SID_fread_all(&group_backmatch_id, sizeof(int), 1, &fp_in_bridge_backmatch);
                SID_fread_all(&group_backmatch_file, sizeof(int), 1, &fp_in_bridge_backmatch);
                SID_fread_all(&group_backmatch_index, sizeof(int), 1, &fp_in_bridge_backmatch);
                SID_fread_all(&group_backmatch_score, sizeof(float), 1, &fp_in_bridge_backmatch);
                SID_fread_all(&group_backmatch_score_prog, sizeof(float), 1, &fp_in_bridge_backmatch);
                SID_fskip(sizeof(int), 1, &fp_in_bridge_forematch); // skip subhalo count
                SID_fskip(sizeof(int), 1, &fp_in_bridge_backmatch); // skip subhalo count
                fread_catalog_file(&fp_properties_groups, NULL, NULL, &properties_groups, NULL, i_group);

                // Process group
                process_local(trees,
                              i_pass,
                              filename_out_root,
                              radius2,
                              M_min,
                              i_snap,
                              i_group,
                              0,
                              i_group,
                              group_id,
                              group_file_offset,
                              group_type,
                              group_n_particles_peak,
                              group_tree_id,
                              group_index,
                              group_descendant_id,
                              group_forematch_first_index,
                              group_forematch_first_file,
                              group_backmatch_index,
                              group_backmatch_file,
                              group_id,
                              &properties_groups,
                              NULL,
                              &n_groups_list,
                              group_list,
                              x_cen,
                              y_cen,
                              z_cen);

                for(int j_subgroup = 0; j_subgroup < n_subgroups_group; i_subgroup++, j_subgroup++) {
                    // Read subgroup
                    int   subgroup_id;
                    int   subgroup_type;
                    int   subgroup_descendant_id;
                    int   subgroup_tree_id;
                    int   subgroup_file_offset;
                    int   subgroup_index;
                    int   subgroup_n_particles_peak;
                    int   subgroup_forematch_id;
                    int   subgroup_forematch_first_file;
                    int   subgroup_forematch_first_index;
                    float subgroup_forematch_first_score;
                    int   subgroup_forematch_default_file;
                    int   subgroup_forematch_default_index;
                    float subgroup_forematch_default_score;
                    int   subgroup_forematch_best_file;
                    int   subgroup_forematch_best_index;
                    float subgroup_forematch_best_score;
                    float subgroup_forematch_score_prog;
                    int   subgroup_backmatch_id;
                    int   subgroup_backmatch_file;
                    int   subgroup_backmatch_index;
                    float subgroup_backmatch_score;
                    float subgroup_backmatch_score_prog;
                    SID_fread_all(&subgroup_id, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_type, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_descendant_id, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_tree_id, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_file_offset, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_index, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_n_particles_peak, sizeof(int), 1, &fp_in_trees);
                    SID_fread_all(&subgroup_forematch_id, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_first_file, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_first_index, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_first_score, sizeof(float), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_default_file, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_default_index, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_default_score, sizeof(float), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_best_file, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_best_index, sizeof(int), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_best_score, sizeof(float), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_forematch_score_prog, sizeof(float), 1, &fp_in_bridge_forematch);
                    SID_fread_all(&subgroup_backmatch_id, sizeof(int), 1, &fp_in_bridge_backmatch);
                    SID_fread_all(&subgroup_backmatch_file, sizeof(int), 1, &fp_in_bridge_backmatch);
                    SID_fread_all(&subgroup_backmatch_index, sizeof(int), 1, &fp_in_bridge_backmatch);
                    SID_fread_all(&subgroup_backmatch_score, sizeof(float), 1, &fp_in_bridge_backmatch);
                    SID_fread_all(&subgroup_backmatch_score_prog, sizeof(float), 1, &fp_in_bridge_backmatch);
                    fread_catalog_file(&fp_properties_subgroups, NULL, NULL, &properties_subgroups, NULL, i_subgroup);

                    // Process subgroup
                    process_local(trees,
                                  i_pass,
                                  filename_out_root,
                                  radius2,
                                  M_min,
                                  i_snap,
                                  i_subgroup,
                                  j_subgroup,
                                  i_group,
                                  subgroup_id,
                                  subgroup_file_offset,
                                  subgroup_type,
                                  subgroup_n_particles_peak,
                                  subgroup_tree_id,
                                  subgroup_index,
                                  subgroup_descendant_id,
                                  subgroup_forematch_first_index,
                                  subgroup_forematch_first_file,
                                  subgroup_backmatch_index,
                                  subgroup_backmatch_file,
                                  group_id,
                                  &properties_subgroups,
                                  &properties_groups,
                                  &n_subgroups_list,
                                  subgroup_list,
                                  x_cen,
                                  y_cen,
                                  z_cen);
                }
            }
            fclose_catalog(&fp_properties_groups);
            fclose_catalog(&fp_properties_subgroups);
            SID_fclose(&fp_in_trees);
            SID_fclose(&fp_in_bridge_forematch);
            SID_fclose(&fp_in_bridge_backmatch);
            if(i_pass == 2)
                SID_log("Done.", SID_LOG_CLOSE);
        } // i_snap
        if(i_pass == 0) {
            group_list       = (int *)SID_malloc(sizeof(int) * n_groups_list);
            subgroup_list    = (int *)SID_malloc(sizeof(int) * n_subgroups_list);
            n_groups_list    = 0;
            n_subgroups_list = 0;
        } else if(i_pass == 1) {
            SID_log("(%d groups and %d subgroups found)...", SID_LOG_CONTINUE, n_groups_list, n_subgroups_list);
            // Write list files
            for(int i_type = 0; i_type < 2; i_type++) {
                char filename_out[SID_MAX_FILENAME_LENGTH];
                int  n_list    = 0;
                int *halo_list = NULL;
                if(i_type == 0) {
                    sprintf(filename_out, "%s_group_list.txt", filename_out_root);
                    n_list    = n_groups_list;
                    halo_list = group_list;
                } else {
                    sprintf(filename_out, "%s_subgroup_list.txt", filename_out_root);
                    n_list    = n_subgroups_list;
                    halo_list = subgroup_list;
                }
                FILE *fp_out = fopen(filename_out, "w");
                fprintf(fp_out, "# Halo IDs in list of tree tracks with base {%s}\n", filename_out_root);
                for(int i_list = 0; i_list < n_list; i_list++)
                    fprintf(fp_out, "%d\n", halo_list[i_list]);
                fclose(fp_out);
            }
        }
        SID_log("Done.", SID_LOG_CLOSE);
    }

    // Clean-up
    SID_free(SID_FARG group_list);
    SID_free(SID_FARG subgroup_list);
    free_trees(&trees);

    SID_log("Done.", SID_LOG_CLOSE);
    SID_Finalize();
}
Exemplo n.º 7
0
size_t SID_fread_chunked_ordered(void   *buffer,
                                 size_t  n_x_read_local,
                                 SID_fp *fp){
  int    i_chunk;
  int    i_group;
  int    i_rank;
  size_t i_x_offset_local;
  size_t n_x_read_local_bcast;
  size_t i_x_read_chunk;
  size_t i_x_offset_global;
  size_t i_x_chunk;
  size_t n_skip;
  size_t n_x_chunk;
  size_t n_x_chunk_max;
  size_t header_size;
  size_t r_val=0;
  char   filename_chunk[256];

  // Operates the same way as SID_fread_chunked() except i_x_offset_local is 
  //   computed here under the assumption that reads are done in order by rank
  i_x_offset_local=0;
#if USE_MPI
  for(i_rank=0;i_rank<SID.n_proc;i_rank++){
    n_x_read_local_bcast=n_x_read_local;
    SID_Bcast(&n_x_read_local_bcast,sizeof(size_t),i_rank,SID.COMM_WORLD);
    if(i_rank<SID.My_rank)
      i_x_offset_local+=n_x_read_local_bcast;
  }
#endif

  i_x_offset_global=fp->last_item;
  for(i_chunk=0,i_x_read_chunk=0,i_x_chunk=i_x_offset_local+i_x_offset_global;
      i_chunk<fp->chunked_header.n_chunk;
      i_chunk++){
    if(fp->i_x_start_chunk[i_chunk]<=i_x_chunk && fp->i_x_last_chunk[i_chunk]>=i_x_chunk){
      n_skip   =i_x_chunk-fp->i_x_start_chunk[i_chunk];
      n_x_chunk=MIN(n_x_read_local-i_x_read_chunk,fp->i_x_step_chunk[i_chunk]-n_skip);
    }
    else{
      n_x_chunk=0;
      n_skip   =0;
    }
    SID_Allreduce(&n_x_chunk,&n_x_chunk_max,1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD);
    if(n_x_chunk_max>0){
      sprintf(filename_chunk,"%s.%d",fp->filename_root,i_chunk);
#if !USE_MPI_IO
      for(i_group=0;i_group<SID.n_proc;i_group++){
        if(SID.My_group==i_group && n_x_chunk>0){
#endif
          SID_fopen(filename_chunk,"r",fp);
          if(n_x_chunk>0){
            SID_fseek(fp,
                      1,
                      fp->header_offset[i_chunk]+n_skip*fp->chunked_header.item_size,
                      SID_SEEK_SET);
            SID_fread((char *)buffer+i_x_read_chunk*fp->chunked_header.item_size,
                      fp->chunked_header.item_size,
                      n_x_chunk,
                      fp);
            i_x_chunk     +=n_x_chunk;
            i_x_read_chunk+=n_x_chunk;
          }
          SID_fclose(fp);
#if !USE_MPI_IO
        }
        SID_Barrier(SID.COMM_WORLD);
      }
#endif
    }
  }
  SID_Allreduce(&i_x_chunk,&(fp->last_item),1,SID_SIZE_T,SID_MAX,SID.COMM_WORLD);
  //fp->last_item--;
  return(r_val);
}
Exemplo n.º 8
0
void read_trees_horizontal(void **groups_in,
                           int *  n_groups_in,
                           void **subgroups_in,
                           int *  n_subgroups_in,
                           int *  n_subgroups_group,
                           int *  n_trees_subgroup_in,
                           int *  n_trees_group_in,
                           int    i_file, // tree snapshot index
                           int    i_read, // actual snapshot index
                           int    j_file,
                           int    n_wrap,
                           char * filename_output_dir,
                           int    mode) {
    char   filename_output_dir_horizontal[SID_MAX_FILENAME_LENGTH];
    char   filename_output_dir_horizontal_trees[SID_MAX_FILENAME_LENGTH];
    char   filename_in[SID_MAX_FILENAME_LENGTH];
    char   filename_output_file_root[SID_MAX_FILENAME_LENGTH];
    SID_fp fp_in;
    SID_log("Reading horizontal trees for snapshot #%03d...", SID_LOG_OPEN, i_read);

    // Parse the mode and set things up accordingly
    if(SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_EXTENDED) &&
            SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_GHOSTS))
        SID_exit_error("Too many storage modes chosen in read_trees_horizontal.", SID_ERROR_LOGIC);
    int flag_read_extended  = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_READ_EXTENDED);
    int flag_store_extended = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_EXTENDED);
    int flag_store_ghosts   = SID_CHECK_BITFIELD_SWITCH(mode, TREE_HORIZONTAL_STORE_GHOSTS);
    if(flag_store_ghosts)
        SID_exit_error("Ghost processing not supported in read_trees_horizontal().", SID_ERROR_LOGIC);

    if(!flag_read_extended)
        SID_exit_error("Reading of non-extended horizontal trees not yet implemented in read_trees_horizontal().",
                       SID_ERROR_LOGIC);

    // Set filename and open file
    strcpy(filename_output_file_root, filename_output_dir);
    strip_path(filename_output_file_root);
    sprintf(filename_output_dir_horizontal, "%s/horizontal", filename_output_dir);
    sprintf(filename_output_dir_horizontal_trees, "%s/trees", filename_output_dir_horizontal);
    if(flag_read_extended)
        sprintf(filename_in, "%s/horizontal_trees_tmp_%03d.dat", filename_output_dir_horizontal_trees, i_read);
    else
        sprintf(filename_in, "%s/horizontal_trees_%03d.dat", filename_output_dir_horizontal_trees, i_read);
    SID_fopen(filename_in, "r", &fp_in);

    // Read header
    int n_step_in;
    int n_search_in;
    int n_groups_max_in;
    int n_subgroups_max_in;
    SID_fread_all(&n_step_in, sizeof(int), 1, &fp_in);
    SID_fread_all(&n_search_in, sizeof(int), 1, &fp_in);
    SID_fread_all(n_groups_in, sizeof(int), 1, &fp_in);
    SID_fread_all(n_subgroups_in, sizeof(int), 1, &fp_in);
    SID_fread_all(&n_groups_max_in, sizeof(int), 1, &fp_in);
    SID_fread_all(&n_subgroups_max_in, sizeof(int), 1, &fp_in);
    SID_fread_all(n_trees_subgroup_in, sizeof(int), 1, &fp_in);
    SID_fread_all(n_trees_group_in, sizeof(int), 1, &fp_in);
    tree_horizontal_extended_info *groups_extended    = NULL;
    tree_horizontal_extended_info *subgroups_extended = NULL;
    if(flag_store_extended) {
        groups_extended    = (tree_horizontal_extended_info *)groups_in[i_file % n_wrap];
        subgroups_extended = (tree_horizontal_extended_info *)subgroups_in[i_file % n_wrap];
    }

    // Perform read
    int i_group;
    int i_subgroup;
    for(i_group = 0, i_subgroup = 0; i_group < (*n_groups_in); i_group++) {
        // Read groups
        int group_id;
        int group_type;
        int group_descendant_id;
        int group_tree_id;
        int group_file_offset;
        int group_index;
        int group_n_particles_peak = 0; // default when reading extended format files
        SID_fread_all(&group_id, sizeof(int), 1, &fp_in);
        SID_fread_all(&group_type, sizeof(int), 1, &fp_in);
        SID_fread_all(&group_descendant_id, sizeof(int), 1, &fp_in);
        SID_fread_all(&group_tree_id, sizeof(int), 1, &fp_in);
        SID_fread_all(&group_file_offset, sizeof(int), 1, &fp_in);
        SID_fread_all(&group_index, sizeof(int), 1, &fp_in);
        if(!flag_read_extended)
            SID_fread_all(&group_n_particles_peak, sizeof(int), 1, &fp_in);
        if(!check_validity_of_tree_case_flag(group_type))
            SID_exit_error("Invalid match type (%d) for i_group=%d", SID_ERROR_LOGIC, group_type, i_group);
        if(flag_store_extended) {
            groups_extended[i_group].id                     = group_id;
            groups_extended[i_group].type                   = group_type;
            groups_extended[i_group].descendant_id          = group_descendant_id;
            groups_extended[i_group].tree_id                = group_tree_id;
            groups_extended[i_group].descendant_file_offset = group_file_offset;
            groups_extended[i_group].descendant_index       = group_index;
        }
        if(flag_read_extended) {
            int   group_file_root;
            int   group_n_particles;
            int   group_n_particles_parent;
            int   group_n_particles_desc;
            int   group_n_particles_proj;
            float group_score_desc;
            float group_score_prog;
            int   group_snap_bridge;
            int   group_file_bridge;
            int   group_index_bridge;
            int   group_id_bridge;
            int   group_first_progenitor_file;
            int   group_first_progenitor_index;
            int   group_next_progenitor_file;
            int   group_next_progenitor_index;
            SID_fread(&group_file_root, sizeof(int), 1, &fp_in);
            SID_fread(&group_n_particles, sizeof(int), 1, &fp_in);
            SID_fread(&group_n_particles_parent, sizeof(int), 1, &fp_in);
            SID_fread(&group_n_particles_desc, sizeof(int), 1, &fp_in);
            SID_fread(&group_n_particles_proj, sizeof(int), 1, &fp_in);
            SID_fread(&group_score_desc, sizeof(float), 1, &fp_in);
            SID_fread(&group_score_prog, sizeof(float), 1, &fp_in);
            SID_fread(&group_snap_bridge, sizeof(int), 1, &fp_in);
            SID_fread(&group_file_bridge, sizeof(int), 1, &fp_in);
            SID_fread(&group_index_bridge, sizeof(int), 1, &fp_in);
            SID_fread(&group_id_bridge, sizeof(int), 1, &fp_in);
            SID_fread(&group_first_progenitor_file, sizeof(int), 1, &fp_in);
            SID_fread(&group_first_progenitor_index, sizeof(int), 1, &fp_in);
            SID_fread(&group_next_progenitor_file, sizeof(int), 1, &fp_in);
            SID_fread(&group_next_progenitor_index, sizeof(int), 1, &fp_in);
            if(flag_store_extended) {
                groups_extended[i_group].file_root          = group_file_root;
                groups_extended[i_group].n_particles_peak   = 0;
                groups_extended[i_group].parent_id          = group_id;
                groups_extended[i_group].substructure_index = 0;
                groups_extended[i_group].n_particles        = group_n_particles;
                groups_extended[i_group].n_particles_parent = group_n_particles_parent;
                groups_extended[i_group].n_particles_desc   = group_n_particles_desc;
                groups_extended[i_group].n_particles_proj   = group_n_particles_proj;
                groups_extended[i_group].score_desc         = group_score_desc;
                groups_extended[i_group].score_prog         = group_score_prog;
                // Only read the bridge information if the halo is marked
                //    TREE_CASE_FRAGMENTED_NEW.  This prevents us from overwriting
                //    the bridge info we need to propagate in order to check for
                //    when a fragmented halo returns to its bridge.
                if(SID_CHECK_BITFIELD_SWITCH(group_type, TREE_CASE_FRAGMENTED_NEW)) {
                    groups_extended[i_group].snap_bridge  = group_snap_bridge;
                    groups_extended[i_group].file_bridge  = group_file_bridge;
                    groups_extended[i_group].index_bridge = group_index_bridge;
                    groups_extended[i_group].id_bridge    = group_id_bridge;
                } else {
                    groups_extended[i_group].snap_bridge  = -1;
                    groups_extended[i_group].file_bridge  = -1;
                    groups_extended[i_group].index_bridge = -1;
                    groups_extended[i_group].id_bridge    = -1;
                }
                groups_extended[i_group].first_progenitor_file  = group_first_progenitor_file;
                groups_extended[i_group].first_progenitor_index = group_first_progenitor_index;
                groups_extended[i_group].next_progenitor_file   = group_next_progenitor_file;
                groups_extended[i_group].next_progenitor_index  = group_next_progenitor_index;
            }
        }
        SID_fread_all(&(n_subgroups_group[i_group]), sizeof(int), 1, &fp_in);

        int j_subgroup;
        for(j_subgroup = 0; j_subgroup < n_subgroups_group[i_group]; i_subgroup++, j_subgroup++) {
            // Read subgroups
            int subgroup_id;
            int subgroup_type;
            int subgroup_descendant_id;
            int subgroup_tree_id;
            int subgroup_file_offset;
            int subgroup_index;
            int subgroup_n_particles_peak = 0; // default when reading extended format files
            SID_fread_all(&subgroup_id, sizeof(int), 1, &fp_in);
            SID_fread_all(&subgroup_type, sizeof(int), 1, &fp_in);
            SID_fread_all(&subgroup_descendant_id, sizeof(int), 1, &fp_in);
            SID_fread_all(&subgroup_tree_id, sizeof(int), 1, &fp_in);
            SID_fread_all(&subgroup_file_offset, sizeof(int), 1, &fp_in);
            SID_fread_all(&subgroup_index, sizeof(int), 1, &fp_in);
            if(!flag_read_extended)
                SID_fread_all(&subgroup_n_particles_peak, sizeof(int), 1, &fp_in);
            if(!check_validity_of_tree_case_flag(subgroup_type))
                SID_exit_error("Invalid match type (%d) for i_group,j_subgroup,i_subgroup=%d,%d,%d", SID_ERROR_LOGIC,
                               subgroup_type, i_group, j_subgroup, i_subgroup);
            if(flag_store_extended) {
                subgroups_extended[i_subgroup].id                     = subgroup_id;
                subgroups_extended[i_subgroup].type                   = subgroup_type;
                subgroups_extended[i_subgroup].descendant_id          = subgroup_descendant_id;
                subgroups_extended[i_subgroup].tree_id                = subgroup_tree_id;
                subgroups_extended[i_subgroup].descendant_file_offset = subgroup_file_offset;
                subgroups_extended[i_subgroup].descendant_index       = subgroup_index;
            }
            if(flag_read_extended) {
                int   subgroup_file_root;
                int   subgroup_n_particles;
                int   subgroup_n_particles_parent;
                int   subgroup_n_particles_desc;
                int   subgroup_n_particles_proj;
                float subgroup_score_desc;
                float subgroup_score_prog;
                int   subgroup_snap_bridge;
                int   subgroup_file_bridge;
                int   subgroup_index_bridge;
                int   subgroup_id_bridge;
                int   subgroup_first_progenitor_file;
                int   subgroup_first_progenitor_index;
                int   subgroup_next_progenitor_file;
                int   subgroup_next_progenitor_index;
                SID_fread(&subgroup_file_root, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_n_particles, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_n_particles_parent, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_n_particles_desc, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_n_particles_proj, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_score_desc, sizeof(float), 1, &fp_in);
                SID_fread(&subgroup_score_prog, sizeof(float), 1, &fp_in);
                SID_fread(&subgroup_snap_bridge, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_file_bridge, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_index_bridge, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_id_bridge, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_first_progenitor_file, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_first_progenitor_index, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_next_progenitor_file, sizeof(int), 1, &fp_in);
                SID_fread(&subgroup_next_progenitor_index, sizeof(int), 1, &fp_in);
                if(flag_store_extended) {
                    subgroups_extended[i_subgroup].file_root          = subgroup_file_root;
                    subgroups_extended[i_subgroup].n_particles_peak   = 0;
                    subgroups_extended[i_subgroup].parent_id          = group_id;
                    subgroups_extended[i_subgroup].substructure_index = i_subgroup;
                    subgroups_extended[i_subgroup].n_particles        = subgroup_n_particles;
                    subgroups_extended[i_subgroup].n_particles_parent = subgroup_n_particles_parent;
                    subgroups_extended[i_subgroup].n_particles_desc   = subgroup_n_particles_desc;
                    subgroups_extended[i_subgroup].n_particles_proj   = subgroup_n_particles_proj;
                    subgroups_extended[i_subgroup].score_desc         = subgroup_score_desc;
                    subgroups_extended[i_subgroup].score_prog         = subgroup_score_prog;
                    // Only read the bridge information if the halo is marked
                    //    TREE_CASE_FRAGMENTED_NEW.  This prevents us from overwriting
                    //    the bridge info we need to propagate in order to check for
                    //    when a fragmented halo returns to its bridge.
                    if(SID_CHECK_BITFIELD_SWITCH(subgroup_type, TREE_CASE_FRAGMENTED_NEW)) {
                        subgroups_extended[i_subgroup].snap_bridge  = subgroup_snap_bridge;
                        subgroups_extended[i_subgroup].file_bridge  = subgroup_file_bridge;
                        subgroups_extended[i_subgroup].index_bridge = subgroup_index_bridge;
                        subgroups_extended[i_subgroup].id_bridge    = subgroup_id_bridge;
                    } else {
                        subgroups_extended[i_subgroup].snap_bridge  = -1;
                        subgroups_extended[i_subgroup].file_bridge  = -1;
                        subgroups_extended[i_subgroup].index_bridge = -1;
                        subgroups_extended[i_subgroup].id_bridge    = -1;
                    }
                    subgroups_extended[i_subgroup].first_progenitor_file  = subgroup_first_progenitor_file;
                    subgroups_extended[i_subgroup].first_progenitor_index = subgroup_first_progenitor_index;
                    subgroups_extended[i_subgroup].next_progenitor_file   = subgroup_next_progenitor_file;
                    subgroups_extended[i_subgroup].next_progenitor_index  = subgroup_next_progenitor_index;
                }
            }
        }
    }
    if(flag_store_extended) {
        for(; i_group < n_groups_max_in; i_group++)
            groups_extended[i_group].type = TREE_CASE_INVALID;
        for(; i_subgroup < n_subgroups_max_in; i_subgroup++)
            subgroups_extended[i_subgroup].type = TREE_CASE_INVALID;
    }
    SID_fclose(&fp_in);
    SID_log("Done.", SID_LOG_CLOSE);
}