static int ingestion_init_s5p_l1b_ra(const harp_ingestion_module *module, coda_product *product, const harp_ingestion_options *options, harp_product_definition **definition, void **user_data) { ingest_info *info; (void)options; info = malloc(sizeof(ingest_info)); if (info == NULL) { harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", sizeof(ingest_info), __FILE__, __LINE__); return -1; } info->product = product; info->band = -1; if (init_cursors(info, NULL) != 0) { ingestion_done(info); return -1; } if (init_dimensions(info, info->observation_cursor, "radiance") != 0) { ingestion_done(info); return -1; } /* Initialize cursors and fill values for datasets which will be read using partial reads. */ if (init_dataset (info->instrument_cursor, "nominal_wavelength", info->num_pixels * info->num_channels, &info->wavelength_cursor, &info->wavelength_fill_value) != 0) { ingestion_done(info); return -1; } if (init_dataset (info->observation_cursor, "radiance", info->num_scanlines * info->num_pixels * info->num_channels, &info->observable_cursor, &info->observable_fill_value) != 0) { ingestion_done(info); return -1; } *definition = *module->product_definition; *user_data = info; return 0; }
static int ingestion_init(const harp_ingestion_module *module, coda_product *product, const harp_ingestion_options *options, harp_product_definition **definition, void **user_data) { const char *option_value; int format_version; ingest_info *info; (void)options; if (coda_get_product_version(product, &format_version) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } info = malloc(sizeof(ingest_info)); if (info == NULL) { harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", sizeof(ingest_info), __FILE__, __LINE__); return -1; } info->product = product; info->format_version = format_version; info->num_vertical = 0; info->model_temperature = 0; info->model_air = 0; info->has_model_air = 0; info->has_absolute_error = 0; if (format_version > 0) { info->has_model_air = 1; } if (format_version >= 2) { info->has_absolute_error = 1; } if (init_dimensions(info) != 0) { ingestion_done(info); return -1; } if (harp_ingestion_options_get_option(options, "temperature", &option_value) == 0) { if (strcmp(option_value, "model") == 0) { info->model_temperature = 1; } } if (harp_ingestion_options_get_option(options, "air", &option_value) == 0) { if (strcmp(option_value, "model") == 0) { info->model_air = 1; } } *definition = *module->product_definition; *user_data = info; return 0; }
int main (int argc, char ** argv) { char * filename; int rc = 0; if (argc < 2 || argc > 3) { print_usage (argc, argv); return -1; } if (argv [1][0] && argv [1][0] == '-') { if ( !strcmp (argv [1], "-f") || !strcmp (argv [1], "--force") ) { do_write_index = 1; filename = argv [2]; } else { print_usage (argc, argv); return -1; } } else { filename = argv [1]; do_write_index = 0; } have_subfiles = 0; struct adios_bp_buffer_struct_v1 * b = 0; uint32_t version = 0; b = malloc (sizeof (struct adios_bp_buffer_struct_v1)); adios_buffer_struct_init (b); int flags = O_RDONLY; if (do_write_index) flags = O_RDWR; int fd = open (filename, flags); if (fd < 0) { fprintf (stderr, "recover: cannot open file %s\n", filename); if (errno) fprintf (stderr, "%s\n", strerror(errno)); return -1; } struct stat statbuf; fstat (fd, &statbuf); uint64_t file_size = statbuf.st_size; b->f = fd; printf ("File size in bytes: %llu\n", file_size); /* Variables to build new index */ struct adios_index_struct_v1 * index = adios_alloc_index_v1(1); //struct adios_index_process_group_struct_v1 * pg_root = 0; struct adios_index_process_group_struct_v1 * pg = 0; //struct adios_index_var_struct_v1 * vars_root = 0; //struct adios_index_attribute_struct_v1 * attrs_root = 0; uint64_t pg_num = 0L; uint64_t curr_offset = 0L; uint64_t new_offset = 0L; uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly)) uint64_t pgsize_actual; // size of current pg based on processing (accurate) int found_pg = 0; printf (DIVIDER); found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported); // pass over the PGs from beginning of file while (found_pg) { curr_offset = new_offset; pg_num++; printf ("PG %llu found at offset %llu\n", pg_num, curr_offset); /* Let's process the group */ /* Allocate PG index struct here to allow to be used below */ pg = (struct adios_index_process_group_struct_v1 *) malloc (sizeof(struct adios_index_process_group_struct_v1)); // setup where to read the process group from (and size) pg->offset_in_file = curr_offset; //b->read_pg_offset = pg->offset_in_file; b->read_pg_offset = curr_offset; b->read_pg_size = pgsize_reported; /* Temporary variables for parsing one PG */ struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; init_dimensions (); // store scalar values from this PG temporarily /* Read the whole PG into a buffer and start parsing */ adios_posix_read_process_group (b); adios_parse_process_group_header_v1 (b, &pg_header); print_process_group_header (pg_num, &pg_header); add_pg_to_index (index, &pg_header, curr_offset); adios_parse_vars_header_v1 (b, &vars_header); print_vars_header (&vars_header); int i; for (i = 0; i < vars_header.count; i++) { var_payload.payload = 0; adios_parse_var_data_header_v1 (b, &var_header); print_var_header (&var_header); if ( var_header.dims == 0) { // Load scalars to save them for handling as dimension values var_payload.payload = malloc (var_header.payload_size + 1); adios_parse_var_data_payload_v1 (b, &var_header, &var_payload ,var_header.payload_size ); } else { // Just parse to move the offset in buffer, don't read data adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0); } store_scalar_dimensions (&var_header, &var_payload); add_var_to_index (index, &pg_header, &var_header, &var_payload); if (var_payload.payload) { free (var_payload.payload); var_payload.payload = 0; } //printf ("\n"); } adios_parse_attributes_header_v1 (b, &attrs_header); print_attrs_header (&attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (b, &attribute); //print_attribute (&attribute); //printf ("\n"); } pgsize_actual = b->offset; printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual); pg = pg->next; printf (DIVIDER); found_pg = 0; if (curr_offset + pgsize_actual < file_size) { new_offset = curr_offset + pgsize_actual; found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported); } if (!found_pg && pgsize_actual != pgsize_reported && curr_offset + pgsize_reported < file_size) { new_offset = curr_offset + pgsize_reported; found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported); } } // The end of the last successfully processed PG // This will be the start of the index data curr_offset += pgsize_actual; printf (DIVIDER); printf ("Found %llu PGs to be processable\n", pg_num); write_index (fd, curr_offset, index); adios_posix_close_internal (b); // will close fd return 0; }
static int ingestion_init_s5p_l1b_ir(const harp_ingestion_module *module, coda_product *product, const harp_ingestion_options *options, harp_product_definition **definition, void **user_data) { ingest_info *info; char product_group_name[17]; int band_available; info = malloc(sizeof(ingest_info)); if (info == NULL) { harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", sizeof(ingest_info), __FILE__, __LINE__); return -1; } info->product = product; info->band = -1; if (parse_option_band(info, options) != 0) { ingestion_done(info); return -1; } snprintf(product_group_name, ARRAY_SIZE(product_group_name), "BAND%d_IRRADIANCE", info->band); if (get_product_group_availability(info->product, product_group_name, &band_available) != 0) { ingestion_done(info); return -1; } if (!band_available) { harp_set_error(HARP_ERROR_INGESTION, "no data for band '%d'", info->band); ingestion_done(info); return -1; } if (init_cursors(info, product_group_name) != 0) { ingestion_done(info); return -1; } if (init_dimensions(info, info->observation_cursor, "irradiance") != 0) { ingestion_done(info); return -1; } /* Initialize cursors and fill values for datasets which will be read using partial reads. */ if (init_dataset (info->instrument_cursor, "calibrated_wavelength", info->num_pixels * info->num_channels, &info->wavelength_cursor, &info->wavelength_fill_value) != 0) { ingestion_done(info); return -1; } if (init_dataset (info->observation_cursor, "irradiance", info->num_scanlines * info->num_pixels * info->num_channels, &info->observable_cursor, &info->observable_fill_value) != 0) { ingestion_done(info); return -1; } assert(info->band >= 1 && info->band <= 8); if (info->band < 7) { *definition = module->product_definition[info->band - 1]; } else { *definition = module->product_definition[info->band - 7]; } *user_data = info; return 0; }