static int ingestion_init_s5p_l1b_ra(const harp_ingestion_module *module, coda_product *product,
const harp_ingestion_options *options, harp_product_definition **definition,
void **user_data)
{
ingest_info *info;
(void)options;
info = malloc(sizeof(ingest_info));
if (info == NULL)
{
harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)",
sizeof(ingest_info), __FILE__, __LINE__);
return -1;
}
info->product = product;
info->band = -1;
if (init_cursors(info, NULL) != 0)
{
ingestion_done(info);
return -1;
}
if (init_dimensions(info, info->observation_cursor, "radiance") != 0)
{
ingestion_done(info);
return -1;
}
/* Initialize cursors and fill values for datasets which will be read using partial reads. */
if (init_dataset
(info->instrument_cursor, "nominal_wavelength", info->num_pixels * info->num_channels, &info->wavelength_cursor,
&info->wavelength_fill_value) != 0)
{
ingestion_done(info);
return -1;
}
if (init_dataset
(info->observation_cursor, "radiance", info->num_scanlines * info->num_pixels * info->num_channels,
&info->observable_cursor, &info->observable_fill_value) != 0)
{
ingestion_done(info);
return -1;
}
*definition = *module->product_definition;
*user_data = info;
return 0;
}
static int ingestion_init(const harp_ingestion_module *module, coda_product *product,
const harp_ingestion_options *options, harp_product_definition **definition, void **user_data)
{
const char *option_value;
int format_version;
ingest_info *info;
(void)options;
if (coda_get_product_version(product, &format_version) != 0)
{
harp_set_error(HARP_ERROR_CODA, NULL);
return -1;
}
info = malloc(sizeof(ingest_info));
if (info == NULL)
{
harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)",
sizeof(ingest_info), __FILE__, __LINE__);
return -1;
}
info->product = product;
info->format_version = format_version;
info->num_vertical = 0;
info->model_temperature = 0;
info->model_air = 0;
info->has_model_air = 0;
info->has_absolute_error = 0;
if (format_version > 0)
{
info->has_model_air = 1;
}
if (format_version >= 2)
{
info->has_absolute_error = 1;
}
if (init_dimensions(info) != 0)
{
ingestion_done(info);
return -1;
}
if (harp_ingestion_options_get_option(options, "temperature", &option_value) == 0)
{
if (strcmp(option_value, "model") == 0)
{
info->model_temperature = 1;
}
}
if (harp_ingestion_options_get_option(options, "air", &option_value) == 0)
{
if (strcmp(option_value, "model") == 0)
{
info->model_air = 1;
}
}
*definition = *module->product_definition;
*user_data = info;
return 0;
}
int main (int argc, char ** argv)
{
char * filename;
int rc = 0;
if (argc < 2 || argc > 3)
{
print_usage (argc, argv);
return -1;
}
if (argv [1][0] && argv [1][0] == '-')
{
if ( !strcmp (argv [1], "-f")
|| !strcmp (argv [1], "--force")
)
{
do_write_index = 1;
filename = argv [2];
}
else
{
print_usage (argc, argv);
return -1;
}
}
else
{
filename = argv [1];
do_write_index = 0;
}
have_subfiles = 0;
struct adios_bp_buffer_struct_v1 * b = 0;
uint32_t version = 0;
b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
adios_buffer_struct_init (b);
int flags = O_RDONLY;
if (do_write_index)
flags = O_RDWR;
int fd = open (filename, flags);
if (fd < 0) {
fprintf (stderr, "recover: cannot open file %s\n", filename);
if (errno)
fprintf (stderr, "%s\n", strerror(errno));
return -1;
}
struct stat statbuf;
fstat (fd, &statbuf);
uint64_t file_size = statbuf.st_size;
b->f = fd;
printf ("File size in bytes: %llu\n", file_size);
/* Variables to build new index */
struct adios_index_struct_v1 * index = adios_alloc_index_v1(1);
//struct adios_index_process_group_struct_v1 * pg_root = 0;
struct adios_index_process_group_struct_v1 * pg = 0;
//struct adios_index_var_struct_v1 * vars_root = 0;
//struct adios_index_attribute_struct_v1 * attrs_root = 0;
uint64_t pg_num = 0L;
uint64_t curr_offset = 0L;
uint64_t new_offset = 0L;
uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly))
uint64_t pgsize_actual; // size of current pg based on processing (accurate)
int found_pg = 0;
printf (DIVIDER);
found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported);
// pass over the PGs from beginning of file
while (found_pg)
{
curr_offset = new_offset;
pg_num++;
printf ("PG %llu found at offset %llu\n", pg_num, curr_offset);
/* Let's process the group */
/* Allocate PG index struct here to allow to be used below */
pg = (struct adios_index_process_group_struct_v1 *)
malloc (sizeof(struct adios_index_process_group_struct_v1));
// setup where to read the process group from (and size)
pg->offset_in_file = curr_offset;
//b->read_pg_offset = pg->offset_in_file;
b->read_pg_offset = curr_offset;
b->read_pg_size = pgsize_reported;
/* Temporary variables for parsing one PG */
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
init_dimensions (); // store scalar values from this PG temporarily
/* Read the whole PG into a buffer and start parsing */
adios_posix_read_process_group (b);
adios_parse_process_group_header_v1 (b, &pg_header);
print_process_group_header (pg_num, &pg_header);
add_pg_to_index (index, &pg_header, curr_offset);
adios_parse_vars_header_v1 (b, &vars_header);
print_vars_header (&vars_header);
int i;
for (i = 0; i < vars_header.count; i++)
{
var_payload.payload = 0;
adios_parse_var_data_header_v1 (b, &var_header);
print_var_header (&var_header);
if ( var_header.dims == 0)
{
// Load scalars to save them for handling as dimension values
var_payload.payload = malloc (var_header.payload_size + 1);
adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
,var_header.payload_size
);
}
else
{
// Just parse to move the offset in buffer, don't read data
adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
}
store_scalar_dimensions (&var_header, &var_payload);
add_var_to_index (index, &pg_header, &var_header, &var_payload);
if (var_payload.payload)
{
free (var_payload.payload);
var_payload.payload = 0;
}
//printf ("\n");
}
adios_parse_attributes_header_v1 (b, &attrs_header);
print_attrs_header (&attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (b, &attribute);
//print_attribute (&attribute);
//printf ("\n");
}
pgsize_actual = b->offset;
printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual);
pg = pg->next;
printf (DIVIDER);
found_pg = 0;
if (curr_offset + pgsize_actual < file_size)
{
new_offset = curr_offset + pgsize_actual;
found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported);
}
if (!found_pg &&
pgsize_actual != pgsize_reported &&
curr_offset + pgsize_reported < file_size)
{
new_offset = curr_offset + pgsize_reported;
found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported);
}
}
// The end of the last successfully processed PG
// This will be the start of the index data
curr_offset += pgsize_actual;
printf (DIVIDER);
printf ("Found %llu PGs to be processable\n", pg_num);
write_index (fd, curr_offset, index);
adios_posix_close_internal (b); // will close fd
return 0;
}
static int ingestion_init_s5p_l1b_ir(const harp_ingestion_module *module, coda_product *product,
const harp_ingestion_options *options, harp_product_definition **definition,
void **user_data)
{
ingest_info *info;
char product_group_name[17];
int band_available;
info = malloc(sizeof(ingest_info));
if (info == NULL)
{
harp_set_error(HARP_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)",
sizeof(ingest_info), __FILE__, __LINE__);
return -1;
}
info->product = product;
info->band = -1;
if (parse_option_band(info, options) != 0)
{
ingestion_done(info);
return -1;
}
snprintf(product_group_name, ARRAY_SIZE(product_group_name), "BAND%d_IRRADIANCE", info->band);
if (get_product_group_availability(info->product, product_group_name, &band_available) != 0)
{
ingestion_done(info);
return -1;
}
if (!band_available)
{
harp_set_error(HARP_ERROR_INGESTION, "no data for band '%d'", info->band);
ingestion_done(info);
return -1;
}
if (init_cursors(info, product_group_name) != 0)
{
ingestion_done(info);
return -1;
}
if (init_dimensions(info, info->observation_cursor, "irradiance") != 0)
{
ingestion_done(info);
return -1;
}
/* Initialize cursors and fill values for datasets which will be read using partial reads. */
if (init_dataset
(info->instrument_cursor, "calibrated_wavelength", info->num_pixels * info->num_channels,
&info->wavelength_cursor, &info->wavelength_fill_value) != 0)
{
ingestion_done(info);
return -1;
}
if (init_dataset
(info->observation_cursor, "irradiance", info->num_scanlines * info->num_pixels * info->num_channels,
&info->observable_cursor, &info->observable_fill_value) != 0)
{
ingestion_done(info);
return -1;
}
assert(info->band >= 1 && info->band <= 8);
if (info->band < 7)
{
*definition = module->product_definition[info->band - 1];
}
else
{
*definition = module->product_definition[info->band - 7];
}
*user_data = info;
return 0;
}
