static int init_num_main(ingest_info *info) { if (coda_cursor_get_num_elements(&info->time_cursor, &info->num_main) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } return 0; }
static int read_dataset(coda_cursor cursor, const char *dataset_name, long num_elements, harp_array data) { long coda_num_elements; harp_scalar fill_value; if (coda_cursor_goto_record_field_by_name(&cursor, dataset_name) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_get_num_elements(&cursor, &coda_num_elements) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_num_elements != num_elements) { harp_set_error(HARP_ERROR_INGESTION, "dataset has %ld elements; expected %ld", coda_num_elements, num_elements); harp_add_coda_cursor_path_to_error_message(&cursor); return -1; } if (coda_cursor_read_float_array(&cursor, data.float_data, coda_array_ordering_c) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_goto(&cursor, "@FillValue[0]") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_read_float(&cursor, &fill_value.float_data) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } /* Replace values equal to the _FillValue variable attribute by NaN. */ harp_array_replace_fill_value(harp_type_float, num_elements, data, fill_value); return 0; }
static int init_dimensions(ingest_info *info) { coda_cursor cursor; if (coda_cursor_set_product(&cursor, info->product) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_goto_record_field_by_name(&cursor, "nl_geolocation") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_get_num_elements(&cursor, &info->num_vertical) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } return 0; }
static int init_dataset(coda_cursor cursor, const char *name, long num_elements, coda_cursor *new_cursor, harp_scalar *fill_value) { long coda_num_elements; if (coda_cursor_goto_record_field_by_name(&cursor, name) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_get_num_elements(&cursor, &coda_num_elements) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_num_elements != num_elements) { harp_set_error(HARP_ERROR_INGESTION, "dataset has %ld elements; expected %ld", coda_num_elements, num_elements); harp_add_coda_cursor_path_to_error_message(&cursor); return -1; } if (coda_cursor_goto(&cursor, "@FillValue[0]") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_read_float(&cursor, &fill_value->float_data) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } coda_cursor_goto_parent(&cursor); coda_cursor_goto_parent(&cursor); coda_cursor_goto_parent(&cursor); *new_cursor = cursor; return 0; }
static void print_data(coda_cursor *cursor, int depth) { coda_type_class type_class; int has_attributes; if (coda_cursor_has_attributes(cursor, &has_attributes) != 0) { handle_coda_error(); } if (has_attributes) { if (coda_cursor_goto_attributes(cursor) != 0) { handle_coda_error(); } fi_printf("{attributes}\n"); INDENT++; print_data(cursor, depth); INDENT--; coda_cursor_goto_parent(cursor); } if (coda_cursor_get_type_class(cursor, &type_class) != 0) { handle_coda_error(); } switch (type_class) { case coda_record_class: { long num_fields; if (coda_cursor_get_num_elements(cursor, &num_fields) != 0) { handle_coda_error(); } if (num_fields > 0) { coda_type *record_type; int is_union; long i; if (coda_cursor_get_type(cursor, &record_type) != 0) { handle_coda_error(); } if (coda_type_get_record_union_status(record_type, &is_union) != 0) { handle_coda_error(); } if (is_union) { const char *field_name; if (coda_cursor_get_available_union_field_index(cursor, &i) != 0) { handle_coda_error(); } if (coda_type_get_record_field_name(record_type, i, &field_name) != 0) { handle_coda_error(); } if (coda_cursor_goto_record_field_by_index(cursor, i) != 0) { handle_coda_error(); } fi_printf("[%s]", field_name); if (print_offsets) { int64_t offset; if (coda_cursor_get_file_bit_offset(cursor, &offset) != 0) { handle_coda_error(); } if (offset >= 0) { char s[21]; coda_str64(offset >> 3, s); ff_printf(":%s", s); if ((offset & 0x7) != 0) { ff_printf(":%d", (int)(offset & 0x7)); } } } ff_printf("\n"); INDENT++; if (max_depth < 0 || depth < max_depth) { print_data(cursor, depth + 1); } else { fi_printf("...\n"); } INDENT--; coda_cursor_goto_parent(cursor); } else { if (coda_cursor_goto_first_record_field(cursor) != 0) { handle_coda_error(); } for (i = 0; i < num_fields; i++) { const char *field_name; if (coda_type_get_record_field_name(record_type, i, &field_name) != 0) { handle_coda_error(); } fi_printf("[%s]", field_name); if (print_offsets) { int64_t offset; if (coda_cursor_get_file_bit_offset(cursor, &offset) != 0) { handle_coda_error(); } if (offset >= 0) { char s[21]; coda_str64(offset >> 3, s); ff_printf(":%s", s); if ((offset & 0x7) != 0) { ff_printf(":%d", (int)(offset & 0x7)); } } } ff_printf("\n"); INDENT++; if (max_depth < 0 || depth < max_depth) { print_data(cursor, depth + 1); } else { fi_printf("...\n"); } INDENT--; if (i < num_fields - 1) { if (coda_cursor_goto_next_record_field(cursor) != 0) { handle_coda_error(); } } } coda_cursor_goto_parent(cursor); }
static int read_basic_type(const coda_cursor *cursor, void *dst) { int32 start[MAX_HDF4_VAR_DIMS]; int32 stride[MAX_HDF4_VAR_DIMS]; int32 edge[MAX_HDF4_VAR_DIMS]; long index; long i; index = cursor->stack[cursor->n - 1].index; assert(cursor->n > 1); switch (((coda_hdf4_type *)cursor->stack[cursor->n - 2].type)->tag) { case tag_hdf4_basic_type_array: { coda_cursor array_cursor; char *buffer; int native_type_size; long num_elements; /* we first read the whole array and then return only the requested element */ array_cursor = *cursor; array_cursor.n--; if (coda_cursor_get_num_elements(&array_cursor, &num_elements) != 0) { return -1; } assert(index < num_elements); native_type_size = get_native_type_size(((coda_hdf4_type *)cursor->stack[cursor->n - 1].type)->definition->read_type); buffer = malloc(num_elements * native_type_size); if (buffer == NULL) { coda_set_error(CODA_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", (long)(num_elements * native_type_size), __FILE__, __LINE__); return -1; } if (read_attribute(&array_cursor, buffer, -1) != 0) { free(buffer); return -1; } memcpy(dst, &buffer[index * native_type_size], native_type_size); free(buffer); } break; case tag_hdf4_attributes: case tag_hdf4_file_attributes: if (read_attribute(cursor, dst, -1) != 0) { return -1; } break; case tag_hdf4_GRImage: { coda_hdf4_GRImage *type; stride[0] = 1; stride[1] = 1; edge[0] = 1; edge[1] = 1; type = (coda_hdf4_GRImage *)cursor->stack[cursor->n - 2].type; if (type->ncomp != 1) { uint8 *buffer; int component_size; int component_index; component_size = get_native_type_size(type->basic_type->definition->read_type); /* HDF4 does not allow reading a single component of a GRImage, so we have to first read all * components and then return only the data item that was requested */ buffer = malloc(component_size * type->ncomp); if (buffer == NULL) { coda_set_error(CODA_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", (long)(component_size * type->ncomp), __FILE__, __LINE__); return -1; } component_index = index % type->ncomp; index /= type->ncomp; /* For GRImage data the first dimension is the fastest running */ start[0] = index % type->dim_sizes[0]; start[1] = index / type->dim_sizes[0]; if (GRreadimage(type->ri_id, start, stride, edge, buffer) != 0) { coda_set_error(CODA_ERROR_HDF4, NULL); free(buffer); return -1; } memcpy(dst, &buffer[component_index * component_size], component_size); free(buffer); } else { /* For GRImage data the first dimension is the fastest running */ start[0] = index % type->dim_sizes[0]; start[1] = index / type->dim_sizes[0]; if (GRreadimage(type->ri_id, start, stride, edge, dst) != 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } } } break; case tag_hdf4_SDS: { coda_hdf4_SDS *type; type = (coda_hdf4_SDS *)cursor->stack[cursor->n - 2].type; if (type->rank == 0) { start[0] = 0; edge[1] = 1; } else { for (i = type->rank - 1; i >= 0; i--) { start[i] = index % type->dimsizes[i]; index /= type->dimsizes[i]; edge[i] = 1; } } if (SDreaddata(type->sds_id, start, NULL, edge, dst) != 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } } break; case tag_hdf4_Vdata_field: { coda_hdf4_Vdata *type; coda_hdf4_Vdata_field *field_type; int record_pos; int order_pos; assert(cursor->n > 2); type = (coda_hdf4_Vdata *)cursor->stack[cursor->n - 3].type; field_type = (coda_hdf4_Vdata_field *)cursor->stack[cursor->n - 2].type; order_pos = index % field_type->order; record_pos = index / field_type->order; if (VSseek(type->vdata_id, record_pos) < 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } if (VSsetfields(type->vdata_id, field_type->field_name) != 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } if (field_type->order > 1) { /* HDF4 does not allow reading part of a vdata field, so we have to first read the full field and * then return only the data item that was requested */ uint8 *buffer; int element_size; int size; size = VSsizeof(type->vdata_id, field_type->field_name); if (size < 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } buffer = malloc(size); if (buffer == NULL) { coda_set_error(CODA_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)", (long)size, __FILE__, __LINE__); return -1; } if (VSread(type->vdata_id, buffer, 1, FULL_INTERLACE) < 0) { coda_set_error(CODA_ERROR_HDF4, NULL); free(buffer); return -1; } /* the size of a field element is the field size divided by the order of the field */ element_size = size / field_type->order; memcpy(dst, &buffer[order_pos * element_size], element_size); free(buffer); } else { if (VSread(type->vdata_id, (uint8 *)dst, 1, FULL_INTERLACE) < 0) { coda_set_error(CODA_ERROR_HDF4, NULL); return -1; } } } break; default: assert(0); exit(1); } return 0; }
static int read_datetime(void *user_data, harp_array data) { ingest_info *info = (ingest_info *)user_data; coda_cursor cursor; harp_scalar fill_value; double time_reference; long coda_num_elements; long i; /* Even though the product specification may not accurately describe this, S5P treats all days as having 86400 * seconds (as does HARP). The time value is thus the sum of: * - the S5P time reference as seconds since 2010 (using 86400 seconds per day) * - the number of seconds since the S5P time reference */ /* Read reference time in seconds since 2010-01-01 */ cursor = info->observation_cursor; if (coda_cursor_goto_record_field_by_name(&cursor, "time") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_get_num_elements(&cursor, &coda_num_elements) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_num_elements != 1) { harp_set_error(HARP_ERROR_INGESTION, "dataset has %ld elements; expected 1", coda_num_elements); harp_add_coda_cursor_path_to_error_message(&cursor); return -1; } if (coda_cursor_goto_first_array_element(&cursor) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_read_double(&cursor, &time_reference) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (time_reference == DEFAULT_FILL_VALUE_INT) { time_reference = coda_NaN(); } /* Read difference in milliseconds (ms) between the time reference and the start of the observation. */ cursor = info->observation_cursor; if (coda_cursor_goto_record_field_by_name(&cursor, "delta_time") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_get_num_elements(&cursor, &coda_num_elements) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_num_elements != info->num_scanlines) { harp_set_error(HARP_ERROR_INGESTION, "dataset has %ld elements; expected %ld", coda_num_elements, info->num_scanlines); harp_add_coda_cursor_path_to_error_message(&cursor); return -1; } if (coda_cursor_read_double_array(&cursor, data.double_data, coda_array_ordering_c) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_goto(&cursor, "@FillValue[0]") != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } if (coda_cursor_read_double(&cursor, &fill_value.double_data) != 0) { harp_set_error(HARP_ERROR_CODA, NULL); return -1; } /* Replace values equal to the _FillValue variable attribute by NaN. */ harp_array_replace_fill_value(harp_type_double, info->num_scanlines, data, fill_value); /* Convert observation start time to seconds since 2010-01-01 */ for (i = 0; i < info->num_scanlines; i++) { data.double_data[i] = time_reference + data.double_data[i] / 1e3; } /* Broadcast the result along the pixel dimension. */ broadcast_array_double(info->num_scanlines, info->num_pixels, data.double_data); return 0; }