int avro_schema_from_json(const char *jsontext, const int32_t len, avro_schema_t *schema, avro_schema_error_t *e) { check_param(EINVAL, jsontext, "JSON text"); check_param(EINVAL, schema, "schema pointer"); json_t *root; json_error_t json_error; AVRO_UNUSED(len); AVRO_UNUSED(e); root = json_loads(jsontext, 0, &json_error); if (!root) { avro_set_error("Error parsing JSON: %s", json_error.text); return EINVAL; } return avro_schema_from_json_root(root, schema); }
int avro_schema_enum_symbol_append(const avro_schema_t enum_schema, const char *symbol) { check_param(EINVAL, is_avro_schema(enum_schema), "enum schema"); check_param(EINVAL, is_avro_enum(enum_schema), "enum schema"); check_param(EINVAL, symbol, "symbol"); char *sym; long idx; struct avro_enum_schema_t *enump = avro_schema_to_enum(enum_schema); sym = avro_strdup(symbol); if (!sym) { avro_set_error("Cannot create copy of symbol name"); return ENOMEM; } idx = enump->symbols->num_entries; st_insert(enump->symbols, (st_data_t) idx, (st_data_t) sym); st_insert(enump->symbols_byname, (st_data_t) sym, (st_data_t) idx); return 0; }
static int avro_skip_file(struct _avro_reader_file_t *reader, int64_t len) { int rval; int64_t needed = len; if (len == 0) { return 0; } if (needed <= bytes_available(reader)) { reader->cur += needed; } else { needed -= bytes_available(reader); buffer_reset(reader); rval = fseek(reader->fp, needed, SEEK_CUR); if (rval < 0) { avro_set_error("Cannot skip %" PRIsz " bytes in file", (size_t) len); return rval; } } return 0; }
int process_frame(avro_value_t *frame_val, frame_reader_t reader, uint64_t wal_pos) { int err = 0, msg_type; size_t num_messages; avro_value_t msg_val, union_val, record_val; check(err, avro_value_get_by_index(frame_val, 0, &msg_val, NULL)); check(err, avro_value_get_size(&msg_val, &num_messages)); for (int i = 0; i < num_messages; i++) { check(err, avro_value_get_by_index(&msg_val, i, &union_val, NULL)); check(err, avro_value_get_discriminant(&union_val, &msg_type)); check(err, avro_value_get_current_branch(&union_val, &record_val)); switch (msg_type) { case PROTOCOL_MSG_BEGIN_TXN: check(err, process_frame_begin_txn(&record_val, reader, wal_pos)); break; case PROTOCOL_MSG_COMMIT_TXN: check(err, process_frame_commit_txn(&record_val, reader, wal_pos)); break; case PROTOCOL_MSG_TABLE_SCHEMA: check(err, process_frame_table_schema(&record_val, reader, wal_pos)); break; case PROTOCOL_MSG_INSERT: check(err, process_frame_insert(&record_val, reader, wal_pos)); break; case PROTOCOL_MSG_UPDATE: check(err, process_frame_update(&record_val, reader, wal_pos)); break; case PROTOCOL_MSG_DELETE: check(err, process_frame_delete(&record_val, reader, wal_pos)); break; default: avro_set_error("Unknown message type %d", msg_type); return EINVAL; } } return err; }
int process_frame_delete(avro_value_t *record_val, frame_reader_t reader, uint64_t wal_pos) { int err = 0, key_present, old_present; avro_value_t relid_val, key_val, old_val, branch_val; int64_t relid; const void *key_bin = NULL, *old_bin = NULL; size_t key_len = 0, old_len = 0; check(err, avro_value_get_by_index(record_val, 0, &relid_val, NULL)); check(err, avro_value_get_by_index(record_val, 1, &key_val, NULL)); check(err, avro_value_get_by_index(record_val, 2, &old_val, NULL)); check(err, avro_value_get_long(&relid_val, &relid)); check(err, avro_value_get_discriminant(&key_val, &key_present)); check(err, avro_value_get_discriminant(&old_val, &old_present)); schema_list_entry *entry = schema_list_lookup(reader, relid); if (!entry) { avro_set_error("Received delete for unknown relid %u", relid); return EINVAL; } if (key_present) { check(err, avro_value_get_current_branch(&key_val, &branch_val)); check(err, avro_value_get_bytes(&branch_val, &key_bin, &key_len)); check(err, read_entirely(&entry->key_value, entry->avro_reader, key_bin, key_len)); } if (old_present) { check(err, avro_value_get_current_branch(&old_val, &branch_val)); check(err, avro_value_get_bytes(&branch_val, &old_bin, &old_len)); check(err, read_entirely(&entry->old_value, entry->avro_reader, old_bin, old_len)); } if (reader->on_delete_row) { check(err, reader->on_delete_row(reader->cb_context, wal_pos, relid, key_bin, key_len, key_bin ? &entry->key_value : NULL, old_bin, old_len, old_bin ? &entry->old_value : NULL)); } return err; }
avro_schema_t avro_schema_enum_ns(const char *name, const char *space) { if (!is_avro_id(name)) { avro_set_error("Invalid Avro identifier"); return NULL; } struct avro_enum_schema_t *enump = (struct avro_enum_schema_t *) avro_new(struct avro_enum_schema_t); if (!enump) { avro_set_error("Cannot allocate new enum schema"); return NULL; } enump->name = avro_strdup(name); if (!enump->name) { avro_set_error("Cannot allocate new enum schema"); avro_freet(struct avro_enum_schema_t, enump); return NULL; } enump->space = space ? avro_strdup(space) : NULL; if (space && !enump->space) { avro_set_error("Cannot allocate new enum schema"); avro_str_free(enump->name); avro_freet(struct avro_enum_schema_t, enump); return NULL; } enump->symbols = st_init_numtable_with_size(DEFAULT_TABLE_SIZE); if (!enump->symbols) { avro_set_error("Cannot allocate new enum schema"); if (enump->space) avro_str_free(enump->space); avro_str_free(enump->name); avro_freet(struct avro_enum_schema_t, enump); return NULL; } enump->symbols_byname = st_init_strtable_with_size(DEFAULT_TABLE_SIZE); if (!enump->symbols_byname) { avro_set_error("Cannot allocate new enum schema"); st_free_table(enump->symbols); if (enump->space) avro_str_free(enump->space); avro_str_free(enump->name); avro_freet(struct avro_enum_schema_t, enump); return NULL; } avro_schema_init(&enump->obj, AVRO_ENUM); return &enump->obj; }
const char *avro_schema_type_name(const avro_schema_t schema) { if (is_avro_record(schema)) { return (avro_schema_to_record(schema))->name; } else if (is_avro_enum(schema)) { return (avro_schema_to_enum(schema))->name; } else if (is_avro_fixed(schema)) { return (avro_schema_to_fixed(schema))->name; } else if (is_avro_union(schema)) { return "union"; } else if (is_avro_array(schema)) { return "array"; } else if (is_avro_map(schema)) { return "map"; } else if (is_avro_int32(schema)) { return "int"; } else if (is_avro_int64(schema)) { return "long"; } else if (is_avro_float(schema)) { return "float"; } else if (is_avro_double(schema)) { return "double"; } else if (is_avro_boolean(schema)) { return "boolean"; } else if (is_avro_null(schema)) { return "null"; } else if (is_avro_string(schema)) { return "string"; } else if (is_avro_bytes(schema)) { return "bytes"; } else if (is_avro_link(schema)) { avro_schema_t target = avro_schema_link_target(schema); return avro_schema_type_name(target); } avro_set_error("Unknown schema type"); return NULL; }
int avro_raw_array_ensure_size(avro_raw_array_t *array, size_t desired_count) { size_t required_size = array->element_size * desired_count; if (array->allocated_size >= required_size) { return 0; } /* * Double the old size when reallocating. */ size_t new_size; if (array->allocated_size == 0) { /* * Start with an arbitrary 10 items. */ new_size = 10 * array->element_size; } else { new_size = array->allocated_size * 2; } if (required_size > new_size) { new_size = required_size; } array->data = avro_realloc(array->data, array->allocated_size, new_size); if (array->data == NULL) { avro_set_error("Cannot allocate space in array for %zu elements", desired_count); return ENOMEM; } array->allocated_size = new_size; return 0; }
static int avro_datum_value_get_by_name(const avro_value_iface_t *iface, const void *vself, const char *name, avro_value_t *child, size_t *index) { AVRO_UNUSED(iface); const avro_datum_t self = (const avro_datum_t) vself; check_param(EINVAL, self, "datum instance"); int rval; avro_datum_t child_datum; if (is_avro_map(self)) { if (index != NULL) { int real_index; check(rval, avro_map_get_index(self, name, &real_index)); *index = real_index; } check(rval, avro_map_get(self, name, &child_datum)); return avro_datum_as_child_value(child, child_datum); } if (is_avro_record(self)) { if (index != NULL) { avro_schema_t schema = avro_datum_get_schema(self); *index = avro_schema_record_field_get_index(schema, name); } check(rval, avro_record_get(self, name, &child_datum)); return avro_datum_as_child_value(child, child_datum); } avro_set_error("Can only get by name from map or record"); return EINVAL; }
static int avro_schema_from_json_t(json_t *json, avro_schema_t *schema, st_table *named_schemas) { #ifdef _WIN32 #pragma message("#warning: Bug: '0' is not of type avro_type_t.") #else #warning "Bug: '0' is not of type avro_type_t." #endif /* We should really have an "AVRO_INVALID" type in * avro_type_t. Suppress warning below in which we set type to 0. */ avro_type_t type = (avro_type_t) 0; unsigned int i; avro_schema_t named_type = NULL; if (avro_type_from_json_t(json, &type, named_schemas, &named_type)) { return EINVAL; } switch (type) { case AVRO_LINK: *schema = avro_schema_link(named_type); break; case AVRO_STRING: *schema = avro_schema_string(); break; case AVRO_BYTES: *schema = avro_schema_bytes(); break; case AVRO_INT32: *schema = avro_schema_int(); break; case AVRO_INT64: *schema = avro_schema_long(); break; case AVRO_FLOAT: *schema = avro_schema_float(); break; case AVRO_DOUBLE: *schema = avro_schema_double(); break; case AVRO_BOOLEAN: *schema = avro_schema_boolean(); break; case AVRO_NULL: *schema = avro_schema_null(); break; case AVRO_RECORD: { json_t *json_name = json_object_get(json, "name"); json_t *json_namespace = json_object_get(json, "namespace"); json_t *json_fields = json_object_get(json, "fields"); unsigned int num_fields; const char *record_name; const char *record_namespace; if (!json_is_string(json_name)) { avro_set_error("Record type must have a \"name\""); return EINVAL; } if (!json_is_array(json_fields)) { avro_set_error("Record type must have \"fields\""); return EINVAL; } num_fields = json_array_size(json_fields); if (num_fields == 0) { avro_set_error("Record type must have at least one field"); return EINVAL; } record_name = json_string_value(json_name); if (!record_name) { avro_set_error("Record type must have a \"name\""); return EINVAL; } if (json_is_string(json_namespace)) { record_namespace = json_string_value(json_namespace); } else { record_namespace = NULL; } *schema = avro_schema_record(record_name, record_namespace); if (save_named_schemas(record_name, *schema, named_schemas)) { avro_set_error("Cannot save record schema"); return ENOMEM; } for (i = 0; i < num_fields; i++) { json_t *json_field = json_array_get(json_fields, i); json_t *json_field_name; json_t *json_field_type; json_t *json_default_value; avro_schema_t json_field_type_schema; int field_rval; if (!json_is_object(json_field)) { avro_set_error("Record field %d must be an array", i); avro_schema_decref(*schema); return EINVAL; } json_field_name = json_object_get(json_field, "name"); if (!json_field_name) { avro_set_error("Record field %d must have a \"name\"", i); avro_schema_decref(*schema); return EINVAL; } json_field_type = json_object_get(json_field, "type"); if (!json_field_type) { avro_set_error("Record field %d must have a \"type\"", i); avro_schema_decref(*schema); return EINVAL; } field_rval = avro_schema_from_json_t(json_field_type, &json_field_type_schema, named_schemas); if (field_rval) { avro_schema_decref(*schema); return field_rval; } json_default_value = json_object_get(json_field, "default"); avro_datum_t default_value = NULL; if (json_default_value) { avro_schema_t default_schema = json_field_type_schema; if (json_field_type_schema->type == AVRO_UNION) { // From the spec: "Default values for union fields correspond // to the first schema in the union." default_schema = avro_schema_union_branch(json_field_type_schema, 0); } default_value = json_t_to_avro_value(default_schema, json_default_value); if (default_value == NULL) { avro_schema_decref(*schema); return EINVAL; } } field_rval = avro_schema_record_field_append(*schema, json_string_value (json_field_name), json_field_type_schema, default_value); avro_schema_decref(json_field_type_schema); if (field_rval != 0) { avro_schema_decref(*schema); return field_rval; } } } break; case AVRO_ENUM: { json_t *json_name = json_object_get(json, "name"); json_t *json_symbols = json_object_get(json, "symbols"); const char *name; unsigned int num_symbols; if (!json_is_string(json_name)) { avro_set_error("Enum type must have a \"name\""); return EINVAL; } if (!json_is_array(json_symbols)) { avro_set_error("Enum type must have \"symbols\""); return EINVAL; } name = json_string_value(json_name); if (!name) { avro_set_error("Enum type must have a \"name\""); return EINVAL; } num_symbols = json_array_size(json_symbols); if (num_symbols == 0) { avro_set_error("Enum type must have at least one symbol"); return EINVAL; } *schema = avro_schema_enum(name); if (save_named_schemas(name, *schema, named_schemas)) { avro_set_error("Cannot save enum schema"); return ENOMEM; } for (i = 0; i < num_symbols; i++) { int enum_rval; json_t *json_symbol = json_array_get(json_symbols, i); const char *symbol; if (!json_is_string(json_symbol)) { avro_set_error("Enum symbol %d must be a string", i); avro_schema_decref(*schema); return EINVAL; } symbol = json_string_value(json_symbol); enum_rval = avro_schema_enum_symbol_append(*schema, symbol); if (enum_rval != 0) { avro_schema_decref(*schema); return enum_rval; } } } break; case AVRO_ARRAY: { int items_rval; json_t *json_items = json_object_get(json, "items"); avro_schema_t items_schema; if (!json_items) { avro_set_error("Array type must have \"items\""); return EINVAL; } items_rval = avro_schema_from_json_t(json_items, &items_schema, named_schemas); if (items_rval) { return items_rval; } *schema = avro_schema_array(items_schema); avro_schema_decref(items_schema); } break; case AVRO_MAP: { int values_rval; json_t *json_values = json_object_get(json, "values"); avro_schema_t values_schema; if (!json_values) { avro_set_error("Map type must have \"values\""); return EINVAL; } values_rval = avro_schema_from_json_t(json_values, &values_schema, named_schemas); if (values_rval) { return values_rval; } *schema = avro_schema_map(values_schema); avro_schema_decref(values_schema); } break; case AVRO_UNION: { unsigned int num_schemas = json_array_size(json); avro_schema_t s; if (num_schemas == 0) { avro_set_error("Union type must have at least one branch"); return EINVAL; } *schema = avro_schema_union(); for (i = 0; i < num_schemas; i++) { int schema_rval; json_t *schema_json = json_array_get(json, i); if (!schema_json) { avro_set_error("Cannot retrieve branch JSON"); return EINVAL; } schema_rval = avro_schema_from_json_t(schema_json, &s, named_schemas); if (schema_rval != 0) { avro_schema_decref(*schema); return schema_rval; } schema_rval = avro_schema_union_append(*schema, s); avro_schema_decref(s); if (schema_rval != 0) { avro_schema_decref(*schema); return schema_rval; } } } break; case AVRO_FIXED: { json_t *json_size = json_object_get(json, "size"); json_t *json_name = json_object_get(json, "name"); json_int_t size; const char *name; if (!json_is_integer(json_size)) { avro_set_error("Fixed type must have a \"size\""); return EINVAL; } if (!json_is_string(json_name)) { avro_set_error("Fixed type must have a \"name\""); return EINVAL; } size = json_integer_value(json_size); name = json_string_value(json_name); *schema = avro_schema_fixed(name, (int64_t) size); if (save_named_schemas(name, *schema, named_schemas)) { avro_set_error("Cannot save fixed schema"); return ENOMEM; } } break; default: avro_set_error("Unknown schema type"); return EINVAL; } return 0; }
static int avro_type_from_json_t(json_t *json, avro_type_t *type, st_table *named_schemas, avro_schema_t *named_type) { json_t *json_type; const char *type_str; if (json_is_array(json)) { *type = AVRO_UNION; return 0; } else if (json_is_object(json)) { json_type = json_object_get(json, "type"); } else { json_type = json; } if (!json_is_string(json_type)) { avro_set_error("\"type\" field must be a string"); return EINVAL; } type_str = json_string_value(json_type); if (!type_str) { avro_set_error("\"type\" field must be a string"); return EINVAL; } /* * TODO: gperf/re2c this */ if (strcmp(type_str, "string") == 0) { *type = AVRO_STRING; } else if (strcmp(type_str, "bytes") == 0) { *type = AVRO_BYTES; } else if (strcmp(type_str, "int") == 0) { *type = AVRO_INT32; } else if (strcmp(type_str, "long") == 0) { *type = AVRO_INT64; } else if (strcmp(type_str, "float") == 0) { *type = AVRO_FLOAT; } else if (strcmp(type_str, "double") == 0) { *type = AVRO_DOUBLE; } else if (strcmp(type_str, "boolean") == 0) { *type = AVRO_BOOLEAN; } else if (strcmp(type_str, "null") == 0) { *type = AVRO_NULL; } else if (strcmp(type_str, "record") == 0) { *type = AVRO_RECORD; } else if (strcmp(type_str, "enum") == 0) { *type = AVRO_ENUM; } else if (strcmp(type_str, "array") == 0) { *type = AVRO_ARRAY; } else if (strcmp(type_str, "map") == 0) { *type = AVRO_MAP; } else if (strcmp(type_str, "fixed") == 0) { *type = AVRO_FIXED; } else if ((*named_type = find_named_schemas(type_str, named_schemas))) { *type = AVRO_LINK; } else { avro_set_error("Unknown Avro \"type\": %s", type_str); return EINVAL; } return 0; }
avro_datum_t avro_datum_from_schema(const avro_schema_t schema) { check_param(NULL, is_avro_schema(schema), "schema"); switch (avro_typeof(schema)) { case AVRO_STRING: return avro_givestring("", NULL); case AVRO_BYTES: return avro_givebytes("", 0, NULL); case AVRO_INT32: return avro_int32(0); case AVRO_INT64: return avro_int64(0); case AVRO_FLOAT: return avro_float(0); case AVRO_DOUBLE: return avro_double(0); case AVRO_BOOLEAN: return avro_boolean(0); case AVRO_NULL: return avro_null(); case AVRO_RECORD: { const struct avro_record_schema_t *record_schema = avro_schema_to_record(schema); avro_datum_t rec = avro_record(schema); int i; for (i = 0; i < record_schema->fields->num_entries; i++) { union { st_data_t data; struct avro_record_field_t *field; } val; st_lookup(record_schema->fields, i, &val.data); avro_datum_t field = avro_datum_from_schema(val.field->type); avro_record_set(rec, val.field->name, field); avro_datum_decref(field); } return rec; } case AVRO_ENUM: return avro_enum(schema, 0); case AVRO_FIXED: { const struct avro_fixed_schema_t *fixed_schema = avro_schema_to_fixed(schema); return avro_givefixed(schema, NULL, fixed_schema->size, NULL); } case AVRO_MAP: return avro_map(schema); case AVRO_ARRAY: return avro_array(schema); case AVRO_UNION: return avro_union(schema, -1, NULL); case AVRO_LINK: { const struct avro_link_schema_t *link_schema = avro_schema_to_link(schema); return avro_datum_from_schema(link_schema->to); } default: avro_set_error("Unknown schema type"); return NULL; } }
avro_schema_t avro_schema_get_subschema(const avro_schema_t schema, const char *name) { if (is_avro_record(schema)) { const struct avro_record_schema_t *rschema = avro_schema_to_record(schema); union { st_data_t data; struct avro_record_field_t *field; } field; if (st_lookup(rschema->fields_byname, (st_data_t) name, &field.data)) { return field.field->type; } avro_set_error("No record field named %s", name); return NULL; } else if (is_avro_union(schema)) { const struct avro_union_schema_t *uschema = avro_schema_to_union(schema); long i; for (i = 0; i < uschema->branches->num_entries; i++) { union { st_data_t data; avro_schema_t schema; } val; st_lookup(uschema->branches, i, &val.data); if (strcmp(avro_schema_type_name(val.schema), name) == 0) { return val.schema; } } avro_set_error("No union branch named %s", name); return NULL; } else if (is_avro_array(schema)) { if (strcmp(name, "[]") == 0) { const struct avro_array_schema_t *aschema = avro_schema_to_array(schema); return aschema->items; } avro_set_error("Array subschema must be called \"[]\""); return NULL; } else if (is_avro_map(schema)) { if (strcmp(name, "{}") == 0) { const struct avro_map_schema_t *mschema = avro_schema_to_map(schema); return mschema->values; } avro_set_error("Map subschema must be called \"{}\""); return NULL; } avro_set_error("Can only retrieve subschemas from record, union, array, or map"); return NULL; }
int avro_consume_binary(avro_reader_t reader, avro_consumer_t *consumer, void *ud) { int rval; const avro_encoding_t *enc = &avro_binary_encoding; check_param(EINVAL, reader, "reader"); check_param(EINVAL, consumer, "consumer"); switch (avro_typeof(consumer->schema)) { case AVRO_NULL: check_prefix(rval, enc->read_null(reader), "Cannot read null value: "); check(rval, avro_consumer_call(consumer, null_value, ud)); break; case AVRO_BOOLEAN: { int8_t b; check_prefix(rval, enc->read_boolean(reader, &b), "Cannot read boolean value: "); check(rval, avro_consumer_call(consumer, boolean_value, b, ud)); } break; case AVRO_STRING: { int64_t len; char *s; check_prefix(rval, enc->read_string(reader, &s, &len), "Cannot read string value: "); check(rval, avro_consumer_call(consumer, string_value, s, len, ud)); } break; case AVRO_INT32: { int32_t i; check_prefix(rval, enc->read_int(reader, &i), "Cannot read int value: "); check(rval, avro_consumer_call(consumer, int_value, i, ud)); } break; case AVRO_INT64: { int64_t l; check_prefix(rval, enc->read_long(reader, &l), "Cannot read long value: "); check(rval, avro_consumer_call(consumer, long_value, l, ud)); } break; case AVRO_FLOAT: { float f; check_prefix(rval, enc->read_float(reader, &f), "Cannot read float value: "); check(rval, avro_consumer_call(consumer, float_value, f, ud)); } break; case AVRO_DOUBLE: { double d; check_prefix(rval, enc->read_double(reader, &d), "Cannot read double value: "); check(rval, avro_consumer_call(consumer, double_value, d, ud)); } break; case AVRO_BYTES: { char *bytes; int64_t len; check_prefix(rval, enc->read_bytes(reader, &bytes, &len), "Cannot read bytes value: "); check(rval, avro_consumer_call(consumer, bytes_value, bytes, len, ud)); } break; case AVRO_FIXED: { char *bytes; int64_t size = avro_schema_to_fixed(consumer->schema)->size; bytes = avro_malloc(size); if (!bytes) { avro_prefix_error("Cannot allocate new fixed value"); return ENOMEM; } rval = avro_read(reader, bytes, size); if (rval) { avro_prefix_error("Cannot read fixed value: "); avro_free(bytes, size); return rval; } rval = avro_consumer_call(consumer, fixed_value, bytes, size, ud); if (rval) { avro_free(bytes, size); return rval; } } break; case AVRO_ENUM: check(rval, read_enum(reader, enc, consumer, ud)); break; case AVRO_ARRAY: check(rval, read_array(reader, enc, consumer, ud)); break; case AVRO_MAP: check(rval, read_map(reader, enc, consumer, ud)); break; case AVRO_UNION: check(rval, read_union(reader, enc, consumer, ud)); break; case AVRO_RECORD: check(rval, read_record(reader, enc, consumer, ud)); break; case AVRO_LINK: avro_set_error("Consumer can't consume a link schema directly"); return EINVAL; } return 0; }