avro_schema_t schema_for_table_schema() { avro_schema_t record_schema = avro_schema_record("TableSchema", PROTOCOL_SCHEMA_NAMESPACE); avro_schema_t field_schema = avro_schema_long(); avro_schema_record_field_append(record_schema, "relid", field_schema); avro_schema_decref(field_schema); field_schema = nullable_schema(avro_schema_string()); avro_schema_record_field_append(record_schema, "keySchema", field_schema); avro_schema_decref(field_schema); field_schema = avro_schema_string(); avro_schema_record_field_append(record_schema, "rowSchema", field_schema); avro_schema_decref(field_schema); return record_schema; }
static int union_free_foreach(int i, avro_schema_t schema, void *arg) { AVRO_UNUSED(i); AVRO_UNUSED(arg); avro_schema_decref(schema); return ST_DELETE; }
avro_schema_t schema_for_delete() { avro_schema_t record_schema = avro_schema_record("Delete", PROTOCOL_SCHEMA_NAMESPACE); avro_schema_t field_schema = avro_schema_long(); avro_schema_record_field_append(record_schema, "relid", field_schema); avro_schema_decref(field_schema); field_schema = nullable_schema(avro_schema_bytes()); avro_schema_record_field_append(record_schema, "key", field_schema); avro_schema_decref(field_schema); field_schema = nullable_schema(avro_schema_bytes()); avro_schema_record_field_append(record_schema, "oldRow", field_schema); avro_schema_decref(field_schema); return record_schema; }
/* typedef struct { int jobid; int vpid; } process_name_t; typedef struct { char *en_vars; char *args; char *host_name; process_name_t proc_name; } launch_context_t; typedef struct { bool is_successful; process_name_t proc_name; } launch_response_t; */ static void build_launch_response(launch_response_t *launch_response_array, int array_size, avro_slice_t **slice) { char filename[FILE_NAME_LEN]; char buf[BUFFER_SIZE]; long len = 0; avro_schema_t schema; avro_value_iface_t *iface; avro_value_t record; avro_value_t results_value, LaunchResult_value, is_successful_value, name_value, jobid_value, vpid_value; size_t index; int i; avro_writer_t writer; sprintf(filename, "%s/%s", SCHEMA_PATH, "LaunchResponseRecordAvro.avsc"); init_schema(filename, &schema); iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &record); avro_value_get_by_name(&record, "results", &results_value, &index); for (i = 0; i < array_size; i++) { avro_value_append(&results_value, &LaunchResult_value, &index); avro_value_get_by_name(&LaunchResult_value, "is_successful", &is_successful_value, &index); avro_value_set_boolean(&is_successful_value, launch_response_array[i].is_successful); avro_value_get_by_name(&LaunchResult_value, "name", &name_value, &index); avro_value_get_by_name(&name_value, "jobid", &jobid_value, &index); avro_value_set_int(&jobid_value, launch_response_array[i].proc_name.jobid); avro_value_get_by_name(&name_value, "vpid", &vpid_value, &index); avro_value_set_int(&vpid_value, launch_response_array[i].proc_name.vpid); } /* create a writer with memory buffer */ writer = avro_writer_memory(buf, sizeof(buf)); /* write record to writer (buffer) */ if (avro_value_write(writer, &record)) { fprintf(stderr, "Unable to write record to memory buffer\n"); fprintf(stderr, "Error: %s\n", avro_strerror()); exit(1); } avro_writer_flush(writer); len = avro_writer_tell(writer); //avro_generic_value_free(&record); avro_value_iface_decref(iface); avro_schema_decref(schema); *slice = xmalloc(sizeof(avro_slice_t)); (*slice)->buffer = xmalloc(len); (*slice)->len = len; memcpy((*slice)->buffer, buf, len); }
avro_schema_t schema_for_begin_txn() { avro_schema_t record_schema = avro_schema_record("BeginTxn", PROTOCOL_SCHEMA_NAMESPACE); avro_schema_t field_schema = avro_schema_long(); avro_schema_record_field_append(record_schema, "xid", field_schema); avro_schema_decref(field_schema); return record_schema; }
static void output_avro_shutdown(LogicalDecodingContext *ctx) { plugin_state *state = ctx->output_plugin_private; MemoryContextDelete(state->memctx); schema_cache_free(state->schema_cache); avro_value_decref(&state->frame_value); avro_value_iface_decref(state->frame_iface); avro_schema_decref(state->frame_schema); }
static int test_fixed(void) { char bytes[] = { 0xD, 0xA, 0xD, 0xA, 0xB, 0xA, 0xB, 0xA }; avro_schema_t schema = avro_schema_fixed("msg", sizeof(bytes)); avro_datum_t datum = avro_wrapfixed("msg", bytes, sizeof(bytes)); write_read_check(schema, NULL, datum, "fixed"); avro_datum_decref(datum); avro_schema_decref(schema); return 0; }
static int test_bytes(void) { char bytes[] = { 0xDE, 0xAD, 0xBE, 0xEF }; avro_schema_t writer_schema = avro_schema_bytes(); avro_datum_t datum = avro_wrapbytes(bytes, sizeof(bytes)); write_read_check(writer_schema, NULL, datum, "bytes"); avro_datum_decref(datum); avro_schema_decref(writer_schema); return 0; }
static int record_free_foreach(int i, struct avro_record_field_t *field, void *arg) { AVRO_UNUSED(i); AVRO_UNUSED(arg); avro_str_free(field->name); avro_schema_decref(field->type); avro_freet(struct avro_record_field_t, field); return ST_DELETE; }
static int test_map(void) { avro_schema_t schema = avro_schema_map(avro_schema_long()); avro_datum_t datum = avro_map(schema); int64_t i = 0; char *nums[] = { "zero", "one", "two", "three", "four", "five", "six", NULL }; while (nums[i]) { avro_datum_t i_datum = avro_int64(i); avro_map_set(datum, nums[i], i_datum); avro_datum_decref(i_datum); i++; } if (avro_array_size(datum) != 7) { fprintf(stderr, "Unexpected map size\n"); exit(EXIT_FAILURE); } avro_datum_t value; const char *key; avro_map_get_key(datum, 2, &key); avro_map_get(datum, key, &value); int64_t val; avro_int64_get(value, &val); if (val != 2) { fprintf(stderr, "Unexpected map value 2\n"); exit(EXIT_FAILURE); } int index; if (avro_map_get_index(datum, "two", &index)) { fprintf(stderr, "Can't get index for key \"two\": %s\n", avro_strerror()); exit(EXIT_FAILURE); } if (index != 2) { fprintf(stderr, "Unexpected index for key \"two\"\n"); exit(EXIT_FAILURE); } if (!avro_map_get_index(datum, "foobar", &index)) { fprintf(stderr, "Unexpected index for key \"foobar\"\n"); exit(EXIT_FAILURE); } write_read_check(schema, datum, NULL, NULL, "map"); test_json(datum, "{\"zero\": 0, \"one\": 1, \"two\": 2, \"three\": 3, " "\"four\": 4, \"five\": 5, \"six\": 6}"); avro_datum_decref(datum); avro_schema_decref(schema); return 0; }
static int test_enum(void) { enum avro_languages { AVRO_C, AVRO_CPP, AVRO_PYTHON, AVRO_RUBY, AVRO_JAVA }; avro_schema_t schema = avro_schema_enum("language"); avro_datum_t datum = avro_enum(schema, AVRO_C); avro_schema_enum_symbol_append(schema, "C"); avro_schema_enum_symbol_append(schema, "C++"); avro_schema_enum_symbol_append(schema, "Python"); avro_schema_enum_symbol_append(schema, "Ruby"); avro_schema_enum_symbol_append(schema, "Java"); if (avro_enum_get(datum) != AVRO_C) { fprintf(stderr, "Unexpected enum value AVRO_C\n"); exit(EXIT_FAILURE); } if (strcmp(avro_enum_get_name(datum), "C") != 0) { fprintf(stderr, "Unexpected enum value name C\n"); exit(EXIT_FAILURE); } write_read_check(schema, datum, NULL, NULL, "enum"); test_json(datum, "\"C\""); avro_enum_set(datum, AVRO_CPP); if (strcmp(avro_enum_get_name(datum), "C++") != 0) { fprintf(stderr, "Unexpected enum value name C++\n"); exit(EXIT_FAILURE); } write_read_check(schema, datum, NULL, NULL, "enum"); test_json(datum, "\"C++\""); avro_enum_set_name(datum, "Python"); if (avro_enum_get(datum) != AVRO_PYTHON) { fprintf(stderr, "Unexpected enum value AVRO_PYTHON\n"); exit(EXIT_FAILURE); } write_read_check(schema, datum, NULL, NULL, "enum"); test_json(datum, "\"Python\""); avro_datum_decref(datum); avro_schema_decref(schema); return 0; }
static int test_float(void) { int i; avro_schema_t schema = avro_schema_double(); for (i = 0; i < 100; i++) { avro_datum_t datum = avro_double(rand_number(-1.0E10, 1.0E10)); write_read_check(schema, NULL, datum, "float"); avro_datum_decref(datum); } avro_schema_decref(schema); return 0; }
static int test_boolean(void) { int i; avro_schema_t schema = avro_schema_boolean(); for (i = 0; i <= 1; i++) { avro_datum_t datum = avro_boolean(i); write_read_check(schema, NULL, datum, "boolean"); avro_datum_decref(datum); } avro_schema_decref(schema); return 0; }
static int test_int64(void) { int i; avro_schema_t writer_schema = avro_schema_long(); for (i = 0; i < 100; i++) { avro_datum_t datum = avro_int64(rand_int64()); write_read_check(writer_schema, NULL, datum, "long"); avro_datum_decref(datum); } avro_schema_decref(writer_schema); return 0; }
/* Returns 0 on success. On failure, sets mapper->error and returns nonzero. */ int table_metadata_update_schema(table_mapper_t mapper, table_metadata_t table, int is_key, const char* schema_json, size_t schema_len) { int prev_schema_id = is_key ? table->key_schema_id : table->row_schema_id; int schema_id = TABLE_MAPPER_SCHEMA_ID_MISSING; int err; if (mapper->registry) { err = schema_registry_request(mapper->registry, rd_kafka_topic_name(table->topic), is_key, schema_json, schema_len, &schema_id); if (err) { mapper_error(mapper, "Failed to register %s schema: %s", is_key ? "key" : "row", mapper->registry->error); return err; } table_metadata_set_schema_id(table, is_key, schema_id); } avro_schema_t schema; /* If running with a schema registry, we can use the registry to detect * if the schema we just saw is the same as the one we remembered * previously (since the registry guarantees to return the same id for * identical schemas). If the registry returns the same id as before, we * can skip parsing the new schema and just keep the previous one. * * However, if we're running without a registry, it's not so easy to detect * whether or not the schema changed, so in that case we just always parse * the new schema. (We could store the previous schema JSON and strcmp() * it with the new JSON, but that probably wouldn't save much over just * parsing the JSON, given this isn't a hot code path.) */ if (prev_schema_id == TABLE_MAPPER_SCHEMA_ID_MISSING || prev_schema_id != schema_id) { if (schema_json) { err = avro_schema_from_json_length(schema_json, schema_len, &schema); if (err) { mapper_error(mapper, "Could not parse %s schema: %s", is_key ? "key" : "row", avro_strerror()); return err; } } else { schema = NULL; } table_metadata_set_schema(table, is_key, schema); if (schema) avro_schema_decref(schema); } return 0; }
void process_file(FILE *input, avro_file_writer_t out, avro_schema_t schema, int verbose, int memstat, int errabort, int strjson, size_t max_str_sz) { json_error_t err; json_t *json; int n = 0; json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err); while (!feof(input)) { n++; if (verbose && !(n % 1000)) printf("Processing record %d\n", n); if (!json) { if (errabort) { fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, aborting.\n", n, err.column, err.position, err.text); return; } fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, skipping to EOL\n", n, err.column, err.position, err.text); while (getc(input) != '\n' && !feof(input)) {}; json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err); continue; } avro_value_t record; avro_value_iface_t *iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &record); if (!schema_traverse(schema, json, NULL, &record, 0, strjson, max_str_sz)) { if (avro_file_writer_append_value(out, &record)) { fprintf(stderr, "ERROR: avro_file_writer_append_value() FAILED: %s\n", avro_strerror()); exit(EXIT_FAILURE); } } else fprintf(stderr, "Error processing record %d, skipping...\n", n); avro_value_iface_decref(iface); avro_value_decref(&record); json_decref(json); if (memstat && !(n % 1000)) memory_status(); json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err); } if (memstat) memory_status(); avro_schema_decref(schema); }
static int l_schema_gc(lua_State *L) { LuaAvroSchema *l_schema = luaL_checkudata(L, 1, MT_AVRO_SCHEMA); if (l_schema->schema != NULL) { avro_schema_decref(l_schema->schema); l_schema->schema = NULL; } if (l_schema->iface != NULL) { avro_value_iface_decref(l_schema->iface); l_schema->iface = NULL; } return 0; }
static int test_boolean(void) { int i; const char *expected_json[] = { "false", "true" }; avro_schema_t schema = avro_schema_boolean(); for (i = 0; i <= 1; i++) { avro_datum_t datum = avro_boolean(i); write_read_check(schema, datum, NULL, NULL, "boolean"); test_json(datum, expected_json[i]); avro_datum_decref(datum); } avro_schema_decref(schema); return 0; }
static void process_file(const char *in_filename, const char *out_filename) { avro_file_reader_t reader; avro_file_writer_t writer; if (in_filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); exit(1); } } else { if (avro_file_reader(in_filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", in_filename, avro_strerror()); exit(1); } } avro_schema_t wschema; avro_value_iface_t *iface; avro_value_t value; wschema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); if (avro_file_writer_create_with_codec (out_filename, wschema, &writer, codec, block_size)) { fprintf(stderr, "Error creating %s:\n %s\n", out_filename, avro_strerror()); exit(1); } while (avro_file_reader_read_value(reader, &value) == 0) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "Error writing to %s:\n %s\n", out_filename, avro_strerror()); exit(1); } avro_value_reset(&value); } avro_file_reader_close(reader); avro_file_writer_close(writer); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); }
/* Frees all the memory structures associated with a frame reader. */ void frame_reader_free(frame_reader_t reader) { avro_reader_free(reader->avro_reader); avro_value_decref(&reader->frame_value); avro_value_iface_decref(reader->frame_iface); avro_schema_decref(reader->frame_schema); for (int i = 0; i < reader->num_schemas; i++) { schema_list_entry *entry = reader->schemas[i]; schema_list_entry_decrefs(entry); free(entry); } free(reader->schemas); free(reader); }
static int test_float(void) { int i; avro_schema_t schema = avro_schema_float(); avro_schema_t double_schema = avro_schema_double(); for (i = 0; i < 100; i++) { float value = rand_number(-1.0E10, 1.0E10); avro_datum_t datum = avro_float(value); avro_datum_t double_datum = avro_double(value); write_read_check(schema, datum, NULL, NULL, "float"); write_read_check(schema, datum, double_schema, double_datum, "float->double"); avro_datum_decref(datum); avro_datum_decref(double_datum); } avro_datum_t datum = avro_float(2000.0); test_json(datum, "2000.0"); avro_datum_decref(datum); avro_schema_decref(schema); avro_schema_decref(double_schema); return 0; }
static int test_int32(void) { int i; avro_schema_t writer_schema = avro_schema_int(); avro_schema_t long_schema = avro_schema_long(); avro_schema_t float_schema = avro_schema_float(); avro_schema_t double_schema = avro_schema_double(); for (i = 0; i < 100; i++) { int32_t value = rand_int32(); avro_datum_t datum = avro_int32(value); avro_datum_t long_datum = avro_int64(value); avro_datum_t float_datum = avro_float(value); avro_datum_t double_datum = avro_double(value); write_read_check(writer_schema, datum, NULL, NULL, "int"); write_read_check(writer_schema, datum, long_schema, long_datum, "int->long"); write_read_check(writer_schema, datum, float_schema, float_datum, "int->float"); write_read_check(writer_schema, datum, double_schema, double_datum, "int->double"); avro_datum_decref(datum); avro_datum_decref(long_datum); avro_datum_decref(float_datum); avro_datum_decref(double_datum); } avro_datum_t datum = avro_int32(10000); test_json(datum, "10000"); avro_datum_decref(datum); avro_schema_decref(writer_schema); avro_schema_decref(long_schema); avro_schema_decref(float_schema); avro_schema_decref(double_schema); return 0; }
avro_schema_t r_kv_get_schema(kv_store_t *kvstore, const char *space, const char *name) { int nsch, i; avro_schema_t *schemas = NULL, schema = NULL; if (!kvstore || !name) { return NULL; } schema = avro_schema_record(name, space); nsch = kv_avro_get_current_schemas(kvstore, &schemas); PRINTF("r_kv_get_schema, name = %s, space = %s, nsch = %d\n", name, space, nsch); for (i = 0; i < nsch; i++) { avro_schema_t sch = schemas[i]; if (avro_typeof(sch) == AVRO_RECORD && avro_schema_equal(schema, sch)) { avro_schema_decref(schema); return sch; } } avro_schema_decref(schema); return NULL; }
static int do_close(AvroDeserializer* self) { if (self->flags & DESERIALIZER_READER_OK) { avro_reader_free(self->datum_reader); self->flags &= ~DESERIALIZER_READER_OK; } if (self->flags & DESERIALIZER_SCHEMA_OK) { avro_schema_decref(self->schema); self->flags &= ~DESERIALIZER_SCHEMA_OK; } if (self->iface != NULL) { avro_value_iface_decref(self->iface); self->iface = NULL; } return 0; }
static int test_union(void) { avro_schema_t schema = avro_schema_union(); avro_datum_t union_datum; avro_datum_t datum; avro_datum_t union_datum1; avro_datum_t datum1; avro_schema_union_append(schema, avro_schema_string()); avro_schema_union_append(schema, avro_schema_int()); avro_schema_union_append(schema, avro_schema_null()); datum = avro_givestring("Follow your bliss.", NULL); union_datum = avro_union(schema, 0, datum); if (avro_union_discriminant(union_datum) != 0) { fprintf(stderr, "Unexpected union discriminant\n"); exit(EXIT_FAILURE); } if (avro_union_current_branch(union_datum) != datum) { fprintf(stderr, "Unexpected union branch datum\n"); exit(EXIT_FAILURE); } union_datum1 = avro_datum_from_schema(schema); avro_union_set_discriminant(union_datum1, 0, &datum1); avro_givestring_set(datum1, "Follow your bliss.", NULL); if (!avro_datum_equal(datum, datum1)) { fprintf(stderr, "Union values should be equal\n"); exit(EXIT_FAILURE); } write_read_check(schema, union_datum, NULL, NULL, "union"); test_json(union_datum, "{\"string\": \"Follow your bliss.\"}"); avro_datum_decref(datum); avro_union_set_discriminant(union_datum, 2, &datum); test_json(union_datum, "null"); avro_datum_decref(union_datum); avro_datum_decref(datum); avro_datum_decref(union_datum1); avro_schema_decref(schema); return 0; }
static int test_double(void) { int i; avro_schema_t schema = avro_schema_double(); for (i = 0; i < 100; i++) { avro_datum_t datum = avro_double(rand_number(-1.0E10, 1.0E10)); write_read_check(schema, datum, NULL, NULL, "double"); avro_datum_decref(datum); } avro_datum_t datum = avro_double(2000.0); test_json(datum, "2000.0"); avro_datum_decref(datum); avro_schema_decref(schema); return 0; }
static int test_string(void) { unsigned int i; const char *strings[] = { "Four score and seven years ago", "our father brought forth on this continent", "a new nation", "conceived in Liberty", "and dedicated to the proposition that all men are created equal." }; avro_schema_t writer_schema = avro_schema_string(); for (i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { avro_datum_t datum = avro_wrapstring(strings[i]); write_read_check(writer_schema, NULL, datum, "string"); avro_datum_decref(datum); } avro_schema_decref(writer_schema); return 0; }
static int read_data() { int rval; int records_read = 0; avro_file_reader_t reader; avro_value_iface_t *iface; avro_value_t value; fprintf(stderr, "\nReading...\n"); rval = avro_file_reader(filename, &reader); if (rval) { fprintf(stderr, "Error: %s\n", avro_strerror()); return -1; } avro_schema_t schema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &value); while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { avro_value_t field; int32_t val; avro_value_get_by_index(&value, 0, &field, NULL); avro_value_get_int(&field, &val); fprintf(stderr, "value = %d\n", val); records_read++; avro_value_reset(&value); } avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(schema); avro_file_reader_close(reader); fprintf(stderr, "read %d records.\n", records_read); if (rval != EOF) { fprintf(stderr, "Error: %s\n", avro_strerror()); return -1; } return records_read; }
static int write_data(int n_records) { int i; avro_schema_t schema; avro_schema_error_t error; avro_file_writer_t writer; avro_value_iface_t *iface; avro_value_t value; fprintf(stderr, "\nWriting...\n"); if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { fprintf(stderr, "Unable to parse schema\n"); return -1; } if (avro_file_writer_create(filename, schema, &writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &value); avro_value_t field; avro_value_get_by_index(&value, 0, &field, NULL); avro_value_set_int(&field, 123); for (i = 0; i < n_records; i++) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "There was an error writing file: %s\n", avro_strerror()); return -1; } } if (avro_file_writer_close(writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(schema); return n_records; }
static int test_map(void) { avro_schema_t schema = avro_schema_map(avro_schema_long()); avro_datum_t datum = avro_map(); int64_t i = 0; char *nums[] = { "zero", "one", "two", "three", "four", "five", "six", NULL }; while (nums[i]) { avro_datum_t i_datum = avro_int64(i); avro_map_set(datum, nums[i], i_datum); avro_datum_decref(i_datum); i++; } write_read_check(schema, NULL, datum, "map"); avro_datum_decref(datum); avro_schema_decref(schema); return 0; }