static void test_fixed_deserialize(void **state) { (void)state; const uint8_t plain_fixed1[] = { 0x0, 0x1, 0x2, 0x3, 0x4 }; size_t plain_fixed1_size = sizeof(plain_fixed1) / sizeof(char); kaa_bytes_t *kaa_fixed1 = kaa_fixed_copy_create(plain_fixed1, plain_fixed1_size); ASSERT_NOT_NULL(kaa_fixed1); size_t expected_size = kaa_fixed_get_size(kaa_fixed1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_fixed_serialize(avro_writer, kaa_fixed1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); kaa_bytes_t *kaa_fixed2 = kaa_fixed_deserialize(avro_reader, &expected_size); ASSERT_NOT_NULL(kaa_fixed2); ASSERT_EQUAL(memcmp(kaa_fixed2->buffer, plain_fixed1, plain_fixed1_size), 0); ASSERT_EQUAL(memcmp(kaa_fixed2->buffer, kaa_fixed1->buffer, plain_fixed1_size), 0); kaa_fixed_destroy(kaa_fixed2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); kaa_fixed_destroy(kaa_fixed1); }
static void test_string_deserialize(void **state) { (void)state; const char *plain_test_str1 = "test"; kaa_string_t *kaa_str1 = kaa_string_copy_create(plain_test_str1); ASSERT_NOT_NULL(kaa_str1); size_t expected_size = kaa_string_get_size(kaa_str1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_string_serialize(avro_writer, kaa_str1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); kaa_string_t *kaa_str2 = kaa_string_deserialize(avro_reader); ASSERT_NOT_NULL(kaa_str2); ASSERT_EQUAL(strcmp(kaa_str2->data, plain_test_str1), 0); ASSERT_EQUAL(strcmp(kaa_str2->data, kaa_str1->data), 0); kaa_string_destroy(kaa_str2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); kaa_string_destroy(kaa_str1); }
/* { "type": "record", "name": "FinishRequest", "fields": [ {"name": "succeed", "type": "boolean"}, {"name": "diagnostics", "type": "string"} ] } */ static void parse_finish_request(avro_slice_t *slice, bool *succeed, char **diag) { char filename[FILE_NAME_LEN]; avro_schema_t schema; avro_value_iface_t *iface; avro_value_t record, succeed_value, diag_value; size_t index; avro_reader_t reader; size_t size; sprintf(filename, "%s/%s", SCHEMA_PATH, "FinishRequestRecordAvro.avsc"); init_schema(filename, &schema); iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &record); reader = avro_reader_memory(slice->buffer, slice->len); if (avro_value_read(reader, &record)) { fprintf(stderr, "Unable to read record from memory buffer\n"); fprintf(stderr, "Error: %s\n", avro_strerror()); exit(1); } avro_value_get_by_name(&record, "succeed", &succeed_value, &index); avro_value_get_boolean(&succeed_value, succeed); avro_value_get_by_name(&record, "diagnostics", &diag_value, &index); avro_value_get_string(&diag_value, diag, &size); //avro_generic_value_free(&record); avro_value_iface_decref(iface); avro_schema_decref(schema); }
int avro_bin_to_json(avro_schema_t schema, const void *val_bin, size_t val_len, char **val_out, size_t *val_len_out) { if (!val_bin) { *val_out = NULL; return 0; } else if (!schema) { log_error("json: got a value where we didn't expect one, and no schema to decode it"); *val_out = NULL; return EINVAL; } avro_reader_t reader = avro_reader_memory(val_bin, val_len); avro_value_iface_t *iface = avro_generic_class_from_schema(schema); if (!iface) { log_error("json: error in avro_generic_class_from_schema: %s", avro_strerror()); avro_reader_free(reader); return EINVAL; } int err; avro_value_t value; err = avro_generic_value_new(iface, &value); if (err) { log_error("json: error in avro_generic_value_new: %s", avro_strerror()); avro_value_iface_decref(iface); avro_reader_free(reader); return err; } err = avro_value_read(reader, &value); if (err) { log_error("json: error decoding Avro value: %s", avro_strerror()); avro_value_decref(&value); avro_value_iface_decref(iface); avro_reader_free(reader); return err; } err = avro_value_to_json(&value, 1, val_out); if (err) { log_error("json: error converting Avro value to JSON: %s", avro_strerror()); avro_value_decref(&value); avro_value_iface_decref(iface); avro_reader_free(reader); return err; } *val_len_out = strlen(*val_out); // not including null terminator - to librdkafka it's just bytes avro_value_decref(&value); avro_value_iface_decref(iface); avro_reader_free(reader); return 0; }
static int AvroDeserializer_init(AvroDeserializer *self, PyObject *args, PyObject *kwds) { int rval; PyObject *types = NULL; const char *schema_json; static char *kwlist[] = {"schema", "types", NULL}; self->flags = 0; self->iface = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|O", kwlist, &schema_json, &types)) { return -1; } rval = avro_schema_from_json(schema_json, 0, &self->schema, NULL); if (rval != 0 || self->schema == NULL) { PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror()); return -1; } self->flags |= DESERIALIZER_SCHEMA_OK; self->iface = avro_generic_class_from_schema(self->schema); if (self->iface == NULL) { PyErr_SetString(PyExc_IOError, "Error creating generic class interface"); return -1; } self->datum_reader = avro_reader_memory(0, 0); if (!self->datum_reader) { PyErr_NoMemory(); return -1; } self->flags |= DESERIALIZER_READER_OK; /* copied verbatim from filereader */ if (types != NULL && PyObject_IsTrue(types)) { /* we still haven't incref'ed types here */ if (Py_TYPE(types) == get_avro_types_type()) { Py_INCREF(types); self->info.types = types; } else { self->info.types = PyObject_CallFunctionObjArgs( (PyObject *) get_avro_types_type(), NULL); if (self->info.types == NULL) { return -1; } declare_types(&self->info, self->schema); } } else { self->info.types = NULL; } return 0; }
static kaa_root_configuration_t *kaa_configuration_manager_deserialize(const char *buffer, size_t buffer_size) { KAA_RETURN_IF_NIL2(buffer, buffer_size, NULL); avro_reader_t reader = avro_reader_memory(buffer, buffer_size); KAA_RETURN_IF_NIL(reader, NULL); kaa_root_configuration_t *result = KAA_CONFIGURATION_DESERIALIZE(reader); avro_reader_free(reader); return result; }
void write_read_check(avro_schema_t writers_schema, avro_datum_t datum, avro_schema_t readers_schema, avro_datum_t expected, char *type) { avro_datum_t datum_out; int validate; for (validate = 0; validate <= 1; validate++) { reader = avro_reader_memory(buf, sizeof(buf)); writer = avro_writer_memory(buf, sizeof(buf)); if (!expected) { expected = datum; } /* Validating read/write */ if (avro_write_data (writer, validate ? writers_schema : NULL, datum)) { fprintf(stderr, "Unable to write %s validate=%d\n %s\n", type, validate, avro_strerror()); exit(EXIT_FAILURE); } int64_t size = avro_size_data(writer, validate ? writers_schema : NULL, datum); if (size != avro_writer_tell(writer)) { fprintf(stderr, "Unable to calculate size %s validate=%d " "(%"PRId64" != %"PRId64")\n %s\n", type, validate, size, avro_writer_tell(writer), avro_strerror()); exit(EXIT_FAILURE); } if (avro_read_data (reader, writers_schema, readers_schema, &datum_out)) { fprintf(stderr, "Unable to read %s validate=%d\n %s\n", type, validate, avro_strerror()); fprintf(stderr, " %s\n", avro_strerror()); exit(EXIT_FAILURE); } if (!avro_datum_equal(expected, datum_out)) { fprintf(stderr, "Unable to encode/decode %s validate=%d\n %s\n", type, validate, avro_strerror()); exit(EXIT_FAILURE); } avro_reader_dump(reader, stderr); avro_datum_decref(datum_out); avro_reader_free(reader); avro_writer_free(writer); } }
static void test_null_deserialize(void **state) { (void)state; size_t expected_size = rand() % 10; char buffer[expected_size]; memset(buffer, rand(), expected_size); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); ASSERT_NULL(kaa_null_deserialize(avro_reader)); avro_reader_free(avro_reader); }
frame_reader_t frame_reader_new() { frame_reader_t reader = malloc(sizeof(frame_reader)); check_alloc(reader); memset(reader, 0, sizeof(frame_reader)); reader->num_schemas = 0; reader->capacity = 16; reader->schemas = malloc(reader->capacity * sizeof(void*)); check_alloc(reader->schemas); reader->frame_schema = schema_for_frame(); reader->frame_iface = avro_generic_class_from_schema(reader->frame_schema); avro_generic_value_new(reader->frame_iface, &reader->frame_value); reader->avro_reader = avro_reader_memory(NULL, 0); return reader; }
int process_frame_table_schema(avro_value_t *record_val, frame_reader_t reader, uint64_t wal_pos) { int err = 0, key_schema_present; avro_value_t relid_val, hash_val, key_schema_val, row_schema_val, branch_val; int64_t relid; const void *hash; const char *key_schema_json = NULL, *row_schema_json; size_t hash_len, key_schema_len = 1, row_schema_len; avro_schema_t key_schema = NULL, row_schema; check(err, avro_value_get_by_index(record_val, 0, &relid_val, NULL)); check(err, avro_value_get_by_index(record_val, 1, &hash_val, NULL)); check(err, avro_value_get_by_index(record_val, 2, &key_schema_val, NULL)); check(err, avro_value_get_by_index(record_val, 3, &row_schema_val, NULL)); check(err, avro_value_get_long(&relid_val, &relid)); check(err, avro_value_get_fixed(&hash_val, &hash, &hash_len)); check(err, avro_value_get_discriminant(&key_schema_val, &key_schema_present)); check(err, avro_value_get_string(&row_schema_val, &row_schema_json, &row_schema_len)); check(err, avro_schema_from_json_length(row_schema_json, row_schema_len - 1, &row_schema)); schema_list_entry *entry = schema_list_replace(reader, relid); entry->relid = relid; entry->hash = *((uint64_t *) hash); entry->row_schema = row_schema; entry->row_iface = avro_generic_class_from_schema(row_schema); avro_generic_value_new(entry->row_iface, &entry->row_value); avro_generic_value_new(entry->row_iface, &entry->old_value); entry->avro_reader = avro_reader_memory(NULL, 0); if (key_schema_present) { check(err, avro_value_get_current_branch(&key_schema_val, &branch_val)); check(err, avro_value_get_string(&branch_val, &key_schema_json, &key_schema_len)); check(err, avro_schema_from_json_length(key_schema_json, key_schema_len - 1, &key_schema)); entry->key_schema = key_schema; entry->key_iface = avro_generic_class_from_schema(key_schema); avro_generic_value_new(entry->key_iface, &entry->key_value); } else { entry->key_schema = NULL; } if (reader->on_table_schema) { check(err, reader->on_table_schema(reader->cb_context, wal_pos, relid, key_schema_json, key_schema_len - 1, key_schema, row_schema_json, row_schema_len - 1, row_schema)); } return err; }
extern int parse_heartbeat_request(avro_slice_t *slice) { char filename[FILE_NAME_LEN]; avro_schema_t schema; avro_value_iface_t *iface; avro_value_t record; size_t index; avro_reader_t reader; size_t size = 0; sprintf(filename, "%s/%s", SCHEMA_PATH, "HeartBeatRequestRecordAvro.avsc"); init_schema(filename, &schema); iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &record); reader = avro_reader_memory(slice->buffer, slice->len); if (avro_value_read(reader, &record)) { fprintf(stderr, "Unable to read record from memory buffer\n"); fprintf(stderr, "Error: %s\n", avro_strerror()); exit(1); } avro_value_get_size(&record, &size); avro_value_iface_decref(iface); avro_schema_decref(schema); // printf("slice->len = %d\n", slice->len); // printf("size = %ld\n", size); // // if (size > 0) { // return 0; // } else { // return -1; // } if (size == 0) { return 0; } else { return -1; } }
static void test_array_deserialize_w_ctx(void **state) { (void)state; const uint8_t plain_fixed[] = { 0x0, 0x1, 0x2, 0x3, 0x4 }; size_t plain_fixed_size = sizeof(plain_fixed) / sizeof(char); size_t array_size = 1 + rand() % 10; kaa_list_t *avro_array1 = kaa_list_create(); size_t i = 0; for (i = 0; i < array_size; ++i) { kaa_list_push_back(avro_array1, kaa_fixed_copy_create(plain_fixed, plain_fixed_size)); } size_t buffer_size = kaa_array_get_size(avro_array1, kaa_fixed_get_size); char buffer[buffer_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, buffer_size); kaa_array_serialize(avro_writer, avro_array1, kaa_fixed_serialize); avro_reader_t avro_reader = avro_reader_memory(buffer, buffer_size); kaa_list_t *avro_array2 = kaa_array_deserialize_w_ctx(avro_reader, (deserialize_w_ctx_fn)kaa_fixed_deserialize, &plain_fixed_size); ASSERT_NOT_NULL(avro_array2); ASSERT_EQUAL(kaa_list_get_size(avro_array2), array_size); kaa_list_node_t *it = kaa_list_begin(avro_array2); while (it) { kaa_bytes_t *fixed = kaa_list_get_data(it); ASSERT_NOT_NULL(fixed); ASSERT_EQUAL((size_t)fixed->size, plain_fixed_size); ASSERT_EQUAL(memcmp(fixed->buffer, plain_fixed, plain_fixed_size), 0); it = kaa_list_next(it); } kaa_list_destroy(avro_array2, kaa_fixed_destroy); avro_reader_free(avro_reader); avro_writer_free(avro_writer); kaa_list_destroy(avro_array1, kaa_fixed_destroy); }
static void test_float_deserialize(void **state) { (void)state; float float_value1 = rand() / rand(); size_t expected_size = kaa_float_get_size(&float_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_float_serialize(avro_writer, &float_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); float *float_value2 = kaa_float_deserialize(avro_reader); ASSERT_EQUAL(*float_value2, float_value1); kaa_data_destroy(float_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_boolean_deserialize(void **state) { (void)state; int8_t boolean_value1 = true; size_t expected_size = kaa_boolean_get_size(&boolean_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_boolean_serialize(avro_writer, &boolean_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); int8_t *boolean_value2 = kaa_boolean_deserialize(avro_reader); ASSERT_EQUAL(*boolean_value2, boolean_value1); kaa_data_destroy(boolean_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_int_deserialize(void **state) { (void)state; int32_t int_value1 = rand(); size_t expected_size = kaa_int_get_size(&int_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_int_serialize(avro_writer, &int_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); int32_t *int_value2 = kaa_int_deserialize(avro_reader); ASSERT_EQUAL(*int_value2, int_value1); kaa_data_destroy(int_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_long_deserialize(void **state) { (void)state; int64_t long_value1 = rand(); size_t expected_size = kaa_long_get_size(&long_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_long_serialize(avro_writer, &long_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); int64_t *long_value2 = kaa_long_deserialize(avro_reader); ASSERT_EQUAL(*long_value2, long_value1); kaa_data_destroy(long_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_enum_deserialize(void **state) { (void)state; test_enum_t enum_value1 = (test_enum_t)rand() % TEST_VAL_5; size_t expected_size = kaa_enum_get_size(&enum_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_enum_serialize(avro_writer, &enum_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); test_enum_t *enum_value2 = (test_enum_t *)kaa_enum_deserialize(avro_reader); ASSERT_EQUAL(*enum_value2, enum_value1); kaa_data_destroy(enum_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_double_deserialize(void **state) { (void)state; double double_value1 = rand() / rand(); size_t expected_size = kaa_double_get_size(&double_value1); char buffer[expected_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size); kaa_double_serialize(avro_writer, &double_value1); avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size); double *double_value2 = kaa_double_deserialize(avro_reader); ASSERT_EQUAL(*double_value2, double_value1); kaa_data_destroy(double_value2); avro_reader_free(avro_reader); avro_writer_free(avro_writer); }
static void test_array_deserialize_wo_ctx(void **state) { (void)state; size_t array_size = 1 + rand() % 10; kaa_list_t *avro_array1 = kaa_list_create(); size_t i = 0; for (i = 0; i < array_size; ++i) { kaa_list_push_back(avro_array1, create_float()); } size_t buffer_size = kaa_array_get_size(avro_array1, kaa_float_get_size); char buffer[buffer_size]; avro_writer_t avro_writer = avro_writer_memory(buffer, buffer_size); kaa_array_serialize(avro_writer, avro_array1, kaa_float_serialize); avro_reader_t avro_reader = avro_reader_memory(buffer, buffer_size); kaa_list_t *avro_array2 = kaa_array_deserialize_wo_ctx(avro_reader, (deserialize_wo_ctx_fn)kaa_float_deserialize); ASSERT_NOT_NULL(avro_array2); ASSERT_EQUAL(kaa_list_get_size(avro_array2), array_size); // /* // * Deserialized array has a back order, so reverse the original. // */ // kaa_list_t *reverse_avro_array = NULL; // kaa_list_t *it = avro_array1; // while (it) { // if (reverse_avro_array) { // reverse_avro_array = kaa_list_push_front(reverse_avro_array, kaa_list_get_data(it)); // } else { // reverse_avro_array = kaa_list_create(kaa_list_get_data(it)); // } // it = kaa_list_next(it); // } // // kaa_list_destroy(avro_array1, kaa_null_destroy); // avro_array1 = reverse_avro_array; /* * Compare origin and deserialized arrays. */ kaa_list_node_t *it1 = kaa_list_begin(avro_array1); kaa_list_node_t *it2 = kaa_list_begin(avro_array2); while (it1 && it2) { float *float_value1 = kaa_list_get_data(it1); float *float_value2 = kaa_list_get_data(it2); ASSERT_NOT_NULL(float_value1); ASSERT_NOT_NULL(float_value2); ASSERT_EQUAL(*float_value2, *float_value1); it1 = kaa_list_next(it1); it2 = kaa_list_next(it2); } kaa_list_destroy(avro_array2, kaa_data_destroy); avro_reader_free(avro_reader); avro_writer_free(avro_writer); kaa_list_destroy(avro_array1, kaa_data_destroy); }