static void test_fixed_deserialize(void **state)
{
(void)state;
const uint8_t plain_fixed1[] = { 0x0, 0x1, 0x2, 0x3, 0x4 };
size_t plain_fixed1_size = sizeof(plain_fixed1) / sizeof(char);
kaa_bytes_t *kaa_fixed1 = kaa_fixed_copy_create(plain_fixed1, plain_fixed1_size);
ASSERT_NOT_NULL(kaa_fixed1);
size_t expected_size = kaa_fixed_get_size(kaa_fixed1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_fixed_serialize(avro_writer, kaa_fixed1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
kaa_bytes_t *kaa_fixed2 = kaa_fixed_deserialize(avro_reader, &expected_size);
ASSERT_NOT_NULL(kaa_fixed2);
ASSERT_EQUAL(memcmp(kaa_fixed2->buffer, plain_fixed1, plain_fixed1_size), 0);
ASSERT_EQUAL(memcmp(kaa_fixed2->buffer, kaa_fixed1->buffer, plain_fixed1_size), 0);
kaa_fixed_destroy(kaa_fixed2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
kaa_fixed_destroy(kaa_fixed1);
}
static void test_string_deserialize(void **state)
{
(void)state;
const char *plain_test_str1 = "test";
kaa_string_t *kaa_str1 = kaa_string_copy_create(plain_test_str1);
ASSERT_NOT_NULL(kaa_str1);
size_t expected_size = kaa_string_get_size(kaa_str1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_string_serialize(avro_writer, kaa_str1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
kaa_string_t *kaa_str2 = kaa_string_deserialize(avro_reader);
ASSERT_NOT_NULL(kaa_str2);
ASSERT_EQUAL(strcmp(kaa_str2->data, plain_test_str1), 0);
ASSERT_EQUAL(strcmp(kaa_str2->data, kaa_str1->data), 0);
kaa_string_destroy(kaa_str2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
kaa_string_destroy(kaa_str1);
}
/*
{
"type": "record",
"name": "FinishRequest",
"fields": [
{"name": "succeed", "type": "boolean"},
{"name": "diagnostics", "type": "string"}
]
}
*/
static void parse_finish_request(avro_slice_t *slice, bool *succeed, char **diag)
{
char filename[FILE_NAME_LEN];
avro_schema_t schema;
avro_value_iface_t *iface;
avro_value_t record, succeed_value, diag_value;
size_t index;
avro_reader_t reader;
size_t size;
sprintf(filename, "%s/%s", SCHEMA_PATH, "FinishRequestRecordAvro.avsc");
init_schema(filename, &schema);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &record);
reader = avro_reader_memory(slice->buffer, slice->len);
if (avro_value_read(reader, &record)) {
fprintf(stderr, "Unable to read record from memory buffer\n");
fprintf(stderr, "Error: %s\n", avro_strerror());
exit(1);
}
avro_value_get_by_name(&record, "succeed", &succeed_value, &index);
avro_value_get_boolean(&succeed_value, succeed);
avro_value_get_by_name(&record, "diagnostics", &diag_value, &index);
avro_value_get_string(&diag_value, diag, &size);
//avro_generic_value_free(&record);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
}
int avro_bin_to_json(avro_schema_t schema,
const void *val_bin, size_t val_len,
char **val_out, size_t *val_len_out) {
if (!val_bin) {
*val_out = NULL;
return 0;
} else if (!schema) {
log_error("json: got a value where we didn't expect one, and no schema to decode it");
*val_out = NULL;
return EINVAL;
}
avro_reader_t reader = avro_reader_memory(val_bin, val_len);
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
if (!iface) {
log_error("json: error in avro_generic_class_from_schema: %s", avro_strerror());
avro_reader_free(reader);
return EINVAL;
}
int err;
avro_value_t value;
err = avro_generic_value_new(iface, &value);
if (err) {
log_error("json: error in avro_generic_value_new: %s", avro_strerror());
avro_value_iface_decref(iface);
avro_reader_free(reader);
return err;
}
err = avro_value_read(reader, &value);
if (err) {
log_error("json: error decoding Avro value: %s", avro_strerror());
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_reader_free(reader);
return err;
}
err = avro_value_to_json(&value, 1, val_out);
if (err) {
log_error("json: error converting Avro value to JSON: %s", avro_strerror());
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_reader_free(reader);
return err;
}
*val_len_out = strlen(*val_out); // not including null terminator - to librdkafka it's just bytes
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_reader_free(reader);
return 0;
}
static int
AvroDeserializer_init(AvroDeserializer *self, PyObject *args, PyObject *kwds)
{
int rval;
PyObject *types = NULL;
const char *schema_json;
static char *kwlist[] = {"schema", "types", NULL};
self->flags = 0;
self->iface = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|O", kwlist,
&schema_json, &types)) {
return -1;
}
rval = avro_schema_from_json(schema_json, 0, &self->schema, NULL);
if (rval != 0 || self->schema == NULL) {
PyErr_Format(PyExc_IOError, "Error reading schema: %s",
avro_strerror());
return -1;
}
self->flags |= DESERIALIZER_SCHEMA_OK;
self->iface = avro_generic_class_from_schema(self->schema);
if (self->iface == NULL) {
PyErr_SetString(PyExc_IOError,
"Error creating generic class interface");
return -1;
}
self->datum_reader = avro_reader_memory(0, 0);
if (!self->datum_reader) {
PyErr_NoMemory();
return -1;
}
self->flags |= DESERIALIZER_READER_OK;
/* copied verbatim from filereader */
if (types != NULL && PyObject_IsTrue(types)) {
/* we still haven't incref'ed types here */
if (Py_TYPE(types) == get_avro_types_type()) {
Py_INCREF(types);
self->info.types = types;
} else {
self->info.types = PyObject_CallFunctionObjArgs(
(PyObject *) get_avro_types_type(), NULL);
if (self->info.types == NULL) {
return -1;
}
declare_types(&self->info, self->schema);
}
} else {
self->info.types = NULL;
}
return 0;
}
static kaa_root_configuration_t *kaa_configuration_manager_deserialize(const char *buffer, size_t buffer_size)
{
KAA_RETURN_IF_NIL2(buffer, buffer_size, NULL);
avro_reader_t reader = avro_reader_memory(buffer, buffer_size);
KAA_RETURN_IF_NIL(reader, NULL);
kaa_root_configuration_t *result = KAA_CONFIGURATION_DESERIALIZE(reader);
avro_reader_free(reader);
return result;
}
void
write_read_check(avro_schema_t writers_schema, avro_datum_t datum,
avro_schema_t readers_schema, avro_datum_t expected, char *type)
{
avro_datum_t datum_out;
int validate;
for (validate = 0; validate <= 1; validate++) {
reader = avro_reader_memory(buf, sizeof(buf));
writer = avro_writer_memory(buf, sizeof(buf));
if (!expected) {
expected = datum;
}
/* Validating read/write */
if (avro_write_data
(writer, validate ? writers_schema : NULL, datum)) {
fprintf(stderr, "Unable to write %s validate=%d\n %s\n",
type, validate, avro_strerror());
exit(EXIT_FAILURE);
}
int64_t size =
avro_size_data(writer, validate ? writers_schema : NULL,
datum);
if (size != avro_writer_tell(writer)) {
fprintf(stderr,
"Unable to calculate size %s validate=%d "
"(%"PRId64" != %"PRId64")\n %s\n",
type, validate, size, avro_writer_tell(writer),
avro_strerror());
exit(EXIT_FAILURE);
}
if (avro_read_data
(reader, writers_schema, readers_schema, &datum_out)) {
fprintf(stderr, "Unable to read %s validate=%d\n %s\n",
type, validate, avro_strerror());
fprintf(stderr, " %s\n", avro_strerror());
exit(EXIT_FAILURE);
}
if (!avro_datum_equal(expected, datum_out)) {
fprintf(stderr,
"Unable to encode/decode %s validate=%d\n %s\n",
type, validate, avro_strerror());
exit(EXIT_FAILURE);
}
avro_reader_dump(reader, stderr);
avro_datum_decref(datum_out);
avro_reader_free(reader);
avro_writer_free(writer);
}
}
static void test_null_deserialize(void **state)
{
(void)state;
size_t expected_size = rand() % 10;
char buffer[expected_size];
memset(buffer, rand(), expected_size);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
ASSERT_NULL(kaa_null_deserialize(avro_reader));
avro_reader_free(avro_reader);
}
frame_reader_t frame_reader_new() {
frame_reader_t reader = malloc(sizeof(frame_reader));
check_alloc(reader);
memset(reader, 0, sizeof(frame_reader));
reader->num_schemas = 0;
reader->capacity = 16;
reader->schemas = malloc(reader->capacity * sizeof(void*));
check_alloc(reader->schemas);
reader->frame_schema = schema_for_frame();
reader->frame_iface = avro_generic_class_from_schema(reader->frame_schema);
avro_generic_value_new(reader->frame_iface, &reader->frame_value);
reader->avro_reader = avro_reader_memory(NULL, 0);
return reader;
}
int process_frame_table_schema(avro_value_t *record_val, frame_reader_t reader, uint64_t wal_pos) {
int err = 0, key_schema_present;
avro_value_t relid_val, hash_val, key_schema_val, row_schema_val, branch_val;
int64_t relid;
const void *hash;
const char *key_schema_json = NULL, *row_schema_json;
size_t hash_len, key_schema_len = 1, row_schema_len;
avro_schema_t key_schema = NULL, row_schema;
check(err, avro_value_get_by_index(record_val, 0, &relid_val, NULL));
check(err, avro_value_get_by_index(record_val, 1, &hash_val, NULL));
check(err, avro_value_get_by_index(record_val, 2, &key_schema_val, NULL));
check(err, avro_value_get_by_index(record_val, 3, &row_schema_val, NULL));
check(err, avro_value_get_long(&relid_val, &relid));
check(err, avro_value_get_fixed(&hash_val, &hash, &hash_len));
check(err, avro_value_get_discriminant(&key_schema_val, &key_schema_present));
check(err, avro_value_get_string(&row_schema_val, &row_schema_json, &row_schema_len));
check(err, avro_schema_from_json_length(row_schema_json, row_schema_len - 1, &row_schema));
schema_list_entry *entry = schema_list_replace(reader, relid);
entry->relid = relid;
entry->hash = *((uint64_t *) hash);
entry->row_schema = row_schema;
entry->row_iface = avro_generic_class_from_schema(row_schema);
avro_generic_value_new(entry->row_iface, &entry->row_value);
avro_generic_value_new(entry->row_iface, &entry->old_value);
entry->avro_reader = avro_reader_memory(NULL, 0);
if (key_schema_present) {
check(err, avro_value_get_current_branch(&key_schema_val, &branch_val));
check(err, avro_value_get_string(&branch_val, &key_schema_json, &key_schema_len));
check(err, avro_schema_from_json_length(key_schema_json, key_schema_len - 1, &key_schema));
entry->key_schema = key_schema;
entry->key_iface = avro_generic_class_from_schema(key_schema);
avro_generic_value_new(entry->key_iface, &entry->key_value);
} else {
entry->key_schema = NULL;
}
if (reader->on_table_schema) {
check(err, reader->on_table_schema(reader->cb_context, wal_pos, relid,
key_schema_json, key_schema_len - 1, key_schema,
row_schema_json, row_schema_len - 1, row_schema));
}
return err;
}
extern int parse_heartbeat_request(avro_slice_t *slice)
{
char filename[FILE_NAME_LEN];
avro_schema_t schema;
avro_value_iface_t *iface;
avro_value_t record;
size_t index;
avro_reader_t reader;
size_t size = 0;
sprintf(filename, "%s/%s", SCHEMA_PATH, "HeartBeatRequestRecordAvro.avsc");
init_schema(filename, &schema);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &record);
reader = avro_reader_memory(slice->buffer, slice->len);
if (avro_value_read(reader, &record)) {
fprintf(stderr, "Unable to read record from memory buffer\n");
fprintf(stderr, "Error: %s\n", avro_strerror());
exit(1);
}
avro_value_get_size(&record, &size);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
// printf("slice->len = %d\n", slice->len);
// printf("size = %ld\n", size);
//
// if (size > 0) {
// return 0;
// } else {
// return -1;
// }
if (size == 0) {
return 0;
} else {
return -1;
}
}
static void test_array_deserialize_w_ctx(void **state)
{
(void)state;
const uint8_t plain_fixed[] = { 0x0, 0x1, 0x2, 0x3, 0x4 };
size_t plain_fixed_size = sizeof(plain_fixed) / sizeof(char);
size_t array_size = 1 + rand() % 10;
kaa_list_t *avro_array1 = kaa_list_create();
size_t i = 0;
for (i = 0; i < array_size; ++i) {
kaa_list_push_back(avro_array1, kaa_fixed_copy_create(plain_fixed, plain_fixed_size));
}
size_t buffer_size = kaa_array_get_size(avro_array1, kaa_fixed_get_size);
char buffer[buffer_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, buffer_size);
kaa_array_serialize(avro_writer, avro_array1, kaa_fixed_serialize);
avro_reader_t avro_reader = avro_reader_memory(buffer, buffer_size);
kaa_list_t *avro_array2 = kaa_array_deserialize_w_ctx(avro_reader, (deserialize_w_ctx_fn)kaa_fixed_deserialize, &plain_fixed_size);
ASSERT_NOT_NULL(avro_array2);
ASSERT_EQUAL(kaa_list_get_size(avro_array2), array_size);
kaa_list_node_t *it = kaa_list_begin(avro_array2);
while (it) {
kaa_bytes_t *fixed = kaa_list_get_data(it);
ASSERT_NOT_NULL(fixed);
ASSERT_EQUAL((size_t)fixed->size, plain_fixed_size);
ASSERT_EQUAL(memcmp(fixed->buffer, plain_fixed, plain_fixed_size), 0);
it = kaa_list_next(it);
}
kaa_list_destroy(avro_array2, kaa_fixed_destroy);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
kaa_list_destroy(avro_array1, kaa_fixed_destroy);
}
static void test_float_deserialize(void **state)
{
(void)state;
float float_value1 = rand() / rand();
size_t expected_size = kaa_float_get_size(&float_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_float_serialize(avro_writer, &float_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
float *float_value2 = kaa_float_deserialize(avro_reader);
ASSERT_EQUAL(*float_value2, float_value1);
kaa_data_destroy(float_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_boolean_deserialize(void **state)
{
(void)state;
int8_t boolean_value1 = true;
size_t expected_size = kaa_boolean_get_size(&boolean_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_boolean_serialize(avro_writer, &boolean_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
int8_t *boolean_value2 = kaa_boolean_deserialize(avro_reader);
ASSERT_EQUAL(*boolean_value2, boolean_value1);
kaa_data_destroy(boolean_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_int_deserialize(void **state)
{
(void)state;
int32_t int_value1 = rand();
size_t expected_size = kaa_int_get_size(&int_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_int_serialize(avro_writer, &int_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
int32_t *int_value2 = kaa_int_deserialize(avro_reader);
ASSERT_EQUAL(*int_value2, int_value1);
kaa_data_destroy(int_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_long_deserialize(void **state)
{
(void)state;
int64_t long_value1 = rand();
size_t expected_size = kaa_long_get_size(&long_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_long_serialize(avro_writer, &long_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
int64_t *long_value2 = kaa_long_deserialize(avro_reader);
ASSERT_EQUAL(*long_value2, long_value1);
kaa_data_destroy(long_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_enum_deserialize(void **state)
{
(void)state;
test_enum_t enum_value1 = (test_enum_t)rand() % TEST_VAL_5;
size_t expected_size = kaa_enum_get_size(&enum_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_enum_serialize(avro_writer, &enum_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
test_enum_t *enum_value2 = (test_enum_t *)kaa_enum_deserialize(avro_reader);
ASSERT_EQUAL(*enum_value2, enum_value1);
kaa_data_destroy(enum_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_double_deserialize(void **state)
{
(void)state;
double double_value1 = rand() / rand();
size_t expected_size = kaa_double_get_size(&double_value1);
char buffer[expected_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, expected_size);
kaa_double_serialize(avro_writer, &double_value1);
avro_reader_t avro_reader = avro_reader_memory(buffer, expected_size);
double *double_value2 = kaa_double_deserialize(avro_reader);
ASSERT_EQUAL(*double_value2, double_value1);
kaa_data_destroy(double_value2);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
}
static void test_array_deserialize_wo_ctx(void **state)
{
(void)state;
size_t array_size = 1 + rand() % 10;
kaa_list_t *avro_array1 = kaa_list_create();
size_t i = 0;
for (i = 0; i < array_size; ++i) {
kaa_list_push_back(avro_array1, create_float());
}
size_t buffer_size = kaa_array_get_size(avro_array1, kaa_float_get_size);
char buffer[buffer_size];
avro_writer_t avro_writer = avro_writer_memory(buffer, buffer_size);
kaa_array_serialize(avro_writer, avro_array1, kaa_float_serialize);
avro_reader_t avro_reader = avro_reader_memory(buffer, buffer_size);
kaa_list_t *avro_array2 = kaa_array_deserialize_wo_ctx(avro_reader, (deserialize_wo_ctx_fn)kaa_float_deserialize);
ASSERT_NOT_NULL(avro_array2);
ASSERT_EQUAL(kaa_list_get_size(avro_array2), array_size);
// /*
// * Deserialized array has a back order, so reverse the original.
// */
// kaa_list_t *reverse_avro_array = NULL;
// kaa_list_t *it = avro_array1;
// while (it) {
// if (reverse_avro_array) {
// reverse_avro_array = kaa_list_push_front(reverse_avro_array, kaa_list_get_data(it));
// } else {
// reverse_avro_array = kaa_list_create(kaa_list_get_data(it));
// }
// it = kaa_list_next(it);
// }
//
// kaa_list_destroy(avro_array1, kaa_null_destroy);
// avro_array1 = reverse_avro_array;
/*
* Compare origin and deserialized arrays.
*/
kaa_list_node_t *it1 = kaa_list_begin(avro_array1);
kaa_list_node_t *it2 = kaa_list_begin(avro_array2);
while (it1 && it2) {
float *float_value1 = kaa_list_get_data(it1);
float *float_value2 = kaa_list_get_data(it2);
ASSERT_NOT_NULL(float_value1);
ASSERT_NOT_NULL(float_value2);
ASSERT_EQUAL(*float_value2, *float_value1);
it1 = kaa_list_next(it1);
it2 = kaa_list_next(it2);
}
kaa_list_destroy(avro_array2, kaa_data_destroy);
avro_reader_free(avro_reader);
avro_writer_free(avro_writer);
kaa_list_destroy(avro_array1, kaa_data_destroy);
}