static PyObject *
create_types_func(PyObject *self, PyObject *args)
{
int rval;
avro_schema_t schema;
PyObject *schema_json;
ConvertInfo info;
PyObject *schema_json_bytes;
if (!PyArg_ParseTuple(args, "O", &schema_json)) {
return NULL;
}
schema_json_bytes = pystring_to_pybytes(schema_json);
rval = avro_schema_from_json(pybytes_to_chars(schema_json_bytes), 0, &schema, NULL);
Py_DECREF(schema_json_bytes);
if (rval != 0 || schema == NULL) {
PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror());
return NULL;
}
info.types = PyObject_CallFunctionObjArgs((PyObject *)get_avro_types_type(), NULL);
if (info.types == NULL) {
/* XXX: is the exception already set? */
return NULL;
}
declare_types(&info, schema);
return info.types;
}
static int file_read_header(avro_reader_t reader,
avro_schema_t * writers_schema, char *sync,
int synclen)
{
int rval;
avro_schema_t meta_schema;
avro_schema_t meta_values_schema;
avro_datum_t meta;
char magic[4];
avro_datum_t schema_bytes;
char *p;
int64_t len;
avro_schema_error_t schema_error;
check(rval, avro_read(reader, magic, sizeof(magic)));
if (magic[0] != 'O' || magic[1] != 'b' || magic[2] != 'j'
|| magic[3] != 1) {
return EILSEQ;
}
meta_values_schema = avro_schema_bytes();
meta_schema = avro_schema_map(meta_values_schema);
rval = avro_read_data(reader, meta_schema, NULL, &meta);
if (rval) {
return EILSEQ;
}
check(rval, avro_map_get(meta, "avro.schema", &schema_bytes));
avro_bytes_get(schema_bytes, &p, &len);
check(rval,
avro_schema_from_json(p, len, writers_schema, &schema_error));
avro_schema_decref(meta);
return avro_read(reader, sync, synclen);
}
int main(void)
{
int pass;
for (pass = 0 ; json_schemas[pass] ; pass++) {
int rval = 0;
size_t len;
static char buf[4096];
avro_writer_t writer;
avro_file_writer_t file_writer;
avro_file_reader_t file_reader;
avro_schema_t schema = NULL;
avro_schema_error_t error = NULL;
char outpath[64];
const char *json_schema = json_schemas[pass];
printf("pass %d with schema %s\n", pass, json_schema);
check(rval, avro_schema_from_json(json_schema, strlen(json_schema),
&schema, &error));
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
avro_value_t val;
avro_generic_value_new(iface, &val);
avro_value_t out;
avro_generic_value_new(iface, &out);
/* create the val */
avro_value_reset(&val);
avro_value_set_string(&val, "test-1691");
/* Write value to file */
snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass);
/* create the writers */
writer = avro_writer_memory(buf, sizeof(buf));
check(rval, avro_file_writer_create(outpath, schema, &file_writer));
check(rval, avro_value_write(writer, &val));
len = avro_writer_tell(writer);
check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
check(rval, avro_file_writer_close(file_writer));
/* Read the value back */
check(rval, avro_file_reader(outpath, &file_reader));
check(rval, avro_file_reader_read_value(file_reader, &out));
if (!avro_value_equal(&val, &out)) {
fprintf(stderr, "fail!\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema);
check(rval, avro_file_reader_close(file_reader));
remove(outpath);
}
exit(EXIT_SUCCESS);
}
static int
AvroDeserializer_init(AvroDeserializer *self, PyObject *args, PyObject *kwds)
{
int rval;
PyObject *types = NULL;
const char *schema_json;
static char *kwlist[] = {"schema", "types", NULL};
self->flags = 0;
self->iface = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|O", kwlist,
&schema_json, &types)) {
return -1;
}
rval = avro_schema_from_json(schema_json, 0, &self->schema, NULL);
if (rval != 0 || self->schema == NULL) {
PyErr_Format(PyExc_IOError, "Error reading schema: %s",
avro_strerror());
return -1;
}
self->flags |= DESERIALIZER_SCHEMA_OK;
self->iface = avro_generic_class_from_schema(self->schema);
if (self->iface == NULL) {
PyErr_SetString(PyExc_IOError,
"Error creating generic class interface");
return -1;
}
self->datum_reader = avro_reader_memory(0, 0);
if (!self->datum_reader) {
PyErr_NoMemory();
return -1;
}
self->flags |= DESERIALIZER_READER_OK;
/* copied verbatim from filereader */
if (types != NULL && PyObject_IsTrue(types)) {
/* we still haven't incref'ed types here */
if (Py_TYPE(types) == get_avro_types_type()) {
Py_INCREF(types);
self->info.types = types;
} else {
self->info.types = PyObject_CallFunctionObjArgs(
(PyObject *) get_avro_types_type(), NULL);
if (self->info.types == NULL) {
return -1;
}
declare_types(&self->info, self->schema);
}
} else {
self->info.types = NULL;
}
return 0;
}
/*把JSON定义的模式解析成模式的数据结构*/
void init(void)
{
avro_schema_error_t error;
if(avro_schema_from_json(STUDENT_SCHEMA,
sizeof(STUDENT_SCHEMA),
&student_schema,&error)) {
fprintf(stderr,"Failed to parse student schema\n");
exit(EXIT_FAILURE);
}
}
static int write_data(int n_records) {
int i;
avro_schema_t schema;
avro_schema_error_t error;
avro_file_writer_t writer;
avro_value_iface_t *iface;
avro_value_t value;
fprintf(stderr, "\nWriting...\n");
if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) {
fprintf(stderr, "Unable to parse schema\n");
return -1;
}
if (avro_file_writer_create(filename, schema, &writer)) {
fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
return -1;
}
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
avro_value_t field;
avro_value_get_by_index(&value, 0, &field, NULL);
avro_value_set_int(&field, 123);
for (i = 0; i < n_records; i++) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "There was an error writing file: %s\n", avro_strerror());
return -1;
}
}
if (avro_file_writer_close(writer)) {
fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
return -1;
}
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
return n_records;
}
static PyObject *
validate_func(PyObject *self, PyObject *args) {
int rval;
PyObject *datum;
char *schema_json;
avro_schema_t schema;
if (!PyArg_ParseTuple(args, "Os", &datum, &schema_json)) {
return NULL;
}
rval = avro_schema_from_json(schema_json, 0, &schema, NULL);
if (rval != 0 || schema == NULL) {
PyErr_Format(PyExc_IOError, "Error reading schema: %s",
avro_strerror());
return NULL;
}
return Py_BuildValue("i", validate(datum, schema));
}
static int test_nested_record(void)
{
const char *json =
"{"
" \"type\": \"record\","
" \"name\": \"list\","
" \"fields\": ["
" { \"name\": \"x\", \"type\": \"int\" },"
" { \"name\": \"y\", \"type\": \"int\" },"
" { \"name\": \"next\", \"type\": [\"null\",\"list\"]}"
" ]"
"}";
int rval;
avro_schema_t schema = NULL;
avro_schema_error_t error;
avro_schema_from_json(json, strlen(json), &schema, &error);
avro_datum_t head = avro_datum_from_schema(schema);
avro_record_set_field_value(rval, head, int32, "x", 10);
avro_record_set_field_value(rval, head, int32, "y", 10);
avro_datum_t next = NULL;
avro_datum_t tail = NULL;
avro_record_get(head, "next", &next);
avro_union_set_discriminant(next, 1, &tail);
avro_record_set_field_value(rval, tail, int32, "x", 20);
avro_record_set_field_value(rval, tail, int32, "y", 20);
avro_record_get(tail, "next", &next);
avro_union_set_discriminant(next, 0, NULL);
write_read_check(schema, head, NULL, NULL, "nested record");
avro_schema_decref(schema);
avro_datum_decref(head);
return 0;
}
int main(void)
{
avro_schema_t schema = NULL;
avro_schema_error_t error;
avro_value_iface_t *simple_array_class;
avro_value_t simple;
/* Initialize the schema structure from JSON */
if (avro_schema_from_json(SIMPLE_ARRAY, sizeof(SIMPLE_ARRAY),
&schema, &error)) {
fprintf(stdout, "Unable to parse schema\n");
exit(EXIT_FAILURE);
}
// Create avro class and value
simple_array_class = avro_generic_class_from_schema( schema );
if ( simple_array_class == NULL )
{
fprintf(stdout, "Unable to create simple array class\n");
exit(EXIT_FAILURE);
}
if ( avro_generic_value_new( simple_array_class, &simple ) )
{
fprintf(stdout, "Error creating instance of record\n" );
exit(EXIT_FAILURE);
}
// Release the avro class and value
avro_value_decref( &simple );
avro_value_iface_decref( simple_array_class );
avro_schema_decref(schema);
return 0;
}
avro_writer_t prepare_writer_status()
{
avro_writer_t writer;
long lSize = 0;
CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : ENTER \n", __FUNCTION__ ));
CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Avro prepares to serialize data\n"));
if ( schema_file_parsed == FALSE )
{
FILE *fp;
/* open schema file */
fp = fopen ( NETWORK_DEVICE_STATUS_AVRO_FILENAME , "rb" );
if ( !fp ) perror( NETWORK_DEVICE_STATUS_AVRO_FILENAME " doesn't exist."), exit(1);
/* seek through file and get file size*/
fseek( fp , 0L , SEEK_END);
lSize = ftell( fp );
/*back to the start of the file*/
rewind( fp );
/* allocate memory for entire content */
ndsschemabuffer = calloc( 1, lSize + 1 );
if ( !ndsschemabuffer ) fclose(fp), fputs("memory alloc fails", stderr), exit(1);
/* copy the file into the buffer */
if ( 1 != fread( ndsschemabuffer , lSize, 1 , fp) )
fclose(fp), free(ndsschemabuffer), ndsschemabuffer=NULL, fputs("entire read fails", stderr), exit(1);
fclose(fp);
avro_schema_error_t error = NULL;
//Master report/datum
avro_schema_t network_device_report_schema = NULL;
avro_schema_from_json(ndsschemabuffer, strlen(ndsschemabuffer),
&network_device_report_schema, &error);
//generate an avro class from our schema and get a pointer to the value interface
iface = avro_generic_class_from_schema(network_device_report_schema);
avro_schema_decref(network_device_report_schema);
schema_file_parsed = TRUE; // parse schema file once only
CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Read Avro schema file ONCE, lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer ));
}
else
{
CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Stored lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer ));
}
memset(&AvroSerializedBuf[0], 0, sizeof(AvroSerializedBuf));
AvroSerializedBuf[0] = MAGIC_NUMBER; /* fill MAGIC number = Empty, i.e. no Schema ID */
memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE ], UUID, sizeof(UUID));
memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE + sizeof(UUID) ], HASH, sizeof(HASH));
writer = avro_writer_memory( (char*)&AvroSerializedBuf[MAGIC_NUMBER_SIZE + SCHEMA_ID_LENGTH],
sizeof(AvroSerializedBuf) - MAGIC_NUMBER_SIZE - SCHEMA_ID_LENGTH );
CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : EXIT \n", __FUNCTION__ ));
return writer;
}
static int
l_schema_new(lua_State *L)
{
if (lua_isstring(L, 1)) {
size_t json_len;
const char *json_str = lua_tolstring(L, 1, &json_len);
avro_schema_t schema;
/* First check for the primitive types */
if (strcmp(json_str, "boolean") == 0) {
schema = avro_schema_boolean();
}
else if (strcmp(json_str, "bytes") == 0) {
schema = avro_schema_bytes();
}
else if (strcmp(json_str, "double") == 0) {
schema = avro_schema_double();
}
else if (strcmp(json_str, "float") == 0) {
schema = avro_schema_float();
}
else if (strcmp(json_str, "int") == 0) {
schema = avro_schema_int();
}
else if (strcmp(json_str, "long") == 0) {
schema = avro_schema_long();
}
else if (strcmp(json_str, "null") == 0) {
schema = avro_schema_null();
}
else if (strcmp(json_str, "string") == 0) {
schema = avro_schema_string();
}
/* Otherwise assume it's JSON */
else {
avro_schema_error_t schema_error;
check(avro_schema_from_json(json_str, json_len, &schema, &schema_error));
}
lua_avro_push_schema(L, schema);
avro_schema_decref(schema);
lua_pushlightuserdata(L, schema);
return 2;
}
if (lua_isuserdata(L, 1)) {
if (lua_getmetatable(L, 1)) {
lua_getfield(L, LUA_REGISTRYINDEX, MT_AVRO_SCHEMA);
if (lua_rawequal(L, -1, -2)) {
/* This is already a schema object, so just return it. */
lua_pop(L, 2); /* remove both metatables */
LuaAvroSchema *l_schema = lua_touserdata(L, 1);
lua_pushlightuserdata(L, l_schema->schema);
return 2;
}
}
}
lua_pushliteral(L, "Invalid input to Schema function");
return lua_error(L);
}
static void
write_data(const char *filename)
{
avro_file_writer_t file;
avro_schema_t writer_schema;
avro_schema_error_t error;
avro_value_iface_t *writer_iface;
avro_value_t writer_value;
avro_value_t field;
// First parse the JSON schema into the C API's internal schema
// representation.
check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error));
// Then create a value that is an instance of that schema. We use the
// built-in "generic" value implementation, which is what you'll usually use
// to create value instances that can actually store data. We only need to
// create one instance, since we can re-use it for all of the values that
// we're going to write into the file.
check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
check_i(avro_generic_value_new(writer_iface, &writer_value));
// Open a new data file for writing, and then write a slew of records into
// it.
check_i(avro_file_writer_create(filename, writer_schema, &file));
/* record 1 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 10));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 11));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 2 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 20));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 21));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 3 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 30));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 31));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 4 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 40));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 41));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 5 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 50));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 51));
check_i(avro_file_writer_append_value(file, &writer_value));
// Close the file and clean up after ourselves.
avro_file_writer_close(file);
avro_value_decref(&writer_value);
avro_value_iface_decref(writer_iface);
avro_schema_decref(writer_schema);
}
static void
read_with_schema_resolution(const char *filename,
const char *reader_schema_json,
const char *field_name)
{
avro_file_reader_t file;
avro_schema_error_t error;
avro_schema_t reader_schema;
avro_schema_t writer_schema;
avro_value_iface_t *writer_iface;
avro_value_iface_t *reader_iface;
avro_value_t writer_value;
avro_value_t reader_value;
// Open an Avro file and grab the writer schema that was used to create the
// file.
check_i(avro_file_reader(filename, &file));
writer_schema = avro_file_reader_get_writer_schema(file);
// Create a value instance that we want to read the data into. Note that
// this is *not* the writer schema!
check_i(avro_schema_from_json
(reader_schema_json, 0, &reader_schema, &error));
check_p(reader_iface = avro_generic_class_from_schema(reader_schema));
check_i(avro_generic_value_new(reader_iface, &reader_value));
// Create a resolved writer that will perform the schema resolution for us.
// If the two schemas aren't compatible, this function will return an error,
// and the error text should describe which parts of the schemas are
// incompatible.
check_p(writer_iface =
avro_resolved_writer_new(writer_schema, reader_schema));
// Create an instance of the resolved writer, and tell it to wrap our reader
// value instance.
check_i(avro_resolved_writer_new_value(writer_iface, &writer_value));
avro_resolved_writer_set_dest(&writer_value, &reader_value);
// Now we've got the same basic loop as above. But we've got two value
// instances floating around! Which do we use? We have the file reader
// fill in `writer_value`, since that's the value that is an instance of the
// file's writer schema. Since it's an instance of a resolved writer,
// though, it doesn't actually store any data itself. Instead, it will
// perform schema resolution on the data read from the file, and fill in its
// wrapped value (which in our case is `reader_value`). That means that
// once the data has been read, we can get its (schema-resolved) contents
// via `reader_value`.
while (avro_file_reader_read_value(file, &writer_value) == 0) {
avro_value_t field;
int32_t value;
check_i(avro_value_get_by_name(&reader_value, field_name, &field, NULL));
check_i(avro_value_get_int(&field, &value));
printf(" %s: %" PRId32 "\n", field_name, value);
}
// Close the file and clean up after ourselves.
avro_file_reader_close(file);
avro_value_decref(&writer_value);
avro_value_iface_decref(writer_iface);
avro_schema_decref(writer_schema);
avro_value_decref(&reader_value);
avro_value_iface_decref(reader_iface);
avro_schema_decref(reader_schema);
}
int main(int argc, char *argv[])
{
int rval;
avro_file_writer_t file_writer;
avro_file_reader_t file_reader;
char outpath[128];
FILE *fp;
char jsontext[16 * 1024];
avro_schema_t schema;
avro_schema_error_t schema_error;
avro_datum_t interop;
avro_datum_t array_datum;
avro_datum_t node_datum;
avro_datum_t union_datum;
avro_datum_t out_datum;
enum Kind {
KIND_A,
KIND_B,
KIND_C
};
if (argc != 3) {
exit(EXIT_FAILURE);
}
snprintf(outpath, sizeof(outpath), "%s/c.avro", argv[2]);
fprintf(stderr, "Writing to %s\n", outpath);
fp = fopen(argv[1], "r");
rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp);
jsontext[rval] = '\0';
check(rval,
avro_schema_from_json(jsontext, rval, &schema, &schema_error));
check(rval, avro_file_writer_create(outpath, schema, &file_writer));
/* TODO: create a method for generating random data from schema */
interop = avro_record("Interop", "org.apache.avro");
avro_record_set(interop, "intField", avro_int32(42));
avro_record_set(interop, "longField", avro_int64(4242));
avro_record_set(interop, "stringField",
avro_wrapstring("Follow your bliss."));
avro_record_set(interop, "boolField", avro_boolean(1));
avro_record_set(interop, "floatField", avro_float(3.14159265));
avro_record_set(interop, "doubleField", avro_double(2.71828183));
avro_record_set(interop, "bytesField", avro_bytes("abcd", 4));
avro_record_set(interop, "nullField", avro_null());
array_datum = avro_array();
avro_array_append_datum(array_datum, avro_double(1.0));
avro_array_append_datum(array_datum, avro_double(2.0));
avro_array_append_datum(array_datum, avro_double(3.0));
avro_record_set(interop, "arrayField", array_datum);
avro_record_set(interop, "mapField", avro_map());
union_datum = avro_union(1, avro_double(1.61803399));
avro_record_set(interop, "unionField", union_datum);
avro_record_set(interop, "enumField", avro_enum("Kind", KIND_A));
avro_record_set(interop, "fixedField",
avro_fixed("MD5", "1234567890123456", 16));
node_datum = avro_record("Node", NULL);
avro_record_set(node_datum, "label",
avro_wrapstring("If you label me, you negate me."));
avro_record_set(node_datum, "children", avro_array());
avro_record_set(interop, "recordField", node_datum);
rval = avro_file_writer_append(file_writer, interop);
if (rval) {
fprintf(stderr, "Unable to append data to interop file!\n");
exit(EXIT_FAILURE);
} else {
fprintf(stderr, "Successfully appended datum to file\n");
}
check(rval, avro_file_writer_close(file_writer));
fprintf(stderr, "Closed writer.\n");
check(rval, avro_file_reader(outpath, &file_reader));
fprintf(stderr, "Re-reading datum to verify\n");
check(rval, avro_file_reader_read(file_reader, NULL, &out_datum));
fprintf(stderr, "Verifying datum...");
if (!avro_datum_equal(interop, out_datum)) {
fprintf(stderr, "fail!\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "ok\n");
check(rval, avro_file_reader_close(file_reader));
fprintf(stderr, "Closed reader.\n");
return 0;
}
