static PyObject * create_types_func(PyObject *self, PyObject *args) { int rval; avro_schema_t schema; PyObject *schema_json; ConvertInfo info; PyObject *schema_json_bytes; if (!PyArg_ParseTuple(args, "O", &schema_json)) { return NULL; } schema_json_bytes = pystring_to_pybytes(schema_json); rval = avro_schema_from_json(pybytes_to_chars(schema_json_bytes), 0, &schema, NULL); Py_DECREF(schema_json_bytes); if (rval != 0 || schema == NULL) { PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror()); return NULL; } info.types = PyObject_CallFunctionObjArgs((PyObject *)get_avro_types_type(), NULL); if (info.types == NULL) { /* XXX: is the exception already set? */ return NULL; } declare_types(&info, schema); return info.types; }
static int file_read_header(avro_reader_t reader, avro_schema_t * writers_schema, char *sync, int synclen) { int rval; avro_schema_t meta_schema; avro_schema_t meta_values_schema; avro_datum_t meta; char magic[4]; avro_datum_t schema_bytes; char *p; int64_t len; avro_schema_error_t schema_error; check(rval, avro_read(reader, magic, sizeof(magic))); if (magic[0] != 'O' || magic[1] != 'b' || magic[2] != 'j' || magic[3] != 1) { return EILSEQ; } meta_values_schema = avro_schema_bytes(); meta_schema = avro_schema_map(meta_values_schema); rval = avro_read_data(reader, meta_schema, NULL, &meta); if (rval) { return EILSEQ; } check(rval, avro_map_get(meta, "avro.schema", &schema_bytes)); avro_bytes_get(schema_bytes, &p, &len); check(rval, avro_schema_from_json(p, len, writers_schema, &schema_error)); avro_schema_decref(meta); return avro_read(reader, sync, synclen); }
int main(void) { int pass; for (pass = 0 ; json_schemas[pass] ; pass++) { int rval = 0; size_t len; static char buf[4096]; avro_writer_t writer; avro_file_writer_t file_writer; avro_file_reader_t file_reader; avro_schema_t schema = NULL; avro_schema_error_t error = NULL; char outpath[64]; const char *json_schema = json_schemas[pass]; printf("pass %d with schema %s\n", pass, json_schema); check(rval, avro_schema_from_json(json_schema, strlen(json_schema), &schema, &error)); avro_value_iface_t *iface = avro_generic_class_from_schema(schema); avro_value_t val; avro_generic_value_new(iface, &val); avro_value_t out; avro_generic_value_new(iface, &out); /* create the val */ avro_value_reset(&val); avro_value_set_string(&val, "test-1691"); /* Write value to file */ snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass); /* create the writers */ writer = avro_writer_memory(buf, sizeof(buf)); check(rval, avro_file_writer_create(outpath, schema, &file_writer)); check(rval, avro_value_write(writer, &val)); len = avro_writer_tell(writer); check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); check(rval, avro_file_writer_close(file_writer)); /* Read the value back */ check(rval, avro_file_reader(outpath, &file_reader)); check(rval, avro_file_reader_read_value(file_reader, &out)); if (!avro_value_equal(&val, &out)) { fprintf(stderr, "fail!\n"); exit(EXIT_FAILURE); } fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema); check(rval, avro_file_reader_close(file_reader)); remove(outpath); } exit(EXIT_SUCCESS); }
static int AvroDeserializer_init(AvroDeserializer *self, PyObject *args, PyObject *kwds) { int rval; PyObject *types = NULL; const char *schema_json; static char *kwlist[] = {"schema", "types", NULL}; self->flags = 0; self->iface = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|O", kwlist, &schema_json, &types)) { return -1; } rval = avro_schema_from_json(schema_json, 0, &self->schema, NULL); if (rval != 0 || self->schema == NULL) { PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror()); return -1; } self->flags |= DESERIALIZER_SCHEMA_OK; self->iface = avro_generic_class_from_schema(self->schema); if (self->iface == NULL) { PyErr_SetString(PyExc_IOError, "Error creating generic class interface"); return -1; } self->datum_reader = avro_reader_memory(0, 0); if (!self->datum_reader) { PyErr_NoMemory(); return -1; } self->flags |= DESERIALIZER_READER_OK; /* copied verbatim from filereader */ if (types != NULL && PyObject_IsTrue(types)) { /* we still haven't incref'ed types here */ if (Py_TYPE(types) == get_avro_types_type()) { Py_INCREF(types); self->info.types = types; } else { self->info.types = PyObject_CallFunctionObjArgs( (PyObject *) get_avro_types_type(), NULL); if (self->info.types == NULL) { return -1; } declare_types(&self->info, self->schema); } } else { self->info.types = NULL; } return 0; }
/*把JSON定义的模式解析成模式的数据结构*/ void init(void) { avro_schema_error_t error; if(avro_schema_from_json(STUDENT_SCHEMA, sizeof(STUDENT_SCHEMA), &student_schema,&error)) { fprintf(stderr,"Failed to parse student schema\n"); exit(EXIT_FAILURE); } }
static int write_data(int n_records) { int i; avro_schema_t schema; avro_schema_error_t error; avro_file_writer_t writer; avro_value_iface_t *iface; avro_value_t value; fprintf(stderr, "\nWriting...\n"); if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { fprintf(stderr, "Unable to parse schema\n"); return -1; } if (avro_file_writer_create(filename, schema, &writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &value); avro_value_t field; avro_value_get_by_index(&value, 0, &field, NULL); avro_value_set_int(&field, 123); for (i = 0; i < n_records; i++) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "There was an error writing file: %s\n", avro_strerror()); return -1; } } if (avro_file_writer_close(writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(schema); return n_records; }
static PyObject * validate_func(PyObject *self, PyObject *args) { int rval; PyObject *datum; char *schema_json; avro_schema_t schema; if (!PyArg_ParseTuple(args, "Os", &datum, &schema_json)) { return NULL; } rval = avro_schema_from_json(schema_json, 0, &schema, NULL); if (rval != 0 || schema == NULL) { PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror()); return NULL; } return Py_BuildValue("i", validate(datum, schema)); }
static int test_nested_record(void) { const char *json = "{" " \"type\": \"record\"," " \"name\": \"list\"," " \"fields\": [" " { \"name\": \"x\", \"type\": \"int\" }," " { \"name\": \"y\", \"type\": \"int\" }," " { \"name\": \"next\", \"type\": [\"null\",\"list\"]}" " ]" "}"; int rval; avro_schema_t schema = NULL; avro_schema_error_t error; avro_schema_from_json(json, strlen(json), &schema, &error); avro_datum_t head = avro_datum_from_schema(schema); avro_record_set_field_value(rval, head, int32, "x", 10); avro_record_set_field_value(rval, head, int32, "y", 10); avro_datum_t next = NULL; avro_datum_t tail = NULL; avro_record_get(head, "next", &next); avro_union_set_discriminant(next, 1, &tail); avro_record_set_field_value(rval, tail, int32, "x", 20); avro_record_set_field_value(rval, tail, int32, "y", 20); avro_record_get(tail, "next", &next); avro_union_set_discriminant(next, 0, NULL); write_read_check(schema, head, NULL, NULL, "nested record"); avro_schema_decref(schema); avro_datum_decref(head); return 0; }
int main(void) { avro_schema_t schema = NULL; avro_schema_error_t error; avro_value_iface_t *simple_array_class; avro_value_t simple; /* Initialize the schema structure from JSON */ if (avro_schema_from_json(SIMPLE_ARRAY, sizeof(SIMPLE_ARRAY), &schema, &error)) { fprintf(stdout, "Unable to parse schema\n"); exit(EXIT_FAILURE); } // Create avro class and value simple_array_class = avro_generic_class_from_schema( schema ); if ( simple_array_class == NULL ) { fprintf(stdout, "Unable to create simple array class\n"); exit(EXIT_FAILURE); } if ( avro_generic_value_new( simple_array_class, &simple ) ) { fprintf(stdout, "Error creating instance of record\n" ); exit(EXIT_FAILURE); } // Release the avro class and value avro_value_decref( &simple ); avro_value_iface_decref( simple_array_class ); avro_schema_decref(schema); return 0; }
avro_writer_t prepare_writer_status() { avro_writer_t writer; long lSize = 0; CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : ENTER \n", __FUNCTION__ )); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Avro prepares to serialize data\n")); if ( schema_file_parsed == FALSE ) { FILE *fp; /* open schema file */ fp = fopen ( NETWORK_DEVICE_STATUS_AVRO_FILENAME , "rb" ); if ( !fp ) perror( NETWORK_DEVICE_STATUS_AVRO_FILENAME " doesn't exist."), exit(1); /* seek through file and get file size*/ fseek( fp , 0L , SEEK_END); lSize = ftell( fp ); /*back to the start of the file*/ rewind( fp ); /* allocate memory for entire content */ ndsschemabuffer = calloc( 1, lSize + 1 ); if ( !ndsschemabuffer ) fclose(fp), fputs("memory alloc fails", stderr), exit(1); /* copy the file into the buffer */ if ( 1 != fread( ndsschemabuffer , lSize, 1 , fp) ) fclose(fp), free(ndsschemabuffer), ndsschemabuffer=NULL, fputs("entire read fails", stderr), exit(1); fclose(fp); avro_schema_error_t error = NULL; //Master report/datum avro_schema_t network_device_report_schema = NULL; avro_schema_from_json(ndsschemabuffer, strlen(ndsschemabuffer), &network_device_report_schema, &error); //generate an avro class from our schema and get a pointer to the value interface iface = avro_generic_class_from_schema(network_device_report_schema); avro_schema_decref(network_device_report_schema); schema_file_parsed = TRUE; // parse schema file once only CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Read Avro schema file ONCE, lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer )); } else { CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Stored lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer )); } memset(&AvroSerializedBuf[0], 0, sizeof(AvroSerializedBuf)); AvroSerializedBuf[0] = MAGIC_NUMBER; /* fill MAGIC number = Empty, i.e. no Schema ID */ memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE ], UUID, sizeof(UUID)); memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE + sizeof(UUID) ], HASH, sizeof(HASH)); writer = avro_writer_memory( (char*)&AvroSerializedBuf[MAGIC_NUMBER_SIZE + SCHEMA_ID_LENGTH], sizeof(AvroSerializedBuf) - MAGIC_NUMBER_SIZE - SCHEMA_ID_LENGTH ); CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : EXIT \n", __FUNCTION__ )); return writer; }
static int l_schema_new(lua_State *L) { if (lua_isstring(L, 1)) { size_t json_len; const char *json_str = lua_tolstring(L, 1, &json_len); avro_schema_t schema; /* First check for the primitive types */ if (strcmp(json_str, "boolean") == 0) { schema = avro_schema_boolean(); } else if (strcmp(json_str, "bytes") == 0) { schema = avro_schema_bytes(); } else if (strcmp(json_str, "double") == 0) { schema = avro_schema_double(); } else if (strcmp(json_str, "float") == 0) { schema = avro_schema_float(); } else if (strcmp(json_str, "int") == 0) { schema = avro_schema_int(); } else if (strcmp(json_str, "long") == 0) { schema = avro_schema_long(); } else if (strcmp(json_str, "null") == 0) { schema = avro_schema_null(); } else if (strcmp(json_str, "string") == 0) { schema = avro_schema_string(); } /* Otherwise assume it's JSON */ else { avro_schema_error_t schema_error; check(avro_schema_from_json(json_str, json_len, &schema, &schema_error)); } lua_avro_push_schema(L, schema); avro_schema_decref(schema); lua_pushlightuserdata(L, schema); return 2; } if (lua_isuserdata(L, 1)) { if (lua_getmetatable(L, 1)) { lua_getfield(L, LUA_REGISTRYINDEX, MT_AVRO_SCHEMA); if (lua_rawequal(L, -1, -2)) { /* This is already a schema object, so just return it. */ lua_pop(L, 2); /* remove both metatables */ LuaAvroSchema *l_schema = lua_touserdata(L, 1); lua_pushlightuserdata(L, l_schema->schema); return 2; } } } lua_pushliteral(L, "Invalid input to Schema function"); return lua_error(L); }
static void write_data(const char *filename) { avro_file_writer_t file; avro_schema_t writer_schema; avro_schema_error_t error; avro_value_iface_t *writer_iface; avro_value_t writer_value; avro_value_t field; // First parse the JSON schema into the C API's internal schema // representation. check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error)); // Then create a value that is an instance of that schema. We use the // built-in "generic" value implementation, which is what you'll usually use // to create value instances that can actually store data. We only need to // create one instance, since we can re-use it for all of the values that // we're going to write into the file. check_p(writer_iface = avro_generic_class_from_schema(writer_schema)); check_i(avro_generic_value_new(writer_iface, &writer_value)); // Open a new data file for writing, and then write a slew of records into // it. check_i(avro_file_writer_create(filename, writer_schema, &file)); /* record 1 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 10)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 11)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 2 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 20)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 21)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 3 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 30)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 31)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 4 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 40)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 41)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 5 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 50)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 51)); check_i(avro_file_writer_append_value(file, &writer_value)); // Close the file and clean up after ourselves. avro_file_writer_close(file); avro_value_decref(&writer_value); avro_value_iface_decref(writer_iface); avro_schema_decref(writer_schema); }
static void read_with_schema_resolution(const char *filename, const char *reader_schema_json, const char *field_name) { avro_file_reader_t file; avro_schema_error_t error; avro_schema_t reader_schema; avro_schema_t writer_schema; avro_value_iface_t *writer_iface; avro_value_iface_t *reader_iface; avro_value_t writer_value; avro_value_t reader_value; // Open an Avro file and grab the writer schema that was used to create the // file. check_i(avro_file_reader(filename, &file)); writer_schema = avro_file_reader_get_writer_schema(file); // Create a value instance that we want to read the data into. Note that // this is *not* the writer schema! check_i(avro_schema_from_json (reader_schema_json, 0, &reader_schema, &error)); check_p(reader_iface = avro_generic_class_from_schema(reader_schema)); check_i(avro_generic_value_new(reader_iface, &reader_value)); // Create a resolved writer that will perform the schema resolution for us. // If the two schemas aren't compatible, this function will return an error, // and the error text should describe which parts of the schemas are // incompatible. check_p(writer_iface = avro_resolved_writer_new(writer_schema, reader_schema)); // Create an instance of the resolved writer, and tell it to wrap our reader // value instance. check_i(avro_resolved_writer_new_value(writer_iface, &writer_value)); avro_resolved_writer_set_dest(&writer_value, &reader_value); // Now we've got the same basic loop as above. But we've got two value // instances floating around! Which do we use? We have the file reader // fill in `writer_value`, since that's the value that is an instance of the // file's writer schema. Since it's an instance of a resolved writer, // though, it doesn't actually store any data itself. Instead, it will // perform schema resolution on the data read from the file, and fill in its // wrapped value (which in our case is `reader_value`). That means that // once the data has been read, we can get its (schema-resolved) contents // via `reader_value`. while (avro_file_reader_read_value(file, &writer_value) == 0) { avro_value_t field; int32_t value; check_i(avro_value_get_by_name(&reader_value, field_name, &field, NULL)); check_i(avro_value_get_int(&field, &value)); printf(" %s: %" PRId32 "\n", field_name, value); } // Close the file and clean up after ourselves. avro_file_reader_close(file); avro_value_decref(&writer_value); avro_value_iface_decref(writer_iface); avro_schema_decref(writer_schema); avro_value_decref(&reader_value); avro_value_iface_decref(reader_iface); avro_schema_decref(reader_schema); }
int main(int argc, char *argv[]) { int rval; avro_file_writer_t file_writer; avro_file_reader_t file_reader; char outpath[128]; FILE *fp; char jsontext[16 * 1024]; avro_schema_t schema; avro_schema_error_t schema_error; avro_datum_t interop; avro_datum_t array_datum; avro_datum_t node_datum; avro_datum_t union_datum; avro_datum_t out_datum; enum Kind { KIND_A, KIND_B, KIND_C }; if (argc != 3) { exit(EXIT_FAILURE); } snprintf(outpath, sizeof(outpath), "%s/c.avro", argv[2]); fprintf(stderr, "Writing to %s\n", outpath); fp = fopen(argv[1], "r"); rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); jsontext[rval] = '\0'; check(rval, avro_schema_from_json(jsontext, rval, &schema, &schema_error)); check(rval, avro_file_writer_create(outpath, schema, &file_writer)); /* TODO: create a method for generating random data from schema */ interop = avro_record("Interop", "org.apache.avro"); avro_record_set(interop, "intField", avro_int32(42)); avro_record_set(interop, "longField", avro_int64(4242)); avro_record_set(interop, "stringField", avro_wrapstring("Follow your bliss.")); avro_record_set(interop, "boolField", avro_boolean(1)); avro_record_set(interop, "floatField", avro_float(3.14159265)); avro_record_set(interop, "doubleField", avro_double(2.71828183)); avro_record_set(interop, "bytesField", avro_bytes("abcd", 4)); avro_record_set(interop, "nullField", avro_null()); array_datum = avro_array(); avro_array_append_datum(array_datum, avro_double(1.0)); avro_array_append_datum(array_datum, avro_double(2.0)); avro_array_append_datum(array_datum, avro_double(3.0)); avro_record_set(interop, "arrayField", array_datum); avro_record_set(interop, "mapField", avro_map()); union_datum = avro_union(1, avro_double(1.61803399)); avro_record_set(interop, "unionField", union_datum); avro_record_set(interop, "enumField", avro_enum("Kind", KIND_A)); avro_record_set(interop, "fixedField", avro_fixed("MD5", "1234567890123456", 16)); node_datum = avro_record("Node", NULL); avro_record_set(node_datum, "label", avro_wrapstring("If you label me, you negate me.")); avro_record_set(node_datum, "children", avro_array()); avro_record_set(interop, "recordField", node_datum); rval = avro_file_writer_append(file_writer, interop); if (rval) { fprintf(stderr, "Unable to append data to interop file!\n"); exit(EXIT_FAILURE); } else { fprintf(stderr, "Successfully appended datum to file\n"); } check(rval, avro_file_writer_close(file_writer)); fprintf(stderr, "Closed writer.\n"); check(rval, avro_file_reader(outpath, &file_reader)); fprintf(stderr, "Re-reading datum to verify\n"); check(rval, avro_file_reader_read(file_reader, NULL, &out_datum)); fprintf(stderr, "Verifying datum..."); if (!avro_datum_equal(interop, out_datum)) { fprintf(stderr, "fail!\n"); exit(EXIT_FAILURE); } fprintf(stderr, "ok\n"); check(rval, avro_file_reader_close(file_reader)); fprintf(stderr, "Closed reader.\n"); return 0; }