static void process_file(const char *in_filename, const char *out_filename) { avro_file_reader_t reader; avro_file_writer_t writer; if (in_filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); exit(1); } } else { if (avro_file_reader(in_filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", in_filename, avro_strerror()); exit(1); } } avro_schema_t wschema; avro_value_iface_t *iface; avro_value_t value; wschema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); if (avro_file_writer_create_with_codec (out_filename, wschema, &writer, codec, block_size)) { fprintf(stderr, "Error creating %s:\n %s\n", out_filename, avro_strerror()); exit(1); } while (avro_file_reader_read_value(reader, &value) == 0) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "Error writing to %s:\n %s\n", out_filename, avro_strerror()); exit(1); } avro_value_reset(&value); } avro_file_reader_close(reader); avro_file_writer_close(writer); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); }
static int read_data() { int rval; int records_read = 0; avro_file_reader_t reader; avro_value_iface_t *iface; avro_value_t value; fprintf(stderr, "\nReading...\n"); rval = avro_file_reader(filename, &reader); if (rval) { fprintf(stderr, "Error: %s\n", avro_strerror()); return -1; } avro_schema_t schema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &value); while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { avro_value_t field; int32_t val; avro_value_get_by_index(&value, 0, &field, NULL); avro_value_get_int(&field, &val); fprintf(stderr, "value = %d\n", val); records_read++; avro_value_reset(&value); } avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(schema); avro_file_reader_close(reader); fprintf(stderr, "read %d records.\n", records_read); if (rval != EOF) { fprintf(stderr, "Error: %s\n", avro_strerror()); return -1; } return records_read; }
static void process_file(const char *filename) { avro_file_reader_t reader; if (filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); exit(1); } } else { if (avro_file_reader(filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", filename, avro_strerror()); exit(1); } } avro_schema_t wschema; avro_value_iface_t *iface; avro_value_t value; wschema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); while (avro_file_reader_read_value(reader, &value) == 0) { char *json; if (avro_value_to_json(&value, 1, &json)) { fprintf(stderr, "Error converting value to JSON: %s\n", avro_strerror()); } else { printf("%s\n", json); free(json); } avro_value_reset(&value); } avro_file_reader_close(reader); avro_value_decref(&value); avro_value_iface_decref(iface); }
static int read_data_datum() { int rval; int records_read = 0; avro_file_reader_t reader; avro_datum_t datum; fprintf(stderr, "\nReading...\n"); rval = avro_file_reader(filename, &reader); if (rval) { fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); return -1; } avro_schema_t schema = avro_file_reader_get_writer_schema(reader); while ((rval = avro_file_reader_read(reader, schema, &datum)) == 0) { avro_datum_t val_datum; int32_t val; if (avro_record_get(datum, "ab", &val_datum)) { fprintf(stderr, "Error getting value: %s\n", avro_strerror()); return -1; } avro_int32_get(val_datum, &val); fprintf(stderr, "value = %d\n", val); records_read++; avro_datum_decref(datum); } avro_schema_decref(schema); avro_file_reader_close(reader); fprintf(stderr, "read %d records using 'datum'.\n", records_read); if (rval != EOF) { fprintf(stderr, "Error using 'datum': %s\n", avro_strerror()); return -1; } return records_read; }
static void read_using_writer_schema(const char *filename) { avro_file_reader_t file; avro_schema_t writer_schema; avro_value_iface_t *writer_iface; avro_value_t writer_value; // Open an Avro file and grab the writer schema that was used to create the // file. check_i(avro_file_reader(filename, &file)); writer_schema = avro_file_reader_get_writer_schema(file); // Then create a value that is an instance of the writer schema. As above, // we use the built-in "generic" value implementation for the value instance // that will actually store the data. check_p(writer_iface = avro_generic_class_from_schema(writer_schema)); check_i(avro_generic_value_new(writer_iface, &writer_value)); // Read values from the file until we run out, printing the contents of each // one. Here, we can read directly into `writer_value` since we know that // it's an instance of the schema that was used to create the file. while (avro_file_reader_read_value(file, &writer_value) == 0) { avro_value_t field; int32_t a; int32_t b; check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_get_int(&field, &a)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_get_int(&field, &b)); printf(" a: %" PRId32 ", b: %" PRId32 "\n", a, b); } // Close the file and clean up after ourselves. avro_file_reader_close(file); avro_value_decref(&writer_value); avro_value_iface_decref(writer_iface); avro_schema_decref(writer_schema); }
int process_file(const char *in_filename, const char *out_filename) { avro_file_reader_t reader; avro_file_writer_t writer; if (in_filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); return 1; } } else { if (avro_file_reader(in_filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", in_filename, avro_strerror()); return 1; } } avro_schema_t wschema; wschema = avro_file_reader_get_writer_schema(reader); /* Check that the reader schema is the same as the writer schema */ { avro_schema_t oschema; avro_file_reader_t oreader; if (avro_file_reader(out_filename, &oreader)) { fprintf(stderr, "Error opening %s:\n %s\n", out_filename, avro_strerror()); avro_file_reader_close(reader); return 1; } oschema = avro_file_reader_get_writer_schema(oreader); if (avro_schema_equal(oschema, wschema) == 0) { fprintf(stderr, "Error: reader and writer schema are not equal.\n"); avro_file_reader_close(oreader); avro_file_reader_close(reader); return 1; } avro_file_reader_close(oreader); avro_schema_decref(oschema); } if (avro_file_writer_open(out_filename, &writer)) { fprintf(stderr, "Error opening %s:\n %s\n", out_filename, avro_strerror()); avro_file_reader_close(reader); return 1; } avro_value_iface_t *iface; avro_value_t value; iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); while (avro_file_reader_read_value(reader, &value) == 0) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "Error writing to %s:\n %s\n", out_filename, avro_strerror()); return 1; } avro_value_reset(&value); } avro_file_reader_close(reader); avro_file_writer_close(writer); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); return 0; }
static void read_with_schema_resolution(const char *filename, const char *reader_schema_json, const char *field_name) { avro_file_reader_t file; avro_schema_error_t error; avro_schema_t reader_schema; avro_schema_t writer_schema; avro_value_iface_t *writer_iface; avro_value_iface_t *reader_iface; avro_value_t writer_value; avro_value_t reader_value; // Open an Avro file and grab the writer schema that was used to create the // file. check_i(avro_file_reader(filename, &file)); writer_schema = avro_file_reader_get_writer_schema(file); // Create a value instance that we want to read the data into. Note that // this is *not* the writer schema! check_i(avro_schema_from_json (reader_schema_json, 0, &reader_schema, &error)); check_p(reader_iface = avro_generic_class_from_schema(reader_schema)); check_i(avro_generic_value_new(reader_iface, &reader_value)); // Create a resolved writer that will perform the schema resolution for us. // If the two schemas aren't compatible, this function will return an error, // and the error text should describe which parts of the schemas are // incompatible. check_p(writer_iface = avro_resolved_writer_new(writer_schema, reader_schema)); // Create an instance of the resolved writer, and tell it to wrap our reader // value instance. check_i(avro_resolved_writer_new_value(writer_iface, &writer_value)); avro_resolved_writer_set_dest(&writer_value, &reader_value); // Now we've got the same basic loop as above. But we've got two value // instances floating around! Which do we use? We have the file reader // fill in `writer_value`, since that's the value that is an instance of the // file's writer schema. Since it's an instance of a resolved writer, // though, it doesn't actually store any data itself. Instead, it will // perform schema resolution on the data read from the file, and fill in its // wrapped value (which in our case is `reader_value`). That means that // once the data has been read, we can get its (schema-resolved) contents // via `reader_value`. while (avro_file_reader_read_value(file, &writer_value) == 0) { avro_value_t field; int32_t value; check_i(avro_value_get_by_name(&reader_value, field_name, &field, NULL)); check_i(avro_value_get_int(&field, &value)); printf(" %s: %" PRId32 "\n", field_name, value); } // Close the file and clean up after ourselves. avro_file_reader_close(file); avro_value_decref(&writer_value); avro_value_iface_decref(writer_iface); avro_schema_decref(writer_schema); avro_value_decref(&reader_value); avro_value_iface_decref(reader_iface); avro_schema_decref(reader_schema); }
static int AvroFileReader_init(AvroFileReader *self, PyObject *args, PyObject *kwds) { int rval; PyObject *pyfile; PyObject *types = NULL; FILE *file; char *schema_json; avro_writer_t schema_json_writer; size_t len; static char *kwlist[] = {"file", "types", NULL}; self->pyfile = NULL; self->flags = 0; self->iface = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, &pyfile, &types)) { return -1; } file = PyFile_AsFile(pyfile); if (file == NULL) { return -1; } self->pyfile = pyfile; Py_INCREF(pyfile); if (avro_file_reader_fp(file, "pyfile", 0, &self->reader)) { PyErr_Format(PyExc_IOError, "Error opening file: %s", avro_strerror()); return -1; } self->flags |= AVROFILE_READER_OK; self->schema = avro_file_reader_get_writer_schema(self->reader); if (self->schema == NULL) { PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror()); return -1; } len = 256; do { /* XXX horrible loop to get a big enough buffer for schema. */ len *= 2; schema_json = (char *)PyMem_Malloc(len); schema_json_writer = avro_writer_memory(schema_json, len); rval = avro_schema_to_json(self->schema, schema_json_writer); if (!rval) { rval = avro_write(schema_json_writer, (void *)"", 1); /* zero terminate */ if (!rval) { self->schema_json = PyString_FromString(schema_json); } } avro_writer_free(schema_json_writer); PyMem_Free(schema_json); } while (rval == ENOSPC); if (rval) { PyErr_Format(PyExc_IOError, "Error saving schema: %s", avro_strerror()); return -1; } self->flags |= AVROFILE_SCHEMA_OK; self->iface = avro_generic_class_from_schema(self->schema); if (self->iface == NULL) { PyErr_SetString(PyExc_IOError, "Error creating generic class interface"); return -1; } if (types != NULL && PyObject_IsTrue(types)) { /* we still haven't incref'ed types here */ if (Py_TYPE(types) == get_avro_types_type()) { Py_INCREF(types); self->info.types = types; } else { self->info.types = PyObject_CallFunctionObjArgs((PyObject *)get_avro_types_type(), NULL); if (self->info.types == NULL) { return -1; } declare_types(&self->info, self->schema); } } else { self->info.types = NULL; } return 0; }
static void process_file(const char *filename) { avro_file_reader_t reader; FILE *fp; int should_close; if (filename == NULL) { fp = stdin; filename = "<stdin>"; should_close = 0; } else { fp = fopen(filename, "rb"); should_close = 1; if (fp == NULL) { fprintf(stderr, "Error opening %s:\n %s\n", filename, strerror(errno)); exit(1); } } if (avro_file_reader_fp(fp, filename, 0, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", filename, avro_strerror()); if (should_close) { fclose(fp); } exit(1); } avro_schema_t wschema; avro_value_iface_t *iface; avro_value_t value; wschema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); int rval; while ((rval = avro_file_reader_read_value(reader, &value)) == 0) { char *json; if (avro_value_to_json(&value, 1, &json)) { fprintf(stderr, "Error converting value to JSON: %s\n", avro_strerror()); } else { printf("%s\n", json); free(json); } avro_value_reset(&value); } // If it was not an EOF that caused it to fail, // print the error. if (rval != EOF) { fprintf(stderr, "Error: %s\n", avro_strerror()); } avro_file_reader_close(reader); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); if (should_close) { fclose(fp); } }