static PyObject *
AvroFileReader_iternext(AvroFileReader *self)
{
avro_value_t value;
PyObject *result;
avro_generic_value_new(self->iface, &value);
int rval = avro_file_reader_read_value(self->reader, &value);
if (rval) {
avro_value_decref(&value);
if (rval == EOF) {
return NULL;
}
set_error_prefix("Error reading: ");
return NULL;
}
result = avro_to_python(&self->info, &value);
avro_value_decref(&value);
return result;
}
int main(void)
{
int pass;
for (pass = 0 ; json_schemas[pass] ; pass++) {
int rval = 0;
size_t len;
static char buf[4096];
avro_writer_t writer;
avro_file_writer_t file_writer;
avro_file_reader_t file_reader;
avro_schema_t schema = NULL;
avro_schema_error_t error = NULL;
char outpath[64];
const char *json_schema = json_schemas[pass];
printf("pass %d with schema %s\n", pass, json_schema);
check(rval, avro_schema_from_json(json_schema, strlen(json_schema),
&schema, &error));
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
avro_value_t val;
avro_generic_value_new(iface, &val);
avro_value_t out;
avro_generic_value_new(iface, &out);
/* create the val */
avro_value_reset(&val);
avro_value_set_string(&val, "test-1691");
/* Write value to file */
snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass);
/* create the writers */
writer = avro_writer_memory(buf, sizeof(buf));
check(rval, avro_file_writer_create(outpath, schema, &file_writer));
check(rval, avro_value_write(writer, &val));
len = avro_writer_tell(writer);
check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
check(rval, avro_file_writer_close(file_writer));
/* Read the value back */
check(rval, avro_file_reader(outpath, &file_reader));
check(rval, avro_file_reader_read_value(file_reader, &out));
if (!avro_value_equal(&val, &out)) {
fprintf(stderr, "fail!\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema);
check(rval, avro_file_reader_close(file_reader));
remove(outpath);
}
exit(EXIT_SUCCESS);
}
static void
process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
exit(1);
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
exit(1);
}
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
if (avro_file_writer_create_with_codec
(out_filename, wschema, &writer, codec, block_size)) {
fprintf(stderr, "Error creating %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
}
static int read_data() {
int rval;
int records_read = 0;
avro_file_reader_t reader;
avro_value_iface_t *iface;
avro_value_t value;
fprintf(stderr, "\nReading...\n");
rval = avro_file_reader(filename, &reader);
if (rval) {
fprintf(stderr, "Error: %s\n", avro_strerror());
return -1;
}
avro_schema_t schema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
avro_value_t field;
int32_t val;
avro_value_get_by_index(&value, 0, &field, NULL);
avro_value_get_int(&field, &val);
fprintf(stderr, "value = %d\n", val);
records_read++;
avro_value_reset(&value);
}
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
avro_file_reader_close(reader);
fprintf(stderr, "read %d records.\n", records_read);
if (rval != EOF) {
fprintf(stderr, "Error: %s\n", avro_strerror());
return -1;
}
return records_read;
}
static void
process_file(const char *filename)
{
avro_file_reader_t reader;
if (filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
exit(1);
}
} else {
if (avro_file_reader(filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, avro_strerror());
exit(1);
}
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
while (avro_file_reader_read_value(reader, &value) == 0) {
char *json;
if (avro_value_to_json(&value, 1, &json)) {
fprintf(stderr, "Error converting value to JSON: %s\n",
avro_strerror());
} else {
printf("%s\n", json);
free(json);
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_value_decref(&value);
avro_value_iface_decref(iface);
}
static void
read_using_writer_schema(const char *filename)
{
avro_file_reader_t file;
avro_schema_t writer_schema;
avro_value_iface_t *writer_iface;
avro_value_t writer_value;
// Open an Avro file and grab the writer schema that was used to create the
// file.
check_i(avro_file_reader(filename, &file));
writer_schema = avro_file_reader_get_writer_schema(file);
// Then create a value that is an instance of the writer schema. As above,
// we use the built-in "generic" value implementation for the value instance
// that will actually store the data.
check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
check_i(avro_generic_value_new(writer_iface, &writer_value));
// Read values from the file until we run out, printing the contents of each
// one. Here, we can read directly into `writer_value` since we know that
// it's an instance of the schema that was used to create the file.
while (avro_file_reader_read_value(file, &writer_value) == 0) {
avro_value_t field;
int32_t a;
int32_t b;
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_get_int(&field, &a));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_get_int(&field, &b));
printf(" a: %" PRId32 ", b: %" PRId32 "\n", a, b);
}
// Close the file and clean up after ourselves.
avro_file_reader_close(file);
avro_value_decref(&writer_value);
avro_value_iface_decref(writer_iface);
avro_schema_decref(writer_schema);
}
int process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
return 1;
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
return 1;
}
}
avro_schema_t wschema;
wschema = avro_file_reader_get_writer_schema(reader);
/* Check that the reader schema is the same as the writer schema */
{
avro_schema_t oschema;
avro_file_reader_t oreader;
if (avro_file_reader(out_filename, &oreader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
oschema = avro_file_reader_get_writer_schema(oreader);
if (avro_schema_equal(oschema, wschema) == 0) {
fprintf(stderr, "Error: reader and writer schema are not equal.\n");
avro_file_reader_close(oreader);
avro_file_reader_close(reader);
return 1;
}
avro_file_reader_close(oreader);
avro_schema_decref(oschema);
}
if (avro_file_writer_open(out_filename, &writer)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
avro_value_iface_t *iface;
avro_value_t value;
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
return 1;
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
return 0;
}
static void
read_with_schema_resolution(const char *filename,
const char *reader_schema_json,
const char *field_name)
{
avro_file_reader_t file;
avro_schema_error_t error;
avro_schema_t reader_schema;
avro_schema_t writer_schema;
avro_value_iface_t *writer_iface;
avro_value_iface_t *reader_iface;
avro_value_t writer_value;
avro_value_t reader_value;
// Open an Avro file and grab the writer schema that was used to create the
// file.
check_i(avro_file_reader(filename, &file));
writer_schema = avro_file_reader_get_writer_schema(file);
// Create a value instance that we want to read the data into. Note that
// this is *not* the writer schema!
check_i(avro_schema_from_json
(reader_schema_json, 0, &reader_schema, &error));
check_p(reader_iface = avro_generic_class_from_schema(reader_schema));
check_i(avro_generic_value_new(reader_iface, &reader_value));
// Create a resolved writer that will perform the schema resolution for us.
// If the two schemas aren't compatible, this function will return an error,
// and the error text should describe which parts of the schemas are
// incompatible.
check_p(writer_iface =
avro_resolved_writer_new(writer_schema, reader_schema));
// Create an instance of the resolved writer, and tell it to wrap our reader
// value instance.
check_i(avro_resolved_writer_new_value(writer_iface, &writer_value));
avro_resolved_writer_set_dest(&writer_value, &reader_value);
// Now we've got the same basic loop as above. But we've got two value
// instances floating around! Which do we use? We have the file reader
// fill in `writer_value`, since that's the value that is an instance of the
// file's writer schema. Since it's an instance of a resolved writer,
// though, it doesn't actually store any data itself. Instead, it will
// perform schema resolution on the data read from the file, and fill in its
// wrapped value (which in our case is `reader_value`). That means that
// once the data has been read, we can get its (schema-resolved) contents
// via `reader_value`.
while (avro_file_reader_read_value(file, &writer_value) == 0) {
avro_value_t field;
int32_t value;
check_i(avro_value_get_by_name(&reader_value, field_name, &field, NULL));
check_i(avro_value_get_int(&field, &value));
printf(" %s: %" PRId32 "\n", field_name, value);
}
// Close the file and clean up after ourselves.
avro_file_reader_close(file);
avro_value_decref(&writer_value);
avro_value_iface_decref(writer_iface);
avro_schema_decref(writer_schema);
avro_value_decref(&reader_value);
avro_value_iface_decref(reader_iface);
avro_schema_decref(reader_schema);
}
static void
process_file(const char *filename)
{
avro_file_reader_t reader;
FILE *fp;
int should_close;
if (filename == NULL) {
fp = stdin;
filename = "<stdin>";
should_close = 0;
} else {
fp = fopen(filename, "rb");
should_close = 1;
if (fp == NULL) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, strerror(errno));
exit(1);
}
}
if (avro_file_reader_fp(fp, filename, 0, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, avro_strerror());
if (should_close) {
fclose(fp);
}
exit(1);
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
int rval;
while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
char *json;
if (avro_value_to_json(&value, 1, &json)) {
fprintf(stderr, "Error converting value to JSON: %s\n",
avro_strerror());
} else {
printf("%s\n", json);
free(json);
}
avro_value_reset(&value);
}
// If it was not an EOF that caused it to fail,
// print the error.
if (rval != EOF) {
fprintf(stderr, "Error: %s\n", avro_strerror());
}
avro_file_reader_close(reader);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
if (should_close) {
fclose(fp);
}
}
