void process_file(FILE *input, avro_file_writer_t out, avro_schema_t schema,
int verbose, int memstat, int errabort, int strjson, size_t max_str_sz) {
json_error_t err;
json_t *json;
int n = 0;
json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
while (!feof(input)) {
n++;
if (verbose && !(n % 1000))
printf("Processing record %d\n", n);
if (!json) {
if (errabort) {
fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, aborting.\n", n, err.column, err.position, err.text);
return;
}
fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, skipping to EOL\n", n, err.column, err.position, err.text);
while (getc(input) != '\n' && !feof(input)) {};
json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
continue;
}
avro_value_t record;
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &record);
if (!schema_traverse(schema, json, NULL, &record, 0, strjson, max_str_sz)) {
if (avro_file_writer_append_value(out, &record)) {
fprintf(stderr, "ERROR: avro_file_writer_append_value() FAILED: %s\n", avro_strerror());
exit(EXIT_FAILURE);
}
} else
fprintf(stderr, "Error processing record %d, skipping...\n", n);
avro_value_iface_decref(iface);
avro_value_decref(&record);
json_decref(json);
if (memstat && !(n % 1000))
memory_status();
json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
}
if (memstat) memory_status();
avro_schema_decref(schema);
}
static void
process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
exit(1);
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
exit(1);
}
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
if (avro_file_writer_create_with_codec
(out_filename, wschema, &writer, codec, block_size)) {
fprintf(stderr, "Error creating %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
}
static int write_data(int n_records) {
int i;
avro_schema_t schema;
avro_schema_error_t error;
avro_file_writer_t writer;
avro_value_iface_t *iface;
avro_value_t value;
fprintf(stderr, "\nWriting...\n");
if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) {
fprintf(stderr, "Unable to parse schema\n");
return -1;
}
if (avro_file_writer_create(filename, schema, &writer)) {
fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
return -1;
}
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
avro_value_t field;
avro_value_get_by_index(&value, 0, &field, NULL);
avro_value_set_int(&field, 123);
for (i = 0; i < n_records; i++) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "There was an error writing file: %s\n", avro_strerror());
return -1;
}
}
if (avro_file_writer_close(writer)) {
fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
return -1;
}
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
return n_records;
}
int process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
return 1;
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
return 1;
}
}
avro_schema_t wschema;
wschema = avro_file_reader_get_writer_schema(reader);
/* Check that the reader schema is the same as the writer schema */
{
avro_schema_t oschema;
avro_file_reader_t oreader;
if (avro_file_reader(out_filename, &oreader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
oschema = avro_file_reader_get_writer_schema(oreader);
if (avro_schema_equal(oschema, wschema) == 0) {
fprintf(stderr, "Error: reader and writer schema are not equal.\n");
avro_file_reader_close(oreader);
avro_file_reader_close(reader);
return 1;
}
avro_file_reader_close(oreader);
avro_schema_decref(oschema);
}
if (avro_file_writer_open(out_filename, &writer)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
avro_value_iface_t *iface;
avro_value_t value;
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
return 1;
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
return 0;
}
static void
write_data(const char *filename)
{
avro_file_writer_t file;
avro_schema_t writer_schema;
avro_schema_error_t error;
avro_value_iface_t *writer_iface;
avro_value_t writer_value;
avro_value_t field;
// First parse the JSON schema into the C API's internal schema
// representation.
check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error));
// Then create a value that is an instance of that schema. We use the
// built-in "generic" value implementation, which is what you'll usually use
// to create value instances that can actually store data. We only need to
// create one instance, since we can re-use it for all of the values that
// we're going to write into the file.
check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
check_i(avro_generic_value_new(writer_iface, &writer_value));
// Open a new data file for writing, and then write a slew of records into
// it.
check_i(avro_file_writer_create(filename, writer_schema, &file));
/* record 1 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 10));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 11));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 2 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 20));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 21));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 3 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 30));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 31));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 4 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 40));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 41));
check_i(avro_file_writer_append_value(file, &writer_value));
/* record 5 */
check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
check_i(avro_value_set_int(&field, 50));
check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
check_i(avro_value_set_int(&field, 51));
check_i(avro_file_writer_append_value(file, &writer_value));
// Close the file and clean up after ourselves.
avro_file_writer_close(file);
avro_value_decref(&writer_value);
avro_value_iface_decref(writer_iface);
avro_schema_decref(writer_schema);
}
