예제 #1
0
void process_file(FILE *input, avro_file_writer_t out, avro_schema_t schema,
                  int verbose, int memstat, int errabort, int strjson, size_t max_str_sz) {

    json_error_t err;
    json_t *json;
    int n = 0;

    json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
    while (!feof(input)) {
        n++;
        if (verbose && !(n % 1000))
            printf("Processing record %d\n", n);
        if (!json) {
            if (errabort) {
                fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, aborting.\n", n, err.column, err.position, err.text);
                return;
            }
            fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, skipping to EOL\n", n, err.column, err.position, err.text);
            while (getc(input) != '\n' && !feof(input)) {};
            json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
            continue;
        }

        avro_value_t record;
        avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
        avro_generic_value_new(iface, &record);

        if (!schema_traverse(schema, json, NULL, &record, 0, strjson, max_str_sz)) {

            if (avro_file_writer_append_value(out, &record)) {
                fprintf(stderr, "ERROR: avro_file_writer_append_value() FAILED: %s\n", avro_strerror());
                exit(EXIT_FAILURE);
            }

        } else
            fprintf(stderr, "Error processing record %d, skipping...\n", n);

        avro_value_iface_decref(iface);
        avro_value_decref(&record);

        json_decref(json);
        if (memstat && !(n % 1000))
            memory_status();

        json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
    }

    if (memstat) memory_status();

    avro_schema_decref(schema);
}
예제 #2
0
static void
process_file(const char *in_filename, const char *out_filename)
{
    avro_file_reader_t  reader;
    avro_file_writer_t  writer;

    if (in_filename == NULL) {
        if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
            fprintf(stderr, "Error opening <stdin>:\n  %s\n",
                    avro_strerror());
            exit(1);
        }
    } else {
        if (avro_file_reader(in_filename, &reader)) {
            fprintf(stderr, "Error opening %s:\n  %s\n",
                    in_filename, avro_strerror());
            exit(1);
        }
    }

    avro_schema_t  wschema;
    avro_value_iface_t  *iface;
    avro_value_t  value;

    wschema = avro_file_reader_get_writer_schema(reader);
    iface = avro_generic_class_from_schema(wschema);
    avro_generic_value_new(iface, &value);

    if (avro_file_writer_create_with_codec
            (out_filename, wschema, &writer, codec, block_size)) {
        fprintf(stderr, "Error creating %s:\n  %s\n",
                out_filename, avro_strerror());
        exit(1);
    }

    while (avro_file_reader_read_value(reader, &value) == 0) {
        if (avro_file_writer_append_value(writer, &value)) {
            fprintf(stderr, "Error writing to %s:\n  %s\n",
                    out_filename, avro_strerror());
            exit(1);
        }
        avro_value_reset(&value);
    }

    avro_file_reader_close(reader);
    avro_file_writer_close(writer);
    avro_value_decref(&value);
    avro_value_iface_decref(iface);
    avro_schema_decref(wschema);
}
예제 #3
0
static int write_data(int n_records) {
	int  i;
	avro_schema_t schema;
	avro_schema_error_t error;
	avro_file_writer_t writer;
	avro_value_iface_t *iface;
	avro_value_t value;

	fprintf(stderr, "\nWriting...\n");

	if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) {
		fprintf(stderr, "Unable to parse schema\n");
		return -1;
	}

	if (avro_file_writer_create(filename, schema, &writer)) {
		fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
		return -1;
	}

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &value);

	avro_value_t field;

	avro_value_get_by_index(&value, 0, &field, NULL);
	avro_value_set_int(&field, 123);

	for (i = 0; i < n_records; i++) {
		if (avro_file_writer_append_value(writer, &value)) {
			fprintf(stderr, "There was an error writing file: %s\n", avro_strerror());
			return -1;
		}
	}

	if (avro_file_writer_close(writer)) {
		fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
		return -1;
	}

	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);

	return n_records;
}
예제 #4
0
int process_file(const char *in_filename, const char *out_filename)
{
	avro_file_reader_t  reader;
	avro_file_writer_t  writer;

	if (in_filename == NULL) {
		if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
			fprintf(stderr, "Error opening <stdin>:\n  %s\n",
				avro_strerror());
			return 1;
		}
	} else {
		if (avro_file_reader(in_filename, &reader)) {
			fprintf(stderr, "Error opening %s:\n  %s\n",
				in_filename, avro_strerror());
			return 1;
		}
	}

	avro_schema_t  wschema;
	wschema = avro_file_reader_get_writer_schema(reader);

	/* Check that the reader schema is the same as the writer schema */
	{
		avro_schema_t oschema;
		avro_file_reader_t oreader;

		if (avro_file_reader(out_filename, &oreader)) {
			fprintf(stderr, "Error opening %s:\n   %s\n",
					out_filename, avro_strerror());
			avro_file_reader_close(reader);
			return 1;
		}

		oschema = avro_file_reader_get_writer_schema(oreader);

		if (avro_schema_equal(oschema, wschema) == 0) {
			fprintf(stderr, "Error: reader and writer schema are not equal.\n");
			avro_file_reader_close(oreader);
			avro_file_reader_close(reader);
			return 1;
		}

		avro_file_reader_close(oreader);
		avro_schema_decref(oschema);
	}

	if (avro_file_writer_open(out_filename, &writer)) {
		fprintf(stderr, "Error opening %s:\n   %s\n",
				out_filename, avro_strerror());
		avro_file_reader_close(reader);
		return 1;
	}

	avro_value_iface_t  *iface;
	avro_value_t  value;

	iface = avro_generic_class_from_schema(wschema);
	avro_generic_value_new(iface, &value);

	while (avro_file_reader_read_value(reader, &value) == 0) {
		if (avro_file_writer_append_value(writer, &value)) {
			fprintf(stderr, "Error writing to %s:\n  %s\n",
				out_filename, avro_strerror());
			return 1;
		}
		avro_value_reset(&value);
	}

	avro_file_reader_close(reader);
	avro_file_writer_close(writer);
	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(wschema);

	return 0;
}
예제 #5
0
static void
write_data(const char *filename)
{
    avro_file_writer_t  file;
    avro_schema_t  writer_schema;
    avro_schema_error_t  error;
    avro_value_iface_t  *writer_iface;
    avro_value_t  writer_value;
    avro_value_t  field;

    // First parse the JSON schema into the C API's internal schema
    // representation.
    check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error));

    // Then create a value that is an instance of that schema.  We use the
    // built-in "generic" value implementation, which is what you'll usually use
    // to create value instances that can actually store data.  We only need to
    // create one instance, since we can re-use it for all of the values that
    // we're going to write into the file.
    check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
    check_i(avro_generic_value_new(writer_iface, &writer_value));

    // Open a new data file for writing, and then write a slew of records into
    // it.
    check_i(avro_file_writer_create(filename, writer_schema, &file));

    /* record 1 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 10));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 11));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 2 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 20));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 21));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 3 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 30));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 31));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 4 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 40));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 41));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 5 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 50));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 51));
    check_i(avro_file_writer_append_value(file, &writer_value));

    // Close the file and clean up after ourselves.
    avro_file_writer_close(file);
    avro_value_decref(&writer_value);
    avro_value_iface_decref(writer_iface);
    avro_schema_decref(writer_schema);
}