コード例 #1
0
ファイル: legacy.c プロジェクト: bigbes/pregel
static int
l_schema_new_raw_value(lua_State *L)
{
	LuaAvroSchema  *l_schema = luaL_checkudata(L, 1, MT_AVRO_SCHEMA);
	if (l_schema->iface == NULL) {
		l_schema->iface = avro_generic_class_from_schema(l_schema->schema);
		if (l_schema->iface == NULL) {
			lua_pushstring(L, avro_strerror());
			return lua_error(L);
		}
	}

	if (lua_gettop(L) >= 2) {
		LuaAvroValue  *l_value = luaL_checkudata(L, 2, MT_AVRO_VALUE);
		if (l_value->should_decref && l_value->value.self != NULL) {
			avro_value_decref(&l_value->value);
		}
		check(avro_generic_value_new(l_schema->iface, &l_value->value));
		l_value->should_decref = true;
		lua_pushvalue(L, 2);
	} else {
		avro_value_t  value;
		check(avro_generic_value_new(l_schema->iface, &value));
		lua_avro_push_value(L, &value, true);
	}
	return 1;
}
コード例 #2
0
rkv_error_t r_kv_create_avro_value(kv_store_t *kvstore,
                                   const char *space,
                                   const char *schemaName,
                                   avro_value_t **ret_avro_value) {
    int ret;
    avro_schema_t schema = NULL;
    avro_value_iface_t *iface = NULL;
    avro_value_t *avro_value = NULL;

    if (!kvstore || !schemaName || !ret_avro_value) {
        return RKV_INVALID_ARGUEMENTS;
    }

    if ((schema = r_kv_get_schema(kvstore, space, schemaName)) == NULL) {
        return RKV_INVALID_SCHEMA;
    }

    ret = rkv_malloc(sizeof(avro_value_t), (void**)&avro_value);
    RETURN_IF_ERR(ret);

    iface = avro_generic_class_from_schema(schema);
    if (avro_generic_value_new(iface, avro_value) != 0) {
        return RKV_ERROR;
    }

    *ret_avro_value = avro_value;
    return RKV_SUCCESS;
}
コード例 #3
0
/*
{
	"type": "record",
	"name": "FinishRequest",
	"fields": [
		{"name": "succeed",  "type": "boolean"},
		{"name": "diagnostics", "type": "string"}
	]
}
 */
static void parse_finish_request(avro_slice_t *slice, bool *succeed, char **diag)
{
	char filename[FILE_NAME_LEN];
	avro_schema_t schema;
	avro_value_iface_t *iface;
	avro_value_t record, succeed_value, diag_value;
	size_t index;
	avro_reader_t reader;
	size_t size;

	sprintf(filename, "%s/%s", SCHEMA_PATH, "FinishRequestRecordAvro.avsc");
	init_schema(filename, &schema);

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &record);

	reader = avro_reader_memory(slice->buffer, slice->len);
	if (avro_value_read(reader, &record)) {
		fprintf(stderr, "Unable to read record from memory buffer\n");
		fprintf(stderr, "Error: %s\n", avro_strerror());
		exit(1);
	}

	avro_value_get_by_name(&record, "succeed", &succeed_value, &index);
	avro_value_get_boolean(&succeed_value, succeed);

	avro_value_get_by_name(&record, "diagnostics", &diag_value, &index);
	avro_value_get_string(&diag_value, diag, &size);

	//avro_generic_value_free(&record);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);
}
コード例 #4
0
ファイル: test_avro_1691.c プロジェクト: jbadorek/avro
int main(void)
{
    int pass;

    for (pass = 0 ; json_schemas[pass] ; pass++) {
        int rval = 0;
        size_t len;
        static char  buf[4096];
        avro_writer_t  writer;
        avro_file_writer_t file_writer;
        avro_file_reader_t file_reader;
        avro_schema_t  schema = NULL;
        avro_schema_error_t  error = NULL;
        char outpath[64];
        const char *json_schema = json_schemas[pass];

        printf("pass %d with schema %s\n", pass, json_schema);
        check(rval, avro_schema_from_json(json_schema, strlen(json_schema),
                                          &schema, &error));

        avro_value_iface_t  *iface = avro_generic_class_from_schema(schema);

        avro_value_t  val;
        avro_generic_value_new(iface, &val);

        avro_value_t  out;
        avro_generic_value_new(iface, &out);

        /* create the val */
        avro_value_reset(&val);
        avro_value_set_string(&val, "test-1691");

        /* Write value to file */
        snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass);

        /* create the writers */
        writer = avro_writer_memory(buf, sizeof(buf));
        check(rval, avro_file_writer_create(outpath, schema, &file_writer));

        check(rval, avro_value_write(writer, &val));

        len = avro_writer_tell(writer);
        check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
        check(rval, avro_file_writer_close(file_writer));

        /* Read the value back */
        check(rval, avro_file_reader(outpath, &file_reader));
        check(rval, avro_file_reader_read_value(file_reader, &out));
        if (!avro_value_equal(&val, &out)) {
            fprintf(stderr, "fail!\n");
            exit(EXIT_FAILURE);
        }
        fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema);
        check(rval, avro_file_reader_close(file_reader));
        remove(outpath);
    }

    exit(EXIT_SUCCESS);
}
コード例 #5
0
int process_frame_table_schema(avro_value_t *record_val, frame_reader_t reader, uint64_t wal_pos) {
    int err = 0, key_schema_present;
    avro_value_t relid_val, hash_val, key_schema_val, row_schema_val, branch_val;
    int64_t relid;
    const void *hash;
    const char *key_schema_json = NULL, *row_schema_json;
    size_t hash_len, key_schema_len = 1, row_schema_len;
    avro_schema_t key_schema = NULL, row_schema;

    check(err, avro_value_get_by_index(record_val, 0, &relid_val,      NULL));
    check(err, avro_value_get_by_index(record_val, 1, &hash_val,       NULL));
    check(err, avro_value_get_by_index(record_val, 2, &key_schema_val, NULL));
    check(err, avro_value_get_by_index(record_val, 3, &row_schema_val, NULL));
    check(err, avro_value_get_long(&relid_val, &relid));
    check(err, avro_value_get_fixed(&hash_val, &hash, &hash_len));
    check(err, avro_value_get_discriminant(&key_schema_val, &key_schema_present));
    check(err, avro_value_get_string(&row_schema_val, &row_schema_json, &row_schema_len));
    check(err, avro_schema_from_json_length(row_schema_json, row_schema_len - 1, &row_schema));

    schema_list_entry *entry = schema_list_replace(reader, relid);
    entry->relid = relid;
    entry->hash = *((uint64_t *) hash);
    entry->row_schema = row_schema;
    entry->row_iface = avro_generic_class_from_schema(row_schema);
    avro_generic_value_new(entry->row_iface, &entry->row_value);
    avro_generic_value_new(entry->row_iface, &entry->old_value);
    entry->avro_reader = avro_reader_memory(NULL, 0);

    if (key_schema_present) {
        check(err, avro_value_get_current_branch(&key_schema_val, &branch_val));
        check(err, avro_value_get_string(&branch_val, &key_schema_json, &key_schema_len));
        check(err, avro_schema_from_json_length(key_schema_json, key_schema_len - 1, &key_schema));
        entry->key_schema = key_schema;
        entry->key_iface = avro_generic_class_from_schema(key_schema);
        avro_generic_value_new(entry->key_iface, &entry->key_value);
    } else {
        entry->key_schema = NULL;
    }

    if (reader->on_table_schema) {
        check(err, reader->on_table_schema(reader->cb_context, wal_pos, relid,
                                           key_schema_json, key_schema_len - 1, key_schema,
                                           row_schema_json, row_schema_len - 1, row_schema));
    }
    return err;
}
コード例 #6
0
/*

typedef struct {
	int jobid;
	int vpid;
} process_name_t;

typedef struct {
	char *en_vars;
	char *args;
	char *host_name;
	process_name_t proc_name;
} launch_context_t;

typedef struct {
	bool is_successful;
	process_name_t proc_name;
} launch_response_t;

*/
static void build_launch_response(launch_response_t *launch_response_array, int array_size, avro_slice_t **slice)
{
	char filename[FILE_NAME_LEN];
	char buf[BUFFER_SIZE];
	long len = 0;
	avro_schema_t schema;
	avro_value_iface_t *iface;
	avro_value_t record;
	avro_value_t results_value, LaunchResult_value, is_successful_value, name_value, jobid_value, vpid_value;
	size_t index;
	int i;

	avro_writer_t writer;

	sprintf(filename, "%s/%s", SCHEMA_PATH, "LaunchResponseRecordAvro.avsc");
	init_schema(filename, &schema);

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &record);

	avro_value_get_by_name(&record, "results", &results_value, &index);

	for (i = 0; i < array_size; i++) {
		avro_value_append(&results_value, &LaunchResult_value, &index);

		avro_value_get_by_name(&LaunchResult_value, "is_successful", &is_successful_value, &index);
		avro_value_set_boolean(&is_successful_value, launch_response_array[i].is_successful);

		avro_value_get_by_name(&LaunchResult_value, "name", &name_value, &index);

		avro_value_get_by_name(&name_value, "jobid", &jobid_value, &index);
		avro_value_set_int(&jobid_value, launch_response_array[i].proc_name.jobid);

		avro_value_get_by_name(&name_value, "vpid", &vpid_value, &index);
		avro_value_set_int(&vpid_value, launch_response_array[i].proc_name.vpid);
	}

	/* create a writer with memory buffer */
	writer = avro_writer_memory(buf, sizeof(buf));
	/* write record to writer (buffer) */
	if (avro_value_write(writer, &record)) {
		fprintf(stderr, "Unable to write record to memory buffer\n");
		fprintf(stderr, "Error: %s\n", avro_strerror());
		exit(1);
	}

	avro_writer_flush(writer);
	len = avro_writer_tell(writer);

	//avro_generic_value_free(&record);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);

	*slice = xmalloc(sizeof(avro_slice_t));
	(*slice)->buffer = xmalloc(len);
	(*slice)->len = len;
	memcpy((*slice)->buffer, buf, len);
}
コード例 #7
0
ファイル: json.c プロジェクト: badboyd/bottledwater-pg
int avro_bin_to_json(avro_schema_t schema,
        const void *val_bin, size_t val_len,
        char **val_out, size_t *val_len_out) {
    if (!val_bin) {
        *val_out = NULL;
        return 0;
    } else if (!schema) {
        log_error("json: got a value where we didn't expect one, and no schema to decode it");
        *val_out = NULL;
        return EINVAL;
    }

    avro_reader_t reader = avro_reader_memory(val_bin, val_len);

    avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
    if (!iface) {
        log_error("json: error in avro_generic_class_from_schema: %s", avro_strerror());
        avro_reader_free(reader);
        return EINVAL;
    }

    int err;

    avro_value_t value;
    err = avro_generic_value_new(iface, &value);
    if (err) {
        log_error("json: error in avro_generic_value_new: %s", avro_strerror());
        avro_value_iface_decref(iface);
        avro_reader_free(reader);
        return err;
    }

    err = avro_value_read(reader, &value);
    if (err) {
        log_error("json: error decoding Avro value: %s", avro_strerror());
        avro_value_decref(&value);
        avro_value_iface_decref(iface);
        avro_reader_free(reader);
        return err;
    }

    err = avro_value_to_json(&value, 1, val_out);
    if (err) {
        log_error("json: error converting Avro value to JSON: %s", avro_strerror());
        avro_value_decref(&value);
        avro_value_iface_decref(iface);
        avro_reader_free(reader);
        return err;
    }

    *val_len_out = strlen(*val_out); // not including null terminator - to librdkafka it's just bytes

    avro_value_decref(&value);
    avro_value_iface_decref(iface);
    avro_reader_free(reader);

    return 0;
}
コード例 #8
0
ファイル: deserializer.c プロジェクト: OberstHorst/pyavroc
static int
AvroDeserializer_init(AvroDeserializer *self, PyObject *args, PyObject *kwds)
{
    int rval;
    PyObject *types = NULL;
    const char *schema_json;
    static char *kwlist[] = {"schema", "types", NULL};

    self->flags = 0;
    self->iface = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|O", kwlist,
                                     &schema_json, &types)) {
        return -1;
    }

    rval = avro_schema_from_json(schema_json, 0, &self->schema, NULL);
    if (rval != 0 || self->schema == NULL) {
        PyErr_Format(PyExc_IOError, "Error reading schema: %s",
                     avro_strerror());
        return -1;
    }
    self->flags |= DESERIALIZER_SCHEMA_OK;

    self->iface = avro_generic_class_from_schema(self->schema);
    if (self->iface == NULL) {
        PyErr_SetString(PyExc_IOError,
                        "Error creating generic class interface");
        return -1;
    }

    self->datum_reader = avro_reader_memory(0, 0);
    if (!self->datum_reader) {
        PyErr_NoMemory();
        return -1;
    }
    self->flags |= DESERIALIZER_READER_OK;

    /* copied verbatim from filereader */
    if (types != NULL && PyObject_IsTrue(types)) {
        /* we still haven't incref'ed types here */
        if (Py_TYPE(types) == get_avro_types_type()) {
            Py_INCREF(types);
            self->info.types = types;
        } else {
            self->info.types = PyObject_CallFunctionObjArgs(
                  (PyObject *) get_avro_types_type(), NULL);
            if (self->info.types == NULL) {
                return -1;
            }
            declare_types(&self->info, self->schema);
        }
    } else {
        self->info.types = NULL;
    }

    return 0;
}
コード例 #9
0
ファイル: json2avro.c プロジェクト: grisha/json2avro
void process_file(FILE *input, avro_file_writer_t out, avro_schema_t schema,
                  int verbose, int memstat, int errabort, int strjson, size_t max_str_sz) {

    json_error_t err;
    json_t *json;
    int n = 0;

    json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
    while (!feof(input)) {
        n++;
        if (verbose && !(n % 1000))
            printf("Processing record %d\n", n);
        if (!json) {
            if (errabort) {
                fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, aborting.\n", n, err.column, err.position, err.text);
                return;
            }
            fprintf(stderr, "JSON error on line %d, column %d, pos %d: %s, skipping to EOL\n", n, err.column, err.position, err.text);
            while (getc(input) != '\n' && !feof(input)) {};
            json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
            continue;
        }

        avro_value_t record;
        avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
        avro_generic_value_new(iface, &record);

        if (!schema_traverse(schema, json, NULL, &record, 0, strjson, max_str_sz)) {

            if (avro_file_writer_append_value(out, &record)) {
                fprintf(stderr, "ERROR: avro_file_writer_append_value() FAILED: %s\n", avro_strerror());
                exit(EXIT_FAILURE);
            }

        } else
            fprintf(stderr, "Error processing record %d, skipping...\n", n);

        avro_value_iface_decref(iface);
        avro_value_decref(&record);

        json_decref(json);
        if (memstat && !(n % 1000))
            memory_status();

        json = json_loadf(input, JSON_DISABLE_EOF_CHECK, &err);
    }

    if (memstat) memory_status();

    avro_schema_decref(schema);
}
コード例 #10
0
ファイル: logdecoder.c プロジェクト: a1exsh/bottledwater-pg
static void output_avro_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt,
        bool is_init) {
    plugin_state *state = palloc(sizeof(plugin_state));
    ctx->output_plugin_private = state;
    opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT;

    state->memctx = AllocSetContextCreate(ctx->context, "Avro decoder context",
            ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);

    state->frame_schema = schema_for_frame();
    state->frame_iface = avro_generic_class_from_schema(state->frame_schema);
    avro_generic_value_new(state->frame_iface, &state->frame_value);
    state->schema_cache = schema_cache_new(ctx->context);
}
コード例 #11
0
ファイル: avromod.c プロジェクト: QwertyManiac/Impala
static void
process_file(const char *in_filename, const char *out_filename)
{
    avro_file_reader_t  reader;
    avro_file_writer_t  writer;

    if (in_filename == NULL) {
        if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
            fprintf(stderr, "Error opening <stdin>:\n  %s\n",
                    avro_strerror());
            exit(1);
        }
    } else {
        if (avro_file_reader(in_filename, &reader)) {
            fprintf(stderr, "Error opening %s:\n  %s\n",
                    in_filename, avro_strerror());
            exit(1);
        }
    }

    avro_schema_t  wschema;
    avro_value_iface_t  *iface;
    avro_value_t  value;

    wschema = avro_file_reader_get_writer_schema(reader);
    iface = avro_generic_class_from_schema(wschema);
    avro_generic_value_new(iface, &value);

    if (avro_file_writer_create_with_codec
            (out_filename, wschema, &writer, codec, block_size)) {
        fprintf(stderr, "Error creating %s:\n  %s\n",
                out_filename, avro_strerror());
        exit(1);
    }

    while (avro_file_reader_read_value(reader, &value) == 0) {
        if (avro_file_writer_append_value(writer, &value)) {
            fprintf(stderr, "Error writing to %s:\n  %s\n",
                    out_filename, avro_strerror());
            exit(1);
        }
        avro_value_reset(&value);
    }

    avro_file_reader_close(reader);
    avro_file_writer_close(writer);
    avro_value_decref(&value);
    avro_value_iface_decref(iface);
    avro_schema_decref(wschema);
}
コード例 #12
0
frame_reader_t frame_reader_new() {
    frame_reader_t reader = malloc(sizeof(frame_reader));
    check_alloc(reader);
    memset(reader, 0, sizeof(frame_reader));
    reader->num_schemas = 0;
    reader->capacity = 16;
    reader->schemas = malloc(reader->capacity * sizeof(void*));
    check_alloc(reader->schemas);

    reader->frame_schema = schema_for_frame();
    reader->frame_iface = avro_generic_class_from_schema(reader->frame_schema);
    avro_generic_value_new(reader->frame_iface, &reader->frame_value);
    reader->avro_reader = avro_reader_memory(NULL, 0);
    return reader;
}
コード例 #13
0
ファイル: test_avro_1906.c プロジェクト: apache/avro
static int write_data(int n_records) {
	int  i;
	avro_schema_t schema;
	avro_schema_error_t error;
	avro_file_writer_t writer;
	avro_value_iface_t *iface;
	avro_value_t value;

	fprintf(stderr, "\nWriting...\n");

	if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) {
		fprintf(stderr, "Unable to parse schema\n");
		return -1;
	}

	if (avro_file_writer_create(filename, schema, &writer)) {
		fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
		return -1;
	}

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &value);

	avro_value_t field;

	avro_value_get_by_index(&value, 0, &field, NULL);
	avro_value_set_int(&field, 123);

	for (i = 0; i < n_records; i++) {
		if (avro_file_writer_append_value(writer, &value)) {
			fprintf(stderr, "There was an error writing file: %s\n", avro_strerror());
			return -1;
		}
	}

	if (avro_file_writer_close(writer)) {
		fprintf(stderr, "There was an error creating file: %s\n", avro_strerror());
		return -1;
	}

	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);

	return n_records;
}
コード例 #14
0
ファイル: test_avro_1906.c プロジェクト: apache/avro
static int read_data() {
	int rval;
	int records_read = 0;

	avro_file_reader_t reader;
	avro_value_iface_t *iface;
	avro_value_t value;

	fprintf(stderr, "\nReading...\n");

	rval = avro_file_reader(filename, &reader);

	if (rval) {
		fprintf(stderr, "Error: %s\n", avro_strerror());
		return -1;
	}

	avro_schema_t schema = avro_file_reader_get_writer_schema(reader);

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &value);

	while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
		avro_value_t field;
		int32_t val;
		avro_value_get_by_index(&value, 0, &field, NULL);
		avro_value_get_int(&field, &val);
		fprintf(stderr, "value = %d\n", val);
		records_read++;
		avro_value_reset(&value);
	}

	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);
	avro_file_reader_close(reader);

	fprintf(stderr, "read %d records.\n", records_read);

	if (rval != EOF) {
		fprintf(stderr, "Error: %s\n", avro_strerror());
		return -1;
	}

	return records_read;
}
コード例 #15
0
ファイル: avrocat.c プロジェクト: Jeffliu/impala
static void
process_file(const char *filename)
{
	avro_file_reader_t  reader;

	if (filename == NULL) {
		if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
			fprintf(stderr, "Error opening <stdin>:\n  %s\n",
				avro_strerror());
			exit(1);
		}
	} else {
		if (avro_file_reader(filename, &reader)) {
			fprintf(stderr, "Error opening %s:\n  %s\n",
				filename, avro_strerror());
			exit(1);
		}
	}

	avro_schema_t  wschema;
	avro_value_iface_t  *iface;
	avro_value_t  value;

	wschema = avro_file_reader_get_writer_schema(reader);
	iface = avro_generic_class_from_schema(wschema);
	avro_generic_value_new(iface, &value);

	while (avro_file_reader_read_value(reader, &value) == 0) {
		char  *json;

		if (avro_value_to_json(&value, 1, &json)) {
			fprintf(stderr, "Error converting value to JSON: %s\n",
				avro_strerror());
		} else {
			printf("%s\n", json);
			free(json);
		}

		avro_value_reset(&value);
	}

	avro_file_reader_close(reader);
	avro_value_decref(&value);
	avro_value_iface_decref(iface);
}
コード例 #16
0
extern int parse_heartbeat_request(avro_slice_t *slice)
{
	char filename[FILE_NAME_LEN];
	avro_schema_t schema;
	avro_value_iface_t *iface;
	avro_value_t record;
	size_t index;
	avro_reader_t reader;
	size_t size = 0;

	sprintf(filename, "%s/%s", SCHEMA_PATH, "HeartBeatRequestRecordAvro.avsc");
	init_schema(filename, &schema);

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &record);

	reader = avro_reader_memory(slice->buffer, slice->len);
	if (avro_value_read(reader, &record)) {
		fprintf(stderr, "Unable to read record from memory buffer\n");
		fprintf(stderr, "Error: %s\n", avro_strerror());
		exit(1);
	}

	avro_value_get_size(&record, &size);

	avro_value_iface_decref(iface);
	avro_schema_decref(schema);

//	printf("slice->len = %d\n", slice->len);
//	printf("size = %ld\n", size);
//
//	if (size > 0) {
//		return 0;
//	} else {
//		return -1;
//	}
	if (size == 0) {
		return 0;
	} else {
		return -1;
	}
}
コード例 #17
0
ファイル: resolved-writer.c プロジェクト: claws/avro-examples
static void
read_using_writer_schema(const char *filename)
{
    avro_file_reader_t  file;
    avro_schema_t  writer_schema;
    avro_value_iface_t  *writer_iface;
    avro_value_t  writer_value;

    // Open an Avro file and grab the writer schema that was used to create the
    // file.
    check_i(avro_file_reader(filename, &file));
    writer_schema = avro_file_reader_get_writer_schema(file);

    // Then create a value that is an instance of the writer schema.  As above,
    // we use the built-in "generic" value implementation for the value instance
    // that will actually store the data.
    check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
    check_i(avro_generic_value_new(writer_iface, &writer_value));

    // Read values from the file until we run out, printing the contents of each
    // one.  Here, we can read directly into `writer_value` since we know that
    // it's an instance of the schema that was used to create the file.
    while (avro_file_reader_read_value(file, &writer_value) == 0) {
        avro_value_t  field;
        int32_t  a;
        int32_t  b;

        check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
        check_i(avro_value_get_int(&field, &a));
        check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
        check_i(avro_value_get_int(&field, &b));
        printf("  a: %" PRId32 ", b: %" PRId32 "\n", a, b);
    }

    // Close the file and clean up after ourselves.
    avro_file_reader_close(file);
    avro_value_decref(&writer_value);
    avro_value_iface_decref(writer_iface);
    avro_schema_decref(writer_schema);
}
コード例 #18
0
ファイル: test_refcount.c プロジェクト: AlexChen12/avro
int main(void)
{
  avro_schema_t schema = NULL;
  avro_schema_error_t error;
  avro_value_iface_t *simple_array_class;
  avro_value_t simple;

  /* Initialize the schema structure from JSON */
  if (avro_schema_from_json(SIMPLE_ARRAY, sizeof(SIMPLE_ARRAY),
                            &schema, &error)) {
    fprintf(stdout, "Unable to parse schema\n");
    exit(EXIT_FAILURE);
  }

  // Create avro class and value
  simple_array_class = avro_generic_class_from_schema( schema );
  if ( simple_array_class == NULL )
  {
    fprintf(stdout, "Unable to create simple array class\n");
    exit(EXIT_FAILURE);
  }

  if ( avro_generic_value_new( simple_array_class, &simple ) )
  {
    fprintf(stdout, "Error creating instance of record\n" );
    exit(EXIT_FAILURE);
  }

  // Release the avro class and value
  avro_value_decref( &simple );
  avro_value_iface_decref( simple_array_class );
  avro_schema_decref(schema);
  
  return 0;

}
コード例 #19
0
ファイル: avroappend.c プロジェクト: 9cc9/Green-Marl
int process_file(const char *in_filename, const char *out_filename)
{
	avro_file_reader_t  reader;
	avro_file_writer_t  writer;

	if (in_filename == NULL) {
		if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
			fprintf(stderr, "Error opening <stdin>:\n  %s\n",
				avro_strerror());
			return 1;
		}
	} else {
		if (avro_file_reader(in_filename, &reader)) {
			fprintf(stderr, "Error opening %s:\n  %s\n",
				in_filename, avro_strerror());
			return 1;
		}
	}

	avro_schema_t  wschema;
	wschema = avro_file_reader_get_writer_schema(reader);

	/* Check that the reader schema is the same as the writer schema */
	{
		avro_schema_t oschema;
		avro_file_reader_t oreader;

		if (avro_file_reader(out_filename, &oreader)) {
			fprintf(stderr, "Error opening %s:\n   %s\n",
					out_filename, avro_strerror());
			avro_file_reader_close(reader);
			return 1;
		}

		oschema = avro_file_reader_get_writer_schema(oreader);

		if (avro_schema_equal(oschema, wschema) == 0) {
			fprintf(stderr, "Error: reader and writer schema are not equal.\n");
			avro_file_reader_close(oreader);
			avro_file_reader_close(reader);
			return 1;
		}

		avro_file_reader_close(oreader);
		avro_schema_decref(oschema);
	}

	if (avro_file_writer_open(out_filename, &writer)) {
		fprintf(stderr, "Error opening %s:\n   %s\n",
				out_filename, avro_strerror());
		avro_file_reader_close(reader);
		return 1;
	}

	avro_value_iface_t  *iface;
	avro_value_t  value;

	iface = avro_generic_class_from_schema(wschema);
	avro_generic_value_new(iface, &value);

	while (avro_file_reader_read_value(reader, &value) == 0) {
		if (avro_file_writer_append_value(writer, &value)) {
			fprintf(stderr, "Error writing to %s:\n  %s\n",
				out_filename, avro_strerror());
			return 1;
		}
		avro_value_reset(&value);
	}

	avro_file_reader_close(reader);
	avro_file_writer_close(writer);
	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(wschema);

	return 0;
}
コード例 #20
0
avro_writer_t prepare_writer_status()
{
  avro_writer_t writer;
  long lSize = 0;

  CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : ENTER \n", __FUNCTION__ ));

  CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Avro prepares to serialize data\n"));

  if ( schema_file_parsed == FALSE )
  {
    FILE *fp;

    /* open schema file */
    fp = fopen ( NETWORK_DEVICE_STATUS_AVRO_FILENAME , "rb" );
    if ( !fp ) perror( NETWORK_DEVICE_STATUS_AVRO_FILENAME " doesn't exist."), exit(1);

    /* seek through file and get file size*/
    fseek( fp , 0L , SEEK_END);
    lSize = ftell( fp );

    /*back to the start of the file*/
    rewind( fp );

    /* allocate memory for entire content */
    ndsschemabuffer = calloc( 1, lSize + 1 );

    if ( !ndsschemabuffer ) fclose(fp), fputs("memory alloc fails", stderr), exit(1);

    /* copy the file into the buffer */
    if ( 1 != fread( ndsschemabuffer , lSize, 1 , fp) )
      fclose(fp), free(ndsschemabuffer), ndsschemabuffer=NULL,  fputs("entire read fails", stderr), exit(1);

    fclose(fp);

    avro_schema_error_t  error = NULL;

    //Master report/datum
    avro_schema_t network_device_report_schema = NULL;
    avro_schema_from_json(ndsschemabuffer, strlen(ndsschemabuffer),
                        &network_device_report_schema, &error);

    //generate an avro class from our schema and get a pointer to the value interface
    iface = avro_generic_class_from_schema(network_device_report_schema);  
    avro_schema_decref(network_device_report_schema);

    schema_file_parsed = TRUE; // parse schema file once only
    CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Read Avro schema file ONCE, lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer ));
  }
  else
  {
    CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, Stored lSize = %ld, pbuffer = 0x%lx.\n", lSize + 1, (ulong)ndsschemabuffer ));
  }

  memset(&AvroSerializedBuf[0], 0, sizeof(AvroSerializedBuf));

  AvroSerializedBuf[0] = MAGIC_NUMBER; /* fill MAGIC number = Empty, i.e. no Schema ID */

  memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE ], UUID, sizeof(UUID));

  memcpy( &AvroSerializedBuf[ MAGIC_NUMBER_SIZE + sizeof(UUID) ], HASH, sizeof(HASH));

  writer = avro_writer_memory( (char*)&AvroSerializedBuf[MAGIC_NUMBER_SIZE + SCHEMA_ID_LENGTH],
                               sizeof(AvroSerializedBuf) - MAGIC_NUMBER_SIZE - SCHEMA_ID_LENGTH );

  CcspLMLiteConsoleTrace(("RDK_LOG_DEBUG, LMLite %s : EXIT \n", __FUNCTION__ ));

  return writer;
}
コード例 #21
0
ファイル: kafka_plugin.c プロジェクト: bzed/pkg-pmacct
void kafka_cache_purge(struct chained_cache *queue[], int index, int safe_action)
{
  struct pkt_primitives *data = NULL;
  struct pkt_bgp_primitives *pbgp = NULL;
  struct pkt_nat_primitives *pnat = NULL;
  struct pkt_mpls_primitives *pmpls = NULL;
  struct pkt_tunnel_primitives *ptun = NULL;
  char *pcust = NULL;
  struct pkt_vlen_hdr_primitives *pvlen = NULL;
  struct pkt_bgp_primitives empty_pbgp;
  struct pkt_nat_primitives empty_pnat;
  struct pkt_mpls_primitives empty_pmpls;
  struct pkt_tunnel_primitives empty_ptun;
  char *empty_pcust = NULL;
  char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN];
  char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_kafka_topic[SRVBUFLEN], *orig_kafka_topic = NULL;
  int i, j, stop, batch_idx, is_topic_dyn = FALSE, qn = 0, ret, saved_index = index;
  int mv_num = 0, mv_num_save = 0;
  time_t start, duration;
  pid_t writer_pid = getpid();

  char *json_buf = NULL;
  int json_buf_off = 0;

#ifdef WITH_AVRO
  avro_writer_t avro_writer;
  char *avro_buf = NULL;
  int avro_buffer_full = FALSE;
#endif

  p_kafka_init_host(&kafkap_kafka_host, config.kafka_config_file);

  /* setting some defaults */
  if (!config.sql_host) config.sql_host = default_kafka_broker_host;
  if (!config.kafka_broker_port) config.kafka_broker_port = default_kafka_broker_port;

  if (!config.sql_table) config.sql_table = default_kafka_topic;
  else {
    if (strchr(config.sql_table, '$')) {
      is_topic_dyn = TRUE;
      orig_kafka_topic = config.sql_table;
    }
  }

  if (config.amqp_routing_key_rr) orig_kafka_topic = config.sql_table;

  p_kafka_init_topic_rr(&kafkap_kafka_host);
  p_kafka_set_topic_rr(&kafkap_kafka_host, config.amqp_routing_key_rr);

  empty_pcust = malloc(config.cpptrs.len);
  if (!empty_pcust) {
    Log(LOG_ERR, "ERROR ( %s/%s ): Unable to malloc() empty_pcust. Exiting.\n", config.name, config.type);
    exit_plugin(1);
  }

  memset(&empty_pbgp, 0, sizeof(struct pkt_bgp_primitives));
  memset(&empty_pnat, 0, sizeof(struct pkt_nat_primitives));
  memset(&empty_pmpls, 0, sizeof(struct pkt_mpls_primitives));
  memset(&empty_ptun, 0, sizeof(struct pkt_tunnel_primitives));
  memset(empty_pcust, 0, config.cpptrs.len);

  p_kafka_connect_to_produce(&kafkap_kafka_host);
  p_kafka_set_broker(&kafkap_kafka_host, config.sql_host, config.kafka_broker_port);
  if (!is_topic_dyn && !config.amqp_routing_key_rr) p_kafka_set_topic(&kafkap_kafka_host, config.sql_table);
  p_kafka_set_partition(&kafkap_kafka_host, config.kafka_partition);
  p_kafka_set_key(&kafkap_kafka_host, config.kafka_partition_key, config.kafka_partition_keylen);

  if (config.message_broker_output & PRINT_OUTPUT_JSON) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_STR);
  else if (config.message_broker_output & PRINT_OUTPUT_AVRO) p_kafka_set_content_type(&kafkap_kafka_host, PM_KAFKA_CNT_TYPE_BIN);
  else {
    Log(LOG_ERR, "ERROR ( %s/%s ): Unsupported kafka_output value specified. Exiting.\n", config.name, config.type);
    exit_plugin(1);
  }

  for (j = 0, stop = 0; (!stop) && P_preprocess_funcs[j]; j++)
    stop = P_preprocess_funcs[j](queue, &index, j);

  Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid);
  start = time(NULL);

  if (config.print_markers) {
    if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
      void *json_obj;
      char *json_str;

      json_obj = compose_purge_init_json(config.name, writer_pid);

      if (json_obj) json_str = compose_json_str(json_obj);
      if (json_str) {
        Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
        ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));

        free(json_str);
        json_str = NULL;
      }
    }
  }

  if (config.message_broker_output & PRINT_OUTPUT_JSON) {
    if (config.sql_multi_values) {
      json_buf = malloc(config.sql_multi_values);

      if (!json_buf) {
	Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (json_buf). Exiting ..\n", config.name, config.type);
	exit_plugin(1);
      }
      else memset(json_buf, 0, config.sql_multi_values);
    }
  }
  else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
    if (!config.avro_buffer_size) config.avro_buffer_size = LARGEBUFLEN;

    avro_buf = malloc(config.avro_buffer_size);

    if (!avro_buf) {
      Log(LOG_ERR, "ERROR ( %s/%s ): malloc() failed (avro_buf). Exiting ..\n", config.name, config.type);
      exit_plugin(1);
    }
    else memset(avro_buf, 0, config.avro_buffer_size);

    avro_writer = avro_writer_memory(avro_buf, config.avro_buffer_size);
#endif
  }

  for (j = 0; j < index; j++) {
    void *json_obj;
    char *json_str;

    if (queue[j]->valid != PRINT_CACHE_COMMITTED) continue;

    data = &queue[j]->primitives;
    if (queue[j]->pbgp) pbgp = queue[j]->pbgp;
    else pbgp = &empty_pbgp;

    if (queue[j]->pnat) pnat = queue[j]->pnat;
    else pnat = &empty_pnat;

    if (queue[j]->pmpls) pmpls = queue[j]->pmpls;
    else pmpls = &empty_pmpls;

    if (queue[j]->ptun) ptun = queue[j]->ptun;
    else ptun = &empty_ptun;

    if (queue[j]->pcust) pcust = queue[j]->pcust;
    else pcust = empty_pcust;

    if (queue[j]->pvlen) pvlen = queue[j]->pvlen;
    else pvlen = NULL;

    if (queue[j]->valid == PRINT_CACHE_FREE) continue;

    if (config.message_broker_output & PRINT_OUTPUT_JSON) {
#ifdef WITH_JANSSON
      json_t *json_obj = json_object();
      int idx;

      for (idx = 0; idx < N_PRIMITIVES && cjhandler[idx]; idx++) cjhandler[idx](json_obj, queue[j]);
      add_writer_name_and_pid_json(json_obj, config.name, writer_pid);

      json_str = compose_json_str(json_obj);
#endif
    }
    else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
      avro_value_iface_t *avro_iface = avro_generic_class_from_schema(avro_acct_schema);
      avro_value_t avro_value = compose_avro(config.what_to_count, config.what_to_count_2, queue[j]->flow_type,
                           &queue[j]->primitives, pbgp, pnat, pmpls, ptun, pcust, pvlen, queue[j]->bytes_counter,
                           queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags,
                           &queue[j]->basetime, queue[j]->stitch, avro_iface);
      size_t avro_value_size;

      add_writer_name_and_pid_avro(avro_value, config.name, writer_pid);
      avro_value_sizeof(&avro_value, &avro_value_size);

      if (avro_value_size > config.avro_buffer_size) {
        Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: insufficient buffer size (avro_buffer_size=%u)\n",
            config.name, config.type, config.avro_buffer_size);
        Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: increase value or look for avro_buffer_size in CONFIG-KEYS document.\n\n",
            config.name, config.type);
        exit_plugin(1);
      }
      else if (avro_value_size >= (config.avro_buffer_size - avro_writer_tell(avro_writer))) {
        avro_buffer_full = TRUE;
        j--;
      }
      else if (avro_value_write(avro_writer, &avro_value)) {
        Log(LOG_ERR, "ERROR ( %s/%s ): AVRO: unable to write value: %s\n",
            config.name, config.type, avro_strerror());
        exit_plugin(1);
      }
      else {
        mv_num++;
      }

      avro_value_decref(&avro_value);
      avro_value_iface_decref(avro_iface);
#else
      if (config.debug) Log(LOG_DEBUG, "DEBUG ( %s/%s ): compose_avro(): AVRO object not created due to missing --enable-avro\n", config.name, config.type);
#endif
    }

    if (config.message_broker_output & PRINT_OUTPUT_JSON) {
      char *tmp_str = NULL;

      if (json_str && config.sql_multi_values) {
        int json_strlen = (strlen(json_str) ? (strlen(json_str) + 1) : 0);

	if (json_strlen >= (config.sql_multi_values - json_buf_off)) {
	  if (json_strlen >= config.sql_multi_values) {
	    Log(LOG_ERR, "ERROR ( %s/%s ): kafka_multi_values not large enough to store JSON elements. Exiting ..\n", config.name, config.type); 
	    exit(1);
	  }

	  tmp_str = json_str;
	  json_str = json_buf;
	}
	else {
	  strcat(json_buf, json_str);
	  mv_num++;

	  string_add_newline(json_buf);
	  json_buf_off = strlen(json_buf);

	  free(json_str);
	  json_str = NULL;
	}
      }

      if (json_str) {
        if (is_topic_dyn) {
          P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]);
          p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
        }

        if (config.amqp_routing_key_rr) {
          P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr);
          p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
        }

        Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
        ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));

	if (config.sql_multi_values) {
	  json_str = tmp_str;
	  strcpy(json_buf, json_str);

	  mv_num_save = mv_num;
	  mv_num = 1;

          string_add_newline(json_buf);
          json_buf_off = strlen(json_buf);
        }

        free(json_str);
        json_str = NULL;

        if (!ret) {
          if (!config.sql_multi_values) qn++;
          else qn += mv_num_save;
        }
        else break;
      }
    }
    else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
      if (!config.sql_multi_values || (mv_num >= config.sql_multi_values) || avro_buffer_full) {
        if (is_topic_dyn) {
          P_handle_table_dyn_strings(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, queue[j]);
          p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
        }

        if (config.amqp_routing_key_rr) {
          P_handle_table_dyn_rr(dyn_kafka_topic, SRVBUFLEN, orig_kafka_topic, &kafkap_kafka_host.topic_rr);
          p_kafka_set_topic(&kafkap_kafka_host, dyn_kafka_topic);
        }

        ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer));
        avro_writer_reset(avro_writer);
        avro_buffer_full = FALSE;
        mv_num_save = mv_num;
        mv_num = 0;

        if (!ret) qn += mv_num_save;
        else break;
      }
#endif
    }
  }

  if (config.sql_multi_values) {
    if (config.message_broker_output & PRINT_OUTPUT_JSON) {
      if (json_buf && json_buf_off) {
	/* no handling of dyn routing keys here: not compatible */
	Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_buf);
	ret = p_kafka_produce_data(&kafkap_kafka_host, json_buf, strlen(json_buf));

	if (!ret) qn += mv_num;
      }
    }
    else if (config.message_broker_output & PRINT_OUTPUT_AVRO) {
#ifdef WITH_AVRO
      if (avro_writer_tell(avro_writer)) {
        ret = p_kafka_produce_data(&kafkap_kafka_host, avro_buf, avro_writer_tell(avro_writer));
        avro_writer_free(avro_writer);

        if (!ret) qn += mv_num;
      }
#endif
    }
  }

  duration = time(NULL)-start;

  if (config.print_markers) {
    if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
      void *json_obj;
      char *json_str;

      json_obj = compose_purge_close_json(config.name, writer_pid, qn, saved_index, duration);

      if (json_obj) json_str = compose_json_str(json_obj);
      if (json_str) {
	sleep(1); /* Let's give a small delay to facilitate purge_close being
		     the last message in batch in case of partitioned topics */
        Log(LOG_DEBUG, "DEBUG ( %s/%s ): %s\n\n", config.name, config.type, json_str);
        ret = p_kafka_produce_data(&kafkap_kafka_host, json_str, strlen(json_str));

        free(json_str);
        json_str = NULL;
      }
    }
  }

  p_kafka_close(&kafkap_kafka_host, FALSE);

  Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n",
		config.name, config.type, writer_pid, qn, saved_index, duration);

  if (config.sql_trigger_exec && !safe_action) P_trigger_exec(config.sql_trigger_exec); 

  if (empty_pcust) free(empty_pcust);

  if (json_buf) free(json_buf);

#ifdef WITH_AVRO
  if (avro_buf) free(avro_buf);
#endif
}
コード例 #22
0
ファイル: avrocat.c プロジェクト: AlexChen12/avro
static void
process_file(const char *filename)
{
	avro_file_reader_t  reader;
	FILE *fp;
	int  should_close;

	if (filename == NULL) {
		fp = stdin;
		filename = "<stdin>";
		should_close = 0;
	} else {
		fp = fopen(filename, "rb");
		should_close = 1;

		if (fp == NULL) {
			fprintf(stderr, "Error opening %s:\n  %s\n",
				filename, strerror(errno));
			exit(1);
		}
	}

	if (avro_file_reader_fp(fp, filename, 0, &reader)) {
		fprintf(stderr, "Error opening %s:\n  %s\n",
			filename, avro_strerror());
		if (should_close) {
			fclose(fp);
		}
		exit(1);
	}

	avro_schema_t  wschema;
	avro_value_iface_t  *iface;
	avro_value_t  value;

	wschema = avro_file_reader_get_writer_schema(reader);
	iface = avro_generic_class_from_schema(wschema);
	avro_generic_value_new(iface, &value);

	int rval;

	while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
		char  *json;

		if (avro_value_to_json(&value, 1, &json)) {
			fprintf(stderr, "Error converting value to JSON: %s\n",
				avro_strerror());
		} else {
			printf("%s\n", json);
			free(json);
		}

		avro_value_reset(&value);
	}

	// If it was not an EOF that caused it to fail,
	// print the error.
	if (rval != EOF) {
		fprintf(stderr, "Error: %s\n", avro_strerror());
	}

	avro_file_reader_close(reader);
	avro_value_decref(&value);
	avro_value_iface_decref(iface);
	avro_schema_decref(wschema);

	if (should_close) {
		fclose(fp);
	}
}
コード例 #23
0
ファイル: filereader.c プロジェクト: OberstHorst/pyavroc
static int
AvroFileReader_init(AvroFileReader *self, PyObject *args, PyObject *kwds)
{
    int rval;
    PyObject *pyfile;
    PyObject *types = NULL;
    FILE *file;
    char *schema_json;
    avro_writer_t schema_json_writer;
    size_t len;
    static char *kwlist[] = {"file", "types", NULL};

    self->pyfile = NULL;
    self->flags = 0;
    self->iface = NULL;

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist,
                                     &pyfile, &types)) {
        return -1;
    }

    file = PyFile_AsFile(pyfile);

    if (file == NULL) {
        return -1;
    }

    self->pyfile = pyfile;
    Py_INCREF(pyfile);

    if (avro_file_reader_fp(file, "pyfile", 0, &self->reader)) {
        PyErr_Format(PyExc_IOError, "Error opening file: %s", avro_strerror());
        return -1;
    }

    self->flags |= AVROFILE_READER_OK;

    self->schema = avro_file_reader_get_writer_schema(self->reader);

    if (self->schema == NULL) {
        PyErr_Format(PyExc_IOError, "Error reading schema: %s", avro_strerror());
        return -1;
    }

    len = 256;
    do {
        /* XXX horrible loop to get a big enough buffer for schema. */
        len *= 2;
        schema_json = (char *)PyMem_Malloc(len);
        schema_json_writer = avro_writer_memory(schema_json, len);
        rval = avro_schema_to_json(self->schema, schema_json_writer);
        if (!rval) {
            rval = avro_write(schema_json_writer, (void *)"", 1);  /* zero terminate */
            if (!rval) {
                self->schema_json = PyString_FromString(schema_json);
            }
        }
        avro_writer_free(schema_json_writer);
        PyMem_Free(schema_json);
    } while (rval == ENOSPC);

    if (rval) {
        PyErr_Format(PyExc_IOError, "Error saving schema: %s", avro_strerror());
        return -1;
    }

    self->flags |= AVROFILE_SCHEMA_OK;

    self->iface = avro_generic_class_from_schema(self->schema);

    if (self->iface == NULL) {
        PyErr_SetString(PyExc_IOError, "Error creating generic class interface");
        return -1;
    }

    if (types != NULL && PyObject_IsTrue(types)) {
        /* we still haven't incref'ed types here */
        if (Py_TYPE(types) == get_avro_types_type()) {
            Py_INCREF(types);
            self->info.types = types;
        } else {
            self->info.types = PyObject_CallFunctionObjArgs((PyObject *)get_avro_types_type(), NULL);
            if (self->info.types == NULL) {
                return -1;
            }
            declare_types(&self->info, self->schema);
        }
    } else {
        self->info.types = NULL;
    }

    return 0;
}
コード例 #24
0
ファイル: resolved-writer.c プロジェクト: claws/avro-examples
static void
read_with_schema_resolution(const char *filename,
                            const char *reader_schema_json,
                            const char *field_name)
{
    avro_file_reader_t  file;
    avro_schema_error_t  error;
    avro_schema_t  reader_schema;
    avro_schema_t  writer_schema;
    avro_value_iface_t  *writer_iface;
    avro_value_iface_t  *reader_iface;
    avro_value_t  writer_value;
    avro_value_t  reader_value;

    // Open an Avro file and grab the writer schema that was used to create the
    // file.
    check_i(avro_file_reader(filename, &file));
    writer_schema = avro_file_reader_get_writer_schema(file);

    // Create a value instance that we want to read the data into.  Note that
    // this is *not* the writer schema!
    check_i(avro_schema_from_json
            (reader_schema_json, 0, &reader_schema, &error));
    check_p(reader_iface = avro_generic_class_from_schema(reader_schema));
    check_i(avro_generic_value_new(reader_iface, &reader_value));

    // Create a resolved writer that will perform the schema resolution for us.
    // If the two schemas aren't compatible, this function will return an error,
    // and the error text should describe which parts of the schemas are
    // incompatible.
    check_p(writer_iface =
            avro_resolved_writer_new(writer_schema, reader_schema));

    // Create an instance of the resolved writer, and tell it to wrap our reader
    // value instance.
    check_i(avro_resolved_writer_new_value(writer_iface, &writer_value));
    avro_resolved_writer_set_dest(&writer_value, &reader_value);

    // Now we've got the same basic loop as above.  But we've got two value
    // instances floating around!  Which do we use?  We have the file reader
    // fill in `writer_value`, since that's the value that is an instance of the
    // file's writer schema.  Since it's an instance of a resolved writer,
    // though, it doesn't actually store any data itself.  Instead, it will
    // perform schema resolution on the data read from the file, and fill in its
    // wrapped value (which in our case is `reader_value`).  That means that
    // once the data has been read, we can get its (schema-resolved) contents
    // via `reader_value`.
    while (avro_file_reader_read_value(file, &writer_value) == 0) {
        avro_value_t  field;
        int32_t  value;

        check_i(avro_value_get_by_name(&reader_value, field_name, &field, NULL));
        check_i(avro_value_get_int(&field, &value));
        printf("  %s: %" PRId32 "\n", field_name, value);
    }

    // Close the file and clean up after ourselves.
    avro_file_reader_close(file);
    avro_value_decref(&writer_value);
    avro_value_iface_decref(writer_iface);
    avro_schema_decref(writer_schema);
    avro_value_decref(&reader_value);
    avro_value_iface_decref(reader_iface);
    avro_schema_decref(reader_schema);
}
コード例 #25
0
ファイル: resolved-writer.c プロジェクト: claws/avro-examples
static void
write_data(const char *filename)
{
    avro_file_writer_t  file;
    avro_schema_t  writer_schema;
    avro_schema_error_t  error;
    avro_value_iface_t  *writer_iface;
    avro_value_t  writer_value;
    avro_value_t  field;

    // First parse the JSON schema into the C API's internal schema
    // representation.
    check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error));

    // Then create a value that is an instance of that schema.  We use the
    // built-in "generic" value implementation, which is what you'll usually use
    // to create value instances that can actually store data.  We only need to
    // create one instance, since we can re-use it for all of the values that
    // we're going to write into the file.
    check_p(writer_iface = avro_generic_class_from_schema(writer_schema));
    check_i(avro_generic_value_new(writer_iface, &writer_value));

    // Open a new data file for writing, and then write a slew of records into
    // it.
    check_i(avro_file_writer_create(filename, writer_schema, &file));

    /* record 1 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 10));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 11));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 2 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 20));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 21));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 3 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 30));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 31));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 4 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 40));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 41));
    check_i(avro_file_writer_append_value(file, &writer_value));

    /* record 5 */
    check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL));
    check_i(avro_value_set_int(&field, 50));
    check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL));
    check_i(avro_value_set_int(&field, 51));
    check_i(avro_file_writer_append_value(file, &writer_value));

    // Close the file and clean up after ourselves.
    avro_file_writer_close(file);
    avro_value_decref(&writer_value);
    avro_value_iface_decref(writer_iface);
    avro_schema_decref(writer_schema);
}
コード例 #26
0
static void build_heartbeat_response(completed_proc_t *completed_proc_array, int array_size, avro_slice_t **slice)
{
	char filename[FILE_NAME_LEN];
	char buf[BUFFER_SIZE];
	long len = 0;
	avro_schema_t schema;
	avro_value_iface_t *iface;
	avro_value_t record;
	avro_value_t completed_processes_value, ProcessStatus_value;
	avro_value_t name_value, ProcessName_value, jobid_value, vpid_value;
	avro_value_t state_value;
	avro_value_t exit_value_value;
	size_t index;
	int i;

	avro_writer_t writer;

	sprintf(filename, "%s/%s", SCHEMA_PATH, "HeartBeatResponseRecordAvro.avsc");
	init_schema(filename, &schema);

	iface = avro_generic_class_from_schema(schema);
	avro_generic_value_new(iface, &record);

	avro_value_get_by_name(&record, "completed_processes", &completed_processes_value, &index);

	for (i = 0; i < array_size; i++) {
		avro_value_append(&completed_processes_value, &ProcessStatus_value, &index);

		avro_value_get_by_name(&ProcessStatus_value, "name", &name_value, &index);
		avro_value_get_by_name(&name_value, "jobid", &jobid_value, &index);
		avro_value_set_int(&jobid_value, completed_proc_array[i].proc_name.jobid);
		avro_value_get_by_name(&name_value, "vpid", &vpid_value, &index);
		avro_value_set_int(&vpid_value, completed_proc_array[i].proc_name.vpid);

		avro_value_get_by_name(&ProcessStatus_value, "state", &state_value, &index);
		avro_value_set_enum(&state_value, completed_proc_array[i].proc_state);

		avro_value_get_by_name(&ProcessStatus_value, "exit_value", &exit_value_value, &index);
		avro_value_set_int(&exit_value_value, completed_proc_array[i].exit_value);
	}

	/* create a writer with memory buffer */
	writer = avro_writer_memory(buf, sizeof(buf));
	/* write record to writer (buffer) */
	if (avro_value_write(writer, &record)) {
		fprintf(stderr, "Unable to write record to memory buffer\n");
		fprintf(stderr, "Error: %s\n", avro_strerror());
		exit(1);
	}

	avro_writer_flush(writer);
	len = avro_writer_tell(writer);

	//avro_generic_value_free(&record);
	avro_value_iface_decref(iface);
	avro_schema_decref(schema);

	*slice = xmalloc(sizeof(avro_slice_t));
	(*slice)->buffer = xmalloc(len);
	(*slice)->len = len;
	memcpy((*slice)->buffer, buf, len);
}