/* Updates the given frame with information about a table row that was deleted.
* This is used only during stream replication. */
int update_frame_with_delete(avro_value_t *frame_val, schema_cache_t cache, Relation rel, HeapTuple oldtuple) {
int err = 0;
schema_cache_entry *entry;
bytea *key_bin = NULL, *old_bin = NULL;
int changed = schema_cache_lookup(cache, rel, &entry);
if (changed < 0) {
return EINVAL;
} else if (changed) {
check(err, update_frame_with_table_schema(frame_val, entry));
}
if (oldtuple) {
check(err, extract_tuple_key(entry, rel, RelationGetDescr(rel), oldtuple, &key_bin));
check(err, avro_value_reset(&entry->row_value));
check(err, tuple_to_avro_row(&entry->row_value, RelationGetDescr(rel), oldtuple));
check(err, try_writing(&old_bin, &write_avro_binary, &entry->row_value));
}
check(err, update_frame_with_delete_raw(frame_val, RelationGetRelid(rel), key_bin, old_bin));
if (key_bin) pfree(key_bin);
if (old_bin) pfree(old_bin);
return err;
}
int main(void)
{
int pass;
for (pass = 0 ; json_schemas[pass] ; pass++) {
int rval = 0;
size_t len;
static char buf[4096];
avro_writer_t writer;
avro_file_writer_t file_writer;
avro_file_reader_t file_reader;
avro_schema_t schema = NULL;
avro_schema_error_t error = NULL;
char outpath[64];
const char *json_schema = json_schemas[pass];
printf("pass %d with schema %s\n", pass, json_schema);
check(rval, avro_schema_from_json(json_schema, strlen(json_schema),
&schema, &error));
avro_value_iface_t *iface = avro_generic_class_from_schema(schema);
avro_value_t val;
avro_generic_value_new(iface, &val);
avro_value_t out;
avro_generic_value_new(iface, &out);
/* create the val */
avro_value_reset(&val);
avro_value_set_string(&val, "test-1691");
/* Write value to file */
snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass);
/* create the writers */
writer = avro_writer_memory(buf, sizeof(buf));
check(rval, avro_file_writer_create(outpath, schema, &file_writer));
check(rval, avro_value_write(writer, &val));
len = avro_writer_tell(writer);
check(rval, avro_file_writer_append_encoded(file_writer, buf, len));
check(rval, avro_file_writer_close(file_writer));
/* Read the value back */
check(rval, avro_file_reader(outpath, &file_reader));
check(rval, avro_file_reader_read_value(file_reader, &out));
if (!avro_value_equal(&val, &out)) {
fprintf(stderr, "fail!\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema);
check(rval, avro_file_reader_close(file_reader));
remove(outpath);
}
exit(EXIT_SUCCESS);
}
/* Updates the given frame with information about a table row that was modified.
* This is used only during stream replication. */
int update_frame_with_update(avro_value_t *frame_val, schema_cache_t cache, Relation rel, HeapTuple oldtuple, HeapTuple newtuple) {
int err = 0;
schema_cache_entry *entry;
bytea *old_bin = NULL, *new_bin = NULL, *old_key_bin = NULL, *new_key_bin = NULL;
int changed = schema_cache_lookup(cache, rel, &entry);
if (changed < 0) {
return EINVAL;
} else if (changed) {
check(err, update_frame_with_table_schema(frame_val, entry));
}
/* oldtuple is non-NULL when replident = FULL, or when replident = DEFAULT and there is no
* primary key, or replident = DEFAULT and the primary key was not modified by the update. */
if (oldtuple) {
check(err, extract_tuple_key(entry, rel, RelationGetDescr(rel), oldtuple, &old_key_bin));
check(err, avro_value_reset(&entry->row_value));
check(err, tuple_to_avro_row(&entry->row_value, RelationGetDescr(rel), oldtuple));
check(err, try_writing(&old_bin, &write_avro_binary, &entry->row_value));
}
check(err, extract_tuple_key(entry, rel, RelationGetDescr(rel), newtuple, &new_key_bin));
check(err, avro_value_reset(&entry->row_value));
check(err, tuple_to_avro_row(&entry->row_value, RelationGetDescr(rel), newtuple));
check(err, try_writing(&new_bin, &write_avro_binary, &entry->row_value));
if (old_key_bin != NULL && (VARSIZE(old_key_bin) != VARSIZE(new_key_bin) ||
memcmp(VARDATA(old_key_bin), VARDATA(new_key_bin), VARSIZE(new_key_bin) - VARHDRSZ) != 0)) {
/* If the primary key changed, turn the update into a delete and an insert. */
check(err, update_frame_with_delete_raw(frame_val, RelationGetRelid(rel), old_key_bin, old_bin));
check(err, update_frame_with_insert_raw(frame_val, RelationGetRelid(rel), new_key_bin, new_bin));
} else {
check(err, update_frame_with_update_raw(frame_val, RelationGetRelid(rel), new_key_bin, old_bin, new_bin));
}
if (old_key_bin) pfree(old_key_bin);
if (new_key_bin) pfree(new_key_bin);
if (old_bin) pfree(old_bin);
pfree(new_bin);
return err;
}
static void
process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
exit(1);
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
exit(1);
}
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
if (avro_file_writer_create_with_codec
(out_filename, wschema, &writer, codec, block_size)) {
fprintf(stderr, "Error creating %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
exit(1);
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
}
static int read_data() {
int rval;
int records_read = 0;
avro_file_reader_t reader;
avro_value_iface_t *iface;
avro_value_t value;
fprintf(stderr, "\nReading...\n");
rval = avro_file_reader(filename, &reader);
if (rval) {
fprintf(stderr, "Error: %s\n", avro_strerror());
return -1;
}
avro_schema_t schema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(schema);
avro_generic_value_new(iface, &value);
while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
avro_value_t field;
int32_t val;
avro_value_get_by_index(&value, 0, &field, NULL);
avro_value_get_int(&field, &val);
fprintf(stderr, "value = %d\n", val);
records_read++;
avro_value_reset(&value);
}
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(schema);
avro_file_reader_close(reader);
fprintf(stderr, "read %d records.\n", records_read);
if (rval != EOF) {
fprintf(stderr, "Error: %s\n", avro_strerror());
return -1;
}
return records_read;
}
static void
process_file(const char *filename)
{
avro_file_reader_t reader;
if (filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
exit(1);
}
} else {
if (avro_file_reader(filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, avro_strerror());
exit(1);
}
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
while (avro_file_reader_read_value(reader, &value) == 0) {
char *json;
if (avro_value_to_json(&value, 1, &json)) {
fprintf(stderr, "Error converting value to JSON: %s\n",
avro_strerror());
} else {
printf("%s\n", json);
free(json);
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_value_decref(&value);
avro_value_iface_decref(iface);
}
/* If we're using a primary key/replica identity index for a given table, this
* function extracts that index' columns from a row tuple, and encodes the values
* as an Avro string using the table's key schema. */
int extract_tuple_key(schema_cache_entry *entry, Relation rel, TupleDesc tupdesc, HeapTuple tuple, bytea **key_out) {
int err = 0;
Relation index_rel;
if (entry->key_schema) {
check(err, avro_value_reset(&entry->key_value));
index_rel = table_key_index(rel);
err = tuple_to_avro_key(&entry->key_value, tupdesc, tuple, rel, index_rel->rd_index);
relation_close(index_rel, AccessShareLock);
if (err) {
return err;
}
check(err, try_writing(key_out, &write_avro_binary, &entry->key_value));
}
return err;
}
/* Updates the given frame value with a tuple inserted into a table. The table
* schema is automatically included in the frame if it's not in the cache. This
* function is used both during snapshot and during stream replication.
*
* The TupleDesc parameter is not redundant. During stream replication, it is just
* RelationGetDescr(rel), but during snapshot it is taken from the result set.
* The difference is that the result set tuple has dropped (logically invisible)
* columns omitted. */
int update_frame_with_insert(avro_value_t *frame_val, schema_cache_t cache, Relation rel, TupleDesc tupdesc, HeapTuple newtuple) {
int err = 0;
schema_cache_entry *entry;
bytea *key_bin = NULL, *new_bin = NULL;
int changed = schema_cache_lookup(cache, rel, &entry);
if (changed) {
check(err, update_frame_with_table_schema(frame_val, entry));
}
check(err, extract_tuple_key(entry, rel, tupdesc, newtuple, &key_bin));
check(err, avro_value_reset(&entry->row_value));
check(err, tuple_to_avro_row(&entry->row_value, tupdesc, newtuple));
check(err, try_writing(&new_bin, &write_avro_binary, &entry->row_value));
check(err, update_frame_with_insert_raw(frame_val, RelationGetRelid(rel), key_bin, new_bin));
if (key_bin) pfree(key_bin);
pfree(new_bin);
return err;
}
int process_file(const char *in_filename, const char *out_filename)
{
avro_file_reader_t reader;
avro_file_writer_t writer;
if (in_filename == NULL) {
if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) {
fprintf(stderr, "Error opening <stdin>:\n %s\n",
avro_strerror());
return 1;
}
} else {
if (avro_file_reader(in_filename, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
in_filename, avro_strerror());
return 1;
}
}
avro_schema_t wschema;
wschema = avro_file_reader_get_writer_schema(reader);
/* Check that the reader schema is the same as the writer schema */
{
avro_schema_t oschema;
avro_file_reader_t oreader;
if (avro_file_reader(out_filename, &oreader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
oschema = avro_file_reader_get_writer_schema(oreader);
if (avro_schema_equal(oschema, wschema) == 0) {
fprintf(stderr, "Error: reader and writer schema are not equal.\n");
avro_file_reader_close(oreader);
avro_file_reader_close(reader);
return 1;
}
avro_file_reader_close(oreader);
avro_schema_decref(oschema);
}
if (avro_file_writer_open(out_filename, &writer)) {
fprintf(stderr, "Error opening %s:\n %s\n",
out_filename, avro_strerror());
avro_file_reader_close(reader);
return 1;
}
avro_value_iface_t *iface;
avro_value_t value;
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
while (avro_file_reader_read_value(reader, &value) == 0) {
if (avro_file_writer_append_value(writer, &value)) {
fprintf(stderr, "Error writing to %s:\n %s\n",
out_filename, avro_strerror());
return 1;
}
avro_value_reset(&value);
}
avro_file_reader_close(reader);
avro_file_writer_close(writer);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
return 0;
}
void reset_frame(plugin_state *state) {
if (avro_value_reset(&state->frame_value)) {
elog(ERROR, "Avro value reset failed: %s", avro_strerror());
}
}
int
avro_value_copy_fast(avro_value_t *dest, const avro_value_t *src)
{
avro_type_t dest_type = avro_value_get_type(dest);
avro_type_t src_type = avro_value_get_type(src);
if (dest_type != src_type) {
return 0;
}
int rval;
check(rval, avro_value_reset(dest));
switch (dest_type) {
case AVRO_BOOLEAN:
{
int val;
check(rval, avro_value_get_boolean(src, &val));
return avro_value_set_boolean(dest, val);
}
case AVRO_BYTES:
{
avro_wrapped_buffer_t val;
check(rval, avro_value_grab_bytes(src, &val));
return avro_value_give_bytes(dest, &val);
}
case AVRO_DOUBLE:
{
double val;
check(rval, avro_value_get_double(src, &val));
return avro_value_set_double(dest, val);
}
case AVRO_FLOAT:
{
float val;
check(rval, avro_value_get_float(src, &val));
return avro_value_set_float(dest, val);
}
case AVRO_INT32:
{
int32_t val;
check(rval, avro_value_get_int(src, &val));
return avro_value_set_int(dest, val);
}
case AVRO_INT64:
{
int64_t val;
check(rval, avro_value_get_long(src, &val));
return avro_value_set_long(dest, val);
}
case AVRO_NULL:
{
check(rval, avro_value_get_null(src));
return avro_value_set_null(dest);
}
case AVRO_STRING:
{
avro_wrapped_buffer_t val;
check(rval, avro_value_grab_string(src, &val));
return avro_value_give_string_len(dest, &val);
}
case AVRO_ARRAY:
{
size_t count;
check(rval, avro_value_get_size(src, &count));
size_t i;
for (i = 0; i < count; i++) {
avro_value_t src_child;
avro_value_t dest_child;
check(rval, avro_value_get_by_index
(src, i, &src_child, NULL));
check(rval, avro_value_append
(dest, &dest_child, NULL));
check(rval, avro_value_copy_fast
(&dest_child, &src_child));
}
return 0;
}
case AVRO_ENUM:
{
int val;
check(rval, avro_value_get_enum(src, &val));
return avro_value_set_enum(dest, val);
}
case AVRO_FIXED:
{
avro_wrapped_buffer_t val;
check(rval, avro_value_grab_fixed(src, &val));
return avro_value_give_fixed(dest, &val);
}
case AVRO_MAP:
{
size_t count;
check(rval, avro_value_get_size(src, &count));
size_t i;
for (i = 0; i < count; i++) {
avro_value_t src_child;
avro_value_t dest_child;
const char *key;
check(rval, avro_value_get_by_index
(src, i, &src_child, &key));
check(rval, avro_value_add
(dest, key, &dest_child, NULL, NULL));
check(rval, avro_value_copy_fast
(&dest_child, &src_child));
}
return 0;
}
case AVRO_RECORD:
{
size_t count;
check(rval, avro_value_get_size(src, &count));
size_t i;
for (i = 0; i < count; i++) {
avro_value_t src_child;
avro_value_t dest_child;
check(rval, avro_value_get_by_index
(src, i, &src_child, NULL));
check(rval, avro_value_get_by_index
(dest, i, &dest_child, NULL));
check(rval, avro_value_copy_fast
(&dest_child, &src_child));
}
return 0;
}
case AVRO_UNION:
{
int disc;
check(rval, avro_value_get_discriminant(src, &disc));
avro_value_t src_branch;
avro_value_t dest_branch;
check(rval, avro_value_get_current_branch(src, &src_branch));
check(rval, avro_value_set_branch(dest, disc, &dest_branch));
return avro_value_copy_fast(&dest_branch, &src_branch);
}
default:
return 0;
}
}
static void
process_file(const char *filename)
{
avro_file_reader_t reader;
FILE *fp;
int should_close;
if (filename == NULL) {
fp = stdin;
filename = "<stdin>";
should_close = 0;
} else {
fp = fopen(filename, "rb");
should_close = 1;
if (fp == NULL) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, strerror(errno));
exit(1);
}
}
if (avro_file_reader_fp(fp, filename, 0, &reader)) {
fprintf(stderr, "Error opening %s:\n %s\n",
filename, avro_strerror());
if (should_close) {
fclose(fp);
}
exit(1);
}
avro_schema_t wschema;
avro_value_iface_t *iface;
avro_value_t value;
wschema = avro_file_reader_get_writer_schema(reader);
iface = avro_generic_class_from_schema(wschema);
avro_generic_value_new(iface, &value);
int rval;
while ((rval = avro_file_reader_read_value(reader, &value)) == 0) {
char *json;
if (avro_value_to_json(&value, 1, &json)) {
fprintf(stderr, "Error converting value to JSON: %s\n",
avro_strerror());
} else {
printf("%s\n", json);
free(json);
}
avro_value_reset(&value);
}
// If it was not an EOF that caused it to fail,
// print the error.
if (rval != EOF) {
fprintf(stderr, "Error: %s\n", avro_strerror());
}
avro_file_reader_close(reader);
avro_value_decref(&value);
avro_value_iface_decref(iface);
avro_schema_decref(wschema);
if (should_close) {
fclose(fp);
}
}