int main(void) { int pass; for (pass = 0 ; json_schemas[pass] ; pass++) { int rval = 0; size_t len; static char buf[4096]; avro_writer_t writer; avro_file_writer_t file_writer; avro_file_reader_t file_reader; avro_schema_t schema = NULL; avro_schema_error_t error = NULL; char outpath[64]; const char *json_schema = json_schemas[pass]; printf("pass %d with schema %s\n", pass, json_schema); check(rval, avro_schema_from_json(json_schema, strlen(json_schema), &schema, &error)); avro_value_iface_t *iface = avro_generic_class_from_schema(schema); avro_value_t val; avro_generic_value_new(iface, &val); avro_value_t out; avro_generic_value_new(iface, &out); /* create the val */ avro_value_reset(&val); avro_value_set_string(&val, "test-1691"); /* Write value to file */ snprintf(outpath, sizeof(outpath), "test-1691-%d.avro", pass); /* create the writers */ writer = avro_writer_memory(buf, sizeof(buf)); check(rval, avro_file_writer_create(outpath, schema, &file_writer)); check(rval, avro_value_write(writer, &val)); len = avro_writer_tell(writer); check(rval, avro_file_writer_append_encoded(file_writer, buf, len)); check(rval, avro_file_writer_close(file_writer)); /* Read the value back */ check(rval, avro_file_reader(outpath, &file_reader)); check(rval, avro_file_reader_read_value(file_reader, &out)); if (!avro_value_equal(&val, &out)) { fprintf(stderr, "fail!\n"); exit(EXIT_FAILURE); } fprintf(stderr, "pass %d: ok: schema %s\n", pass, json_schema); check(rval, avro_file_reader_close(file_reader)); remove(outpath); } exit(EXIT_SUCCESS); }
static void process_file(const char *in_filename, const char *out_filename) { avro_file_reader_t reader; avro_file_writer_t writer; if (in_filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); exit(1); } } else { if (avro_file_reader(in_filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", in_filename, avro_strerror()); exit(1); } } avro_schema_t wschema; avro_value_iface_t *iface; avro_value_t value; wschema = avro_file_reader_get_writer_schema(reader); iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); if (avro_file_writer_create_with_codec (out_filename, wschema, &writer, codec, block_size)) { fprintf(stderr, "Error creating %s:\n %s\n", out_filename, avro_strerror()); exit(1); } while (avro_file_reader_read_value(reader, &value) == 0) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "Error writing to %s:\n %s\n", out_filename, avro_strerror()); exit(1); } avro_value_reset(&value); } avro_file_reader_close(reader); avro_file_writer_close(writer); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); }
static int write_data(int n_records) { int i; avro_schema_t schema; avro_schema_error_t error; avro_file_writer_t writer; avro_value_iface_t *iface; avro_value_t value; fprintf(stderr, "\nWriting...\n"); if (avro_schema_from_json(PERSON_SCHEMA, 0, &schema, &error)) { fprintf(stderr, "Unable to parse schema\n"); return -1; } if (avro_file_writer_create(filename, schema, &writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } iface = avro_generic_class_from_schema(schema); avro_generic_value_new(iface, &value); avro_value_t field; avro_value_get_by_index(&value, 0, &field, NULL); avro_value_set_int(&field, 123); for (i = 0; i < n_records; i++) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "There was an error writing file: %s\n", avro_strerror()); return -1; } } if (avro_file_writer_close(writer)) { fprintf(stderr, "There was an error creating file: %s\n", avro_strerror()); return -1; } avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(schema); return n_records; }
int process_file(const char *in_filename, const char *out_filename) { avro_file_reader_t reader; avro_file_writer_t writer; if (in_filename == NULL) { if (avro_file_reader_fp(stdin, "<stdin>", 0, &reader)) { fprintf(stderr, "Error opening <stdin>:\n %s\n", avro_strerror()); return 1; } } else { if (avro_file_reader(in_filename, &reader)) { fprintf(stderr, "Error opening %s:\n %s\n", in_filename, avro_strerror()); return 1; } } avro_schema_t wschema; wschema = avro_file_reader_get_writer_schema(reader); /* Check that the reader schema is the same as the writer schema */ { avro_schema_t oschema; avro_file_reader_t oreader; if (avro_file_reader(out_filename, &oreader)) { fprintf(stderr, "Error opening %s:\n %s\n", out_filename, avro_strerror()); avro_file_reader_close(reader); return 1; } oschema = avro_file_reader_get_writer_schema(oreader); if (avro_schema_equal(oschema, wschema) == 0) { fprintf(stderr, "Error: reader and writer schema are not equal.\n"); avro_file_reader_close(oreader); avro_file_reader_close(reader); return 1; } avro_file_reader_close(oreader); avro_schema_decref(oschema); } if (avro_file_writer_open(out_filename, &writer)) { fprintf(stderr, "Error opening %s:\n %s\n", out_filename, avro_strerror()); avro_file_reader_close(reader); return 1; } avro_value_iface_t *iface; avro_value_t value; iface = avro_generic_class_from_schema(wschema); avro_generic_value_new(iface, &value); while (avro_file_reader_read_value(reader, &value) == 0) { if (avro_file_writer_append_value(writer, &value)) { fprintf(stderr, "Error writing to %s:\n %s\n", out_filename, avro_strerror()); return 1; } avro_value_reset(&value); } avro_file_reader_close(reader); avro_file_writer_close(writer); avro_value_decref(&value); avro_value_iface_decref(iface); avro_schema_decref(wschema); return 0; }
int main(int argc, char *argv[]) { FILE *input; avro_schema_t schema; avro_file_writer_t out; const char *key; int opt, opterr = 0, verbose = 0, memstat = 0, errabort = 0, strjson = 0; char *schema_arg = NULL; char *codec = NULL; char *endptr = NULL; char *outpath = NULL; size_t block_sz = 0; size_t max_str_sz = 0; extern char *optarg; extern int optind, optopt; while ((opt = getopt(argc, argv, "c:s:S:b:z:dmxjh")) != -1) { switch (opt) { case 's': schema_arg = optarg; break; case 'S': schema_arg = read_schema_file(optarg); break; case 'b': block_sz = strtol(optarg, &endptr, 0); if (*endptr) { fprintf(stderr, "ERROR: Invalid block size for -b: %s\n", optarg); opterr++; } break; case 'z': max_str_sz = strtol(optarg, &endptr, 0); if (*endptr) { fprintf(stderr, "ERROR: Invalid maximum string size for -z: %s\n", optarg); opterr++; } break; case 'c': codec = optarg; break; case 'd': verbose = 1; break; case 'x': errabort = 1; break; case 'j': strjson = 1; break; case 'm': #if defined(__linux__) memstat = 1; #else usage_error(argv[0], "Memory stats is a Linux-only feature!"); #endif break; case 'h': print_help(argv[0]); exit(0); case ':': fprintf(stderr, "ERROR: Option -%c requires an operand\n", optopt); opterr++; break; case '?': fprintf(stderr, "ERROR: Unrecognized option: -%c\n", optopt); opterr++; } } int file_args_cnt = (argc - optind); if (file_args_cnt == 0) { usage_error(argv[0], "Please provide at least one file name argument"); } if (file_args_cnt > 2) { fprintf(stderr, "Too many file name arguments: %d!\n", file_args_cnt); usage_error(argv[0], 0); } if (opterr) usage_error(argv[0], 0); if (!schema_arg) usage_error(argv[0], "Please provide correct schema!"); if (!codec) codec = "null"; else if (strcmp(codec, "snappy") && strcmp(codec, "deflate") && strcmp(codec, "lzma") && strcmp(codec, "null")) { fprintf(stderr, "ERROR: Invalid codec %s, valid codecs: snappy, deflate, lzma, null\n", codec); exit(EXIT_FAILURE); } if ((argc - optind) == 1) { input = stdin; outpath = argv[optind]; } else { outpath = argv[optind+1]; input = fopen(argv[optind], "rb"); if ( errno != 0 ) { fprintf(stderr, "ERROR: Cannot open input file: %s: ", argv[optind]); perror(0); exit(EXIT_FAILURE); } } if (avro_schema_from_json_length(schema_arg, strlen(schema_arg), &schema)) { fprintf(stderr, "ERROR: Unable to parse schema: '%s'\n", schema_arg); exit(EXIT_FAILURE); } if (!strcmp(outpath, "-")) { if (avro_file_writer_create_with_codec_fp(stdout, outpath, 0, schema, &out, codec, block_sz)) { fprintf(stderr, "ERROR: avro_file_writer_create_with_codec_fp FAILED: %s\n", avro_strerror()); exit(EXIT_FAILURE); } } else { remove(outpath); if (avro_file_writer_create_with_codec(outpath, schema, &out, codec, block_sz)) { fprintf(stderr, "ERROR: avro_file_writer_create_with_codec FAILED: %s\n", avro_strerror()); exit(EXIT_FAILURE); } } if (verbose) fprintf(stderr, "Using codec: %s\n", codec); process_file(input, out, schema, verbose, memstat, errabort, strjson, max_str_sz); if (verbose) printf("Closing writer....\n"); avro_file_writer_close(out); }
static void write_data(const char *filename) { avro_file_writer_t file; avro_schema_t writer_schema; avro_schema_error_t error; avro_value_iface_t *writer_iface; avro_value_t writer_value; avro_value_t field; // First parse the JSON schema into the C API's internal schema // representation. check_i(avro_schema_from_json(WRITER_SCHEMA, 0, &writer_schema, &error)); // Then create a value that is an instance of that schema. We use the // built-in "generic" value implementation, which is what you'll usually use // to create value instances that can actually store data. We only need to // create one instance, since we can re-use it for all of the values that // we're going to write into the file. check_p(writer_iface = avro_generic_class_from_schema(writer_schema)); check_i(avro_generic_value_new(writer_iface, &writer_value)); // Open a new data file for writing, and then write a slew of records into // it. check_i(avro_file_writer_create(filename, writer_schema, &file)); /* record 1 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 10)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 11)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 2 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 20)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 21)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 3 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 30)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 31)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 4 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 40)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 41)); check_i(avro_file_writer_append_value(file, &writer_value)); /* record 5 */ check_i(avro_value_get_by_name(&writer_value, "a", &field, NULL)); check_i(avro_value_set_int(&field, 50)); check_i(avro_value_get_by_name(&writer_value, "b", &field, NULL)); check_i(avro_value_set_int(&field, 51)); check_i(avro_file_writer_append_value(file, &writer_value)); // Close the file and clean up after ourselves. avro_file_writer_close(file); avro_value_decref(&writer_value); avro_value_iface_decref(writer_iface); avro_schema_decref(writer_schema); }
int main(void) { int rval; avro_file_reader_t dbreader; avro_file_writer_t db; avro_schema_t extraction_schema, name_schema, phone_schema; int64_t i; const char *dbname = "student.db"; init(); /*如果student.db存在,则删除*/ unlink(dbname); /*创建数据库文件*/ rval = avro_file_writer_create(dbname, student_schema, &db); if (rval) { fprintf(stderr, "Failed to create %s\n", dbname); exit(EXIT_FAILURE); } /*向数据库文件中添加学生信息*/ add_student(db, "Zhanghua", "Law", "15201161111", 25); add_student(db, "Lili", "Economy", "15201162222", 24); add_student(db,"Wangyu","Information","15201163333", 25); add_student(db, "Zhaoxin", "Art", "15201164444", 23); add_student(db, "Sunqin", "Physics", "15201165555", 25); add_student(db, "Zhouping", "Math", "15201166666", 23); avro_file_writer_close(db); fprintf(stdout, "\nPrint all the records from database\n"); /*读取并输出所有的学生信息*/ avro_file_reader(dbname, &dbreader); for (i = 0; i < id; i++) { if (show_student(dbreader, NULL)) { fprintf(stderr, "Error printing student\n"); exit(EXIT_FAILURE); } } avro_file_reader_close(dbreader); /*输出学生的姓名和电话信息*/ extraction_schema = avro_schema_record("Student", NULL); name_schema = avro_schema_string(); phone_schema = avro_schema_string(); avro_schema_record_field_append(extraction_schema, "Name", name_schema); avro_schema_record_field_append(extraction_schema, "Phone", phone_schema); /*只读取每个学生的姓名和电话*/ fprintf(stdout, "\n\nExtract Name & Phone of the records from database\n"); avro_file_reader(dbname, &dbreader); for (i = 0; i < id; i++) { if (show_student(dbreader, extraction_schema)) { fprintf(stderr, "Error printing student\n"); exit(EXIT_FAILURE); } } avro_file_reader_close(dbreader); avro_schema_decref(name_schema); avro_schema_decref(phone_schema); avro_schema_decref(extraction_schema); /*最后释放学生模式*/ avro_schema_decref(student_schema); return 0; }
int main(int argc, char *argv[]) { int rval; avro_file_writer_t file_writer; avro_file_reader_t file_reader; char outpath[128]; FILE *fp; char jsontext[16 * 1024]; avro_schema_t schema; avro_schema_error_t schema_error; avro_datum_t interop; avro_datum_t array_datum; avro_datum_t node_datum; avro_datum_t union_datum; avro_datum_t out_datum; enum Kind { KIND_A, KIND_B, KIND_C }; if (argc != 3) { exit(EXIT_FAILURE); } snprintf(outpath, sizeof(outpath), "%s/c.avro", argv[2]); fprintf(stderr, "Writing to %s\n", outpath); fp = fopen(argv[1], "r"); rval = fread(jsontext, 1, sizeof(jsontext) - 1, fp); jsontext[rval] = '\0'; check(rval, avro_schema_from_json(jsontext, rval, &schema, &schema_error)); check(rval, avro_file_writer_create(outpath, schema, &file_writer)); /* TODO: create a method for generating random data from schema */ interop = avro_record("Interop", "org.apache.avro"); avro_record_set(interop, "intField", avro_int32(42)); avro_record_set(interop, "longField", avro_int64(4242)); avro_record_set(interop, "stringField", avro_wrapstring("Follow your bliss.")); avro_record_set(interop, "boolField", avro_boolean(1)); avro_record_set(interop, "floatField", avro_float(3.14159265)); avro_record_set(interop, "doubleField", avro_double(2.71828183)); avro_record_set(interop, "bytesField", avro_bytes("abcd", 4)); avro_record_set(interop, "nullField", avro_null()); array_datum = avro_array(); avro_array_append_datum(array_datum, avro_double(1.0)); avro_array_append_datum(array_datum, avro_double(2.0)); avro_array_append_datum(array_datum, avro_double(3.0)); avro_record_set(interop, "arrayField", array_datum); avro_record_set(interop, "mapField", avro_map()); union_datum = avro_union(1, avro_double(1.61803399)); avro_record_set(interop, "unionField", union_datum); avro_record_set(interop, "enumField", avro_enum("Kind", KIND_A)); avro_record_set(interop, "fixedField", avro_fixed("MD5", "1234567890123456", 16)); node_datum = avro_record("Node", NULL); avro_record_set(node_datum, "label", avro_wrapstring("If you label me, you negate me.")); avro_record_set(node_datum, "children", avro_array()); avro_record_set(interop, "recordField", node_datum); rval = avro_file_writer_append(file_writer, interop); if (rval) { fprintf(stderr, "Unable to append data to interop file!\n"); exit(EXIT_FAILURE); } else { fprintf(stderr, "Successfully appended datum to file\n"); } check(rval, avro_file_writer_close(file_writer)); fprintf(stderr, "Closed writer.\n"); check(rval, avro_file_reader(outpath, &file_reader)); fprintf(stderr, "Re-reading datum to verify\n"); check(rval, avro_file_reader_read(file_reader, NULL, &out_datum)); fprintf(stderr, "Verifying datum..."); if (!avro_datum_equal(interop, out_datum)) { fprintf(stderr, "fail!\n"); exit(EXIT_FAILURE); } fprintf(stderr, "ok\n"); check(rval, avro_file_reader_close(file_reader)); fprintf(stderr, "Closed reader.\n"); return 0; }