int ds_var_field_writer_init2(struct ds_var_field_writer *dsvfw, const char *filename, int64_t type, int64_t element_type) { unlink(filename); if (0 > vmbuf_init(&dsvfw->ofs_table, 4096) || 0 > file_writer_init(&dsvfw->data, filename)) return -1; struct ds_var_field_header header = { -type, -1 , element_type }; /* set type to negative value so load fails if not finalized */ if (0 > file_writer_write(&dsvfw->data, &header, sizeof(header))) return -1; return 0; }
int mysql_dumper_dump(struct mysql_login_info *mysql_login_info, const char *outputdir, const char *dbname, const char *tablename, const char *query, size_t query_len, struct mysql_dumper_type *types) { MYSQL mysql; MYSQL_STMT *stmt = NULL; mysql_init(&mysql); my_bool b_flag = 1; if (0 != mysql_options(&mysql, MYSQL_OPT_RECONNECT, (const char *)&b_flag)) return report_error(&mysql); if (NULL == mysql_real_connect(&mysql, mysql_login_info->host, mysql_login_info->user, mysql_login_info->pass, mysql_login_info->db, mysql_login_info->port, NULL, CLIENT_COMPRESS)) return report_error(&mysql); b_flag = 0; if (0 != mysql_options(&mysql, MYSQL_REPORT_DATA_TRUNCATION, (const char *)&b_flag)) return report_error(&mysql); stmt = mysql_stmt_init(&mysql); if (!stmt) return report_error(&mysql); if (0 != mysql_stmt_prepare(stmt, query, query_len)) return report_stmt_error(&mysql, stmt); MYSQL_RES *rs = mysql_stmt_result_metadata(stmt); if (!rs) return report_stmt_error(&mysql, stmt); unsigned int n = mysql_num_fields(rs); MYSQL_FIELD *fields = mysql_fetch_fields(rs); int field_types[n]; MYSQL_BIND bind[n]; my_bool is_null[n]; unsigned long length[n]; my_bool error[n]; memset(bind, 0, sizeof(MYSQL_BIND) * n); int null_terminate_str[n]; memset(null_terminate_str, 0, sizeof(int) * n); struct file_writer ffields[n]; struct file_writer vfields[2][n]; struct vmbuf buf = VMBUF_INITIALIZER; vmbuf_init(&buf, 4096); vmbuf_sprintf(&buf, "%s/%s/%s/schema.txt", outputdir, dbname, tablename); mkdir_for_file_recursive(vmbuf_data(&buf)); int fdschema = creat(vmbuf_data(&buf), 0644); struct vmfile ds_txt = VMFILE_INITIALIZER; vmbuf_reset(&buf); vmbuf_sprintf(&buf, "%s/%s/%s/ds.txt", outputdir, dbname, tablename); if (0 > vmfile_init(&ds_txt, vmbuf_data(&buf), 4096)) return LOGGER_ERROR("failed to create: %s", vmbuf_data(&buf)), vmbuf_free(&buf), -1; vmfile_sprintf(&ds_txt, "DS_LOADER_BEGIN()\n"); vmfile_sprintf(&ds_txt, "/*\n * DB: %s\n */\n", dbname); vmfile_sprintf(&ds_txt, "#undef DB_NAME\n#define DB_NAME %s\n", dbname); ssize_t len = 80 - strlen(tablename); if (len < 0) len = 4; char header[len]; memset(header, '=', len); vmfile_sprintf(&ds_txt, "/* %.*s[ %s ]%.*s */\n", (int)len / 2, header, tablename, (int)(len - (len / 2)), header); vmfile_sprintf(&ds_txt, "# undef TABLE_NAME\n# define TABLE_NAME %s\n", tablename); /* * initialize output files */ unsigned int i; for (i = 0; i < n; ++i) { file_writer_make(&ffields[i]); file_writer_make(&vfields[0][i]); file_writer_make(&vfields[1][i]); } struct hashtable ht_types = HASHTABLE_INITIALIZER; hashtable_init(&ht_types, 32); if (NULL != types) { struct mysql_dumper_type *t = types; for (; t->name; ++t) { /* storing ptr here which points to static str */ hashtable_insert(&ht_types, t->name, strlen(t->name), t, sizeof(struct mysql_dumper_type)); } } /* * parse meta data and construct bind array */ int err = 0; for (i = 0; i < n; ++i) { field_types[i] = fields[i].type; bind[i].is_unsigned = IS_UNSIGNED(fields[i].flags); int64_t ds_type = -1; const char *ds_type_str = "VAR"; /* * handle overrides */ while (NULL != types) { uint32_t ofs = hashtable_lookup(&ht_types, fields[i].name, strlen(fields[i].name)); if (!ofs) break; struct mysql_dumper_type *type = (struct mysql_dumper_type *)hashtable_get_val(&ht_types, ofs); null_terminate_str[i] = MYSQL_DUMPER_CSTR & type->flags; if (type->mysql_type) field_types[i] = type->mysql_type; bind[i].is_unsigned = (type->flags & MYSQL_DUMPER_UNSIGNED) > 0 ? 1 : 0; break; } vmbuf_reset(&buf); vmbuf_sprintf(&buf, "%s/%s/%s/%s", outputdir, dbname, tablename, fields[i].name); mkdir_for_file_recursive(vmbuf_data(&buf)); if (is_var_length_field(field_types[i])) { size_t ofs = vmbuf_wlocpos(&buf); vmbuf_sprintf(&buf, ".ofs"); if (0 > (err = file_writer_init(&vfields[0][i], vmbuf_data(&buf)))) break; vmbuf_wlocset(&buf, ofs); vmbuf_sprintf(&buf, ".dat"); if (0 > (err = file_writer_init(&vfields[1][i], vmbuf_data(&buf)))) break; } else { ds_type = get_ds_type(field_types[i], bind[i].is_unsigned); const char *s = get_ds_type_str(ds_type); if (*s) ds_type_str = s; if (0 > (err = file_writer_init(&ffields[i], vmbuf_data(&buf))) || 0 > (err = file_writer_write(&ffields[i], &ds_type, sizeof(ds_type)))) break;; } len = ribs_mysql_get_storage_size(field_types[i], fields[i].length); if (fdschema > 0) dprintf(fdschema, "%03d name = %s, size=%zu, length=%lu, type=%s (%s), is_prikey=%d, ds_type=%s\n", i, fields[i].name, len, fields[i].length, ribs_mysql_get_type_name(field_types[i]), bind[i].is_unsigned ? "unsigned" : "signed", IS_PRI_KEY(fields[i].flags), ds_type_str); if (is_var_length_field(field_types[i])) { vmfile_sprintf(&ds_txt, " DS_VAR_FIELD_LOADER(%s)\n", fields[i].name); } else { vmfile_sprintf(&ds_txt, " DS_FIELD_LOADER(%s, %s)\n", ds_type_str, fields[i].name); } bind[i].buffer_type = field_types[i]; bind[i].buffer_length = len; bind[i].buffer = malloc(len); bind[i].is_null = &is_null[i]; bind[i].length = &length[i]; bind[i].error = &error[i]; } hashtable_free(&ht_types); mysql_free_result(rs); close(fdschema); //vmfile_sprintf(&ds_txt, "/*\n * TABLE END: %s\n */\n", tablename); vmfile_sprintf(&ds_txt, "DS_LOADER_END()\n"); vmfile_close(&ds_txt); /* * execute & bind */ if (0 != err || 0 != mysql_stmt_execute(stmt) || 0 != mysql_stmt_bind_result(stmt, bind)) { err = -1; report_stmt_error(&mysql, stmt); goto dumper_close_writer; } char zeros[4096]; memset(zeros, 0, sizeof(zeros)); int mysql_err = 0; size_t count = 0, num_rows_errors = 0; /* * write all rows to output files */ while (0 == (mysql_err = mysql_stmt_fetch(stmt))) { int b = 0; for (i = 0; i < n && !b; ++i) b = b || error[i]; if (b) { ++num_rows_errors; continue; } for (i = 0; i < n; ++i) { if (is_var_length_field(field_types[i])) { size_t ofs = file_writer_wlocpos(&vfields[1][i]); if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))) || 0 > (err = file_writer_write(&vfields[1][i], is_null[i] ? NULL : bind[i].buffer, is_null[i] ? 0 : length[i]))) goto dumper_error; if (null_terminate_str[i]) { const char c = '\0'; if (0 > (err = file_writer_write(&vfields[1][i], &c, sizeof(c)))) goto dumper_error; } } else { if (0 > (err = file_writer_write(&ffields[i], is_null[i] ? zeros : bind[i].buffer, bind[i].buffer_length))) goto dumper_error; } } ++count; } /* no dumper errors */ goto dumper_ok; dumper_error: LOGGER_ERROR("failed to write data, aborting"); dumper_ok: /* we are done with mysql, close it */ mysql_stmt_close(stmt); mysql_close(&mysql); LOGGER_INFO("%s: %zu records, %zu skipped", tablename, count, num_rows_errors); /* check for mysql errors */ if (mysql_err != MYSQL_NO_DATA) { LOGGER_ERROR("mysql_stmt_fetch returned an error (code=%d)\n", mysql_err); err = -1; } dumper_close_writer: /* * finalize & free memory */ for (i = 0; i < n; ++i) { if (is_var_length_field(field_types[i])) { size_t ofs = file_writer_wlocpos(&vfields[1][i]); if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs)))) LOGGER_ERROR("failed to write offset"); file_writer_close(&vfields[0][i]); file_writer_close(&vfields[1][i]); } else { file_writer_close(&ffields[i]); } free(bind[i].buffer); } vmbuf_free(&buf); return err; }
int main(int argc, char *argv[]) { size_t num_files = 0; char **files = (char **) calloc(MAX_FILE_SUPPORT, MAX_FILE_SUPPORT * sizeof(char *)); if (0 > init_log_config(&logconf, argc, argv)) { exit(EXIT_FAILURE); } if (!SSTRISEMPTY(logconf.target) && !SSTRISEMPTY(logconf.interface)) { LOGGER_ERROR("%s", "cannot write to target and interface together. please choose one"); exit(EXIT_FAILURE); } char *f = logconf.watch_files; if (!SSTRISEMPTY(f)) { while (f != NULL) { char *fprime = strsep(&f, ","); if (fprime != NULL) { files[num_files] = strdup(fprime); ++num_files; } } } else { LOGGER_ERROR("%s", "no files..no watch!"); exit(EXIT_FAILURE); } if (0 > epoll_worker_init()) { LOGGER_ERROR("%s", "epoll_worker_init"); exit(EXIT_FAILURE); } ribs_timer(60*1000, dump_stats); tab_event_fds = thashtable_create(); delta_push = thashtable_create(); vmbuf_init(&write_buffer, 4096); vmbuf_init(&mb, 4096); if (SSTRISEMPTY(logconf.interface) && !SSTRISEMPTY(logconf.target)) { file_writer_make(&fw); if (0 > file_writer_init(&fw, logconf.target)) { LOGGER_ERROR("%s", "flie_writer"); exit(EXIT_FAILURE); } write_to_file = true; } else if (!SSTRISEMPTY(logconf.interface)) { if (0 > http_client_pool_init(&client_pool, 20, 20)) { LOGGER_ERROR("http_client_pool_init"); exit(EXIT_FAILURE); } memset(&eserv, 0, sizeof(eserv)); vmbuf_reset(&write_buffer); _replace(logconf.interface, &write_buffer, "http://www.", ""); _replace(logconf.interface, &write_buffer, "http://", ""); char *interface = vmbuf_data(&write_buffer); eserv.context = ribs_strdup(strstr(interface, "/")); char *ln = strchr(interface, '/'); ln = ribs_malloc_sprintf("%.*s", ((int)strlen(interface) - (int)strlen(ln)), interface); if (0 > parse_host_to_inet(ln, eserv.hostname, &eserv.server, &eserv.port)) { LOGGER_ERROR("%s", "server details invalid. cannot parse server"); exit(EXIT_FAILURE); } } else { LOGGER_ERROR("%s", "no target defined. please use target or interface"); exit(EXIT_FAILURE); } char _hostname[1024]; gethostname(_hostname, 1024); hostname = ribs_strdup(_hostname); int wd = inotify_init1(IN_NONBLOCK); if (0 >= wd) { LOGGER_ERROR("%s", "failed to init inotify. cannot proceed. make sure you've inotify and is accessible to this user."); exit(EXIT_FAILURE); } if (!recursive_flush_events(wd, files, num_files)) { LOGGER_ERROR("%s", "collection failed"); abort(); } return 0; }