/* Plays rounds of allocations/librations on partial URL */ static void test_dup (void) { struct oio_url_s *u0 = oio_url_empty (); oio_url_set (u0, OIOURL_NS, "NS"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_ACCOUNT, "ACCT"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_USER, "JFS"); _round_dup_cleanup (u0); (void) oio_url_get_id (u0); (void) oio_url_get_id_size (u0); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_TYPE, "mail"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_PATH, "path"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_CONTENTID, "XYZT"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_CONTENTID, "0000"); _round_dup_cleanup (u0); oio_url_set (u0, OIOURL_VERSION, "0"); _round_dup_cleanup (u0); oio_url_pclean (&u0); }
static struct oio_url_s * _metacd_load_url (struct req_args_s *args) { const gchar *s; struct oio_url_s *url = oio_url_empty(); if (NULL != (s = NS())) oio_url_set (url, OIOURL_NS, s); if (NULL != (s = ACCOUNT())) oio_url_set (url, OIOURL_ACCOUNT, s); if (NULL != (s = REF())) oio_url_set (url, OIOURL_USER, s); if (NULL != (s = TYPE())) oio_url_set (url, OIOURL_TYPE, s); if (NULL != (s = PATH())) { oio_url_set (url, OIOURL_PATH, s); if (NULL != (s = VERSION())) oio_url_set (url, OIOURL_VERSION, s); } if (NULL != (s = CID())) oio_url_set (url, OIOURL_HEXID, s); if (NULL != (s = CONTENT())) oio_url_set (url, OIOURL_CONTENTID, s); return url; }
static GError * _get_peers(struct sqlx_service_s *ss, struct sqlx_name_s *n, gboolean nocache, gchar ***result) { EXTRA_ASSERT(ss != NULL); EXTRA_ASSERT(result != NULL); gint retries = 1; gchar **peers = NULL; GError *err = NULL; gint64 seq = 1; struct oio_url_s *u = oio_url_empty (); oio_url_set(u, OIOURL_NS, ss->ns_name); if (!sqlx_name_extract (n, u, NAME_SRVTYPE_META2, &seq)) { oio_url_pclean (&u); return BADREQ("Invalid type name: '%s'", n->type); } retry: if (nocache) { hc_decache_reference_service(ss->resolver, u, n->type); if (!result) { oio_url_pclean (&u); return NULL; } } peers = NULL; err = hc_resolve_reference_service(ss->resolver, u, n->type, &peers); if (NULL != err) { g_prefix_error(&err, "Peer resolution error: "); oio_url_clean(u); return err; } gchar **out = filter_services_and_clean(ss, peers, seq, n->type); if (!out) { if (retries-- > 0) { nocache = TRUE; goto retry; } err = NEWERROR(CODE_CONTAINER_NOTFOUND, "Base not managed"); } if (err) { if (out) g_strfreev (out); *result = NULL; } else { *result = out; } oio_url_clean(u); return err; }
static struct oio_url_s * _init_url (struct test_data_s *td) { struct oio_url_s *url = oio_url_empty (); if (td->ns) oio_url_set (url, OIOURL_NS, td->ns); if (td->account) oio_url_set (url, OIOURL_ACCOUNT, td->account); if (td->ref) oio_url_set (url, OIOURL_USER, td->ref); if (td->type) oio_url_set (url, OIOURL_TYPE, td->type); if (td->path) oio_url_set (url, OIOURL_PATH, td->path); return url; }
static void meta2_on_close(struct sqlx_sqlite3_s *sq3, gboolean deleted, gpointer cb_data) { gint64 seq = 1; EXTRA_ASSERT(sq3 != NULL); if (!deleted) return; struct oio_url_s *u = oio_url_empty (); oio_url_set (u, OIOURL_NS, PSRV(cb_data)->ns_name); if (!sqlx_name_extract ((struct sqlx_name_s*)&sq3->name, u, NAME_SRVTYPE_META2, &seq)) { GRID_WARN("Invalid base name [%s]", sq3->name.base); return; } hc_decache_reference_service(PSRV(cb_data)->resolver, u, NAME_SRVTYPE_META2); oio_url_pclean(&u); }
static void _test_reference_cycle_round (void) { GError *err = NULL; struct oio_url_s *url = oio_url_empty (); _random_url (url); struct oio_directory_s *dir = oio_directory__create_proxy (ns); g_assert_nonnull (dir); /* link with no reference */ gchar **srvtab = NULL; err = oio_directory__link (dir, url, NAME_SRVTYPE_META2, FALSE, &srvtab); g_assert_error (err, GQ(), CODE_USER_NOTFOUND); g_assert_null (srvtab); g_clear_error (&err); /* create */ err = oio_directory__create (dir, url); g_assert_no_error (err); /* link with a reference */ err = oio_directory__link (dir, url, NAME_SRVTYPE_META2, FALSE, &srvtab); g_assert_no_error (err); g_assert_nonnull (srvtab); g_strfreev (srvtab); /* list */ gchar **dirtab = NULL; err = oio_directory__list (dir, url, NAME_SRVTYPE_META2, &dirtab, &srvtab); g_assert_no_error (err); g_assert_nonnull (dirtab); g_assert_nonnull (srvtab); g_strfreev (dirtab); g_strfreev (srvtab); oio_url_pclean (&url); oio_directory__destroy (dir); }
struct oio_url_s * metautils_message_extract_url (MESSAGE m) { struct oio_url_s *url = oio_url_empty (); for (struct map_s *p = url2msg_map; p->f ;++p) { // TODO call really often, so make it zero-copy gchar *s = metautils_message_extract_string_copy (m, p->f); if (s) { if (!p->avoid || strcmp(p->avoid, s)) oio_url_set (url, p->u, s); g_free0 (s); } } container_id_t cid; GError *e = metautils_message_extract_cid (m, NAME_MSGKEY_CONTAINERID, &cid); if (e) g_clear_error (&e); else oio_url_set_id (url, cid); return url; }
static struct oio_error_s * _roundtrip_common (struct oio_sds_s *client, struct oio_url_s *url, const char *path) { struct file_info_s fi, fi0; struct oio_error_s *err = NULL; int has = 0; err = (struct oio_error_s*) _checksum_file (path, &fi0); MAYBERETURN(err, "Checksum error (original): "); gchar tmppath[256] = ""; g_snprintf (tmppath, sizeof(tmppath), "/tmp/test-roundtrip-%d-%lu-", getpid(), time(0)); _append_random_chars (tmppath, random_chars, 16); gchar content_id[65] = ""; _append_random_chars (content_id, hex_chars, 64); GRID_INFO ("Roundtrip on local(%s) distant(%s) content_id(%s)", tmppath, oio_url_get (url, OIOURL_WHOLE), content_id); /* Check the content is not preset yet */ err = oio_sds_has (client, url, &has); if (!err && has) err = (struct oio_error_s*) NEWERROR(0,"content already present"); MAYBERETURN(err, "Check error"); GRID_INFO("Content absent as expected"); /* Then upload it */ struct oio_sds_ul_src_s ul_src = { .type = OIO_UL_SRC_FILE, .data = { .file = { .path = path, .offset = 0, .size = 0, }, }, }; struct oio_sds_ul_dst_s ul_dst = { .url = url, .autocreate = 1, .out_size = 0, .content_id = content_id, }; err = oio_sds_upload (client, &ul_src, &ul_dst); MAYBERETURN(err, "Upload error"); GRID_INFO("Content uploaded"); /* Check it is now present */ has = 0; err = oio_sds_has (client, url, &has); if (!err && !has) err = (struct oio_error_s*) NEWERROR(0, "content not found"); MAYBERETURN(err, "Check error"); GRID_INFO("Content present as expected"); /* Get it to validate the content is accessible */ err = oio_sds_download_to_file (client, url, tmppath); MAYBERETURN(err, "Download error"); GRID_INFO("Content downloaded to a file"); /* Validate the original and the copy match */ err = (struct oio_error_s*) _checksum_file (tmppath, &fi); MAYBERETURN(err, "Checksum error (copy): "); if (fi.fs != fi0.fs) MAYBERETURN(NEWERROR(0, "Copy sizes mismatch"), "Validation error"); if (0 != memcmp(fi.h, fi0.h, fi.hs)) MAYBERETURN(NEWERROR(0, "Copy hash mismatch"), "Validation error"); GRID_INFO("The original file and its copy match"); /* Get it an other way in a buffer. */ guint8 buf[1024]; struct oio_sds_dl_dst_s dl_dst = { .type = OIO_DL_DST_BUFFER, .data = {.buffer = {.ptr = buf, .length=1024}} }; struct oio_sds_dl_src_s dl_src = { .url = url, .ranges = NULL, }; err = oio_sds_download (client, &dl_src, &dl_dst); MAYBERETURN(err, "Download error"); GRID_INFO("Content downloaded to a buffer"); /* link the container */ struct oio_url_s *url1 = oio_url_dup (url); oio_url_set (url1, OIOURL_PATH, tmppath); err = oio_sds_link (client, url1, content_id); oio_url_pclean (&url1); MAYBERETURN(err, "Link error: "); /* List the container, the content must appear */ struct oio_sds_list_param_s list_in = { .url = url, .prefix = NULL, .marker = NULL, .end = NULL, .delimiter = 0, .flag_allversions = 0, .flag_nodeleted = 0, }; struct oio_sds_list_listener_s list_out = { .ctx = NULL, .on_item = _on_item, .on_prefix = NULL, .on_bound = NULL, }; err = oio_sds_list (client, &list_in, &list_out); MAYBERETURN(err, "List error"); /* Remove the content from the content */ err = oio_sds_delete (client, url); MAYBERETURN(err, "Delete error"); GRID_INFO("Content removed"); /* Check the content is not preset anymore */ has = 0; err = oio_sds_has (client, url, &has); if (!err && has) err = (struct oio_error_s*) NEWERROR(0, "content still present"); MAYBERETURN(err, "Check error"); GRID_INFO("Content absent as expected"); g_remove (tmppath); oio_error_pfree (&err); return NULL; } static struct oio_error_s * _roundtrip_autocontainer (struct oio_sds_s *client, struct oio_url_s *url, const char *path) { struct file_info_s fi; GError *err = _checksum_file (path, &fi); /* compute the autocontainer with the SHA1, consider only the first 17 bits */ struct oio_str_autocontainer_config_s cfg = { .src_offset = 0, .src_size = 0, .dst_bits = 17, }; char tmp[65]; const char *auto_container = oio_str_autocontainer_hash (fi.h, fi.hs, tmp, &cfg); /* build a new URL with the computed container name */ struct oio_url_s *url_auto = oio_url_dup (url); oio_url_set (url_auto, OIOURL_USER, auto_container); err = (GError*) _roundtrip_common (client, url_auto, path); oio_url_pclean (&url_auto); return (struct oio_error_s*) err; } int main(int argc, char **argv) { oio_log_to_stderr(); oio_sds_default_autocreate = 1; for (int i=0; i<4 ;i++) oio_log_more (); prng = g_rand_new (); if (argc != 2) { g_printerr ("Usage: %s PATH\n", argv[0]); return 1; } const char *path = argv[1]; struct oio_url_s *url = oio_url_empty (); oio_url_set (url, OIOURL_NS, g_getenv("OIO_NS")); oio_url_set (url, OIOURL_ACCOUNT, g_getenv("OIO_ACCOUNT")); oio_url_set (url, OIOURL_USER, g_getenv("OIO_USER")); oio_url_set (url, OIOURL_PATH, g_getenv("OIO_PATH")); if (!oio_url_has_fq_path(url)) { g_printerr ("Partial URL [%s]: requires a NS (%s), an ACCOUNT (%s)," " an USER (%s) and a PATH (%s)\n", oio_url_get (url, OIOURL_WHOLE), oio_url_has (url, OIOURL_NS)?"ok":"missing", oio_url_has (url, OIOURL_ACCOUNT)?"ok":"missing", oio_url_has (url, OIOURL_USER)?"ok":"missing", oio_url_has (url, OIOURL_PATH)?"ok":"missing"); return 3; } GRID_INFO("URL valid [%s]", oio_url_get (url, OIOURL_WHOLE)); struct oio_sds_s *client = NULL; struct oio_error_s *err = NULL; /* Initiate a client */ err = oio_sds_init (&client, oio_url_get(url, OIOURL_NS)); if (err) { g_printerr ("Client init error: (%d) %s\n", oio_error_code(err), oio_error_message(err)); return 4; } GRID_INFO("Client ready to [%s]", oio_url_get (url, OIOURL_NS)); err = _roundtrip_common (client, url, path); if (!err) err = _roundtrip_autocontainer (client, url, path); int rc = err != NULL; oio_error_pfree (&err); oio_sds_pfree (&client); oio_url_pclean (&url); return rc; }
GError* _open_and_lock(struct meta1_backend_s *m1, struct oio_url_s *url, enum m1v2_open_type_e how, struct sqlx_sqlite3_s **handle) { EXTRA_ASSERT(m1 != NULL); EXTRA_ASSERT(url != NULL); EXTRA_ASSERT(handle != NULL); GRID_TRACE2("%s(%p,%p,%d,%p)", __FUNCTION__, (void*)m1, oio_url_get (url, OIOURL_HEXID), how, (void*)handle); if (!oio_url_has (url, OIOURL_HEXID)) return NEWERROR (CODE_BAD_REQUEST, "Partial URL (missing HEXID)"); if (!m1b_check_ns_url (m1, url)) return NEWERROR(CODE_NAMESPACE_NOTMANAGED, "Invalid NS"); gchar base[5]; const guint8 *cid = oio_url_get_id(url); g_snprintf(base, sizeof(base), "%02X%02X", cid[0], cid[1]); if (!meta1_prefixes_is_managed(m1->prefixes, cid)) return NEWERROR(CODE_RANGE_NOTFOUND, "prefix [%s] not managed", base); /* Now open/lock the base in a way suitable for our op */ struct sqlx_name_s n = {.base=base, .type=NAME_SRVTYPE_META1, .ns=m1->ns_name}; GError *err = sqlx_repository_open_and_lock(m1->repo, &n, m1_to_sqlx(how), handle, NULL); if (err != NULL) { if (!CODE_IS_REDIRECT(err->code)) g_prefix_error(&err, "Open/Lock error: "); return err; } EXTRA_ASSERT(*handle != NULL); GRID_TRACE("Opened and locked [%s][%s] -> [%s][%s]", base, NAME_SRVTYPE_META1, (*handle)->name.base, (*handle)->name.type); return NULL; } GError* __create_user(struct sqlx_sqlite3_s *sq3, struct oio_url_s *url) { if (!oio_url_has_fq_container (url)) return NEWERROR(CODE_BAD_REQUEST, "Partial URL"); static const gchar *sql = "INSERT INTO users ('cid','account','user') VALUES (?,?,?)"; GError *err = NULL; sqlite3_stmt *stmt = NULL; int rc; EXTRA_ASSERT(sq3 != NULL); EXTRA_ASSERT(sq3->db != NULL); /* Prepare the statement */ sqlite3_prepare_debug(rc, sq3->db, sql, -1, &stmt, NULL); if (rc != SQLITE_OK) err = M1_SQLITE_GERROR(sq3->db, rc); else { sqlite3_bind_blob(stmt, 1, oio_url_get_id(url), oio_url_get_id_size(url), NULL); sqlite3_bind_text(stmt, 2, oio_url_get(url, OIOURL_ACCOUNT), -1, NULL); sqlite3_bind_text(stmt, 3, oio_url_get(url, OIOURL_USER), -1, NULL); /* Run the results */ do { rc = sqlite3_step(stmt); } while (rc == SQLITE_ROW); if (rc != SQLITE_OK && rc != SQLITE_DONE) { err = M1_SQLITE_GERROR(sq3->db, rc); if (rc == SQLITE_CONSTRAINT) { g_prefix_error(&err, "Already created? "); err->code = CODE_CONTAINER_EXISTS; } } sqlite3_finalize_debug(rc, stmt); } if (err) GRID_DEBUG("User creation failed : (%d) %s", err->code, err->message); return err; } GError* __info_user(struct sqlx_sqlite3_s *sq3, struct oio_url_s *url, gboolean ac, struct oio_url_s ***result) { GError *err = NULL; sqlite3_stmt *stmt = NULL; GPtrArray *gpa; int rc; gboolean found; EXTRA_ASSERT(sq3 != NULL); EXTRA_ASSERT(sq3->db != NULL); EXTRA_ASSERT(url != NULL); retry: /* Prepare the statement */ sqlite3_prepare_debug(rc, sq3->db, "SELECT account,user FROM users WHERE cid = ?", -1, &stmt, NULL); if (rc != SQLITE_OK) return M1_SQLITE_GERROR(sq3->db, rc); (void) sqlite3_bind_blob(stmt, 1, oio_url_get_id (url), oio_url_get_id_size (url), NULL); /* Run the results */ found = FALSE; gpa = result ? g_ptr_array_new() : NULL; do { if (SQLITE_ROW == (rc = sqlite3_step(stmt))) { found = TRUE; if (!gpa) continue; struct oio_url_s *u = oio_url_empty (); oio_url_set (u, OIOURL_NS, oio_url_get (url, OIOURL_NS)); oio_url_set (u, OIOURL_ACCOUNT, (char*)sqlite3_column_text(stmt, 0)); oio_url_set (u, OIOURL_USER, (char*)sqlite3_column_text(stmt, 1)); oio_url_set (u, OIOURL_HEXID, oio_url_get (url, OIOURL_HEXID)); g_ptr_array_add(gpa, u); } } while (rc == SQLITE_ROW); if (rc != SQLITE_DONE && rc != SQLITE_OK) { err = M1_SQLITE_GERROR(sq3->db, rc); g_prefix_error(&err, "DB error: "); } sqlite3_finalize_debug(rc,stmt); stmt = NULL; if (err) { if (gpa) { g_ptr_array_set_free_func (gpa, (GDestroyNotify)oio_url_clean); g_ptr_array_free (gpa, TRUE); } return err; } if (!found) { if (gpa) g_ptr_array_free (gpa, TRUE); if (ac) { ac = FALSE; /* do not retry */ err = __create_user (sq3, url); if (!err) goto retry; } return NEWERROR(CODE_USER_NOTFOUND, "no such container"); } if (gpa) *result = (struct oio_url_s**) metautils_gpa_to_array(gpa, TRUE); return NULL; }