static void
_test_field (const char *v, struct oio_url_s *u, enum oio_url_field_e f)
{
const char *found = oio_url_get (u, f);
if (v) {
g_assert (oio_url_has (u, f));
g_assert (!strcmp (v, found));
} else {
g_assert (!oio_url_has (u, f));
g_assert (NULL == found);
}
}
static void
_test_url (guint idx, struct oio_url_s *u, struct test_data_s *td)
{
(void) idx;
_test_field (td->whole, u, OIOURL_WHOLE);
_test_field (td->ns, u, OIOURL_NS);
_test_field (td->account, u, OIOURL_ACCOUNT);
_test_field (td->ref, u, OIOURL_USER);
_test_field (td->type, u, OIOURL_TYPE);
_test_field (td->path, u, OIOURL_PATH);
if (td->hexa) {
g_assert (oio_url_has (u, OIOURL_HEXID));
g_assert (NULL != oio_url_get_id (u));
g_assert (!g_ascii_strcasecmp (oio_url_get (u, OIOURL_HEXID), td->hexa));
} else {
g_assert (!oio_url_has (u, OIOURL_HEXID));
g_assert (NULL == oio_url_get_id (u));
g_assert (NULL == oio_url_get (u, OIOURL_HEXID));
}
}
int
meta2_filter_check_url_cid (struct gridd_filter_ctx_s *ctx,
struct gridd_reply_ctx_s *reply)
{
(void) reply;
struct oio_url_s *url = meta2_filter_ctx_get_url(ctx);
TRACE_FILTER();
if (url && oio_url_has(url, OIOURL_HEXID))
return FILTER_OK;
meta2_filter_ctx_set_error (ctx, BADREQ("Invalid URL"));
return FILTER_KO;
}
int
meta2_filter_check_url_cid (struct gridd_filter_ctx_s *ctx,
struct gridd_reply_ctx_s *reply)
{
(void) reply;
struct oio_url_s *url = meta2_filter_ctx_get_url(ctx);
TRACE_FILTER();
if (url && oio_url_has(url, OIOURL_HEXID))
return FILTER_OK;
meta2_filter_ctx_set_error (ctx, NEWERROR(CODE_BAD_REQUEST, "No/partial URL"));
return FILTER_KO;
}
void
metautils_message_add_url_no_type (MESSAGE m, struct oio_url_s *url)
{
if (!m)
return;
for (struct map_s *p = url2msg_map; p->f ;++p) {
if (p->u != OIOURL_TYPE && oio_url_has (url, p->u)) {
const char *s = oio_url_get (url, p->u);
if (!p->avoid || strcmp(p->avoid, s))
metautils_message_add_field_str(m, p->f, s);
}
}
const guint8 *id = oio_url_get_id (url);
if (id)
metautils_message_add_field (m, NAME_MSGKEY_CONTAINERID, id, oio_url_get_id_size (url));
}
static struct oio_error_s *
_roundtrip_common (struct oio_sds_s *client, struct oio_url_s *url,
const char *path)
{
struct file_info_s fi, fi0;
struct oio_error_s *err = NULL;
int has = 0;
err = (struct oio_error_s*) _checksum_file (path, &fi0);
MAYBERETURN(err, "Checksum error (original): ");
gchar tmppath[256] = "";
g_snprintf (tmppath, sizeof(tmppath), "/tmp/test-roundtrip-%d-%lu-", getpid(), time(0));
_append_random_chars (tmppath, random_chars, 16);
gchar content_id[65] = "";
_append_random_chars (content_id, hex_chars, 64);
GRID_INFO ("Roundtrip on local(%s) distant(%s) content_id(%s)", tmppath,
oio_url_get (url, OIOURL_WHOLE), content_id);
/* Check the content is not preset yet */
err = oio_sds_has (client, url, &has);
if (!err && has) err = (struct oio_error_s*) NEWERROR(0,"content already present");
MAYBERETURN(err, "Check error");
GRID_INFO("Content absent as expected");
/* Then upload it */
struct oio_sds_ul_src_s ul_src = {
.type = OIO_UL_SRC_FILE,
.data = { .file = { .path = path, .offset = 0, .size = 0, }, },
};
struct oio_sds_ul_dst_s ul_dst = {
.url = url, .autocreate = 1, .out_size = 0, .content_id = content_id,
};
err = oio_sds_upload (client, &ul_src, &ul_dst);
MAYBERETURN(err, "Upload error");
GRID_INFO("Content uploaded");
/* Check it is now present */
has = 0;
err = oio_sds_has (client, url, &has);
if (!err && !has) err = (struct oio_error_s*) NEWERROR(0, "content not found");
MAYBERETURN(err, "Check error");
GRID_INFO("Content present as expected");
/* Get it to validate the content is accessible */
err = oio_sds_download_to_file (client, url, tmppath);
MAYBERETURN(err, "Download error");
GRID_INFO("Content downloaded to a file");
/* Validate the original and the copy match */
err = (struct oio_error_s*) _checksum_file (tmppath, &fi);
MAYBERETURN(err, "Checksum error (copy): ");
if (fi.fs != fi0.fs)
MAYBERETURN(NEWERROR(0, "Copy sizes mismatch"), "Validation error");
if (0 != memcmp(fi.h, fi0.h, fi.hs))
MAYBERETURN(NEWERROR(0, "Copy hash mismatch"), "Validation error");
GRID_INFO("The original file and its copy match");
/* Get it an other way in a buffer. */
guint8 buf[1024];
struct oio_sds_dl_dst_s dl_dst = {
.type = OIO_DL_DST_BUFFER,
.data = {.buffer = {.ptr = buf, .length=1024}}
};
struct oio_sds_dl_src_s dl_src = {
.url = url,
.ranges = NULL,
};
err = oio_sds_download (client, &dl_src, &dl_dst);
MAYBERETURN(err, "Download error");
GRID_INFO("Content downloaded to a buffer");
/* link the container */
struct oio_url_s *url1 = oio_url_dup (url);
oio_url_set (url1, OIOURL_PATH, tmppath);
err = oio_sds_link (client, url1, content_id);
oio_url_pclean (&url1);
MAYBERETURN(err, "Link error: ");
/* List the container, the content must appear */
struct oio_sds_list_param_s list_in = {
.url = url,
.prefix = NULL, .marker = NULL, .end = NULL, .delimiter = 0,
.flag_allversions = 0, .flag_nodeleted = 0,
};
struct oio_sds_list_listener_s list_out = {
.ctx = NULL,
.on_item = _on_item, .on_prefix = NULL, .on_bound = NULL,
};
err = oio_sds_list (client, &list_in, &list_out);
MAYBERETURN(err, "List error");
/* Remove the content from the content */
err = oio_sds_delete (client, url);
MAYBERETURN(err, "Delete error");
GRID_INFO("Content removed");
/* Check the content is not preset anymore */
has = 0;
err = oio_sds_has (client, url, &has);
if (!err && has) err = (struct oio_error_s*) NEWERROR(0, "content still present");
MAYBERETURN(err, "Check error");
GRID_INFO("Content absent as expected");
g_remove (tmppath);
oio_error_pfree (&err);
return NULL;
}
static struct oio_error_s *
_roundtrip_autocontainer (struct oio_sds_s *client, struct oio_url_s *url,
const char *path)
{
struct file_info_s fi;
GError *err = _checksum_file (path, &fi);
/* compute the autocontainer with the SHA1, consider only the first 17 bits */
struct oio_str_autocontainer_config_s cfg = {
.src_offset = 0, .src_size = 0,
.dst_bits = 17,
};
char tmp[65];
const char *auto_container = oio_str_autocontainer_hash (fi.h, fi.hs, tmp, &cfg);
/* build a new URL with the computed container name */
struct oio_url_s *url_auto = oio_url_dup (url);
oio_url_set (url_auto, OIOURL_USER, auto_container);
err = (GError*) _roundtrip_common (client, url_auto, path);
oio_url_pclean (&url_auto);
return (struct oio_error_s*) err;
}
int
main(int argc, char **argv)
{
oio_log_to_stderr();
oio_sds_default_autocreate = 1;
for (int i=0; i<4 ;i++)
oio_log_more ();
prng = g_rand_new ();
if (argc != 2) {
g_printerr ("Usage: %s PATH\n", argv[0]);
return 1;
}
const char *path = argv[1];
struct oio_url_s *url = oio_url_empty ();
oio_url_set (url, OIOURL_NS, g_getenv("OIO_NS"));
oio_url_set (url, OIOURL_ACCOUNT, g_getenv("OIO_ACCOUNT"));
oio_url_set (url, OIOURL_USER, g_getenv("OIO_USER"));
oio_url_set (url, OIOURL_PATH, g_getenv("OIO_PATH"));
if (!oio_url_has_fq_path(url)) {
g_printerr ("Partial URL [%s]: requires a NS (%s), an ACCOUNT (%s),"
" an USER (%s) and a PATH (%s)\n",
oio_url_get (url, OIOURL_WHOLE),
oio_url_has (url, OIOURL_NS)?"ok":"missing",
oio_url_has (url, OIOURL_ACCOUNT)?"ok":"missing",
oio_url_has (url, OIOURL_USER)?"ok":"missing",
oio_url_has (url, OIOURL_PATH)?"ok":"missing");
return 3;
}
GRID_INFO("URL valid [%s]", oio_url_get (url, OIOURL_WHOLE));
struct oio_sds_s *client = NULL;
struct oio_error_s *err = NULL;
/* Initiate a client */
err = oio_sds_init (&client, oio_url_get(url, OIOURL_NS));
if (err) {
g_printerr ("Client init error: (%d) %s\n", oio_error_code(err),
oio_error_message(err));
return 4;
}
GRID_INFO("Client ready to [%s]", oio_url_get (url, OIOURL_NS));
err = _roundtrip_common (client, url, path);
if (!err)
err = _roundtrip_autocontainer (client, url, path);
int rc = err != NULL;
oio_error_pfree (&err);
oio_sds_pfree (&client);
oio_url_pclean (&url);
return rc;
}
GError*
_open_and_lock(struct meta1_backend_s *m1, struct oio_url_s *url,
enum m1v2_open_type_e how, struct sqlx_sqlite3_s **handle)
{
EXTRA_ASSERT(m1 != NULL);
EXTRA_ASSERT(url != NULL);
EXTRA_ASSERT(handle != NULL);
GRID_TRACE2("%s(%p,%p,%d,%p)", __FUNCTION__, (void*)m1,
oio_url_get (url, OIOURL_HEXID), how, (void*)handle);
if (!oio_url_has (url, OIOURL_HEXID))
return NEWERROR (CODE_BAD_REQUEST, "Partial URL (missing HEXID)");
if (!m1b_check_ns_url (m1, url))
return NEWERROR(CODE_NAMESPACE_NOTMANAGED, "Invalid NS");
gchar base[5];
const guint8 *cid = oio_url_get_id(url);
g_snprintf(base, sizeof(base), "%02X%02X", cid[0], cid[1]);
if (!meta1_prefixes_is_managed(m1->prefixes, cid))
return NEWERROR(CODE_RANGE_NOTFOUND, "prefix [%s] not managed", base);
/* Now open/lock the base in a way suitable for our op */
struct sqlx_name_s n = {.base=base, .type=NAME_SRVTYPE_META1, .ns=m1->ns_name};
GError *err = sqlx_repository_open_and_lock(m1->repo, &n, m1_to_sqlx(how), handle, NULL);
if (err != NULL) {
if (!CODE_IS_REDIRECT(err->code))
g_prefix_error(&err, "Open/Lock error: ");
return err;
}
EXTRA_ASSERT(*handle != NULL);
GRID_TRACE("Opened and locked [%s][%s] -> [%s][%s]",
base, NAME_SRVTYPE_META1,
(*handle)->name.base, (*handle)->name.type);
return NULL;
}
GError*
__create_user(struct sqlx_sqlite3_s *sq3, struct oio_url_s *url)
{
if (!oio_url_has_fq_container (url))
return NEWERROR(CODE_BAD_REQUEST, "Partial URL");
static const gchar *sql = "INSERT INTO users ('cid','account','user') VALUES (?,?,?)";
GError *err = NULL;
sqlite3_stmt *stmt = NULL;
int rc;
EXTRA_ASSERT(sq3 != NULL);
EXTRA_ASSERT(sq3->db != NULL);
/* Prepare the statement */
sqlite3_prepare_debug(rc, sq3->db, sql, -1, &stmt, NULL);
if (rc != SQLITE_OK)
err = M1_SQLITE_GERROR(sq3->db, rc);
else {
sqlite3_bind_blob(stmt, 1, oio_url_get_id(url), oio_url_get_id_size(url), NULL);
sqlite3_bind_text(stmt, 2, oio_url_get(url, OIOURL_ACCOUNT), -1, NULL);
sqlite3_bind_text(stmt, 3, oio_url_get(url, OIOURL_USER), -1, NULL);
/* Run the results */
do { rc = sqlite3_step(stmt); } while (rc == SQLITE_ROW);
if (rc != SQLITE_OK && rc != SQLITE_DONE) {
err = M1_SQLITE_GERROR(sq3->db, rc);
if (rc == SQLITE_CONSTRAINT) {
g_prefix_error(&err, "Already created? ");
err->code = CODE_CONTAINER_EXISTS;
}
}
sqlite3_finalize_debug(rc, stmt);
}
if (err)
GRID_DEBUG("User creation failed : (%d) %s", err->code, err->message);
return err;
}
GError*
__info_user(struct sqlx_sqlite3_s *sq3, struct oio_url_s *url, gboolean ac,
struct oio_url_s ***result)
{
GError *err = NULL;
sqlite3_stmt *stmt = NULL;
GPtrArray *gpa;
int rc;
gboolean found;
EXTRA_ASSERT(sq3 != NULL);
EXTRA_ASSERT(sq3->db != NULL);
EXTRA_ASSERT(url != NULL);
retry:
/* Prepare the statement */
sqlite3_prepare_debug(rc, sq3->db, "SELECT account,user FROM users WHERE cid = ?", -1, &stmt, NULL);
if (rc != SQLITE_OK)
return M1_SQLITE_GERROR(sq3->db, rc);
(void) sqlite3_bind_blob(stmt, 1, oio_url_get_id (url), oio_url_get_id_size (url), NULL);
/* Run the results */
found = FALSE;
gpa = result ? g_ptr_array_new() : NULL;
do { if (SQLITE_ROW == (rc = sqlite3_step(stmt))) {
found = TRUE;
if (!gpa) continue;
struct oio_url_s *u = oio_url_empty ();
oio_url_set (u, OIOURL_NS, oio_url_get (url, OIOURL_NS));
oio_url_set (u, OIOURL_ACCOUNT, (char*)sqlite3_column_text(stmt, 0));
oio_url_set (u, OIOURL_USER, (char*)sqlite3_column_text(stmt, 1));
oio_url_set (u, OIOURL_HEXID, oio_url_get (url, OIOURL_HEXID));
g_ptr_array_add(gpa, u);
} } while (rc == SQLITE_ROW);
if (rc != SQLITE_DONE && rc != SQLITE_OK) {
err = M1_SQLITE_GERROR(sq3->db, rc);
g_prefix_error(&err, "DB error: ");
}
sqlite3_finalize_debug(rc,stmt);
stmt = NULL;
if (err) {
if (gpa) {
g_ptr_array_set_free_func (gpa, (GDestroyNotify)oio_url_clean);
g_ptr_array_free (gpa, TRUE);
}
return err;
}
if (!found) {
if (gpa) g_ptr_array_free (gpa, TRUE);
if (ac) {
ac = FALSE; /* do not retry */
err = __create_user (sq3, url);
if (!err) goto retry;
}
return NEWERROR(CODE_USER_NOTFOUND, "no such container");
}
if (gpa)
*result = (struct oio_url_s**) metautils_gpa_to_array(gpa, TRUE);
return NULL;
}
static GError *
m2b_open(struct meta2_backend_s *m2, struct oio_url_s *url,
enum m2v2_open_type_e how, struct sqlx_sqlite3_s **result)
{
GError *err = NULL;
struct sqlx_sqlite3_s *sq3 = NULL;
EXTRA_ASSERT(url != NULL);
EXTRA_ASSERT(result != NULL);
EXTRA_ASSERT(m2 != NULL);
EXTRA_ASSERT(m2->backend.repo != NULL);
/* TODO */
gboolean no_peers = FALSE;
if (no_peers) {
how &= ~M2V2_OPEN_REPLIMODE;
how |= M2V2_OPEN_LOCAL|M2V2_OPEN_NOREFCHECK;
}
struct sqlx_name_mutable_s n;
sqlx_name_fill (&n, url, NAME_SRVTYPE_META2, 1);
err = sqlx_repository_open_and_lock(m2->backend.repo,
sqlx_name_mutable_to_const(&n), m2_to_sqlx(how), &sq3, NULL);
sqlx_name_clean (&n);
if (NULL != err) {
if (err->code == CODE_CONTAINER_NOTFOUND)
err->domain = GQ();
return err;
}
sq3->no_peers = how & (M2V2_OPEN_LOCAL|M2V2_OPEN_NOREFCHECK);
// XXX If the container is being deleted, this is sad ...
// This MIGHT happen if a cache is present (and this is the
// common case for m2v2), because the deletion will happen
// when the base exit the cache.
// In facts this SHOULD NOT happend because a base being deleted
// is closed with an instruction to exit the cache immediately.
// TODO FIXME this is maybe a good place for an assert().
if (sq3->deleted) {
err = NEWERROR(CODE_CONTAINER_FROZEN, "destruction pending");
m2b_close(sq3);
return err;
}
// Complete URL with full VNS and container name
void set(gchar *k, int f) {
if (oio_url_has(url, f))
return;
gchar *s = sqlx_admin_get_str (sq3, k);
if (s) {
oio_url_set (url, f, s);
g_free (s);
}
}
set (SQLX_ADMIN_NAMESPACE, OIOURL_NS);
set (SQLX_ADMIN_ACCOUNT, OIOURL_ACCOUNT);
set (SQLX_ADMIN_USERNAME, OIOURL_USER);
set (SQLX_ADMIN_USERTYPE, OIOURL_TYPE);
*result = sq3;
return NULL;
}
