static GError *
__notify_services(struct meta1_backend_s *m1, struct sqlx_sqlite3_s *sq3,
struct oio_url_s *url)
{
if (!m1->notifier)
return NULL;
struct meta1_service_url_s **services = NULL;
GError *err = __get_container_all_services(sq3, url, NULL, &services);
if (!err) {
struct meta1_service_url_s **services2 = expand_urlv(services);
GString *notif = g_string_sized_new(128);
g_string_append (notif, "{\"event\":\""NAME_SRVTYPE_META1".account.services\"");
g_string_append_printf (notif, ",\"when\":%"G_GINT64_FORMAT, oio_ext_real_time());
g_string_append (notif, ",\"data\":{");
g_string_append_printf (notif, "\"url\":\"%s\"", oio_url_get(url, OIOURL_WHOLE));
g_string_append (notif, ",\"services\":[");
if (services2) {
for (struct meta1_service_url_s **svc = services2; *svc ; svc++) {
if (svc != services2) // not at the beginning
g_string_append(notif, ",");
meta1_service_url_encode_json(notif, *svc);
}
}
g_string_append(notif, "]}}");
oio_events_queue__send (m1->notifier, g_string_free(notif, FALSE));
meta1_service_url_cleanv(services2);
meta1_service_url_cleanv(services);
}
return err;
}
static GError *
_create_container_init_phase(struct sqlx_sqlite3_s *sq3,
struct oio_url_s *url, struct m2v2_create_params_s *params)
{
GError *err = NULL;
struct sqlx_repctx_s *repctx = NULL;
if (!params->local && (err = _transaction_begin(sq3, url, &repctx)))
return err;
if (!err && params->storage_policy)
err = m2db_set_storage_policy(sq3, params->storage_policy, 0);
if (!err && params->version_policy) {
gint64 max = g_ascii_strtoll(params->version_policy, NULL, 10);
m2db_set_max_versions(sq3, max);
}
if (!err) {
m2db_set_ctime (sq3, oio_ext_real_time());
sqlx_admin_init_i64(sq3, META2_INIT_FLAG, 1);
}
if (!err && params->properties) {
for (gchar **p=params->properties; *p && *(p+1) ;p+=2)
sqlx_admin_set_str (sq3, *p, *(p+1));
}
if (!params->local)
err = sqlx_transaction_end(repctx, err);
return err;
}
static gboolean
hook_increment(gpointer k, gpointer v, gpointer u)
{
(void) k; (void) u;
OV(v)->version ++;
OV(v)->when = oio_ext_real_time() / G_TIME_SPAN_SECOND;
return FALSE;
}
GError *
meta2_backend_create_container(struct meta2_backend_s *m2,
struct oio_url_s *url, struct m2v2_create_params_s *params)
{
GError *err = NULL;
enum m2v2_open_type_e open_mode = 0;
struct sqlx_sqlite3_s *sq3 = NULL;
GRID_DEBUG("CREATE(%s,%s,%s)%s", oio_url_get(url, OIOURL_WHOLE),
params?params->storage_policy:NULL,
params?params->version_policy:NULL,
(params && params->local)? " (local)" : "");
/* We must check storage policy BEFORE opening the base if we don't
* want to have an empty base in case of invalid policy */
if (params->storage_policy) {
if (NULL != (err = _check_policy(m2, params->storage_policy)))
return err;
}
if (params->local) // NOREFCHECK: do not call get_peers()
open_mode = M2V2_OPEN_LOCAL|M2V2_OPEN_NOREFCHECK;
else
open_mode = M2V2_OPEN_MASTERONLY;
open_mode |= M2V2_OPEN_AUTOCREATE;
err = m2b_open(m2, url, open_mode, &sq3);
if (sq3 && !err) {
if (sqlx_admin_has(sq3, META2_INIT_FLAG))
err = NEWERROR(CODE_CONTAINER_EXISTS, "Container already initiated");
else {
err = _create_container_init_phase(sq3, url, params);
if (err) {
m2b_destroy(sq3);
return err;
}
if (!params->local && sq3->election == ELECTION_LEADER && m2->notify.hook) {
GString *gs = g_string_new ("{");
g_string_append (gs, "\"event\":\""NAME_SRVTYPE_META2".container.create\"");
g_string_append_printf (gs, ",\"when\":%"G_GINT64_FORMAT, oio_ext_real_time());
g_string_append (gs, ",\"data\":{");
g_string_append (gs, "\"url\":{");
_append_url (gs, url);
g_string_append (gs, "}}}");
m2->notify.hook (m2->notify.udata, g_string_free (gs, FALSE));
}
}
m2b_close(sq3);
}
return err;
}
GError*
m2v2_json_load_single_chunk (struct json_object *j, gpointer *pbean)
{
GError *err = NULL;
GByteArray *hid = NULL, *hash = NULL;
struct bean_CHUNKS_s *chunk = NULL;
struct json_object *jid, *jcontent, *jhash, *jsize, *jctime, *jpos;
struct oio_ext_json_mapping_s mapping[] = {
{"id", &jid, json_type_string, 1},
{"hash", &jhash, json_type_string, 1},
{"size", &jsize, json_type_int, 1},
{"ctime", &jctime, json_type_int, 0},
{"content", &jcontent, json_type_string, 1},
{"pos", &jpos, json_type_string, 1},
{NULL, NULL, 0, 0}
};
*pbean = NULL;
if (NULL != (err = oio_ext_extract_json (j, mapping)))
return err;
hid = metautils_gba_from_hexstring(json_object_get_string(jcontent));
if (!hid) {
err = NEWERROR(CODE_BAD_REQUEST, "Invalid header, not hexa id");
goto exit;
}
hash = metautils_gba_from_hexstring(json_object_get_string(jhash));
if (!hash) {
err = NEWERROR(CODE_BAD_REQUEST, "Invalid chunk, not hexa header id");
goto exit;
}
chunk = _bean_create (&descr_struct_CHUNKS);
CHUNKS_set2_id (chunk, json_object_get_string(jid));
CHUNKS_set_hash (chunk, hash);
CHUNKS_set_size (chunk, json_object_get_int64(jsize));
CHUNKS_set_ctime (chunk, !jctime ? oio_ext_real_time() / G_TIME_SPAN_SECOND : json_object_get_int64(jctime));
CHUNKS_set_content (chunk, hid);
CHUNKS_set2_position (chunk, json_object_get_string (jpos));
*pbean = chunk;
chunk = NULL;
exit:
metautils_gba_unref (hid);
metautils_gba_unref (hash);
_bean_clean (chunk);
return err;
}
static enum http_rc_e
_registration (struct req_args_s *args, enum reg_op_e op, struct json_object *jsrv)
{
GError *err;
if (!jsrv || !json_object_is_type (jsrv, json_type_object))
return _reply_common_error (args, BADREQ("Expected: json object"));
if (!push_queue)
return _reply_bad_gateway(args, NEWERROR(CODE_INTERNAL_ERROR,
"Service upstream disabled"));
if (NULL != (err = _cs_check_tokens(args)))
return _reply_notfound_error (args, err);
struct service_info_s *si = NULL;
err = service_info_load_json_object (jsrv, &si, TRUE);
if (err) {
if (err->code == CODE_BAD_REQUEST)
return _reply_format_error (args, err);
else
return _reply_system_error (args, err);
}
if (!si->type[0]) {
service_info_clean (si);
return _reply_format_error (args, BADREQ("Service type not specified"));
}
if (!si->ns_name[0]) {
g_strlcpy (si->ns_name, nsname, sizeof(si->ns_name));
} else if (!validate_namespace (si->ns_name)) {
service_info_clean (si);
return _reply_format_error (args, NEWERROR (CODE_NAMESPACE_NOTMANAGED,
"Unexpected NS"));
}
si->score.timestamp = oio_ext_real_time () / G_TIME_SPAN_SECOND;
if (op == REGOP_PUSH)
si->score.value = SCORE_UNSET;
else if (op == REGOP_UNLOCK)
si->score.value = SCORE_UNLOCK;
else /* if (op == REGOP_LOCK) */
si->score.value = CLAMP(si->score.value, SCORE_DOWN, SCORE_MAX);
// TODO follow the DRY principle and factorize this!
if (flag_cache_enabled) {
GString *gstr = g_string_new ("");
service_info_encode_json (gstr, si, TRUE);
PUSH_DO(lru_tree_insert(push_queue, service_info_key(si), si));
return _reply_success_json (args, gstr);
} else {
CSURL(cs);
GSList l = {.data = si, .next = NULL};
if (NULL != (err = conscience_remote_push_services (cs, &l))) {
service_info_clean (si);
return _reply_common_error (args, err);
} else {
GString *gstr = g_string_new ("");
service_info_encode_json (gstr, si, TRUE);
service_info_clean (si);
return _reply_success_json (args, gstr);
}
}
}
static struct oio_error_s *
_roundtrip_common (struct oio_sds_s *client, struct oio_url_s *url,
const char *path)
{
struct file_info_s fi, fi0;
struct oio_error_s *err = NULL;
int has = 0;
err = (struct oio_error_s*) _checksum_file (path, &fi0);
MAYBERETURN(err, "Checksum error (original): ");
gchar tmppath[256] = "";
g_snprintf (tmppath, sizeof(tmppath),
"/tmp/test-roundtrip-%d-%"G_GINT64_FORMAT"-",
getpid(), oio_ext_real_time());
_append_random_chars (tmppath, random_chars, 16);
gchar content_id[17] = "";
_append_random_chars (content_id, hex_chars, 16);
GRID_INFO ("Roundtrip on local(%s) distant(%s) content_id(%s)", tmppath,
oio_url_get (url, OIOURL_WHOLE), content_id);
/* Check the content is not present yet */
err = oio_sds_has (client, url, &has);
if (!err && has) err = (struct oio_error_s*) NEWERROR(0,"content already present");
MAYBERETURN(err, "Check error");
GRID_INFO("Content absent as expected");
/* Then upload it */
struct oio_sds_ul_dst_s ul_dst = {
.url = url, .autocreate = 1, .out_size = 0, .content_id = content_id,
};
err = oio_sds_upload_from_file (client, &ul_dst, path, 0, 0);
MAYBERETURN(err, "Upload error");
GRID_INFO("Content uploaded");
/* Check it is now present */
has = 0;
err = oio_sds_has (client, url, &has);
if (!err && !has) err = (struct oio_error_s*) NEWERROR(0, "content not found");
MAYBERETURN(err, "Check error");
GRID_INFO("Content present as expected");
/* Get it to validate the content is accessible */
err = oio_sds_download_to_file (client, url, tmppath);
MAYBERETURN(err, "Download error");
GRID_INFO("Content downloaded to a file");
/* Validate the original and the copy match */
err = (struct oio_error_s*) _checksum_file (tmppath, &fi);
MAYBERETURN(err, "Checksum error (copy): ");
if (fi.fs != fi0.fs)
MAYBERETURN(NEWERROR(0, "Copy sizes mismatch (expected %zu, got %zu)",
fi0.fs, fi.fs), "Validation error");
if (0 != memcmp(fi.h, fi0.h, fi.hs))
MAYBERETURN(NEWERROR(0, "Copy hash mismatch"), "Validation error");
GRID_INFO("The original file and its copy match");
/* Get it an other way in a buffer. */
guint8 buf[1024];
struct oio_sds_dl_dst_s dl_dst = {
.type = OIO_DL_DST_BUFFER,
.data = {.buffer = {.ptr = buf, .length=1024}}
};
struct oio_sds_dl_src_s dl_src = {
.url = url,
.ranges = NULL,
};
err = oio_sds_download (client, &dl_src, &dl_dst);
MAYBERETURN(err, "Download error");
GRID_INFO("Content downloaded to a buffer");
/* link the container */
struct oio_url_s *url1 = oio_url_dup (url);
oio_url_set (url1, OIOURL_PATH, tmppath);
err = oio_sds_link (client, url1, content_id);
oio_url_pclean (&url1);
MAYBERETURN(err, "Link error: ");
/* List the container, the content must appear */
struct oio_sds_list_param_s list_in = {
.url = url,
.prefix = NULL, .marker = NULL, .end = NULL, .delimiter = 0,
.flag_allversions = 0, .flag_nodeleted = 0,
};
struct oio_sds_list_listener_s list_out = {
.ctx = NULL,
.on_item = _on_item, .on_prefix = NULL, .on_bound = NULL,
};
err = oio_sds_list (client, &list_in, &list_out);
MAYBERETURN(err, "List error");
/* Remove the content from the content */
err = oio_sds_delete (client, url);
MAYBERETURN(err, "Delete error");
GRID_INFO("Content removed");
/* Check the content is not preset anymore */
has = 0;
err = oio_sds_has (client, url, &has);
if (!err && has) err = (struct oio_error_s*) NEWERROR(0, "content still present");
MAYBERETURN(err, "Check error");
GRID_INFO("Content absent as expected");
g_remove (tmppath);
oio_error_pfree (&err);
return NULL;
}
static struct oio_error_s *
_roundtrip_autocontainer (struct oio_sds_s *client, struct oio_url_s *url,
const char *path)
{
struct file_info_s fi;
GError *err = _checksum_file (path, &fi);
/* compute the autocontainer with the SHA1, consider only the first 17 bits */
struct oio_str_autocontainer_config_s cfg = {
.src_offset = 0, .src_size = 0,
.dst_bits = 17,
};
char tmp[65];
const char *auto_container = oio_str_autocontainer_hash (fi.h, fi.hs, tmp, &cfg);
/* build a new URL with the computed container name */
struct oio_url_s *url_auto = oio_url_dup (url);
oio_url_set (url_auto, OIOURL_USER, auto_container);
err = (GError*) _roundtrip_common (client, url_auto, path);
oio_url_pclean (&url_auto);
return (struct oio_error_s*) err;
}
int
main(int argc, char **argv)
{
oio_log_to_stderr();
oio_sds_default_autocreate = 1;
oio_log_init_level_from_env("G_DEBUG_LEVEL");
if (argc != 2) {
g_printerr ("Usage: %s PATH\n", argv[0]);
return 1;
}
const char *path = argv[1];
struct oio_url_s *url = oio_url_empty ();
oio_url_set (url, OIOURL_NS, g_getenv("OIO_NS"));
oio_url_set (url, OIOURL_ACCOUNT, g_getenv("OIO_ACCOUNT"));
oio_url_set (url, OIOURL_USER, g_getenv("OIO_USER"));
oio_url_set (url, OIOURL_TYPE, g_getenv("OIO_TYPE"));
oio_url_set (url, OIOURL_PATH, g_getenv("OIO_PATH"));
if (!oio_url_has_fq_path(url)) {
g_printerr ("Partial URL [%s]: requires a NS (%s), an ACCOUNT (%s),"
" an USER (%s) and a PATH (%s) (+ optional TYPE: %s)\n",
oio_url_get (url, OIOURL_WHOLE),
oio_url_has (url, OIOURL_NS)?"ok":"missing",
oio_url_has (url, OIOURL_ACCOUNT)?"ok":"missing",
oio_url_has (url, OIOURL_USER)?"ok":"missing",
oio_url_has (url, OIOURL_PATH)?"ok":"missing",
oio_url_has (url, OIOURL_TYPE)?"ok":"missing");
return 3;
}
GRID_INFO("URL valid [%s]", oio_url_get (url, OIOURL_WHOLE));
struct oio_sds_s *client = NULL;
struct oio_error_s *err = NULL;
/* Initiate a client */
err = oio_sds_init (&client, oio_url_get(url, OIOURL_NS));
if (err) {
g_printerr ("Client init error: (%d) %s\n", oio_error_code(err),
oio_error_message(err));
return 4;
}
GRID_INFO("Client ready to [%s]", oio_url_get (url, OIOURL_NS));
err = _roundtrip_common (client, url, path);
if (!err)
err = _roundtrip_autocontainer (client, url, path);
int rc = err != NULL;
oio_error_pfree (&err);
oio_sds_pfree (&client);
oio_url_pclean (&url);
return rc;
}
time_t
oio_ext_real_seconds (void)
{
return oio_ext_real_time () / G_TIME_SPAN_SECOND;
}
static gboolean
manage_service(struct service_info_s *si)
{
GError *error_local;
struct service_info_s *old_si = NULL;
struct service_tag_s *tag_first = NULL;
struct namespace_data_s *ns_data;
gsize key_size;
gchar key[LIMIT_LENGTH_SRVTYPE + STRLEN_ADDRINFO], str_addr[STRLEN_ADDRINFO];
if (!si) {
ERROR("Invalid parameter");
return FALSE;
}
key_size = agent_get_service_key(si, key, sizeof(key));
grid_addrinfo_to_string(&(si->addr), str_addr, sizeof(str_addr));
/*this service must refer to known namespace and service type*/
error_local = NULL;
if (!(ns_data = get_namespace(si->ns_name, &error_local))) {
ERROR("Namespace unavailable for service [ns=%s type=%s addr=%s] : %s",
si->ns_name, si->type, str_addr, gerror_get_message(error_local));
if (error_local)
g_error_free(error_local);
return FALSE;
}
/*Info trace when a service of a new type is used */
if (error_local)
g_error_free(error_local);
if (!conscience_get_srvtype(ns_data->conscience, &error_local, si->type, MODE_STRICT)) {
/*to avoid traces flooding, if the service already exists, no trace is sent */
if (!g_hash_table_lookup(ns_data->local_services, key)
&& !g_hash_table_lookup(ns_data->down_services, key)) {
INFO("New service type discovered [ns=%s type=%s addr=%s]", ns_data->name, si->type, str_addr);
}
}
/*replace MACRO tags by their true values */
if (error_local)
g_clear_error(&error_local);
metautils_srvinfo_ensure_tags (si);
si->score.value = SCORE_UNSET;
si->score.timestamp = oio_ext_real_time() / G_TIME_SPAN_SECOND;
/*then keep the score */
g_hash_table_remove(ns_data->down_services, key);
/* save first lauched tag if still in old si */
old_si = g_hash_table_lookup(ns_data->local_services, key);
if (old_si != NULL) {
tag_first = service_info_get_tag(old_si->tags, NAME_TAGNAME_RAWX_FIRST);
if (tag_first != NULL)
service_tag_set_value_boolean(service_info_ensure_tag(si->tags, NAME_TAGNAME_RAWX_FIRST), tag_first->value.b);
}
service_tag_set_value_boolean(service_info_ensure_tag(si->tags, "tag.up"), TRUE);
g_hash_table_insert(ns_data->local_services, g_strndup(key, key_size), si);
DEBUG("Service registration [ns=%s type=%s addr=%s]", ns_data->name, si->type, str_addr);
if (error_local)
g_error_free(error_local);
return TRUE;
}
