GPtrArray* meta0_utils_list_to_array(GSList *list) { GSList *l; GPtrArray *result = NULL; EXTRA_ASSERT(list != NULL); result = meta0_utils_array_create(); for (l=list; l ;l=l->next) { guint16 *p, *max; struct meta0_info_s *m0i; if (!(m0i = l->data)) continue; gchar url[STRLEN_ADDRINFO]; grid_addrinfo_to_string(&(m0i->addr), url, sizeof(url)); p = (guint16*) m0i->prefixes; max = (guint16*) (m0i->prefixes + m0i->prefixes_size); for (; p<max; p++) meta0_utils_array_add(result, (guint8*)p, url); } return result; }
GTree* meta0_utils_list_to_tree(GSList *list) { GSList *l; GTree *result = NULL; EXTRA_ASSERT(list != NULL); result = g_tree_new_full( hashstr_quick_cmpdata, NULL, g_free, (GDestroyNotify)garray_free); for (l=list; l ;l=l->next) { struct meta0_info_s *m0i; if (!(m0i = l->data)) continue; gchar url[STRLEN_ADDRINFO]; grid_addrinfo_to_string(&(m0i->addr), url, sizeof(url)); gsize len = m0i->prefixes_size; len = len / 2; GArray *pfx = g_array_new(FALSE, FALSE, sizeof(guint16)); g_array_append_vals(pfx, m0i->prefixes, len); g_tree_replace(result, hashstr_create(url), pfx); } return result; }
// TODO: factorize the two following functions enum http_rc_e action_admin_meta0_list(struct req_args_s *args) { GError *err = NULL; GSList *m0_lst = NULL; GSList *m1_lst = NULL; GString *json = NULL; err = conscience_get_services(NS(), NAME_SRVTYPE_META0, FALSE, &m0_lst); if (!err) { for (GSList *l = m0_lst; l; l = l->next) { g_clear_error(&err); service_info_t *m0 = l->data; gchar m0_url[STRLEN_ADDRINFO] = {0}; grid_addrinfo_to_string(&(m0->addr), m0_url, sizeof(m0_url)); err = meta0_remote_get_meta1_all(m0_url, &m1_lst); if (!err || !CODE_IS_NETWORK_ERROR(err->code)) break; } g_slist_free_full(m0_lst, (GDestroyNotify)service_info_clean); } if (m1_lst) { json = _m0_mapping_from_m1_list(m1_lst); meta0_utils_list_clean(m1_lst); return _reply_json(args, HTTP_CODE_OK, "OK", json); } return _reply_common_error(args, err); }
enum http_rc_e action_admin_meta0_force(struct req_args_s *args) { GError *err = NULL; GSList *m0_lst = NULL; err = conscience_get_services(NS(), NAME_SRVTYPE_META0, FALSE, &m0_lst); if (!err) { for (GSList *l = m0_lst; l; l = l->next) { g_clear_error(&err); service_info_t *m0 = l->data; gchar m0_url[STRLEN_ADDRINFO] = {0}; grid_addrinfo_to_string(&(m0->addr), m0_url, sizeof(m0_url)); err = meta0_remote_force(m0_url, (gchar*) args->rq->body->data); if (!err) { err = meta0_remote_cache_refresh(m0_url); break; } else if (!CODE_IS_NETWORK_ERROR(err->code)) { break; } } g_slist_free_full(m0_lst, (GDestroyNotify)service_info_clean); } if (err) return _reply_common_error(args, err); return _reply_nocontent(args); }
gchar * service_info_key (const struct service_info_s *si) { gchar addr[STRLEN_ADDRINFO]; const char *explicit = service_info_get_tag_value(si, "tag.id", NULL); if (explicit) return oio_make_service_key(si->ns_name, si->type, explicit); grid_addrinfo_to_string(&si->addr, addr, sizeof(struct addr_info_s)); return oio_make_service_key(si->ns_name, si->type, addr); }
gsize agent_get_service_key(struct service_info_s *si, gchar * dst, gsize dst_size) { gsize writen; writen = 0; writen += g_snprintf(dst + writen, dst_size - writen, "%s:", si->type); writen += grid_addrinfo_to_string(&(si->addr), dst + writen, dst_size - writen); return writen; }
GError * gcluster_push_services(addr_info_t * addr, long timeout, GSList *ls) { EXTRA_ASSERT (addr != NULL); gchar target[STRLEN_ADDRINFO]; grid_addrinfo_to_string (addr, target, sizeof(target)); MESSAGE req = metautils_message_create_named (NAME_MSGNAME_CS_PUSH_SRV); metautils_message_add_body_unref (req, service_info_marshall_gba (ls, NULL)); return gridd_client_exec (target, ((gdouble)timeout)/1000.0, message_marshall_gba_and_clean(req)); }
static void _dump_addr (const char *s) { gchar str[256], hexa[1024]; struct addr_info_s addr; if (grid_string_to_addrinfo(s, &addr)) { memset(str, 0, sizeof(str)); grid_addrinfo_to_string(&addr, str, sizeof(str)); memset(hexa, 0, sizeof(hexa)); oio_str_bin2hex(&addr, sizeof(addr), hexa, sizeof(hexa)); g_print("%s %s\n", str, hexa); } }
// TODO: export? static gpointer _gen_chunk_bean(struct service_info_s *si) { gchar straddr[STRLEN_ADDRINFO], strid[STRLEN_CHUNKID]; gchar *chunkid = NULL; struct bean_CHUNKS_s *chunk = NULL; grid_addrinfo_to_string(&(si->addr), straddr, sizeof(straddr)); SHA256_randomized_string(strid, sizeof(strid)); chunk = _bean_create(&descr_struct_CHUNKS); chunkid = m2v2_build_chunk_url (straddr, strid); CHUNKS_set2_id(chunk, chunkid); g_free(chunkid); return (gpointer)chunk; }
GSList * gcluster_get_service_types(addr_info_t *addr, long timeout, GError ** error) { EXTRA_ASSERT (addr != NULL); gchar target[STRLEN_ADDRINFO]; grid_addrinfo_to_string (addr, target, sizeof(target)); GSList *out = NULL; MESSAGE req = metautils_message_create_named (NAME_MSGNAME_CS_GET_SRVNAMES); GError *err = gridd_client_exec_and_decode (target, ((gdouble)timeout)/1000.0, message_marshall_gba_and_clean(req), &out, strings_unmarshall); if (err) { g_prefix_error (&err, "Remote: "); g_error_transmit(error, err); } return out; }
static struct meta1_service_url_s* _siv_to_url(struct service_info_s **siv) { struct meta1_service_url_s *m1u = g_malloc0 (sizeof(*m1u)); GString *u = g_string_new(""); for (struct service_info_s **p=siv; *p ; p++) { gchar str[STRLEN_ADDRINFO]; grid_addrinfo_to_string(&((*p)->addr), str, sizeof(str)); if (u->len > 0) g_string_append_c(u, ','); g_string_append(u, str); } g_strlcpy (m1u->host, u->str, sizeof(m1u->host)); g_string_free(u, TRUE); return m1u; }
void service_info_encode_json(GString *gstr, const struct service_info_s *si, gboolean full) { if (!si) return; gchar straddr[STRLEN_ADDRINFO]; grid_addrinfo_to_string(&(si->addr), straddr, sizeof(straddr)); if (full) { g_string_append_printf(gstr, "{\"ns\":\"%s\",\"type\":\"%s\",\"addr\":\"%s\",\"score\":%d,\"tags\":{", si->ns_name, si->type, straddr, si->score.value); } else { g_string_append_printf(gstr, "{\"addr\":\"%s\",\"score\":%d,\"tags\":{", straddr, si->score.value); } _append_all_tags(gstr, si->tags); g_string_append(gstr, "}}"); }
static GError* _client_connect_addr(struct gridd_client_s *client, const struct addr_info_s *ai) { EXTRA_ASSERT(client != NULL); EXTRA_ASSERT(client->abstract.vtable == &VTABLE_CLIENT); if (NULL == ai || !ai->port) return NEWERROR(CODE_INTERNAL_ERROR, "Invalid parameter"); g_assert(client != NULL); _client_reset_cnx(client); _client_reset_target(client); _client_reset_reply(client); grid_addrinfo_to_string(ai, client->orig_url, URL_MAXLEN); memcpy(client->url, client->orig_url, URL_MAXLEN); client->step = NONE; return NULL; }
struct conscience_srv_s * conscience_srvtype_register_srv(struct conscience_srvtype_s *srvtype, GError ** err, const struct conscience_srvid_s *srvid) { gsize desc_size; struct conscience_srv_s *service; if (!srvtype || !srvid) { GSETCODE(err, CODE_INTERNAL_ERROR, "Invalid parameter"); return NULL; } service = g_malloc0(sizeof(struct conscience_srv_s)); memcpy(&(service->id), srvid, sizeof(struct conscience_srvid_s)); service->tags = g_ptr_array_new(); service->locked = FALSE; service->score.timestamp = 0; service->score.value = -1; service->srvtype = srvtype; /*build the service description once for all*/ desc_size = g_snprintf(service->description,sizeof(service->description),"%s/%s/", srvtype->conscience->ns_info.name, srvtype->type_name); grid_addrinfo_to_string(&(service->id.addr), service->description+desc_size,sizeof(service->description)-desc_size); /*register the service with its ID*/ g_hash_table_insert(srvtype->services_ht, &(service->id), service); /*ring insertion */ srvtype->services_ring.next->prev = service; service->prev = &(srvtype->services_ring); service->next = srvtype->services_ring.next; srvtype->services_ring.next = service; return service; }
static GError* _cache_load_from_m0(struct meta1_prefixes_set_s *m1ps, const gchar *ns_name, const struct addr_info_s *local_addr, struct addr_info_s *m0_addr, GArray **updated_prefixes, gboolean *meta0_ok) { GError *err = NULL; GSList *m0info_list = NULL; EXTRA_ASSERT(m1ps != NULL); GRID_TRACE2("%s(%p,%s,%p,%p)", __FUNCTION__, m1ps, ns_name, local_addr, m0_addr); (void)ns_name; gchar m0[STRLEN_ADDRINFO]; grid_addrinfo_to_string (m0_addr, m0, sizeof(m0)); err = meta0_remote_get_meta1_all(m0, &m0info_list); if (err) { g_prefix_error(&err, "Remote error: "); return err; } if (!m0info_list) { GRID_DEBUG("META0 has no prefix configured!"); return NULL; } *meta0_ok = TRUE; guint8 *cache = _cache_from_m0l(m0info_list, local_addr); GPtrArray *by_prefix = meta0_utils_list_to_array(m0info_list); g_mutex_lock(&m1ps->lock); GRID_DEBUG("Got %u prefixes from M0, %u in place", by_prefix->len, m1ps->by_prefix ? m1ps->by_prefix->len : 0); if ( m1ps->by_prefix ) { guint prefix; *updated_prefixes = g_array_new(FALSE, FALSE, sizeof(guint16)); for( prefix=0 ; prefix <65536 ;prefix++) { if ( _cache_is_managed(m1ps->cache,(guint8 *)&prefix) != _cache_is_managed( cache,(guint8 *)&prefix)) { g_array_append_vals(*updated_prefixes, &prefix, 1); } } } SWAP_PTR(m1ps->by_prefix, by_prefix); SWAP_PTR(m1ps->cache, cache); g_mutex_unlock(&m1ps->lock); if (by_prefix) meta0_utils_array_clean(by_prefix); by_prefix = NULL; if (cache) g_free(cache); cache = NULL; g_slist_foreach(m0info_list, meta0_info_gclean, NULL); g_slist_free(m0info_list); return NULL; }
static gboolean manage_service(struct service_info_s *si) { GError *error_local; struct service_info_s *old_si = NULL; struct service_tag_s *tag_first = NULL; struct namespace_data_s *ns_data; gsize key_size; gchar key[LIMIT_LENGTH_SRVTYPE + STRLEN_ADDRINFO], str_addr[STRLEN_ADDRINFO]; if (!si) { ERROR("Invalid parameter"); return FALSE; } key_size = agent_get_service_key(si, key, sizeof(key)); grid_addrinfo_to_string(&(si->addr), str_addr, sizeof(str_addr)); /*this service must refer to known namespace and service type*/ error_local = NULL; if (!(ns_data = get_namespace(si->ns_name, &error_local))) { ERROR("Namespace unavailable for service [ns=%s type=%s addr=%s] : %s", si->ns_name, si->type, str_addr, gerror_get_message(error_local)); if (error_local) g_error_free(error_local); return FALSE; } /*Info trace when a service of a new type is used */ if (error_local) g_error_free(error_local); if (!conscience_get_srvtype(ns_data->conscience, &error_local, si->type, MODE_STRICT)) { /*to avoid traces flooding, if the service already exists, no trace is sent */ if (!g_hash_table_lookup(ns_data->local_services, key) && !g_hash_table_lookup(ns_data->down_services, key)) { INFO("New service type discovered [ns=%s type=%s addr=%s]", ns_data->name, si->type, str_addr); } } /*replace MACRO tags by their true values */ if (error_local) g_clear_error(&error_local); metautils_srvinfo_ensure_tags (si); si->score.value = SCORE_UNSET; si->score.timestamp = oio_ext_real_time() / G_TIME_SPAN_SECOND; /*then keep the score */ g_hash_table_remove(ns_data->down_services, key); /* save first lauched tag if still in old si */ old_si = g_hash_table_lookup(ns_data->local_services, key); if (old_si != NULL) { tag_first = service_info_get_tag(old_si->tags, NAME_TAGNAME_RAWX_FIRST); if (tag_first != NULL) service_tag_set_value_boolean(service_info_ensure_tag(si->tags, NAME_TAGNAME_RAWX_FIRST), tag_first->value.b); } service_tag_set_value_boolean(service_info_ensure_tag(si->tags, "tag.up"), TRUE); g_hash_table_insert(ns_data->local_services, g_strndup(key, key_size), si); DEBUG("Service registration [ns=%s type=%s addr=%s]", ns_data->name, si->type, str_addr); if (error_local) g_error_free(error_local); return TRUE; }
static GError * _init_assign(gchar *ns_name, GList **working_m1list,GSList **unref_m1list) { GError *error = NULL; GSList *m1_list = NULL; m1_list = list_namespace_services(ns_name, "meta1", &error); if (!m1_list) { if ( error) { GRID_ERROR("failed to init meta1 service list :(%d) %s", error->code, error->message); goto errorLabel; } } GRID_INFO("nb m1 cs %d",g_slist_length(m1_list)); if ( context->replica > g_slist_length(m1_list)) { GRID_ERROR("Number of meta1 services [%d] less than number of replication [%d]",g_slist_length(m1_list),context->replica); error = NEWERROR(EINVAL, "Number of meta1 services [%d] less than number of replication [%d]",g_slist_length(m1_list),context->replica); goto errorLabel; } if ( context->replica <= 0 ) { GRID_ERROR("Invalid replica number [%d]",context->replica); error = NEWERROR(EINVAL, "Invalid replica number [%d]",context->replica); goto errorLabel; } // Duplicate the current prefix distribution and build a List GSList *prefixByMeta1 = meta0_utils_array_to_list(context->array_meta1_by_prefix); GSList *l=NULL; for (;m1_list;m1_list=m1_list->next) { struct meta0_assign_meta1_s *aM1; struct service_info_s *sInfo; gchar url[128]; url[0] = '\0'; aM1 = g_malloc0(sizeof(struct meta0_assign_meta1_s)); sInfo=m1_list->data; grid_addrinfo_to_string(&(sInfo->addr), url, sizeof(url)); aM1->addr=g_strdup(url); aM1->score=0; aM1->available=FALSE; aM1->used=TRUE; l = prefixByMeta1; for (;l;l=l->next) { struct meta0_info_s *m0info; if (!(m0info = l->data)) continue; if (addr_info_equal(&(m0info->addr),&(sInfo->addr))) { guint16 *p, *max; guint i=0; GArray *pfx = g_array_new(FALSE, FALSE, 2); p = (guint16*) m0info->prefixes; max = (guint16*) (m0info->prefixes + m0info->prefixes_size); for (; p<max; p++) { i++; pfx=g_array_append_vals(pfx,(guint8*)p,1); } aM1->assignPrefixes=pfx; aM1->score=i; GRID_DEBUG("aM1 %s , score %d",aM1->addr,aM1->score); prefixByMeta1=g_slist_remove(prefixByMeta1,m0info); meta0_info_clean(m0info); break; } } struct meta0_assign_meta1_s *m1ref = g_hash_table_lookup(context->map_meta1_ref,aM1->addr); if ( m1ref && !m1ref->used) { // unref meta1 aM1->used=FALSE; if (aM1->score != 0 ) { // meta1 refer always prefixe *unref_m1list=g_slist_prepend(*unref_m1list,aM1); } } else { *working_m1list = g_list_prepend(*working_m1list,aM1); } g_hash_table_insert(context->working_map_meta1_ref,strdup(aM1->addr),aM1); } GRID_TRACE("len working %d, len reste pref %d",g_list_length(*working_m1list),g_slist_length(prefixByMeta1)); guint nb_M1 = g_list_length(*working_m1list) + g_slist_length(prefixByMeta1); //defined the average assign score if (nb_M1 == 0 ) { GRID_ERROR("No Meta1 available"); error = NEWERROR(0, "No Meta1 service available"); goto errorLabel; } context->avgscore = (65536* context->replica)/nb_M1; GRID_DEBUG("average meta1 score %d",context->avgscore); GList *work = g_list_first(*working_m1list); for (;work;work=work->next) { struct meta0_assign_meta1_s *aM1 = work->data; if ( aM1->score > context->avgscore) { aM1->available=TRUE; } } GRID_DEBUG("init meta1 list, find %d meta1",g_list_length(*working_m1list)); GRID_DEBUG("init unref meta1 list, find %d meta1",g_slist_length(*unref_m1list)); meta0_utils_list_clean(prefixByMeta1); errorLabel : if (m1_list) { g_slist_foreach(m1_list, service_info_gclean, NULL); g_slist_free(m1_list); } return error; }