static void _m1policy_update_thread(gpointer data, gpointer p) { gs_error_t *err = NULL; GError *gerr=NULL; gchar **tmp=NULL; container_id_t cid; struct thread_user_data_s *user_data = (struct thread_user_data_s *)p; gchar *base = data; if(!hex2bin(base, cid, 2, &gerr)) { GRID_WARN("Invalid prefix: %d %s",gerr->code, gerr->message); g_clear_error(&gerr); return; } err = _m1policy_update(user_data->action,user_data->ns,TRUE,cid, "meta2", user_data->excludesrv, user_data->checkonly, &tmp); g_mutex_lock(user_data->lock); _pack_result(&(user_data->gpa),base, tmp, err, user_data->cpts); user_data->indexreach ++; g_mutex_unlock(user_data->lock); if ( err ) { gs_error_free(err); err = NULL; } if ( tmp ) g_strfreev (tmp); }
gs_error_t * hc_delete_content(gs_grid_storage_t *hc, struct hc_url_s *url) { gs_error_t *e = NULL; gs_container_t *c = NULL; gs_content_t *content = NULL; c = gs_get_storage_container(hc, hc_url_get(url, HCURL_REFERENCE),NULL, 0, &e); if(NULL != c) { const gchar *version = hc_url_get(url, HCURL_VERSION); // First try content = gs_get_content_from_path_and_version (c, hc_url_get(url, HCURL_PATH), version, &e); if (content == NULL && e != NULL && e->code == CODE_CONTENT_NOTFOUND && !version) { // Last version is probably marked deleted, so a "get" without // version fails. We need to specify we want the latest, even // if it's deleted, so we can undelete it. version = HCURL_LATEST_VERSION; gs_error_free(e); e = NULL; // Second try content = gs_get_content_from_path_and_version (c, hc_url_get(url, HCURL_PATH), version, &e); } if (NULL != content) { if(gs_destroy_content (content, &e)) { GRID_DEBUG("Content %s deleted\n", hc_url_get(url, HCURL_PATH)); } gs_content_free(content); } gs_container_free(c); } return e; }
static int _my_content_filter(gs_content_t * content, void *user_data) { gs_error_t *err = NULL; static gs_content_info_t info; struct list_content_s *lc = (struct list_content_s *)user_data; if (!content) return -1; if (!gs_content_get_info(content, &info, &err)) { g_printerr("cannot read the information about a content (%s)\n", gs_error_get_message(err)); gs_error_free(err); return -1; } if(lc->show_info) { /* load content info from meta2 */ if(!gs_content_reload(content, TRUE, FALSE, &err)) { g_printerr("Failed to get content informations from meta2 : (%s)\n", gs_error_get_message(err)); gs_error_free(err); return -1; } if(lc->xml) _filter_info_xml(content, lc); else _filter_info(content, lc); } else { struct ls_utils_s *lu = g_malloc0(sizeof(struct ls_utils_s)); lu->path = g_strdup(content->info.path); lu->deleted = content->deleted; lu->version = g_ascii_strtoll(content->version, NULL, 10); GString *tmp = g_string_new(""); if (lc->xml) { g_string_append_printf(tmp, " <Content>\n"); g_string_append_printf(tmp, " <Path>%s</Path>\n" " </Content>\n",info.path); } else { g_string_append_printf(tmp, "%s\n", info.path); } lu->str = g_string_free(tmp, FALSE); lc->listed = g_slist_prepend(lc->listed, lu); } return 1; }
static int list_container(gs_grid_storage_t * gs, const char *cName) { int rc = -1; gs_error_t *err = NULL; gs_container_t *container = NULL; if (!gs || !cName) return -1; container = gs_get_storage_container(gs, cName, NULL, 0, &err); if (!container) { PRINT_ERROR("cannot find %s\n", cName); goto exit_label; } else PRINT_DEBUG("%s found\n", cName); /* Dump container properties */ if (flag_prop && !dump_container_properties(container)) return -1; nb_elements = 0; if (!gs_list_container(container, NULL, my_content_filter, NULL, &err)) { PRINT_ERROR("cannot download all the contents of %s\n", cName); goto exit_label; } else PRINT_DEBUG("%s listed\n", cName); g_print("total:%i elements in [grid://%s/%s/]\n", nb_elements, meta0_url, cName); rc = 0; exit_label: if (rc < 0) { if (err) { PRINT_ERROR("Failed to list [%s] cause:\n", cName); PRINT_DEBUG("\t%s\n", gs_error_get_message(err)); gs_error_free(err); } else { PRINT_ERROR("Failed to list [%s]\n", cName); } } gs_container_free(container); return rc; }
gs_error_t * hc_create_container(gs_grid_storage_t *hc, struct hc_url_s *url, const char *stgpol, const char *versioning) { GError *err = NULL; gs_error_t *e = NULL; gs_container_t *c = NULL; struct m2v2_create_params_s params = {stgpol, versioning, FALSE}; c = gs_get_storage_container2(hc, hc_url_get(url, HCURL_REFERENCE), ¶ms, 0, &e); if (c != NULL) { gchar m2[STRLEN_ADDRINFO] = {0}; addr_info_to_string(&(c->meta2_addr), m2, STRLEN_ADDRINFO); err = m2v2_remote_execute_HAS(m2, NULL, url); if (err == NULL) { e = gs_error_new(CODE_CONTAINER_EXISTS, "Failed to create container [%s]: " "container already exists in namespace [%s]\n", hc_url_get(url, HCURL_REFERENCE), hc_url_get(url, HCURL_NS)); goto end_label; } else if (err->code != CODE_CONTAINER_NOTFOUND) { GSERRORCAUSE(&e, err, "Failed to check container existence in meta2: "); goto end_label; } else { GRID_WARN("Container exists in meta1 but not in meta2"); } } gs_error_free(e); e = NULL; c = gs_get_storage_container2(hc, hc_url_get(url, HCURL_REFERENCE), ¶ms, 1, &e); if (c) GRID_INFO("Container [%s] created in namespace [%s].\n\n", hc_url_get(url, HCURL_REFERENCE), hc_url_get(url, HCURL_NS)); end_label: gs_container_free(c); g_clear_error(&err); return e; }
gs_error_t* hcadmin_meta1_policy_update(char *ns,gchar *action, gboolean checkonly, gchar **globalresult, gchar ***result, char ** args) { gs_error_t *err = NULL; GPtrArray *gpa = NULL; gchar *m1pAction = NULL; struct global_cpt_s *cpts = NULL; gchar *excludesrv = NULL; gchar *srvtype = "meta2"; gchar *hexID = NULL; GError *gerr = NULL; container_id_t cid; gchar **tmp = NULL; if ((!args) || ( g_strv_length(args) == 0)) { GSERRORCODE(&err,500, "Invalid argument list\n"); goto failed; } // extract ID hexID = args[0]; // extract specific arg / action type if ( g_strcmp0(action, "meta1_policy_apply") == 0) { m1pAction = g_strdup_printf("UPDATE"); if ( g_strv_length(args) > 1 ) if (strlen(args[1]) > 0) srvtype = args[1]; } else if ( g_strcmp0(action, "meta1_policy_exclude") == 0 ) { m1pAction = g_strdup_printf("EXCLUDE"); if ( g_strv_length(args) > 1 ) excludesrv = args[1]; else { GSERRORCODE(&err, 500, "Missing service url \n"); } } else { GSERRORCODE(&err, 500, "INVALID meta1 policy action %s\n",action); } if (err) goto failed; if ( g_strcmp0(hexID,"ALL") == 0) { guint idx; struct thread_user_data_s *user_data = NULL; GThreadPool *pool = NULL; cpts = g_malloc0(sizeof(struct global_cpt_s)); user_data = g_malloc0(sizeof(struct thread_user_data_s)); user_data->ns = ns; user_data->action = m1pAction; user_data->type = g_strdup_printf(srvtype); user_data->excludesrv = excludesrv; user_data->checkonly = checkonly; user_data->lock = g_mutex_new(); user_data->indexreach=0; user_data->cpts = cpts; user_data->gpa = NULL; pool = g_thread_pool_new (_m1policy_update_thread,user_data,10,TRUE,&gerr); for ( idx=0; idx<65536 ;idx++) { gchar base[5]; guint8 *prefix=(guint8 *)(&idx); g_snprintf(base, sizeof(base), "%02X%02X", prefix[0], prefix[1]); g_thread_pool_push(pool, g_strdup(base) ,&gerr); if ( gerr != NULL ) { GRID_WARN("Failed to push new data thread %d, %d %s",idx,gerr->code,gerr->message); g_clear_error(&gerr); gerr = NULL; } } guint lastindex = 0; while(1) { if ( user_data->indexreach - lastindex > 500) { lastindex = user_data->indexreach; GRID_INFO("%d prefix checked",user_data->indexreach); } if ( user_data->indexreach >= 65536 ) break; usleep(1000000); } if ( excludesrv && excludesrv[0] ) { struct hc_url_s *url = NULL; url = hc_url_empty(); hc_url_set(url,HCURL_NS,ns); m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url); hc_url_clean(url); } g_thread_pool_free(pool, FALSE, TRUE); g_mutex_free(user_data->lock); g_free(user_data); gpa = user_data->gpa; } else { // prefix ?? if ( strlen(hexID) == 4 ) { if(!hex2bin(hexID, cid, 2, &gerr)) { GSERRORCAUSE(&err, gerr, "Invalid prefix\n"); goto failed; } err = _m1policy_update(m1pAction,ns,TRUE,cid, srvtype, excludesrv, checkonly, &tmp); _pack_result(&gpa,hexID,tmp,err,NULL); if ( err ) { gs_error_free(err); err = NULL; } else { if ( excludesrv && excludesrv[0] ) { struct hc_url_s *url = NULL; url = hc_url_empty(); hc_url_set(url,HCURL_NS,ns); m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url); hc_url_clean(url); } } // CID ?? } else if (strlen(hexID) == 64 ) { if (!hex2bin(hexID, cid, sizeof(container_id_t), &gerr)) { GSERRORCAUSE(&err, gerr, "Invalid container_id\n"); goto failed; } err = _m1policy_update(m1pAction,ns,FALSE,cid, srvtype, excludesrv, checkonly, &tmp); _pack_result(&gpa,hexID,tmp,err,NULL); if ( err ) { gs_error_free(err); err = NULL; } else { if ( excludesrv && excludesrv[0] ) { struct hc_url_s *url = NULL; url = hc_url_empty(); hc_url_set(url,HCURL_NS,ns); hc_url_set(url,HCURL_HEXID,hexID); m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url); hc_url_clean(url); } } // other ? --> error } else { GSERRORCODE(&err,500,"invalid ID %s, %d \n",hexID, sizeof(hexID)); } } // free all memory if ( gerr) g_clear_error(&gerr); if ( gpa ) { g_ptr_array_add(gpa, NULL); if ( cpts ) { *globalresult = g_strdup_printf("Global result : References checked %lu , updated %lu , failed %lu",cpts->checkedRef,cpts->updatedRef,cpts->failedRef); } *result = (gchar**)g_ptr_array_free(gpa,FALSE); } if ( tmp ) g_strfreev (tmp); if ( cpts ) g_free(cpts); failed : if (m1pAction) g_free(m1pAction); return err; }
static status_t _get_container_global_property(gs_grid_storage_t *hc, struct metacnx_ctx_s *cnx, container_id_t cid, GHashTable **ht, gs_error_t **gserr) { GSList *prop_list = NULL, *l = NULL; GError *gerr = NULL; // get all properties with current meta2 if (!meta2_remote_list_all_container_properties(cnx, cid, &prop_list, &gerr)) { GSList *m2_list = NULL; addr_info_t *addr = NULL; GSList *m2 = NULL; gchar str_addr[STRLEN_ADDRINFO]; struct metacnx_ctx_s cnxOther; gs_error_t *e = NULL; gboolean bResult = FALSE; // search all meta2 fo current contener m2_list = gs_resolve_meta2(hc, cid, &gerr); if (!m2_list) { GSERRORCAUSE(gserr, gerr, "Failed to get container admin entries, Cannot find meta2(s)"); if (gerr) g_error_free(gerr); return 0; } // test each meta2... for (m2=m2_list; m2 ;m2=m2->next) { addr = m2->data; if (addr) { addr_info_to_string(addr, str_addr, sizeof(str_addr)); DEBUG("Failed to get container admin entries -> test with next meta2 [%s]", str_addr); if (!_open_meta2_connection(&cnxOther, str_addr/*ctx->loc->m2_url[0]*/, &e)) { GSERRORCODE(gserr, e->code, "Failed to open connection to meta2 (%s)\n", str_addr); gs_error_free(e); continue; } if (!meta2_remote_list_all_container_properties(&cnxOther, cid, &prop_list, &gerr)) { _close_meta2_connection(&cnxOther); GSERRORCAUSE(gserr, gerr, "Failed to get container admin entries: %s\n",str_addr); continue; } else { _close_meta2_connection(&cnxOther); // no error bResult = TRUE; break; } } } if (m2_list) { g_slist_foreach(m2_list, addr_info_gclean, NULL); g_slist_free(m2_list); } if (gerr) g_error_free(gerr); if (bResult == FALSE) return 0; } // here: reading properties ok *ht = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); for (l = prop_list; l && l->data; l = l->next) { gchar *tmp = l->data; gchar **tok = g_strsplit(tmp, "=", 2); if (tok[0] && tok[1]) g_hash_table_insert(*ht, g_strdup(tok[0]), g_strdup(tok[1])); g_strfreev(tok); } g_slist_free_full(prop_list, g_free); /* Ensure we have some mandatory properties */ if(!g_hash_table_lookup(*ht, M2V2_ADMIN_STORAGE_POLICY)) g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_STORAGE_POLICY), g_strdup("namespace default")); if(!g_hash_table_lookup(*ht, M2V2_ADMIN_VERSIONING_POLICY)) g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_VERSIONING_POLICY), g_strdup("namespace default")); if(!g_hash_table_lookup(*ht, M2V2_ADMIN_QUOTA)) g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_QUOTA), g_strdup("namespace default")); return 1; }
int main(int argc, char **args) { int rc = ~0; gs_error_t *err = NULL; gs_grid_storage_t *gs = NULL; close(0); if (!parse_opt(argc, args)) { PRINT_ERROR("cannot parse the options, see the help section (--help option)\n"); return ~0; } if (flag_help) { help(); return 0; } if (!conf_check()) { PRINT_ERROR("invalid configuration, see the help section (--help option)\n"); help(); return ~0; } if (0 > chdir(base_dir)) { PRINT_ERROR("Failed to change the working directory : %s\n", strerror(errno)); return ~0; } gs = gs_grid_storage_init(meta0_url, &err); if (gs == NULL) { if (optind<argc){ free(meta0_url); meta0_url=strtok(args[optind],"/"); free(container_name); container_name=strtok(NULL,"/"); if(container_name!=NULL){ gs = gs_grid_storage_init(meta0_url, &err); if(!gs){ PRINT_ERROR("cannot init the GridStorage client\n"); goto error_gs; } } else{ PRINT_ERROR("missing options\n"); help(); return 0; } } else{ PRINT_ERROR("missing options\n"); help(); return 0; } } if (0 > list_container(gs, container_name)) { PRINT_ERROR("cannot list %s in namespace %s\n", container_name, meta0_url); goto error_list; } rc = 0; error_list: gs_grid_storage_free(gs); error_gs: if (err) gs_error_free(err); return rc; }
static int my_content_filter(gs_content_t * content, void *user_data) { char original_path[1024], path[1024], *ptr; int output_fd; gs_error_t *err = NULL; static gs_content_info_t info; static gs_container_info_t container_info; gs_download_info_t dl_info; (void) user_data; if (!content) { PRINT_ERROR("Invalid content received\n"); return -1; } if (!gs_content_get_info(content, &info, &err) || !gs_container_get_info(info.container, &container_info, &err)) { PRINT_ERROR("cannot read the information about a content (%s)\n", gs_error_get_message(err)); gs_error_free(err); return -1; } /*purify the path */ g_strlcpy(original_path, info.path, sizeof(original_path)); for (ptr = original_path; *ptr; ptr++) { register int c; c = *ptr; if (c == '/' || g_ascii_isspace(c)) *ptr = '_'; } PRINT_DEBUG("About to download [grid://%s/%s/%s] (%"G_GINT64_FORMAT" bytes) in [%s/%s]\n", meta0_url, container_info.name, info.path, info.size, base_dir, original_path); /*open the destination file */ g_snprintf(path, sizeof(path), "%s/%s", base_dir, original_path); output_fd = open(path, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); if (output_fd < 0) { PRINT_ERROR("Failed to open [%s] as the download target : %s\n", path, strerror(errno)); return -1; } /*download the content */ memset(&dl_info, 0x00, sizeof(dl_info)); dl_info.offset = 0; dl_info.size = 0; dl_info.writer = write_to_fd; dl_info.user_data = &output_fd; if (GS_OK != gs_download_content(content, &dl_info, &err)) { close(output_fd); unlink(path); if (flag_verbose) { PRINT_ERROR("grid storage error: cannot download [grid://%s/%s/%s] in [%s]. Cause:\n\t%s\r\n", meta0_url, container_name, info.path, path, gs_error_get_message(err)); } else { PRINT_ERROR("grid storage error: cannot download [grid://%s/%s/%s] in [%s]\n", meta0_url, container_name, info.path, path); } return -1; } /*then adjust the file rights and reply the success */ fchmod(output_fd, S_IRGRP | S_IRUSR | S_IWUSR); close(output_fd); g_print("download: [grid://%s/%s/%s] in [%s] (%"G_GINT64_FORMAT" bytes)\n", meta0_url, container_name, info.path, path, info.size); nb_elements++; /* Save content properties */ if (flag_prop && !dump_content_properties(content)) return -1; return 0; }
int main (int argc, char ** args) { int rc = -1; gs_error_t *err = NULL; gs_grid_storage_t *hc; gs_container_t *container; char *ns; char cname[60]; srand(time(NULL)); if (argc != 2) { g_error("Usage: %s NS\n", args[0]); return rc; } ns = args[1]; bzero(cname, sizeof(cname)); g_snprintf(cname, sizeof(cname), "SOLR%d", rand()); printf("Working with container [%s]\n", cname); hc = gs_grid_storage_init( ns, &err ); if(!hc) { printf("failed to init hc\n"); return rc; } container = gs_get_container(hc, cname, 1, &err); if(!container) { printf("Failed to resolve container\n"); goto end_label; } gs_service_t **srv_array = NULL; srv_array = gs_container_service_get_available(container, "meta0", &err); char url[256]; bzero(url, sizeof(url)); gs_service_get_url(srv_array[0], url, sizeof(url)); printf("New service linked\n"); printf("service url = [%s]\n", url); if(srv_array) gs_service_free_array(srv_array); srv_array = gs_container_service_get_all(container, "meta0", &err); bzero(url, sizeof(url)); gs_service_get_url(srv_array[0], url, sizeof(url)); printf("Already linked service :\n"); printf("service url = [%s]\n", url); if(srv_array) gs_service_free_array(srv_array); end_label: if(container) { gs_container_free(container); container = NULL; } if(hc) { gs_grid_storage_free(hc); hc = NULL; } if(err) { gs_error_free(err); err= NULL; } }
static void _gs_error_clear(gs_error_t ** e) { gs_error_free(*e); *e = NULL; }