static void
_m1policy_update_thread(gpointer data, gpointer p)
{
gs_error_t *err = NULL;
GError *gerr=NULL;
gchar **tmp=NULL;
container_id_t cid;
struct thread_user_data_s *user_data = (struct thread_user_data_s *)p;
gchar *base = data;
if(!hex2bin(base, cid, 2, &gerr)) {
GRID_WARN("Invalid prefix: %d %s",gerr->code, gerr->message);
g_clear_error(&gerr);
return;
}
err = _m1policy_update(user_data->action,user_data->ns,TRUE,cid, "meta2", user_data->excludesrv, user_data->checkonly, &tmp);
g_mutex_lock(user_data->lock);
_pack_result(&(user_data->gpa),base, tmp, err, user_data->cpts);
user_data->indexreach ++;
g_mutex_unlock(user_data->lock);
if ( err ) {
gs_error_free(err);
err = NULL;
}
if ( tmp )
g_strfreev (tmp);
}
gs_error_t *
hc_delete_content(gs_grid_storage_t *hc, struct hc_url_s *url)
{
gs_error_t *e = NULL;
gs_container_t *c = NULL;
gs_content_t *content = NULL;
c = gs_get_storage_container(hc, hc_url_get(url, HCURL_REFERENCE),NULL, 0, &e);
if(NULL != c) {
const gchar *version = hc_url_get(url, HCURL_VERSION);
// First try
content = gs_get_content_from_path_and_version (c, hc_url_get(url, HCURL_PATH), version, &e);
if (content == NULL && e != NULL && e->code == CODE_CONTENT_NOTFOUND && !version) {
// Last version is probably marked deleted, so a "get" without
// version fails. We need to specify we want the latest, even
// if it's deleted, so we can undelete it.
version = HCURL_LATEST_VERSION;
gs_error_free(e);
e = NULL;
// Second try
content = gs_get_content_from_path_and_version (c, hc_url_get(url, HCURL_PATH), version, &e);
}
if (NULL != content) {
if(gs_destroy_content (content, &e)) {
GRID_DEBUG("Content %s deleted\n", hc_url_get(url, HCURL_PATH));
}
gs_content_free(content);
}
gs_container_free(c);
}
return e;
}
static int
_my_content_filter(gs_content_t * content, void *user_data)
{
gs_error_t *err = NULL;
static gs_content_info_t info;
struct list_content_s *lc = (struct list_content_s *)user_data;
if (!content)
return -1;
if (!gs_content_get_info(content, &info, &err)) {
g_printerr("cannot read the information about a content (%s)\n", gs_error_get_message(err));
gs_error_free(err);
return -1;
}
if(lc->show_info) {
/* load content info from meta2 */
if(!gs_content_reload(content, TRUE, FALSE, &err)) {
g_printerr("Failed to get content informations from meta2 : (%s)\n", gs_error_get_message(err));
gs_error_free(err);
return -1;
}
if(lc->xml)
_filter_info_xml(content, lc);
else
_filter_info(content, lc);
} else {
struct ls_utils_s *lu = g_malloc0(sizeof(struct ls_utils_s));
lu->path = g_strdup(content->info.path);
lu->deleted = content->deleted;
lu->version = g_ascii_strtoll(content->version, NULL, 10);
GString *tmp = g_string_new("");
if (lc->xml) {
g_string_append_printf(tmp, " <Content>\n");
g_string_append_printf(tmp, " <Path>%s</Path>\n"
" </Content>\n",info.path);
} else {
g_string_append_printf(tmp, "%s\n", info.path);
}
lu->str = g_string_free(tmp, FALSE);
lc->listed = g_slist_prepend(lc->listed, lu);
}
return 1;
}
static int
list_container(gs_grid_storage_t * gs, const char *cName)
{
int rc = -1;
gs_error_t *err = NULL;
gs_container_t *container = NULL;
if (!gs || !cName)
return -1;
container = gs_get_storage_container(gs, cName, NULL, 0, &err);
if (!container) {
PRINT_ERROR("cannot find %s\n", cName);
goto exit_label;
}
else
PRINT_DEBUG("%s found\n", cName);
/* Dump container properties */
if (flag_prop && !dump_container_properties(container))
return -1;
nb_elements = 0;
if (!gs_list_container(container, NULL, my_content_filter, NULL, &err)) {
PRINT_ERROR("cannot download all the contents of %s\n", cName);
goto exit_label;
}
else
PRINT_DEBUG("%s listed\n", cName);
g_print("total:%i elements in [grid://%s/%s/]\n", nb_elements, meta0_url, cName);
rc = 0;
exit_label:
if (rc < 0) {
if (err) {
PRINT_ERROR("Failed to list [%s] cause:\n", cName);
PRINT_DEBUG("\t%s\n", gs_error_get_message(err));
gs_error_free(err);
}
else {
PRINT_ERROR("Failed to list [%s]\n", cName);
}
}
gs_container_free(container);
return rc;
}
gs_error_t *
hc_create_container(gs_grid_storage_t *hc, struct hc_url_s *url,
const char *stgpol, const char *versioning)
{
GError *err = NULL;
gs_error_t *e = NULL;
gs_container_t *c = NULL;
struct m2v2_create_params_s params = {stgpol, versioning, FALSE};
c = gs_get_storage_container2(hc, hc_url_get(url, HCURL_REFERENCE),
¶ms, 0, &e);
if (c != NULL) {
gchar m2[STRLEN_ADDRINFO] = {0};
addr_info_to_string(&(c->meta2_addr), m2, STRLEN_ADDRINFO);
err = m2v2_remote_execute_HAS(m2, NULL, url);
if (err == NULL) {
e = gs_error_new(CODE_CONTAINER_EXISTS,
"Failed to create container [%s]: "
"container already exists in namespace [%s]\n",
hc_url_get(url, HCURL_REFERENCE), hc_url_get(url, HCURL_NS));
goto end_label;
} else if (err->code != CODE_CONTAINER_NOTFOUND) {
GSERRORCAUSE(&e, err,
"Failed to check container existence in meta2: ");
goto end_label;
} else {
GRID_WARN("Container exists in meta1 but not in meta2");
}
}
gs_error_free(e);
e = NULL;
c = gs_get_storage_container2(hc, hc_url_get(url, HCURL_REFERENCE),
¶ms, 1, &e);
if (c)
GRID_INFO("Container [%s] created in namespace [%s].\n\n",
hc_url_get(url, HCURL_REFERENCE), hc_url_get(url, HCURL_NS));
end_label:
gs_container_free(c);
g_clear_error(&err);
return e;
}
gs_error_t*
hcadmin_meta1_policy_update(char *ns,gchar *action, gboolean checkonly, gchar **globalresult, gchar ***result, char ** args)
{
gs_error_t *err = NULL;
GPtrArray *gpa = NULL;
gchar *m1pAction = NULL;
struct global_cpt_s *cpts = NULL;
gchar *excludesrv = NULL;
gchar *srvtype = "meta2";
gchar *hexID = NULL;
GError *gerr = NULL;
container_id_t cid;
gchar **tmp = NULL;
if ((!args) || ( g_strv_length(args) == 0)) {
GSERRORCODE(&err,500, "Invalid argument list\n");
goto failed;
}
// extract ID
hexID = args[0];
// extract specific arg / action type
if ( g_strcmp0(action, "meta1_policy_apply") == 0) {
m1pAction = g_strdup_printf("UPDATE");
if ( g_strv_length(args) > 1 )
if (strlen(args[1]) > 0)
srvtype = args[1];
} else if ( g_strcmp0(action, "meta1_policy_exclude") == 0 ) {
m1pAction = g_strdup_printf("EXCLUDE");
if ( g_strv_length(args) > 1 )
excludesrv = args[1];
else {
GSERRORCODE(&err, 500, "Missing service url \n");
}
} else {
GSERRORCODE(&err, 500, "INVALID meta1 policy action %s\n",action);
}
if (err)
goto failed;
if ( g_strcmp0(hexID,"ALL") == 0) {
guint idx;
struct thread_user_data_s *user_data = NULL;
GThreadPool *pool = NULL;
cpts = g_malloc0(sizeof(struct global_cpt_s));
user_data = g_malloc0(sizeof(struct thread_user_data_s));
user_data->ns = ns;
user_data->action = m1pAction;
user_data->type = g_strdup_printf(srvtype);
user_data->excludesrv = excludesrv;
user_data->checkonly = checkonly;
user_data->lock = g_mutex_new();
user_data->indexreach=0;
user_data->cpts = cpts;
user_data->gpa = NULL;
pool = g_thread_pool_new (_m1policy_update_thread,user_data,10,TRUE,&gerr);
for ( idx=0; idx<65536 ;idx++) {
gchar base[5];
guint8 *prefix=(guint8 *)(&idx);
g_snprintf(base, sizeof(base), "%02X%02X",
prefix[0], prefix[1]);
g_thread_pool_push(pool, g_strdup(base) ,&gerr);
if ( gerr != NULL ) {
GRID_WARN("Failed to push new data thread %d, %d %s",idx,gerr->code,gerr->message);
g_clear_error(&gerr);
gerr = NULL;
}
}
guint lastindex = 0;
while(1) {
if ( user_data->indexreach - lastindex > 500) {
lastindex = user_data->indexreach;
GRID_INFO("%d prefix checked",user_data->indexreach);
}
if ( user_data->indexreach >= 65536 )
break;
usleep(1000000);
}
if ( excludesrv && excludesrv[0] ) {
struct hc_url_s *url = NULL;
url = hc_url_empty();
hc_url_set(url,HCURL_NS,ns);
m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url);
hc_url_clean(url);
}
g_thread_pool_free(pool, FALSE, TRUE);
g_mutex_free(user_data->lock);
g_free(user_data);
gpa = user_data->gpa;
} else {
// prefix ??
if ( strlen(hexID) == 4 ) {
if(!hex2bin(hexID, cid, 2, &gerr)) {
GSERRORCAUSE(&err, gerr, "Invalid prefix\n");
goto failed;
}
err = _m1policy_update(m1pAction,ns,TRUE,cid, srvtype, excludesrv, checkonly, &tmp);
_pack_result(&gpa,hexID,tmp,err,NULL);
if ( err ) {
gs_error_free(err);
err = NULL;
} else {
if ( excludesrv && excludesrv[0] ) {
struct hc_url_s *url = NULL;
url = hc_url_empty();
hc_url_set(url,HCURL_NS,ns);
m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url);
hc_url_clean(url);
}
}
// CID ??
} else if (strlen(hexID) == 64 ) {
if (!hex2bin(hexID, cid, sizeof(container_id_t), &gerr)) {
GSERRORCAUSE(&err, gerr, "Invalid container_id\n");
goto failed;
}
err = _m1policy_update(m1pAction,ns,FALSE,cid, srvtype, excludesrv, checkonly, &tmp);
_pack_result(&gpa,hexID,tmp,err,NULL);
if ( err ) {
gs_error_free(err);
err = NULL;
} else {
if ( excludesrv && excludesrv[0] ) {
struct hc_url_s *url = NULL;
url = hc_url_empty();
hc_url_set(url,HCURL_NS,ns);
hc_url_set(url,HCURL_HEXID,hexID);
m2v2_remote_execute_EXITELECTION(excludesrv,NULL,url);
hc_url_clean(url);
}
}
// other ? --> error
} else {
GSERRORCODE(&err,500,"invalid ID %s, %d \n",hexID, sizeof(hexID));
}
}
// free all memory
if ( gerr)
g_clear_error(&gerr);
if ( gpa ) {
g_ptr_array_add(gpa, NULL);
if ( cpts ) {
*globalresult = g_strdup_printf("Global result : References checked %lu , updated %lu , failed %lu",cpts->checkedRef,cpts->updatedRef,cpts->failedRef);
}
*result = (gchar**)g_ptr_array_free(gpa,FALSE);
}
if ( tmp )
g_strfreev (tmp);
if ( cpts )
g_free(cpts);
failed :
if (m1pAction)
g_free(m1pAction);
return err;
}
static status_t
_get_container_global_property(gs_grid_storage_t *hc, struct metacnx_ctx_s *cnx, container_id_t cid,
GHashTable **ht, gs_error_t **gserr)
{
GSList *prop_list = NULL, *l = NULL;
GError *gerr = NULL;
// get all properties with current meta2
if (!meta2_remote_list_all_container_properties(cnx, cid, &prop_list, &gerr)) {
GSList *m2_list = NULL;
addr_info_t *addr = NULL;
GSList *m2 = NULL;
gchar str_addr[STRLEN_ADDRINFO];
struct metacnx_ctx_s cnxOther;
gs_error_t *e = NULL;
gboolean bResult = FALSE;
// search all meta2 fo current contener
m2_list = gs_resolve_meta2(hc, cid, &gerr);
if (!m2_list) {
GSERRORCAUSE(gserr, gerr, "Failed to get container admin entries, Cannot find meta2(s)");
if (gerr)
g_error_free(gerr);
return 0;
}
// test each meta2...
for (m2=m2_list; m2 ;m2=m2->next) {
addr = m2->data;
if (addr) {
addr_info_to_string(addr, str_addr, sizeof(str_addr));
DEBUG("Failed to get container admin entries -> test with next meta2 [%s]", str_addr);
if (!_open_meta2_connection(&cnxOther, str_addr/*ctx->loc->m2_url[0]*/, &e)) {
GSERRORCODE(gserr, e->code,
"Failed to open connection to meta2 (%s)\n", str_addr);
gs_error_free(e);
continue;
}
if (!meta2_remote_list_all_container_properties(&cnxOther, cid, &prop_list, &gerr)) {
_close_meta2_connection(&cnxOther);
GSERRORCAUSE(gserr, gerr, "Failed to get container admin entries: %s\n",str_addr);
continue;
} else {
_close_meta2_connection(&cnxOther);
// no error
bResult = TRUE;
break;
}
}
}
if (m2_list) {
g_slist_foreach(m2_list, addr_info_gclean, NULL);
g_slist_free(m2_list);
}
if (gerr)
g_error_free(gerr);
if (bResult == FALSE)
return 0;
}
// here: reading properties ok
*ht = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
for (l = prop_list; l && l->data; l = l->next) {
gchar *tmp = l->data;
gchar **tok = g_strsplit(tmp, "=", 2);
if (tok[0] && tok[1])
g_hash_table_insert(*ht, g_strdup(tok[0]), g_strdup(tok[1]));
g_strfreev(tok);
}
g_slist_free_full(prop_list, g_free);
/* Ensure we have some mandatory properties */
if(!g_hash_table_lookup(*ht, M2V2_ADMIN_STORAGE_POLICY))
g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_STORAGE_POLICY), g_strdup("namespace default"));
if(!g_hash_table_lookup(*ht, M2V2_ADMIN_VERSIONING_POLICY))
g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_VERSIONING_POLICY), g_strdup("namespace default"));
if(!g_hash_table_lookup(*ht, M2V2_ADMIN_QUOTA))
g_hash_table_insert(*ht, g_strdup(M2V2_ADMIN_QUOTA), g_strdup("namespace default"));
return 1;
}
int
main(int argc, char **args)
{
int rc = ~0;
gs_error_t *err = NULL;
gs_grid_storage_t *gs = NULL;
close(0);
if (!parse_opt(argc, args)) {
PRINT_ERROR("cannot parse the options, see the help section (--help option)\n");
return ~0;
}
if (flag_help) {
help();
return 0;
}
if (!conf_check()) {
PRINT_ERROR("invalid configuration, see the help section (--help option)\n");
help();
return ~0;
}
if (0 > chdir(base_dir)) {
PRINT_ERROR("Failed to change the working directory : %s\n", strerror(errno));
return ~0;
}
gs = gs_grid_storage_init(meta0_url, &err);
if (gs == NULL) {
if (optind<argc){
free(meta0_url);
meta0_url=strtok(args[optind],"/");
free(container_name);
container_name=strtok(NULL,"/");
if(container_name!=NULL){
gs = gs_grid_storage_init(meta0_url, &err);
if(!gs){
PRINT_ERROR("cannot init the GridStorage client\n");
goto error_gs;
}
}
else{
PRINT_ERROR("missing options\n");
help();
return 0;
}
}
else{
PRINT_ERROR("missing options\n");
help();
return 0;
}
}
if (0 > list_container(gs, container_name)) {
PRINT_ERROR("cannot list %s in namespace %s\n", container_name, meta0_url);
goto error_list;
}
rc = 0;
error_list:
gs_grid_storage_free(gs);
error_gs:
if (err)
gs_error_free(err);
return rc;
}
static int
my_content_filter(gs_content_t * content, void *user_data)
{
char original_path[1024], path[1024], *ptr;
int output_fd;
gs_error_t *err = NULL;
static gs_content_info_t info;
static gs_container_info_t container_info;
gs_download_info_t dl_info;
(void) user_data;
if (!content) {
PRINT_ERROR("Invalid content received\n");
return -1;
}
if (!gs_content_get_info(content, &info, &err)
|| !gs_container_get_info(info.container, &container_info, &err)) {
PRINT_ERROR("cannot read the information about a content (%s)\n", gs_error_get_message(err));
gs_error_free(err);
return -1;
}
/*purify the path */
g_strlcpy(original_path, info.path, sizeof(original_path));
for (ptr = original_path; *ptr; ptr++) {
register int c;
c = *ptr;
if (c == '/' || g_ascii_isspace(c))
*ptr = '_';
}
PRINT_DEBUG("About to download [grid://%s/%s/%s] (%"G_GINT64_FORMAT" bytes) in [%s/%s]\n",
meta0_url, container_info.name, info.path, info.size, base_dir, original_path);
/*open the destination file */
g_snprintf(path, sizeof(path), "%s/%s", base_dir, original_path);
output_fd = open(path, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
if (output_fd < 0) {
PRINT_ERROR("Failed to open [%s] as the download target : %s\n", path, strerror(errno));
return -1;
}
/*download the content */
memset(&dl_info, 0x00, sizeof(dl_info));
dl_info.offset = 0;
dl_info.size = 0;
dl_info.writer = write_to_fd;
dl_info.user_data = &output_fd;
if (GS_OK != gs_download_content(content, &dl_info, &err)) {
close(output_fd);
unlink(path);
if (flag_verbose) {
PRINT_ERROR("grid storage error: cannot download [grid://%s/%s/%s] in [%s]. Cause:\n\t%s\r\n",
meta0_url, container_name, info.path, path, gs_error_get_message(err));
}
else {
PRINT_ERROR("grid storage error: cannot download [grid://%s/%s/%s] in [%s]\n",
meta0_url, container_name, info.path, path);
}
return -1;
}
/*then adjust the file rights and reply the success */
fchmod(output_fd, S_IRGRP | S_IRUSR | S_IWUSR);
close(output_fd);
g_print("download: [grid://%s/%s/%s] in [%s] (%"G_GINT64_FORMAT" bytes)\n", meta0_url, container_name, info.path, path,
info.size);
nb_elements++;
/* Save content properties */
if (flag_prop && !dump_content_properties(content))
return -1;
return 0;
}
int main (int argc, char ** args)
{
int rc = -1;
gs_error_t *err = NULL;
gs_grid_storage_t *hc;
gs_container_t *container;
char *ns;
char cname[60];
srand(time(NULL));
if (argc != 2) {
g_error("Usage: %s NS\n", args[0]);
return rc;
}
ns = args[1];
bzero(cname, sizeof(cname));
g_snprintf(cname, sizeof(cname), "SOLR%d", rand());
printf("Working with container [%s]\n", cname);
hc = gs_grid_storage_init( ns, &err );
if(!hc) {
printf("failed to init hc\n");
return rc;
}
container = gs_get_container(hc, cname, 1, &err);
if(!container) {
printf("Failed to resolve container\n");
goto end_label;
}
gs_service_t **srv_array = NULL;
srv_array = gs_container_service_get_available(container, "meta0", &err);
char url[256];
bzero(url, sizeof(url));
gs_service_get_url(srv_array[0], url, sizeof(url));
printf("New service linked\n");
printf("service url = [%s]\n", url);
if(srv_array)
gs_service_free_array(srv_array);
srv_array = gs_container_service_get_all(container, "meta0", &err);
bzero(url, sizeof(url));
gs_service_get_url(srv_array[0], url, sizeof(url));
printf("Already linked service :\n");
printf("service url = [%s]\n", url);
if(srv_array)
gs_service_free_array(srv_array);
end_label:
if(container) {
gs_container_free(container);
container = NULL;
}
if(hc) {
gs_grid_storage_free(hc);
hc = NULL;
}
if(err) {
gs_error_free(err);
err= NULL;
}
}
static void
_gs_error_clear(gs_error_t ** e)
{
gs_error_free(*e);
*e = NULL;
}
