/** * Get the list of children of a given parent (or list of parents). * \param parent_list [in] list of parents to get the child of * \param parent_count [in] number of ids in parent list * \param attr_mask [in] required attributes for children * \param child_id_list [out] ptr to array of child ids * \param child_attr_list [out] ptr to array of child attrs * \param child_count [out] number of returned children */ int ListMgr_GetChild( lmgr_t * p_mgr, const lmgr_filter_t * p_filter, const wagon_t * parent_list, unsigned int parent_count, int attr_mask, wagon_t ** child_id_list, attr_set_t ** child_attr_list, unsigned int * child_count) { result_handle_t result; char *curr; int filter_main = 0; int filter_annex = 0; int main_attrs = 0; int dnames_attrs = 0; int annex_attrs = 0; char query[4096]; char fieldlist_main[1024] = ""; char fieldlist_dnames[1024] = ""; char fieldlist_annex[1024] = ""; char filter_str_main[1024] = ""; char filter_str_annex[1024] = ""; char tmp[2048]; char *path = NULL; int path_len; char * pc; int rc, i; /* TODO: querying children from several parent cannot work, since * we need to get the paths of the children. Or we could do a * lookup into parent_list to find the right one. In the meantime, * try not to mess up the code. */ if (parent_count != 1) RBH_BUG("cannot get children for several parent simultaneously"); /* always request for name to build fullpath in wagon */ attr_mask |= ATTR_MASK_name; /* request is always on the MAIN table (which contains [parent_id, id] relationship */ /* /!\ possible cases: * - simplest: the fields of the filter and the attributes to be retrieved are in the MAIN table * - harder: the fields of the filter and attributes are in a different table */ /* 1) location of filters */ if ( p_filter ) { char dummy_str[1024]; unsigned int dummy_uint; if (dir_filter(p_mgr, dummy_str, p_filter, &dummy_uint) != FILTERDIR_NONE) { DisplayLog( LVL_MAJOR, LISTMGR_TAG, "Directory filter not supported in %s()", __func__ ); return DB_NOT_SUPPORTED; } else if (func_filter(p_mgr, dummy_str, p_filter, T_MAIN, FALSE, FALSE)) { DisplayLog( LVL_MAJOR, LISTMGR_TAG, "Function filter not supported in %s()", __func__ ); return DB_NOT_SUPPORTED; } /* There is always a filter on T_DNAMES, which is the parent condition. * Look for optional filters: */ filter_main = filter2str( p_mgr, filter_str_main, p_filter, T_MAIN, FALSE, TRUE ); if ( annex_table ) filter_annex = filter2str( p_mgr, filter_str_annex, p_filter, T_ANNEX, FALSE, TRUE ); else filter_annex = 0; /* @TODO to be implemented */ #if 0 filter_stripe_info = filter2str( p_mgr, filter_str_stripe_info, p_filter, T_STRIPE_INFO, ( filter_main > 0 ) || ( filter_annex > 0 ), TRUE ); filter_stripe_items = filter2str( p_mgr, filter_str_stripe_items, p_filter, T_STRIPE_ITEMS, ( filter_main > 0 ) || ( filter_annex > 0 ) || ( filter_stripe_info > 0 ), TRUE ); #endif } /* 2) location of requested attributes */ if (attr_mask) { /* retrieve source info for generated fields */ add_source_fields_for_gen( &attr_mask ); main_attrs = attrmask2fieldlist( fieldlist_main, attr_mask, T_MAIN, /* leading comma */ TRUE, /* for update */ FALSE, /* prefix */ MAIN_TABLE".", /* postfix */ "" ); dnames_attrs += attrmask2fieldlist( fieldlist_dnames, attr_mask, T_DNAMES, /* leading comma */ TRUE, /* for update */ FALSE, /* prefix */ DNAMES_TABLE".", /* postfix */ "" ); if ( annex_table ) annex_attrs = attrmask2fieldlist( fieldlist_annex, attr_mask, T_ANNEX, /* leading comma */ TRUE, /* for update */ FALSE, /* prefix */ ANNEX_TABLE".", /* postfix */ "" ); else annex_attrs = 0; } else { /* no returned attrs */ if (child_attr_list) *child_attr_list = NULL; } pc = parent_cond(p_mgr, tmp, sizeof(tmp), parent_list, parent_count, DNAMES_TABLE"."); if (!pc) return DB_BUFFER_TOO_SMALL; curr = query; /* SELECT clause */ /* id + dname fields */ curr += sprintf(curr, "SELECT "DNAMES_TABLE".id%s", fieldlist_dnames); /* main attrs */ if (main_attrs) curr += sprintf(curr, "%s", fieldlist_main); /* annex attrs */ if (annex_attrs) curr += sprintf(curr, "%s", fieldlist_annex); /* FROM clause */ curr += sprintf(curr, " FROM "DNAMES_TABLE); if (main_attrs || filter_main) curr += sprintf(curr, " LEFT JOIN "MAIN_TABLE " ON "DNAMES_TABLE".id="MAIN_TABLE".id"); if (annex_attrs || filter_annex) curr += sprintf(curr, " LEFT JOIN "ANNEX_TABLE " ON "DNAMES_TABLE".id="ANNEX_TABLE".id"); /* WHERE clause */ curr += sprintf(curr, " WHERE %s", pc); if (filter_main) curr += sprintf(curr, " AND %s", filter_str_main); if (filter_annex) curr += sprintf(curr, " AND %s", filter_str_annex); retry: rc = db_exec_sql(&p_mgr->conn, query, &result); if (lmgr_delayed_retry(p_mgr, rc)) goto retry; else if (rc) return rc; /* copy result to output structures */ *child_count = db_result_nb_records(&p_mgr->conn, &result); /* allocate entry_id array */ *child_id_list = MemCalloc(*child_count, sizeof(wagon_t)); if (*child_id_list == NULL) return DB_NO_MEMORY; if (child_attr_list) { *child_attr_list = MemCalloc(*child_count, sizeof(attr_set_t)); if (*child_attr_list == NULL) { rc = DB_NO_MEMORY; goto array_free; } } /* Allocate a string long enough to contain the parent path and a * child name. */ path_len = strlen(parent_list[0].fullname) + RBH_NAME_MAX + 2; path = malloc(path_len); if (!path) { DisplayLog( LVL_MAJOR, LISTMGR_TAG, "Can't alloc enough memory (%d bytes)", path_len ); rc = DB_NO_MEMORY; goto array_free; } for (i = 0; i < *child_count; i++) { char *res[128]; /* 128 fields per row is large enough */ rc = db_next_record(&p_mgr->conn, &result, res, 128); if ( rc ) goto array_free; /* copy id to array */ pk2entry_id(p_mgr, res[0], &((*child_id_list)[i].id)); /* copy attributes to array */ if (child_attr_list) { (*child_attr_list)[i].attr_mask = attr_mask; /* first id, then dnames attrs, then main attrs, then annex attrs */ if (dnames_attrs) { /* shift of 1 for id */ rc = result2attrset( T_DNAMES, res + 1, dnames_attrs, &((*child_attr_list)[i]) ); if ( rc ) goto array_free; } if (main_attrs) { /* first id, then main attrs, then annex attrs */ /* shift of 1 for id */ rc = result2attrset( T_MAIN, res + dnames_attrs + 1, main_attrs, &((*child_attr_list)[i]) ); if ( rc ) goto array_free; } if (annex_attrs) { /* shift of main_attrs count */ rc = result2attrset( T_ANNEX, res + dnames_attrs + main_attrs + 1, annex_attrs, &((*child_attr_list)[i]) ); if ( rc ) goto array_free; } #ifdef _LUSTRE if (stripe_fields(attr_mask)) { if (get_stripe_info( p_mgr, res[0], &ATTR(&(*child_attr_list)[i], stripe_info), &ATTR(&(*child_attr_list)[i], stripe_items) )) { ATTR_MASK_UNSET(&(*child_attr_list)[i], stripe_info); ATTR_MASK_UNSET(&(*child_attr_list)[i], stripe_items); } } #endif generate_fields(&((*child_attr_list)[i])); /* Note: path is properly sized already to not overflow. */ sprintf(path, "%s/%s", parent_list[0].fullname, (*child_attr_list)[i].attr_values.name); (*child_id_list)[i].fullname = strdup(path); } } if (path) free(path); db_result_free( &p_mgr->conn, &result ); return 0; array_free: if (path) free(path); if (child_attr_list && *child_attr_list) { MemFree(*child_attr_list); *child_attr_list = NULL; } MemFree(*child_id_list); *child_id_list = NULL; return rc; }
/** * Initialize a recovery process. * \param p_filter[in] (optional) filter partial filesystem recovery * \retval DB_SUCCESS the recovery process successfully started; * the stats indicate the recovery states we can expect. * \retval DB_ALREADY_EXISTS a recovery process already started * and was not properly completed. stats indicate the current status. * \retval error another error occurred. */ int ListMgr_RecovInit( lmgr_t * p_mgr, const lmgr_filter_t * p_filter, lmgr_recov_stat_t * p_stats ) { int rc; db_value_t report_val; unsigned int nb; struct lmgr_report_t * report; report_field_descr_t report_count = {-1, REPORT_COUNT, SORT_NONE, false, 0, FV_NULL}; /** @TODO use glib strings */ char query[4096]; char filter_str[4096] = ""; char *filter_curr = filter_str; #define has_filters (filter_curr != filter_str) int distinct = 0; rc = ListMgr_RecovStatus( p_mgr, p_stats ); if (rc == 0) { if (p_stats->total != 0) /* RECOVERY table exists and is not empty */ return DB_ALREADY_EXISTS; } else if ( rc != DB_NOT_EXISTS ) /* other error */ return rc; if ( rc == 0 ) { DisplayLog( LVL_EVENT, LISTMGR_TAG, "Dropping any previous "RECOV_TABLE" table" ); /* start from clean state (no table, no indexes, no addl field) */ rc = db_drop_component(&p_mgr->conn, DBOBJ_TABLE, RECOV_TABLE); if ( rc ) return rc; } if ( p_filter ) { /* dummy vars */ char filter_dir_str[512] = ""; unsigned int filter_dir_index = 0; if (dir_filter(p_mgr, filter_dir_str, p_filter, &filter_dir_index, NULL) != FILTERDIR_NONE) { DisplayLog( LVL_CRIT, LISTMGR_TAG, "Directory filter not supported for recovery"); return DB_NOT_SUPPORTED; } if (filter2str(p_mgr, filter_curr, p_filter, T_MAIN, AOF_PREFIX) > 0) filter_curr += strlen(filter_curr); if (filter2str(p_mgr, filter_curr, p_filter, T_ANNEX, (has_filters ? AOF_LEADING_SEP : 0) | AOF_PREFIX) > 0) filter_curr += strlen(filter_curr); if (filter2str(p_mgr, filter_curr, p_filter, T_DNAMES, (has_filters ? AOF_LEADING_SEP : 0) | AOF_PREFIX) > 0) { filter_curr += strlen(filter_curr); distinct = 1; } if (filter2str(p_mgr, filter_curr, p_filter, T_STRIPE_INFO, (has_filters ? AOF_LEADING_SEP : 0) | AOF_PREFIX) > 0) filter_curr += strlen(filter_curr); if (filter2str(p_mgr, filter_curr, p_filter, T_STRIPE_ITEMS, (has_filters ? AOF_LEADING_SEP : 0) | AOF_PREFIX) > 0) { filter_curr += strlen(filter_curr); distinct = 1; } } DisplayLog( LVL_EVENT, LISTMGR_TAG, "Populating "RECOV_TABLE" table (this can take a few minutes)..." ); /* create the recovery table */ if (distinct) { /* need to select only 1 instance of each object when joining with STRIPE_ITEMS or NAMES */ strcpy(query, "CREATE TABLE "RECOV_TABLE " SELECT DISTINCT("MAIN_TABLE".id)," BUILD_RECOV_LIST_FIELDS_NAMES " FROM "MAIN_TABLE" LEFT JOIN "ANNEX_TABLE" ON " "("MAIN_TABLE".id = "ANNEX_TABLE".id)" " LEFT JOIN "DNAMES_TABLE" ON " "("MAIN_TABLE".id = "DNAMES_TABLE".id)" " LEFT JOIN "STRIPE_INFO_TABLE" ON " "("MAIN_TABLE".id = "STRIPE_INFO_TABLE".id)" " LEFT JOIN "STRIPE_ITEMS_TABLE" ON " "("MAIN_TABLE".id = "STRIPE_ITEMS_TABLE".id)"); } else { strcpy(query, "CREATE TABLE "RECOV_TABLE " SELECT "MAIN_TABLE".id," BUILD_RECOV_LIST_FIELDS " FROM "MAIN_TABLE" LEFT JOIN "ANNEX_TABLE" ON " "("MAIN_TABLE".id = "ANNEX_TABLE".id)" " LEFT JOIN "STRIPE_INFO_TABLE" ON " "("MAIN_TABLE".id = "STRIPE_INFO_TABLE".id)"); } if (has_filters) { strcat(query, " WHERE "); strcat(query, filter_str); } /* the whole function is not atomic as we try to preserve the progress * in case of DB engine failure. So we retry each step independently. */ retry1: rc = db_exec_sql(&p_mgr->conn, query, NULL); if (lmgr_delayed_retry(p_mgr, rc)) goto retry1; else if (rc) return rc; DisplayLog( LVL_EVENT, LISTMGR_TAG, "Building indexes on "RECOV_TABLE" table..." ); /* create pk */ retry2: rc = db_exec_sql( &p_mgr->conn, "ALTER TABLE "RECOV_TABLE" ADD PRIMARY KEY (id)", NULL ); if (lmgr_delayed_retry(p_mgr, rc)) goto retry2; else if (rc) return rc; /* add recov_status column */ retry3: rc = db_exec_sql( &p_mgr->conn, "ALTER TABLE "RECOV_TABLE" ADD COLUMN recov_status INTEGER", NULL ); if (lmgr_delayed_retry(p_mgr, rc)) goto retry3; else if (rc) return rc; /* add index on status */ retry4: rc = db_exec_sql( &p_mgr->conn, "CREATE INDEX recov_st_index ON "RECOV_TABLE"(recov_status)", NULL ); if (lmgr_delayed_retry(p_mgr, rc)) goto retry4; else if (rc) return rc; /* count entries of each status */ expected_recov_status( p_mgr, p_stats ); /* if there is a filter on OSTs, report distinct ids */ if (distinct) report_count.report_type = REPORT_COUNT_DISTINCT; /* double check entry count before deleting entries */ report = ListMgr_Report(p_mgr, &report_count, 1, NULL, p_filter, NULL); if (report == NULL) return DB_REQUEST_FAILED; nb = 1; rc = ListMgr_GetNextReportItem(report, &report_val, &nb, NULL); ListMgr_CloseReport(report); if (rc) return rc; if ( nb == 0 ) return DB_REQUEST_FAILED; if ( report_val.value_u.val_biguint != p_stats->total ) { DisplayLog( LVL_CRIT, LISTMGR_TAG, "ERROR: recovery count (%llu) is different from entry count in main table (%lld): preserving entries", p_stats->total, report_val.value_u.val_biguint ); return DB_REQUEST_FAILED; } /* clean previous DB content */ return ListMgr_MassRemove( p_mgr, p_filter, NULL ); }
/** * Get the list of children of a given parent (or list of parents). * \param parent_list [in] list of parents to get the child of * \param parent_count [in] number of ids in parent list * \param attr_mask [in] required attributes for children * \param child_id_list [out] ptr to array of child ids * \param child_attr_list [out] ptr to array of child attrs * \param child_count [out] number of returned children */ int ListMgr_GetChild(lmgr_t *p_mgr, const lmgr_filter_t *p_filter, const wagon_t *parent_list, unsigned int parent_count, attr_mask_t attr_mask, wagon_t **child_id_list, attr_set_t **child_attr_list, unsigned int *child_count) { result_handle_t result; char *path = NULL; int path_len; int rc, i; GString *req = NULL; GString *fields = NULL; GString *from = NULL; GString *where = NULL; struct field_count field_cnt = {0}; struct field_count filter_cnt = {0}; table_enum query_tab = T_DNAMES; bool distinct = false; int retry_status; /* XXX: querying children from several parent cannot work, since * we need to get the paths of the children. Or we could do a * lookup into parent_list to find the right one. In the meantime, * try not to mess up the code. */ if (unlikely(parent_count != 1)) RBH_BUG("cannot get children for several parent simultaneously"); /* always request for name to build fullpath in wagon */ attr_mask_set_index(&attr_mask, ATTR_INDEX_name); fields = g_string_new(NULL); /* append fields for all tables */ if (!attr_mask_is_null(attr_mask)) { /* retrieve source info for generated fields */ add_source_fields_for_gen(&attr_mask.std); field_cnt.nb_names = attrmask2fieldlist(fields, attr_mask, T_DNAMES, DNAMES_TABLE".", "", AOF_LEADING_SEP); field_cnt.nb_main = attrmask2fieldlist(fields, attr_mask, T_MAIN, MAIN_TABLE".", "", AOF_LEADING_SEP); field_cnt.nb_annex = attrmask2fieldlist(fields, attr_mask, T_ANNEX, ANNEX_TABLE".", "", AOF_LEADING_SEP); } else { /* no returned attrs */ if (child_attr_list != NULL) *child_attr_list = NULL; } where = g_string_new(NULL); /* starts with condition on parent */ rc = append_parent_cond(p_mgr, where, parent_list, parent_count, DNAMES_TABLE"."); if (rc != DB_SUCCESS) goto free_str; /* check filters on other tables */ if (!no_filter(p_filter)) { if (unlikely(dir_filter(p_mgr, NULL, p_filter, NULL, NULL) != FILTERDIR_NONE)) { DisplayLog(LVL_MAJOR, LISTMGR_TAG, "Directory filter not supported in %s()", __func__); rc = DB_NOT_SUPPORTED; goto free_str; } else if (unlikely(func_filter(p_mgr, NULL, p_filter, T_MAIN, 0))) { DisplayLog(LVL_MAJOR, LISTMGR_TAG, "Function filter not supported in %s()", __func__); rc = DB_NOT_SUPPORTED; goto free_str; } /* There is always a filter on T_DNAMES, which is the parent condition. * Look for optional filters. */ filter_where(p_mgr, p_filter, &filter_cnt, where, AOF_LEADING_SEP | AOF_SKIP_NAME); /** @FIXME process other filters on NAMES */ } from = g_string_new(DNAMES_TABLE); /* add filter_count + field_count to build the FROM clause. * Preserve field count which is needed to interpret the result. */ filter_cnt.nb_main += field_cnt.nb_main; filter_cnt.nb_annex += field_cnt.nb_annex; filter_cnt.nb_names += field_cnt.nb_names; /* query tab is DNAMES, skip_name=true, is_first_tab=T_DNAMES */ filter_from(p_mgr, &filter_cnt, from, &query_tab, &distinct, AOF_LEADING_SEP | AOF_SKIP_NAME); /* request is always on the DNAMES table (which contains [parent_id, id] relationship */ if (distinct) req = g_string_new("SELECT DISTINCT("DNAMES_TABLE".id) as id"); else req = g_string_new("SELECT "DNAMES_TABLE".id as id"); /* build the whole request */ g_string_append_printf(req, "%s FROM %s WHERE %s", fields->str, from->str, where->str); retry: rc = db_exec_sql(&p_mgr->conn, req->str, &result); retry_status = lmgr_delayed_retry(p_mgr, rc); if (retry_status == 1) goto retry; else if (retry_status == 2) { rc = DB_RBH_SIG_SHUTDOWN; goto free_str; } else if (rc) goto free_str; /* copy result to output structures */ *child_count = db_result_nb_records(&p_mgr->conn, &result); /* allocate entry_id array */ *child_id_list = MemCalloc(*child_count, sizeof(wagon_t)); if (*child_id_list == NULL) { rc = DB_NO_MEMORY; goto free_str; } if (child_attr_list) { *child_attr_list = MemCalloc(*child_count, sizeof(attr_set_t)); if (*child_attr_list == NULL) { rc = DB_NO_MEMORY; goto array_free; } } /* Allocate a string long enough to contain the parent path and a * child name. */ path_len = strlen(parent_list[0].fullname) + RBH_NAME_MAX + 2; path = malloc(path_len); if (!path) { DisplayLog(LVL_MAJOR, LISTMGR_TAG, "Can't alloc enough memory (%d bytes)", path_len); rc = DB_NO_MEMORY; goto array_free; } for (i = 0; i < *child_count; i++) { char *res[128]; /* 128 fields per record is large enough */ rc = db_next_record(&p_mgr->conn, &result, res, sizeof(res)/sizeof(*res)); if (rc) goto array_free; /* copy id to array */ pk2entry_id(p_mgr, res[0], &((*child_id_list)[i].id)); /* copy attributes to array */ if (child_attr_list) { unsigned int shift = 1; /* first was NAMES.id */ (*child_attr_list)[i].attr_mask = attr_mask; /* first id, then dnames attrs, then main attrs, then annex attrs */ if (field_cnt.nb_names > 0) { /* shift of 1 for id */ rc = result2attrset(T_DNAMES, res + shift, field_cnt.nb_names, &((*child_attr_list)[i])); if (rc) goto array_free; shift += field_cnt.nb_names; } if (field_cnt.nb_main > 0) { /* first id, then main attrs, then annex attrs */ /* shift of 1 for id */ rc = result2attrset(T_MAIN, res + shift, field_cnt.nb_main, &((*child_attr_list)[i])); if (rc) goto array_free; shift += field_cnt.nb_main; } if (field_cnt.nb_annex > 0) { /* shift of main_attrs count */ rc = result2attrset(T_ANNEX, res + shift, field_cnt.nb_annex, &((*child_attr_list)[i])); if (rc) goto array_free; shift += field_cnt.nb_annex; } #ifdef _LUSTRE if (stripe_fields(attr_mask)) { if (get_stripe_info(p_mgr, res[0], &ATTR(&(*child_attr_list)[i], stripe_info), &ATTR(&(*child_attr_list)[i], stripe_items))) { ATTR_MASK_UNSET(&(*child_attr_list)[i], stripe_info); ATTR_MASK_UNSET(&(*child_attr_list)[i], stripe_items); } } #endif generate_fields(&((*child_attr_list)[i])); /* Note: path is properly sized already to not overflow. */ snprintf(path, path_len, "%s/%s", parent_list[0].fullname, (*child_attr_list)[i].attr_values.name); (*child_id_list)[i].fullname = strdup(path); } } if (path) free(path); db_result_free(&p_mgr->conn, &result); g_string_free(req, TRUE); g_string_free(fields, TRUE); g_string_free(from, TRUE); g_string_free(where, TRUE); return 0; array_free: if (path) free(path); if (child_attr_list && *child_attr_list) { MemFree(*child_attr_list); *child_attr_list = NULL; } MemFree(*child_id_list); *child_id_list = NULL; free_str: g_string_free(req, TRUE); g_string_free(fields, TRUE); g_string_free(from, TRUE); g_string_free(where, TRUE); return rc; }