void print_summary_file(FILE *stream, struct summary *s, int include_field)
{
if( field_flag(RULE) & fields_flags ) fprintf(stream, "%d ", s->rule);
if( field_flag(WALL_TIME) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "t: " : "", s->wall_time);
if( field_flag(PROCESSES) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "p: " : "", s->max_concurrent_processes);
if( field_flag(CPU_TIME) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "c: " : "", s->cpu_time);
if( field_flag(VIRTUAL) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "v: " : "", s->virtual_memory);
if( field_flag(RESIDENT) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "m: " : "", s->resident_memory);
if( field_flag(B_READ) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "r: " : "", s->bytes_read);
if( field_flag(B_WRITTEN) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "w: " : "", s->bytes_written);
if( field_flag(WDIR_FILES) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "n: " : "", s->workdir_number_files_dirs);
if( field_flag(WDIR_FOOTPRINT) & fields_flags ) fprintf(stream, "%s%10lf ", include_field ? "z: " : "", s->workdir_footprint);
fprintf(stream, "\n");
}
double summary_accumulate(struct summary *s)
{
double acc = 0;
if( field_flag(WALL_TIME) & fields_flags ) acc += s->wall_time;
if( field_flag(PROCESSES) & fields_flags ) acc += s->max_concurrent_processes;
if( field_flag(CPU_TIME) & fields_flags ) acc += s->cpu_time;
if( field_flag(VIRTUAL) & fields_flags ) acc += s->virtual_memory;
if( field_flag(RESIDENT) & fields_flags ) acc += s->resident_memory;
if( field_flag(B_READ) & fields_flags ) acc += s->bytes_read;
if( field_flag(B_WRITTEN) & fields_flags ) acc += s->bytes_written;
if( field_flag(WDIR_FILES) & fields_flags ) acc += s->workdir_number_files_dirs;
if( field_flag(WDIR_FOOTPRINT) & fields_flags ) acc += s->workdir_footprint;
return acc;
}
void covariance_matrix_merge(struct cluster *c, struct cluster *left, struct cluster *right)
{
enum fields row, col;
struct summary *uA, *uB;
double uxA, uxB;
double uyA, uyB;
double sA, sB;
int nA, nB;
uA = left->centroid;
uB = right->centroid;
nA = left->count;
nB = right->count;
/* To NUM_FIELDS - 1 because we do not want RULE */
for(row = 1; row < NUM_FIELDS; row++)
{
if(!(field_flag(row) & fields_flags))
continue;
uxA = n_to_field(uA, row);
uxB = n_to_field(uB, row);
for(col = row; col < NUM_FIELDS; col++)
{
if(!(field_flag(col) & fields_flags))
continue;
uyA = n_to_field(uA, col);
uyB = n_to_field(uB, col);
sA = left->covariance[row][col];
sB = right->covariance[row][col];
c->covariance[row][col] = covariance_scalar_merge(uxA, uyA, sA, nA, uxB, uyB, sB, nB);
c->covariance[col][row] = c->covariance[row][col];
}
}
}
void print_covariance_matrix(FILE *stream, double covariance[NUM_FIELDS][NUM_FIELDS])
{
enum fields row, col;
fprintf(stream, "# ");
if( field_flag(WALL_TIME) & fields_flags ) fprintf(stream, "%10s ", "t ");
if( field_flag(PROCESSES) & fields_flags ) fprintf(stream, "%10s ", "p ");
if( field_flag(CPU_TIME) & fields_flags ) fprintf(stream, "%10s ", "c ");
if( field_flag(VIRTUAL) & fields_flags ) fprintf(stream, "%10s ", "v ");
if( field_flag(RESIDENT) & fields_flags ) fprintf(stream, "%10s ", "m ");
if( field_flag(B_READ) & fields_flags ) fprintf(stream, "%10s ", "r ");
if( field_flag(B_WRITTEN) & fields_flags ) fprintf(stream, "%10s ", "w ");
if( field_flag(WDIR_FILES) & fields_flags ) fprintf(stream, "%10s ", "n ");
if( field_flag(WDIR_FOOTPRINT) & fields_flags ) fprintf(stream, "%10s ", "z ");
fprintf(stream, "\n");
/* To NUM_FIELDS - 1 because we do not want RULE */
for(row = 1; row < NUM_FIELDS; row++)
{
if(!(field_flag(row) & fields_flags))
continue;
switch(row)
{
case WALL_TIME: fprintf(stream, "%s ", "# t "); break;
case PROCESSES: fprintf(stream, "%s ", "# p "); break;
case CPU_TIME: fprintf(stream, "%s ", "# c "); break;
case VIRTUAL: fprintf(stream, "%s ", "# v "); break;
case RESIDENT: fprintf(stream, "%s ", "# m "); break;
case B_READ: fprintf(stream, "%s ", "# r "); break;
case B_WRITTEN: fprintf(stream, "%s ", "# w "); break;
case WDIR_FILES: fprintf(stream, "%s ", "# n "); break;
case WDIR_FOOTPRINT: fprintf(stream, "%s ", "# z "); break;
case RULE: break;
}
for(col = 1; col < NUM_FIELDS; col++)
{
if(!(field_flag(col) & fields_flags))
continue;
fprintf(stream, "%10lf ", covariance[row][col]);
}
fprintf(stream, "\n");
}
fprintf(stream, "# \n");
}
/**
* Builds a report from database.
*/
struct lmgr_report_t *ListMgr_Report(lmgr_t *p_mgr,
const report_field_descr_t *
report_desc_array,
unsigned int report_descr_count,
const profile_field_descr_t *
profile_descr,
const lmgr_filter_t *p_filter,
const lmgr_iter_opt_t *p_opt)
{
unsigned int i;
char attrname[128];
lmgr_report_t *p_report;
int rc;
table_enum query_tab;
/* supported report fields: ENTRIES, ANNEX_INFO or ACCT */
bool use_acct_table = false;
lmgr_iter_opt_t opt = { 0 };
unsigned int profile_len = 0;
unsigned int ratio = 0;
struct field_count fcnt = { 0 };
GString *req = NULL;
GString *fields = NULL;
GString *where = NULL;
GString *having = NULL;
GString *group_by = NULL;
GString *order_by = NULL;
GString *filter_name = NULL;
/* check profile argument and increase output array if needed */
if (profile_descr != NULL) {
if (profile_descr->attr_index != ATTR_INDEX_size) {
DisplayLog(LVL_CRIT, LISTMGR_TAG,
"Profile on attribute '%s' (index=%u) is not supported",
field_name(profile_descr->attr_index),
profile_descr->attr_index);
return NULL;
}
profile_len = SZ_PROFIL_COUNT;
if (profile_descr->range_ratio_len > 0)
ratio = 1;
}
/* allocate a new report structure */
p_report = (lmgr_report_t *) MemAlloc(sizeof(lmgr_report_t));
if (!p_report)
return NULL;
p_report->p_mgr = p_mgr;
p_report->result = (struct result *)MemCalloc(report_descr_count
+ profile_len + ratio,
sizeof(struct result));
if (!p_report->result)
goto free_report;
p_report->result_count = report_descr_count + profile_len + ratio;
p_report->profile_count = profile_len;
p_report->ratio_count = ratio;
if (profile_descr != NULL)
p_report->profile_attr = ATTR_INDEX_size;
/* initially, no char * tab allocated */
p_report->str_tab = NULL;
if (p_opt)
opt = *p_opt;
fields = g_string_new(NULL);
group_by = g_string_new(NULL);
order_by = g_string_new(NULL);
having = g_string_new(NULL);
where = g_string_new(NULL);
if (full_acct(report_desc_array, report_descr_count, p_filter)
&& !opt.force_no_acct) {
listmgr_optimizedstat(p_report, p_mgr, report_descr_count,
report_desc_array, profile_descr, fields,
group_by, order_by, having, where);
use_acct_table = true;
} else { /* not only ACCT table */
/* sorting by ratio first */
if (profile_descr && profile_descr->range_ratio_len > 0) {
if (profile_descr->attr_index == ATTR_INDEX_size) {
coma_if_needed(order_by);
if (profile_descr->range_ratio_sort == SORT_ASC)
g_string_append(order_by, "sizeratio ASC");
else
g_string_append(order_by, "sizeratio DESC");
}
}
for (i = 0; i < report_descr_count; i++) {
/* no field for count or distinct count */
if (report_desc_array[i].report_type != REPORT_COUNT &&
report_desc_array[i].report_type != REPORT_COUNT_DISTINCT) {
/* in what table is this field ? */
if (is_main_field(report_desc_array[i].attr_index))
fcnt.nb_main++;
else if (is_annex_field(report_desc_array[i].attr_index))
fcnt.nb_annex++;
else {
/* Not supported yet */
DisplayLog(LVL_CRIT, LISTMGR_TAG,
"Error: report on attribute '%s' (index=%u) is not supported (report item #%u).",
field_name(report_desc_array[i].attr_index),
report_desc_array[i].attr_index, i);
rc = DB_NOT_SUPPORTED;
goto free_str;
}
}
sprintf(attrname, "attr%u", i);
/* what kind of stat on this field ? */
switch (report_desc_array[i].report_type) {
case REPORT_MIN:
coma_if_needed(fields);
g_string_append_printf(fields, "MIN(%s) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_MAX:
coma_if_needed(fields);
g_string_append_printf(fields, "MAX(%s) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_AVG:
coma_if_needed(fields);
g_string_append_printf(fields, "ROUND(AVG(%s)) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_SUM:
coma_if_needed(fields);
g_string_append_printf(fields, "SUM(%s) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_COUNT:
coma_if_needed(fields);
g_string_append_printf(fields, "COUNT(*) as %s", attrname);
p_report->result[i].type = DB_BIGUINT;
break;
case REPORT_COUNT_DISTINCT:
coma_if_needed(fields);
g_string_append_printf(fields, "COUNT(DISTINCT(%s)) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type = DB_BIGUINT;
break;
case REPORT_GROUP_BY:
coma_if_needed(fields);
g_string_append_printf(fields, "%s as %s",
field_str(report_desc_array[i].
attr_index), attrname);
coma_if_needed(group_by);
g_string_append(group_by, attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
}
/* is this field sorted ? */
append_sort_order(order_by, attrname,
report_desc_array[i].sort_flag);
/* is this field filtered ? */
listmgr_fieldfilter(p_report, p_mgr, report_desc_array, attrname,
having, where, i);
p_report->result[i].flags =
field_flag(report_desc_array[i].attr_index);
}
/* generate size profile */
if (profile_descr != NULL) {
if (profile_descr->attr_index == ATTR_INDEX_size) {
coma_if_needed(fields);
g_string_append(fields, "SUM(size=0)");
for (i = 1; i < SZ_PROFIL_COUNT - 1; i++)
g_string_append_printf(fields,
",SUM(" SZRANGE_FUNC "(size)=%u)",
i - 1);
g_string_append_printf(fields,
",SUM(" SZRANGE_FUNC "(size)>=%u)",
SZ_PROFIL_COUNT - 1);
for (i = 0; i < SZ_PROFIL_COUNT; i++)
p_report->result[i + report_descr_count].type = DB_BIGUINT;
if (profile_descr->range_ratio_len > 0) {
/* add ratio field and sort it */
coma_if_needed(fields);
g_string_append_printf(fields, "SUM(size>=%llu",
SZ_MIN_BY_INDEX(profile_descr->
range_ratio_start));
/* is the last range = 1T->inf ? */
if (profile_descr->range_ratio_start +
profile_descr->range_ratio_len >= SZ_PROFIL_COUNT)
g_string_append(fields, ")");
else
g_string_append_printf(fields, " and size<%llu)",
SZ_MIN_BY_INDEX(profile_descr->range_ratio_start
+ profile_descr->range_ratio_len));
g_string_append(fields, "/COUNT(*) as sizeratio");
}
}
}
}
/* process filter */
if (!(no_filter(p_filter))) {
if (full_acct(report_desc_array, report_descr_count, p_filter)
&& !opt.force_no_acct) {
int filter_acct;
/* filter on acct fields only */
filter_acct = filter2str(p_mgr, where, p_filter, T_ACCT,
(!GSTRING_EMPTY(where) ? AOF_LEADING_SEP : 0)
| AOF_PREFIX);
if (filter_acct > 0)
use_acct_table = true;
} else {
/* process NAMES filters apart, as with must then join with
* DISTINCT(id) */
filter_where(p_mgr, p_filter, &fcnt, where,
(!GSTRING_EMPTY(where) ? AOF_LEADING_SEP : 0)
| AOF_SKIP_NAME);
filter_name = g_string_new(NULL);
fcnt.nb_names =
filter2str(p_mgr, filter_name, p_filter, T_DNAMES, 0);
}
}
/* start building the whole request */
req = g_string_new("SELECT ");
g_string_append_printf(req, "%s FROM ", fields->str);
/* FROM clause */
if (use_acct_table) {
g_string_append(req, ACCT_TABLE);
query_tab = T_ACCT;
} else {
bool distinct;
filter_from(p_mgr, &fcnt, req, &query_tab, &distinct, AOF_SKIP_NAME);
if (filter_name != NULL && !GSTRING_EMPTY(filter_name)) {
g_string_append_printf(req, " INNER JOIN (SELECT DISTINCT(id)"
" FROM " DNAMES_TABLE " WHERE %s) N"
" ON %s.id=N.id", filter_name->str,
table2name(query_tab));
/* FIXME: what if NAMES is the query tab? */
}
/* FIXME: do the same for stripe items */
}
/* Build the request */
if (!GSTRING_EMPTY(where))
g_string_append_printf(req, " WHERE %s", where->str);
if (!GSTRING_EMPTY(group_by))
g_string_append_printf(req, " GROUP BY %s", group_by->str);
if (!GSTRING_EMPTY(having))
g_string_append_printf(req, " HAVING %s", having->str);
if (!GSTRING_EMPTY(order_by))
g_string_append_printf(req, " ORDER BY %s", order_by->str);
/* iterator opt */
if (opt.list_count_max > 0)
g_string_append_printf(req, " LIMIT %u", opt.list_count_max);
retry:
/* execute request (expect that ACCT table does not exists) */
if (use_acct_table)
rc = db_exec_sql_quiet(&p_mgr->conn, req->str,
&p_report->select_result);
else
rc = db_exec_sql(&p_mgr->conn, req->str, &p_report->select_result);
if (lmgr_delayed_retry(p_mgr, rc))
goto retry;
/* if the ACCT table does exist, switch to standard mode */
if (use_acct_table && (rc == DB_NOT_EXISTS)) {
lmgr_iter_opt_t new_opt;
if (p_opt != NULL)
new_opt = *p_opt;
else
new_opt.list_count_max = 0;
new_opt.force_no_acct = true;
DisplayLog(LVL_EVENT, LISTMGR_TAG,
"No accounting info: switching to standard query mode");
g_string_free(req, TRUE);
g_string_free(fields, TRUE);
g_string_free(group_by, TRUE);
g_string_free(order_by, TRUE);
g_string_free(having, TRUE);
g_string_free(where, TRUE);
if (filter_name != NULL)
g_string_free(filter_name, TRUE);
return ListMgr_Report(p_mgr, report_desc_array, report_descr_count,
profile_descr, p_filter, &new_opt);
}
free_str:
/* these are always allocated */
g_string_free(fields, TRUE);
g_string_free(group_by, TRUE);
g_string_free(order_by, TRUE);
g_string_free(having, TRUE);
g_string_free(where, TRUE);
/* these may not be allocated */
if (req != NULL)
g_string_free(req, TRUE);
if (filter_name != NULL)
g_string_free(filter_name, TRUE);
if (rc == DB_SUCCESS)
return p_report;
/* error */
MemFree(p_report->result);
free_report:
MemFree(p_report);
return NULL;
} /* ListMgr_Report */
static void listmgr_optimizedstat(lmgr_report_t *p_report, lmgr_t *p_mgr,
unsigned int report_descr_count,
const report_field_descr_t *
report_desc_array,
const profile_field_descr_t *profile_descr,
GString *fields, GString *group_by,
GString *order_by, GString *having,
GString *where)
{
int i;
/* sorting by ratio first */
if (profile_descr && profile_descr->range_ratio_len > 0) {
if (profile_descr->attr_index == ATTR_INDEX_size)
append_sort_order(order_by, "sizeratio",
profile_descr->range_ratio_sort);
}
for (i = 0; i < report_descr_count; i++) {
char attrname[128];
snprintf(attrname, sizeof(attrname), "attr%u", i);
if ((report_desc_array[i].report_type == REPORT_COUNT)
|| is_acct_pk(report_desc_array[i].attr_index)
|| is_acct_field(report_desc_array[i].attr_index)) {
switch (report_desc_array[i].report_type) {
case REPORT_MIN:
case REPORT_MAX:
coma_if_needed(fields);
g_string_append_printf(fields, "NULL as %s", attrname);
p_report->result[i].type = DB_TEXT;
break;
case REPORT_AVG:
coma_if_needed(fields);
g_string_append_printf(fields,
"ROUND(SUM(%s)/SUM(" ACCT_FIELD_COUNT
")) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_SUM:
coma_if_needed(fields);
g_string_append_printf(fields, "SUM(%s) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
case REPORT_COUNT:
coma_if_needed(fields);
g_string_append_printf(fields,
"SUM(" ACCT_FIELD_COUNT ") as %s",
attrname);
p_report->result[i].type = DB_BIGUINT;
break;
case REPORT_COUNT_DISTINCT:
coma_if_needed(fields);
g_string_append_printf(fields, "COUNT(DISTINCT(%s)) as %s",
field_str(report_desc_array[i].
attr_index), attrname);
p_report->result[i].type = DB_BIGUINT;
break;
case REPORT_GROUP_BY:
coma_if_needed(fields);
g_string_append_printf(fields, "%s as %s",
field_str(report_desc_array[i].
attr_index), attrname);
coma_if_needed(group_by);
g_string_append(group_by, attrname);
p_report->result[i].type =
field_type(report_desc_array[i].attr_index);
break;
}
/* is this field sorted ? */
append_sort_order(order_by, attrname,
report_desc_array[i].sort_flag);
} else {
coma_if_needed(fields);
g_string_append_printf(fields, "NULL as %s", attrname);
p_report->result[i].type = DB_TEXT;
}
listmgr_fieldfilter(p_report, p_mgr, report_desc_array, attrname,
having, where, i);
p_report->result[i].flags = field_flag(report_desc_array[i].attr_index);
}
if (profile_descr) {
/* XXX only size profile in managed for now */
if (profile_descr->attr_index == ATTR_INDEX_size) {
for (i = 0; i < SZ_PROFIL_COUNT; i++) {
coma_if_needed(fields);
g_string_append_printf(fields, "SUM(%s)", sz_field[i]);
/* count */
p_report->result[i + report_descr_count].type = DB_BIGUINT;
}
if (profile_descr->range_ratio_len > 0) {
/* add ratio field and sort it */
coma_if_needed(fields);
/* case i == 0 */
g_string_append_printf(fields, "SUM(%s",
sz_field[profile_descr->
range_ratio_start]);
for (i = 1; i < profile_descr->range_ratio_len; i++) {
g_string_append_printf(fields, "+%s",
sz_field[profile_descr->
range_ratio_start + i]);
}
g_string_append(fields,
")/SUM(" ACCT_FIELD_COUNT ") as sizeratio");
}
}
}
}
