/*
* Carry out aggregation on a complete data set held in a table
* This is an alternative entry point to the class that does not need
* the setting up of a session.
* The table should identify keys, time, sequence and duration in the
* standard way as defined by FHA spec.
* Returns a TABLE of results compliant to the FHA spec, _time will be
* set to the last time of the dataset, _seq to 0. _dur is not set
* The result is independent of dataset's memory allocation, sothe caller
* needs to run table_destroy() to free its memory.
* Returns NULL if there is an error, if dataset is NULL or if there
* was insufficent data.
*/
TABLE cascade_aggregate(enum cascade_fn func, /* aggregation function */
TABLE dataset /* multi-sample, multi-key
* dataset in a table */ )
{
TREE *inforow, *keyvals, *databykey, *colnames;
char *keycol, *colname, *tmpstr, *type;
int duration, haskey=0;
TABLE itab, result;
TABSET tset;
ITREE *col;
double val, tmpval1, tmpval2;
time_t t1, t2, tdiff;
/* assert special cases */
if ( ! dataset ) {
elog_printf(DIAG, "no dataset given to aggregate");
return NULL;
}
if (table_nrows(dataset) == 0) {
elog_printf(DIAG, "no rows to aggregate in dataset");
}
if ( ! table_hascol(dataset, "_time")) {
tmpstr = table_outheader(dataset);
elog_printf(ERROR, "attempting to aggregate a table without _time "
"column (columns: %s)", tmpstr);
nfree(tmpstr);
return NULL;
}
/* find any keys that might exist */
inforow = table_getinforow(dataset, "key");
if (inforow) {
keycol = tree_search(inforow, "1", 2);
if (keycol) {
keyvals = table_uniqcolvals(dataset, keycol, NULL);
if (keyvals) {
/* separate the combined data set into ones of
* separate keys */
haskey++;
databykey = tree_create();
tset = tableset_create(dataset);
tree_traverse(keyvals) {
/* select out the data */
tableset_reset(tset);
tableset_where(tset, keycol, eq,
tree_getkey(keyvals));
itab = tableset_into(tset);
tree_add(databykey, tree_getkey(keyvals), itab);
}
tableset_destroy(tset);
}
tree_destroy(keyvals);
}
tree_destroy(inforow);
}
/* if there were no keys found, pretend that we have a single one */
if ( ! haskey ) {
databykey = tree_create();
tree_add(databykey, "nokey", dataset);
}
/* find the time span and duration of the dataset */
table_first(dataset);
if (table_hascol(dataset, "_dur"))
duration = strtol(table_getcurrentcell(dataset, "_dur"), NULL, 10);
else
duration = 0;
t1 = strtol(table_getcurrentcell(dataset, "_time"), NULL, 10);
table_last(dataset);
t2 = strtol(table_getcurrentcell(dataset, "_time"), NULL, 10);
tdiff = t2-t1+duration;
/* go over the keyed table and apply our operators to each column
* in turn */
result = table_create_fromdonor(dataset);
table_addcol(result, "_seq", NULL); /* make col before make row */
table_addcol(result, "_time", NULL);
table_addcol(result, "_dur", NULL);
tree_traverse(databykey) {
table_addemptyrow(result);
itab = tree_get(databykey);
colnames = table_getheader(itab);
tree_traverse(colnames) {
colname = tree_getkey(colnames);
if ( ! table_hascol(result, colname)) {
tmpstr = xnstrdup(colname);
table_addcol(result, tmpstr, NULL);
table_freeondestroy(result, tmpstr);
}
col = table_getcol(itab, colname);
type = table_getinfocell(itab, "type", colname);
if (type && strcmp(type, "str") == 0) {
/* string value: report the last one */
itree_last(col);
table_replacecurrentcell(result, colname, itree_get(col));
} else if (strcmp(colname, "_dur") == 0) {
/* _dur: use the last value, can treat as string */
itree_last(col);
table_replacecurrentcell(result, "_dur", itree_get(col));
} else if (strcmp(colname, "_seq") == 0) {
/* _seq: only one result is produced so must be 0 */
table_replacecurrentcell(result, "_seq", "0");
} else if (strcmp(colname, "_time") == 0) {
/* _time: use the last value, can treat as string */
itree_last(col);
table_replacecurrentcell(result, "_time", itree_get(col));
} else {
/* numeric value: treat as a float and report it */
switch (func) {
case CASCADE_AVG:
/* average of column's values */
val = 0.0;
itree_traverse(col)
val += atof( itree_get(col) );
val = val / itree_n(col);
break;
case CASCADE_MIN:
/* minimum of column's values */
val = DBL_MAX;
itree_traverse(col) {
tmpval1 = atof( itree_get(col) );
if (tmpval1 < val)
val = tmpval1;
}
break;
case CASCADE_MAX:
/* maximum of column's values */
val = DBL_MIN;
itree_traverse(col) {
tmpval1 = atof( itree_get(col) );
if (tmpval1 > val)
val = tmpval1;
}
break;
case CASCADE_SUM:
/* sum of column's values */
val = 0.0;
itree_traverse(col)
val += atof( itree_get(col) );
break;
case CASCADE_LAST:
/* last value */
itree_last(col);
val = atof( itree_get(col) );
break;
case CASCADE_FIRST:
/* last value */
itree_first(col);
val = atof( itree_get(col) );
break;
case CASCADE_DIFF:
/* the difference in values of first and last */
itree_first(col);
tmpval1 = atof( itree_get(col) );
itree_last(col);
tmpval2 = atof( itree_get(col) );
val = tmpval2 - tmpval1;
break;
case CASCADE_RATE:
/* difference in values (as CASCADE_DIFF) then
* divided by the number of seconds in the set */
itree_first(col);
tmpval1 = atof( itree_get(col) );
itree_last(col);
tmpval2 = atof( itree_get(col) );
val = tmpval2 - tmpval1;
val = val / tdiff;
break;
}
/* save the floating point value */
table_replacecurrentcell_alloc(result, colname,
util_ftoa(val));
}
itree_destroy(col);
}
/* make sure that there are values for the special columns */
if ( ! table_hascol(dataset, "_time"))
table_replacecurrentcell(result, "_time",
util_decdatetime(time(NULL)));
if ( ! table_hascol(dataset, "_seq"))
table_replacecurrentcell(result, "_seq", "0");
if ( ! table_hascol(dataset, "_dur"))
table_replacecurrentcell(result, "_dur", "0");
}
/* clear up */
if (haskey) {
tree_traverse(databykey) {
itab = tree_get(databykey);
table_destroy(itab);
}
}
tree_destroy(databykey);
return result;
}
/*
* Linux specific routines
*/
void plinps_collect(TABLE tab) {
DIR *dir;
struct dirent *d;
char pfile[PATH_MAX];
void *data;
TABLE filtered_tab;
/* open procfs */
dir = opendir("/proc");
if ( ! dir ) {
elog_printf(ERROR, "can't open /proc: %d %s",
errno, strerror(errno));
return;
}
/* traverse process entries.
* remember to take into consideration the transigent nature of
* processes, which may not be there when we come to opening them */
while ((d = readdir(dir))) {
/* the linux /proc contains pids, dot files and system status files
* we are only interested in the pid directories whose filenames
* contain only digits */
if (strcmp(d->d_name, ".") == 0)
continue;
if (strcmp(d->d_name, "..") == 0)
continue;
if ( ! isdigit(*d->d_name) )
continue;
/* open pid's short stat file */
sprintf(pfile, "/proc/%s/stat", d->d_name);
data = probe_readfile(pfile);
if (data) {
/* point of no return: start collecting table data */
table_addemptyrow(tab);
/*plinps_col_fperm(tab, pfile, plinps_uidtoname);*/
plinps_col_stat(tab, data, plinps_uidtoname);
table_freeondestroy(tab, data);
} else
continue;
/* add information from the longer status file */
sprintf(pfile, "/proc/%s/status", d->d_name);
data = probe_readfile(pfile);
if (data) {
plinps_col_status(tab, data, plinps_uidtoname);
table_freeondestroy(tab, data);
}
/* add memory stats */
sprintf(pfile, "/proc/%s/statm", d->d_name);
data = probe_readfile(pfile);
if (data) {
plinps_col_statm(tab, data, plinps_uidtoname);
table_freeondestroy(tab, data);
}
/* find the command line */
sprintf(pfile, "/proc/%s/cmdline", d->d_name);
data = probe_readfile(pfile);
if (data) {
plinps_col_cmd(tab, data);
table_freeondestroy(tab, data);
}
}
/* close procfs and clean up */
closedir(dir);
/* check to see if there is any change in the route content (and
* thus the filter clause), then compile the filter on this tab
* (inefficient, ought to compile once) */
plinps_load_filter(NULL);
plinps_compile_filter(tab);
if (plinps_filter_tset) {
/* filter the main table into a subset, then replace the original
* with the subset */
filtered_tab = tableset_into(plinps_filter_tset);
table_rmallrows(tab);
if (table_nrows(filtered_tab))
if (table_addtable(tab, filtered_tab, 0) == -1) {
elog_printf(FATAL, "unable to replace table");
}
table_destroy(filtered_tab);
}
}
/*
* FreeResources: Free resource table.
*/
void FreeResources(void)
{
table_destroy(t, FreeRsc);
}
int destroy_tests( void ) {
table_destroy(&test_table, NULL);
return 0;
}
int main(void)
{
Table *table = table_create();
printf("-------------------------------------------\n");
printf("Commands:\n");
printf("a <key> <value> - push to table\n");
printf("d <key> - delete from table\n");
printf("s - sort table\n");
printf("p - print table\n");
printf("f <key> - find element by key\n");
printf("c - clear table\n");
printf("q - exit\n");
printf("-------------------------------------------\n");
size_t cnt = 0;
char last_cmd = '\n';
while (true) {
if (!cnt || last_cmd == '\n')
printf("> ");
char cmd;
bool is_finished = false;
scanf("%c", &cmd);
size_t pos;
//LinearlistElement *this = NULL;
Key *key = key_create();
switch (cmd) {
case 'q':
is_finished = true;
break;
case 'a':
table_push(table);
break;
case 'd':
//scanf("%d", &pos);
key_set_value_from_input(key);
table_note_delete(table, key);
break;
case 's':
table_sort(table);
break;
case 'p':
table_print(table);
break;
case 'c':
table_clear(table);
break;
case 'f':
key_set_value_from_input(key);
Note *found = table_search(table, key, &pos);
if (found == NULL) {
printf("There are no notes with this key in the table\n");
break;
}
note_print(table, found);
break;
case '\n':
break;
default:
printf("Invalid command `%c`\n", cmd);
break;
}
last_cmd = cmd;
key_destroy(&key);
if (is_finished) break;
cnt++;
cnt %= 2;
//printf("First: %lld\n", (long long) linearlist_get_value(linearlist_get_first(linearlist)));
//printf("Last: %lld\n", (long long) linearlist_get_value(linearlist_get_last(linearlist)));
}
printf("Goodbye!\n");
table_destroy(&table);
return 0;
}
int main(void)
{
t_table *table = table_init(128);
if (table == NULL) {
fprintf(stderr, "Table could not be initialized!\n");
return EXIT_FAILURE;
}
print_table_info(table);
print_table(table);
int data_pole[NUM] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
t_string key_pole[NUM];
int i;
for (i = 0; i < NUM; i++) {
stringInit(&key_pole[i]);
}
for (i = 0; i < NUM; i++) {
stringAddChar(&key_pole[i], 'a' + i);
}
int ret = 0;
t_tab_data * data_ptr;
for (i = 0; i < NUM; i++) {
ret = table_insert(table, &key_pole[i], &data_ptr);
switch (ret) {
case TAB_INSERT_OK:
data_ptr->a = data_pole[i];
break;
case TAB_INSERT_FOUND:
fprintf(stderr, "Item already inserted\n");
break;
case TAB_INSERT_ALL_ERR:
fprintf(stderr, "Allocation error\n");
break;
}
}
print_table_info(table);
print_table(table);
/*
//1. raz
printf("INSERT\n\n");
for (i = 0; i < NUM; i++) {
ret = table_insert(table, key_pole[i], 7, (t_tab_data **) &data);
switch (ret) {
case TAB_INSERT_FAIL:
fprintf(stderr, "Item already inserted.\n");
break;
case TAB_INSERT_FULL:
while (ret == TAB_INSERT_FULL) {
if (table_resize(table) == TAB_RESIZE_FAIL) {
fprintf(stderr, "RESIZE FAIL\n");
table_destroy(table);
return EXIT_FAILURE;
}
ret = table_insert(table, key_pole[i], 7, (t_tab_data **)&data);
}
if (ret == TAB_INSERT_FAIL) {
fprintf(stderr, "Item already inserted.\n");
break;
}
if (ret == TAB_INSERT_ALL_ERR) {
fprintf(stderr, "ALLOC ERR\n");
table_destroy(table);
return EXIT_FAILURE;
}
*(int *)data = data_pole[i];
break;
case TAB_INSERT_OK:
*(int *)data = data_pole[i];
break;
case TAB_INSERT_ALL_ERR:
fprintf(stderr, "ALLOC ERROR\n");
table_destroy(table);
return EXIT_FAILURE;
}
}
print_table(table);
*/
printf("-----------------------------------------------------------------------------------------\n\n");
printf("GET DATA\n\n");
for (i = 0; i < NUM; i++) {
ret = table_search(table, &key_pole[i], (t_tab_data **) &data_ptr);
if (ret == TAB_SEARCH_OK) {
printf("Key: %s\n", key_pole[i].string);
printf("Data: %d\n\n", data_ptr == NULL ? -1 : ((t_tab_data *)data_ptr)->a);
}
}
/*
print_table(table);
*/
for (i = 0; i < NUM; i++) {
stringFree(&key_pole[i]);
}
table_destroy(table);
return EXIT_SUCCESS;
}
/*
* Configure the tableset from text, returning an error if parse fails
* Currently, this is a very simple format consisting of individual lines,
* each introduced by the verb token. Conditions are AND'ed together
* Syntax: where <col> <op> <val>
* unless <col> <op> <val>
* <op>: eq - equal (string context)
* ne - not equal (string context)
* gt - greater than (numeric context)
* lt - less than (numeric context)
* ge - greater than or equal to (numeric context)
* le - less than or equal to (numeric context)
* begins - text string begins with (string context)
* example: "where col1 eq fred" or "unless col4 begins kev"
* Returns 1 for success or 0 for failure, errors are sent to elog
* The commands text buffer does not have to be kept, as it is duplicated
* internally.
*/
int tableset_configure(TABSET t, char *commands) {
TABLE tabcmds;
int r, i;
char *cmds, *mode, *col, *optxt, *val;
enum tableset_op op;
/* check arguments are present */
if ( ! t )
elog_die(FATAL, "No tableset specified");
if ( ! commands || ! *commands ) {
elog_printf(DIAG, "No commands supplied with which to configure");
return 0; /* failure */
}
cmds = xnstrdup(commands);
/* scan text commands into an unheaded table, which is to manage memory */
tabcmds = table_create();
r = table_scan(tabcmds, cmds, " \t", TABLE_MULTISEP, TABLE_NOCOLNAMES,
TABLE_NORULER);
if (r == -1) {
elog_printf(ERROR, "Unable to scan commands: '%s'", commands);
nfree(cmds);
return 0; /* failure */
}
table_freeondestroy(tabcmds, cmds);
/* now interpret the commands into the tableset */
table_traverse(tabcmds) {
/* First column is the column/attribute name, second is the
* operation to apply, third is the value */
mode = table_getcurrentcell(tabcmds, "column_0");
col = table_getcurrentcell(tabcmds, "column_1");
optxt = table_getcurrentcell(tabcmds, "column_2");
val = table_getcurrentcell(tabcmds, "column_3");
if ((mode && col && optxt && val) == 0) {
elog_printf(ERROR, "None of the rows has the right number of "
"columns (should be 4: mode col optxt val)");
table_destroy(tabcmds);
return 0; /* failure */
}
/* translate the operation text into enum */
for (i=0; tableset_optxt[i]; i++)
if (strcmp(tableset_optxt[i], optxt) == 0) {
op = i;
break;
}
if (tableset_optxt[i] == NULL) {
elog_printf(ERROR, "Line %d has an unknown operator '%s'; "
"not using this line",
table_getcurrentrowkey(tabcmds) + 1, optxt);
table_destroy(tabcmds);
return 0; /* failure */
}
/* match the command to carry out operation */
if (strncmp(mode, "where", 5) == 0) {
tableset_where(t, col, op, val);
} else if (strncmp(mode, "unless", 6) == 0) {
tableset_unless(t, col, op, val);
} else {
elog_printf(ERROR, "Unable to recognise configuration "
"statement, line %d: %s %s %s %s; not using this "
"line", table_getcurrentrowkey(tabcmds) + 1,
mode, col, optxt, val);
}
}
table_destroy(tabcmds);
return 1; /* success */
}
/**
* Destroys the supplied transaction.
*
* @param tx
*/
void htp_tx_destroy(htp_tx_t *tx) {
bstr_free(tx->request_line);
bstr_free(tx->request_method);
bstr_free(tx->request_uri);
bstr_free(tx->request_uri_normalized);
bstr_free(tx->request_protocol);
if (tx->parsed_uri != NULL) {
bstr_free(tx->parsed_uri->scheme);
bstr_free(tx->parsed_uri->username);
bstr_free(tx->parsed_uri->password);
bstr_free(tx->parsed_uri->hostname);
bstr_free(tx->parsed_uri->port);
bstr_free(tx->parsed_uri->path);
bstr_free(tx->parsed_uri->query);
bstr_free(tx->parsed_uri->fragment);
free(tx->parsed_uri);
}
if (tx->parsed_uri_incomplete != NULL) {
bstr_free(tx->parsed_uri_incomplete->scheme);
bstr_free(tx->parsed_uri_incomplete->username);
bstr_free(tx->parsed_uri_incomplete->password);
bstr_free(tx->parsed_uri_incomplete->hostname);
bstr_free(tx->parsed_uri_incomplete->port);
bstr_free(tx->parsed_uri_incomplete->path);
bstr_free(tx->parsed_uri_incomplete->query);
bstr_free(tx->parsed_uri_incomplete->fragment);
free(tx->parsed_uri_incomplete);
}
// Destroy request_header_lines
htp_header_line_t *hl = NULL;
if (tx->request_header_lines != NULL) {
list_iterator_reset(tx->request_header_lines);
while ((hl = list_iterator_next(tx->request_header_lines)) != NULL) {
bstr_free(hl->line);
bstr_free(hl->terminators);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
list_destroy(tx->request_header_lines);
}
// Destroy request_headers
htp_header_t *h = NULL;
if (tx->request_headers != NULL) {
table_iterator_reset(tx->request_headers);
while (table_iterator_next(tx->request_headers, (void **) & h) != NULL) {
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
table_destroy(tx->request_headers);
}
if (tx->request_headers_raw != NULL) {
bstr_free(tx->request_headers_raw);
}
bstr_free(tx->response_line);
bstr_free(tx->response_protocol);
bstr_free(tx->response_status);
bstr_free(tx->response_message);
// Destroy response_header_lines
hl = NULL;
if (tx->response_header_lines != NULL) {
list_iterator_reset(tx->response_header_lines);
while ((hl = list_iterator_next(tx->response_header_lines)) != NULL) {
bstr_free(hl->line);
bstr_free(hl->terminators);
// No need to destroy hl->header because
// htp_header_line_t does not own it.
free(hl);
}
list_destroy(tx->response_header_lines);
}
// Destroy response headers
h = NULL;
if (tx->response_headers) {
table_iterator_reset(tx->response_headers);
while (table_iterator_next(tx->response_headers, (void **) & h) != NULL) {
bstr_free(h->name);
bstr_free(h->value);
free(h);
}
table_destroy(tx->response_headers);
}
// Tell the connection to remove this transaction
// from the list
htp_conn_remove_tx(tx->conn, tx);
// Invalidate the pointer to this transactions held
// by the connection parser. This is to allow a transaction
// to be destroyed from within the final response callback.
if (tx->connp != NULL) {
if (tx->connp->out_tx == tx) {
tx->connp->out_tx = NULL;
}
}
free(tx);
}
void
purge_config(uint8_t what)
{
struct listener *l;
struct table *t;
struct rule *r;
struct pki *p;
const char *k;
void *iter_dict;
if (what & PURGE_LISTENERS) {
while ((l = TAILQ_FIRST(env->sc_listeners)) != NULL) {
TAILQ_REMOVE(env->sc_listeners, l, entry);
free(l);
}
free(env->sc_listeners);
env->sc_listeners = NULL;
}
if (what & PURGE_TABLES) {
while (dict_root(env->sc_tables_dict, NULL, (void **)&t))
table_destroy(t);
free(env->sc_tables_dict);
env->sc_tables_dict = NULL;
}
if (what & PURGE_RULES) {
while ((r = TAILQ_FIRST(env->sc_rules)) != NULL) {
TAILQ_REMOVE(env->sc_rules, r, r_entry);
free(r);
}
free(env->sc_rules);
env->sc_rules = NULL;
}
if (what & PURGE_PKI) {
while (dict_poproot(env->sc_pki_dict, (void **)&p)) {
explicit_bzero(p->pki_cert, p->pki_cert_len);
free(p->pki_cert);
if (p->pki_key) {
explicit_bzero(p->pki_key, p->pki_key_len);
free(p->pki_key);
}
if (p->pki_pkey)
EVP_PKEY_free(p->pki_pkey);
free(p);
}
free(env->sc_pki_dict);
env->sc_pki_dict = NULL;
} else if (what & PURGE_PKI_KEYS) {
iter_dict = NULL;
while (dict_iter(env->sc_pki_dict, &iter_dict, &k,
(void **)&p)) {
explicit_bzero(p->pki_cert, p->pki_cert_len);
free(p->pki_cert);
p->pki_cert = NULL;
if (p->pki_key) {
explicit_bzero(p->pki_key, p->pki_key_len);
free(p->pki_key);
p->pki_key = NULL;
}
if (p->pki_pkey)
EVP_PKEY_free(p->pki_pkey);
p->pki_pkey = NULL;
}
}
}
void ui_print_val(uint8_t type, uint8_t *data, uint32_t length)
{
uint32_t bytes = 0;
if(data == NULL)
{
printf("NULL");
return;
}
switch(type)
{
case AMP_TYPE_VAR:
{
var_t *cd = var_deserialize(data, length, &bytes);
char *str = var_to_string(cd);
printf("%s", str);
SRELEASE(str);
var_release(cd);
}
break;
case AMP_TYPE_INT:
printf("%d", utils_deserialize_int(data, length, &bytes));
break;
case AMP_TYPE_TS:
case AMP_TYPE_UINT:
printf("%d", utils_deserialize_uint(data, length, &bytes));
break;
case AMP_TYPE_VAST:
printf(VAST_FIELDSPEC, utils_deserialize_vast(data, length, &bytes));
break;
case AMP_TYPE_SDNV:
case AMP_TYPE_UVAST:
printf(UVAST_FIELDSPEC, utils_deserialize_uvast(data, length, &bytes));
break;
case AMP_TYPE_REAL32:
printf("%f", utils_deserialize_real32(data, length, &bytes));
break;
case AMP_TYPE_REAL64:
printf("%f", utils_deserialize_real64(data, length, &bytes));
break;
case AMP_TYPE_STRING:
{
char* tmp = NULL;
tmp = utils_deserialize_string(data, length, &bytes);
printf("%s", tmp);
SRELEASE(tmp);
}
break;
case AMP_TYPE_BLOB:
{
blob_t *blob = blob_deserialize(data, length, &bytes);
char *str = blob_to_str(blob);
printf("%s", str);
SRELEASE(str);
SRELEASE(blob);
}
break;
case AMP_TYPE_DC:
{
uint32_t bytes = 0;
Lyst dc = dc_deserialize(data, length, &bytes);
ui_print_dc(dc);
dc_destroy(&dc);
}
break;
case AMP_TYPE_MID:
{
uint32_t bytes = 0;
mid_t *mid = mid_deserialize(data, length, &bytes);
ui_print_mid(mid);
mid_release(mid);
}
break;
case AMP_TYPE_MC:
{
uint32_t bytes = 0;
Lyst mc = midcol_deserialize(data, length, &bytes);
ui_print_mc(mc);
midcol_destroy(&mc);
}
break;
// \todo: Expression has no priority. Need to re-think priority.
case AMP_TYPE_EXPR:
{
uint32_t bytes = 0;
expr_t *expr = expr_deserialize(data, length, &bytes);
ui_print_expr(expr);
expr_release(expr);
}
break;
/* case DTNMP_TYPE_DEF:
{
uint32_t bytes = 0;
def_gen_t *def = def_deserialize_gen(data, length, &bytes);
ui_print_def(def);
def_release_gen(def);
}
break;
*/
case AMP_TYPE_TRL:
{
uint32_t bytes = 0;
trl_t *trl = trl_deserialize(data, length, &bytes);
ui_print_trl(trl);
trl_release(trl);
}
break;
case AMP_TYPE_TABLE:
{
uint32_t bytes = 0;
table_t *table = table_deserialize(data, length, &bytes);
ui_print_table(table);
table_destroy(table, 1);
}
break;
case AMP_TYPE_SRL:
{
uint32_t bytes = 0;
srl_t *srl = srl_deserialize(data, length, &bytes);
ui_print_srl(srl);
srl_release(srl);
}
break;
default:
printf("Unknown.");
}
}
/**
* Transcode all parameters supplied in the table.
*
* @param connp
* @param params
* @param destroy_old
*/
int htp_transcode_params(htp_connp_t *connp, table_t **params, int destroy_old) {
table_t *input_params = *params;
// No transcoding unless necessary
if (connp->cfg->internal_encoding == NULL) {
return HTP_OK;
}
// Create a new table that will hold transcoded parameters
table_t *output_params = connp->cfg->create_table(table_size(input_params));
if (output_params == NULL) {
return HTP_ERROR;
}
// Initialize iconv
iconv_t cd = iconv_open(connp->cfg->internal_encoding, connp->cfg->request_encoding);
if (cd == (iconv_t) -1) {
// TODO Report iconv initialization error
table_destroy(&output_params);
return HTP_ERROR;
}
#if (_LIBICONV_VERSION >= 0x0108)
int iconv_param = 0;
iconvctl(cd, ICONV_SET_TRANSLITERATE, &iconv_param);
iconv_param = 1;
iconvctl(cd, ICONV_SET_DISCARD_ILSEQ, &iconv_param);
#endif
// Convert the parameters, one by one
bstr *name;
void *tvalue;
table_iterator_reset(input_params);
while ((name = table_iterator_next(input_params, &tvalue)) != NULL) {
bstr *new_name = NULL, *new_value = NULL;
bstr *value = (bstr *)tvalue;
// Convert name
htp_transcode_bstr(cd, name, &new_name);
if (new_name == NULL) {
iconv_close(cd);
table_iterator_reset(output_params);
while(table_iterator_next(output_params, &tvalue) != NULL) {
bstr *b = (bstr *)tvalue;
bstr_free(&b);
}
table_destroy(&output_params);
return HTP_ERROR;
}
// Convert value
htp_transcode_bstr(cd, value, &new_value);
if (new_value == NULL) {
bstr_free(&new_name);
iconv_close(cd);
table_iterator_reset(output_params);
while(table_iterator_next(output_params, &tvalue) != NULL) {
bstr *b = (bstr *)tvalue;
bstr_free(&b);
}
table_destroy(&output_params);
return HTP_ERROR;
}
// Add to new table
table_addn(output_params, new_name, new_value);
}
// Replace the old parameter table
*params = output_params;
// Destroy the old parameter table if necessary
if (destroy_old) {
table_iterator_reset(input_params);
while(table_iterator_next(input_params, &tvalue) != NULL) {
bstr *b = (bstr *)tvalue;
bstr_free(&b);
}
table_destroy(&input_params);
}
iconv_close(cd);
return HTP_OK;
}
static void insert_row(DB_ENV *db_env, struct table *t, DB_TXN *txn, long a, long b, long c, long d) {
int r;
// generate the primary key
char key_buffer[8];
a = htonl64(a);
memcpy(key_buffer, &a, sizeof a);
// generate the primary value
char val_buffer[3*8];
b = htonl64(b);
memcpy(val_buffer+0, &b, sizeof b);
c = htonl64(c);
memcpy(val_buffer+8, &c, sizeof c);
d = htonl64(d);
memcpy(val_buffer+16, &d, sizeof d);
DBT key = { .data = key_buffer, .size = sizeof key_buffer };
DBT value = { .data = val_buffer, .size = sizeof val_buffer };
#if defined(TOKUDB)
if (!force_multiple && t->ndbs == 1) {
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, t->mult_flags[0]); assert(r == 0);
} else {
r = db_env->put_multiple(db_env, t->dbs[0], txn, &key, &value, t->ndbs, &t->dbs[0], t->mult_keys, t->mult_vals, t->mult_flags); assert(r == 0);
}
#else
assert(db_env);
r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, 0); assert(r == 0);
#endif
}
static inline float tdiff (struct timeval *a, struct timeval *b) {
return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
}
static void insert_all(DB_ENV *db_env, struct table *t, long nrows, long max_rows_per_txn, long key_range, long rows_per_report, bool do_txn) {
int r;
struct timeval tstart;
r = gettimeofday(&tstart, NULL); assert(r == 0);
struct timeval tlast = tstart;
DB_TXN *txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
}
long n_rows_per_txn = 0;
long rowi;
for (rowi = 0; rowi < nrows; rowi++) {
long a = rowi;
long b = random64() % key_range;
long c = random64() % key_range;
long d = random64() % key_range;
insert_row(db_env, t, txn, a, b, c, d);
n_rows_per_txn++;
// maybe commit
if (do_txn && n_rows_per_txn == max_rows_per_txn) {
r = txn->commit(txn, 0); assert(r == 0);
r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
n_rows_per_txn = 0;
}
// maybe report performance
if (((rowi + 1) % rows_per_report) == 0) {
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
float last_time = tdiff(&tnow, &tlast);
float total_time = tdiff(&tnow, &tstart);
printf("%ld %.3f %.0f/s %.0f/s\n", rowi + 1, last_time, rows_per_report/last_time, rowi/total_time); fflush(stdout);
tlast = tnow;
}
}
if (do_txn) {
r = txn->commit(txn, 0); assert(r == 0);
}
struct timeval tnow;
r = gettimeofday(&tnow, NULL); assert(r == 0);
printf("total %ld %.3f %.0f/s\n", nrows, tdiff(&tnow, &tstart), nrows/tdiff(&tnow, &tstart)); fflush(stdout);
}
int main(int argc, char *argv[]) {
#if defined(TOKDUB)
char *db_env_dir = "insertm.env.tokudb";
#else
char *db_env_dir = "insertm.env.bdb";
#endif
int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG;
long rows = 100000000;
long rows_per_txn = 1000;
long rows_per_report = 100000;
long key_range = 100000;
bool do_txn = true;
u_int32_t pagesize = 0;
u_int64_t cachesize = 1000000000;
int ndbs = 4;
#if defined(TOKUDB)
u_int32_t checkpoint_period = 60;
#endif
int i;
for (i = 1; i < argc; i++) {
char *arg = argv[i];
if (strcmp(arg, "--verbose") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
ndbs = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--rows") == 0 && i+1 < argc) {
rows = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_txn") == 0 && i+1 < argc) {
rows_per_txn = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--rows_per_report") == 0 && i+1 < argc) {
rows_per_report = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--key_range") == 0 && i+1 < argc) {
key_range = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--txn") == 0 && i+1 < argc) {
do_txn = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--pagesize") == 0 && i+1 < argc) {
pagesize = atoi(argv[++i]);
continue;
}
if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
cachesize = atol(argv[++i]);
continue;
}
if (strcmp(arg, "--force_multiple") == 0 && i+1 < argc) {
force_multiple = atoi(argv[++i]);
continue;
}
#if defined(TOKUDB)
if (strcmp(arg, "--checkpoint_period") == 0 && i+1 < argc) {
checkpoint_period = atoi(argv[++i]);
continue;
}
#endif
assert(0);
}
int r;
char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
r = system(rm_cmd); assert(r == 0);
r = mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
// create and open the env
DB_ENV *db_env = NULL;
r = db_env_create(&db_env, 0); assert(r == 0);
if (!do_txn)
db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
if (cachesize) {
const u_int64_t gig = 1 << 30;
r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
}
#if defined(TOKUDB)
r = db_env->set_generate_row_callback_for_put(db_env, my_generate_row_for_put); assert(r == 0);
#endif
r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
if (checkpoint_period) {
r = db_env->checkpointing_set_period(db_env, checkpoint_period); assert(r == 0);
u_int32_t period;
r = db_env->checkpointing_get_period(db_env, &period); assert(r == 0 && period == checkpoint_period);
}
#endif
// create the db
DB *dbs[ndbs];
for (i = 0; i < ndbs; i++) {
DB *db = NULL;
r = db_create(&db, db_env, 0); assert(r == 0);
DB_TXN *create_txn = NULL;
if (do_txn) {
r = db_env->txn_begin(db_env, NULL, &create_txn, 0); assert(r == 0);
}
if (pagesize) {
r = db->set_pagesize(db, pagesize); assert(r == 0);
}
char db_filename[32]; sprintf(db_filename, "test%d", i);
r = db->open(db, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
#if defined(TOKUDB)
DESCRIPTOR_S new_descriptor;
int index_num = htonl(i);
new_descriptor.dbt.data = &index_num;
new_descriptor.dbt.size = sizeof i;
r = db->change_descriptor(db, create_txn, &new_descriptor.dbt, 0); assert(r == 0);
#else
db->app_private = (void *) (intptr_t) i;
if (i > 0) {
r = dbs[0]->associate(dbs[0], create_txn, db, my_secondary_key, 0); assert(r == 0);
}
#endif
if (do_txn) {
r = create_txn->commit(create_txn, 0); assert(r == 0);
}
dbs[i] = db;
}
// insert all rows
struct table table;
table_init(&table, ndbs, dbs, 4 * 8, 4 * 8);
insert_all(db_env, &table, rows, rows_per_txn, key_range, rows_per_report, do_txn);
table_destroy(&table);
// shutdown
for (i = 0; i < ndbs; i++) {
DB *db = dbs[i];
r = db->close(db, 0); assert(r == 0); db = NULL;
}
r = db_env->close(db_env, 0); assert(r == 0); db_env = NULL;
return 0;
}
/*
* This one handles all table-related commands
* ipfw table NAME create ...
* ipfw table NAME modify ...
* ipfw table NAME destroy
* ipfw table NAME swap NAME
* ipfw table NAME lock
* ipfw table NAME unlock
* ipfw table NAME add addr[/masklen] [value]
* ipfw table NAME add [addr[/masklen] value] [addr[/masklen] value] ..
* ipfw table NAME delete addr[/masklen] [addr[/masklen]] ..
* ipfw table NAME lookup addr
* ipfw table {NAME | all} flush
* ipfw table {NAME | all} list
* ipfw table {NAME | all} info
* ipfw table {NAME | all} detail
*/
void
ipfw_table_handler(int ac, char *av[])
{
int do_add, is_all;
int atomic, error, tcmd;
ipfw_xtable_info i;
ipfw_obj_header oh;
char *tablename;
uint32_t set;
void *arg;
memset(&oh, 0, sizeof(oh));
is_all = 0;
if (co.use_set != 0)
set = co.use_set - 1;
else
set = 0;
ac--; av++;
NEED1("table needs name");
tablename = *av;
if (table_check_name(tablename) == 0) {
table_fill_ntlv(&oh.ntlv, *av, set, 1);
oh.idx = 1;
} else {
if (strcmp(tablename, "all") == 0)
is_all = 1;
else
errx(EX_USAGE, "table name %s is invalid", tablename);
}
ac--; av++;
NEED1("table needs command");
tcmd = get_token(tablecmds, *av, "table command");
/* Check if atomic operation was requested */
atomic = 0;
if (tcmd == TOK_ATOMIC) {
ac--; av++;
NEED1("atomic needs command");
tcmd = get_token(tablecmds, *av, "table command");
switch (tcmd) {
case TOK_ADD:
break;
default:
errx(EX_USAGE, "atomic is not compatible with %s", *av);
}
atomic = 1;
}
switch (tcmd) {
case TOK_LIST:
case TOK_INFO:
case TOK_DETAIL:
case TOK_FLUSH:
break;
default:
if (is_all != 0)
errx(EX_USAGE, "table name required");
}
switch (tcmd) {
case TOK_ADD:
case TOK_DEL:
do_add = **av == 'a';
ac--; av++;
table_modify_record(&oh, ac, av, do_add, co.do_quiet,
co.do_quiet, atomic);
break;
case TOK_CREATE:
ac--; av++;
table_create(&oh, ac, av);
break;
case TOK_MODIFY:
ac--; av++;
table_modify(&oh, ac, av);
break;
case TOK_DESTROY:
if (table_destroy(&oh) != 0)
err(EX_OSERR, "failed to destroy table %s", tablename);
break;
case TOK_FLUSH:
if (is_all == 0) {
if ((error = table_flush(&oh)) != 0)
err(EX_OSERR, "failed to flush table %s info",
tablename);
} else {
error = tables_foreach(table_flush_one, &oh, 1);
if (error != 0)
err(EX_OSERR, "failed to flush tables list");
}
break;
case TOK_SWAP:
ac--; av++;
NEED1("second table name required");
table_swap(&oh, *av);
break;
case TOK_LOCK:
case TOK_UNLOCK:
table_lock(&oh, (tcmd == TOK_LOCK));
break;
case TOK_DETAIL:
case TOK_INFO:
arg = (tcmd == TOK_DETAIL) ? (void *)1 : NULL;
if (is_all == 0) {
if ((error = table_get_info(&oh, &i)) != 0)
err(EX_OSERR, "failed to request table info");
table_show_info(&i, arg);
} else {
error = tables_foreach(table_show_info, arg, 1);
if (error != 0)
err(EX_OSERR, "failed to request tables list");
}
break;
case TOK_LIST:
if (is_all == 0) {
ipfw_xtable_info i;
if ((error = table_get_info(&oh, &i)) != 0)
err(EX_OSERR, "failed to request table info");
table_show_one(&i, NULL);
} else {
error = tables_foreach(table_show_one, NULL, 1);
if (error != 0)
err(EX_OSERR, "failed to request tables list");
}
break;
case TOK_LOOKUP:
ac--; av++;
table_lookup(&oh, ac, av);
break;
}
}
/* callback to edit the current job table used by clockwork */
G_MODULE_EXPORT void
on_edit_jobs (GtkObject *object, gpointer user_data)
{
TABLE clockcf=NULL;
char *clockpurl, *jobpurl, jobpurl_t[1000];
int r;
/* Check clockwork is running to see if we can read its configuration */
if (uicollect_is_clockwork_running(NULL, NULL, NULL, NULL, 1)) {
/* Get the latest configuration from the local running clockwork */
clockpurl = util_strjoin("http://localhost:", HTTPD_PORT_HTTP_STR,
"/cftsv", NULL);
clockcf = route_tread(clockpurl, NULL);
if ( ! clockcf ) {
elog_printf(DIAG, "Unable to read clockwork configuration "
"(%s), although it is running; possibly security "
"is an issue", clockpurl);
elog_printf(FATAL, "<big><b>Unable to Load Collector "
"Configuration</b></big>\n"
"The collector is running but the configuration "
"can't be read. Check Habitat's security "
"configuration");
nfree(clockpurl);
return;
}
nfree(clockpurl);
r = table_search(clockcf, "name", "jobs");
if (r == -1) {
elog_printf(DIAG, "Clockwork configuration read but 'jobs' "
"declaration is missing");
elog_printf(FATAL, "<big><b>Unable to Load Collector "
"Configuration</b></big>\n"
"The collector does not have a configured job "
"table. Please check your configuration");
table_destroy(clockcf);
return;
}
jobpurl = table_getcurrentcell(clockcf, "value");
} else {
/* Clockwork is not running, so find the job table from the
* current configuration (the jobs directive). This has a flaw as
* clockwork may be started manually with a job switch (-j or -J) */
jobpurl = cf_getstr(iiab_cf, "jobs");
if ( ! jobpurl) {
elog_printf(FATAL, "Unable to load collection jobs, as there "
"was no configuration directive.\n\n"
"Please specify -j, -J or set the directive `jobs' "
"in the configuration file to the route containing "
"a job table. \n\n"
"For example, `jobs=file:/etc/jobs.norm' "
"will look for the file /etc/jobs.norm");
return;
}
}
/* read the job table */
r = route_expand(jobpurl_t, jobpurl, "NOJOB", 0);
if (r == -1 || jobpurl_t[0] == '\0') {
elog_printf(FATAL, "Unable to load collection jobs, as there are "
"no valid configuration directives in the table %s/%s. "
"Please specify -j, -J or set the directive `jobs' in "
"the configuration file to the route containing a job "
"table. For example, `jobs=file:/etc/jobs.norm' "
"will look for the file /etc/jobs.norm",
jobpurl, jobpurl_t);
return;
}
uiedit_load_route(jobpurl_t, "Collection Jobs");
if (clockcf)
table_destroy(clockcf);
}
/* run though a series of tests for the current cascade configuration */
void test_cascade(enum cascade_fn mode,
char *mode_label,
char *tab_sing,
char *tab_singinfo,
char *tab_singinfokey,
char *tab_mult,
char *tab_multinfo,
char *tab_multinfokey,
char *result_sing,
char *result_singkey,
char *result_mult,
char *result_multkey)
{
int r, resseq, resoff;
ROUTE resrt, samprt;
CASCADE *cas;
char *buf1, *resbuf1, *wantbuf1;
TABLE tab1, restab1, wanttab1;
time_t modt;
/* [1] run cascade aggregation on empty tables */
tab1 = table_create();
restab1 = cascade_aggregate(mode, tab1);
if (restab1)
elog_die(FATAL, "[1] should return NULL when aggregating an "
"empty table");
/* [2] aggregate a single sample table, no info, no key */
buf1 = xnstrdup(tab_sing);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
wanttab1 = table_create();
wantbuf1 = xnstrdup(result_sing);
table_scan(wanttab1, wantbuf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(wanttab1, wantbuf1);
table_rminfo(wanttab1, "info"); /* remove info line */
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[2a] can't aggregate table: "
"mode %s, single sample, no info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
wantbuf1 = table_outtable(wanttab1);
if (strcmp(wantbuf1, resbuf1))
elog_die(FATAL, "[2b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, wantbuf1);
nfree(buf1);
nfree(resbuf1);
nfree(wantbuf1);
table_destroy(tab1);
table_destroy(restab1);
table_destroy(wanttab1);
/* [3] aggregate a single sample table, with info but no key */
tab1 = table_create();
buf1 = xnstrdup(tab_singinfo);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[3a] can't aggregate table: "
"mode %s, single sample, with info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_sing, resbuf1))
elog_die(FATAL, "[3b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_sing);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [4] aggregate a single sample table, with info and key */
tab1 = table_create();
buf1 = xnstrdup(tab_singinfokey);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[4a] can't aggregate table: "
"mode %s, single sample, with info and key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_singkey, resbuf1))
elog_die(FATAL, "[4b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_singkey);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [5] aggregate a multi sample table, no info, no key */
tab1 = table_create();
buf1 = xnstrdup(tab_mult);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
wanttab1 = table_create();
wantbuf1 = xnstrdup(result_mult);
table_scan(wanttab1, wantbuf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(wanttab1, wantbuf1);
table_rminfo(wanttab1, "info"); /* remove info line */
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[5a] can't aggregate table: "
"mode %s, multi sample, no info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
wantbuf1 = table_outtable(wanttab1);
if (strcmp(wantbuf1, resbuf1))
elog_die(FATAL, "[5b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, wantbuf1);
nfree(buf1);
nfree(resbuf1);
nfree(wantbuf1);
table_destroy(tab1);
table_destroy(restab1);
table_destroy(wanttab1);
/* [6] aggregate a multi sample table, with info but no key */
tab1 = table_create();
buf1 = xnstrdup(tab_multinfo);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[6a] can't aggregate table: "
"mode %s, multi sample, with info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_mult, resbuf1))
elog_die(FATAL, "[6b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_mult);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [7] aggregate a multi sample table, with info and key */
tab1 = table_create();
buf1 = xnstrdup(tab_multinfokey);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
restab1 = cascade_aggregate(mode, tab1);
if ( ! restab1)
elog_die(FATAL, "[7a] can't aggregate table: "
"mode %s, multi sample, with info and key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_multkey, resbuf1))
elog_die(FATAL, "[7b] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_multkey);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/*
* And now the route based methods.
* We run through the same tests using routes to store the samples
* also the results
*/
/* unlink previous storage */
unlink(RS_SAMPFILE);
unlink(RS_RESFILE);
/* [8] create sample and result routes */
resrt = route_open(RS_RESPURL, "Output of testing results", NULL, 20);
if ( ! resrt )
elog_die(FATAL, "[8a] Can't open result route");
samprt = route_open(RS_SAMPPURL, "Samples under test", NULL, 20);
if ( ! samprt )
elog_die(FATAL, "[8b] Can't open result route");
/* [9] run cascade on an empty ring and sample several times
* where there is no change */
cas = cascade_init(mode, RS_SAMPPURL);
if ( ! cas)
elog_die(FATAL, "[9a] can't start cascade");
r = cascade_sample(cas, out, err);
if ( r )
elog_die(FATAL, "[9b] cascade sample failed");
r = cascade_sample(cas, out, err);
if ( r )
elog_die(FATAL, "[9c] cascade sample failed");
/* [10] add tab_sing to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_sing);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
wanttab1 = table_create();
wantbuf1 = xnstrdup(result_sing);
table_scan(wanttab1, wantbuf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(wanttab1, wantbuf1);
table_rminfo(wanttab1, "info"); /* remove info line */
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[10a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[10b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[10c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[10d] can't read result ring: "
"mode %s, single sample, no info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
wantbuf1 = table_outtable(wanttab1);
if (strcmp(wantbuf1, resbuf1))
elog_die(FATAL, "[10e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, wantbuf1);
nfree(buf1);
nfree(resbuf1);
nfree(wantbuf1);
table_destroy(tab1);
table_destroy(restab1);
table_destroy(wanttab1);
/* [11] add tab_singinfo to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_singinfo);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[11a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[11b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[11c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[11d] can't read result ring: "
"mode %s, single sample, with info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_sing, resbuf1))
elog_die(FATAL, "[11e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_sing);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [12] add tab_singinfokey to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_singinfokey);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[12a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[12b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[12c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[12d] can't read result ring: "
"mode %s, single sample, with info and key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_singkey, resbuf1))
elog_die(FATAL, "[12e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_sing);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [13] add tab_mult to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_mult);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
wanttab1 = table_create();
wantbuf1 = xnstrdup(result_mult);
table_scan(wanttab1, wantbuf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(wanttab1, wantbuf1);
table_rminfo(wanttab1, "info"); /* remove info line */
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[13a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[13b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[13c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[13d] can't read result ring: "
"mode %s, multi sample, no info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
wantbuf1 = table_outtable(wanttab1);
if (strcmp(wantbuf1, resbuf1))
elog_die(FATAL, "[13e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, wantbuf1);
nfree(buf1);
nfree(resbuf1);
nfree(wantbuf1);
table_destroy(tab1);
table_destroy(restab1);
table_destroy(wanttab1);
/* [14] add tab_multinfo to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_multinfo);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[14a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[14b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[14c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[14d] can't read result ring: "
"mode %s, multi sample, with info, no key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_mult, resbuf1))
elog_die(FATAL, "[14e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_mult);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* [15] add tab_multinfokey to table and sample */
tab1 = table_create();
buf1 = xnstrdup(tab_multinfokey);
table_scan(tab1, buf1, "\t", TABLE_SINGLESEP, TABLE_HASCOLNAMES,
TABLE_HASRULER);
table_freeondestroy(tab1, buf1);
r = route_twrite(samprt, tab1);
if ( r < 0 )
elog_die(FATAL, "[15a] add table failed");
r = cascade_sample(cas, resrt, err);
if ( r )
elog_die(FATAL, "[15b] cascade sample failed, mode %s", mode_label);
r = route_tell(resrt, &resseq, &resoff, &modt);
if ( !r )
elog_die(FATAL, "[15c] can't route_tell(), mode %s", mode_label);
restab1 = route_seektread(resrt, resseq, 0);
if ( ! restab1)
elog_die(FATAL, "[15d] can't read result ring: "
"mode %s, multi sample, with info and key", mode_label);
table_rmcol(restab1, "_dur");
table_rmcol(restab1, "_seq");
table_replaceinfocell(restab1, "type", "_time", "i32");
table_replaceinfocell(restab1, "key", "_time", "-");
buf1 = table_outtable(tab1);
resbuf1 = table_outtable(restab1);
if (strcmp(result_multkey, resbuf1))
elog_die(FATAL, "[15e] aggregation failed, mode %s:-\n"
"--- in ---\n%s\n--- out ---\n%s\n--- want ---\n%s",
mode_label, buf1, resbuf1, result_multkey);
nfree(buf1);
nfree(resbuf1);
table_destroy(tab1);
table_destroy(restab1);
/* shutdown */
cascade_fini(cas);
route_close(resrt);
route_close(samprt);
}
int main(int argc, char **argv) {
ROUTE err, saveroute;
int i;
TABLE tab;
char *str;
route_init(NULL, 0);
route_register(&rt_filea_method);
route_register(&rt_fileov_method);
route_register(&rt_stdin_method);
route_register(&rt_stdout_method);
route_register(&rt_stderr_method);
route_register(&rt_rs_method);
rs_init();
if ( ! elog_init(1, argv[0], NULL))
elog_die(FATAL, "Didn't initialise elog\n");
err = route_open("stderr", NULL, NULL, 0);
/* first lot of messages sent to the default places */
elog_send(INFO, "This is an eventlog test");
elog_send(INFO, NULL);
elog_send(INFO, "");
elog_send(INFO, "Event!!");
elog_send(DEBUG, "Event!!");
elog_send(WARNING, "Event!!");
elog_send(ERROR, "Event!!");
elog_send(INFO, "Event!!");
/* change origin */
elog_setorigin("etest");
elog_send(INFO, "test of set origin");
/* set one new purl route */
elog_setsevpurl(DEBUG, FILE1);
elog_send(INFO, "on screen");
elog_send(DEBUG, "in file");
elog_send(WARNING, "on screen");
/* set second identical purl route to reuse the previous one */
elog_setsevpurl(ERROR, FILE1);
if (elog_opendest[DEBUG].route !=
elog_opendest[ERROR].route)
route_die(err, "[13] didnt reuse already open DEBUG route\n");
elog_send(ERROR, "in file");
/* set identical below purl route */
if ( !elog_setbelowpurl(INFO, FILE1))
route_die(err, "[14] unable to setbelowpurl() file\n");
if (elog_opendest[DEBUG].route !=
elog_opendest[ERROR].route ||
elog_opendest[INFO].route !=
elog_opendest[ERROR].route)
route_die(err, "[14] didnt reuse already open ERROR route\n");
elog_send(DEBUG, "in file");
elog_send(INFO, "in file");
elog_send(WARNING, "on screen");
elog_send(ERROR, "in file");
elog_send(FATAL, "on screen");
/* set identical above purl route */
if ( !elog_setabovepurl(ERROR, FILE1))
route_die(err, "[19] unable to setabovepurl() file\n");
if (elog_opendest[ERROR].route !=
elog_opendest[INFO].route ||
elog_opendest[FATAL].route !=
elog_opendest[INFO].route)
route_die(err, "[19] didnt reuse already open INFO route\n");
elog_send(DEBUG, "in file");
elog_send(INFO, "in file");
elog_send(WARNING, "on screen");
elog_send(ERROR, "in file");
elog_send(FATAL, "in file");
/* set identical all purl route */
saveroute = elog_opendest[DEBUG].route;
if ( !elog_setallpurl(FILE1))
route_die(err, "[24] unable to setallpurl() file\n");
for (i=0; i < ELOG_NSEVERITIES; i++)
if (elog_opendest[i].route != saveroute)
route_die(err, "[24] didnt reuse already open %s route\n",
elog_sevstring[i]);
elog_send(DEBUG, "in file");
elog_send(INFO, "in file");
elog_send(WARNING, "in file");
elog_send(ERROR, "in file");
elog_send(FATAL, "in file");
/* set one different purl - timestore that we currently have to set up
* ourselves */
saveroute = route_open(RS1, "event log test", NULL, 10);
if ( ! saveroute)
route_die(err, "[29] unable to create/open timestore\n");
route_close(saveroute);
if ( ! elog_setsevpurl(INFO, RS1))
route_die(err, "[29] unable to setsevpurl() timestore\n");
if (elog_opendest[INFO].route ==
elog_opendest[WARNING].route)
route_die(err, "[29] different route same as WARNING\n");
elog_send(DEBUG, "in file");
elog_send(INFO, "in timestore");
elog_send(WARNING, "in file");
elog_send(ERROR, "in file");
elog_send(FATAL, "in file");
/* set one different route */
#if 0
elog_send(INFO, "");
elog_send(INFO, "Event!!");
elog_send(DEBUG, "Event!!");
elog_send(WARNING, "Event!!");
elog_send(ERROR, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
elog_send(INFO, "Event!!");
#endif
/* change format */
elog_setsevroute(WARNING, err);
elog_setformat(WARNING, "%s %s");
elog_send(WARNING, "still works??");
/* safe logging */
elog_safeprintf(INFO, "This is an eventlog test 35");
elog_safeprintf(INFO, NULL);
elog_safeprintf(INFO, "");
elog_safeprintf(INFO, "Event!! 38");
elog_safeprintf(DEBUG, "Event!! 39");
elog_safeprintf(WARNING, "Event!! 40");
elog_safeprintf(ERROR, "Event!! 41");
elog_safeprintf(INFO, "Event!!");
/* print the status out */
tab = elog_getstatus();
str = table_printcols_a(tab, elog_colnames);
printf("%s\n", str);
nfree(str);
table_destroy(tab);
rs_fini();
elog_fini();
route_close(err);
route_fini();
exit(0);
}
void ht_destroy(Hashtable *ht)
{
table_destroy(ht->table, ht->real_capacity, ht->free_value);
pcs_free(ht);
}
void main(int argc, char **argv) {
if ( argc != 3 ) {
printf("Usage: %s (filename) (number of entries)\n", argv[0]);
exit(1);
}
FILE *fp;
char buf[12];
table t;
mpz_t u;
ulint i, count, invalid, found;
unsigned long long phonenum;
unsigned long long *numlist;
struct timeval starttime, endtime;
ulint starttime_us, endtime_us;
if ( (fp = fopen(argv[1], "r")) == NULL ) {
printf("Could not open file: %s\n", argv[1]);
}
sscanf(argv[2], "%ld", &count);
numlist = (unsigned long long *)calloc( count, sizeof(unsigned long long));
mpz_init(u);
mpz_set_ui(u, 1000 * 1000);
mpz_mul_ui(u, u, 10000);
table_init(&t, count, u);
srand(time(NULL));
i = 0;
invalid = 0;
while ( fgets(buf, 12, fp) != NULL ) {
sscanf(buf, "%lld\n", &phonenum);
numlist[i] = phonenum;
if (rand() <= PROB_INVALID * RAND_MAX) {
numlist[i] = numlist[i] / 4;
invalid++;
}
table_insert(&t, phonenum);
i++;
}
fclose(fp);
printf("Building table...\n");
table_build(&t);
printf("Done inserting %ld numbers (%ld invalids)\n", i, invalid);
i = 0;
found = 0;
invalid = 0;
gettimeofday(&starttime, NULL);
while ( i < count ) {
if ( table_find(&t, numlist[i]) ) {
found++;
} else {
invalid++;
}
i++;
}
gettimeofday(&endtime, NULL);
starttime_us = starttime.tv_sec*1000000 + starttime.tv_usec;
endtime_us = endtime.tv_sec*1000000 + endtime.tv_usec;
printf("Found %ld numbers, rejected %ld\n", found, invalid);
printf("Total find time: %ld us\n", endtime_us-starttime_us);
free(numlist);
table_destroy(&t);
mpz_clear(u);
}
