void test_nopolicy (flux_t *h)
{
flux_reduce_t *r;
int i, errors;
clear_counts ();
ok ((r = flux_reduce_create (h, reduce_ops, 0., NULL, 0)) != NULL,
"nopolicy: flux_reduce_create works");
errors = 0;
for (i = 0; i < 100; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
errors++;
}
ok (errors == 0,
"nopolicy: flux_reduce_append added 100 items in batch 0");
cmp_ok (forward_calls, "==", 0,
"nopolicy: op.forward not called as we are rank 0");
cmp_ok (reduce_calls, "==", 0,
"nopolicy: op.reduce not called as we have no flush policy");
cmp_ok (sink_calls, "==", 100,
"nopolicy: op.sink called 100 times");
cmp_ok (sink_items, "==", 100,
"nopolicy: op.sink processed 100 items");
flux_reduce_destroy (r);
}
void reset( void )
{
reset_cbuff();
clear_counts();
#ifdef DEBUG
dump_cbuff();
#endif
}
void read_stringpairs() {
char *my_string = NULL, *token1, *token2;
char str1[1024], str2[1024];
size_t nbytes;
int bytes_read;
while ((bytes_read = getline(&my_string, &nbytes, stdin)) != -1) {
if (g_input_format == INPUT_FORMAT_L2P) {
if (sscanf(my_string, "%1023s %1023s", &str1[0], &str2[0]) == 2)
add_string_pair(str1, str2);
} else if (g_input_format == INPUT_FORMAT_NEWS) {
token1 = strtok(my_string, "\t\n");
token2 = strtok(NULL, "\t\n");
if (token1 != NULL && token2 != NULL)
add_string_pair(token1, token2);
}
}
clear_counts();
initial_align();
}
void test_hwm (flux_t *h)
{
flux_reduce_t *r;
int i, errors;
unsigned int hwm;
clear_counts ();
ok ((r = flux_reduce_create (h, reduce_ops, 0., NULL,
FLUX_REDUCE_HWMFLUSH)) != NULL,
"hwm: flux_reduce_create works");
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 0,
"hwm: hwm is initially zero");
/* batch 0 is a training batch.
* It looks just like no policy.
*/
errors = 0;
for (i = 0; i < 100; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
errors++;
}
ok (errors == 0,
"hwm.0: flux_reduce_append added 100 items");
cmp_ok (reduce_calls, "==", 0,
"hwm.0: op.reduce not called (training)");
cmp_ok (sink_calls, "==", 100,
"hwm.0: op.sink called 100 times");
cmp_ok (sink_items, "==", 100,
"hwm.0: op.sink processed 100 items");
clear_counts ();
/* batch 1 has a hwm. Put in one short of hwm items.
*/
errors = 0;
for (i = 0; i < 99; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 1) < 0)
errors++;
}
ok (errors == 0,
"hwm.1: flux_reduce_append added 99 items");
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 100,
"hwm.0: hwm is 100");
cmp_ok (reduce_calls, "==", 98,
"hwm.1: op.reduce called 98 times");
cmp_ok (sink_calls, "==", 0,
"hwm.1: op.sink not called yet");
/* Now finish batch 1 with one item. Everything should go thru.
*/
ok (flux_reduce_append (r, xstrdup ("hi"), 1) == 0,
"hwm.1: flux_reduce_append added 1 item");
cmp_ok (reduce_calls, "==", 99,
"hwm.1: op.reduce called");
cmp_ok (sink_calls, "==", 1,
"hwm.1: op.sink called 1 time");
cmp_ok (sink_items, "==", 100,
"hwm.1: op.sink handled 100 items");
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 100,
"hwm.1: hwm is 100");
clear_counts ();
/* Straggler test
* Start batch 2, then append one item from batch 1.
* This should cause last hwm to be recomputed to be 101 instead of 100.
* Straggler should immediately be sinked.
*/
ok (flux_reduce_append (r, xstrdup ("hi"), 2) == 0,
"hwm.2: flux_reduce_append added 1 item");
cmp_ok (reduce_calls, "==", 0,
"hwm.2: op.reduce not called");
cmp_ok (sink_calls, "==", 0,
"hwm.2: op.sink not called");
ok (flux_reduce_append (r, xstrdup ("hi"), 1) == 0,
"hwm.1: flux_reduce_append added 1 straggler");
cmp_ok (reduce_calls, "==", 0,
"hwm.1: op.reduce not called");
cmp_ok (sink_calls, "==", 1,
"hwm.1: op.sink called 1 time");
cmp_ok (sink_items, "==", 1,
"hwm.1: op.sink handled 1 item");
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 101,
"hwm.1: hwm is 101");
sink_items = sink_calls = 0; // don't count batch 1 straggler below
/* At this point we have one batch 2 item in queue.
* Put in 99 more and we should be one short of 101 hwm.
*/
errors = 0;
for (i = 0; i < 99; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 2) < 0)
errors++;
}
ok (errors == 0,
"hwm.2: flux_reduce_append added 99 items");
cmp_ok (reduce_calls, "==", 99,
"hwm.2: op.reduce called 99 times");
cmp_ok (sink_calls, "==", 0,
"hwm.2: op.sink not called yet");
ok (flux_reduce_append (r, xstrdup ("hi"), 2) == 0,
"hwm.2: flux_reduce_append added 1 item");
cmp_ok (sink_calls, "==", 1,
"hwm.2: op.sink called 1 time");
cmp_ok (sink_items, "==", 101,
"hwm.2: op.sink handled 101 items");
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 101,
"hwm.2: hwm is 101");
clear_counts ();
/* Manually set the hwm to 10.
* Append 20 items to batch 3.
* Reduce is called on the first set of 10.
* The second set of 10 will be immediately flushed.
* Put in one batch 4 item and verify the HWM is still 10.
*/
hwm = 10;
ok (flux_reduce_opt_set (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0,
"hwm.3: hwm set to 10");
errors = 0;
for (i = 0; i < 20; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 3) < 0)
errors++;
}
ok (errors == 0,
"hwm.3: flux_reduce_append added 20 items");
cmp_ok (reduce_calls, "==", 9,
"hwm.3: op.reduce called 9 times");
cmp_ok (sink_calls, "==", 11,
"hwm.3: op.sink called 11 times");
cmp_ok (sink_items, "==", 20,
"hwm.3: op.sink handled 20 items");
ok (flux_reduce_append (r, xstrdup ("hi"), 4) == 0,
"hwm.4: flux_reduce_append added one item");
hwm = 0;
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_HWM, &hwm, sizeof (hwm)) == 0
&& hwm == 10,
"hwm.4: hwm is still 10");
flux_reduce_destroy (r);
}
void test_timed (flux_t *h)
{
flux_reduce_t *r;
int i, errors;
double timeout;
clear_counts ();
ok ((r = flux_reduce_create (h, reduce_ops, 0.1, NULL,
FLUX_REDUCE_TIMEDFLUSH)) != NULL,
"timed: flux_reduce_create works");
if (!r)
BAIL_OUT();
ok (flux_reduce_opt_get (r, FLUX_REDUCE_OPT_TIMEOUT, &timeout,
sizeof (timeout)) == 0 && timeout == 0.1,
"timed: flux_reduce_opt_get TIMEOUT returned timeout");
/* Append 100 items in batch 0 before starting reactor.
* Reduction occurs at each append.
* Nothing should be sinked.
*/
errors = 0;
for (i = 0; i < 100; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 0) < 0)
errors++;
}
ok (errors == 0,
"timed.0: flux_reduce_append added 100 items");
cmp_ok (reduce_calls, "==", 99,
"timed.0: op.reduce called 99 times");
cmp_ok (sink_calls, "==", 0,
"timed.0: op.sink called 0 times");
/* Start reactor so timeout handler can run.
* It should fire once and sink all items in one sink call.
*/
ok (flux_reactor_run (flux_get_reactor (h), 0) == 0,
"timed.0: reactor completed normally");
cmp_ok (sink_calls, "==", 1,
"timed.0: op.sink called 1 time");
cmp_ok (sink_items, "==", 100,
"timed.0: op.sink processed 100 items");
clear_counts ();
/* Now append one more item to batch 0.
* It should be immediately flushed.
*/
ok (flux_reduce_append (r, xstrdup ("hi"), 0) == 0,
"timed.0: flux_reduce_append added 1 more item");
cmp_ok (reduce_calls, "==", 0,
"timed.0: op.reduce not called");
cmp_ok (sink_calls, "==", 1,
"timed.0: op.sink called 1 time");
cmp_ok (sink_items, "==", 1,
"timed.0: op.sink processed 1 items");
clear_counts ();
/* Append 100 items to batch 1.
* It should behave like the first batch.
*/
errors = 0;
for (i = 0; i < 100; i++) {
if (flux_reduce_append (r, xstrdup ("hi"), 1) < 0)
errors++;
}
ok (errors == 0,
"timed.1: flux_reduce_append added 100 items");
cmp_ok (reduce_calls, "==", 99,
"timed.1: op.reduce called 99 times");
cmp_ok (sink_calls, "==", 0,
"timed.1: op.sink called 0 times");
/* Start reactor so timeout handler can run.
* It should fire once and sink all items in one sink call.
*/
ok (flux_reactor_run (flux_get_reactor (h), 0) == 0,
"timed.1: reactor completed normally");
cmp_ok (sink_calls, "==", 1,
"timed.1: op.sink called 1 time");
cmp_ok (sink_items, "==", 100,
"timed.1: op.sink processed 100 items");
flux_reduce_destroy (r);
}
uint8_t GLZAformat(size_t insize, uint8_t * inbuf, size_t * outsize_ptr, uint8_t ** outbuf) {
const uint32_t CHARS_TO_WRITE = 0x40000;
uint8_t this_char, prev_char, next_char, user_cap_encoded, user_cap_lock_encoded, user_delta_encoded, stride;
uint8_t *in_char_ptr, *end_char_ptr, *out_char_ptr;
uint32_t i, j, k;
uint32_t num_AZ, num_az_pre_AZ, num_az_post_AZ, num_spaces;
uint32_t order_1_counts[0x100][0x100];
uint32_t symbol_counts[0x100];
double order_1_entropy, best_stride_entropy, saved_entropy[4];
// format byte: B0: cap encoded, B3:B1 = stride (0 - 4), B5:B4 = log2 delta length (0 - 2), B6: little endian
user_cap_encoded = 0;
user_cap_lock_encoded = 0;
user_delta_encoded = 0;
*outbuf = (uint8_t *)malloc(2 * insize + 1);
if (*outbuf == 0)
return(0);
end_char_ptr = inbuf + insize;
num_AZ = 0;
num_az_pre_AZ = 0;
num_az_post_AZ = 0;
num_spaces = 0;
if (insize > 4) {
in_char_ptr = inbuf;
this_char = *in_char_ptr++;
if (this_char == 0x20)
num_spaces++;
if ((this_char >= 'A') && (this_char <= 'Z')) {
num_AZ++;
next_char = *in_char_ptr;
if (((next_char >= 'a') && (next_char <= 'z')) || ((next_char >= 'A') && (next_char <= 'Z')))
num_az_post_AZ++;
}
while (in_char_ptr != end_char_ptr) {
this_char = *in_char_ptr++;
if (this_char == 0x20)
num_spaces++;
if ((this_char >= 'A') && (this_char <= 'Z')) {
num_AZ++;
prev_char = *(in_char_ptr - 2);
next_char = *in_char_ptr;
if (((next_char >= 'a') && (next_char <= 'z')) || ((next_char >= 'A') && (next_char <= 'Z')))
num_az_post_AZ++;
if (((prev_char >= 'a') && (prev_char <= 'z')) || ((prev_char >= 'A') && (prev_char <= 'Z')))
num_az_pre_AZ++;
}
}
}
out_char_ptr = *outbuf;
if (((num_AZ && (4 * num_az_post_AZ > num_AZ) && (num_az_post_AZ > num_az_pre_AZ)
&& (num_spaces > insize / 50)) && (user_cap_encoded != 1)) || (user_cap_encoded == 2)) {
#ifdef PRINTON
fprintf(stderr,"Converting textual data\n");
#endif
*out_char_ptr++ = 1;
in_char_ptr = inbuf;
while (in_char_ptr != end_char_ptr) {
if ((*in_char_ptr >= 'A') && (*in_char_ptr <= 'Z')) {
if (((*(in_char_ptr + 1) >= 'A') && (*(in_char_ptr + 1) <= 'Z') && (user_cap_lock_encoded != 1))
&& ((*(in_char_ptr + 2) < 'a') || (*(in_char_ptr + 2) > 'z'))) {
*out_char_ptr++ = 'B';
*out_char_ptr++ = *in_char_ptr++ + ('a' - 'A');
*out_char_ptr++ = *in_char_ptr++ + ('a' - 'A');
while ((*in_char_ptr >= 'A') && (*in_char_ptr <= 'Z'))
*out_char_ptr++ = *in_char_ptr++ + ('a' - 'A');
if ((*in_char_ptr >= 'a') && (*in_char_ptr <= 'z'))
*out_char_ptr++ = 'C';
}
else {
*out_char_ptr++ = 'C';
*out_char_ptr++ = *in_char_ptr++ + ('a' - 'A');
}
}
else if (*in_char_ptr >= 0xFE) {
*out_char_ptr++ = *in_char_ptr++;
*out_char_ptr++ = 0xFF;
}
else if (*in_char_ptr == 0xA) {
in_char_ptr++;
*out_char_ptr++ = 0xA;
*out_char_ptr++ = ' ';
}
else
*out_char_ptr++ = *in_char_ptr++;
}
}
else if ((user_delta_encoded != 1) && (insize > 4)) {
clear_counts(symbol_counts, order_1_counts);
for (i = 0 ; i < insize - 1 ; i++) {
symbol_counts[inbuf[i]]++;
order_1_counts[inbuf[i]][inbuf[i+1]]++;
}
symbol_counts[inbuf[insize-1]]++;
order_1_counts[inbuf[insize-1]][0x80]++;
order_1_entropy = calculate_order_1_entropy(symbol_counts, order_1_counts);
best_stride_entropy = order_1_entropy;
stride = 0;
for (k = 1 ; k <= 100 ; k++) {
if (insize <= (size_t)k)
break;
clear_counts(symbol_counts, order_1_counts);
if ((k == 2) | (k == 4)) {
for (i = 0 ; i < k ; i++) {
symbol_counts[inbuf[i]]++;
order_1_counts[inbuf[i]][0xFF & (inbuf[i+k] - inbuf[i])]++;
}
for (i = k ; i < (uint32_t)insize - k ; i++) {
symbol_counts[0xFF & (inbuf[i] - inbuf[i-k])]++;
order_1_counts[0xFF & (inbuf[i] - inbuf[i-k])][0xFF & (inbuf[i+k] - inbuf[i])]++;
}
for (i = (uint32_t)insize - k ; i < insize ; i++) {
symbol_counts[0xFF & (inbuf[i] - inbuf[i-k])]++;
order_1_counts[0xFF & (inbuf[i] - inbuf[i-k])][0x80]++;
}
order_1_entropy = calculate_order_1_entropy(symbol_counts, order_1_counts);
if ((order_1_entropy < 0.95 * best_stride_entropy) || ((stride != 0) && (order_1_entropy < best_stride_entropy))) {
stride = k;
best_stride_entropy = order_1_entropy;
}
}
else {
for (i = 0 ; i < k - 1 ; i++) {
symbol_counts[inbuf[i]]++;
order_1_counts[inbuf[i]][inbuf[i+1]]++;
}
symbol_counts[inbuf[k-1]]++;
order_1_counts[inbuf[k-1]][0xFF & (inbuf[k]-inbuf[0])]++;
uint8_t failed_test = 0;
i = k;
if (insize > 100000) {
uint32_t initial_test_size = 100000 + ((insize - 100000) >> 3);
while (i < initial_test_size) {
symbol_counts[0xFF & (inbuf[i] - inbuf[i-k])]++;
order_1_counts[0xFF & (inbuf[i] - inbuf[i-k])][0xFF & (inbuf[i+1] - inbuf[i+1-k])]++;
i++;
}
order_1_entropy = calculate_order_1_entropy(symbol_counts, order_1_counts);
if (order_1_entropy >= 1.05 * best_stride_entropy * (double)initial_test_size / (double)insize)
failed_test = 1;
}
if (failed_test == 0) {
while (i < insize - 1) {
symbol_counts[0xFF & (inbuf[i] - inbuf[i-k])]++;
order_1_counts[0xFF & (inbuf[i] - inbuf[i-k])][0xFF & (inbuf[i+1] - inbuf[i+1-k])]++;
i++;
}
symbol_counts[0xFF & (inbuf[insize-1] - inbuf[insize-1-k])]++;
order_1_counts[0xFF & (inbuf[insize-1] - inbuf[insize-1-k])][0x80]++;
order_1_entropy = calculate_order_1_entropy(symbol_counts, order_1_counts);
if ((order_1_entropy < 0.95 * best_stride_entropy) || ((stride != 0) && (order_1_entropy < best_stride_entropy))) {
stride = k;
best_stride_entropy = order_1_entropy;
}
}
}
}
