static tb_void_t tb_demo_test_cstr_h(tb_size_t index)
{
// the count
tb_size_t count = 1000000;
// save func
g_func_indx = index;
g_func_prev = tb_item_func_str(tb_true);
// the func
tb_item_func_t func = g_func_prev;
func.hash = tb_demo_test_hash_func;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 1, count, func);
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_char_t s[256] = {0};
tb_hong_t t = tb_mclock();
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random();
// format it
tb_snprintf(s, sizeof(s) - 1, "%ld", value);
// set value to filter
if (!tb_bloom_filter_set(filter, s))
{
// repeat++
r++;
}
}
t = tb_mclock() - t;
// trace
tb_trace_i("cstr: index: %lu, repeat: %lu, time: %lld ms", index, r, t);
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_void_t tb_demo_test_uint32_h(tb_size_t index)
{
// the count
tb_size_t count = 1000000;
// save func
g_func_indx = index;
g_func_prev = tb_item_func_uint32();
// the func
tb_item_func_t func = g_func_prev;
func.hash = tb_demo_test_hash_func;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 1, count, func);
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_hong_t t = tb_mclock();
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random_range(tb_random_generator(), 0, TB_MAXU32);
// set value to filter
if (!tb_bloom_filter_set(filter, (tb_cpointer_t)value))
{
// repeat++
r++;
}
}
t = tb_mclock() - t;
// trace
tb_trace_i("uint32: index: %lu, repeat: %lu, time: %lld ms", index, r, t);
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_void_t tb_demo_test_cstr_p()
{
// the count
tb_size_t count = 10000000;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_01, 3, count, tb_item_func_str(tb_true));
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_char_t s[256] = {0};
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random();
// format it
tb_snprintf(s, sizeof(s) - 1, "%ld", value);
// set value to filter
if (!tb_bloom_filter_set(filter, s))
{
// repeat++
r++;
}
}
// trace
#ifdef TB_CONFIG_TYPE_FLOAT
tb_trace_i("cstr: count: %lu, repeat: %lu, repeat_p ~= p: %lf", count, r, (tb_double_t)r / count);
#else
tb_trace_i("cstr: count: %lu, repeat: %lu", count, r);
#endif
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_bool_t tb_demo_spider_task_done(tb_demo_spider_t* spider, tb_char_t const* iurl, tb_bool_t* full)
{
// check
tb_assert_and_check_return_val(spider && iurl, tb_false);
// killed?
tb_check_return_val(TB_STATE_OK == tb_atomic_get(&spider->state), tb_false);
// only for home?
if (spider->home_only && !tb_stristr(iurl, spider->home_domain))
{
// trace
tb_trace_d("task: done: %s: skip", iurl);
return tb_true;
}
// enter
tb_spinlock_enter(&spider->lock);
// done
tb_bool_t ok = tb_false;
tb_size_t size = 0;
tb_demo_spider_task_t* task = tb_null;
tb_bool_t repeat = tb_false;
do
{
// check
tb_assert_and_check_break(spider->filter && spider->pool);
// the task count
size = tb_fixed_pool_size(spider->pool);
// make the output url
if (!tb_demo_spider_make_ourl(spider, iurl, spider->ourl, sizeof(spider->ourl) - 1)) break;
// have been done already?
if (!tb_bloom_filter_set(spider->filter, spider->ourl))
{
// trace
tb_trace_d("task: size: %lu, done: %s: repeat", size, iurl);
// ok
ok = tb_true;
repeat = tb_true;
break;
}
// trace
tb_trace_d("task: size: %lu, done: %s: ..", size, iurl);
// full?
tb_check_break(size < TB_DEMO_SPIDER_TASK_MAXN);
// make task
task = (tb_demo_spider_task_t*)tb_fixed_pool_malloc0(spider->pool);
tb_assert_and_check_break(task);
// init task
task->spider = spider;
tb_strlcpy(task->iurl, iurl, sizeof(task->iurl) - 1);
tb_strlcpy(task->ourl, spider->ourl, sizeof(task->ourl) - 1);
// ok
ok = tb_true;
} while (0);
// leave
tb_spinlock_leave(&spider->lock);
// ok?
if (ok && !repeat)
{
// done
ok = tb_false;
do
{
// check
tb_assert_and_check_break(task);
// killed?
tb_check_break(TB_STATE_OK == tb_atomic_get(&spider->state));
// repeat?
if (tb_file_info(task->ourl, tb_null))
{
// trace
tb_trace_d("task: size: %lu, done: %s: repeat", size, iurl);
// ok
ok = tb_true;
repeat = tb_true;
break;
}
// done task
ok = tb_transfer_pool_done(tb_transfer_pool(), task->iurl, task->ourl, 0, spider->limited_rate, tb_demo_spider_task_save, tb_demo_spider_task_ctrl, task);
tb_assert_and_check_break(ok);
} while (0);
}
// repeat or failed?
if (repeat || !ok)
{
// exit task
if (task) tb_demo_spider_task_exit(task);
task = tb_null;
// failed?
if (!full && !ok)
{
// trace
tb_trace_e("task: size: %lu, done: %s: post failed", size, iurl);
}
// save full
if (full) *full = size < TB_DEMO_SPIDER_TASK_MAXN? tb_false : tb_true;
}
// ok?
return ok;
}
