static tb_void_t tb_demo_test_cstr_h(tb_size_t index)
{
// the count
tb_size_t count = 1000000;
// save func
g_func_indx = index;
g_func_prev = tb_item_func_str(tb_true);
// the func
tb_item_func_t func = g_func_prev;
func.hash = tb_demo_test_hash_func;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 1, count, func);
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_char_t s[256] = {0};
tb_hong_t t = tb_mclock();
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random();
// format it
tb_snprintf(s, sizeof(s) - 1, "%ld", value);
// set value to filter
if (!tb_bloom_filter_set(filter, s))
{
// repeat++
r++;
}
}
t = tb_mclock() - t;
// trace
tb_trace_i("cstr: index: %lu, repeat: %lu, time: %lld ms", index, r, t);
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_void_t tb_demo_test_uint32_h(tb_size_t index)
{
// the count
tb_size_t count = 1000000;
// save func
g_func_indx = index;
g_func_prev = tb_item_func_uint32();
// the func
tb_item_func_t func = g_func_prev;
func.hash = tb_demo_test_hash_func;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 1, count, func);
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_hong_t t = tb_mclock();
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random_range(tb_random_generator(), 0, TB_MAXU32);
// set value to filter
if (!tb_bloom_filter_set(filter, (tb_cpointer_t)value))
{
// repeat++
r++;
}
}
t = tb_mclock() - t;
// trace
tb_trace_i("uint32: index: %lu, repeat: %lu, time: %lld ms", index, r, t);
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_void_t tb_demo_test_cstr_p()
{
// the count
tb_size_t count = 10000000;
// init filter
tb_bloom_filter_ref_t filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_01, 3, count, tb_item_func_str(tb_true));
if (filter)
{
// clear random
tb_random_clear(tb_random_generator());
// done
tb_size_t i = 0;
tb_size_t r = 0;
tb_char_t s[256] = {0};
for (i = 0; i < count; i++)
{
// the value
tb_long_t value = tb_random();
// format it
tb_snprintf(s, sizeof(s) - 1, "%ld", value);
// set value to filter
if (!tb_bloom_filter_set(filter, s))
{
// repeat++
r++;
}
}
// trace
#ifdef TB_CONFIG_TYPE_FLOAT
tb_trace_i("cstr: count: %lu, repeat: %lu, repeat_p ~= p: %lf", count, r, (tb_double_t)r / count);
#else
tb_trace_i("cstr: count: %lu, repeat: %lu", count, r);
#endif
// exit filter
tb_bloom_filter_exit(filter);
}
}
static tb_bool_t tb_demo_spider_init(tb_demo_spider_t* spider, tb_int_t argc, tb_char_t** argv)
{
// check
tb_assert_and_check_return_val(spider && argc && argv, tb_false);
// done
tb_bool_t ok = tb_false;
do
{
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
// init option
spider->option = tb_option_init("spider", "the spider demo", g_options);
tb_assert_and_check_break(spider->option);
// done option
if (!tb_option_done(spider->option, argc - 1, &argv[1])) break;
// check
tb_assert_and_check_break(tb_option_find(spider->option, "home"));
// init home
if (!tb_url_init(&spider->home)) break;
tb_url_set(&spider->home, tb_option_item_cstr(spider->option, "home"));
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init only home?
if (tb_option_find(spider->option, "only"))
spider->home_only = tb_option_item_bool(spider->option, "only");
// init root
tb_char_t const* root = tb_option_item_cstr(spider->option, "directory");
// init user agent
spider->user_agent = tb_option_item_cstr(spider->option, "agent");
// init timeout
if (tb_option_find(spider->option, "timeout"))
spider->timeout = tb_option_item_sint32(spider->option, "timeout");
// init limited rate
if (tb_option_find(spider->option, "rate"))
spider->limited_rate = tb_option_item_uint32(spider->option, "rate");
#else
// check
tb_assert_and_check_break(argv[1]);
// init home
if (!tb_url_init(&spider->home)) break;
spider->home = argv[1]? argv[1] : tb_null;
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init root
tb_char_t const* root = argv[2];
#endif
// the home host
tb_char_t const* host = tb_url_host_get(&spider->home);
tb_assert_and_check_break(host);
// init home domain
tb_char_t const* domain = tb_strchr(host, '.');
if (domain)
{
tb_strlcpy(spider->home_domain, domain, sizeof(spider->home_domain) - 1);
spider->home_domain[sizeof(spider->home_domain) - 1] = '\0';
}
// using the default root
if (root) tb_strlcpy(spider->root, root, sizeof(spider->root) - 1);
else
{
// the temporary root
tb_directory_temp(spider->root, sizeof(spider->root) - 1);
// append spider
tb_strcat(spider->root, "/spider");
}
tb_trace_d("root: %s", spider->root);
// using the default user agent
if (!spider->user_agent) spider->user_agent = TB_DEMO_SPIDER_USER_AGENT;
// using the default timeout
if (!spider->timeout) spider->timeout = TB_DEMO_SPIDER_TASK_TIMEOUT;
// using the default rate
if (!spider->limited_rate) spider->limited_rate = TB_DEMO_SPIDER_TASK_RATE;
// strip root tail: '/' or '\\'
tb_size_t size = tb_strlen(spider->root);
if (size && (spider->root[size - 1] == '/' || spider->root[size - 1] == '\\')) spider->root[size - 1] = '\0';
// init state
spider->state = TB_STATE_OK;
// init lock
if (!tb_spinlock_init(&spider->lock)) break;
// init pool
spider->pool = tb_fixed_pool_init(tb_null, TB_DEMO_SPIDER_TASK_MAXN >> 2, sizeof(tb_demo_spider_task_t), tb_null, tb_null, tb_null);
tb_assert_and_check_break(spider->pool);
// init filter
spider->filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 3, TB_DEMO_SPIDER_FILTER_MAXN, tb_item_func_str(tb_true));
tb_assert_and_check_break(spider->filter);
// register lock profiler
#ifdef TB_LOCK_PROFILER_ENABLE
tb_lock_profiler_register(tb_lock_profiler(), (tb_pointer_t)&spider->lock, "spider");
#endif
// ok
ok = tb_true;
} while (0);
// failed? help it
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
if (!ok && spider->option) tb_option_help(spider->option);
#endif
// ok?
return ok;
}
