static __tb_inline__ tb_bool_t tb_option_check(tb_option_impl_t* impl)
{
// check
tb_assert_and_check_return_val(impl && impl->list && impl->opts, tb_false);
// walk
tb_bool_t ok = tb_true;
tb_option_item_t* item = impl->opts;
while (item && ok)
{
switch (item->mode)
{
case TB_OPTION_MODE_KEY:
case TB_OPTION_MODE_KEY_VAL:
item++;
break;
case TB_OPTION_MODE_VAL:
{
if (item->lname && !tb_option_find(impl, item->lname)) ok = tb_false;
item++;
}
break;
case TB_OPTION_MODE_MORE:
case TB_OPTION_MODE_END:
default:
item = tb_null;
break;
}
}
// ok?
return ok;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* main
*/
tb_int_t tb_demo_utils_option_main(tb_int_t argc, tb_char_t** argv)
{
// init option
tb_option_ref_t option = tb_option_init("option", "the option command test demo", g_options);
if (option)
{
// done option
if (tb_option_done(option, argc - 1, &argv[1]))
{
// done dump
tb_option_dump(option);
// done help
if (tb_option_find(option, "help"))
tb_option_help(option);
// done version
else if (tb_option_find(option, "version"))
{
tb_version_t const* version = tb_version();
if (version) tb_trace_i("version: tbox-v%u.%u.%u.%llu", version->major, version->minor, version->alter, version->build);
}
else
{
// done integer
if (tb_option_find(option, "i"))
tb_trace_i("integer: %lld", tb_option_item_sint64(option, "i"));
// done string
if (tb_option_find(option, "s"))
tb_trace_i("string: %s", tb_option_item_cstr(option, "s"));
#ifdef TB_CONFIG_TYPE_HAVE_FLOAT
// done float
if (tb_option_find(option, "f"))
tb_trace_i("float: %f", tb_option_item_float(option, "f"));
#endif
// done boolean
if (tb_option_find(option, "b"))
tb_trace_i("boolean: %s", tb_option_item_bool(option, "b")? "y" : "n");
// done demo
if (tb_option_find(option, "demo"))
tb_trace_i("demo: %s", tb_option_item_cstr(option, "demo"));
// done file0
if (tb_option_find(option, "file0"))
tb_trace_i("file0: %s", tb_option_item_cstr(option, "file0"));
// done file1
if (tb_option_find(option, "file1"))
tb_trace_i("file1: %s", tb_option_item_cstr(option, "file1"));
// done more
tb_size_t more = 0;
while (1)
{
tb_char_t name[64] = {0};
tb_snprintf(name, 63, "more%lu", more++);
if (tb_option_find(option, name))
tb_trace_i("%s: %s", name, tb_option_item_cstr(option, name));
else break;
}
}
}
else tb_option_help(option);
// exit option
tb_option_exit(option);
}
return 0;
}
static tb_bool_t tb_demo_istream_open_func(tb_async_stream_ref_t stream, tb_size_t state, tb_cpointer_t priv)
{
// check
tb_demo_context_t* context = (tb_demo_context_t*)priv;
tb_assert_and_check_return_val(stream && context && context->option, tb_false);
// done
tb_bool_t ok = tb_false;
do
{
// check
if (state != TB_STATE_OK)
{
// print verbose info
if (context->verbose)
{
tb_char_t const* url = tb_null;
tb_async_stream_ctrl(stream, TB_STREAM_CTRL_GET_URL, &url);
tb_printf("open: %s: %s\n", url, tb_state_cstr(state));
}
break;
}
// trace
if (context->verbose) tb_printf("open: ok\n");
// init ostream
if (tb_option_find(context->option, "more0"))
{
// the path
tb_char_t const* path = tb_option_item_cstr(context->option, "more0");
// init
context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (context->verbose) tb_printf("save: %s: ..\n", path);
}
else
{
// the name
tb_char_t const* name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '/');
if (!name) name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '\\');
if (!name) name = "/async_stream.file";
// the path
tb_char_t path[TB_PATH_MAXN] = {0};
if (tb_directory_curt(path, TB_PATH_MAXN))
{
// append name
tb_strcat(path, name);
// init file
context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (context->verbose) tb_printf("save: %s: ..\n", path);
}
}
tb_assert_and_check_break(context->ostream);
// init transfer
context->transfer = tb_async_transfer_init(tb_null, tb_true);
tb_assert_and_check_break(context->transfer);
// init transfer stream
if (!tb_async_transfer_init_istream(context->transfer, stream)) break;
if (!tb_async_transfer_init_ostream(context->transfer, context->ostream)) break;
// the limit rate
if (tb_option_find(context->option, "limitrate"))
tb_async_transfer_limitrate(context->transfer, tb_option_item_uint32(context->option, "limitrate"));
// open and done transfer
if (!tb_async_transfer_open_done(context->transfer, 0, tb_demo_transfer_done_func, context)) break;
// ok
ok = tb_true;
} while (0);
// failed or closed? exit wait
if (state != TB_STATE_OK && context->event) tb_event_post(context->event);
// ok?
return ok;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* main
*/
tb_int_t tb_demo_stream_async_stream_main(tb_int_t argc, tb_char_t** argv)
{
// done
tb_demo_context_t context = {0};
do
{
// init option
context.option = tb_option_init("astream", "the astream demo", g_options);
tb_assert_and_check_break(context.option);
// done option
if (tb_option_done(context.option, argc - 1, &argv[1]))
{
// debug and verbose
context.debug = tb_option_find(context.option, "debug")? tb_false : tb_true;
context.verbose = tb_option_find(context.option, "no-verbose")? tb_false : tb_true;
// done url
if (tb_option_find(context.option, "url"))
{
// init event
context.event = tb_event_init();
tb_assert_and_check_break(context.event);
// init istream
context.istream = tb_async_stream_init_from_url(tb_null, tb_option_item_cstr(context.option, "url"));
tb_assert_and_check_break(context.istream);
// ctrl http
if (tb_async_stream_type(context.istream) == TB_STREAM_TYPE_HTTP)
{
// enable gzip?
if (tb_option_find(context.option, "gzip"))
{
// auto unzip
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break;
// need gzip
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break;
}
// enable debug?
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, context.debug? tb_demo_istream_head_func : tb_null)) break;
// custem header?
if (tb_option_find(context.option, "header"))
{
// init
tb_string_t key;
tb_string_t val;
tb_string_init(&key);
tb_string_init(&val);
// done
tb_bool_t k = tb_true;
tb_char_t const* p = tb_option_item_cstr(context.option, "header");
while (*p)
{
// is key?
if (k)
{
if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++);
else if (*p == ':')
{
// skip ':'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// is val now
k = tb_false;
}
else p++;
}
// is val?
else
{
if (*p != ';') tb_string_chrcat(&val, *p++);
else
{
// skip ';'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// is key now
k = tb_true;
// clear key & val
tb_string_clear(&key);
tb_string_clear(&val);
}
}
}
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// exit
tb_string_exit(&key);
tb_string_exit(&val);
}
// keep alive?
if (tb_option_find(context.option, "keep-alive"))
{
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break;
}
// post-data?
if (tb_option_find(context.option, "post-data"))
{
tb_char_t const* post_data = tb_option_item_cstr(context.option, "post-data");
tb_hize_t post_size = tb_strlen(post_data);
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break;
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (context.debug) tb_printf("post: %llu\n", post_size);
}
// post-file?
else if (tb_option_find(context.option, "post-file"))
{
tb_char_t const* url = tb_option_item_cstr(context.option, "post-file");
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break;
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (context.debug) tb_printf("post: %s\n", url);
}
}
// set range
if (tb_option_find(context.option, "range"))
{
tb_char_t const* p = tb_option_item_cstr(context.option, "range");
if (p)
{
// the bof
tb_hize_t eof = 0;
tb_hize_t bof = tb_atoll(p);
while (*p && tb_isdigit(*p)) p++;
if (*p == '-')
{
p++;
eof = tb_atoll(p);
}
if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break;
}
}
// set timeout
if (tb_option_find(context.option, "timeout"))
{
tb_size_t timeout = tb_option_item_uint32(context.option, "timeout");
tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_SET_TIMEOUT, &timeout);
}
// print verbose info
if (context.verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(context.option, "url"));
// open istream
if (!tb_async_stream_open(context.istream, tb_demo_istream_open_func, &context))
{
// print verbose info
if (context.verbose) tb_printf("open: failed\n");
break;
}
// wait it
tb_event_wait(context.event, -1);
}
else tb_option_help(context.option);
}
else tb_option_help(context.option);
} while (0);
// exit transfer
if (context.transfer) tb_async_transfer_exit(context.transfer);
context.transfer = tb_null;
// exit istream
if (context.istream) tb_async_stream_exit(context.istream);
context.istream = tb_null;
// exit ostream
if (context.ostream) tb_async_stream_exit(context.ostream);
context.ostream = tb_null;
// exit option
if (context.option) tb_option_exit(context.option);
context.option = tb_null;
// exit event
if (context.event) tb_event_exit(context.event);
context.event = tb_null;
return 0;
}
static tb_bool_t tb_demo_spider_init(tb_demo_spider_t* spider, tb_int_t argc, tb_char_t** argv)
{
// check
tb_assert_and_check_return_val(spider && argc && argv, tb_false);
// done
tb_bool_t ok = tb_false;
do
{
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
// init option
spider->option = tb_option_init("spider", "the spider demo", g_options);
tb_assert_and_check_break(spider->option);
// done option
if (!tb_option_done(spider->option, argc - 1, &argv[1])) break;
// check
tb_assert_and_check_break(tb_option_find(spider->option, "home"));
// init home
if (!tb_url_init(&spider->home)) break;
tb_url_set(&spider->home, tb_option_item_cstr(spider->option, "home"));
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init only home?
if (tb_option_find(spider->option, "only"))
spider->home_only = tb_option_item_bool(spider->option, "only");
// init root
tb_char_t const* root = tb_option_item_cstr(spider->option, "directory");
// init user agent
spider->user_agent = tb_option_item_cstr(spider->option, "agent");
// init timeout
if (tb_option_find(spider->option, "timeout"))
spider->timeout = tb_option_item_sint32(spider->option, "timeout");
// init limited rate
if (tb_option_find(spider->option, "rate"))
spider->limited_rate = tb_option_item_uint32(spider->option, "rate");
#else
// check
tb_assert_and_check_break(argv[1]);
// init home
if (!tb_url_init(&spider->home)) break;
spider->home = argv[1]? argv[1] : tb_null;
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init root
tb_char_t const* root = argv[2];
#endif
// the home host
tb_char_t const* host = tb_url_host_get(&spider->home);
tb_assert_and_check_break(host);
// init home domain
tb_char_t const* domain = tb_strchr(host, '.');
if (domain)
{
tb_strlcpy(spider->home_domain, domain, sizeof(spider->home_domain) - 1);
spider->home_domain[sizeof(spider->home_domain) - 1] = '\0';
}
// using the default root
if (root) tb_strlcpy(spider->root, root, sizeof(spider->root) - 1);
else
{
// the temporary root
tb_directory_temp(spider->root, sizeof(spider->root) - 1);
// append spider
tb_strcat(spider->root, "/spider");
}
tb_trace_d("root: %s", spider->root);
// using the default user agent
if (!spider->user_agent) spider->user_agent = TB_DEMO_SPIDER_USER_AGENT;
// using the default timeout
if (!spider->timeout) spider->timeout = TB_DEMO_SPIDER_TASK_TIMEOUT;
// using the default rate
if (!spider->limited_rate) spider->limited_rate = TB_DEMO_SPIDER_TASK_RATE;
// strip root tail: '/' or '\\'
tb_size_t size = tb_strlen(spider->root);
if (size && (spider->root[size - 1] == '/' || spider->root[size - 1] == '\\')) spider->root[size - 1] = '\0';
// init state
spider->state = TB_STATE_OK;
// init lock
if (!tb_spinlock_init(&spider->lock)) break;
// init pool
spider->pool = tb_fixed_pool_init(tb_null, TB_DEMO_SPIDER_TASK_MAXN >> 2, sizeof(tb_demo_spider_task_t), tb_null, tb_null, tb_null);
tb_assert_and_check_break(spider->pool);
// init filter
spider->filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 3, TB_DEMO_SPIDER_FILTER_MAXN, tb_item_func_str(tb_true));
tb_assert_and_check_break(spider->filter);
// register lock profiler
#ifdef TB_LOCK_PROFILER_ENABLE
tb_lock_profiler_register(tb_lock_profiler(), (tb_pointer_t)&spider->lock, "spider");
#endif
// ok
ok = tb_true;
} while (0);
// failed? help it
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
if (!ok && spider->option) tb_option_help(spider->option);
#endif
// ok?
return ok;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* main
*/
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
// done
tb_option_ref_t option = tb_null;
tb_stream_ref_t istream = tb_null;
tb_stream_ref_t ostream = tb_null;
tb_stream_ref_t pstream = tb_null;
do
{
// init option
option = tb_option_init("stream", "the stream demo", g_options);
tb_assert_and_check_break(option);
// done option
if (tb_option_done(option, argc - 1, &argv[1]))
{
// debug & verbose
tb_bool_t debug = tb_option_find(option, "debug");
tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true;
// done url
if (tb_option_find(option, "url"))
{
// init istream
istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url"));
tb_assert_and_check_break(istream);
// ctrl http
if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP)
{
// enable gzip?
if (tb_option_find(option, "gzip"))
{
// auto unzip
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break;
// need gzip
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break;
}
// enable debug?
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break;
// custem header?
if (tb_option_find(option, "header"))
{
// init
tb_string_t key;
tb_string_t val;
tb_string_init(&key);
tb_string_init(&val);
// done
tb_bool_t k = tb_true;
tb_char_t const* p = tb_option_item_cstr(option, "header");
while (*p)
{
// is key?
if (k)
{
if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++);
else if (*p == ':')
{
// skip ':'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// is val now
k = tb_false;
}
else p++;
}
// is val?
else
{
if (*p != ';') tb_string_chrcat(&val, *p++);
else
{
// skip ';'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// is key now
k = tb_true;
// clear key & val
tb_string_clear(&key);
tb_string_clear(&val);
}
}
}
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// exit
tb_string_exit(&key);
tb_string_exit(&val);
}
// keep alive?
if (tb_option_find(option, "keep-alive"))
{
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break;
}
// post-data?
if (tb_option_find(option, "post-data"))
{
tb_char_t const* post_data = tb_option_item_cstr(option, "post-data");
tb_hize_t post_size = tb_strlen(post_data);
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (debug) tb_printf("post: %llu\n", post_size);
}
// post-file?
else if (tb_option_find(option, "post-file"))
{
tb_char_t const* url = tb_option_item_cstr(option, "post-file");
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (debug) tb_printf("post: %s\n", url);
}
}
// set range
if (tb_option_find(option, "range"))
{
tb_char_t const* p = tb_option_item_cstr(option, "range");
if (p)
{
// the bof
tb_hize_t eof = 0;
tb_hize_t bof = tb_atoll(p);
while (*p && tb_isdigit(*p)) p++;
if (*p == '-')
{
p++;
eof = tb_atoll(p);
}
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break;
}
}
// set timeout
if (tb_option_find(option, "timeout"))
{
tb_size_t timeout = tb_option_item_uint32(option, "timeout");
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break;
}
// print verbose info
if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url"));
// open istream
if (!tb_stream_open(istream))
{
// print verbose info
if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream)));
break;
}
// print verbose info
if (verbose) tb_printf("open: ok\n");
// init ostream
if (tb_option_find(option, "more0"))
{
// the path
tb_char_t const* path = tb_option_item_cstr(option, "more0");
// init
ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (verbose) tb_printf("save: %s\n", path);
}
else
{
// the name
tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/');
if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\');
if (!name) name = "/stream.file";
// the path
tb_char_t path[TB_PATH_MAXN] = {0};
if (tb_directory_current(path, TB_PATH_MAXN))
tb_strcat(path, name);
else break;
// init file
ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (verbose) tb_printf("save: %s\n", path);
}
tb_assert_and_check_break(ostream);
// the limit rate
tb_size_t limitrate = 0;
if (tb_option_find(option, "limitrate"))
limitrate = tb_option_item_uint32(option, "limitrate");
// save it
tb_hong_t save = 0;
tb_demo_context_t context = {0};
context.verbose = verbose;
if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break;
}
else tb_option_help(option);
}
else tb_option_help(option);
} while (0);
// exit pstream
if (pstream) tb_stream_exit(pstream);
pstream = tb_null;
// exit istream
if (istream) tb_stream_exit(istream);
istream = tb_null;
// exit ostream
if (ostream) tb_stream_exit(ostream);
ostream = tb_null;
// exit option
if (option) tb_option_exit(option);
option = tb_null;
return 0;
}
