static __tb_inline__ tb_bool_t tb_option_check(tb_option_impl_t* impl) { // check tb_assert_and_check_return_val(impl && impl->list && impl->opts, tb_false); // walk tb_bool_t ok = tb_true; tb_option_item_t* item = impl->opts; while (item && ok) { switch (item->mode) { case TB_OPTION_MODE_KEY: case TB_OPTION_MODE_KEY_VAL: item++; break; case TB_OPTION_MODE_VAL: { if (item->lname && !tb_option_find(impl, item->lname)) ok = tb_false; item++; } break; case TB_OPTION_MODE_MORE: case TB_OPTION_MODE_END: default: item = tb_null; break; } } // ok? return ok; }
/* ////////////////////////////////////////////////////////////////////////////////////// * main */ tb_int_t tb_demo_utils_option_main(tb_int_t argc, tb_char_t** argv) { // init option tb_option_ref_t option = tb_option_init("option", "the option command test demo", g_options); if (option) { // done option if (tb_option_done(option, argc - 1, &argv[1])) { // done dump tb_option_dump(option); // done help if (tb_option_find(option, "help")) tb_option_help(option); // done version else if (tb_option_find(option, "version")) { tb_version_t const* version = tb_version(); if (version) tb_trace_i("version: tbox-v%u.%u.%u.%llu", version->major, version->minor, version->alter, version->build); } else { // done integer if (tb_option_find(option, "i")) tb_trace_i("integer: %lld", tb_option_item_sint64(option, "i")); // done string if (tb_option_find(option, "s")) tb_trace_i("string: %s", tb_option_item_cstr(option, "s")); #ifdef TB_CONFIG_TYPE_HAVE_FLOAT // done float if (tb_option_find(option, "f")) tb_trace_i("float: %f", tb_option_item_float(option, "f")); #endif // done boolean if (tb_option_find(option, "b")) tb_trace_i("boolean: %s", tb_option_item_bool(option, "b")? "y" : "n"); // done demo if (tb_option_find(option, "demo")) tb_trace_i("demo: %s", tb_option_item_cstr(option, "demo")); // done file0 if (tb_option_find(option, "file0")) tb_trace_i("file0: %s", tb_option_item_cstr(option, "file0")); // done file1 if (tb_option_find(option, "file1")) tb_trace_i("file1: %s", tb_option_item_cstr(option, "file1")); // done more tb_size_t more = 0; while (1) { tb_char_t name[64] = {0}; tb_snprintf(name, 63, "more%lu", more++); if (tb_option_find(option, name)) tb_trace_i("%s: %s", name, tb_option_item_cstr(option, name)); else break; } } } else tb_option_help(option); // exit option tb_option_exit(option); } return 0; }
static tb_bool_t tb_demo_istream_open_func(tb_async_stream_ref_t stream, tb_size_t state, tb_cpointer_t priv) { // check tb_demo_context_t* context = (tb_demo_context_t*)priv; tb_assert_and_check_return_val(stream && context && context->option, tb_false); // done tb_bool_t ok = tb_false; do { // check if (state != TB_STATE_OK) { // print verbose info if (context->verbose) { tb_char_t const* url = tb_null; tb_async_stream_ctrl(stream, TB_STREAM_CTRL_GET_URL, &url); tb_printf("open: %s: %s\n", url, tb_state_cstr(state)); } break; } // trace if (context->verbose) tb_printf("open: ok\n"); // init ostream if (tb_option_find(context->option, "more0")) { // the path tb_char_t const* path = tb_option_item_cstr(context->option, "more0"); // init context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (context->verbose) tb_printf("save: %s: ..\n", path); } else { // the name tb_char_t const* name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '/'); if (!name) name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '\\'); if (!name) name = "/async_stream.file"; // the path tb_char_t path[TB_PATH_MAXN] = {0}; if (tb_directory_curt(path, TB_PATH_MAXN)) { // append name tb_strcat(path, name); // init file context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (context->verbose) tb_printf("save: %s: ..\n", path); } } tb_assert_and_check_break(context->ostream); // init transfer context->transfer = tb_async_transfer_init(tb_null, tb_true); tb_assert_and_check_break(context->transfer); // init transfer stream if (!tb_async_transfer_init_istream(context->transfer, stream)) break; if (!tb_async_transfer_init_ostream(context->transfer, context->ostream)) break; // the limit rate if (tb_option_find(context->option, "limitrate")) tb_async_transfer_limitrate(context->transfer, tb_option_item_uint32(context->option, "limitrate")); // open and done transfer if (!tb_async_transfer_open_done(context->transfer, 0, tb_demo_transfer_done_func, context)) break; // ok ok = tb_true; } while (0); // failed or closed? exit wait if (state != TB_STATE_OK && context->event) tb_event_post(context->event); // ok? return ok; }
/* ////////////////////////////////////////////////////////////////////////////////////// * main */ tb_int_t tb_demo_stream_async_stream_main(tb_int_t argc, tb_char_t** argv) { // done tb_demo_context_t context = {0}; do { // init option context.option = tb_option_init("astream", "the astream demo", g_options); tb_assert_and_check_break(context.option); // done option if (tb_option_done(context.option, argc - 1, &argv[1])) { // debug and verbose context.debug = tb_option_find(context.option, "debug")? tb_false : tb_true; context.verbose = tb_option_find(context.option, "no-verbose")? tb_false : tb_true; // done url if (tb_option_find(context.option, "url")) { // init event context.event = tb_event_init(); tb_assert_and_check_break(context.event); // init istream context.istream = tb_async_stream_init_from_url(tb_null, tb_option_item_cstr(context.option, "url")); tb_assert_and_check_break(context.istream); // ctrl http if (tb_async_stream_type(context.istream) == TB_STREAM_TYPE_HTTP) { // enable gzip? if (tb_option_find(context.option, "gzip")) { // auto unzip if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break; // need gzip if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break; } // enable debug? if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, context.debug? tb_demo_istream_head_func : tb_null)) break; // custem header? if (tb_option_find(context.option, "header")) { // init tb_string_t key; tb_string_t val; tb_string_init(&key); tb_string_init(&val); // done tb_bool_t k = tb_true; tb_char_t const* p = tb_option_item_cstr(context.option, "header"); while (*p) { // is key? if (k) { if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++); else if (*p == ':') { // skip ':' p++; // skip space while (*p && tb_isspace(*p)) p++; // is val now k = tb_false; } else p++; } // is val? else { if (*p != ';') tb_string_chrcat(&val, *p++); else { // skip ';' p++; // skip space while (*p && tb_isspace(*p)) p++; // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // is key now k = tb_true; // clear key & val tb_string_clear(&key); tb_string_clear(&val); } } } // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // exit tb_string_exit(&key); tb_string_exit(&val); } // keep alive? if (tb_option_find(context.option, "keep-alive")) { if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break; } // post-data? if (tb_option_find(context.option, "post-data")) { tb_char_t const* post_data = tb_option_item_cstr(context.option, "post-data"); tb_hize_t post_size = tb_strlen(post_data); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (context.debug) tb_printf("post: %llu\n", post_size); } // post-file? else if (tb_option_find(context.option, "post-file")) { tb_char_t const* url = tb_option_item_cstr(context.option, "post-file"); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (context.debug) tb_printf("post: %s\n", url); } } // set range if (tb_option_find(context.option, "range")) { tb_char_t const* p = tb_option_item_cstr(context.option, "range"); if (p) { // the bof tb_hize_t eof = 0; tb_hize_t bof = tb_atoll(p); while (*p && tb_isdigit(*p)) p++; if (*p == '-') { p++; eof = tb_atoll(p); } if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break; } } // set timeout if (tb_option_find(context.option, "timeout")) { tb_size_t timeout = tb_option_item_uint32(context.option, "timeout"); tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_SET_TIMEOUT, &timeout); } // print verbose info if (context.verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(context.option, "url")); // open istream if (!tb_async_stream_open(context.istream, tb_demo_istream_open_func, &context)) { // print verbose info if (context.verbose) tb_printf("open: failed\n"); break; } // wait it tb_event_wait(context.event, -1); } else tb_option_help(context.option); } else tb_option_help(context.option); } while (0); // exit transfer if (context.transfer) tb_async_transfer_exit(context.transfer); context.transfer = tb_null; // exit istream if (context.istream) tb_async_stream_exit(context.istream); context.istream = tb_null; // exit ostream if (context.ostream) tb_async_stream_exit(context.ostream); context.ostream = tb_null; // exit option if (context.option) tb_option_exit(context.option); context.option = tb_null; // exit event if (context.event) tb_event_exit(context.event); context.event = tb_null; return 0; }
static tb_bool_t tb_demo_spider_init(tb_demo_spider_t* spider, tb_int_t argc, tb_char_t** argv) { // check tb_assert_and_check_return_val(spider && argc && argv, tb_false); // done tb_bool_t ok = tb_false; do { #ifdef TB_CONFIG_MODULE_HAVE_OBJECT // init option spider->option = tb_option_init("spider", "the spider demo", g_options); tb_assert_and_check_break(spider->option); // done option if (!tb_option_done(spider->option, argc - 1, &argv[1])) break; // check tb_assert_and_check_break(tb_option_find(spider->option, "home")); // init home if (!tb_url_init(&spider->home)) break; tb_url_set(&spider->home, tb_option_item_cstr(spider->option, "home")); tb_trace_d("home: %s", tb_url_get(&spider->home)); // init only home? if (tb_option_find(spider->option, "only")) spider->home_only = tb_option_item_bool(spider->option, "only"); // init root tb_char_t const* root = tb_option_item_cstr(spider->option, "directory"); // init user agent spider->user_agent = tb_option_item_cstr(spider->option, "agent"); // init timeout if (tb_option_find(spider->option, "timeout")) spider->timeout = tb_option_item_sint32(spider->option, "timeout"); // init limited rate if (tb_option_find(spider->option, "rate")) spider->limited_rate = tb_option_item_uint32(spider->option, "rate"); #else // check tb_assert_and_check_break(argv[1]); // init home if (!tb_url_init(&spider->home)) break; spider->home = argv[1]? argv[1] : tb_null; tb_trace_d("home: %s", tb_url_get(&spider->home)); // init root tb_char_t const* root = argv[2]; #endif // the home host tb_char_t const* host = tb_url_host_get(&spider->home); tb_assert_and_check_break(host); // init home domain tb_char_t const* domain = tb_strchr(host, '.'); if (domain) { tb_strlcpy(spider->home_domain, domain, sizeof(spider->home_domain) - 1); spider->home_domain[sizeof(spider->home_domain) - 1] = '\0'; } // using the default root if (root) tb_strlcpy(spider->root, root, sizeof(spider->root) - 1); else { // the temporary root tb_directory_temp(spider->root, sizeof(spider->root) - 1); // append spider tb_strcat(spider->root, "/spider"); } tb_trace_d("root: %s", spider->root); // using the default user agent if (!spider->user_agent) spider->user_agent = TB_DEMO_SPIDER_USER_AGENT; // using the default timeout if (!spider->timeout) spider->timeout = TB_DEMO_SPIDER_TASK_TIMEOUT; // using the default rate if (!spider->limited_rate) spider->limited_rate = TB_DEMO_SPIDER_TASK_RATE; // strip root tail: '/' or '\\' tb_size_t size = tb_strlen(spider->root); if (size && (spider->root[size - 1] == '/' || spider->root[size - 1] == '\\')) spider->root[size - 1] = '\0'; // init state spider->state = TB_STATE_OK; // init lock if (!tb_spinlock_init(&spider->lock)) break; // init pool spider->pool = tb_fixed_pool_init(tb_null, TB_DEMO_SPIDER_TASK_MAXN >> 2, sizeof(tb_demo_spider_task_t), tb_null, tb_null, tb_null); tb_assert_and_check_break(spider->pool); // init filter spider->filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 3, TB_DEMO_SPIDER_FILTER_MAXN, tb_item_func_str(tb_true)); tb_assert_and_check_break(spider->filter); // register lock profiler #ifdef TB_LOCK_PROFILER_ENABLE tb_lock_profiler_register(tb_lock_profiler(), (tb_pointer_t)&spider->lock, "spider"); #endif // ok ok = tb_true; } while (0); // failed? help it #ifdef TB_CONFIG_MODULE_HAVE_OBJECT if (!ok && spider->option) tb_option_help(spider->option); #endif // ok? return ok; }
/* ////////////////////////////////////////////////////////////////////////////////////// * main */ tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv) { // done tb_option_ref_t option = tb_null; tb_stream_ref_t istream = tb_null; tb_stream_ref_t ostream = tb_null; tb_stream_ref_t pstream = tb_null; do { // init option option = tb_option_init("stream", "the stream demo", g_options); tb_assert_and_check_break(option); // done option if (tb_option_done(option, argc - 1, &argv[1])) { // debug & verbose tb_bool_t debug = tb_option_find(option, "debug"); tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true; // done url if (tb_option_find(option, "url")) { // init istream istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url")); tb_assert_and_check_break(istream); // ctrl http if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP) { // enable gzip? if (tb_option_find(option, "gzip")) { // auto unzip if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break; // need gzip if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break; } // enable debug? if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break; // custem header? if (tb_option_find(option, "header")) { // init tb_string_t key; tb_string_t val; tb_string_init(&key); tb_string_init(&val); // done tb_bool_t k = tb_true; tb_char_t const* p = tb_option_item_cstr(option, "header"); while (*p) { // is key? if (k) { if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++); else if (*p == ':') { // skip ':' p++; // skip space while (*p && tb_isspace(*p)) p++; // is val now k = tb_false; } else p++; } // is val? else { if (*p != ';') tb_string_chrcat(&val, *p++); else { // skip ';' p++; // skip space while (*p && tb_isspace(*p)) p++; // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // is key now k = tb_true; // clear key & val tb_string_clear(&key); tb_string_clear(&val); } } } // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // exit tb_string_exit(&key); tb_string_exit(&val); } // keep alive? if (tb_option_find(option, "keep-alive")) { if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break; } // post-data? if (tb_option_find(option, "post-data")) { tb_char_t const* post_data = tb_option_item_cstr(option, "post-data"); tb_hize_t post_size = tb_strlen(post_data); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (debug) tb_printf("post: %llu\n", post_size); } // post-file? else if (tb_option_find(option, "post-file")) { tb_char_t const* url = tb_option_item_cstr(option, "post-file"); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break; if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (debug) tb_printf("post: %s\n", url); } } // set range if (tb_option_find(option, "range")) { tb_char_t const* p = tb_option_item_cstr(option, "range"); if (p) { // the bof tb_hize_t eof = 0; tb_hize_t bof = tb_atoll(p); while (*p && tb_isdigit(*p)) p++; if (*p == '-') { p++; eof = tb_atoll(p); } if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break; } } // set timeout if (tb_option_find(option, "timeout")) { tb_size_t timeout = tb_option_item_uint32(option, "timeout"); if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break; } // print verbose info if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url")); // open istream if (!tb_stream_open(istream)) { // print verbose info if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream))); break; } // print verbose info if (verbose) tb_printf("open: ok\n"); // init ostream if (tb_option_find(option, "more0")) { // the path tb_char_t const* path = tb_option_item_cstr(option, "more0"); // init ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (verbose) tb_printf("save: %s\n", path); } else { // the name tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/'); if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\'); if (!name) name = "/stream.file"; // the path tb_char_t path[TB_PATH_MAXN] = {0}; if (tb_directory_current(path, TB_PATH_MAXN)) tb_strcat(path, name); else break; // init file ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (verbose) tb_printf("save: %s\n", path); } tb_assert_and_check_break(ostream); // the limit rate tb_size_t limitrate = 0; if (tb_option_find(option, "limitrate")) limitrate = tb_option_item_uint32(option, "limitrate"); // save it tb_hong_t save = 0; tb_demo_context_t context = {0}; context.verbose = verbose; if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break; } else tb_option_help(option); } else tb_option_help(option); } while (0); // exit pstream if (pstream) tb_stream_exit(pstream); pstream = tb_null; // exit istream if (istream) tb_stream_exit(istream); istream = tb_null; // exit ostream if (ostream) tb_stream_exit(ostream); ostream = tb_null; // exit option if (option) tb_option_exit(option); option = tb_null; return 0; }