tb_bool_t tb_async_transfer_init_ostream_from_data(tb_async_transfer_ref_t transfer, tb_byte_t* data, tb_size_t size) { // check tb_async_transfer_impl_t* impl = (tb_async_transfer_impl_t*)transfer; tb_assert_and_check_return_val(impl && impl->aicp && data && size, tb_false); // muse be closed tb_assert_and_check_return_val(TB_STATE_CLOSED == tb_atomic_get(&impl->state), tb_false); // exit the previous stream first if be not data stream if (impl->ostream && tb_async_stream_type(impl->ostream) != TB_STREAM_TYPE_DATA) { if (impl->oowner) tb_async_stream_exit(impl->ostream); impl->ostream = tb_null; } // using the previous stream? if (impl->ostream) { // ctrl stream if (!tb_async_stream_ctrl(impl->ostream, TB_STREAM_CTRL_DATA_SET_DATA, data, size)) return tb_false; } else { // init stream impl->ostream = tb_async_stream_init_from_data(impl->aicp, data, size); tb_assert_and_check_return_val(impl->ostream, tb_false); // init owner impl->oowner = 1; } // ok return tb_true; }
tb_bool_t tb_async_transfer_init_istream_from_url(tb_async_transfer_ref_t transfer, tb_char_t const* url) { // check tb_async_transfer_impl_t* impl = (tb_async_transfer_impl_t*)transfer; tb_assert_and_check_return_val(impl && impl->aicp && url, tb_false); // muse be closed tb_assert_and_check_return_val(TB_STATE_CLOSED == tb_atomic_get(&impl->state), tb_false); // check stream type if (impl->istream) { // probe protocol tb_size_t protocol = tb_url_protocol_probe(url); tb_assert_static((tb_size_t)TB_URL_PROTOCOL_FILE == (tb_size_t)TB_STREAM_TYPE_FILE); tb_assert_static((tb_size_t)TB_URL_PROTOCOL_HTTP == (tb_size_t)TB_STREAM_TYPE_HTTP); tb_assert_static((tb_size_t)TB_URL_PROTOCOL_SOCK == (tb_size_t)TB_STREAM_TYPE_SOCK); tb_assert_static((tb_size_t)TB_URL_PROTOCOL_DATA == (tb_size_t)TB_STREAM_TYPE_DATA); // protocol => type tb_size_t type = protocol; if (!type || type > TB_STREAM_TYPE_DATA) { tb_trace_e("unknown stream for url: %s", url); return tb_false; } // exit the previous stream first if be different stream type if (tb_async_stream_type(impl->istream) != type) { if (impl->iowner) tb_async_stream_exit(impl->istream); impl->istream = tb_null; } } // using the previous stream? if (impl->istream) { // ctrl stream if (!tb_async_stream_ctrl(impl->istream, TB_STREAM_CTRL_SET_URL, url)) return tb_false; } else { // init stream impl->istream = tb_async_stream_init_from_url(impl->aicp, url); tb_assert_and_check_return_val(impl->istream, tb_false); // init owner impl->iowner = 1; } // ok return tb_true; }
static tb_bool_t tb_demo_spider_task_ctrl(tb_async_stream_ref_t istream, tb_async_stream_ref_t ostream, tb_cpointer_t priv) { // check tb_demo_spider_task_t* task = (tb_demo_spider_task_t*)priv; tb_assert_and_check_return_val(task && task->spider, tb_false); tb_assert_and_check_return_val(istream && ostream, tb_false); tb_assert_and_check_return_val(tb_async_stream_type(istream) == TB_STREAM_TYPE_HTTP, tb_false); // the url tb_char_t const* url = tb_null; if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_GET_URL, &url)) return tb_false; // trace tb_trace_d("ctrl: %s: ..", url); // set timeout if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, task->spider->timeout)) return tb_false; #if defined(TB_CONFIG_MODULE_HAVE_ZIP) && defined(TB_CONFIG_THIRD_HAVE_ZLIB) // need gzip if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) return tb_false; // auto unzip if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) return tb_false; #endif // user agent if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "User-Agent", task->spider->user_agent)) return tb_false; // enable cookies if (!tb_async_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_COOKIES, tb_cookies())) return tb_false; // ok return tb_true; }
static tb_bool_t tb_demo_istream_open_func(tb_async_stream_ref_t stream, tb_size_t state, tb_cpointer_t priv) { // check tb_demo_context_t* context = (tb_demo_context_t*)priv; tb_assert_and_check_return_val(stream && context && context->option, tb_false); // done tb_bool_t ok = tb_false; do { // check if (state != TB_STATE_OK) { // print verbose info if (context->verbose) { tb_char_t const* url = tb_null; tb_async_stream_ctrl(stream, TB_STREAM_CTRL_GET_URL, &url); tb_printf("open: %s: %s\n", url, tb_state_cstr(state)); } break; } // trace if (context->verbose) tb_printf("open: ok\n"); // init ostream if (tb_option_find(context->option, "more0")) { // the path tb_char_t const* path = tb_option_item_cstr(context->option, "more0"); // init context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (context->verbose) tb_printf("save: %s: ..\n", path); } else { // the name tb_char_t const* name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '/'); if (!name) name = tb_strrchr(tb_option_item_cstr(context->option, "url"), '\\'); if (!name) name = "/async_stream.file"; // the path tb_char_t path[TB_PATH_MAXN] = {0}; if (tb_directory_curt(path, TB_PATH_MAXN)) { // append name tb_strcat(path, name); // init file context->ostream = tb_async_stream_init_from_file(tb_async_stream_aicp((tb_async_stream_ref_t)stream), path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC); // print verbose info if (context->verbose) tb_printf("save: %s: ..\n", path); } } tb_assert_and_check_break(context->ostream); // init transfer context->transfer = tb_async_transfer_init(tb_null, tb_true); tb_assert_and_check_break(context->transfer); // init transfer stream if (!tb_async_transfer_init_istream(context->transfer, stream)) break; if (!tb_async_transfer_init_ostream(context->transfer, context->ostream)) break; // the limit rate if (tb_option_find(context->option, "limitrate")) tb_async_transfer_limitrate(context->transfer, tb_option_item_uint32(context->option, "limitrate")); // open and done transfer if (!tb_async_transfer_open_done(context->transfer, 0, tb_demo_transfer_done_func, context)) break; // ok ok = tb_true; } while (0); // failed or closed? exit wait if (state != TB_STATE_OK && context->event) tb_event_post(context->event); // ok? return ok; }
/* ////////////////////////////////////////////////////////////////////////////////////// * main */ tb_int_t tb_demo_stream_async_stream_main(tb_int_t argc, tb_char_t** argv) { // done tb_demo_context_t context = {0}; do { // init option context.option = tb_option_init("astream", "the astream demo", g_options); tb_assert_and_check_break(context.option); // done option if (tb_option_done(context.option, argc - 1, &argv[1])) { // debug and verbose context.debug = tb_option_find(context.option, "debug")? tb_false : tb_true; context.verbose = tb_option_find(context.option, "no-verbose")? tb_false : tb_true; // done url if (tb_option_find(context.option, "url")) { // init event context.event = tb_event_init(); tb_assert_and_check_break(context.event); // init istream context.istream = tb_async_stream_init_from_url(tb_null, tb_option_item_cstr(context.option, "url")); tb_assert_and_check_break(context.istream); // ctrl http if (tb_async_stream_type(context.istream) == TB_STREAM_TYPE_HTTP) { // enable gzip? if (tb_option_find(context.option, "gzip")) { // auto unzip if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break; // need gzip if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break; } // enable debug? if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, context.debug? tb_demo_istream_head_func : tb_null)) break; // custem header? if (tb_option_find(context.option, "header")) { // init tb_string_t key; tb_string_t val; tb_string_init(&key); tb_string_init(&val); // done tb_bool_t k = tb_true; tb_char_t const* p = tb_option_item_cstr(context.option, "header"); while (*p) { // is key? if (k) { if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++); else if (*p == ':') { // skip ':' p++; // skip space while (*p && tb_isspace(*p)) p++; // is val now k = tb_false; } else p++; } // is val? else { if (*p != ';') tb_string_chrcat(&val, *p++); else { // skip ';' p++; // skip space while (*p && tb_isspace(*p)) p++; // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // is key now k = tb_true; // clear key & val tb_string_clear(&key); tb_string_clear(&val); } } } // set header if (tb_string_size(&key) && tb_string_size(&val)) { if (context.debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val)); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break; } // exit tb_string_exit(&key); tb_string_exit(&val); } // keep alive? if (tb_option_find(context.option, "keep-alive")) { if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break; } // post-data? if (tb_option_find(context.option, "post-data")) { tb_char_t const* post_data = tb_option_item_cstr(context.option, "post-data"); tb_hize_t post_size = tb_strlen(post_data); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (context.debug) tb_printf("post: %llu\n", post_size); } // post-file? else if (tb_option_find(context.option, "post-file")) { tb_char_t const* url = tb_option_item_cstr(context.option, "post-file"); if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break; if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break; if (context.debug) tb_printf("post: %s\n", url); } } // set range if (tb_option_find(context.option, "range")) { tb_char_t const* p = tb_option_item_cstr(context.option, "range"); if (p) { // the bof tb_hize_t eof = 0; tb_hize_t bof = tb_atoll(p); while (*p && tb_isdigit(*p)) p++; if (*p == '-') { p++; eof = tb_atoll(p); } if (!tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break; } } // set timeout if (tb_option_find(context.option, "timeout")) { tb_size_t timeout = tb_option_item_uint32(context.option, "timeout"); tb_async_stream_ctrl(context.istream, TB_STREAM_CTRL_SET_TIMEOUT, &timeout); } // print verbose info if (context.verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(context.option, "url")); // open istream if (!tb_async_stream_open(context.istream, tb_demo_istream_open_func, &context)) { // print verbose info if (context.verbose) tb_printf("open: failed\n"); break; } // wait it tb_event_wait(context.event, -1); } else tb_option_help(context.option); } else tb_option_help(context.option); } while (0); // exit transfer if (context.transfer) tb_async_transfer_exit(context.transfer); context.transfer = tb_null; // exit istream if (context.istream) tb_async_stream_exit(context.istream); context.istream = tb_null; // exit ostream if (context.ostream) tb_async_stream_exit(context.ostream); context.ostream = tb_null; // exit option if (context.option) tb_option_exit(context.option); context.option = tb_null; // exit event if (context.event) tb_event_exit(context.event); context.event = tb_null; return 0; }