/* //////////////////////////////////////////////////////////////////////////////////////
* interfaces
*/
tb_size_t tb_database_sqlite3_probe(tb_url_ref_t url)
{
// check
tb_assert_and_check_return_val(url, 0);
// done
tb_size_t score = 0;
tb_stream_ref_t stream = tb_null;
do
{
// the url arguments
tb_char_t const* args = tb_url_args(url);
if (args)
{
// find the database type
tb_char_t const* ptype = tb_stristr(args, "type=");
if (ptype && !tb_strnicmp(ptype + 5, "sqlite3", 7))
{
// ok
score = 100;
break;
}
}
// has host or port? no sqlite3
if (tb_url_host(url) || tb_url_port(url)) break;
// the database path
tb_char_t const* path = tb_url_cstr((tb_url_ref_t)url);
tb_assert_and_check_break(path);
// is file?
if (tb_url_protocol(url) == TB_URL_PROTOCOL_FILE) score += 20;
// init stream
stream = tb_stream_init_from_url(path);
tb_assert_and_check_break(stream);
// open stream
if (!tb_stream_open(stream)) break;
// read head
tb_char_t head[16] = {0};
if (!tb_stream_bread(stream, (tb_byte_t*)head, 15)) break;
// is sqlite3?
if (!tb_stricmp(head, "SQLite format 3")) score = 100;
} while (0);
// exit stream
if (stream) tb_stream_exit(stream);
stream = tb_null;
// trace
tb_trace_d("probe: %s, score: %lu", tb_url_cstr((tb_url_ref_t)url), score);
// ok?
return score;
}
tb_bool_t tb_object_data_writ_to_url(tb_object_ref_t object, tb_char_t const* url)
{
// check
tb_object_data_t* data = tb_object_data_cast(object);
tb_assert_and_check_return_val(data && tb_object_data_getp((tb_object_ref_t)data) && url, tb_false);
// make stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_false);
// ctrl
if (tb_stream_type(stream) == TB_STREAM_TYPE_FILE)
tb_stream_ctrl(stream, TB_STREAM_CTRL_FILE_SET_MODE, TB_FILE_MODE_WO | TB_FILE_MODE_CREAT | TB_FILE_MODE_TRUNC);
// open stream
tb_bool_t ok = tb_false;
if (tb_stream_open(stream))
{
// writ stream
if (tb_stream_bwrit(stream, (tb_byte_t const*)tb_object_data_getp((tb_object_ref_t)data), tb_object_data_size((tb_object_ref_t)data))) ok = tb_true;
}
// exit stream
tb_stream_exit(stream);
// ok?
return ok;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* interfaces
*/
tb_object_ref_t tb_object_data_init_from_url(tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_null);
// make stream
tb_object_ref_t object = tb_null;
if (tb_stream_open(stream))
{
// read all data
tb_size_t size = 0;
tb_byte_t* data = (tb_byte_t*)tb_stream_bread_all(stream, tb_false, &size);
if (data)
{
// make object
object = tb_object_data_init_from_data(data, size);
// exit data
tb_free(data);
}
// exit stream
tb_stream_exit(stream);
}
// ok?
return object;
}
tb_int_t tb_demo_stream_zip_main(tb_int_t argc, tb_char_t** argv)
{
// init istream
tb_stream_ref_t istream = tb_stream_init_from_url(argv[1]);
// init ostream
tb_stream_ref_t ostream = tb_stream_init_from_file(argv[2], TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// filter istream or ostream?
tb_stream_ref_t iostream = istream;
// tb_stream_ref_t iostream = ostream;
// init fstream
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_RLC, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_RLC, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIB, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIB, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_GZIP, TB_ZIP_ACTION_INFLATE);
tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_GZIP, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIBRAW, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIBRAW, TB_ZIP_ACTION_DEFLATE);
// done
if (istream && ostream && fstream)
{
// save it
tb_hong_t save = 0;
if (iostream == istream) save = tb_transfer_done(fstream, ostream, 0, tb_null, tb_null);
else save = tb_transfer_done(istream, fstream, 0, tb_null, tb_null);
// trace
tb_trace_i("save: %lld bytes, size: %lld bytes", save, tb_stream_size(istream));
}
// exit fstream
tb_stream_exit(fstream);
// exit istream
tb_stream_exit(istream);
// exit ostream
tb_stream_exit(ostream);
return 0;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* interfaces
*/
tb_bool_t tb_dns_init()
{
// done
tb_size_t count = 0;
if (tb_file_info("/etc/resolv.conf", tb_null))
{
/* try get list from "/etc/resolv.conf"
*
* # Generated by NetworkManager
* nameserver 10.1.20.10
* nameserver 8.8.8.8
*
*/
tb_stream_ref_t stream = tb_stream_init_from_url("/etc/resolv.conf");
if (stream)
{
// open
if (tb_stream_open(stream))
{
// read
tb_long_t size = 0;
tb_char_t line[8192];
while ((size = tb_stream_bread_line(stream, line, 8192)) >= 0)
{
if (size && !tb_strnicmp(line, "nameserver", 10))
{
// seek to server
tb_char_t const* p = line + 10;
while (*p && !tb_isdigit(*p)) p++;
tb_check_continue(*p);
// add server
tb_dns_server_add(p);
// count++
count++;
}
}
}
// exit
tb_stream_exit(stream);
}
}
// no server? add the default server
if (!count)
{
tb_dns_server_add("8.8.8.8");
tb_dns_server_add("8.8.8.4");
}
// ok
return tb_true;
}
tb_void_t tb_xml_reader_exit(tb_xml_reader_ref_t reader)
{
// check
tb_xml_reader_impl_t* impl = (tb_xml_reader_impl_t*)reader;
tb_assert_and_check_return(impl);
// clos it first
tb_xml_reader_clos(reader);
// exit the filter stream
if (impl->fstream) tb_stream_exit(impl->fstream);
// exit text
tb_string_exit(&impl->text);
// exit version
tb_string_exit(&impl->version);
// exit charset
tb_string_exit(&impl->charset);
// exit element
tb_string_exit(&impl->element);
// exit element name
tb_string_exit(&impl->element_name);
// exit attribute name
tb_string_exit(&impl->attribute_name);
// exit attribute data
tb_string_exit(&impl->attribute_data);
// exit attributes
tb_long_t i = 0;
for (i = 0; i < TB_XML_READER_ATTRIBUTES_MAXN; i++)
{
tb_xml_node_ref_t node = (tb_xml_node_ref_t)(impl->attributes + i);
tb_string_exit(&node->name);
tb_string_exit(&node->data);
}
// free it
tb_free(impl);
}
tb_void_t tb_xml_reader_clos(tb_xml_reader_ref_t reader)
{
// check
tb_xml_reader_impl_t* impl = (tb_xml_reader_impl_t*)reader;
tb_assert_and_check_return(impl);
// clos the reader stream
if (impl->rstream) tb_stream_clos(impl->rstream);
impl->rstream = tb_null;
// exit the input stream
if (impl->istream && impl->bowner) tb_stream_exit(impl->istream);
impl->istream = tb_null;
// clear level
impl->level = 0;
// clear owner
impl->bowner = tb_false;
// clear text
tb_string_clear(&impl->text);
// clear element
tb_string_clear(&impl->element);
// clear name
tb_string_clear(&impl->element_name);
// clear attribute name
tb_string_clear(&impl->attribute_name);
// clear attribute data
tb_string_clear(&impl->attribute_data);
// clear attributes
tb_long_t i = 0;
for (i = 0; i < TB_XML_READER_ATTRIBUTES_MAXN; i++)
{
tb_xml_node_ref_t node = (tb_xml_node_ref_t)(impl->attributes + i);
tb_string_clear(&node->name);
tb_string_clear(&node->data);
}
}
gb_bitmap_ref_t gb_bitmap_init_from_url(tb_size_t pixfmt, tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(GB_PIXFMT_OK(pixfmt) && url, tb_null);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_null);
// init bitmap from stream
gb_bitmap_ref_t bitmap = tb_null;
if (tb_stream_open(stream)) bitmap = gb_bitmap_init_from_stream(pixfmt, stream);
// exit stream
tb_stream_exit(stream);
// ok?
return bitmap;
}
tb_void_t tb_dump_data_from_url(tb_char_t const* url)
{
// check
tb_assert_and_check_return(url);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
if (stream)
{
// open stream
if (tb_stream_open(stream))
{
// dump stream
tb_dump_data_from_stream(stream);
}
// exit stream
tb_stream_exit(stream);
}
}
tb_long_t tb_object_writ_to_data(tb_object_ref_t object, tb_byte_t* data, tb_size_t size, tb_size_t format)
{
// check
tb_assert_and_check_return_val(object && data && size, -1);
// make stream
tb_long_t writ = -1;
tb_stream_ref_t stream = tb_stream_init_from_data(data, size);
if (stream)
{
// open and writ stream
if (tb_stream_open(stream)) writ = tb_object_writ(object, stream, format);
// exit stream
tb_stream_exit(stream);
}
// ok?
return writ;
}
tb_void_t tb_dump_data(tb_byte_t const* data, tb_size_t size)
{
// check
tb_assert_and_check_return(data && size);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_data(data, size);
if (stream)
{
// open stream
if (tb_stream_open(stream))
{
// dump stream
tb_dump_data_from_stream(stream);
}
// exit stream
tb_stream_exit(stream);
}
}
tb_object_ref_t tb_object_read_from_data(tb_byte_t const* data, tb_size_t size)
{
// check
tb_assert_and_check_return_val(data && size, tb_null);
// init
tb_object_ref_t object = tb_null;
// make stream
tb_stream_ref_t stream = tb_stream_init_from_data(data, size);
tb_assert_and_check_return_val(stream, tb_null);
// read object
if (tb_stream_open(stream)) object = tb_object_read(stream);
// exit stream
tb_stream_exit(stream);
// ok?
return object;
}
tb_object_ref_t tb_object_read_from_url(tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// init
tb_object_ref_t object = tb_null;
// make stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_null);
// read object
if (tb_stream_open(stream)) object = tb_object_read(stream);
// exit stream
tb_stream_exit(stream);
// ok?
return object;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* interfaces
*/
tb_object_ref_t tb_object_data_init_from_url(tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_null);
// make stream
tb_object_ref_t object = tb_null;
if (tb_stream_open(stream))
{
// size
tb_hong_t size = tb_stream_size(stream);
if (size > 0 && size < TB_MAXS32)
{
tb_byte_t* data = tb_malloc0_bytes((tb_size_t)size);
if (data)
{
if (tb_stream_bread(stream, data, (tb_size_t)size))
object = tb_object_data_init_from_data(data, (tb_size_t)size);
tb_free(data);
}
}
else object = tb_object_data_init_from_data(tb_null, 0);
// check, TODO: read stream if no size
tb_assert(size >= 0 && size < TB_MAXS32);
// exit stream
tb_stream_exit(stream);
}
// ok?
return object;
}
static tb_void_t tb_demo_spider_parser_exit(tb_thread_pool_worker_ref_t worker, tb_cpointer_t priv)
{
// check
tb_demo_spider_parser_t* parser = (tb_demo_spider_parser_t*)priv;
tb_assert_and_check_return(parser);
// exit stream
if (parser->stream) tb_stream_exit(parser->stream);
parser->stream = tb_null;
// exit reader
if (parser->reader) tb_xml_reader_exit(parser->reader);
parser->reader = tb_null;
// exit cache
if (parser->cache) tb_circle_queue_exit(parser->cache);
parser->cache = tb_null;
// exit iurl
tb_url_exit(&parser->iurl);
// exit it
tb_free(parser);
}
tb_long_t tb_object_writ_to_url(tb_object_ref_t object, tb_char_t const* url, tb_size_t format)
{
// check
tb_assert_and_check_return_val(object && url, -1);
// make stream
tb_long_t writ = -1;
tb_stream_ref_t stream = tb_stream_init_from_url(url);
if (stream)
{
// ctrl stream
if (tb_stream_type(stream) == TB_STREAM_TYPE_FILE)
tb_stream_ctrl(stream, TB_STREAM_CTRL_FILE_SET_MODE, TB_FILE_MODE_RW | TB_FILE_MODE_BINARY | TB_FILE_MODE_CREAT | TB_FILE_MODE_TRUNC);
// open and writ stream
if (tb_stream_open(stream)) writ = tb_object_writ(object, stream, format);
// exit stream
tb_stream_exit(stream);
}
// ok?
return writ;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* main
*/
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
// done
tb_option_ref_t option = tb_null;
tb_stream_ref_t istream = tb_null;
tb_stream_ref_t ostream = tb_null;
tb_stream_ref_t pstream = tb_null;
do
{
// init option
option = tb_option_init("stream", "the stream demo", g_options);
tb_assert_and_check_break(option);
// done option
if (tb_option_done(option, argc - 1, &argv[1]))
{
// debug & verbose
tb_bool_t debug = tb_option_find(option, "debug");
tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true;
// done url
if (tb_option_find(option, "url"))
{
// init istream
istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url"));
tb_assert_and_check_break(istream);
// ctrl http
if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP)
{
// enable gzip?
if (tb_option_find(option, "gzip"))
{
// auto unzip
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break;
// need gzip
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break;
}
// enable debug?
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break;
// custem header?
if (tb_option_find(option, "header"))
{
// init
tb_string_t key;
tb_string_t val;
tb_string_init(&key);
tb_string_init(&val);
// done
tb_bool_t k = tb_true;
tb_char_t const* p = tb_option_item_cstr(option, "header");
while (*p)
{
// is key?
if (k)
{
if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++);
else if (*p == ':')
{
// skip ':'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// is val now
k = tb_false;
}
else p++;
}
// is val?
else
{
if (*p != ';') tb_string_chrcat(&val, *p++);
else
{
// skip ';'
p++;
// skip space
while (*p && tb_isspace(*p)) p++;
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// is key now
k = tb_true;
// clear key & val
tb_string_clear(&key);
tb_string_clear(&val);
}
}
}
// set header
if (tb_string_size(&key) && tb_string_size(&val))
{
if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
}
// exit
tb_string_exit(&key);
tb_string_exit(&val);
}
// keep alive?
if (tb_option_find(option, "keep-alive"))
{
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break;
}
// post-data?
if (tb_option_find(option, "post-data"))
{
tb_char_t const* post_data = tb_option_item_cstr(option, "post-data");
tb_hize_t post_size = tb_strlen(post_data);
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (debug) tb_printf("post: %llu\n", post_size);
}
// post-file?
else if (tb_option_find(option, "post-file"))
{
tb_char_t const* url = tb_option_item_cstr(option, "post-file");
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break;
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
if (debug) tb_printf("post: %s\n", url);
}
}
// set range
if (tb_option_find(option, "range"))
{
tb_char_t const* p = tb_option_item_cstr(option, "range");
if (p)
{
// the bof
tb_hize_t eof = 0;
tb_hize_t bof = tb_atoll(p);
while (*p && tb_isdigit(*p)) p++;
if (*p == '-')
{
p++;
eof = tb_atoll(p);
}
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break;
}
}
// set timeout
if (tb_option_find(option, "timeout"))
{
tb_size_t timeout = tb_option_item_uint32(option, "timeout");
if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break;
}
// print verbose info
if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url"));
// open istream
if (!tb_stream_open(istream))
{
// print verbose info
if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream)));
break;
}
// print verbose info
if (verbose) tb_printf("open: ok\n");
// init ostream
if (tb_option_find(option, "more0"))
{
// the path
tb_char_t const* path = tb_option_item_cstr(option, "more0");
// init
ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (verbose) tb_printf("save: %s\n", path);
}
else
{
// the name
tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/');
if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\');
if (!name) name = "/stream.file";
// the path
tb_char_t path[TB_PATH_MAXN] = {0};
if (tb_directory_current(path, TB_PATH_MAXN))
tb_strcat(path, name);
else break;
// init file
ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// print verbose info
if (verbose) tb_printf("save: %s\n", path);
}
tb_assert_and_check_break(ostream);
// the limit rate
tb_size_t limitrate = 0;
if (tb_option_find(option, "limitrate"))
limitrate = tb_option_item_uint32(option, "limitrate");
// save it
tb_hong_t save = 0;
tb_demo_context_t context = {0};
context.verbose = verbose;
if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break;
}
else tb_option_help(option);
}
else tb_option_help(option);
} while (0);
// exit pstream
if (pstream) tb_stream_exit(pstream);
pstream = tb_null;
// exit istream
if (istream) tb_stream_exit(istream);
istream = tb_null;
// exit ostream
if (ostream) tb_stream_exit(ostream);
ostream = tb_null;
// exit option
if (option) tb_option_exit(option);
option = tb_null;
return 0;
}
