static tb_demo_spider_parser_t* tb_demo_spider_parser_init(tb_thread_pool_worker_ref_t worker)
{
// check
tb_assert_and_check_return_val(worker, tb_null);
// done
tb_bool_t ok = tb_false;
tb_demo_spider_parser_t* parser = tb_null;
do
{
// attempt to get the parser
parser = (tb_demo_spider_parser_t*)tb_thread_pool_worker_getp(worker, 0);
if (!parser)
{
// make parser
parser = tb_malloc0_type(tb_demo_spider_parser_t);
tb_assert_and_check_break(parser);
// save parser
tb_thread_pool_worker_setp(worker, 0, tb_demo_spider_parser_exit, (tb_cpointer_t)parser);
// init stream
parser->stream = tb_stream_init_file();
tb_assert_and_check_break(parser->stream);
// init reader
parser->reader = tb_xml_reader_init();
tb_assert_and_check_break(parser->reader);
// init cache
parser->cache = tb_circle_queue_init(255, tb_item_func_str(tb_true));
tb_assert_and_check_break(parser->cache);
// init iurl
if (!tb_url_init(&parser->iurl)) break;
}
// ok
ok = tb_true;
} while (0);
// failed?
if (!ok)
{
// exit it
if (parser) tb_demo_spider_parser_exit(worker, (tb_cpointer_t)parser);
parser = tb_null;
}
// ok
return parser;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* implementation
*/
tb_database_sql_ref_t tb_database_sql_init(tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// done
tb_bool_t ok = tb_false;
tb_database_sql_ref_t database = tb_null;
tb_url_t database_url;
do
{
// trace
tb_trace_d("init: %s: ..", url);
// init url
if (!tb_url_init(&database_url)) break;
// make url
if (!tb_url_cstr_set(&database_url, url)) break;
// check protocol
tb_size_t protocol = tb_url_protocol(&database_url);
tb_assert_and_check_break(protocol == TB_URL_PROTOCOL_SQL || protocol == TB_URL_PROTOCOL_FILE);
// the probe func
static tb_size_t (*s_probe[])(tb_url_ref_t) =
{
tb_null
#ifdef TB_CONFIG_PACKAGE_HAVE_MYSQL
, tb_database_mysql_probe
#endif
#ifdef TB_CONFIG_PACKAGE_HAVE_SQLITE3
, tb_database_sqlite3_probe
#endif
};
// the init func
static tb_database_sql_ref_t (*s_init[])(tb_url_ref_t) =
{
tb_null
#ifdef TB_CONFIG_PACKAGE_HAVE_MYSQL
, tb_database_mysql_init
#endif
#ifdef TB_CONFIG_PACKAGE_HAVE_SQLITE3
, tb_database_sqlite3_init
#endif
};
// probe the database type
tb_size_t i = 1;
tb_size_t n = tb_arrayn(s_probe);
tb_size_t s = 0;
tb_size_t m = 0;
for (; i < n; i++)
{
if (s_probe[i])
{
// probe it
tb_size_t score = s_probe[i](&database_url);
if (score > s)
{
// save the max score
s = score;
m = i;
// ok?
if (score == 100) break;
}
}
}
tb_check_break(m < n && s_init[m]);
// init it
database = s_init[m](&database_url);
tb_assert_and_check_break(database);
// trace
tb_trace_d("init: %s: ok", url);
// ok
ok = tb_true;
} while (0);
// exit url
tb_url_exit(&database_url);
// failed?
if (!ok)
{
// trace
tb_trace_d("init: %s: no", url);
// exit database
if (database) tb_database_sql_exit(database);
database = tb_null;
}
// ok?
return database;
}
tb_database_sql_ref_t tb_database_sqlite3_init(tb_url_ref_t url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// done
tb_bool_t ok = tb_false;
tb_database_sqlite3_t* sqlite = tb_null;
do
{
// make database
sqlite = tb_malloc0_type(tb_database_sqlite3_t);
tb_assert_and_check_break(sqlite);
// init database
sqlite->base.type = TB_DATABASE_SQL_TYPE_SQLITE3;
sqlite->base.open = tb_database_sqlite3_open;
sqlite->base.clos = tb_database_sqlite3_clos;
sqlite->base.exit = tb_database_sqlite3_exit;
sqlite->base.done = tb_database_sqlite3_done;
sqlite->base.begin = tb_database_sqlite3_begin;
sqlite->base.commit = tb_database_sqlite3_commit;
sqlite->base.rollback = tb_database_sqlite3_rollback;
sqlite->base.result_load = tb_database_sqlite3_result_load;
sqlite->base.result_exit = tb_database_sqlite3_result_exit;
sqlite->base.statement_init = tb_database_sqlite3_statement_init;
sqlite->base.statement_exit = tb_database_sqlite3_statement_exit;
sqlite->base.statement_done = tb_database_sqlite3_statement_done;
sqlite->base.statement_bind = tb_database_sqlite3_statement_bind;
// init row operation
static tb_iterator_op_t row_op =
{
tb_database_sqlite3_result_row_iterator_size
, tb_database_sqlite3_result_row_iterator_head
, tb_null
, tb_database_sqlite3_result_row_iterator_tail
, tb_database_sqlite3_result_row_iterator_prev
, tb_database_sqlite3_result_row_iterator_next
, tb_database_sqlite3_result_row_iterator_item
, tb_null
, tb_null
, tb_null
, tb_null
};
// init col operation
static tb_iterator_op_t col_op =
{
tb_database_sqlite3_result_col_iterator_size
, tb_database_sqlite3_result_col_iterator_head
, tb_null
, tb_database_sqlite3_result_col_iterator_tail
, tb_database_sqlite3_result_col_iterator_prev
, tb_database_sqlite3_result_col_iterator_next
, tb_database_sqlite3_result_col_iterator_item
, tb_null
, tb_null
, tb_null
, tb_null
};
// init result row iterator
sqlite->result.itor.priv = (tb_pointer_t)sqlite;
sqlite->result.itor.step = 0;
sqlite->result.itor.mode = 0;
sqlite->result.itor.op = &row_op;
// init result col iterator
sqlite->result.row.itor.priv = (tb_pointer_t)sqlite;
sqlite->result.row.itor.step = 0;
sqlite->result.row.itor.mode = TB_ITERATOR_MODE_RACCESS | TB_ITERATOR_MODE_READONLY;
sqlite->result.row.itor.op = &col_op;
// init url
if (!tb_url_init(&sqlite->base.url)) break;
// copy url
tb_url_copy(&sqlite->base.url, url);
// init state
sqlite->base.state = TB_STATE_OK;
// ok
ok = tb_true;
} while (0);
// failed?
if (!ok)
{
// exit database
if (sqlite) tb_database_sqlite3_exit((tb_database_sql_impl_t*)sqlite);
sqlite = tb_null;
}
// ok?
return (tb_database_sql_ref_t)sqlite;
}
static tb_bool_t tb_demo_spider_init(tb_demo_spider_t* spider, tb_int_t argc, tb_char_t** argv)
{
// check
tb_assert_and_check_return_val(spider && argc && argv, tb_false);
// done
tb_bool_t ok = tb_false;
do
{
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
// init option
spider->option = tb_option_init("spider", "the spider demo", g_options);
tb_assert_and_check_break(spider->option);
// done option
if (!tb_option_done(spider->option, argc - 1, &argv[1])) break;
// check
tb_assert_and_check_break(tb_option_find(spider->option, "home"));
// init home
if (!tb_url_init(&spider->home)) break;
tb_url_set(&spider->home, tb_option_item_cstr(spider->option, "home"));
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init only home?
if (tb_option_find(spider->option, "only"))
spider->home_only = tb_option_item_bool(spider->option, "only");
// init root
tb_char_t const* root = tb_option_item_cstr(spider->option, "directory");
// init user agent
spider->user_agent = tb_option_item_cstr(spider->option, "agent");
// init timeout
if (tb_option_find(spider->option, "timeout"))
spider->timeout = tb_option_item_sint32(spider->option, "timeout");
// init limited rate
if (tb_option_find(spider->option, "rate"))
spider->limited_rate = tb_option_item_uint32(spider->option, "rate");
#else
// check
tb_assert_and_check_break(argv[1]);
// init home
if (!tb_url_init(&spider->home)) break;
spider->home = argv[1]? argv[1] : tb_null;
tb_trace_d("home: %s", tb_url_get(&spider->home));
// init root
tb_char_t const* root = argv[2];
#endif
// the home host
tb_char_t const* host = tb_url_host_get(&spider->home);
tb_assert_and_check_break(host);
// init home domain
tb_char_t const* domain = tb_strchr(host, '.');
if (domain)
{
tb_strlcpy(spider->home_domain, domain, sizeof(spider->home_domain) - 1);
spider->home_domain[sizeof(spider->home_domain) - 1] = '\0';
}
// using the default root
if (root) tb_strlcpy(spider->root, root, sizeof(spider->root) - 1);
else
{
// the temporary root
tb_directory_temp(spider->root, sizeof(spider->root) - 1);
// append spider
tb_strcat(spider->root, "/spider");
}
tb_trace_d("root: %s", spider->root);
// using the default user agent
if (!spider->user_agent) spider->user_agent = TB_DEMO_SPIDER_USER_AGENT;
// using the default timeout
if (!spider->timeout) spider->timeout = TB_DEMO_SPIDER_TASK_TIMEOUT;
// using the default rate
if (!spider->limited_rate) spider->limited_rate = TB_DEMO_SPIDER_TASK_RATE;
// strip root tail: '/' or '\\'
tb_size_t size = tb_strlen(spider->root);
if (size && (spider->root[size - 1] == '/' || spider->root[size - 1] == '\\')) spider->root[size - 1] = '\0';
// init state
spider->state = TB_STATE_OK;
// init lock
if (!tb_spinlock_init(&spider->lock)) break;
// init pool
spider->pool = tb_fixed_pool_init(tb_null, TB_DEMO_SPIDER_TASK_MAXN >> 2, sizeof(tb_demo_spider_task_t), tb_null, tb_null, tb_null);
tb_assert_and_check_break(spider->pool);
// init filter
spider->filter = tb_bloom_filter_init(TB_BLOOM_FILTER_PROBABILITY_0_001, 3, TB_DEMO_SPIDER_FILTER_MAXN, tb_item_func_str(tb_true));
tb_assert_and_check_break(spider->filter);
// register lock profiler
#ifdef TB_LOCK_PROFILER_ENABLE
tb_lock_profiler_register(tb_lock_profiler(), (tb_pointer_t)&spider->lock, "spider");
#endif
// ok
ok = tb_true;
} while (0);
// failed? help it
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
if (!ok && spider->option) tb_option_help(spider->option);
#endif
// ok?
return ok;
}
