Ejemplo n.º 1
0
tb_void_t tb_circle_queue_clear(tb_circle_queue_ref_t self)
{
    // check
    tb_circle_queue_t* queue = (tb_circle_queue_t*)self;
    tb_assert_and_check_return(queue);
    
    // clear it
    while (!tb_circle_queue_null(self)) tb_circle_queue_pop(self);
    queue->head = 0;
    queue->tail = 0;
    queue->size = 0;
}
Ejemplo n.º 2
0
static tb_void_t tb_demo_spider_parser_task_done(tb_thread_pool_worker_ref_t worker, tb_cpointer_t priv)
{
    // check
    tb_demo_spider_task_t* task = (tb_demo_spider_task_t*)priv;
    tb_assert_and_check_return(worker && task && task->spider);

    // init parser
    tb_demo_spider_parser_t* parser = tb_demo_spider_parser_init(worker);
    tb_assert_and_check_return(parser && parser->stream && parser->reader && parser->cache);

    // open stream
    if (tb_demo_spider_parser_open_html(parser->stream, task->ourl))
    {
        // open reader
        if (tb_xml_reader_open(parser->reader, parser->stream, tb_false))
        {
            // trace
            tb_trace_d("parser: open: %s", task->ourl);

            // init url
            tb_url_set(&parser->iurl, task->iurl);

            // parse url
            while (     TB_STATE_OK == tb_atomic_get(&task->spider->state)
                    &&  tb_demo_spider_parser_get_url(parser->reader, &parser->iurl))
            {
                // trace
                tb_trace_d("parser: done: %s", tb_url_get(&parser->iurl));

                // done task
                tb_bool_t full = tb_false;
                if (!tb_demo_spider_task_done(task->spider, tb_url_get(&parser->iurl), &full))
                {
                    // full?
                    tb_assert_and_check_break(full);

                    // cache url
                    if (!tb_circle_queue_full(parser->cache)) tb_circle_queue_put(parser->cache, tb_url_get(&parser->iurl));

                    // trace
                    tb_trace_d("parser: cache: save: %s, size: %lu", tb_url_get(&parser->iurl), tb_circle_queue_size(parser->cache));
                }
            }

            // clos reader
            tb_xml_reader_clos(parser->reader);
        }

        // clos stream
        tb_stream_clos(parser->stream);
    }

    // done task from the cache
    while (!tb_circle_queue_null(parser->cache))
    {
        // the url
        tb_char_t const* url = (tb_char_t const*)tb_circle_queue_get(parser->cache);
        tb_assert_and_check_break(url);

        // done task
        if (!tb_demo_spider_task_done(task->spider, url, tb_null)) break;

        // trace
        tb_trace_d("parser: cache: load: %s, size: %lu", url, tb_circle_queue_size(parser->cache));

        // pop it
        tb_circle_queue_pop(parser->cache);
    }
}