tb_int_t tb_demo_stream_zip_main(tb_int_t argc, tb_char_t** argv)
{
// init istream
tb_stream_ref_t istream = tb_stream_init_from_url(argv[1]);
// init ostream
tb_stream_ref_t ostream = tb_stream_init_from_file(argv[2], TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
// filter istream or ostream?
tb_stream_ref_t iostream = istream;
// tb_stream_ref_t iostream = ostream;
// init fstream
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_RLC, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_RLC, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIB, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIB, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_GZIP, TB_ZIP_ACTION_INFLATE);
tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_GZIP, TB_ZIP_ACTION_DEFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIBRAW, TB_ZIP_ACTION_INFLATE);
// tb_stream_ref_t fstream = tb_stream_init_filter_from_zip(iostream, TB_ZIP_ALGO_ZLIBRAW, TB_ZIP_ACTION_DEFLATE);
// done
if (istream && ostream && fstream)
{
// save it
tb_hong_t save = 0;
if (iostream == istream) save = tb_transfer_done(fstream, ostream, 0, tb_null, tb_null);
else save = tb_transfer_done(istream, fstream, 0, tb_null, tb_null);
// trace
tb_trace_i("save: %lld bytes, size: %lld bytes", save, tb_stream_size(istream));
}
// exit fstream
tb_stream_exit(fstream);
// exit istream
tb_stream_exit(istream);
// exit ostream
tb_stream_exit(ostream);
return 0;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* interfaces
*/
tb_object_ref_t tb_object_data_init_from_url(tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(url, tb_null);
// init stream
tb_stream_ref_t stream = tb_stream_init_from_url(url);
tb_assert_and_check_return_val(stream, tb_null);
// make stream
tb_object_ref_t object = tb_null;
if (tb_stream_open(stream))
{
// size
tb_hong_t size = tb_stream_size(stream);
if (size > 0 && size < TB_MAXS32)
{
tb_byte_t* data = tb_malloc0_bytes((tb_size_t)size);
if (data)
{
if (tb_stream_bread(stream, data, (tb_size_t)size))
object = tb_object_data_init_from_data(data, (tb_size_t)size);
tb_free(data);
}
}
else object = tb_object_data_init_from_data(tb_null, 0);
// check, TODO: read stream if no size
tb_assert(size >= 0 && size < TB_MAXS32);
// exit stream
tb_stream_exit(stream);
}
// ok?
return object;
}
static tb_bool_t tb_database_sqlite3_statement_bind(tb_database_sql_impl_t* database, tb_database_sql_statement_ref_t statement, tb_database_sql_value_t const* list, tb_size_t size)
{
// check
tb_database_sqlite3_t* sqlite = tb_database_sqlite3_cast(database);
tb_assert_and_check_return_val(sqlite && sqlite->database && statement && list && size, tb_false);
// the param count
tb_size_t param_count = (tb_size_t)sqlite3_bind_parameter_count((sqlite3_stmt*)statement);
tb_assert_and_check_return_val(size == param_count, tb_false);
// walk
tb_size_t i = 0;
for (i = 0; i < size; i++)
{
// the value
tb_database_sql_value_t const* value = &list[i];
// done
tb_int_t ok = SQLITE_ERROR;
tb_byte_t* data = tb_null;
switch (value->type)
{
case TB_DATABASE_SQL_VALUE_TYPE_TEXT:
tb_trace_i("sqlite3: test %lu %s", i, value->u.text.data);
ok = sqlite3_bind_text((sqlite3_stmt*)statement, (tb_int_t)(i + 1), value->u.text.data, (tb_int_t)tb_database_sql_value_size(value), tb_null);
break;
case TB_DATABASE_SQL_VALUE_TYPE_INT64:
case TB_DATABASE_SQL_VALUE_TYPE_UINT64:
ok = sqlite3_bind_int64((sqlite3_stmt*)statement, (tb_int_t)(i + 1), tb_database_sql_value_int64(value));
break;
case TB_DATABASE_SQL_VALUE_TYPE_INT32:
case TB_DATABASE_SQL_VALUE_TYPE_INT16:
case TB_DATABASE_SQL_VALUE_TYPE_INT8:
case TB_DATABASE_SQL_VALUE_TYPE_UINT32:
case TB_DATABASE_SQL_VALUE_TYPE_UINT16:
case TB_DATABASE_SQL_VALUE_TYPE_UINT8:
ok = sqlite3_bind_int((sqlite3_stmt*)statement, (tb_int_t)(i + 1), (tb_int_t)tb_database_sql_value_int32(value));
break;
case TB_DATABASE_SQL_VALUE_TYPE_BLOB16:
case TB_DATABASE_SQL_VALUE_TYPE_BLOB8:
ok = sqlite3_bind_blob((sqlite3_stmt*)statement, (tb_int_t)(i + 1), value->u.blob.data, (tb_int_t)value->u.blob.size, tb_null);
break;
case TB_DATABASE_SQL_VALUE_TYPE_BLOB32:
{
if (value->u.blob.stream)
{
// done
do
{
// the stream size
tb_hong_t size = tb_stream_size(value->u.blob.stream);
tb_assert_and_check_break(size >= 0);
// make data
data = tb_malloc0_bytes((tb_size_t)size);
tb_assert_and_check_break(data);
// read data
if (!tb_stream_bread(value->u.blob.stream, data, (tb_size_t)size)) break;
// bind it
ok = sqlite3_bind_blob((sqlite3_stmt*)statement, (tb_int_t)(i + 1), data, (tb_int_t)size, tb_database_sqlite3_statement_bind_exit);
} while (0);
}
else ok = sqlite3_bind_blob((sqlite3_stmt*)statement, (tb_int_t)(i + 1), value->u.blob.data, (tb_int_t)value->u.blob.size, tb_null);
}
break;
#ifdef TB_CONFIG_TYPE_HAVE_FLOAT
case TB_DATABASE_SQL_VALUE_TYPE_FLOAT:
case TB_DATABASE_SQL_VALUE_TYPE_DOUBLE:
ok = sqlite3_bind_double((sqlite3_stmt*)statement, (tb_int_t)(i + 1), (tb_double_t)tb_database_sql_value_double(value));
break;
#endif
case TB_DATABASE_SQL_VALUE_TYPE_NULL:
ok = sqlite3_bind_null((sqlite3_stmt*)statement, (tb_int_t)(i + 1));
break;
default:
tb_trace_e("statement: bind: unknown value type: %lu", value->type);
break;
}
// failed?
if (SQLITE_OK != ok)
{
// exit data
if (data) tb_free(data);
data = tb_null;
// save state
sqlite->base.state = tb_database_sqlite3_state_from_errno(sqlite3_errcode(sqlite->database));
// trace
tb_trace_e("statement: bind value[%lu] failed, error[%d]: %s", i, sqlite3_errcode(sqlite->database), sqlite3_errmsg(sqlite->database));
break;
}
}
// ok?
return (i == size)? tb_true : tb_false;
}
/* //////////////////////////////////////////////////////////////////////////////////////
* implementation
*/
static tb_bool_t tb_demo_spider_parser_open_html(tb_stream_ref_t stream, tb_char_t const* url)
{
// check
tb_assert_and_check_return_val(stream && url, tb_false);
// done
tb_bool_t ok = tb_false;
do
{
// find the .suffix
tb_char_t const* p = tb_strrchr(url, '.');
if (p)
{
// not html?
tb_check_break ( tb_stricmp(p, ".css")
&& tb_stricmp(p, ".js")
&& tb_stricmp(p, ".png")
&& tb_stricmp(p, ".jpg")
&& tb_stricmp(p, ".gif")
&& tb_stricmp(p, ".ico")
&& tb_stricmp(p, ".bmp")
&& tb_stricmp(p, ".mp4")
&& tb_stricmp(p, ".mp3")
&& tb_stricmp(p, ".flv")
&& tb_stricmp(p, ".avi")
&& tb_stricmp(p, ".exe")
&& tb_stricmp(p, ".msi")
&& tb_stricmp(p, ".zip")
&& tb_stricmp(p, ".rar")
&& tb_stricmp(p, ".7z"));
}
// ctrl stream
if (!tb_stream_ctrl(stream, TB_STREAM_CTRL_SET_URL, url)) break;
// open stream
if (!tb_stream_open(stream)) break;
// the stream size
tb_hong_t size = tb_stream_size(stream);
tb_check_break(size);
// prefetch some data
tb_byte_t* data = tb_null;
tb_size_t need = tb_min((tb_size_t)size, 256);
if (!tb_stream_need(stream, &data, need)) break;
// is html?
if (tb_strnistr((tb_char_t const*)data, need, "<!DOCTYPE html>"))
{
ok = tb_true;
break;
}
// is html?
ok = tb_strnistr((tb_char_t const*)data, need, "<html")? tb_true : tb_false;
} while (0);
// failed?
if (!ok)
{
// clos stream
if (stream) tb_stream_clos(stream);
}
// ok?
return ok;
}
static tb_object_ref_t tb_object_bplist_reader_done(tb_stream_ref_t stream)
{
// check
tb_assert_and_check_return_val(stream, tb_null);
// init root
tb_object_ref_t root = tb_null;
// init reader
tb_object_bplist_reader_t reader = {0};
reader.stream = stream;
// init size
tb_hize_t size = tb_stream_size(stream);
tb_assert_and_check_return_val(size, tb_null);
// init data
tb_byte_t data[32] = {0};
// read magic & version
if (!tb_stream_bread(stream, data, 8)) return tb_null;
// check magic & version
if (tb_strncmp((tb_char_t const*)data, "bplist00", 8)) return tb_null;
// seek to tail
if (!tb_stream_seek(stream, size - 26)) return tb_null;
// read offset size
tb_size_t offset_size = tb_stream_bread_u8(stream);
tb_trace_d("offset_size: %lu", offset_size);
// read item size for array and dictionary
tb_size_t item_size = tb_stream_bread_u8(stream);
tb_trace_d("item_size: %lu", item_size);
// read object count
tb_size_t object_count = (tb_size_t)tb_stream_bread_u64_be(stream);
tb_trace_d("object_count: %lu", object_count);
// read root object
tb_size_t root_object = (tb_size_t)tb_stream_bread_u64_be(stream);
tb_trace_d("root_object: %lu", root_object);
// read offset table index
tb_size_t offset_table_index = (tb_size_t)tb_stream_bread_u64_be(stream);
tb_trace_d("offset_table_index: %lu", offset_table_index);
// check
tb_assert_and_check_return_val(item_size && offset_size && object_count, tb_null);
// init object hash
tb_object_ref_t* object_hash = (tb_object_ref_t*)tb_malloc0(sizeof(tb_object_ref_t) * object_count);
tb_assert_and_check_return_val(object_hash, tb_null);
// done
tb_bool_t failed = tb_false;
do
{
// walk
tb_size_t i = 0;
for (i = 0; i < object_count; i++)
{
// seek to the offset entry
if (!tb_stream_seek(stream, offset_table_index + i * offset_size))
{
failed = tb_true;
break;
}
// read the object offset
tb_hize_t offset = 0;
switch (offset_size)
{
case 1:
offset = tb_stream_bread_u8(stream);
break;
case 2:
offset = tb_stream_bread_u16_be(stream);
break;
case 4:
offset = tb_stream_bread_u32_be(stream);
break;
case 8:
offset = tb_stream_bread_u64_be(stream);
break;
default:
return tb_null;
break;
}
// seek to the object offset
if (!tb_stream_seek(stream, offset))
{
failed = tb_true;
break;
}
// read object
object_hash[i] = tb_object_bplist_reader_func_object(&reader, item_size);
// if (object_hash[i]) tb_object_dump(object_hash[i]);
}
// failed?
tb_check_break(!failed);
// build array & dictionary items
for (i = 0; i < object_count; i++)
{
tb_object_ref_t object = object_hash[i];
if (object)
{
switch (tb_object_type(object))
{
case TB_OBJECT_TYPE_ARRAY:
{
// the priv data
tb_byte_t* priv = (tb_byte_t*)tb_object_getp(object);
if (priv)
{
// count
tb_size_t count = (tb_size_t)tb_bits_get_u32_ne(priv);
if (count)
{
// goto item data
tb_byte_t const* p = priv + sizeof(tb_uint32_t);
// walk items
tb_size_t j = 0;
for (i = 0; j < count; j++)
{
// the item index
tb_size_t item = tb_object_bplist_bits_get(p + j * item_size, item_size);
tb_assert(item < object_count && object_hash[item]);
// tb_trace_d("item: %d", item);
// append item
if (item < object_count && object_hash[item])
{
tb_object_inc(object_hash[item]);
tb_object_array_append(object, object_hash[item]);
}
}
}
// exit priv
tb_free(priv);
tb_object_setp(object, tb_null);
// tb_object_dump(object);
}
}
break;
case TB_OBJECT_TYPE_DICTIONARY:
{
// the priv data
tb_byte_t* priv = (tb_byte_t*)tb_object_getp(object);
if (priv)
{
// count
tb_size_t count = (tb_size_t)tb_bits_get_u32_ne(priv);
if (count)
{
// goto item data
tb_byte_t const* p = priv + sizeof(tb_uint32_t);
// walk items
tb_size_t j = 0;
for (i = 0; j < count; j++)
{
// the key & val
tb_size_t key = tb_object_bplist_bits_get(p + j * item_size, item_size);
tb_size_t val = tb_object_bplist_bits_get(p + (count + j) * item_size, item_size);
tb_assert(key < object_count && object_hash[key]);
tb_assert(val < object_count && object_hash[val]);
// tb_trace_d("key_val: %u => %lu", key, val);
// append the key & val
if (key < object_count && val < object_count && object_hash[key] && object_hash[val])
{
// key must be string now.
tb_assert(tb_object_type(object_hash[key]) == TB_OBJECT_TYPE_STRING);
if (tb_object_type(object_hash[key]) == TB_OBJECT_TYPE_STRING)
{
// set key => val
tb_char_t const* skey = tb_object_string_cstr(object_hash[key]);
if (skey)
{
tb_object_inc(object_hash[val]);
tb_object_dictionary_set(object, skey, object_hash[val]);
}
tb_assert(skey);
}
}
}
}
// exit priv
tb_free(priv);
tb_object_setp(object, tb_null);
// tb_object_dump(object);
}
}
break;
default:
break;
}
}
}
} while (0);
// exit object hash
if (object_hash)
{
// root
if (root_object < object_count) root = object_hash[root_object];
// refn--
tb_size_t i;
for (i = 0; i < object_count; i++)
{
if (object_hash[i] && i != root_object)
tb_object_dec(object_hash[i]);
}
// exit object hash
tb_free(object_hash);
object_hash = tb_null;
}
// ok?
return root;
}
