tb_long_t tb_charset_conv_bst(tb_size_t ftype, tb_size_t ttype, tb_static_stream_ref_t fst, tb_static_stream_ref_t tst) { // check tb_assert_and_check_return_val(TB_CHARSET_TYPE_OK(ftype) && TB_CHARSET_TYPE_OK(ttype) && fst && tst, -1); tb_assert_and_check_return_val(tb_static_stream_valid(fst) && tb_static_stream_valid(tst), -1); // init the charset tb_charset_ref_t fr = tb_charset_find_by_type(ftype); tb_charset_ref_t to = tb_charset_find_by_type(ttype); tb_assert_and_check_return_val(fr && to && fr->get && fr->set, -1); // no data? tb_check_return_val(tb_static_stream_left(fst), 0); // big endian? tb_bool_t fbe = !(ftype & TB_CHARSET_TYPE_LE)? tb_true : tb_false; tb_bool_t tbe = !(ttype & TB_CHARSET_TYPE_LE)? tb_true : tb_false; // walk tb_uint32_t ch; tb_byte_t const* tp = tb_static_stream_pos(tst); while (tb_static_stream_left(fst) && tb_static_stream_left(tst)) { // get ucs4 character tb_long_t ok = 0; if ((ok = fr->get(fst, fbe, &ch)) > 0) { // set ucs4 character if (to->set(tst, tbe, ch) < 0) break; } else if (ok < 0) break; } // ok? return tb_static_stream_pos(tst) - tp; }
tb_long_t tb_charset_utf16_get(tb_static_stream_ref_t sstream, tb_bool_t be, tb_uint32_t* ch) { // init tb_byte_t const* p = tb_static_stream_pos(sstream); tb_byte_t const* q = p; tb_size_t n = tb_static_stream_left(sstream); // not enough? break it tb_check_return_val(n > 1, -1); // the first character tb_uint32_t c = be? tb_bits_get_u16_be(p) : tb_bits_get_u16_le(p); p += 2; // large? if (c >= 0xd800 && c <= 0xdbff) { // not enough? break it tb_check_return_val(n > 3, -1); // the next character tb_uint32_t c2 = be? tb_bits_get_u16_be(p) : tb_bits_get_u16_le(p); if (c2 >= 0xdc00 && c2 <= 0xdfff) { c = ((c - 0xd800) << 10) + (c2 - 0xdc00) + 0x0010000; p += 2; }; }; // next if (p > q) tb_static_stream_skip(sstream, p - q); // set character *ch = c; // ok? return p > q? 1 : 0; }
tb_long_t tb_filter_spak(tb_filter_ref_t self, tb_byte_t const* data, tb_size_t size, tb_byte_t const** pdata, tb_size_t need, tb_long_t sync) { // check tb_filter_t* filter = (tb_filter_t*)self; tb_assert_and_check_return_val(filter && filter->spak && pdata, -1); // init odata *pdata = tb_null; // save the input offset filter->offset += size; // eof? if (filter->limit >= 0 && filter->offset == filter->limit) filter->beof = tb_true; // eof? sync it if (filter->beof) sync = -1; // the idata tb_byte_t const* idata = tb_buffer_data(&filter->idata); tb_size_t isize = tb_buffer_size(&filter->idata); if (data && size) { // append data to cache if have the cache data if (idata && isize) { // trace tb_trace_d("[%p]: append idata: %lu", self, size); // append data idata = tb_buffer_memncat(&filter->idata, data, size); isize = tb_buffer_size(&filter->idata); } // using the data directly if no cache data else { // trace tb_trace_d("[%p]: using idata directly: %lu", self, size); // using it directly idata = data; isize = size; } } // sync data if null else { // check sync tb_assert_and_check_return_val(sync, 0); } // the need if (!need) need = tb_max(size, tb_queue_buffer_maxn(&filter->odata)); tb_assert_and_check_return_val(need, -1); // init pull tb_size_t omaxn = 0; tb_byte_t* odata = tb_queue_buffer_pull_init(&filter->odata, &omaxn); if (odata) { // the osize tb_long_t osize = omaxn >= need? need : 0; // exit pull if (odata) tb_queue_buffer_pull_exit(&filter->odata, osize > 0? osize : 0); // enough? if (osize > 0) { // append to the cache if idata is not belong to the cache if (size && idata == data) tb_buffer_memncat(&filter->idata, data, size); // return it directly *pdata = odata; return osize; } } // grow odata maxn if not enough if (need > tb_queue_buffer_maxn(&filter->odata)) tb_queue_buffer_resize(&filter->odata, need); // the odata omaxn = 0; odata = tb_queue_buffer_push_init(&filter->odata, &omaxn); tb_assert_and_check_return_val(odata && omaxn, -1); // init stream tb_static_stream_t istream = {0}; tb_static_stream_t ostream = {0}; if (idata && isize) { // @note istream maybe null for sync the end data if (!tb_static_stream_init(&istream, (tb_byte_t*)idata, isize)) return -1; } if (!tb_static_stream_init(&ostream, (tb_byte_t*)odata, omaxn)) return -1; // trace tb_trace_d("[%p]: spak: ileft: %lu, oleft: %lu, offset: %llu, limit: %lld, beof: %d: ..", self, tb_buffer_size(&filter->idata), tb_queue_buffer_size(&filter->odata), filter->offset, filter->limit, filter->beof); // spak data tb_long_t osize = filter->spak(filter, &istream, &ostream, sync); // eof? if (osize < 0) filter->beof = tb_true; // no data and eof? if (!osize && !tb_static_stream_left(&istream) && filter->beof) osize = -1; // eof? sync it if (filter->beof) sync = -1; // exit odata tb_queue_buffer_push_exit(&filter->odata, osize > 0? osize : 0); // have the left idata? tb_size_t left = tb_static_stream_left(&istream); if (left) { // move to the cache head if idata is belong to the cache if (idata != data) { // trace tb_trace_d("[%p]: move to the cache head: %lu", self, left); tb_buffer_memnmov(&filter->idata, tb_static_stream_offset(&istream), left); } // append to the cache if idata is not belong to the cache else { // trace tb_trace_d("[%p]: append to the cache: %lu", self, left); tb_buffer_memncat(&filter->idata, tb_static_stream_pos(&istream), left); } } // clear the cache else tb_buffer_clear(&filter->idata); // init pull omaxn = 0; odata = tb_queue_buffer_pull_init(&filter->odata, &omaxn); // no sync? cache the output data if (!sync) osize = omaxn >= need? need : 0; // sync and has data? return it directly else if (omaxn) osize = tb_min(omaxn, need); // sync, no data or end? // else osize = osize; // exit pull if (odata) tb_queue_buffer_pull_exit(&filter->odata, osize > 0? osize : 0); // return it if have the odata if (osize > 0) *pdata = odata; // trace tb_trace_d("[%p]: spak: ileft: %lu, oleft: %lu, offset: %llu, limit: %lld, beof: %d: %ld", self, tb_buffer_size(&filter->idata), tb_queue_buffer_size(&filter->odata), filter->offset, filter->limit, filter->beof, osize); // ok? return osize; }
/* chunked_data * * head data tail * ea5\r\n ..........\r\n e65\r\n..............\r\n 0\r\n\r\n * ---------------------- ------------------------- --------- * chunk0 chunk1 end */ static tb_long_t tb_filter_chunked_spak(tb_filter_t* filter, tb_static_stream_ref_t istream, tb_static_stream_ref_t ostream, tb_long_t sync) { // check tb_filter_chunked_t* cfilter = tb_filter_chunked_cast(filter); tb_assert_and_check_return_val(cfilter && istream && ostream, -1); tb_assert_and_check_return_val(tb_static_stream_valid(istream) && tb_static_stream_valid(ostream), -1); // the idata tb_byte_t const* ip = tb_static_stream_pos(istream); tb_byte_t const* ie = tb_static_stream_end(istream); // trace tb_trace_d("[%p]: isize: %lu, beof: %d", cfilter, tb_static_stream_size(istream), filter->beof); // find the eof: '\r\n 0\r\n\r\n' if ( !filter->beof && ip + 6 < ie && ie[-7] == '\r' && ie[-6] == '\n' && ie[-5] == '0' && ie[-4] == '\r' && ie[-3] == '\n' && ie[-2] == '\r' && ie[-1] == '\n') { // is eof filter->beof = tb_true; } // the odata tb_byte_t* op = (tb_byte_t*)tb_static_stream_pos(ostream); tb_byte_t* oe = (tb_byte_t*)tb_static_stream_end(ostream); tb_byte_t* ob = op; // parse chunked head and chunked tail if (!cfilter->size || cfilter->read >= cfilter->size) { // walk while (ip < ie) { // the charactor tb_char_t ch = *ip++; // trace tb_trace_d("[%p]: character: %x", cfilter, ch); // check tb_assert_and_check_return_val(ch, -1); // append char to line if (ch != '\n') tb_string_chrcat(&cfilter->line, ch); // is line end? else { // check tb_char_t const* pb = tb_string_cstr(&cfilter->line); tb_size_t pn = tb_string_size(&cfilter->line); tb_assert_and_check_return_val(pb, -1); // trace tb_trace_d("[%p]: line: %s", cfilter, tb_string_cstr(&cfilter->line)); // strip '\r' if exists if (pb[pn - 1] == '\r') tb_string_strip(&cfilter->line, pn - 1); // is chunked tail? only "\r\n" if (!tb_string_size(&cfilter->line)) { // reset size cfilter->read = 0; cfilter->size = 0; // trace tb_trace_d("[%p]: tail", cfilter); // continue continue ; } // is chunked head? parse size else { // parse size cfilter->size = tb_s16tou32(pb); // trace tb_trace_d("[%p]: size: %lu", cfilter, cfilter->size); // clear data tb_string_clear(&cfilter->line); // is eof? "0\r\n\r\n" if (!cfilter->size) { // trace tb_trace_d("[%p]: eof", cfilter); // is eof filter->beof = tb_true; // continue to spak the end data continue ; } // ok break; } } } } // check tb_assert_and_check_return_val(cfilter->read <= cfilter->size, -1); // read chunked data tb_size_t size = tb_min3(ie - ip, oe - op, cfilter->size - cfilter->read); if (size) { // copy data tb_memcpy((tb_byte_t*)op, ip, size); ip += size; op += size; // update read cfilter->read += size; } // update stream tb_static_stream_goto(istream, (tb_byte_t*)ip); tb_static_stream_goto(ostream, (tb_byte_t*)op); // trace tb_trace_d("[%p]: read: %lu, size: %lu, beof: %u, ileft: %lu", cfilter, cfilter->read, cfilter->size, filter->beof, tb_static_stream_left(istream)); // ok return (op - ob); }