static int http_request_parse_target(struct http_request_parser *parser)
{
struct http_message_parser *_parser = &parser->parser;
const unsigned char *p = parser->parser.cur;
/* We'll just parse anything up to the first SP or a control char.
We could also implement workarounds for buggy HTTP clients and
parse anything up to the HTTP-version and return 301 with the
target properly encoded (FIXME). */
while (p < _parser->end && *p > ' ')
p++;
/* target is too long when explicit limit is exceeded or when input buffer
runs out of space */
/* FIXME: put limit on full request line rather than target and method
separately */
/* FIXME: is it wise to keep target in stream buffer? It can become very
large for some applications, increasing the stream buffer size */
if ((uoff_t)(p - _parser->cur) > parser->max_target_length ||
(p == _parser->end && ((uoff_t)(p - _parser->cur) >=
i_stream_get_max_buffer_size(_parser->input)))) {
parser->error_code = HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG;
parser->parser.error = "HTTP request target is too long";
return -1;
}
if (p == _parser->end)
return 0;
parser->request_target = p_strdup_until(_parser->msg.pool, _parser->cur, p);
parser->parser.cur = p;
return 1;
}
static int
mdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output)
{
struct dbox_metadata_header meta_hdr;
const char *line;
size_t buf_size;
int ret;
if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0)
return ret;
o_stream_nsend(output, &meta_hdr, sizeof(meta_hdr));
buf_size = i_stream_get_max_buffer_size(file->input);
/* use unlimited line length for metadata */
i_stream_set_max_buffer_size(file->input, (size_t)-1);
while ((line = i_stream_read_next_line(file->input)) != NULL) {
if (*line == '\0') {
/* end of metadata */
break;
}
o_stream_nsend_str(output, line);
o_stream_nsend(output, "\n", 1);
}
i_stream_set_max_buffer_size(file->input, buf_size);
if (line == NULL) {
dbox_file_set_corrupted(file, "missing end-of-metadata line");
return 0;
}
o_stream_nsend(output, "\n", 1);
return 1;
}
static struct istream *
mail_crypt_cache_open(struct mail_crypt_user *muser, struct mail *mail,
struct istream *input)
{
struct mail_crypt_cache *cache = &muser->cache;
struct istream *inputs[2];
string_t *temp_prefix = t_str_new(128);
mail_crypt_cache_close(muser);
input->seekable = FALSE;
inputs[0] = input;
inputs[1] = NULL;
mail_user_set_get_temp_prefix(temp_prefix, mail->box->storage->user->set);
input = i_stream_create_seekable_path(inputs,
i_stream_get_max_buffer_size(inputs[0]),
str_c(temp_prefix));
i_stream_unref(&inputs[0]);
if (mail->uid > 0) {
cache->to = timeout_add(MAIL_CRYPT_MAIL_CACHE_EXPIRE_MSECS,
mail_crypt_cache_close, muser);
cache->box = mail->box;
cache->uid = mail->uid;
cache->input = input;
/* index-mail wants the stream to be destroyed at close, so create
a new stream instead of just increasing reference. */
return i_stream_create_limit(cache->input, (uoff_t)-1);
}
return input;
}
void i_stream_grow_buffer(struct istream_private *stream, size_t bytes)
{
size_t old_size, max_size;
old_size = stream->buffer_size;
stream->buffer_size = stream->pos + bytes;
if (stream->buffer_size <= stream->init_buffer_size)
stream->buffer_size = stream->init_buffer_size;
else
stream->buffer_size = nearest_power(stream->buffer_size);
max_size = i_stream_get_max_buffer_size(&stream->istream);
i_assert(max_size > 0);
if (stream->buffer_size > max_size)
stream->buffer_size = max_size;
if (stream->buffer_size <= old_size)
stream->buffer_size = old_size;
else {
stream->w_buffer = i_realloc(stream->w_buffer, old_size,
stream->buffer_size);
stream->buffer = stream->w_buffer;
}
}
static ssize_t i_stream_fs_file_read(struct istream_private *stream)
{
struct fs_file_istream *fstream = (struct fs_file_istream *)stream;
struct istream *input;
if (fstream->istream.parent == NULL) {
input = fs_read_stream(fstream->file,
i_stream_get_max_buffer_size(&stream->istream));
i_stream_init_parent(stream, input);
i_stream_unref(&input);
}
i_stream_seek(stream->parent, stream->parent_start_offset +
stream->istream.v_offset);
return i_stream_read_copy_from_parent(&stream->istream);
}
bool i_stream_try_alloc(struct istream_private *stream,
size_t wanted_size, size_t *size_r)
{
i_assert(wanted_size > 0);
if (wanted_size > stream->buffer_size - stream->pos) {
if (stream->skip > 0) {
/* remove the unused bytes from beginning of buffer */
i_stream_compress(stream);
} else if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) {
/* buffer is full - grow it */
i_stream_grow_buffer(stream, I_STREAM_MIN_SIZE);
}
}
*size_r = stream->buffer_size - stream->pos;
if (stream->try_alloc_limit > 0 &&
*size_r > stream->try_alloc_limit)
*size_r = stream->try_alloc_limit;
return *size_r > 0;
}
static struct istream *
zlib_mail_cache_open(struct zlib_user *zuser, struct mail *mail,
struct istream *input)
{
struct zlib_mail_cache *cache = &zuser->cache;
struct istream *inputs[2];
string_t *temp_prefix = t_str_new(128);
zlib_mail_cache_close(zuser);
/* zlib istream is seekable, but very slow. create a seekable istream
which we can use to quickly seek around in the stream that's been
read so far. usually the partial IMAP FETCHes continue from where
the previous left off, so this isn't strictly necessary, but with
the way lib-imap-storage's CRLF-cache works it has to seek backwards
somewhat, which causes a zlib stream reset. And the CRLF-cache isn't
easy to fix.. */
input->seekable = FALSE;
inputs[0] = input;
inputs[1] = NULL;
mail_user_set_get_temp_prefix(temp_prefix, mail->box->storage->user->set);
input = i_stream_create_seekable_path(inputs,
i_stream_get_max_buffer_size(inputs[0]),
str_c(temp_prefix));
i_stream_unref(&inputs[0]);
cache->to = timeout_add(ZLIB_MAIL_CACHE_EXPIRE_MSECS,
zlib_mail_cache_close, zuser);
cache->box = mail->box;
cache->uid = mail->uid;
cache->input = input;
/* index-mail wants the stream to be destroyed at close, so create
a new stream instead of just increasing reference. */
return i_stream_create_limit(cache->input, (uoff_t)-1);
}
static ssize_t i_stream_lzma_read(struct istream_private *stream)
{
struct lzma_istream *zstream = (struct lzma_istream *)stream;
const unsigned char *data;
uoff_t high_offset;
size_t size, out_size;
lzma_ret ret;
high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
if (zstream->eof_offset == high_offset) {
i_assert(zstream->high_pos == 0 ||
zstream->high_pos == stream->pos);
stream->istream.eof = TRUE;
return -1;
}
if (stream->pos < zstream->high_pos) {
/* we're here because we seeked back within the read buffer. */
ret = zstream->high_pos - stream->pos;
stream->pos = zstream->high_pos;
zstream->high_pos = 0;
if (zstream->eof_offset != (uoff_t)-1) {
high_offset = stream->istream.v_offset +
(stream->pos - stream->skip);
i_assert(zstream->eof_offset == high_offset);
stream->istream.eof = TRUE;
}
return ret;
}
zstream->high_pos = 0;
if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
/* try to keep at least CHUNK_SIZE available */
if (!zstream->marked && stream->skip > 0) {
/* don't try to keep anything cached if we don't
have a seek mark. */
i_stream_compress(stream);
}
if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
i_stream_grow_buffer(stream, CHUNK_SIZE);
if (stream->pos == stream->buffer_size) {
if (stream->skip > 0) {
/* lose our buffer cache */
i_stream_compress(stream);
}
if (stream->pos == stream->buffer_size)
return -2; /* buffer full */
}
}
if (i_stream_read_more(stream->parent, &data, &size) < 0) {
if (stream->parent->stream_errno != 0) {
stream->istream.stream_errno =
stream->parent->stream_errno;
} else {
i_assert(stream->parent->eof);
lzma_stream_end(zstream);
stream->istream.eof = TRUE;
}
return -1;
}
if (size == 0) {
/* no more input */
i_assert(!stream->istream.blocking);
return 0;
}
zstream->strm.next_in = data;
zstream->strm.avail_in = size;
out_size = stream->buffer_size - stream->pos;
zstream->strm.next_out = stream->w_buffer + stream->pos;
zstream->strm.avail_out = out_size;
ret = lzma_code(&zstream->strm, LZMA_RUN);
out_size -= zstream->strm.avail_out;
stream->pos += out_size;
i_stream_skip(stream->parent, size - zstream->strm.avail_in);
switch (ret) {
case LZMA_OK:
break;
case LZMA_DATA_ERROR:
case LZMA_BUF_ERROR:
lzma_read_error(zstream, "corrupted data");
stream->istream.stream_errno = EINVAL;
return -1;
case LZMA_FORMAT_ERROR:
lzma_read_error(zstream, "wrong magic in header (not xz file?)");
stream->istream.stream_errno = EINVAL;
return -1;
case LZMA_OPTIONS_ERROR:
lzma_read_error(zstream, "Unsupported xz options");
stream->istream.stream_errno = EINVAL;
return -1;
case LZMA_MEM_ERROR:
i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
i_stream_get_name(&stream->istream));
case LZMA_STREAM_END:
lzma_stream_end(zstream);
if (out_size == 0) {
stream->istream.eof = TRUE;
return -1;
}
break;
default:
lzma_read_error(zstream, t_strdup_printf(
"lzma_code() failed with %d", ret));
stream->istream.stream_errno = EINVAL;
return -1;
}
if (out_size == 0) {
/* read more input */
return i_stream_lzma_read(stream);
}
return out_size;
}
static int astream_read_next(struct attachment_istream *astream, bool *retry_r)
{
struct istream_private *stream = &astream->istream;
struct message_block block;
size_t old_size, new_size;
const char *error;
int ret;
*retry_r = FALSE;
if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream))
return -2;
if (astream->failed) {
stream->istream.stream_errno = EINVAL;
return -1;
}
old_size = stream->pos - stream->skip;
switch (message_parser_parse_next_block(astream->parser, &block)) {
case -1:
/* done / error */
ret = astream_end_of_part(astream, &error);
if (ret > 0) {
/* final data */
new_size = stream->pos - stream->skip;
return new_size - old_size;
}
stream->istream.eof = TRUE;
stream->istream.stream_errno = stream->parent->stream_errno;
if (ret < 0) {
io_stream_set_error(&stream->iostream, "%s", error);
stream->istream.stream_errno = EINVAL;
astream->failed = TRUE;
}
astream->cur_part = NULL;
return -1;
case 0:
/* need more data */
return 0;
default:
break;
}
if (block.part != astream->cur_part && astream->cur_part != NULL) {
/* end of a MIME part */
if (astream_end_of_part(astream, &error) < 0) {
io_stream_set_error(&stream->iostream, "%s", error);
stream->istream.stream_errno = EINVAL;
astream->failed = TRUE;
return -1;
}
}
astream->cur_part = block.part;
if (block.hdr != NULL) {
/* parsing a header */
astream_parse_header(astream, block.hdr);
} else if (block.size == 0) {
/* end of headers */
if (astream_want_attachment(astream, block.part)) {
astream->part.state = MAIL_ATTACHMENT_STATE_MAYBE;
astream->part.start_offset = stream->parent->v_offset;
}
} else {
astream_add_body(astream, &block);
}
new_size = stream->pos - stream->skip;
*retry_r = new_size == old_size;
return new_size - old_size;
}
static ssize_t i_stream_zlib_read(struct istream_private *stream)
{
struct zlib_istream *zstream = (struct zlib_istream *)stream;
const unsigned char *data;
uoff_t high_offset;
size_t size, out_size;
int ret;
high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
if (zstream->eof_offset == high_offset) {
i_assert(zstream->high_pos == 0 ||
zstream->high_pos == stream->pos);
if (!zstream->trailer_read) {
do {
ret = i_stream_zlib_read_trailer(zstream);
} while (ret == 0 && stream->istream.blocking);
if (ret <= 0)
return ret;
}
if (!zstream->gz || i_stream_is_eof(stream->parent)) {
stream->istream.eof = TRUE;
return -1;
}
/* gzip file with concatenated content */
zstream->eof_offset = (uoff_t)-1;
zstream->stream_size = (uoff_t)-1;
zstream->header_read = FALSE;
zstream->trailer_read = FALSE;
zstream->crc32 = 0;
(void)inflateEnd(&zstream->zs);
i_stream_zlib_init(zstream);
}
if (!zstream->header_read) {
do {
ret = i_stream_zlib_read_header(stream);
} while (ret == 0 && stream->istream.blocking);
if (ret <= 0)
return ret;
zstream->header_read = TRUE;
}
if (stream->pos < zstream->high_pos) {
/* we're here because we seeked back within the read buffer. */
ret = zstream->high_pos - stream->pos;
stream->pos = zstream->high_pos;
zstream->high_pos = 0;
if (zstream->trailer_read) {
high_offset = stream->istream.v_offset +
(stream->pos - stream->skip);
i_assert(zstream->eof_offset == high_offset);
stream->istream.eof = TRUE;
}
return ret;
}
zstream->high_pos = 0;
if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
/* try to keep at least CHUNK_SIZE available */
if (!zstream->marked && stream->skip > 0) {
/* don't try to keep anything cached if we don't
have a seek mark. */
i_stream_compress(stream);
}
if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
i_stream_grow_buffer(stream, CHUNK_SIZE);
if (stream->pos == stream->buffer_size) {
if (stream->skip > 0) {
/* lose our buffer cache */
i_stream_compress(stream);
}
if (stream->pos == stream->buffer_size)
return -2; /* buffer full */
}
}
if (i_stream_read_more(stream->parent, &data, &size) < 0) {
if (stream->parent->stream_errno != 0) {
stream->istream.stream_errno =
stream->parent->stream_errno;
} else {
i_assert(stream->parent->eof);
zlib_read_error(zstream, "unexpected EOF");
stream->istream.stream_errno = EPIPE;
}
return -1;
}
if (size == 0) {
/* no more input */
i_assert(!stream->istream.blocking);
return 0;
}
zstream->zs.next_in = (void *)data;
zstream->zs.avail_in = size;
out_size = stream->buffer_size - stream->pos;
zstream->zs.next_out = stream->w_buffer + stream->pos;
zstream->zs.avail_out = out_size;
ret = inflate(&zstream->zs, Z_SYNC_FLUSH);
out_size -= zstream->zs.avail_out;
zstream->crc32 = crc32_data_more(zstream->crc32,
stream->w_buffer + stream->pos,
out_size);
stream->pos += out_size;
i_stream_skip(stream->parent, size - zstream->zs.avail_in);
switch (ret) {
case Z_OK:
break;
case Z_NEED_DICT:
zlib_read_error(zstream, "can't read file without dict");
stream->istream.stream_errno = EINVAL;
return -1;
case Z_DATA_ERROR:
zlib_read_error(zstream, "corrupted data");
stream->istream.stream_errno = EINVAL;
return -1;
case Z_MEM_ERROR:
i_fatal_status(FATAL_OUTOFMEM, "zlib.read(%s): Out of memory",
i_stream_get_name(&stream->istream));
case Z_STREAM_END:
zstream->eof_offset = stream->istream.v_offset +
(stream->pos - stream->skip);
zstream->stream_size = zstream->eof_offset;
zstream->zs.avail_in = 0;
if (!zstream->trailer_read) {
/* try to read and verify the trailer, we might not
be called again. */
if (i_stream_zlib_read_trailer(zstream) < 0)
return -1;
}
break;
default:
i_fatal("inflate() failed with %d", ret);
}
if (out_size == 0) {
/* read more input */
return i_stream_zlib_read(stream);
}
return out_size;
}
static ssize_t i_stream_ssl_read_real(struct istream_private *stream)
{
struct ssl_istream *sstream = (struct ssl_istream *)stream;
struct ssl_iostream *ssl_io = sstream->ssl_io;
unsigned char buffer[IO_BLOCK_SIZE];
size_t max_buffer_size = i_stream_get_max_buffer_size(&stream->istream);
size_t orig_max_buffer_size = stream->max_buffer_size;
size_t size;
ssize_t ret, total_ret;
if (sstream->seen_eof) {
stream->istream.eof = TRUE;
return -1;
}
if (stream->pos >= max_buffer_size) {
i_stream_compress(stream);
if (stream->pos >= max_buffer_size)
return -2;
}
ret = openssl_iostream_more(ssl_io);
if (ret <= 0) {
if (ret < 0) {
/* handshake failed */
i_assert(errno != 0);
io_stream_set_error(&stream->iostream,
"%s", ssl_io->last_error);
stream->istream.stream_errno = errno;
}
return ret;
}
if (!i_stream_try_alloc(stream, 1, &size))
i_unreached();
if (stream->pos + size > max_buffer_size) {
i_assert(max_buffer_size > stream->pos);
size = max_buffer_size - stream->pos;
}
while ((ret = SSL_read(ssl_io->ssl,
stream->w_buffer + stream->pos, size)) <= 0) {
/* failed to read anything */
ret = openssl_iostream_handle_error(ssl_io, ret, "SSL_read");
if (ret <= 0) {
if (ret == 0)
return 0;
if (ssl_io->last_error != NULL) {
io_stream_set_error(&stream->iostream,
"%s", ssl_io->last_error);
}
if (errno != EPIPE)
stream->istream.stream_errno = errno;
stream->istream.eof = TRUE;
sstream->seen_eof = TRUE;
return -1;
}
/* we did some BIO I/O, try reading again */
}
stream->pos += ret;
total_ret = ret;
/* now make sure that we read everything already buffered in OpenSSL
into the stream (without reading anything more). this makes I/O loop
behave similary for ssl-istream as file-istream. */
sstream->ssl_io->input_handler = FALSE;
stream->max_buffer_size = (size_t)-1;
while ((ret = SSL_read(ssl_io->ssl, buffer, sizeof(buffer))) > 0) {
memcpy(i_stream_alloc(stream, ret), buffer, ret);
stream->pos += ret;
total_ret += ret;
}
stream->max_buffer_size = orig_max_buffer_size;
return total_ret;
}
static ssize_t i_stream_bzlib_read(struct istream_private *stream)
{
struct bzlib_istream *zstream = (struct bzlib_istream *)stream;
const unsigned char *data;
uoff_t high_offset;
size_t size, out_size;
int ret;
high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
if (zstream->eof_offset == high_offset) {
i_assert(zstream->high_pos == 0 ||
zstream->high_pos == stream->pos);
stream->istream.eof = TRUE;
return -1;
}
if (stream->pos < zstream->high_pos) {
/* we're here because we seeked back within the read buffer. */
ret = zstream->high_pos - stream->pos;
stream->pos = zstream->high_pos;
zstream->high_pos = 0;
if (zstream->eof_offset != (uoff_t)-1) {
high_offset = stream->istream.v_offset +
(stream->pos - stream->skip);
i_assert(zstream->eof_offset == high_offset);
stream->istream.eof = TRUE;
}
return ret;
}
zstream->high_pos = 0;
if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
/* try to keep at least CHUNK_SIZE available */
if (!zstream->marked && stream->skip > 0) {
/* don't try to keep anything cached if we don't
have a seek mark. */
i_stream_compress(stream);
}
if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
i_stream_grow_buffer(stream, CHUNK_SIZE);
if (stream->pos == stream->buffer_size) {
if (stream->skip > 0) {
/* lose our buffer cache */
i_stream_compress(stream);
}
if (stream->pos == stream->buffer_size)
return -2; /* buffer full */
}
}
if (i_stream_read_more(stream->parent, &data, &size) < 0) {
if (stream->parent->stream_errno != 0) {
stream->istream.stream_errno =
stream->parent->stream_errno;
} else {
i_assert(stream->parent->eof);
bzlib_read_error(zstream, "unexpected EOF");
stream->istream.stream_errno = EINVAL;
}
return -1;
}
if (size == 0) {
/* no more input */
i_assert(!stream->istream.blocking);
return 0;
}
zstream->zs.next_in = (char *)data;
zstream->zs.avail_in = size;
out_size = stream->buffer_size - stream->pos;
zstream->zs.next_out = (char *)stream->w_buffer + stream->pos;
zstream->zs.avail_out = out_size;
ret = BZ2_bzDecompress(&zstream->zs);
out_size -= zstream->zs.avail_out;
stream->pos += out_size;
i_stream_skip(stream->parent, size - zstream->zs.avail_in);
switch (ret) {
case BZ_OK:
break;
case BZ_PARAM_ERROR:
i_unreached();
case BZ_DATA_ERROR:
bzlib_read_error(zstream, "corrupted data");
stream->istream.stream_errno = EINVAL;
return -1;
case BZ_DATA_ERROR_MAGIC:
bzlib_read_error(zstream,
"wrong magic in header (not bz2 file?)");
stream->istream.stream_errno = EINVAL;
return -1;
case BZ_MEM_ERROR:
i_fatal_status(FATAL_OUTOFMEM, "bzlib.read(%s): Out of memory",
i_stream_get_name(&stream->istream));
case BZ_STREAM_END:
zstream->eof_offset = stream->istream.v_offset +
(stream->pos - stream->skip);
zstream->stream_size = zstream->eof_offset;
if (out_size == 0) {
stream->istream.eof = TRUE;
return -1;
}
break;
default:
i_fatal("BZ2_bzDecompress() failed with %d", ret);
}
if (out_size == 0) {
/* read more input */
return i_stream_bzlib_read(stream);
}
return out_size;
}
