static void test_message_header_parser_partial(void)
{
struct message_header_parser_ctx *parser;
struct message_header_line *hdr;
struct istream *input;
unsigned int i, max = (strlen(test1_msg)-TEST1_MSG_BODY_LEN)*2;
string_t *str;
int ret;
test_begin("message header parser partial");
input = test_istream_create(test1_msg);
test_istream_set_allow_eof(input, FALSE);
str = t_str_new(max);
parser = message_parse_header_init(input, NULL, 0);
for (i = 0; i <= max; i++) {
test_istream_set_size(input, i/2);
while ((ret = message_parse_header_next(parser, &hdr)) > 0)
hdr_write(str, hdr);
test_assert((ret == 0 && i < max) ||
(ret < 0 && i == max));
}
message_parse_header_deinit(&parser);
str_append(str, " body");
test_assert(strcmp(str_c(str), test1_msg) == 0);
i_stream_unref(&input);
test_end();
}
static void test_message_header_parser_no_eoh(void)
{
static const char *str = "a:b\n";
struct message_header_parser_ctx *parser;
struct message_header_line *hdr;
struct istream *input;
test_begin("message header parser no EOH");
input = test_istream_create(str);
parser = message_parse_header_init(input, NULL, 0);
test_assert(message_parse_header_next(parser, &hdr) > 0 &&
strcmp(hdr->name, "a") == 0);
test_assert(message_parse_header_next(parser, &hdr) < 0);
message_parse_header_deinit(&parser);
test_assert(input->stream_errno == 0);
i_stream_unref(&input);
test_end();
}
void message_parse_header(struct istream *input, struct message_size *hdr_size,
enum message_header_parser_flags flags,
message_header_callback_t *callback, void *context)
{
struct message_header_parser_ctx *hdr_ctx;
struct message_header_line *hdr;
int ret;
hdr_ctx = message_parse_header_init(input, hdr_size, flags);
while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0)
callback(hdr, context);
i_assert(ret != 0);
message_parse_header_deinit(&hdr_ctx);
/* call after the final skipping */
callback(NULL, context);
}
static void
test_message_header_parser_long_lines_str(const char *str,
unsigned int buffer_size,
struct message_size *size_r)
{
struct message_header_parser_ctx *parser;
struct message_header_line *hdr;
struct istream *input;
unsigned int i, len = strlen(str);
input = test_istream_create(str);
test_istream_set_max_buffer_size(input, buffer_size);
parser = message_parse_header_init(input, size_r, 0);
for (i = 1; i <= len; i++) {
test_istream_set_size(input, i);
while (message_parse_header_next(parser, &hdr) > 0) ;
}
message_parse_header_deinit(&parser);
i_stream_unref(&input);
}
bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox,
struct mail_index_view *view, uint32_t seq)
{
struct mbox_sync_mail_context ctx;
struct message_header_parser_ctx *hdr_ctx;
struct message_header_line *hdr;
struct header_func *func;
struct mbox_md5_context *mbox_md5_ctx;
const void *data;
bool expunged;
uint32_t uid;
int ret;
/* we only wish to be sure that this mail actually is what we expect
it to be. If there's X-UID header and it matches our UID, we use it.
Otherwise it could mean that the X-UID header is invalid and it's
just not yet been rewritten. In that case use MD5 sum, if it
exists. */
mail_index_lookup_uid(view, seq, &uid);
memset(&ctx, 0, sizeof(ctx));
mbox_md5_ctx = mbox->md5_v.init();
hdr_ctx = message_parse_header_init(mbox->mbox_stream, NULL, 0);
while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
if (hdr->eoh)
break;
func = bsearch(hdr->name, header_funcs,
N_ELEMENTS(header_funcs), sizeof(*header_funcs),
mbox_sync_bsearch_header_func_cmp);
if (func != NULL) {
if (strcasecmp(hdr->name, "X-UID") == 0) {
if (hdr->continues) {
hdr->use_full_value = TRUE;
continue;
}
(void)parse_x_uid(&ctx, hdr);
if (ctx.mail.uid == uid)
break;
}
} else {
mbox->md5_v.more(mbox_md5_ctx, hdr);
}
}
i_assert(ret != 0);
message_parse_header_deinit(&hdr_ctx);
mbox->md5_v.finish(mbox_md5_ctx, ctx.hdr_md5_sum);
if (ctx.mail.uid == uid)
return TRUE;
/* match by MD5 sum */
mbox->mbox_save_md5 = TRUE;
mail_index_lookup_ext(view, seq, mbox->md5hdr_ext_idx,
&data, &expunged);
return data == NULL ? 0 :
memcmp(data, ctx.hdr_md5_sum, 16) == 0;
}
int mbox_sync_parse_next_mail(struct istream *input,
struct mbox_sync_mail_context *ctx)
{
struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
struct message_header_parser_ctx *hdr_ctx;
struct message_header_line *hdr;
struct mbox_sync_header_func *func;
struct mbox_md5_context *mbox_md5_ctx;
size_t line_start_pos;
int i, ret;
ctx->hdr_offset = ctx->mail.offset;
ctx->mail.flags = MAIL_RECENT; /* default to having recent flag */
ctx->header_first_change = (size_t)-1;
ctx->header_last_change = 0;
for (i = 0; i < MBOX_HDR_COUNT; i++)
ctx->hdr_pos[i] = (size_t)-1;
ctx->content_length = (uoff_t)-1;
str_truncate(ctx->header, 0);
mbox_md5_ctx = ctx->sync_ctx->mbox->md5_v.init();
line_start_pos = 0;
hdr_ctx = message_parse_header_init(input, NULL, 0);
while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
if (hdr->eoh) {
ctx->have_eoh = TRUE;
break;
}
if (!hdr->continued) {
line_start_pos = str_len(ctx->header);
str_append(ctx->header, hdr->name);
str_append_n(ctx->header, hdr->middle, hdr->middle_len);
}
func = bsearch(hdr->name, header_funcs,
N_ELEMENTS(header_funcs), sizeof(*header_funcs),
mbox_sync_bsearch_header_func_cmp);
if (func != NULL) {
if (hdr->continues) {
hdr->use_full_value = TRUE;
continue;
}
if (!func->func(ctx, hdr)) {
/* this header is broken, remove it */
ctx->need_rewrite = TRUE;
str_truncate(ctx->header, line_start_pos);
if (ctx->header_first_change == (size_t)-1) {
ctx->header_first_change =
line_start_pos;
}
continue;
}
buffer_append(ctx->header, hdr->full_value,
hdr->full_value_len);
} else {
ctx->sync_ctx->mbox->md5_v.more(mbox_md5_ctx, hdr);
buffer_append(ctx->header, hdr->value,
hdr->value_len);
}
if (!hdr->no_newline) {
if (hdr->crlf_newline)
str_append_c(ctx->header, '\r');
str_append_c(ctx->header, '\n');
}
}
i_assert(ret != 0);
message_parse_header_deinit(&hdr_ctx);
ctx->sync_ctx->mbox->md5_v.finish(mbox_md5_ctx, ctx->hdr_md5_sum);
if ((ctx->seq == 1 && !ctx->seen_imapbase) ||
(ctx->seq > 1 && sync_ctx->dest_first_mail)) {
/* missing X-IMAPbase */
ctx->need_rewrite = TRUE;
if (sync_ctx->base_uid_validity == 0) {
/* figure out a new UIDVALIDITY for us. */
sync_ctx->base_uid_validity =
sync_ctx->hdr->uid_validity != 0 &&
!sync_ctx->renumber_uids ?
sync_ctx->hdr->uid_validity :
I_MAX((uint32_t)ioloop_time, 1);
}
}
ctx->body_offset = input->v_offset;
if (input->stream_errno != 0) {
mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s",
i_stream_get_name(input), i_stream_get_error(input));
return -1;
}
return 0;
}
static int parse_next_header(struct message_parser_ctx *ctx,
struct message_block *block_r)
{
struct message_part *part = ctx->part;
struct message_header_line *hdr;
size_t size;
int ret;
if (ctx->skip > 0) {
i_stream_skip(ctx->input, ctx->skip);
ctx->skip = 0;
}
ret = message_parse_header_next(ctx->hdr_parser_ctx, &hdr);
if (ret == 0 || (ret < 0 && ctx->input->stream_errno != 0)) {
(void)i_stream_get_data(ctx->input, &size);
ctx->want_count = size + 1;
return ret;
}
if (hdr != NULL) {
if (hdr->eoh)
;
else if (strcasecmp(hdr->name, "Mime-Version") == 0) {
/* it's MIME. Content-* headers are valid */
part->flags |= MESSAGE_PART_FLAG_IS_MIME;
} else if (strcasecmp(hdr->name, "Content-Type") == 0) {
if ((ctx->flags &
MESSAGE_PARSER_FLAG_MIME_VERSION_STRICT) == 0)
part->flags |= MESSAGE_PART_FLAG_IS_MIME;
if (hdr->continues)
hdr->use_full_value = TRUE;
else T_BEGIN {
parse_content_type(ctx, hdr);
} T_END;
}
block_r->hdr = hdr;
block_r->size = 0;
return 1;
}
/* end of headers */
if ((part->flags & MESSAGE_PART_FLAG_MULTIPART) != 0 &&
ctx->last_boundary == NULL) {
/* multipart type but no message boundary */
part->flags = 0;
}
if ((part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
/* It's not MIME. Reset everything we found from
Content-Type. */
part->flags = 0;
ctx->last_boundary = NULL;
}
if (!ctx->part_seen_content_type ||
(part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
if (part->parent != NULL &&
(part->parent->flags &
MESSAGE_PART_FLAG_MULTIPART_DIGEST) != 0) {
/* when there's no content-type specified and we're
below multipart/digest, assume message/rfc822
content-type */
part->flags |= MESSAGE_PART_FLAG_MESSAGE_RFC822;
} else {
/* otherwise we default to text/plain */
part->flags |= MESSAGE_PART_FLAG_TEXT;
}
}
if (message_parse_header_has_nuls(ctx->hdr_parser_ctx))
part->flags |= MESSAGE_PART_FLAG_HAS_NULS;
message_parse_header_deinit(&ctx->hdr_parser_ctx);
i_assert((part->flags & MUTEX_FLAGS) != MUTEX_FLAGS);
ctx->last_chr = '\n';
if (ctx->last_boundary != NULL) {
parse_next_body_multipart_init(ctx);
ctx->parse_next_block = parse_next_body_to_boundary;
} else if (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822)
ctx->parse_next_block = parse_next_body_message_rfc822_init;
else if (ctx->boundaries != NULL)
ctx->parse_next_block = parse_next_body_to_boundary;
else
ctx->parse_next_block = parse_next_body_to_eof;
ctx->want_count = 1;
/* return empty block as end of headers */
block_r->hdr = NULL;
block_r->size = 0;
return 1;
}
static void
test_message_header_parser_one(struct message_header_parser_ctx *parser,
enum message_header_parser_flags hdr_flags)
{
struct message_header_line *hdr;
bool use_full_value;
use_full_value = hdr_flags != 0;
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 0);
if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP) == 0)
test_assert(hdr->full_value_offset == 4);
else
test_assert(hdr->full_value_offset == 5);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h1") == 0);
if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP) == 0) {
test_assert(hdr->middle_len == 2 && memcmp(hdr->middle, ": ", 2) == 0);
test_assert(hdr->value_len == 3 && memcmp(hdr->value, " v1", 3) == 0);
} else {
test_assert(hdr->middle_len == 3 && memcmp(hdr->middle, ": ", 3) == 0);
test_assert(hdr->value_len == 2 && memcmp(hdr->value, "v1", 2) == 0);
}
test_assert(!hdr->continues && !hdr->continued && !hdr->eoh &&
!hdr->no_newline && !hdr->crlf_newline);
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 8 && hdr->full_value_offset == 11);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h2") == 0);
test_assert(hdr->middle_len == 1 && memcmp(hdr->middle, ":", 1) == 0);
test_assert(hdr->value_len == 0);
test_assert(hdr->continues && !hdr->continued && !hdr->eoh &&
!hdr->no_newline && !hdr->crlf_newline);
if (use_full_value) hdr->use_full_value = TRUE;
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 8 && hdr->full_value_offset == 11);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h2") == 0);
test_assert(hdr->middle_len == 1 && memcmp(hdr->middle, ":", 1) == 0);
test_assert(hdr->value_len == 3 && memcmp(hdr->value, " v2", 3) == 0);
test_assert(!hdr->continues && hdr->continued && !hdr->eoh &&
!hdr->no_newline && hdr->crlf_newline);
if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE) != 0) {
test_assert(hdr->full_value_len == 3 &&
memcmp(hdr->full_value, " v2", 3) == 0);
} else if (use_full_value) {
test_assert(hdr->full_value_len == 4 &&
memcmp(hdr->full_value, "\n v2", 4) == 0);
}
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 17 && hdr->full_value_offset == 21);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h3") == 0);
test_assert(hdr->middle_len == 2 && memcmp(hdr->middle, ": ", 2) == 0);
test_assert(hdr->value_len == 0);
test_assert(hdr->continues && !hdr->continued && !hdr->eoh &&
!hdr->no_newline && hdr->crlf_newline);
if (use_full_value) hdr->use_full_value = TRUE;
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 17 && hdr->full_value_offset == 21);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h3") == 0);
test_assert(hdr->middle_len == 2 && memcmp(hdr->middle, ": ", 2) == 0);
test_assert(hdr->value_len == 3 && memcmp(hdr->value, "\tv3", 3) == 0);
test_assert(hdr->continues && hdr->continued && !hdr->eoh &&
!hdr->no_newline && !hdr->crlf_newline);
if (use_full_value) hdr->use_full_value = TRUE;
if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE) != 0) {
test_assert(hdr->full_value_len == 3 &&
memcmp(hdr->full_value, " v3", 3) == 0);
} else if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_DROP_CR) != 0) {
test_assert(hdr->full_value_len == 4 &&
memcmp(hdr->full_value, "\n\tv3", 4) == 0);
} else if (use_full_value) {
test_assert(hdr->full_value_len == 5 &&
memcmp(hdr->full_value, "\r\n\tv3", 5) == 0);
}
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 17 && hdr->full_value_offset == 21);
test_assert(hdr->name_len == 2 && strcmp(hdr->name, "h3") == 0);
test_assert(hdr->middle_len == 2 && memcmp(hdr->middle, ": ", 2) == 0);
test_assert(hdr->value_len == 3 && memcmp(hdr->value, "\tw3", 3) == 0);
test_assert(!hdr->continues && hdr->continued && !hdr->eoh &&
!hdr->no_newline && hdr->crlf_newline);
if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE) != 0) {
test_assert(hdr->full_value_len == 6 &&
memcmp(hdr->full_value, " v3 w3", 6) == 0);
} else if ((hdr_flags & MESSAGE_HEADER_PARSER_FLAG_DROP_CR) != 0) {
test_assert(hdr->full_value_len == 8 &&
memcmp(hdr->full_value, "\n\tv3\n\tw3", 8) == 0);
} else if (use_full_value) {
test_assert(hdr->full_value_len == 9 &&
memcmp(hdr->full_value, "\r\n\tv3\n\tw3", 9) == 0);
}
test_assert(message_parse_header_next(parser, &hdr) > 0);
test_assert(hdr->name_offset == 32 && hdr->full_value_offset == 32);
test_assert(hdr->name_len == 0 && hdr->middle_len == 0 && hdr->value_len == 0);
test_assert(!hdr->continues && !hdr->continued && hdr->eoh &&
!hdr->no_newline && !hdr->crlf_newline);
test_assert(message_parse_header_next(parser, &hdr) < 0);
}
static int parse_next_header(struct message_parser_ctx *ctx,
struct message_block *block_r)
{
struct message_part *part = ctx->part;
struct message_header_line *hdr;
struct message_boundary *boundary;
bool full;
int ret;
if ((ret = message_parser_read_more(ctx, block_r, &full)) == 0)
return ret;
if (ret > 0 && block_is_at_eoh(block_r) &&
ctx->last_boundary != NULL &&
(part->flags & MESSAGE_PART_FLAG_IS_MIME) != 0) {
/* we are at the end of headers and we've determined that we're
going to start a multipart. add the boundary already here
at this point so we can reliably determine whether the
"\n--boundary" belongs to us or to a previous boundary.
this is a problem if the boundary prefixes are identical,
because MIME requires only the prefix to match. */
parse_next_body_multipart_init(ctx);
ctx->multipart = TRUE;
}
/* before parsing the header see if we can find a --boundary from here.
we're guaranteed to be at the beginning of the line here. */
if (ret > 0) {
ret = ctx->boundaries == NULL ? -1 :
boundary_line_find(ctx, block_r->data,
block_r->size, full, &boundary);
if (ret > 0 && boundary->part == ctx->part) {
/* our own body begins with our own --boundary.
we don't want to handle that yet. */
ret = -1;
}
}
if (ret < 0) {
/* no boundary */
ret = message_parse_header_next(ctx->hdr_parser_ctx, &hdr);
if (ret == 0 || (ret < 0 && ctx->input->stream_errno != 0)) {
ctx->want_count = i_stream_get_data_size(ctx->input) + 1;
return ret;
}
} else if (ret == 0) {
/* need more data */
return 0;
} else {
/* boundary found. stop parsing headers here. The previous
[CR]LF belongs to the MIME boundary though. */
if (ctx->prev_hdr_newline_size > 0) {
i_assert(ctx->part->header_size.lines > 0);
/* remove the newline size from the MIME header */
ctx->part->header_size.lines--;
ctx->part->header_size.physical_size -=
ctx->prev_hdr_newline_size;
ctx->part->header_size.virtual_size -= 2;
/* add the newline size to the parent's body */
ctx->part->parent->body_size.lines++;
ctx->part->parent->body_size.physical_size +=
ctx->prev_hdr_newline_size;
ctx->part->parent->body_size.virtual_size += 2;
}
hdr = NULL;
}
if (hdr != NULL) {
if (hdr->eoh)
;
else if (strcasecmp(hdr->name, "Mime-Version") == 0) {
/* it's MIME. Content-* headers are valid */
part->flags |= MESSAGE_PART_FLAG_IS_MIME;
} else if (strcasecmp(hdr->name, "Content-Type") == 0) {
if ((ctx->flags &
MESSAGE_PARSER_FLAG_MIME_VERSION_STRICT) == 0)
part->flags |= MESSAGE_PART_FLAG_IS_MIME;
if (hdr->continues)
hdr->use_full_value = TRUE;
else T_BEGIN {
parse_content_type(ctx, hdr);
} T_END;
}
block_r->hdr = hdr;
block_r->size = 0;
ctx->prev_hdr_newline_size = hdr->no_newline ? 0 :
(hdr->crlf_newline ? 2 : 1);
return 1;
}
/* end of headers */
if ((part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
/* It's not MIME. Reset everything we found from
Content-Type. */
i_assert(!ctx->multipart);
part->flags = 0;
}
ctx->last_boundary = NULL;
if (!ctx->part_seen_content_type ||
(part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
if (part->parent != NULL &&
(part->parent->flags &
MESSAGE_PART_FLAG_MULTIPART_DIGEST) != 0) {
/* when there's no content-type specified and we're
below multipart/digest, assume message/rfc822
content-type */
part->flags |= MESSAGE_PART_FLAG_MESSAGE_RFC822;
} else {
/* otherwise we default to text/plain */
part->flags |= MESSAGE_PART_FLAG_TEXT;
}
}
if (message_parse_header_has_nuls(ctx->hdr_parser_ctx))
part->flags |= MESSAGE_PART_FLAG_HAS_NULS;
message_parse_header_deinit(&ctx->hdr_parser_ctx);
i_assert((part->flags & MUTEX_FLAGS) != MUTEX_FLAGS);
ctx->last_chr = '\n';
if (ctx->multipart) {
i_assert(ctx->last_boundary == NULL);
ctx->multipart = FALSE;
ctx->parse_next_block = parse_next_body_to_boundary;
} else if ((part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822) != 0)
ctx->parse_next_block = parse_next_body_message_rfc822_init;
else if (ctx->boundaries != NULL)
ctx->parse_next_block = parse_next_body_to_boundary;
else
ctx->parse_next_block = parse_next_body_to_eof;
ctx->want_count = 1;
/* return empty block as end of headers */
block_r->hdr = NULL;
block_r->size = 0;
return 1;
}
