static GMimeStream*
get_mime_stream (GMimeObject *obj, GError **err)
{
GMimeStream *mstream;
GMimeDataWrapper *wrapper;
if (!GMIME_IS_PART(obj)) {
char *str;
GByteArray *ar;
str = g_mime_object_to_string (obj);
ar = g_byte_array_new_take ((guchar*)str, strlen(str));
mstream = g_mime_stream_mem_new_with_byte_array (ar);
g_mime_stream_mem_set_owner (
GMIME_STREAM_MEM(mstream), TRUE);
return mstream;
}
wrapper = get_data_wrapper (obj, err);
if (!wrapper)
return NULL;
mstream = g_mime_data_wrapper_get_stream (wrapper);
if (!mstream || g_mime_stream_reset (mstream) != 0) {
g_set_error (err, G_IO_ERROR, G_IO_ERROR_FAILED,
"invalid mime-stream");
return NULL;
}
return mstream;
}
static GByteArray *
read_all_bytes (const char *path, gboolean is_text)
{
GMimeStream *filtered, *stream, *mem;
GMimeFilter *filter;
GByteArray *buffer;
buffer = g_byte_array_new ();
stream = g_mime_stream_fs_open (path, O_RDONLY, 0644, NULL);
mem = g_mime_stream_mem_new_with_byte_array (buffer);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) mem, FALSE);
if (is_text) {
filtered = g_mime_stream_filter_new (mem);
filter = g_mime_filter_dos2unix_new (FALSE);
g_mime_stream_filter_add ((GMimeStreamFilter *) filtered, filter);
g_object_unref (filter);
g_object_unref (mem);
} else {
filtered = mem;
}
g_mime_stream_write_to_stream (stream, filtered);
g_mime_stream_flush (filtered);
g_object_unref (filtered);
g_object_unref (stream);
return buffer;
}
static void
test_html (const char *datadir, const char *input, const char *output, guint32 citation)
{
guint32 flags = GMIME_FILTER_HTML_CONVERT_NL | GMIME_FILTER_HTML_CONVERT_SPACES |
GMIME_FILTER_HTML_CONVERT_URLS | GMIME_FILTER_HTML_CONVERT_ADDRESSES;
const char *what = "GMimeFilterHtml", *mode;
GByteArray *expected, *actual;
GMimeStream *stream;
GMimeFilter *filter;
char *path;
switch (citation) {
case GMIME_FILTER_HTML_BLOCKQUOTE_CITATION: mode = "blockquote"; break;
case GMIME_FILTER_HTML_MARK_CITATION: mode = "mark"; break;
case GMIME_FILTER_HTML_CITE: mode = "cite"; break;
default: mode = "none"; break;
}
testsuite_check ("%s (%s %s)", what, input, mode);
actual = g_byte_array_new ();
stream = g_mime_stream_mem_new_with_byte_array (actual);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) stream, FALSE);
filter = g_mime_filter_html_new (flags | citation, 0x008888);
path = g_build_filename (datadir, input, NULL);
pump_data_through_filter (filter, path, stream, TRUE, TRUE);
g_mime_filter_reset (filter);
g_object_unref (stream);
g_object_unref (filter);
g_free (path);
path = g_build_filename (datadir, output, NULL);
expected = read_all_bytes (path, TRUE);
g_free (path);
if (actual->len != expected->len) {
testsuite_check_failed ("%s failed: stream lengths do not match: expected=%u; actual=%u",
what, expected->len, actual->len);
stream = g_mime_stream_fs_open (output, O_WRONLY | O_CREAT, 0644, NULL);
g_mime_stream_write (stream, (const char *) actual->data, actual->len);
g_mime_stream_flush (stream);
g_object_unref (stream);
goto error;
}
if (memcmp (actual->data, expected->data, actual->len) != 0) {
testsuite_check_failed ("%s failed: stream contents do not match", what);
goto error;
}
testsuite_check_passed ();
error:
g_byte_array_free (expected, TRUE);
g_byte_array_free (actual, TRUE);
}
static void extract_part(GMimeObject *part, PartExtractorData *a_data) {
GMimeDataWrapper *attachment_wrapper = g_mime_part_get_content_object(GMIME_PART(part));
GMimeStream *attachment_mem_stream = g_mime_stream_mem_new();
g_mime_stream_mem_set_owner(GMIME_STREAM_MEM(attachment_mem_stream), FALSE);
g_mime_data_wrapper_write_to_stream(attachment_wrapper, attachment_mem_stream);
g_mime_stream_flush(attachment_mem_stream);
a_data->content = g_mime_stream_mem_get_byte_array(GMIME_STREAM_MEM(attachment_mem_stream));
g_object_unref(attachment_mem_stream);
}
static void
test_gzip (const char *datadir, const char *filename)
{
char *name = g_strdup_printf ("%s.gz", filename);
char *path = g_build_filename (datadir, filename, NULL);
const char *what = "GMimeFilterGzip::zip";
GByteArray *actual, *expected;
GMimeStream *stream;
GMimeFilter *filter;
testsuite_check ("%s", what);
actual = g_byte_array_new ();
stream = g_mime_stream_mem_new_with_byte_array (actual);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) stream, FALSE);
filter = g_mime_filter_gzip_new (GMIME_FILTER_GZIP_MODE_ZIP, 9);
g_mime_filter_gzip_set_filename ((GMimeFilterGZip *) filter, filename);
g_mime_filter_gzip_set_comment ((GMimeFilterGZip *) filter, "This is a comment.");
pump_data_through_filter (filter, path, stream, TRUE, TRUE);
g_mime_filter_reset (filter);
g_object_unref (stream);
g_object_unref (filter);
g_free (path);
path = g_build_filename (datadir, name, NULL);
expected = read_all_bytes (path, FALSE);
g_free (name);
g_free (path);
if (actual->len != 1233 && (actual->len != expected->len || memcmp (actual->data, expected->data, actual->len) != 0)) {
if (actual->len != expected->len)
testsuite_check_failed ("%s failed: streams are not the same length: %u", what, actual->len);
else
testsuite_check_failed ("%s failed: streams do not match", what);
name = g_strdup_printf ("%s.1.gz", filename);
path = g_build_filename (datadir, name, NULL);
g_free (name);
stream = g_mime_stream_fs_open (path, O_WRONLY | O_CREAT | O_TRUNC, 0644, NULL);
g_free (path);
g_mime_stream_write (stream, (const char *) actual->data, actual->len);
g_mime_stream_flush (stream);
g_object_unref (stream);
} else {
testsuite_check_passed ();
}
g_byte_array_free (expected, TRUE);
g_byte_array_free (actual, TRUE);
}
static void
test_charset_conversion (const char *datadir, const char *base, const char *from, const char *to)
{
const char *what = "GMimeFilterCharset";
GByteArray *actual, *expected;
GMimeStream *stream;
GMimeFilter *filter;
char *path, *name;
testsuite_check ("%s (%s %s -> %s)", what, base, from, to);
actual = g_byte_array_new ();
stream = g_mime_stream_mem_new_with_byte_array (actual);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) stream, FALSE);
filter = g_mime_filter_charset_new (from, to);
name = g_strdup_printf ("%s.%s.txt", base, from);
path = g_build_filename (datadir, name, NULL);
pump_data_through_filter (filter, path, stream, TRUE, TRUE);
g_mime_filter_reset (filter);
g_object_unref (stream);
g_object_unref (filter);
g_free (path);
g_free (name);
name = g_strdup_printf ("%s.%s.txt", base, to);
path = g_build_filename (datadir, name, NULL);
expected = read_all_bytes (path, TRUE);
g_free (path);
g_free (name);
if (actual->len != expected->len) {
testsuite_check_failed ("%s failed: stream lengths do not match: expected=%u; actual=%u",
what, expected->len, actual->len);
goto error;
}
if (memcmp (actual->data, expected->data, actual->len) != 0) {
testsuite_check_failed ("%s failed: stream contents do not match", what);
goto error;
}
testsuite_check_passed ();
error:
g_byte_array_free (expected, TRUE);
g_byte_array_free (actual, TRUE);
}
static void
test_enriched (const char *datadir, const char *input, const char *output)
{
const char *what = "GMimeFilterEnriched";
GByteArray *actual, *expected;
GMimeStream *stream;
GMimeFilter *filter;
char *path;
testsuite_check ("%s (%s)", what, input);
actual = g_byte_array_new ();
stream = g_mime_stream_mem_new_with_byte_array (actual);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) stream, FALSE);
filter = g_mime_filter_enriched_new (0);
path = g_build_filename (datadir, input, NULL);
pump_data_through_filter (filter, path, stream, TRUE, TRUE);
g_mime_filter_reset (filter);
g_object_unref (stream);
g_object_unref (filter);
g_free (path);
path = g_build_filename (datadir, output, NULL);
expected = read_all_bytes (path, TRUE);
g_free (path);
if (actual->len != expected->len) {
testsuite_check_failed ("%s failed: stream lengths do not match: expected=%u; actual=%u",
what, expected->len, actual->len);
printf ("enriched: -->%.*s<--\n", (int) actual->len, (char *) actual->data);
goto error;
}
if (memcmp (actual->data, expected->data, actual->len) != 0) {
testsuite_check_failed ("%s failed: stream contents do not match", what);
goto error;
}
testsuite_check_passed ();
error:
g_byte_array_free (expected, TRUE);
g_byte_array_free (actual, TRUE);
}
static void
test_gunzip (const char *datadir, const char *filename)
{
char *name = g_strdup_printf ("%s.gz", filename);
char *path = g_build_filename (datadir, name, NULL);
const char *what = "GMimeFilterGzip::unzip";
GByteArray *actual, *expected;
GMimeStream *stream;
GMimeFilter *filter;
const char *value;
testsuite_check ("%s", what);
actual = g_byte_array_new ();
stream = g_mime_stream_mem_new_with_byte_array (actual);
g_mime_stream_mem_set_owner ((GMimeStreamMem *) stream, FALSE);
filter = g_mime_filter_gzip_new (GMIME_FILTER_GZIP_MODE_UNZIP, 9);
pump_data_through_filter (filter, path, stream, FALSE, TRUE);
g_object_unref (stream);
g_free (path);
g_free (name);
path = g_build_filename (datadir, filename, NULL);
expected = read_all_bytes (path, TRUE);
g_free (path);
if (actual->len != expected->len) {
testsuite_check_failed ("%s failed: stream lengths do not match: expected=%u; actual=%u",
what, expected->len, actual->len);
goto error;
}
if (memcmp (actual->data, expected->data, actual->len) != 0) {
testsuite_check_failed ("%s failed: stream contents do not match", what);
goto error;
}
value = g_mime_filter_gzip_get_filename ((GMimeFilterGZip *) filter);
if (!value || strcmp (value, filename) != 0) {
testsuite_check_failed ("%s failed: filename does not match: %s", what, value);
goto error;
}
value = g_mime_filter_gzip_get_comment ((GMimeFilterGZip *) filter);
if (!value || strcmp (value, "This is a comment.") != 0) {
testsuite_check_failed ("%s failed: comment does not match: %s", what, value);
goto error;
}
testsuite_check_passed ();
error:
g_byte_array_free (expected, TRUE);
g_byte_array_free (actual, TRUE);
g_mime_filter_reset (filter);
g_object_unref (filter);
}
static void collect_part(GMimeObject *part, PartCollectorData *fdata, gboolean multipart_parent) {
GMimeContentType *content_type = g_mime_object_get_content_type(part);
GMimeContentDisposition *disposition = g_mime_object_get_content_disposition(part);
if (!content_type)
return;
GMimeDataWrapper *wrapper = g_mime_part_get_content_object(GMIME_PART(part));
if (!wrapper)
return;
// All the information will be collected in the CollectedPart
CollectedPart *c_part = new_collected_part(fdata->part_id);
gboolean is_attachment = FALSE;
if (disposition) {
c_part->disposition = g_ascii_strdown(disposition->disposition, -1);
is_attachment = !g_ascii_strcasecmp(disposition->disposition, GMIME_DISPOSITION_ATTACHMENT);
}
// If a filename is given, collect it always
const gchar *filename = g_mime_part_get_filename(GMIME_PART(part));
if (filename)
c_part->filename = g_strdup(filename);
// If a contentID is given, collect it always
const char* content_id = g_mime_part_get_content_id (GMIME_PART(part));
if (content_id)
c_part->content_id = g_strdup(content_id);
// Get the contentType in lowercase
gchar *content_type_str = g_mime_content_type_to_string(content_type);
c_part->content_type = g_ascii_strdown(content_type_str, -1);
g_free(content_type_str);
// To qualify as a message body, a MIME entity MUST NOT have a Content-Disposition header with the value "attachment".
if (!is_attachment && g_mime_content_type_is_type(content_type, "text", "*")) {
gboolean is_text_plain = g_mime_content_type_is_type(content_type, "text", "plain");
gboolean is_text_html = g_mime_content_type_is_type(content_type, "text", "html");
gboolean is_text_rtf = g_mime_content_type_is_type(content_type, "text", "rtf");
gboolean is_text_enriched = g_mime_content_type_is_type(content_type, "text", "enriched");
gboolean is_new_text = !fdata->text_part && is_text_plain;
gboolean is_new_html = !fdata->html_part && (is_text_html || is_text_enriched || is_text_rtf);
GMimeStream *mem_stream = g_mime_stream_mem_new();
g_mime_stream_mem_set_owner(GMIME_STREAM_MEM(mem_stream), FALSE);
GMimeStream *filtered_mem_stream = g_mime_stream_filter_new(mem_stream);
const gchar *charset = g_mime_object_get_content_type_parameter(part, "charset");
if (charset && g_ascii_strcasecmp(charset, UTF8_CHARSET)) {
GMimeFilter *utf8_charset_filter = g_mime_filter_charset_new(charset, UTF8_CHARSET);
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), utf8_charset_filter);
g_object_unref(utf8_charset_filter);
}
if (!fdata->raw && is_new_text) {
GMimeFilter *strip_filter = g_mime_filter_strip_new();
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), strip_filter);
g_object_unref(strip_filter);
GMimeFilter *crlf_filter = g_mime_filter_crlf_new(FALSE, FALSE);
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), crlf_filter);
g_object_unref(crlf_filter);
GMimeFilter *html_filter = g_mime_filter_html_new(
GMIME_FILTER_HTML_CONVERT_NL |
GMIME_FILTER_HTML_CONVERT_SPACES |
GMIME_FILTER_HTML_CONVERT_URLS |
GMIME_FILTER_HTML_MARK_CITATION |
GMIME_FILTER_HTML_CONVERT_ADDRESSES |
GMIME_FILTER_HTML_CITE, CITATION_COLOUR);
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), html_filter);
g_object_unref(html_filter);
}
if (!fdata->raw && (is_new_text || is_new_html)) {
GMimeFilter *from_filter = g_mime_filter_from_new(GMIME_FILTER_FROM_MODE_ESCAPE);
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), from_filter);
g_object_unref(from_filter);
}
// Add Enriched/RTF filter for this content
if (!fdata->raw && (is_new_html && (is_text_enriched || is_text_rtf))) {
guint flags = 0;
if (is_text_rtf)
flags = GMIME_FILTER_ENRICHED_IS_RICHTEXT;
GMimeFilter *enriched_filter = g_mime_filter_enriched_new(flags);
g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered_mem_stream), enriched_filter);
g_object_unref(enriched_filter);
}
g_mime_data_wrapper_write_to_stream(wrapper, filtered_mem_stream);
// Very important! Flush the the stream and get all content through.
g_mime_stream_flush(filtered_mem_stream);
// Freed by the mem_stream on its own (owner) [transfer none]
c_part->content = g_mime_stream_mem_get_byte_array(GMIME_STREAM_MEM(mem_stream));
// After we unref the mem_stream, part_content is NOT available anymore
g_object_unref(filtered_mem_stream);
g_object_unref(mem_stream);
// Without content, the collected body part is of no use, so we ignore it.
if (c_part->content->len == 0) {
free_collected_part(c_part);
return;
}
// We accept only the first text and first html content, everything
// else is considered an alternative body
if (is_new_text) {
fdata->text_part = c_part;
} else if (is_new_html) {
fdata->html_part = c_part;
} else {
g_ptr_array_add(fdata->alternative_bodies, c_part);
}
} else {
GMimeStream *attachment_mem_stream = g_mime_stream_mem_new();
g_mime_stream_mem_set_owner (GMIME_STREAM_MEM(attachment_mem_stream), FALSE);
g_mime_data_wrapper_write_to_stream(wrapper, attachment_mem_stream);
g_mime_stream_flush(attachment_mem_stream);
c_part->content = g_mime_stream_mem_get_byte_array(GMIME_STREAM_MEM(attachment_mem_stream));
g_object_unref(attachment_mem_stream);
// Some content may not have disposition defined so we need to determine better what it is
if ((disposition && !g_ascii_strcasecmp(disposition->disposition, GMIME_DISPOSITION_INLINE)) ||
g_mime_part_get_content_id(GMIME_PART(part))) {
g_ptr_array_add(fdata->inlines, c_part);
} else {
// All other disposition should be kept within attachments
g_ptr_array_add(fdata->attachments, c_part);
}
}
}
static void
rspamd_process_file (struct rspamd_config *cfg, const gchar *fname, gint mode)
{
struct rspamd_task *task;
gint fd;
gpointer map;
struct stat st;
GError *err = NULL;
#if 0
GMimeMessage *message;
GMimeParser *parser;
GMimeStream *stream;
GByteArray tmp;
#endif
struct rspamd_mime_part *part;
guint i;
gdouble ts1, ts2;
fd = open (fname, O_RDONLY);
if (fd == -1) {
rspamd_fprintf (stderr, "cannot open %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
if (fstat (fd, &st) == -1) {
rspamd_fprintf (stderr, "cannot stat %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
close (fd);
if (map == MAP_FAILED) {
rspamd_fprintf (stderr, "cannot mmap %s: %s\n", fname, strerror (errno));
exit (EXIT_FAILURE);
}
task = rspamd_task_new (NULL, cfg);
task->msg.begin = map;
task->msg.len = st.st_size;
ts1 = rspamd_get_ticks ();
if (mode == MODE_NORMAL) {
if (!rspamd_mime_parse_task (task, &err)) {
rspamd_fprintf (stderr, "cannot parse %s: %e\n", fname, err);
g_error_free (err);
}
}
#if 0
else if (mode == MODE_GMIME) {
tmp.data = map;
tmp.len = st.st_size;
stream = g_mime_stream_mem_new_with_byte_array (&tmp);
g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
parser = g_mime_parser_new_with_stream (stream);
message = g_mime_parser_construct_message (parser);
}
#endif
ts2 = rspamd_get_ticks ();
total_time += ts2 - ts1;
if (mode == MODE_NORMAL) {
for (i = 0; i < task->parts->len; i ++) {
part = g_ptr_array_index (task->parts, i);
if (part->ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
rspamd_show_multipart (part);
}
else if (part->ct->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
rspamd_show_message (part);
}
else {
rspamd_show_normal (part);
}
}
}
#if 0
else if (mode == MODE_GMIME) {
g_mime_message_foreach (message, mime_foreach_callback, NULL);
}
#endif
rspamd_task_free (task);
munmap (map, st.st_size);
#if 0
if (mode == MODE_GMIME) {
g_object_unref (message);
}
#endif
}
