static void test_compress_file(const char *in_path, const char *out_path) { const struct compression_handler *handler; struct istream *input, *file_input; struct ostream *output, *file_output; int fd_in, fd_out; struct sha1_ctxt sha1; unsigned char output_sha1[SHA1_RESULTLEN], input_sha1[SHA1_RESULTLEN]; const unsigned char *data; size_t size; ssize_t ret; handler = compression_lookup_handler_from_ext(out_path); if (handler == NULL) i_fatal("Can't detect compression algorithm from path %s", out_path); if (handler->create_ostream == NULL) i_fatal("Support not compiled in for %s", handler->name); /* write the compressed output file */ fd_in = open(in_path, O_RDONLY); if (fd_in == -1) i_fatal("open(%s) failed: %m", in_path); fd_out = open(out_path, O_TRUNC | O_CREAT | O_RDWR, 0600); if (fd_out == -1) i_fatal("creat(%s) failed: %m", out_path); sha1_init(&sha1); file_output = o_stream_create_fd_file(fd_out, 0, FALSE); output = handler->create_ostream(file_output, 1); input = i_stream_create_fd_autoclose(&fd_in, IO_BLOCK_SIZE); while (i_stream_read_data(input, &data, &size, 0) > 0) { sha1_loop(&sha1, data, size); o_stream_nsend(output, data, size); i_stream_skip(input, size); } if (o_stream_nfinish(output) < 0) { i_fatal("write(%s) failed: %s", out_path, o_stream_get_error(output)); } i_stream_destroy(&input); o_stream_destroy(&output); o_stream_destroy(&file_output); sha1_result(&sha1, output_sha1); /* verify that we can read the compressed file */ sha1_init(&sha1); file_input = i_stream_create_fd(fd_out, IO_BLOCK_SIZE, FALSE); input = handler->create_istream(file_input, FALSE); while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { sha1_loop(&sha1, data, size); i_stream_skip(input, size); } i_stream_destroy(&input); i_stream_destroy(&file_input); sha1_result(&sha1, input_sha1); if (memcmp(input_sha1, output_sha1, sizeof(input_sha1)) != 0) i_fatal("Decompression couldn't get the original input"); i_close_fd(&fd_out); }
int maildir_save_begin(struct mail_save_context *_ctx, struct istream *input) { struct maildir_save_context *ctx = (struct maildir_save_context *)_ctx; struct maildir_filename *mf; /* new mail, new failure state */ ctx->failed = FALSE; T_BEGIN { /* create a new file in tmp/ directory */ const char *fname; ctx->fd = maildir_create_tmp(ctx->mbox, ctx->tmpdir, &fname); if (ctx->fd == -1) ctx->failed = TRUE; else { if (ctx->mbox->storage->storage.set->mail_save_crlf) ctx->input = i_stream_create_crlf(input); else ctx->input = i_stream_create_lf(input); mf = maildir_save_add(_ctx, fname, NULL); if (_ctx->data.guid != NULL) { maildir_save_set_dest_basename(_ctx, mf, _ctx->data.guid); } } } T_END; if (!ctx->failed) { _ctx->data.output = o_stream_create_fd_file(ctx->fd, 0, FALSE); o_stream_cork(_ctx->data.output); ctx->last_save_finished = FALSE; } return ctx->failed ? -1 : 0; }
static int mail_index_recreate(struct mail_index *index) { struct mail_index_map *map = index->map; struct ostream *output; unsigned int base_size; const char *path; int ret = 0, fd; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); i_assert(map->hdr.indexid == index->indexid); i_assert((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) == 0); i_assert(index->indexid != 0); fd = mail_index_create_tmp_file(index, index->filepath, &path); if (fd == -1) return -1; output = o_stream_create_fd_file(fd, 0, FALSE); o_stream_cork(output); base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr)); o_stream_nsend(output, &map->hdr, base_size); o_stream_nsend(output, CONST_PTR_OFFSET(map->hdr_base, base_size), map->hdr.header_size - base_size); o_stream_nsend(output, map->rec_map->records, map->rec_map->records_count * map->hdr.record_size); o_stream_nflush(output); if (o_stream_nfinish(output) < 0) { mail_index_file_set_syscall_error(index, path, "write()"); ret = -1; } o_stream_destroy(&output); if (ret == 0 && index->fsync_mode != FSYNC_MODE_NEVER) { if (fdatasync(fd) < 0) { mail_index_file_set_syscall_error(index, path, "fdatasync()"); ret = -1; } } if (close(fd) < 0) { mail_index_file_set_syscall_error(index, path, "close()"); ret = -1; } if ((index->flags & MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS) != 0) (void)mail_index_create_backup(index); if (ret == 0 && rename(path, index->filepath) < 0) { mail_index_set_error(index, "rename(%s, %s) failed: %m", path, index->filepath); ret = -1; } if (ret < 0) i_unlink(path); return ret; }
int mbox_move(struct mbox_sync_context *sync_ctx, uoff_t dest, uoff_t source, uoff_t size) { struct istream *input; struct ostream *output; off_t ret; i_assert(size < OFF_T_MAX); if (size == 0 || source == dest) return 0; i_stream_sync(sync_ctx->input); output = o_stream_create_fd_file(sync_ctx->write_fd, (uoff_t)-1, FALSE); i_stream_seek(sync_ctx->file_input, source); if (o_stream_seek(output, dest) < 0) { mbox_set_syscall_error(sync_ctx->mbox, "o_stream_seek()"); o_stream_unref(&output); return -1; } input = i_stream_create_limit(sync_ctx->file_input, size); ret = o_stream_send_istream(output, input); i_stream_unref(&input); if (ret == (off_t)size) ret = 0; else if (ret >= 0) { mbox_sync_set_critical(sync_ctx, "mbox_move(%"PRIuUOFF_T", %"PRIuUOFF_T", %"PRIuUOFF_T ") moved only %"PRIuUOFF_T" bytes", dest, source, size, (uoff_t)ret); ret = -1; } else if (ret < 0) { errno = output->stream_errno; mbox_set_syscall_error(sync_ctx->mbox, "o_stream_send_istream()"); } mbox_sync_file_updated(sync_ctx, FALSE); o_stream_destroy(&output); return (int)ret; }
static int astream_decode_base64(struct attachment_istream *astream) { struct attachment_istream_part *part = &astream->part; buffer_t *extra_buf = NULL; struct istream *input, *base64_input; struct ostream *output; const unsigned char *data; size_t size; ssize_t ret; buffer_t *buf; int outfd; bool failed = FALSE; if (part->base64_bytes < astream->set.min_size || part->temp_output->offset > part->base64_bytes + BASE64_ATTACHMENT_MAX_EXTRA_BYTES) { /* only a small part of the MIME part is base64-encoded. */ return -1; } if (part->base64_line_blocks == 0) { /* only one line of base64 */ part->base64_line_blocks = part->cur_base64_blocks; i_assert(part->base64_line_blocks > 0); } /* decode base64 data and write it to another temp file */ outfd = astream->set.open_temp_fd(astream->context); if (outfd == -1) return -1; buf = buffer_create_dynamic(default_pool, 1024); input = i_stream_create_fd(part->temp_fd, IO_BLOCK_SIZE); base64_input = i_stream_create_limit(input, part->base64_bytes); output = o_stream_create_fd_file(outfd, 0, FALSE); o_stream_cork(output); hash_format_reset(astream->set.hash_format); size_t bytes_needed = 1; while ((ret = i_stream_read_bytes(base64_input, &data, &size, bytes_needed)) > 0) { buffer_set_used_size(buf, 0); if (base64_decode(data, size, &size, buf) < 0) { i_error("istream-attachment: BUG: " "Attachment base64 data unexpectedly broke"); failed = TRUE; break; } i_stream_skip(base64_input, size); o_stream_nsend(output, buf->data, buf->used); hash_format_loop(astream->set.hash_format, buf->data, buf->used); bytes_needed = i_stream_get_data_size(base64_input) + 1; } if (ret != -1) { i_assert(failed); } else if (base64_input->stream_errno != 0) { i_error("istream-attachment: read(%s) failed: %s", i_stream_get_name(base64_input), i_stream_get_error(base64_input)); failed = TRUE; } if (o_stream_nfinish(output) < 0) { i_error("istream-attachment: write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); failed = TRUE; } buffer_free(&buf); i_stream_unref(&base64_input); o_stream_unref(&output); if (input->v_offset != part->temp_output->offset && !failed) { /* write the rest of the data to the message stream */ extra_buf = buffer_create_dynamic(default_pool, 1024); while ((ret = i_stream_read_more(input, &data, &size)) > 0) { buffer_append(extra_buf, data, size); i_stream_skip(input, size); } i_assert(ret == -1); if (input->stream_errno != 0) { i_error("istream-attachment: read(%s) failed: %s", i_stream_get_name(input), i_stream_get_error(input)); failed = TRUE; } } i_stream_unref(&input); if (failed) { i_close_fd(&outfd); return -1; } /* successfully wrote it. switch to using it. */ o_stream_destroy(&part->temp_output); i_close_fd(&part->temp_fd); part->temp_fd = outfd; if (extra_buf != NULL) { stream_add_data(astream, extra_buf->data, extra_buf->used); buffer_free(&extra_buf); } return 0; }
static int mbox_save_init_file(struct mbox_save_context *ctx, struct mbox_transaction_context *t, bool want_mail) { struct mailbox_transaction_context *_t = &t->ictx.mailbox_ctx; struct mbox_mailbox *mbox = ctx->mbox; struct mail_storage *storage = &mbox->storage->storage; bool empty = FALSE; int ret; if (ctx->mbox->box.backend_readonly) { mail_storage_set_error(storage, MAIL_ERROR_PERM, "Read-only mbox"); return -1; } if ((_t->flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) != 0 || ctx->ctx.uid != 0) want_mail = TRUE; if (ctx->append_offset == (uoff_t)-1) { /* first appended mail in this transaction */ if (mbox->mbox_lock_type != F_WRLCK) { if (mbox->mbox_lock_type == F_RDLCK) { /* FIXME: we shouldn't fail here. it's just a locking issue that should be possible to fix.. */ mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE, "Can't copy mails inside same mailbox"); return -1; } if (mbox_lock(mbox, F_WRLCK, &t->mbox_lock_id) <= 0) return -1; } if (mbox->mbox_fd == -1) { if (mbox_file_open(mbox) < 0) return -1; } /* update mbox_sync_dirty state */ ret = mbox_sync_has_changed_full(mbox, TRUE, &empty); if (ret < 0) return -1; if (!want_mail && ret == 0) { /* we're not required to assign UIDs for the appended mails immediately. do it only if it doesn't require syncing. */ mbox_save_init_sync(_t); } } if (!ctx->synced && (want_mail || empty)) { /* we'll need to assign UID for the mail immediately. */ if (mbox_sync(mbox, 0) < 0) return -1; mbox_save_init_sync(_t); } /* the syncing above could have changed the append offset */ if (ctx->append_offset == (uoff_t)-1) { if (mbox_seek_to_end(ctx, &ctx->append_offset) < 0) return -1; ctx->output = o_stream_create_fd_file(mbox->mbox_fd, ctx->append_offset, FALSE); o_stream_cork(ctx->output); } return 0; }
int subsfile_set_subscribed(struct mailbox_list *list, const char *path, const char *temp_prefix, const char *name, bool set) { const struct mail_storage_settings *mail_set = list->mail_set; struct dotlock_settings dotlock_set; struct dotlock *dotlock; struct mailbox_permissions perm; const char *line, *dir, *fname, *escaped_name; struct istream *input = NULL; struct ostream *output; int fd_in, fd_out; enum mailbox_list_path_type type; bool found, changed = FALSE, failed = FALSE; unsigned int version = 2; if (strcasecmp(name, "INBOX") == 0) name = "INBOX"; memset(&dotlock_set, 0, sizeof(dotlock_set)); dotlock_set.use_excl_lock = mail_set->dotlock_use_excl; dotlock_set.nfs_flush = mail_set->mail_nfs_storage; dotlock_set.temp_prefix = temp_prefix; dotlock_set.timeout = SUBSCRIPTION_FILE_LOCK_TIMEOUT; dotlock_set.stale_timeout = SUBSCRIPTION_FILE_CHANGE_TIMEOUT; mailbox_list_get_root_permissions(list, &perm); fd_out = file_dotlock_open_group(&dotlock_set, path, 0, perm.file_create_mode, perm.file_create_gid, perm.file_create_gid_origin, &dotlock); if (fd_out == -1 && errno == ENOENT) { /* directory hasn't been created yet. */ type = list->set.control_dir != NULL ? MAILBOX_LIST_PATH_TYPE_CONTROL : MAILBOX_LIST_PATH_TYPE_DIR; fname = strrchr(path, '/'); if (fname != NULL) { dir = t_strdup_until(path, fname); if (mailbox_list_mkdir_root(list, dir, type) < 0) return -1; } fd_out = file_dotlock_open_group(&dotlock_set, path, 0, perm.file_create_mode, perm.file_create_gid, perm.file_create_gid_origin, &dotlock); } if (fd_out == -1) { if (errno == EAGAIN) { mailbox_list_set_error(list, MAIL_ERROR_TEMP, "Timeout waiting for subscription file lock"); } else { subswrite_set_syscall_error(list, "file_dotlock_open()", path); } return -1; } fd_in = nfs_safe_open(path, O_RDONLY); if (fd_in == -1 && errno != ENOENT) { subswrite_set_syscall_error(list, "open()", path); file_dotlock_delete(&dotlock); return -1; } if (fd_in != -1) { input = i_stream_create_fd_autoclose(&fd_in, list->mailbox_name_max_length+1); i_stream_set_return_partial_line(input, TRUE); subsfile_list_read_header(list, input, &version); } found = FALSE; output = o_stream_create_fd_file(fd_out, 0, FALSE); o_stream_cork(output); if (version >= 2) o_stream_send_str(output, version2_header); if (version < 2 || name[0] == '\0') escaped_name = name; else { const char *const *tmp; char separators[2]; string_t *str = t_str_new(64); separators[0] = mailbox_list_get_hierarchy_sep(list); separators[1] = '\0'; tmp = t_strsplit(name, separators); str_append_tabescaped(str, *tmp); for (tmp++; *tmp != NULL; tmp++) { str_append_c(str, '\t'); str_append_tabescaped(str, *tmp); } escaped_name = str_c(str); } if (input != NULL) { while ((line = next_line(list, path, input, &failed, FALSE)) != NULL) { if (strcmp(line, escaped_name) == 0) { found = TRUE; if (!set) { changed = TRUE; continue; } } o_stream_nsend_str(output, line); o_stream_nsend(output, "\n", 1); } i_stream_destroy(&input); } if (!failed && set && !found) { /* append subscription */ line = t_strconcat(escaped_name, "\n", NULL); o_stream_nsend_str(output, line); changed = TRUE; } if (changed && !failed) { if (o_stream_nfinish(output) < 0) { subswrite_set_syscall_error(list, "write()", path); failed = TRUE; } else if (mail_set->parsed_fsync_mode != FSYNC_MODE_NEVER) { if (fsync(fd_out) < 0) { subswrite_set_syscall_error(list, "fsync()", path); failed = TRUE; } } } else { o_stream_ignore_last_errors(output); } o_stream_destroy(&output); if (failed || !changed) { if (file_dotlock_delete(&dotlock) < 0) { subswrite_set_syscall_error(list, "file_dotlock_delete()", path); failed = TRUE; } } else { enum dotlock_replace_flags flags = DOTLOCK_REPLACE_FLAG_VERIFY_OWNER; if (file_dotlock_replace(&dotlock, flags) < 0) { subswrite_set_syscall_error(list, "file_dotlock_replace()", path); failed = TRUE; } } return failed ? -1 : (changed ? 1 : 0); }
static void test_compression_handler(const struct compression_handler *handler) { const char *path = "test-compression.tmp"; struct istream *file_input, *input; struct ostream *file_output, *output; unsigned char buf[IO_BLOCK_SIZE]; const unsigned char *data; size_t size; struct sha1_ctxt sha1; unsigned char output_sha1[SHA1_RESULTLEN], input_sha1[SHA1_RESULTLEN]; unsigned int i; int fd; ssize_t ret; test_begin(t_strdup_printf("compression handler %s", handler->name)); /* write compressed data */ fd = open(path, O_TRUNC | O_CREAT | O_RDWR, 0600); if (fd == -1) i_fatal("creat(%s) failed: %m", path); file_output = o_stream_create_fd_file(fd, 0, FALSE); output = handler->create_ostream(file_output, 1); sha1_init(&sha1); /* 1) write lots of easily compressible data */ memset(buf, 0, sizeof(buf)); for (i = 0; i < 1024*1024*4 / sizeof(buf); i++) { sha1_loop(&sha1, buf, sizeof(buf)); test_assert(o_stream_send(output, buf, sizeof(buf)) == sizeof(buf)); } /* 2) write uncompressible data */ for (i = 0; i < 1024*128 / sizeof(buf); i++) { random_fill_weak(buf, sizeof(buf)); sha1_loop(&sha1, buf, sizeof(buf)); test_assert(o_stream_send(output, buf, sizeof(buf)) == sizeof(buf)); } /* 3) write semi-compressible data */ for (i = 0; i < sizeof(buf); i++) { if (rand () % 3 == 0) buf[i] = rand() % 4; else buf[i] = i; } for (i = 0; i < 1024*128 / sizeof(buf); i++) { sha1_loop(&sha1, buf, sizeof(buf)); test_assert(o_stream_send(output, buf, sizeof(buf)) == sizeof(buf)); } o_stream_destroy(&output); o_stream_destroy(&file_output); sha1_result(&sha1, output_sha1); /* read and uncompress the data */ sha1_init(&sha1); file_input = i_stream_create_fd(fd, IO_BLOCK_SIZE, FALSE); input = handler->create_istream(file_input, FALSE); while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { sha1_loop(&sha1, data, size); i_stream_skip(input, size); } test_assert(ret == -1); i_stream_destroy(&input); i_stream_destroy(&file_input); sha1_result(&sha1, input_sha1); test_assert(memcmp(input_sha1, output_sha1, sizeof(input_sha1)) == 0); if (unlink(path) < 0) i_error("unlink(%s) failed: %m", path); test_end(); }
int subsfile_set_subscribed(struct mailbox_list *list, const char *path, const char *temp_prefix, const char *name, bool set) { const struct mail_storage_settings *mail_set = list->mail_set; struct dotlock_settings dotlock_set; struct dotlock *dotlock; const char *line, *origin; struct istream *input; struct ostream *output; int fd_in, fd_out; mode_t mode; gid_t gid; bool found, changed = FALSE, failed = FALSE; if (strcasecmp(name, "INBOX") == 0) name = "INBOX"; memset(&dotlock_set, 0, sizeof(dotlock_set)); dotlock_set.use_excl_lock = mail_set->dotlock_use_excl; dotlock_set.nfs_flush = mail_set->mail_nfs_storage; dotlock_set.temp_prefix = temp_prefix; dotlock_set.timeout = SUBSCRIPTION_FILE_LOCK_TIMEOUT; dotlock_set.stale_timeout = SUBSCRIPTION_FILE_CHANGE_TIMEOUT; mailbox_list_get_permissions(list, NULL, &mode, &gid, &origin); fd_out = file_dotlock_open_group(&dotlock_set, path, 0, mode, gid, origin, &dotlock); if (fd_out == -1 && errno == ENOENT) { /* directory hasn't been created yet. */ if (mailbox_list_create_parent_dir(list, NULL, path) < 0) return -1; fd_out = file_dotlock_open_group(&dotlock_set, path, 0, mode, gid, origin, &dotlock); } if (fd_out == -1) { if (errno == EAGAIN) { mailbox_list_set_error(list, MAIL_ERROR_TEMP, "Timeout waiting for subscription file lock"); } else { subswrite_set_syscall_error(list, "file_dotlock_open()", path); } return -1; } fd_in = nfs_safe_open(path, O_RDONLY); if (fd_in == -1 && errno != ENOENT) { subswrite_set_syscall_error(list, "open()", path); (void)file_dotlock_delete(&dotlock); return -1; } input = fd_in == -1 ? NULL : i_stream_create_fd(fd_in, list->mailbox_name_max_length+1, TRUE); output = o_stream_create_fd_file(fd_out, 0, FALSE); o_stream_cork(output); found = FALSE; while ((line = next_line(list, path, input, &failed, FALSE)) != NULL) { if (strcmp(line, name) == 0) { found = TRUE; if (!set) { changed = TRUE; continue; } } (void)o_stream_send_str(output, line); (void)o_stream_send(output, "\n", 1); } if (!failed && set && !found) { /* append subscription */ line = t_strconcat(name, "\n", NULL); (void)o_stream_send_str(output, line); changed = TRUE; } if (changed && !failed) { if (o_stream_flush(output) < 0) { subswrite_set_syscall_error(list, "write()", path); failed = TRUE; } else if (mail_set->parsed_fsync_mode != FSYNC_MODE_NEVER) { if (fsync(fd_out) < 0) { subswrite_set_syscall_error(list, "fsync()", path); failed = TRUE; } } } if (input != NULL) i_stream_destroy(&input); o_stream_destroy(&output); if (failed || !changed) { if (file_dotlock_delete(&dotlock) < 0) { subswrite_set_syscall_error(list, "file_dotlock_delete()", path); failed = TRUE; } } else { enum dotlock_replace_flags flags = DOTLOCK_REPLACE_FLAG_VERIFY_OWNER; if (file_dotlock_replace(&dotlock, flags) < 0) { subswrite_set_syscall_error(list, "file_dotlock_replace()", path); failed = TRUE; } } return failed ? -1 : (changed ? 1 : 0); }
int dbox_file_fix(struct dbox_file *file, uoff_t start_offset) { struct ostream *output; const char *dir, *p, *temp_path, *broken_path; bool deleted; int fd, ret; i_assert(dbox_file_is_open(file)); p = strrchr(file->cur_path, '/'); i_assert(p != NULL); dir = t_strdup_until(file->cur_path, p); temp_path = t_strdup_printf("%s/%s", dir, dbox_generate_tmp_filename()); fd = file->storage->v.file_create_fd(file, temp_path, FALSE); if (fd == -1) return -1; output = o_stream_create_fd_file(fd, 0, FALSE); ret = dbox_file_fix_write_stream(file, start_offset, temp_path, output); o_stream_unref(&output); if (close(fd) < 0) { mail_storage_set_critical(&file->storage->storage, "close(%s) failed: %m", temp_path); ret = -1; } if (ret < 0) { if (unlink(temp_path) < 0) { mail_storage_set_critical(&file->storage->storage, "unlink(%s) failed: %m", temp_path); } return -1; } /* keep a copy of the original file in case someone wants to look at it */ broken_path = t_strconcat(file->cur_path, DBOX_MAIL_FILE_BROKEN_COPY_SUFFIX, NULL); if (link(file->cur_path, broken_path) < 0) { mail_storage_set_critical(&file->storage->storage, "link(%s, %s) failed: %m", file->cur_path, broken_path); } else { i_warning("dbox: Copy of the broken file saved to %s", broken_path); } if (rename(temp_path, file->cur_path) < 0) { mail_storage_set_critical(&file->storage->storage, "rename(%s, %s) failed: %m", temp_path, file->cur_path); return -1; } /* file was successfully recreated - reopen it */ dbox_file_close(file); if (dbox_file_open(file, &deleted) <= 0) { mail_storage_set_critical(&file->storage->storage, "dbox_file_fix(%s): reopening file failed", file->cur_path); return -1; } return 0; }