void backend_disconnect(struct backend *s) { char buf[1024]; if (!s || s->sock == -1) return; if (!prot_error(s->in)) { if (s->prot->logout_cmd.cmd) { prot_printf(s->out, "%s\r\n", s->prot->logout_cmd.cmd); prot_flush(s->out); for (;;) { if (!prot_fgets(buf, sizeof(buf), s->in)) { /* connection closed? */ break; } else if (s->prot->logout_cmd.unsol && !strncmp(s->prot->logout_cmd.unsol, buf, strlen(s->prot->logout_cmd.unsol))) { /* unsolicited response */ continue; } else { /* success/fail response -- don't care either way */ break; } } } } /* Flush the incoming buffer */ prot_NONBLOCK(s->in); prot_fill(s->in); #ifdef HAVE_SSL /* Free tlsconn */ if (s->tlsconn) { tls_reset_servertls(&s->tlsconn); s->tlsconn = NULL; } #endif /* HAVE_SSL */ /* close/free socket & prot layer */ cyrus_close_sock(s->sock); s->sock = -1; prot_free(s->in); prot_free(s->out); s->in = s->out = NULL; /* Free saslconn */ if(s->saslconn) { sasl_dispose(&(s->saslconn)); s->saslconn = NULL; } /* free last_result buffer */ buf_free(&s->last_result); forget_capabilities(s); }
/* verify that the matching MAILBOX exists within the claimed chunk * for each mailbox or mailbox_message in the index */ static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk, struct gzuncat *gzuc, int verbose, FILE *out) { /* * get list of mailboxes in chunk * get list of mailbox_messages in chunk * index mailboxes list by uniqueid * index mailbox_messages list by uniqueid:uid * open chunk * foreach line in chunk * read dlist * skip if it's not a mailbox * if details in dlist match details in mailbox * remove from mailbox list/index * foreach record in dlist * if details in dlist match details in mailbox_message * remove from mailbox_message list/index * failed if either list of mailboxes or list of mailbox_messages is not empty */ struct backup_mailbox_list *mailbox_list = NULL; struct backup_mailbox_message_list *mailbox_message_list = NULL; hash_table mailbox_list_index = HASH_TABLE_INITIALIZER; hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER; struct backup_mailbox *mailbox = NULL; struct backup_mailbox_message *mailbox_message = NULL; int r; if (out && verbose) fprintf(out, "checking chunk %d mailbox links...\n", chunk->id); mailbox_list = backup_get_mailboxes(backup, chunk->id, 0); mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id); if (mailbox_list->count == 0 && mailbox_message_list->count == 0) { /* nothing we care about in this chunk */ free(mailbox_list); free(mailbox_message_list); if (out && verbose) fprintf(out, "ok\n"); return 0; } /* XXX consider whether the two hashes should use pools */ if (mailbox_list->count) { /* build an index of the mailbox list */ construct_hash_table(&mailbox_list_index, mailbox_list->count, 0); mailbox = mailbox_list->head; while (mailbox) { hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index); mailbox = mailbox->next; } } if (mailbox_message_list->count) { /* build an index of the mailbox message list */ construct_hash_table(&mailbox_message_list_index, mailbox_message_list->count, 0); mailbox_message = mailbox_message_list->head; while (mailbox_message) { char keybuf[1024]; // FIXME whatever snprintf(keybuf, sizeof(keybuf), "%s:%d", mailbox_message->mailbox_uniqueid, mailbox_message->uid); hash_insert(keybuf, mailbox_message, &mailbox_message_list_index); mailbox_message = mailbox_message->next; } } r = gzuc_member_start_from(gzuc, chunk->offset); if (r) { syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s", __func__, chunk->id, chunk->offset, zError(r)); if (out) fprintf(out, "error reading chunk %i at offset %jd: %s", chunk->id, chunk->offset, zError(r)); goto done; } struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); prot_setisclient(ps, 1); /* don't sync literals */ struct buf cmd = BUF_INITIALIZER; while (1) { struct dlist *dl = NULL; struct dlist *record = NULL; struct dlist *di = NULL; const char *uniqueid = NULL; int c = parse_backup_line(ps, NULL, &cmd, &dl); if (c == EOF) { const char *error = prot_error(ps); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "%s: error reading chunk %i data at offset %jd, byte %i: %s", __func__, chunk->id, chunk->offset, prot_bytes_in(ps), error); if (out) fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s", chunk->id, chunk->offset, prot_bytes_in(ps), error); r = EOF; } break; } if (strcmp(buf_cstring(&cmd), "APPLY") != 0) goto next_line; if (strcmp(dl->name, "MAILBOX") != 0) goto next_line; if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) goto next_line; if (mailbox_list->count) { mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index); if (mailbox && mailbox_matches(mailbox, dl)) { backup_mailbox_list_remove(mailbox_list, mailbox); hash_del(uniqueid, &mailbox_list_index); backup_mailbox_free(&mailbox); } } if (mailbox_message_list->count) { if (!dlist_getlist(dl, "RECORD", &record)) goto next_line; for (di = record->head; di; di = di->next) { char keybuf[1024]; // FIXME whatever uint32_t uid; if (!dlist_getnum32(di, "UID", &uid)) continue; snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid); mailbox_message = (struct backup_mailbox_message *) hash_lookup( keybuf, &mailbox_message_list_index); if (!mailbox_message) continue; if (!mailbox_message_matches(mailbox_message, di)) continue; backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message); hash_del(keybuf, &mailbox_message_list_index); backup_mailbox_message_free(&mailbox_message); } } next_line: if (dl) { dlist_unlink_files(dl); dlist_free(&dl); } } buf_free(&cmd); prot_free(ps); gzuc_member_end(gzuc, NULL); /* anything left in either of the lists is missing from the chunk data. bad! */ mailbox = mailbox_list->head; while (mailbox) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n", __func__, chunk->id, mailbox->uniqueid, mailbox->mboxname); if (out) fprintf(out, "chunk %d missing mailbox data for %s (%s)\n", chunk->id, mailbox->uniqueid, mailbox->mboxname); mailbox = mailbox->next; } mailbox_message = mailbox_message_list->head; while (mailbox_message) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n", __func__, chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); if (out) fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n", chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); mailbox_message = mailbox_message->next; } if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0; done: free_hash_table(&mailbox_list_index, NULL); free_hash_table(&mailbox_message_list_index, NULL); backup_mailbox_list_empty(mailbox_list); free(mailbox_list); backup_mailbox_message_list_empty(mailbox_message_list); free(mailbox_message_list); syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id, r ? "failed" : "passed"); if (out && verbose) fprintf(out, "%s\n", r ? "error" : "ok"); return r; }
HIDDEN void ws_input(struct transaction_t *txn) { struct ws_context *ctx = (struct ws_context *) txn->ws_ctx; wslay_event_context_ptr ev = ctx->event; int want_read = wslay_event_want_read(ev); int want_write = wslay_event_want_write(ev); int goaway = txn->flags.conn & CONN_CLOSE; syslog(LOG_DEBUG, "ws_input() eof: %d, want read: %d, want write: %d", txn->conn->pin->eof, want_read, want_write); if (want_read && !goaway) { /* Read frame(s) */ if (txn->conn->sess_ctx) { /* Data has been read into request body */ ctx->h2_pin = prot_readmap(buf_base(&txn->req_body.payload), buf_len(&txn->req_body.payload)); } int r = wslay_event_recv(ev); if (txn->conn->sess_ctx) { buf_reset(&txn->req_body.payload); prot_free(ctx->h2_pin); } if (!r) { /* Successfully received frames */ syslog(LOG_DEBUG, "ws_event_recv: success"); } else if (r == WSLAY_ERR_NO_MORE_MSG) { /* Client closed connection */ syslog(LOG_DEBUG, "client closed connection"); txn->flags.conn = CONN_CLOSE; } else { /* Failure */ syslog(LOG_DEBUG, "ws_event_recv: %s (%s)", wslay_strerror(r), prot_error(txn->conn->pin)); goaway = 1; if (r == WSLAY_ERR_CALLBACK_FAILURE) { /* Client timeout */ txn->error.desc = prot_error(txn->conn->pin); } else { txn->error.desc = wslay_strerror(r); } } } else if (!want_write) { /* Connection is done */ syslog(LOG_DEBUG, "connection closed"); txn->flags.conn = CONN_CLOSE; } if (goaway) { /* Tell client we are closing session */ syslog(LOG_WARNING, "%s, closing connection", txn->error.desc); syslog(LOG_DEBUG, "wslay_event_queue_close()"); int r = wslay_event_queue_close(ev, WSLAY_CODE_GOING_AWAY, (uint8_t *) txn->error.desc, strlen(txn->error.desc)); if (r) { syslog(LOG_ERR, "wslay_event_queue_close: %s", wslay_strerror(r)); } txn->flags.conn = CONN_CLOSE; } /* Write frame(s) */ ws_output(txn); return; }
static int _verify_message_cb(const struct backup_message *message, void *rock) { struct verify_message_rock *vmrock = (struct verify_message_rock *) rock; struct dlist *dl = NULL; struct dlist *di = NULL; FILE *out = vmrock->out; int r; /* cache the dlist so that multiple reads from the same offset don't * cause expensive reverse seeks in decompression stream */ if (!vmrock->cached_dlist || vmrock->cached_offset != message->offset) { if (vmrock->cached_dlist) { dlist_unlink_files(vmrock->cached_dlist); dlist_free(&vmrock->cached_dlist); } r = gzuc_seekto(vmrock->gzuc, message->offset); if (r) return r; struct protstream *ps = prot_readcb(_prot_fill_cb, vmrock->gzuc); prot_setisclient(ps, 1); /* don't sync literals */ r = parse_backup_line(ps, NULL, NULL, &dl); prot_free(ps); if (r == EOF) { const char *error = prot_error(ps); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "%s: error reading message %i at offset %jd, byte %i: %s", __func__, message->id, message->offset, prot_bytes_in(ps), error); if (out) fprintf(out, "error reading message %i at offset %jd, byte %i: %s", message->id, message->offset, prot_bytes_in(ps), error); } return r; } vmrock->cached_dlist = dl; vmrock->cached_offset = message->offset; } else { dl = vmrock->cached_dlist; } r = strcmp(dl->name, "MESSAGE"); if (r) return r; r = -1; for (di = dl->head; di; di = di->next) { struct message_guid *guid = NULL; const char *fname = NULL; if (!dlist_tofile(di, NULL, &guid, NULL, &fname)) continue; r = message_guid_cmp(guid, message->guid); if (!r) { if (vmrock->verify_guid) { const char *msg_base = NULL; size_t msg_len = 0; struct message_guid computed_guid; int fd; fd = open(fname, O_RDWR); if (fd != -1) { map_refresh(fd, 1, &msg_base, &msg_len, MAP_UNKNOWN_LEN, fname, NULL); message_guid_generate(&computed_guid, msg_base, msg_len); r = message_guid_cmp(&computed_guid, message->guid); if (r && out) fprintf(out, "guid mismatch for message %i\n", message->id); map_free(&msg_base, &msg_len); close(fd); } else { syslog(LOG_ERR, "IOERROR: %s open %s: %m", __func__, fname); if (out) fprintf(out, "error reading staging file for message %i\n", message->id); r = -1; } } break; } } return r; }
/* returns: * 0 on success * 1 if compact was not needed * negative on error */ EXPORTED int backup_compact(const char *name, enum backup_open_nonblock nonblock, int force, int verbose, FILE *out) { struct backup *original = NULL; struct backup *compact = NULL; struct backup_chunk_list *all_chunks = NULL; struct backup_chunk_list *keep_chunks = NULL; struct backup_chunk *chunk = NULL; struct sync_msgid_list *keep_message_guids = NULL; struct gzuncat *gzuc = NULL; struct protstream *in = NULL; time_t since, chunk_start_time, ts; int r; compact_readconfig(); r = compact_open(name, &original, &compact, nonblock); if (r) return r; /* calculate current time after obtaining locks, in case of a wait */ const time_t now = time(NULL); const int retention_days = config_getint(IMAPOPT_BACKUP_RETENTION_DAYS); if (retention_days > 0) { since = now - (retention_days * 24 * 60 * 60); } else { /* zero or negative retention days means "keep forever" */ since = -1; } all_chunks = backup_get_chunks(original); if (!all_chunks) goto error; keep_chunks = backup_get_live_chunks(original, since); if (!keep_chunks) goto error; if (!force && !compact_required(all_chunks, keep_chunks)) { /* nothing to do */ backup_chunk_list_free(&all_chunks); backup_chunk_list_free(&keep_chunks); backup_unlink(&compact); backup_close(&original); return 1; } if (verbose) { fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count); for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { fprintf(out, " %d", chunk->id); } fprintf(out, "\n"); } gzuc = gzuc_new(original->fd); if (!gzuc) goto error; chunk_start_time = -1; ts = 0; struct buf cmd = BUF_INITIALIZER; for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { keep_message_guids = sync_msgid_list_create(0); r = backup_message_foreach(original, chunk->id, &since, _keep_message_guids_cb, keep_message_guids); if (r) goto error; gzuc_member_start_from(gzuc, chunk->offset); in = prot_readcb(_prot_fill_cb, gzuc); while (1) { struct dlist *dl = NULL; int c = parse_backup_line(in, &ts, &cmd, &dl); if (c == EOF) { const char *error = prot_error(in); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "IOERROR: %s: error reading chunk at offset %jd, byte %i: %s\n", name, chunk->offset, prot_bytes_in(in), error); if (out) fprintf(out, "error reading chunk at offset %jd, byte %i: %s\n", chunk->offset, prot_bytes_in(in), error); r = IMAP_IOERROR; goto error; } break; } if (chunk_start_time == -1) { r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; chunk_start_time = ts; } // XXX if this line is worth keeping if (want_append(original, chunk->id, dl, keep_message_guids)) { // FIXME if message is removed due to unneeded chunk, // subsequent mailbox lines for it will fail here // so we need to be able to tell which lines apply to messages we don't want anymore r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; } dlist_unlink_files(dl); dlist_free(&dl); // if this line put us over compact_maxsize if (want_split(chunk, &compact->append_state->wrote)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; if (verbose) { fprintf(out, "splitting chunk %d\n", chunk->id); } } } // if we're due to start a new chunk if (compact->append_state && compact->append_state->mode) { if (!want_combine(compact->append_state->wrote, chunk->next)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; } else if (verbose) { fprintf(out, "combining chunks %d and %d\n", chunk->id, chunk->next->id); } } prot_free(in); in = NULL; gzuc_member_end(gzuc, NULL); sync_msgid_list_free(&keep_message_guids); } buf_free(&cmd); if (compact->append_state && compact->append_state->mode) backup_append_end(compact, &ts); gzuc_free(&gzuc); backup_chunk_list_free(&keep_chunks); /* if we get here okay, then the compact succeeded */ r = compact_closerename(&original, &compact, now); if (r) goto error; return 0; error: if (in) prot_free(in); if (gzuc) gzuc_free(&gzuc); if (keep_message_guids) sync_msgid_list_free(&keep_message_guids); if (all_chunks) backup_chunk_list_free(&all_chunks); if (keep_chunks) backup_chunk_list_free(&keep_chunks); if (compact) backup_unlink(&compact); if (original) backup_close(&original); return r ? r : -1; }