static QMGR_MESSAGE *qmgr_message_create(const char *queue_name, const char *queue_id, int qflags) { QMGR_MESSAGE *message; message = (QMGR_MESSAGE *) mymalloc(sizeof(QMGR_MESSAGE)); qmgr_message_count++; message->flags = 0; message->qflags = qflags; message->tflags = 0; message->tflags_offset = 0; message->rflags = QMGR_READ_FLAG_DEFAULT; message->fp = 0; message->refcount = 0; message->single_rcpt = 0; message->arrival_time.tv_sec = message->arrival_time.tv_usec = 0; message->create_time = 0; GETTIMEOFDAY(&message->active_time); message->queued_time = sane_time(); message->refill_time = 0; message->data_offset = 0; message->queue_id = mystrdup(queue_id); message->queue_name = mystrdup(queue_name); message->encoding = 0; message->sender = 0; message->dsn_envid = 0; message->dsn_ret = 0; message->smtputf8 = 0; message->filter_xport = 0; message->inspect_xport = 0; message->redirect_addr = 0; message->data_size = 0; message->cont_length = 0; message->warn_offset = 0; message->warn_time = 0; message->rcpt_offset = 0; message->verp_delims = 0; message->client_name = 0; message->client_addr = 0; message->client_port = 0; message->client_proto = 0; message->client_helo = 0; message->sasl_method = 0; message->sasl_username = 0; message->sasl_sender = 0; message->log_ident = 0; message->rewrite_context = 0; recipient_list_init(&message->rcpt_list, RCPT_LIST_INIT_QUEUE); message->rcpt_count = 0; message->rcpt_limit = var_qmgr_msg_rcpt_limit; message->rcpt_unread = 0; QMGR_LIST_INIT(message->job_list); return (message); }
static int qmgr_message_read(QMGR_MESSAGE *message) { VSTRING *buf; int rec_type; long curr_offset; long save_offset = message->rcpt_offset; /* save a flag */ int save_unread = message->rcpt_unread; /* save a count */ char *start; int recipient_limit; const char *error_text; char *name; char *value; char *orig_rcpt = 0; int count; int dsn_notify = 0; char *dsn_orcpt = 0; int n; int have_log_client_attr = 0; /* * Initialize. No early returns or we have a memory leak. */ buf = vstring_alloc(100); /* * If we re-open this file, skip over on-file recipient records that we * already looked at, and refill the in-core recipient address list. * * For the first time, the message recipient limit is calculated from the * global recipient limit. This is to avoid reading little recipients * when the active queue is near empty. When the queue becomes full, only * the necessary amount is read in core. Such priming is necessary * because there are no message jobs yet. * * For the next time, the recipient limit is based solely on the message * jobs' positions in the job lists and/or job stacks. */ if (message->rcpt_offset) { if (message->rcpt_list.len) msg_panic("%s: recipient list not empty on recipient reload", message->queue_id); if (vstream_fseek(message->fp, message->rcpt_offset, SEEK_SET) < 0) msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); message->rcpt_offset = 0; recipient_limit = message->rcpt_limit - message->rcpt_count; } else { recipient_limit = var_qmgr_rcpt_limit - qmgr_recipient_count; if (recipient_limit < message->rcpt_limit) recipient_limit = message->rcpt_limit; } /* Keep interrupt latency in check. */ if (recipient_limit > 5000) recipient_limit = 5000; if (recipient_limit <= 0) msg_panic("%s: no recipient slots available", message->queue_id); if (msg_verbose) msg_info("%s: recipient limit %d", message->queue_id, recipient_limit); /* * Read envelope records. XXX Rely on the front-end programs to enforce * record size limits. Read up to recipient_limit recipients from the * queue file, to protect against memory exhaustion. Recipient records * may appear before or after the message content, so we keep reading * from the queue file until we have enough recipients (rcpt_offset != 0) * and until we know all the non-recipient information. * * Note that the total recipient count record is accurate only for fresh * queue files. After some of the recipients are marked as done and the * queue file is deferred, it can be used as upper bound estimate only. * Fortunately, this poses no major problem on the scheduling algorithm, * as the only impact is that the already deferred messages are not * chosen by qmgr_job_candidate() as often as they could. * * On the first open, we must examine all non-recipient records. * * Optimization: when we know that recipient records are not mixed with * non-recipient records, as is typical with mailing list mail, then we * can avoid having to examine all the queue file records before we can * start deliveries. This avoids some file system thrashing with huge * mailing lists. */ for (;;) { if ((curr_offset = vstream_ftell(message->fp)) < 0) msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); if (curr_offset == message->data_offset && curr_offset > 0) { if (vstream_fseek(message->fp, message->data_size, SEEK_CUR) < 0) msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); curr_offset += message->data_size; } rec_type = rec_get_raw(message->fp, buf, 0, REC_FLAG_NONE); start = vstring_str(buf); if (msg_verbose > 1) msg_info("record %c %s", rec_type, start); if (rec_type == REC_TYPE_PTR) { if ((rec_type = rec_goto(message->fp, start)) == REC_TYPE_ERROR) break; /* Need to update curr_offset after pointer jump. */ continue; } if (rec_type <= 0) { msg_warn("%s: message rejected: missing end record", message->queue_id); break; } if (rec_type == REC_TYPE_END) { message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT; break; } /* * Map named attributes to pseudo record types, so that we don't have * to pollute the queue file with records that are incompatible with * past Postfix versions. Preferably, people should be able to back * out from an upgrade without losing mail. */ if (rec_type == REC_TYPE_ATTR) { if ((error_text = split_nameval(start, &name, &value)) != 0) { msg_warn("%s: ignoring bad attribute: %s: %.200s", message->queue_id, error_text, start); rec_type = REC_TYPE_ERROR; break; } if ((n = rec_attr_map(name)) != 0) { start = value; rec_type = n; } } /* * Process recipient records. */ if (rec_type == REC_TYPE_RCPT) { /* See also below for code setting orig_rcpt etc. */ if (message->rcpt_offset == 0) { message->rcpt_unread--; recipient_list_add(&message->rcpt_list, curr_offset, dsn_orcpt ? dsn_orcpt : "", dsn_notify ? dsn_notify : 0, orig_rcpt ? orig_rcpt : "", start); if (dsn_orcpt) { myfree(dsn_orcpt); dsn_orcpt = 0; } if (orig_rcpt) { myfree(orig_rcpt); orig_rcpt = 0; } if (dsn_notify) dsn_notify = 0; if (message->rcpt_list.len >= recipient_limit) { if ((message->rcpt_offset = vstream_ftell(message->fp)) < 0) msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT) /* We already examined all non-recipient records. */ break; if (message->rflags & QMGR_READ_FLAG_MIXED_RCPT_OTHER) /* Examine all remaining non-recipient records. */ continue; /* Optimizations for "pure recipient" record sections. */ if (curr_offset > message->data_offset) { /* We already examined all non-recipient records. */ message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT; break; } /* Examine non-recipient records in extracted segment. */ if (vstream_fseek(message->fp, message->data_offset + message->data_size, SEEK_SET) < 0) msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); continue; } } continue; } if (rec_type == REC_TYPE_DONE || rec_type == REC_TYPE_DRCP) { if (message->rcpt_offset == 0) { message->rcpt_unread--; if (dsn_orcpt) { myfree(dsn_orcpt); dsn_orcpt = 0; } if (orig_rcpt) { myfree(orig_rcpt); orig_rcpt = 0; } if (dsn_notify) dsn_notify = 0; } continue; } if (rec_type == REC_TYPE_DSN_ORCPT) { /* See also above for code clearing dsn_orcpt. */ if (dsn_orcpt != 0) { msg_warn("%s: ignoring out-of-order DSN original recipient address <%.200s>", message->queue_id, dsn_orcpt); myfree(dsn_orcpt); dsn_orcpt = 0; } if (message->rcpt_offset == 0) dsn_orcpt = mystrdup(start); continue; } if (rec_type == REC_TYPE_DSN_NOTIFY) { /* See also above for code clearing dsn_notify. */ if (dsn_notify != 0) { msg_warn("%s: ignoring out-of-order DSN notify flags <%d>", message->queue_id, dsn_notify); dsn_notify = 0; } if (message->rcpt_offset == 0) { if (!alldig(start) || (n = atoi(start)) == 0 || !DSN_NOTIFY_OK(n)) msg_warn("%s: ignoring malformed DSN notify flags <%.200s>", message->queue_id, start); else dsn_notify = n; continue; } } if (rec_type == REC_TYPE_ORCP) { /* See also above for code clearing orig_rcpt. */ if (orig_rcpt != 0) { msg_warn("%s: ignoring out-of-order original recipient <%.200s>", message->queue_id, orig_rcpt); myfree(orig_rcpt); orig_rcpt = 0; } if (message->rcpt_offset == 0) orig_rcpt = mystrdup(start); continue; } /* * Process non-recipient records. */ if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT) /* We already examined all non-recipient records. */ continue; if (rec_type == REC_TYPE_SIZE) { if (message->data_offset == 0) { if ((count = sscanf(start, "%ld %ld %d %d %ld %d", &message->data_size, &message->data_offset, &message->rcpt_unread, &message->rflags, &message->cont_length, &message->smtputf8)) >= 3) { /* Postfix >= 1.0 (a.k.a. 20010228). */ if (message->data_offset <= 0 || message->data_size <= 0) { msg_warn("%s: invalid size record: %.100s", message->queue_id, start); rec_type = REC_TYPE_ERROR; break; } if (message->rflags & ~QMGR_READ_FLAG_USER) { msg_warn("%s: invalid flags in size record: %.100s", message->queue_id, start); rec_type = REC_TYPE_ERROR; break; } } else if (count == 1) { /* Postfix < 1.0 (a.k.a. 20010228). */ qmgr_message_oldstyle_scan(message); } else { /* Can't happen. */ msg_warn("%s: message rejected: weird size record", message->queue_id); rec_type = REC_TYPE_ERROR; break; } } /* Postfix < 2.4 compatibility. */ if (message->cont_length == 0) { message->cont_length = message->data_size; } else if (message->cont_length < 0) { msg_warn("%s: invalid size record: %.100s", message->queue_id, start); rec_type = REC_TYPE_ERROR; break; } continue; } if (rec_type == REC_TYPE_TIME) { if (message->arrival_time.tv_sec == 0) REC_TYPE_TIME_SCAN(start, message->arrival_time); continue; } if (rec_type == REC_TYPE_CTIME) { if (message->create_time == 0) message->create_time = atol(start); continue; } if (rec_type == REC_TYPE_FILT) { if (message->filter_xport != 0) myfree(message->filter_xport); message->filter_xport = mystrdup(start); continue; } if (rec_type == REC_TYPE_INSP) { if (message->inspect_xport != 0) myfree(message->inspect_xport); message->inspect_xport = mystrdup(start); continue; } if (rec_type == REC_TYPE_RDR) { if (message->redirect_addr != 0) myfree(message->redirect_addr); message->redirect_addr = mystrdup(start); continue; } if (rec_type == REC_TYPE_FROM) { if (message->sender == 0) { message->sender = mystrdup(start); opened(message->queue_id, message->sender, message->cont_length, message->rcpt_unread, "queue %s", message->queue_name); } continue; } if (rec_type == REC_TYPE_DSN_ENVID) { if (message->dsn_envid == 0) message->dsn_envid = mystrdup(start); } if (rec_type == REC_TYPE_DSN_RET) { if (message->dsn_ret == 0) { if (!alldig(start) || (n = atoi(start)) == 0 || !DSN_RET_OK(n)) msg_warn("%s: ignoring malformed DSN RET flags in queue file record:%.100s", message->queue_id, start); else message->dsn_ret = n; } } if (rec_type == REC_TYPE_ATTR) { /* Allow extra segment to override envelope segment info. */ if (strcmp(name, MAIL_ATTR_ENCODING) == 0) { if (message->encoding != 0) myfree(message->encoding); message->encoding = mystrdup(value); } /* * Backwards compatibility. Before Postfix 2.3, the logging * attributes were called client_name, etc. Now they are called * log_client_name. etc., and client_name is used for the actual * client information. To support old queue files we accept both * names for the purpose of logging; the new name overrides the * old one. * * XXX Do not use the "legacy" client_name etc. attribute values for * initializing the logging attributes, when this file already * contains the "modern" log_client_name etc. logging attributes. * Otherwise, logging attributes that are not present in the * queue file would be set with information from the real client. */ else if (strcmp(name, MAIL_ATTR_ACT_CLIENT_NAME) == 0) { if (have_log_client_attr == 0 && message->client_name == 0) message->client_name = mystrdup(value); } else if (strcmp(name, MAIL_ATTR_ACT_CLIENT_ADDR) == 0) { if (have_log_client_attr == 0 && message->client_addr == 0) message->client_addr = mystrdup(value); } else if (strcmp(name, MAIL_ATTR_ACT_CLIENT_PORT) == 0) { if (have_log_client_attr == 0 && message->client_port == 0) message->client_port = mystrdup(value); } else if (strcmp(name, MAIL_ATTR_ACT_PROTO_NAME) == 0) { if (have_log_client_attr == 0 && message->client_proto == 0) message->client_proto = mystrdup(value); } else if (strcmp(name, MAIL_ATTR_ACT_HELO_NAME) == 0) { if (have_log_client_attr == 0 && message->client_helo == 0) message->client_helo = mystrdup(value); } /* Original client attributes. */ else if (strcmp(name, MAIL_ATTR_LOG_CLIENT_NAME) == 0) { if (message->client_name != 0) myfree(message->client_name); message->client_name = mystrdup(value); have_log_client_attr = 1; } else if (strcmp(name, MAIL_ATTR_LOG_CLIENT_ADDR) == 0) { if (message->client_addr != 0) myfree(message->client_addr); message->client_addr = mystrdup(value); have_log_client_attr = 1; } else if (strcmp(name, MAIL_ATTR_LOG_CLIENT_PORT) == 0) { if (message->client_port != 0) myfree(message->client_port); message->client_port = mystrdup(value); have_log_client_attr = 1; } else if (strcmp(name, MAIL_ATTR_LOG_PROTO_NAME) == 0) { if (message->client_proto != 0) myfree(message->client_proto); message->client_proto = mystrdup(value); have_log_client_attr = 1; } else if (strcmp(name, MAIL_ATTR_LOG_HELO_NAME) == 0) { if (message->client_helo != 0) myfree(message->client_helo); message->client_helo = mystrdup(value); have_log_client_attr = 1; } else if (strcmp(name, MAIL_ATTR_SASL_METHOD) == 0) { if (message->sasl_method == 0) message->sasl_method = mystrdup(value); else msg_warn("%s: ignoring multiple %s attribute: %s", message->queue_id, MAIL_ATTR_SASL_METHOD, value); } else if (strcmp(name, MAIL_ATTR_SASL_USERNAME) == 0) { if (message->sasl_username == 0) message->sasl_username = mystrdup(value); else msg_warn("%s: ignoring multiple %s attribute: %s", message->queue_id, MAIL_ATTR_SASL_USERNAME, value); } else if (strcmp(name, MAIL_ATTR_SASL_SENDER) == 0) { if (message->sasl_sender == 0) message->sasl_sender = mystrdup(value); else msg_warn("%s: ignoring multiple %s attribute: %s", message->queue_id, MAIL_ATTR_SASL_SENDER, value); } else if (strcmp(name, MAIL_ATTR_LOG_IDENT) == 0) { if (message->log_ident == 0) message->log_ident = mystrdup(value); else msg_warn("%s: ignoring multiple %s attribute: %s", message->queue_id, MAIL_ATTR_LOG_IDENT, value); } else if (strcmp(name, MAIL_ATTR_RWR_CONTEXT) == 0) { if (message->rewrite_context == 0) message->rewrite_context = mystrdup(value); else msg_warn("%s: ignoring multiple %s attribute: %s", message->queue_id, MAIL_ATTR_RWR_CONTEXT, value); } /* * Optional tracing flags (verify, sendmail -v, sendmail -bv). * This record is killed after a trace logfile report is sent and * after the logfile is deleted. */ else if (strcmp(name, MAIL_ATTR_TRACE_FLAGS) == 0) { message->tflags = DEL_REQ_TRACE_FLAGS(atoi(value)); if (message->tflags == DEL_REQ_FLAG_RECORD) message->tflags_offset = curr_offset; else message->tflags_offset = 0; } continue; } if (rec_type == REC_TYPE_WARN) { if (message->warn_offset == 0) { message->warn_offset = curr_offset; REC_TYPE_WARN_SCAN(start, message->warn_time); } continue; } if (rec_type == REC_TYPE_VERP) { if (message->verp_delims == 0) { if (message->sender == 0 || message->sender[0] == 0) { msg_warn("%s: ignoring VERP request for null sender", message->queue_id); } else if (verp_delims_verify(start) != 0) { msg_warn("%s: ignoring bad VERP request: \"%.100s\"", message->queue_id, start); } else { if (msg_verbose) msg_info("%s: enabling VERP for sender \"%.100s\"", message->queue_id, message->sender); message->single_rcpt = 1; message->verp_delims = mystrdup(start); } } continue; } } /* * Grr. */ if (dsn_orcpt != 0) { if (rec_type > 0) msg_warn("%s: ignoring out-of-order DSN original recipient <%.200s>", message->queue_id, dsn_orcpt); myfree(dsn_orcpt); } if (orig_rcpt != 0) { if (rec_type > 0) msg_warn("%s: ignoring out-of-order original recipient <%.200s>", message->queue_id, orig_rcpt); myfree(orig_rcpt); } /* * After sending a "delayed" warning, request sender notification when * message delivery is completed. While "mail delayed" notifications are * bad enough because they multiply the amount of email traffic, "delay * cleared" notifications are even worse because they come in a sudden * burst when the queue drains after a network outage. */ if (var_dsn_delay_cleared && message->warn_time < 0) message->tflags |= DEL_REQ_FLAG_REC_DLY_SENT; /* * Remember when we have read the last recipient batch. Note that we do * it here after reading as reading might have used considerable amount * of time. */ message->refill_time = sane_time(); /* * Avoid clumsiness elsewhere in the program. When sending data across an * IPC channel, sending an empty string is more convenient than sending a * null pointer. */ if (message->dsn_envid == 0) message->dsn_envid = mystrdup(""); if (message->encoding == 0) message->encoding = mystrdup(MAIL_ATTR_ENC_NONE); if (message->client_name == 0) message->client_name = mystrdup(""); if (message->client_addr == 0) message->client_addr = mystrdup(""); if (message->client_port == 0) message->client_port = mystrdup(""); if (message->client_proto == 0) message->client_proto = mystrdup(""); if (message->client_helo == 0) message->client_helo = mystrdup(""); if (message->sasl_method == 0) message->sasl_method = mystrdup(""); if (message->sasl_username == 0) message->sasl_username = mystrdup(""); if (message->sasl_sender == 0) message->sasl_sender = mystrdup(""); if (message->log_ident == 0) message->log_ident = mystrdup(""); if (message->rewrite_context == 0) message->rewrite_context = mystrdup(MAIL_ATTR_RWR_LOCAL); /* Postfix < 2.3 compatibility. */ if (message->create_time == 0) message->create_time = message->arrival_time.tv_sec; /* * Clean up. */ vstring_free(buf); /* * Sanity checks. Verify that all required information was found, * including the queue file end marker. */ if (message->rcpt_unread < 0 || (message->rcpt_offset == 0 && message->rcpt_unread != 0)) { msg_warn("%s: rcpt count mismatch (%d)", message->queue_id, message->rcpt_unread); message->rcpt_unread = 0; } if (rec_type <= 0) { /* Already logged warning. */ } else if (message->arrival_time.tv_sec == 0) { msg_warn("%s: message rejected: missing arrival time record", message->queue_id); } else if (message->sender == 0) { msg_warn("%s: message rejected: missing sender record", message->queue_id); } else if (message->data_offset == 0) { msg_warn("%s: message rejected: missing size record", message->queue_id); } else { return (0); } message->rcpt_offset = save_offset; /* restore flag */ message->rcpt_unread = save_unread; /* restore count */ recipient_list_free(&message->rcpt_list); recipient_list_init(&message->rcpt_list, RCPT_LIST_INIT_QUEUE); return (-1); }
static QMGR_JOB *qmgr_job_candidate(QMGR_JOB *current) { QMGR_TRANSPORT *transport = current->transport; QMGR_JOB *job, *best_job = 0; double score, best_score = 0.0; int max_slots, max_needed_entries, max_total_entries; int delay; time_t now = sane_time(); /* * Fetch the result directly from the cache if the cache is still valid. * * Note that we cache negative results too, so the cache must be invalidated * by resetting the cached current job pointer, not the candidate pointer * itself. * * In case the cache is valid and contains no candidate, we can ignore the * time change, as it affects only which candidate is the best, not if * one exists. However, this feature requires that we no longer relax the * cache resetting rules, depending on the automatic cache timeout. */ if (transport->candidate_cache_current == current && (transport->candidate_cache_time == now || transport->candidate_cache == 0)) return (transport->candidate_cache); /* * Estimate the minimum amount of delivery slots that can ever be * accumulated for the given job. All jobs that won't fit into these * slots are excluded from the candidate selection. */ max_slots = (MIN_ENTRIES(current) - current->selected_entries + current->slots_available) / transport->slot_cost; /* * Select the candidate with best time_since_queued/total_recipients * score. In addition to jobs which don't meet the max_slots limit, skip * also jobs which don't have any selectable entries at the moment. * * Instead of traversing the whole job list we traverse it just from the * current job forward. This has several advantages. First, we skip some * of the blocker jobs and the current job itself right away. But the * really important advantage is that we are sure that we don't consider * any jobs that are already stack children of the current job. Thanks to * this we can easily include all encountered jobs which are leaf * children of some of the preempting stacks as valid candidates. All we * need to do is to make sure we do not include any of the stack parents. * And, because the leaf children are not ordered by the time since * queued, we have to exclude them from the early loop end test. * * However, don't bother searching if we can't find anything suitable * anyway. */ if (max_slots > 0) { for (job = current->transport_peers.next; job; job = job->transport_peers.next) { if (job->stack_children.next != 0 || IS_BLOCKER(job, transport)) continue; max_total_entries = MAX_ENTRIES(job); max_needed_entries = max_total_entries - job->selected_entries; delay = now - job->message->queued_time + 1; if (max_needed_entries > 0 && max_needed_entries <= max_slots) { score = (double) delay / max_total_entries; if (score > best_score) { best_score = score; best_job = job; } } /* * Stop early if the best score is as good as it can get. */ if (delay <= best_score && job->stack_level == 0) break; } } /* * Cache the result for later use. */ transport->candidate_cache = best_job; transport->candidate_cache_current = current; transport->candidate_cache_time = now; return (best_job); }