QMGR_QUEUE *qmgr_error_queue(const char *service, DSN *dsn)
{
QMGR_TRANSPORT *transport;
QMGR_QUEUE *queue;
char *nexthop;
/*
* Find or create transport.
*/
if ((transport = qmgr_error_transport(service)) == 0)
return (0);
/*
* Find or create queue.
*/
nexthop = qmgr_error_nexthop(dsn);
if ((queue = qmgr_queue_find(transport, nexthop)) == 0)
queue = qmgr_queue_create(transport, nexthop, nexthop);
myfree(nexthop);
if (QMGR_QUEUE_THROTTLED(queue))
return (0);
/*
* Done.
*/
return (queue);
}
void qmgr_queue_throttle(QMGR_QUEUE *queue, DSN *dsn)
{
const char *myname = "qmgr_queue_throttle";
QMGR_TRANSPORT *transport = queue->transport;
double feedback;
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_READY(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
if (queue->dsn)
msg_panic("%s: queue %s: spurious reason %s",
myname, queue->name, queue->dsn->reason);
if (msg_verbose)
msg_info("%s: queue %s: %s %s",
myname, queue->name, dsn->status, dsn->reason);
/*
* Don't restart the positive feedback hysteresis cycle with every
* negative feedback. Restart it only when we make a negative concurrency
* adjustment (i.e. at the start of a negative feedback hysteresis
* cycle). Otherwise positive feedback would be too weak (positive
* feedback does not take effect until the end of its hysteresis cycle).
*/
/*
* This queue is declared dead after a configurable number of
* pseudo-cohort failures.
*/
if (QMGR_QUEUE_READY(queue)) {
queue->fail_cohorts += 1.0 / queue->window;
if (transport->fail_cohort_limit > 0
&& queue->fail_cohorts >= transport->fail_cohort_limit)
queue->window = QMGR_QUEUE_STAT_THROTTLED;
}
/*
* Decrease the destination's concurrency limit until we reach 1. Base
* adjustments on the concurrency limit itself, instead of using the
* actual concurrency. The latter fluctuates wildly when deliveries
* complete in bursts (artificial benchmark measurements).
*
* Even after reaching 1, we maintain the negative hysteresis cycle so that
* negative feedback can cancel out positive feedback.
*/
if (QMGR_QUEUE_READY(queue)) {
feedback = QMGR_FEEDBACK_VAL(transport->neg_feedback, queue->window);
QMGR_LOG_FEEDBACK(feedback);
queue->failure -= feedback;
/* Prepare for overshoot (feedback > hysteresis, rounding error). */
while (queue->failure - feedback / 2 < 0) {
queue->window -= transport->neg_feedback.hysteresis;
queue->success = 0;
queue->failure += transport->neg_feedback.hysteresis;
}
/* Prepare for overshoot. */
if (queue->window < 1)
queue->window = 1;
}
/*
* Special case for a site that just was declared dead.
*/
if (QMGR_QUEUE_THROTTLED(queue)) {
queue->dsn = DSN_COPY(dsn);
event_request_timer(qmgr_queue_unthrottle_wrapper,
(char *) queue, var_min_backoff_time);
queue->dflags = 0;
}
QMGR_LOG_WINDOW(queue);
}
void qmgr_queue_unthrottle(QMGR_QUEUE *queue)
{
const char *myname = "qmgr_queue_unthrottle";
QMGR_TRANSPORT *transport = queue->transport;
double feedback;
if (msg_verbose)
msg_info("%s: queue %s", myname, queue->name);
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_READY(queue) && !QMGR_QUEUE_THROTTLED(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
/*
* Don't restart the negative feedback hysteresis cycle with every
* positive feedback. Restart it only when we make a positive concurrency
* adjustment (i.e. at the end of a positive feedback hysteresis cycle).
* Otherwise negative feedback would be too aggressive: negative feedback
* takes effect immediately at the start of its hysteresis cycle.
*/
queue->fail_cohorts = 0;
/*
* Special case when this site was dead.
*/
if (QMGR_QUEUE_THROTTLED(queue)) {
event_cancel_timer(qmgr_queue_unthrottle_wrapper, (char *) queue);
if (queue->dsn == 0)
msg_panic("%s: queue %s: window 0 status 0", myname, queue->name);
dsn_free(queue->dsn);
queue->dsn = 0;
/* Back from the almost grave, best concurrency is anyone's guess. */
if (queue->busy_refcount > 0)
queue->window = queue->busy_refcount;
else
queue->window = transport->init_dest_concurrency;
queue->success = queue->failure = 0;
QMGR_LOG_WINDOW(queue);
return;
}
/*
* Increase the destination's concurrency limit until we reach the
* transport's concurrency limit. Allow for a margin the size of the
* initial destination concurrency, so that we're not too gentle.
*
* Why is the concurrency increment based on preferred concurrency and not
* on the number of outstanding delivery requests? The latter fluctuates
* wildly when deliveries complete in bursts (artificial benchmark
* measurements), and does not account for cached connections.
*
* Keep the window within reasonable distance from actual concurrency
* otherwise negative feedback will be ineffective. This expression
* assumes that busy_refcount changes gradually. This is invalid when
* deliveries complete in bursts (artificial benchmark measurements).
*/
if (transport->dest_concurrency_limit == 0
|| transport->dest_concurrency_limit > queue->window)
if (queue->window < queue->busy_refcount + transport->init_dest_concurrency) {
feedback = QMGR_FEEDBACK_VAL(transport->pos_feedback, queue->window);
QMGR_LOG_FEEDBACK(feedback);
queue->success += feedback;
/* Prepare for overshoot (feedback > hysteresis, rounding error). */
while (queue->success + feedback / 2 >= transport->pos_feedback.hysteresis) {
queue->window += transport->pos_feedback.hysteresis;
queue->success -= transport->pos_feedback.hysteresis;
queue->failure = 0;
}
/* Prepare for overshoot. */
if (transport->dest_concurrency_limit > 0
&& queue->window > transport->dest_concurrency_limit)
queue->window = transport->dest_concurrency_limit;
}
QMGR_LOG_WINDOW(queue);
}
static void qmgr_deliver_update(int unused_event, char *context)
{
QMGR_ENTRY *entry = (QMGR_ENTRY *) context;
QMGR_QUEUE *queue = entry->queue;
QMGR_TRANSPORT *transport = queue->transport;
QMGR_MESSAGE *message = entry->message;
static DSN_BUF *dsb;
int status;
/*
* Release the delivery agent from a "hot" queue entry.
*/
#define QMGR_DELIVER_RELEASE_AGENT(entry) do { \
event_disable_readwrite(vstream_fileno(entry->stream)); \
(void) vstream_fclose(entry->stream); \
entry->stream = 0; \
qmgr_deliver_concurrency--; \
} while (0)
if (dsb == 0)
dsb = dsb_create();
/*
* The message transport has responded. Stop the watchdog timer.
*/
event_cancel_timer(qmgr_deliver_abort, context);
/*
* Retrieve the delivery agent status report. The numerical status code
* indicates if delivery should be tried again. The reason text is sent
* only when a site should be avoided for a while, so that the queue
* manager can log why it does not even try to schedule delivery to the
* affected recipients.
*/
status = qmgr_deliver_final_reply(entry->stream, dsb);
/*
* The mail delivery process failed for some reason (although delivery
* may have been successful). Back off with this transport type for a
* while. Dispose of queue entries for this transport that await
* selection (the todo lists). Stay away from queue entries that have
* been selected (the busy lists), or we would have dangling pointers.
* The queue itself won't go away before we dispose of the current queue
* entry.
*/
if (status == DELIVER_STAT_CRASH) {
message->flags |= DELIVER_STAT_DEFER;
#if 0
whatsup = concatenate("unknown ", transport->name,
" mail transport error", (char *) 0);
qmgr_transport_throttle(transport,
DSN_SIMPLE(&dsb->dsn, "4.3.0", whatsup));
myfree(whatsup);
#else
qmgr_transport_throttle(transport,
DSN_SIMPLE(&dsb->dsn, "4.3.0",
"unknown mail transport error"));
#endif
msg_warn("transport %s failure -- see a previous warning/fatal/panic logfile record for the problem description",
transport->name);
/*
* Assume the worst and write a defer logfile record for each
* recipient. This omission was already present in the first queue
* manager implementation of 199703, and was fixed 200511.
*
* To avoid the synchronous qmgr_defer_recipient() operation for each
* recipient of this queue entry, release the delivery process and
* move the entry back to the todo queue. Let qmgr_defer_transport()
* log the recipient asynchronously if possible, and get out of here.
* Note: if asynchronous logging is not possible,
* qmgr_defer_transport() eventually invokes qmgr_entry_done() and
* the entry becomes a dangling pointer.
*/
QMGR_DELIVER_RELEASE_AGENT(entry);
qmgr_entry_unselect(queue, entry);
qmgr_defer_transport(transport, &dsb->dsn);
return;
}
/*
* This message must be tried again.
*
* If we have a problem talking to this site, back off with this site for a
* while; dispose of queue entries for this site that await selection
* (the todo list); stay away from queue entries that have been selected
* (the busy list), or we would have dangling pointers. The queue itself
* won't go away before we dispose of the current queue entry.
*
* XXX Caution: DSN_COPY() will panic on empty status or reason.
*/
#define SUSPENDED "delivery temporarily suspended: "
if (status == DELIVER_STAT_DEFER) {
message->flags |= DELIVER_STAT_DEFER;
if (VSTRING_LEN(dsb->status)) {
/* Sanitize the DSN status/reason from the delivery agent. */
if (!dsn_valid(vstring_str(dsb->status)))
vstring_strcpy(dsb->status, "4.0.0");
if (VSTRING_LEN(dsb->reason) == 0)
vstring_strcpy(dsb->reason, "unknown error");
vstring_prepend(dsb->reason, SUSPENDED, sizeof(SUSPENDED) - 1);
if (QMGR_QUEUE_READY(queue)) {
qmgr_queue_throttle(queue, DSN_FROM_DSN_BUF(dsb));
if (QMGR_QUEUE_THROTTLED(queue))
qmgr_defer_todo(queue, &dsb->dsn);
}
}
}
/*
* No problems detected. Mark the transport and queue as alive. The queue
* itself won't go away before we dispose of the current queue entry.
*/
if (status != DELIVER_STAT_CRASH && VSTRING_LEN(dsb->reason) == 0) {
qmgr_transport_unthrottle(transport);
qmgr_queue_unthrottle(queue);
}
/*
* Release the delivery process, and give some other queue entry a chance
* to be delivered. When all recipients for a message have been tried,
* decide what to do next with this message: defer, bounce, delete.
*/
QMGR_DELIVER_RELEASE_AGENT(entry);
qmgr_entry_done(entry, QMGR_QUEUE_BUSY);
}
static void qmgr_message_resolve(QMGR_MESSAGE *message)
{
static ARGV *defer_xport_argv;
RECIPIENT_LIST list = message->rcpt_list;
RECIPIENT *recipient;
QMGR_TRANSPORT *transport = 0;
QMGR_QUEUE *queue = 0;
RESOLVE_REPLY reply;
VSTRING *queue_name;
char *at;
char **cpp;
char *nexthop;
ssize_t len;
int status;
DSN dsn;
MSG_STATS stats;
DSN *saved_dsn;
#define STREQ(x,y) (strcmp(x,y) == 0)
#define STR vstring_str
#define LEN VSTRING_LEN
resolve_clnt_init(&reply);
queue_name = vstring_alloc(1);
for (recipient = list.info; recipient < list.info + list.len; recipient++) {
/*
* Redirect overrides all else. But only once (per entire message).
* For consistency with the remainder of Postfix, rewrite the address
* to canonical form before resolving it.
*/
if (message->redirect_addr) {
if (recipient > list.info) {
recipient->u.queue = 0;
continue;
}
message->rcpt_offset = 0;
message->rcpt_unread = 0;
rewrite_clnt_internal(REWRITE_CANON, message->redirect_addr,
reply.recipient);
RECIPIENT_UPDATE(recipient->address, STR(reply.recipient));
if (qmgr_resolve_one(message, recipient,
recipient->address, &reply) < 0)
continue;
if (!STREQ(recipient->address, STR(reply.recipient)))
RECIPIENT_UPDATE(recipient->address, STR(reply.recipient));
}
/*
* Content filtering overrides the address resolver.
*
* XXX Bypass content_filter inspection for user-generated probes
* (sendmail -bv). MTA-generated probes never have the "please filter
* me" bits turned on, but we handle them here anyway for the sake of
* future proofing.
*/
#define FILTER_WITHOUT_NEXTHOP(filter, next) \
(((next) = split_at((filter), ':')) == 0 || *(next) == 0)
#define RCPT_WITHOUT_DOMAIN(rcpt, next) \
((next = strrchr(rcpt, '@')) == 0 || *++(next) == 0)
else if (message->filter_xport
&& (message->tflags & DEL_REQ_TRACE_ONLY_MASK) == 0) {
reply.flags = 0;
vstring_strcpy(reply.transport, message->filter_xport);
if (FILTER_WITHOUT_NEXTHOP(STR(reply.transport), nexthop)
&& *(nexthop = var_def_filter_nexthop) == 0
&& RCPT_WITHOUT_DOMAIN(recipient->address, nexthop))
nexthop = var_myhostname;
vstring_strcpy(reply.nexthop, nexthop);
vstring_strcpy(reply.recipient, recipient->address);
}
/*
* Resolve the destination to (transport, nexthop, address). The
* result address may differ from the one specified by the sender.
*/
else {
if (qmgr_resolve_one(message, recipient,
recipient->address, &reply) < 0)
continue;
if (!STREQ(recipient->address, STR(reply.recipient)))
RECIPIENT_UPDATE(recipient->address, STR(reply.recipient));
}
/*
* Bounce null recipients. This should never happen, but is most
* likely the result of a fault in a different program, so aborting
* the queue manager process does not help.
*/
if (recipient->address[0] == 0) {
QMGR_REDIRECT(&reply, MAIL_SERVICE_ERROR,
"5.1.3 null recipient address");
}
/*
* Discard mail to the local double bounce address here, so this
* system can run without a local delivery agent. They'd still have
* to configure something for mail directed to the local postmaster,
* though, but that is an RFC requirement anyway.
*
* XXX This lookup should be done in the resolver, and the mail should
* be directed to a general-purpose null delivery agent.
*/
if (reply.flags & RESOLVE_CLASS_LOCAL) {
at = strrchr(STR(reply.recipient), '@');
len = (at ? (at - STR(reply.recipient))
: strlen(STR(reply.recipient)));
if (strncasecmp(STR(reply.recipient), var_double_bounce_sender,
len) == 0
&& !var_double_bounce_sender[len]) {
status = sent(message->tflags, message->queue_id,
QMGR_MSG_STATS(&stats, message), recipient,
"none", DSN_SIMPLE(&dsn, "2.0.0",
"undeliverable postmaster notification discarded"));
if (status == 0) {
deliver_completed(message->fp, recipient->offset);
#if 0
/* It's the default verification probe sender address. */
msg_warn("%s: undeliverable postmaster notification discarded",
message->queue_id);
#endif
} else
message->flags |= status;
continue;
}
}
/*
* Optionally defer deliveries over specific transports, unless the
* restriction is lifted temporarily.
*/
if (*var_defer_xports && (message->qflags & QMGR_FLUSH_DFXP) == 0) {
if (defer_xport_argv == 0)
defer_xport_argv = argv_split(var_defer_xports, CHARS_COMMA_SP);
for (cpp = defer_xport_argv->argv; *cpp; cpp++)
if (strcmp(*cpp, STR(reply.transport)) == 0)
break;
if (*cpp) {
QMGR_REDIRECT(&reply, MAIL_SERVICE_RETRY,
"4.3.2 deferred transport");
}
}
/*
* Look up or instantiate the proper transport.
*/
if (transport == 0 || !STREQ(transport->name, STR(reply.transport))) {
if ((transport = qmgr_transport_find(STR(reply.transport))) == 0)
transport = qmgr_transport_create(STR(reply.transport));
queue = 0;
}
/*
* This message is being flushed. If need-be unthrottle the
* transport.
*/
if ((message->qflags & QMGR_FLUSH_EACH) != 0
&& QMGR_TRANSPORT_THROTTLED(transport))
qmgr_transport_unthrottle(transport);
/*
* This transport is dead. Defer delivery to this recipient.
*/
if (QMGR_TRANSPORT_THROTTLED(transport)) {
saved_dsn = transport->dsn;
if ((transport = qmgr_error_transport(MAIL_SERVICE_RETRY)) != 0) {
nexthop = qmgr_error_nexthop(saved_dsn);
vstring_strcpy(reply.nexthop, nexthop);
myfree(nexthop);
queue = 0;
} else {
qmgr_defer_recipient(message, recipient, saved_dsn);
continue;
}
}
/*
* The nexthop destination provides the default name for the
* per-destination queue. When the delivery agent accepts only one
* recipient per delivery, give each recipient its own queue, so that
* deliveries to different recipients of the same message can happen
* in parallel, and so that we can enforce per-recipient concurrency
* limits and prevent one recipient from tying up all the delivery
* agent resources. We use recipient@nexthop as queue name rather
* than the actual recipient domain name, so that one recipient in
* multiple equivalent domains cannot evade the per-recipient
* concurrency limit. Split the address on the recipient delimiter if
* one is defined, so that extended addresses don't get extra
* delivery slots.
*
* Fold the result to lower case so that we don't have multiple queues
* for the same name.
*
* Important! All recipients in a queue must have the same nexthop
* value. It is OK to have multiple queues with the same nexthop
* value, but only when those queues are named after recipients.
*
* The single-recipient code below was written for local(8) like
* delivery agents, and assumes that all domains that deliver to the
* same (transport + nexthop) are aliases for $nexthop. Delivery
* concurrency is changed from per-domain into per-recipient, by
* changing the queue name from nexthop into localpart@nexthop.
*
* XXX This assumption is incorrect when different destinations share
* the same (transport + nexthop). In reality, such transports are
* rarely configured to use single-recipient deliveries. The fix is
* to decouple the per-destination recipient limit from the
* per-destination concurrency.
*/
vstring_strcpy(queue_name, STR(reply.nexthop));
if (strcmp(transport->name, MAIL_SERVICE_ERROR) != 0
&& strcmp(transport->name, MAIL_SERVICE_RETRY) != 0
&& transport->recipient_limit == 1) {
/* Copy the recipient localpart. */
at = strrchr(STR(reply.recipient), '@');
len = (at ? (at - STR(reply.recipient))
: strlen(STR(reply.recipient)));
vstring_strncpy(queue_name, STR(reply.recipient), len);
/* Remove the address extension from the recipient localpart. */
if (*var_rcpt_delim && split_addr(STR(queue_name), var_rcpt_delim))
vstring_truncate(queue_name, strlen(STR(queue_name)));
/* Assume the recipient domain is equivalent to nexthop. */
vstring_sprintf_append(queue_name, "@%s", STR(reply.nexthop));
}
lowercase(STR(queue_name));
/*
* This transport is alive. Find or instantiate a queue for this
* recipient.
*/
if (queue == 0 || !STREQ(queue->name, STR(queue_name))) {
if ((queue = qmgr_queue_find(transport, STR(queue_name))) == 0)
queue = qmgr_queue_create(transport, STR(queue_name),
STR(reply.nexthop));
}
/*
* This message is being flushed. If need-be unthrottle the queue.
*/
if ((message->qflags & QMGR_FLUSH_EACH) != 0
&& QMGR_QUEUE_THROTTLED(queue))
qmgr_queue_unthrottle(queue);
/*
* This queue is dead. Defer delivery to this recipient.
*/
if (QMGR_QUEUE_THROTTLED(queue)) {
saved_dsn = queue->dsn;
if ((queue = qmgr_error_queue(MAIL_SERVICE_RETRY, saved_dsn)) == 0) {
qmgr_defer_recipient(message, recipient, saved_dsn);
continue;
}
}
/*
* This queue is alive. Bind this recipient to this queue instance.
*/
recipient->u.queue = queue;
}
resolve_clnt_free(&reply);
vstring_free(queue_name);
}