static void qmgr_queue_resume(int event, char *context)
{
QMGR_QUEUE *queue = (QMGR_QUEUE *) context;
const char *myname = "qmgr_queue_resume";
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_SUSPENDED(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
/*
* We can't simply force delivery on this queue: the transport's pending
* count may already be maxed out, and there may be other constraints
* that definitely should be none of our business. The best we can do is
* to play by the same rules as everyone else: let qmgr_active_drain()
* and round-robin selection take care of message selection.
*/
queue->window = 1;
/*
* Every event handler that leaves a queue in the "ready" state should
* remove the queue when it is empty.
*/
if (QMGR_QUEUE_READY(queue) && queue->todo.next == 0 && queue->busy.next == 0)
qmgr_queue_done(queue);
}
void qmgr_queue_done(QMGR_QUEUE *queue)
{
const char *myname = "qmgr_queue_done";
QMGR_TRANSPORT *transport = queue->transport;
/*
* Sanity checks. It is an error to delete an in-core queue with pending
* messages or timers.
*/
if (queue->busy_refcount != 0 || queue->todo_refcount != 0)
msg_panic("%s: refcount: %d", myname,
queue->busy_refcount + queue->todo_refcount);
if (queue->todo.next || queue->busy.next)
msg_panic("%s: queue not empty: %s", myname, queue->name);
if (!QMGR_QUEUE_READY(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
if (queue->dsn)
msg_panic("%s: queue %s: spurious reason %s",
myname, queue->name, queue->dsn->reason);
/*
* Clean up this in-core queue.
*/
QMGR_LIST_UNLINK(transport->queue_list, QMGR_QUEUE *, queue, peers);
htable_delete(transport->queue_byname, queue->name, (void (*) (char *)) 0);
myfree(queue->name);
myfree(queue->nexthop);
qmgr_queue_count--;
myfree((char *) queue);
}
static void qmgr_queue_unthrottle_wrapper(int unused_event, char *context)
{
QMGR_QUEUE *queue = (QMGR_QUEUE *) context;
/*
* This routine runs when a wakeup timer goes off; it does not run in the
* context of some queue manipulation. Therefore, it is safe to discard
* this in-core queue when it is empty and when this site is not dead.
*/
qmgr_queue_unthrottle(queue);
if (QMGR_QUEUE_READY(queue) && queue->todo.next == 0 && queue->busy.next == 0)
qmgr_queue_done(queue);
}
void qmgr_queue_suspend(QMGR_QUEUE *queue, int delay)
{
const char *myname = "qmgr_queue_suspend";
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_READY(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
if (queue->busy_refcount > 0)
msg_panic("%s: queue is busy", myname);
/*
* Set the queue status to "suspended". No-one is supposed to remove a
* queue in suspended state.
*/
queue->window = QMGR_QUEUE_STAT_SUSPENDED;
event_request_timer(qmgr_queue_resume, (char *) queue, delay);
}
static void qmgr_queue_resume(int event, char *context)
{
QMGR_QUEUE *queue = (QMGR_QUEUE *) context;
const char *myname = "qmgr_queue_resume";
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_SUSPENDED(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
/*
* We can't simply force delivery on this queue: the transport's pending
* count may already be maxed out, and there may be other constraints
* that definitely should be none of our business. The best we can do is
* to play by the same rules as everyone else: let qmgr_active_drain()
* and round-robin selection take care of message selection.
*/
queue->window = 1;
/*
* Every event handler that leaves a queue in the "ready" state should
* remove the queue when it is empty.
*
* XXX Do not omit the redundant test below. It is here to simplify code
* consistency checks. The check is trivially eliminated by the compiler
* optimizer. There is no need to sacrifice code clarity for the sake of
* performance.
*
* XXX Do not expose the blocker job logic here. Rate-limited queues are not
* a performance-critical feature. Here, too, there is no need to sacrifice
* code clarity for the sake of performance.
*/
if (QMGR_QUEUE_READY(queue) && queue->todo.next == 0 && queue->busy.next == 0)
qmgr_queue_done(queue);
else
qmgr_job_blocker_update(queue);
}
QMGR_TRANSPORT *qmgr_transport_select(void)
{
QMGR_TRANSPORT *xport;
QMGR_QUEUE *queue;
int need;
/*
* If we find a suitable transport, rotate the list of transports to
* effectuate round-robin selection. See similar selection code in
* qmgr_peer_select().
*
* This function is called repeatedly until all transports have maxed out
* the number of pending delivery agent connections, until all delivery
* agent concurrency windows are maxed out, or until we run out of "todo"
* queue entries.
*/
#define MIN5af51743e4eef(x, y) ((x) < (y) ? (x) : (y))
for (xport = qmgr_transport_list.next; xport; xport = xport->peers.next) {
if ((xport->flags & QMGR_TRANSPORT_STAT_DEAD) != 0
|| (xport->flags & QMGR_TRANSPORT_STAT_RATE_LOCK) != 0
|| xport->pending >= QMGR_TRANSPORT_MAX_PEND)
continue;
need = xport->pending + 1;
for (queue = xport->queue_list.next; queue; queue = queue->peers.next) {
if (QMGR_QUEUE_READY(queue) == 0)
continue;
if ((need -= MIN5af51743e4eef(queue->window - queue->busy_refcount,
queue->todo_refcount)) <= 0) {
QMGR_LIST_ROTATE(qmgr_transport_list, xport, peers);
if (msg_verbose)
msg_info("qmgr_transport_select: %s", xport->name);
return (xport);
}
}
}
return (0);
}
void qmgr_queue_throttle(QMGR_QUEUE *queue, DSN *dsn)
{
const char *myname = "qmgr_queue_throttle";
QMGR_TRANSPORT *transport = queue->transport;
double feedback;
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_READY(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
if (queue->dsn)
msg_panic("%s: queue %s: spurious reason %s",
myname, queue->name, queue->dsn->reason);
if (msg_verbose)
msg_info("%s: queue %s: %s %s",
myname, queue->name, dsn->status, dsn->reason);
/*
* Don't restart the positive feedback hysteresis cycle with every
* negative feedback. Restart it only when we make a negative concurrency
* adjustment (i.e. at the start of a negative feedback hysteresis
* cycle). Otherwise positive feedback would be too weak (positive
* feedback does not take effect until the end of its hysteresis cycle).
*/
/*
* This queue is declared dead after a configurable number of
* pseudo-cohort failures.
*/
if (QMGR_QUEUE_READY(queue)) {
queue->fail_cohorts += 1.0 / queue->window;
if (transport->fail_cohort_limit > 0
&& queue->fail_cohorts >= transport->fail_cohort_limit)
queue->window = QMGR_QUEUE_STAT_THROTTLED;
}
/*
* Decrease the destination's concurrency limit until we reach 1. Base
* adjustments on the concurrency limit itself, instead of using the
* actual concurrency. The latter fluctuates wildly when deliveries
* complete in bursts (artificial benchmark measurements).
*
* Even after reaching 1, we maintain the negative hysteresis cycle so that
* negative feedback can cancel out positive feedback.
*/
if (QMGR_QUEUE_READY(queue)) {
feedback = QMGR_FEEDBACK_VAL(transport->neg_feedback, queue->window);
QMGR_LOG_FEEDBACK(feedback);
queue->failure -= feedback;
/* Prepare for overshoot (feedback > hysteresis, rounding error). */
while (queue->failure - feedback / 2 < 0) {
queue->window -= transport->neg_feedback.hysteresis;
queue->success = 0;
queue->failure += transport->neg_feedback.hysteresis;
}
/* Prepare for overshoot. */
if (queue->window < 1)
queue->window = 1;
}
/*
* Special case for a site that just was declared dead.
*/
if (QMGR_QUEUE_THROTTLED(queue)) {
queue->dsn = DSN_COPY(dsn);
event_request_timer(qmgr_queue_unthrottle_wrapper,
(char *) queue, var_min_backoff_time);
queue->dflags = 0;
}
QMGR_LOG_WINDOW(queue);
}
void qmgr_queue_unthrottle(QMGR_QUEUE *queue)
{
const char *myname = "qmgr_queue_unthrottle";
QMGR_TRANSPORT *transport = queue->transport;
double feedback;
if (msg_verbose)
msg_info("%s: queue %s", myname, queue->name);
/*
* Sanity checks.
*/
if (!QMGR_QUEUE_READY(queue) && !QMGR_QUEUE_THROTTLED(queue))
msg_panic("%s: bad queue status: %s", myname, QMGR_QUEUE_STATUS(queue));
/*
* Don't restart the negative feedback hysteresis cycle with every
* positive feedback. Restart it only when we make a positive concurrency
* adjustment (i.e. at the end of a positive feedback hysteresis cycle).
* Otherwise negative feedback would be too aggressive: negative feedback
* takes effect immediately at the start of its hysteresis cycle.
*/
queue->fail_cohorts = 0;
/*
* Special case when this site was dead.
*/
if (QMGR_QUEUE_THROTTLED(queue)) {
event_cancel_timer(qmgr_queue_unthrottle_wrapper, (char *) queue);
if (queue->dsn == 0)
msg_panic("%s: queue %s: window 0 status 0", myname, queue->name);
dsn_free(queue->dsn);
queue->dsn = 0;
/* Back from the almost grave, best concurrency is anyone's guess. */
if (queue->busy_refcount > 0)
queue->window = queue->busy_refcount;
else
queue->window = transport->init_dest_concurrency;
queue->success = queue->failure = 0;
QMGR_LOG_WINDOW(queue);
return;
}
/*
* Increase the destination's concurrency limit until we reach the
* transport's concurrency limit. Allow for a margin the size of the
* initial destination concurrency, so that we're not too gentle.
*
* Why is the concurrency increment based on preferred concurrency and not
* on the number of outstanding delivery requests? The latter fluctuates
* wildly when deliveries complete in bursts (artificial benchmark
* measurements), and does not account for cached connections.
*
* Keep the window within reasonable distance from actual concurrency
* otherwise negative feedback will be ineffective. This expression
* assumes that busy_refcount changes gradually. This is invalid when
* deliveries complete in bursts (artificial benchmark measurements).
*/
if (transport->dest_concurrency_limit == 0
|| transport->dest_concurrency_limit > queue->window)
if (queue->window < queue->busy_refcount + transport->init_dest_concurrency) {
feedback = QMGR_FEEDBACK_VAL(transport->pos_feedback, queue->window);
QMGR_LOG_FEEDBACK(feedback);
queue->success += feedback;
/* Prepare for overshoot (feedback > hysteresis, rounding error). */
while (queue->success + feedback / 2 >= transport->pos_feedback.hysteresis) {
queue->window += transport->pos_feedback.hysteresis;
queue->success -= transport->pos_feedback.hysteresis;
queue->failure = 0;
}
/* Prepare for overshoot. */
if (transport->dest_concurrency_limit > 0
&& queue->window > transport->dest_concurrency_limit)
queue->window = transport->dest_concurrency_limit;
}
QMGR_LOG_WINDOW(queue);
}
static void qmgr_deliver_update(int unused_event, char *context)
{
QMGR_ENTRY *entry = (QMGR_ENTRY *) context;
QMGR_QUEUE *queue = entry->queue;
QMGR_TRANSPORT *transport = queue->transport;
QMGR_MESSAGE *message = entry->message;
static DSN_BUF *dsb;
int status;
/*
* Release the delivery agent from a "hot" queue entry.
*/
#define QMGR_DELIVER_RELEASE_AGENT(entry) do { \
event_disable_readwrite(vstream_fileno(entry->stream)); \
(void) vstream_fclose(entry->stream); \
entry->stream = 0; \
qmgr_deliver_concurrency--; \
} while (0)
if (dsb == 0)
dsb = dsb_create();
/*
* The message transport has responded. Stop the watchdog timer.
*/
event_cancel_timer(qmgr_deliver_abort, context);
/*
* Retrieve the delivery agent status report. The numerical status code
* indicates if delivery should be tried again. The reason text is sent
* only when a site should be avoided for a while, so that the queue
* manager can log why it does not even try to schedule delivery to the
* affected recipients.
*/
status = qmgr_deliver_final_reply(entry->stream, dsb);
/*
* The mail delivery process failed for some reason (although delivery
* may have been successful). Back off with this transport type for a
* while. Dispose of queue entries for this transport that await
* selection (the todo lists). Stay away from queue entries that have
* been selected (the busy lists), or we would have dangling pointers.
* The queue itself won't go away before we dispose of the current queue
* entry.
*/
if (status == DELIVER_STAT_CRASH) {
message->flags |= DELIVER_STAT_DEFER;
#if 0
whatsup = concatenate("unknown ", transport->name,
" mail transport error", (char *) 0);
qmgr_transport_throttle(transport,
DSN_SIMPLE(&dsb->dsn, "4.3.0", whatsup));
myfree(whatsup);
#else
qmgr_transport_throttle(transport,
DSN_SIMPLE(&dsb->dsn, "4.3.0",
"unknown mail transport error"));
#endif
msg_warn("transport %s failure -- see a previous warning/fatal/panic logfile record for the problem description",
transport->name);
/*
* Assume the worst and write a defer logfile record for each
* recipient. This omission was already present in the first queue
* manager implementation of 199703, and was fixed 200511.
*
* To avoid the synchronous qmgr_defer_recipient() operation for each
* recipient of this queue entry, release the delivery process and
* move the entry back to the todo queue. Let qmgr_defer_transport()
* log the recipient asynchronously if possible, and get out of here.
* Note: if asynchronous logging is not possible,
* qmgr_defer_transport() eventually invokes qmgr_entry_done() and
* the entry becomes a dangling pointer.
*/
QMGR_DELIVER_RELEASE_AGENT(entry);
qmgr_entry_unselect(queue, entry);
qmgr_defer_transport(transport, &dsb->dsn);
return;
}
/*
* This message must be tried again.
*
* If we have a problem talking to this site, back off with this site for a
* while; dispose of queue entries for this site that await selection
* (the todo list); stay away from queue entries that have been selected
* (the busy list), or we would have dangling pointers. The queue itself
* won't go away before we dispose of the current queue entry.
*
* XXX Caution: DSN_COPY() will panic on empty status or reason.
*/
#define SUSPENDED "delivery temporarily suspended: "
if (status == DELIVER_STAT_DEFER) {
message->flags |= DELIVER_STAT_DEFER;
if (VSTRING_LEN(dsb->status)) {
/* Sanitize the DSN status/reason from the delivery agent. */
if (!dsn_valid(vstring_str(dsb->status)))
vstring_strcpy(dsb->status, "4.0.0");
if (VSTRING_LEN(dsb->reason) == 0)
vstring_strcpy(dsb->reason, "unknown error");
vstring_prepend(dsb->reason, SUSPENDED, sizeof(SUSPENDED) - 1);
if (QMGR_QUEUE_READY(queue)) {
qmgr_queue_throttle(queue, DSN_FROM_DSN_BUF(dsb));
if (QMGR_QUEUE_THROTTLED(queue))
qmgr_defer_todo(queue, &dsb->dsn);
}
}
}
/*
* No problems detected. Mark the transport and queue as alive. The queue
* itself won't go away before we dispose of the current queue entry.
*/
if (status != DELIVER_STAT_CRASH && VSTRING_LEN(dsb->reason) == 0) {
qmgr_transport_unthrottle(transport);
qmgr_queue_unthrottle(queue);
}
/*
* Release the delivery process, and give some other queue entry a chance
* to be delivered. When all recipients for a message have been tried,
* decide what to do next with this message: defer, bounce, delete.
*/
QMGR_DELIVER_RELEASE_AGENT(entry);
qmgr_entry_done(entry, QMGR_QUEUE_BUSY);
}
