/*
* Callback function for the cases kssl_input() had to submit an asynchronous
* job and need to come back when done to carry on the input processing.
* This routine follows the conventions of timeout and interrupt handlers.
* (no blocking, ...)
*/
static void
tcp_kssl_input_callback(void *arg, mblk_t *mp, kssl_cmd_t kssl_cmd)
{
tcp_t *tcp = (tcp_t *)arg;
conn_t *connp;
mblk_t *sqmp;
ASSERT(tcp != NULL);
connp = tcp->tcp_connp;
ASSERT(connp != NULL);
switch (kssl_cmd) {
case KSSL_CMD_SEND:
/* I'm coming from an outside perimeter */
if (mp != NULL) {
/*
* See comment in tcp_kssl_input() call to tcp_output()
*/
mutex_enter(&tcp->tcp_non_sq_lock);
tcp->tcp_squeue_bytes += msgdsize(mp);
mutex_exit(&tcp->tcp_non_sq_lock);
}
CONN_INC_REF(connp);
SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
/* FALLTHROUGH */
case KSSL_CMD_NONE:
break;
case KSSL_CMD_DELIVER_PROXY:
case KSSL_CMD_DELIVER_SSL:
/*
* Keep accumulating if not yet accepted.
*/
if (tcp->tcp_listener != NULL) {
tcp_rcv_enqueue(tcp, mp, msgdsize(mp), NULL);
} else {
putnext(connp->conn_rq, mp);
}
break;
case KSSL_CMD_NOT_SUPPORTED:
/* Stop the SSL processing */
kssl_release_ctx(tcp->tcp_kssl_ctx);
tcp->tcp_kssl_ctx = NULL;
}
/*
* Process any input that may have accumulated while we're waiting for
* the call-back.
* We need to re-enter the squeue for this connp, and a new mp is
* necessary.
*/
if ((sqmp = allocb(1, BPRI_MED)) != NULL) {
CONN_INC_REF(connp);
SQUEUE_ENTER_ONE(connp->conn_sqp, sqmp, tcp_kssl_input_asynch,
connp, NULL, SQ_FILL, SQTAG_TCP_KSSL_INPUT);
} else {
DTRACE_PROBE(kssl_err__allocb_failed);
}
CONN_DEC_REF(connp);
}
/*
* tcp_input_data() calls this routine for all packet destined to a
* connection to the SSL port, when the SSL kernel proxy is configured
* to intercept and process those packets.
* A packet may carry multiple SSL records, so the function
* calls kssl_input() in a loop, until all records are
* handled.
* As long as this connection is in handshake, that is until the first
* time kssl_input() returns a record to be delivered ustreams,
* we maintain the tcp_kssl_inhandshake, and keep an extra reference on
* the tcp/connp across the call to kssl_input(). The reason is, that
* function may return KSSL_CMD_QUEUED after scheduling an asynchronous
* request and cause tcp_kssl_callback() to be called on a different CPU,
* which could decrement the conn/tcp reference before we get to increment it.
*/
void
tcp_kssl_input(tcp_t *tcp, mblk_t *mp, cred_t *cr)
{
struct conn_s *connp = tcp->tcp_connp;
tcp_t *listener;
mblk_t *ind_mp;
kssl_cmd_t kssl_cmd;
mblk_t *outmp;
struct T_conn_ind *tci;
boolean_t more = B_FALSE;
boolean_t conn_held = B_FALSE;
boolean_t is_v4;
void *addr;
if (is_system_labeled() && mp != NULL) {
ASSERT(cr != NULL || msg_getcred(mp, NULL) != NULL);
/*
* Provide for protocols above TCP such as RPC. NOPID leaves
* db_cpid unchanged.
* The cred could have already been set.
*/
if (cr != NULL)
mblk_setcred(mp, cr, NOPID);
}
/* First time here, allocate the SSL context */
if (tcp->tcp_kssl_ctx == NULL) {
ASSERT(tcp->tcp_kssl_pending);
is_v4 = (connp->conn_ipversion == IPV4_VERSION);
if (is_v4) {
addr = &connp->conn_faddr_v4;
} else {
addr = &connp->conn_faddr_v6;
}
if (kssl_init_context(tcp->tcp_kssl_ent,
addr, is_v4, tcp->tcp_mss,
&(tcp->tcp_kssl_ctx)) != KSSL_STS_OK) {
tcp->tcp_kssl_pending = B_FALSE;
kssl_release_ent(tcp->tcp_kssl_ent, NULL,
KSSL_NO_PROXY);
tcp->tcp_kssl_ent = NULL;
goto no_can_do;
}
tcp->tcp_kssl_inhandshake = B_TRUE;
/* we won't be needing this one after now */
kssl_release_ent(tcp->tcp_kssl_ent, NULL, KSSL_NO_PROXY);
tcp->tcp_kssl_ent = NULL;
}
if (tcp->tcp_kssl_inhandshake) {
CONN_INC_REF(connp);
conn_held = B_TRUE;
}
do {
kssl_cmd = kssl_input(tcp->tcp_kssl_ctx, mp, &outmp,
&more, tcp_kssl_input_callback, (void *)tcp);
switch (kssl_cmd) {
case KSSL_CMD_SEND:
DTRACE_PROBE(kssl_cmd_send);
/*
* We need to increment tcp_squeue_bytes to account
* for the extra bytes internally injected to the
* outgoing flow. tcp_output() will decrement it
* as they are sent out.
*/
mutex_enter(&tcp->tcp_non_sq_lock);
tcp->tcp_squeue_bytes += msgdsize(outmp);
mutex_exit(&tcp->tcp_non_sq_lock);
tcp_output(connp, outmp, NULL, NULL);
/* FALLTHROUGH */
case KSSL_CMD_NONE:
DTRACE_PROBE(kssl_cmd_none);
if (tcp->tcp_kssl_pending) {
mblk_t *ctxmp;
/*
* SSL handshake successfully started -
* pass up the T_CONN_IND
*/
mp = NULL;
listener = tcp->tcp_listener;
tcp->tcp_kssl_pending = B_FALSE;
ind_mp = tcp->tcp_conn.tcp_eager_conn_ind;
ASSERT(ind_mp != NULL);
ctxmp = allocb(sizeof (kssl_ctx_t), BPRI_MED);
/*
* Give this session a chance to fall back to
* userland SSL
*/
if (ctxmp == NULL)
goto no_can_do;
/*
* attach the kssl_ctx to the conn_ind and
* transform it to a T_SSL_PROXY_CONN_IND.
* Hold it so that it stays valid till it
* reaches the stream head.
*/
kssl_hold_ctx(tcp->tcp_kssl_ctx);
*((kssl_ctx_t *)ctxmp->b_rptr) =
tcp->tcp_kssl_ctx;
ctxmp->b_wptr = ctxmp->b_rptr +
sizeof (kssl_ctx_t);
ind_mp->b_cont = ctxmp;
tci = (struct T_conn_ind *)ind_mp->b_rptr;
tci->PRIM_type = T_SSL_PROXY_CONN_IND;
/*
* The code below is copied from tcp_input_data
* delivering the T_CONN_IND on a TCPS_SYN_RCVD,
* and all conn ref cnt comments apply.
*/
tcp->tcp_conn.tcp_eager_conn_ind = NULL;
tcp->tcp_tconnind_started = B_TRUE;
CONN_INC_REF(connp);
CONN_INC_REF(listener->tcp_connp);
if (listener->tcp_connp->conn_sqp ==
connp->conn_sqp) {
tcp_send_conn_ind(listener->tcp_connp,
ind_mp,
listener->tcp_connp->conn_sqp);
CONN_DEC_REF(listener->tcp_connp);
} else {
SQUEUE_ENTER_ONE(
listener->tcp_connp->conn_sqp,
ind_mp, tcp_send_conn_ind,
listener->tcp_connp, NULL, SQ_FILL,
SQTAG_TCP_CONN_IND);
}
}
break;
case KSSL_CMD_QUEUED:
DTRACE_PROBE(kssl_cmd_queued);
/*
* We hold the conn_t here because an asynchronous
* request have been queued and
* tcp_kssl_input_callback() will be called later.
* It will release the conn_t
*/
CONN_INC_REF(connp);
break;
case KSSL_CMD_DELIVER_PROXY:
case KSSL_CMD_DELIVER_SSL:
DTRACE_PROBE(kssl_cmd_proxy__ssl);
/*
* Keep accumulating if not yet accepted.
*/
if (tcp->tcp_listener != NULL) {
DTRACE_PROBE1(kssl_mblk__input_rcv_enqueue,
mblk_t *, outmp);
tcp_rcv_enqueue(tcp, outmp, msgdsize(outmp),
NULL);
} else {
DTRACE_PROBE1(kssl_mblk__input_putnext,
mblk_t *, outmp);
putnext(connp->conn_rq, outmp);
}
/*
* We're at a phase where records are sent upstreams,
* past the handshake
*/
tcp->tcp_kssl_inhandshake = B_FALSE;
break;
case KSSL_CMD_NOT_SUPPORTED:
DTRACE_PROBE(kssl_cmd_not_supported);
/*
* Stop the SSL processing by the proxy, and
* switch to the userland SSL
*/
if (tcp->tcp_kssl_pending) {
tcp->tcp_kssl_pending = B_FALSE;
no_can_do:
DTRACE_PROBE1(kssl_no_can_do, tcp_t *, tcp);
listener = tcp->tcp_listener;
ind_mp = tcp->tcp_conn.tcp_eager_conn_ind;
ASSERT(ind_mp != NULL);
if (tcp->tcp_kssl_ctx != NULL) {
kssl_release_ctx(tcp->tcp_kssl_ctx);
tcp->tcp_kssl_ctx = NULL;
}
/*
* Make this a T_SSL_PROXY_CONN_IND, for the
* stream head to deliver it to the SSL
* fall-back listener
*/
tci = (struct T_conn_ind *)ind_mp->b_rptr;
tci->PRIM_type = T_SSL_PROXY_CONN_IND;
/*
* The code below is copied from tcp_input_data
* delivering the T_CONN_IND on a TCPS_SYN_RCVD,
* and all conn ref cnt comments apply.
*/
tcp->tcp_conn.tcp_eager_conn_ind = NULL;
tcp->tcp_tconnind_started = B_TRUE;
CONN_INC_REF(connp);
CONN_INC_REF(listener->tcp_connp);
if (listener->tcp_connp->conn_sqp ==
connp->conn_sqp) {
tcp_send_conn_ind(listener->tcp_connp,
ind_mp,
listener->tcp_connp->conn_sqp);
CONN_DEC_REF(listener->tcp_connp);
} else {
SQUEUE_ENTER_ONE(
listener->tcp_connp->conn_sqp,
ind_mp, tcp_send_conn_ind,
listener->tcp_connp, NULL,
SQ_FILL, SQTAG_TCP_CONN_IND);
}
}
if (mp != NULL)
tcp_rcv_enqueue(tcp, mp, msgdsize(mp), NULL);
break;
}
mp = NULL;
} while (more);
if (conn_held) {
CONN_DEC_REF(connp);
}
}
/*
* Fusion output routine, called by tcp_output() and tcp_wput_proto().
*/
boolean_t
tcp_fuse_output(tcp_t *tcp, mblk_t *mp, uint32_t send_size)
{
tcp_t *peer_tcp = tcp->tcp_loopback_peer;
uint_t max_unread;
boolean_t flow_stopped;
boolean_t urgent = (DB_TYPE(mp) != M_DATA);
ASSERT(tcp->tcp_fused);
ASSERT(peer_tcp != NULL && peer_tcp->tcp_loopback_peer == tcp);
ASSERT(tcp->tcp_connp->conn_sqp == peer_tcp->tcp_connp->conn_sqp);
ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_PROTO ||
DB_TYPE(mp) == M_PCPROTO);
max_unread = peer_tcp->tcp_fuse_rcv_unread_hiwater;
/* If this connection requires IP, unfuse and use regular path */
if (TCP_LOOPBACK_IP(tcp) || TCP_LOOPBACK_IP(peer_tcp) ||
IPP_ENABLED(IPP_LOCAL_OUT|IPP_LOCAL_IN)) {
TCP_STAT(tcp_fusion_aborted);
tcp_unfuse(tcp);
return (B_FALSE);
}
if (send_size == 0) {
freemsg(mp);
return (B_TRUE);
}
/*
* Handle urgent data; we either send up SIGURG to the peer now
* or do it later when we drain, in case the peer is detached
* or if we're short of memory for M_PCSIG mblk.
*/
if (urgent) {
/*
* We stop synchronous streams when we have urgent data
* queued to prevent tcp_fuse_rrw() from pulling it. If
* for some reasons the urgent data can't be delivered
* below, synchronous streams will remain stopped until
* someone drains the tcp_rcv_list.
*/
TCP_FUSE_SYNCSTR_PLUG_DRAIN(peer_tcp);
tcp_fuse_output_urg(tcp, mp);
}
mutex_enter(&peer_tcp->tcp_fuse_lock);
/*
* Wake up and signal the peer; it is okay to do this before
* enqueueing because we are holding the lock. One of the
* advantages of synchronous streams is the ability for us to
* find out when the application performs a read on the socket,
* by way of tcp_fuse_rrw() entry point being called. Every
* data that gets enqueued onto the receiver is treated as if
* it has arrived at the receiving endpoint, thus generating
* SIGPOLL/SIGIO for asynchronous socket just as in the strrput()
* case. However, we only wake up the application when necessary,
* i.e. during the first enqueue. When tcp_fuse_rrw() is called
* it will send everything upstream.
*/
if (peer_tcp->tcp_direct_sockfs && !urgent &&
!TCP_IS_DETACHED(peer_tcp)) {
if (peer_tcp->tcp_rcv_list == NULL)
STR_WAKEUP_SET(STREAM(peer_tcp->tcp_rq));
/* Update poll events and send SIGPOLL/SIGIO if necessary */
STR_SENDSIG(STREAM(peer_tcp->tcp_rq));
}
/*
* Enqueue data into the peer's receive list; we may or may not
* drain the contents depending on the conditions below.
*/
tcp_rcv_enqueue(peer_tcp, mp, send_size);
/* In case it wrapped around and also to keep it constant */
peer_tcp->tcp_rwnd += send_size;
/*
* Exercise flow-control when needed; we will get back-enabled
* in either tcp_accept_finish(), tcp_unfuse(), or tcp_fuse_rrw().
* If tcp_direct_sockfs is on or if the peer endpoint is detached,
* we emulate streams flow control by checking the peer's queue
* size and high water mark; otherwise we simply use canputnext()
* to decide if we need to stop our flow.
*
* The outstanding unread data block check does not apply for a
* detached receiver; this is to avoid unnecessary blocking of the
* sender while the accept is currently in progress and is quite
* similar to the regular tcp.
*/
if (TCP_IS_DETACHED(peer_tcp) || max_unread == 0)
max_unread = UINT_MAX;
flow_stopped = tcp->tcp_flow_stopped;
if (!flow_stopped &&
(((peer_tcp->tcp_direct_sockfs || TCP_IS_DETACHED(peer_tcp)) &&
(peer_tcp->tcp_rcv_cnt >= peer_tcp->tcp_fuse_rcv_hiwater ||
++peer_tcp->tcp_fuse_rcv_unread_cnt >= max_unread)) ||
(!peer_tcp->tcp_direct_sockfs &&
!TCP_IS_DETACHED(peer_tcp) && !canputnext(peer_tcp->tcp_rq)))) {
tcp_setqfull(tcp);
flow_stopped = B_TRUE;
TCP_STAT(tcp_fusion_flowctl);
DTRACE_PROBE4(tcp__fuse__output__flowctl, tcp_t *, tcp,
uint_t, send_size, uint_t, peer_tcp->tcp_rcv_cnt,
uint_t, peer_tcp->tcp_fuse_rcv_unread_cnt);
} else if (flow_stopped &&