/**
* See if any pending writes have been completed, and cleanup if so.
* Cleaning up means removing any publication data that was stored because the write did
* not originally complete.
*/
void MQTTProtocol_checkPendingWrites()
{
FUNC_ENTRY;
if (state.pending_writes.count > 0)
{
ListElement* le = state.pending_writes.first;
while (le)
{
pending_write* pw = (pending_write*)(le->content);
if (Socket_noPendingWrites(pw->socket))
{
Clients* client = pw->client;
MQTTProtocol_removePublication(pw->p);
state.pending_writes.current = le;
ListRemove(&(state.pending_writes), le->content); /* does NextElement itself */
le = state.pending_writes.current;
/* now we might be able to write the next message in the queue */
MQTTProtocol_processQueued(client);
}
else
ListNextElement(&(state.pending_writes), &le);
}
}
FUNC_EXIT;
}
/**
* MQTT retry protocol and socket pending writes processing.
* @param now current time
* @param doRetry boolean - retries as well as pending writes?
* @param regardless boolean - retry packets regardless of retry interval (used on reconnect)
*/
void MQTTProtocol_retry(time_t now, int doRetry, int regardless)
{
ListElement* current = NULL;
FUNC_ENTRY;
ListNextElement(bstate->clients, ¤t);
/* look through the outbound message list of each client, checking to see if a retry is necessary */
while (current)
{
Clients* client = (Clients*)(current->content);
ListNextElement(bstate->clients, ¤t);
if (client->connected == 0)
continue;
if (client->good == 0)
{
MQTTProtocol_closeSession(client, 1);
continue;
}
if (Socket_noPendingWrites(client->net.socket) == 0)
continue;
if (doRetry)
MQTTProtocol_retries(now, client, regardless);
}
FUNC_EXIT;
}
/**
* MQTT retry protocol and socket pending writes processing.
* @param now current time
* @param doRetry boolean - retries as well as pending writes?
* @return not actually used
*/
int MQTTProtocol_retry(time_t now, int doRetry)
{
Node* current = NULL;
int rc = 0;
FUNC_ENTRY;
current = TreeNextElement(bstate->clients, current);
/* look through the outbound message list of each client, checking to see if a retry is necessary */
while (current)
{
Clients* client = (Clients*)(current->content);
current = TreeNextElement(bstate->clients, current);
if (client->connected == 0)
{
#if defined(MQTTS)
if (client->protocol == PROTOCOL_MQTTS)
{
if (difftime(now,client->lastContact) > bstate->retry_interval)
{
int rc2 = 0;
/* NB: no dup bit for these packets */
if (client->connect_state == 1) /* TODO: handle err */
rc2 = MQTTSPacket_send_willTopicReq(client);
else if (client->connect_state == 2) /* TODO: handle err */
rc2 = MQTTSPacket_send_willMsgReq(client);
if (rc2 == SOCKET_ERROR)
{
client->good = 0;
Log(LOG_WARNING, 29, NULL, client->clientID, client->socket);
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
}
}
#endif
continue;
}
if (client->good == 0)
{
MQTTProtocol_closeSession(client, 1);
continue;
}
if (Socket_noPendingWrites(client->socket) == 0)
continue;
if (doRetry)
MQTTProtocol_retries(now, client);
if (client)
{
if (MQTTProtocol_processQueued(client))
rc = 1;
}
}
FUNC_EXIT_RC(rc);
return rc;
}
/**
* Don't accept work from a client unless it is accepting work back, i.e. its socket is writeable
* this seems like a reasonable form of flow control, and practically, seems to work.
* @param socket the socket to check
* @param read_set the socket read set (see select doc)
* @param write_set the socket write set (see select doc)
* @return boolean - is the socket ready to go?
*/
int isReady(int socket, fd_set* read_set, fd_set* write_set)
{
int rc = 1;
FUNC_ENTRY;
if (ListFindItem(s.connect_pending, &socket, intcompare) && FD_ISSET(socket, write_set))
ListRemoveItem(s.connect_pending, &socket, intcompare);
else
rc = FD_ISSET(socket, read_set) && FD_ISSET(socket, write_set) && Socket_noPendingWrites(socket);
FUNC_EXIT_RC(rc);
return rc;
}
/**
* Attempts to write a series of buffers to a socket in *one* system call so that they are
* sent as one packet.
* @param socket the socket to write to
* @param buf0 the first buffer
* @param buf0len the length of data in the first buffer
* @param count number of buffers
* @param buffers an array of buffers to write
* @param buflens an array of corresponding buffer lengths
* @return completion code, especially TCPSOCKET_INTERRUPTED
*/
int Socket_putdatas(int socket, char* buf0, int buf0len, int count, char** buffers, int* buflens)
{
unsigned long bytes = 0L;
iobuf iovecs[5];
int rc = TCPSOCKET_INTERRUPTED, i, total = buf0len;
FUNC_ENTRY;
if (!Socket_noPendingWrites(socket))
{
Log(LOG_SEVERE, -1, "Trying to write to socket %d for which there is already pending output", socket);
rc = SOCKET_ERROR;
goto exit;
}
for (i = 0; i < count; i++)
total += buflens[i];
iovecs[0].iov_base = buf0;
iovecs[0].iov_len = buf0len;
for (i = 0; i < count; i++)
{
iovecs[i+1].iov_base = buffers[i];
iovecs[i+1].iov_len = buflens[i];
}
if ((rc = Socket_writev(socket, iovecs, count+1, &bytes)) != SOCKET_ERROR)
{
if (bytes == total)
rc = TCPSOCKET_COMPLETE;
else
{
int* sockmem = (int*)malloc(sizeof(int));
Log(TRACE_MIN, -1, "Partial write: %ld bytes of %d actually written on socket %d",
bytes, total, socket);
#if defined(OPENSSL)
SocketBuffer_pendingWrite(socket, NULL, count+1, iovecs, total, bytes);
#else
SocketBuffer_pendingWrite(socket, count+1, iovecs, total, bytes);
#endif
*sockmem = socket;
ListAppend(s.write_pending, sockmem, sizeof(int));
FD_SET(socket, &(s.pending_wset));
rc = TCPSOCKET_INTERRUPTED;
}
}
exit:
FUNC_EXIT_RC(rc);
return rc;
}
/**
* Kick off a new publication, queue it or send it as appropriate.
* @param pubclient the client to send the publication to
* @param publish the publication data
* @param qos the MQTT QoS to use
* @param retained boolean - whether to set the MQTT retained flag
* @param mm - pointer to the message to send
* @return the completion code
*/
int MQTTProtocol_startOrQueuePublish(Clients* pubclient, Publish* publish, int qos, int retained, int priority, Messages** mm)
{
int rc = TCPSOCKET_COMPLETE;
int socket = pubclient->socket; /* used when MQTT_MP protocol is included */
FUNC_ENTRY;
#if defined(MQTTMP)
if (pubclient->protocol == PROTOCOL_MQTT_MP)
socket = pubclient->actualSock;
#endif
if (pubclient->connected && pubclient->good && /* client is connected and has no errors */
Socket_noPendingWrites(socket) && /* there aren't any previous packets still stacked up on the socket */
queuedMsgsCount(pubclient) == 0 && /* there are no messages ahead in the queue */
pubclient->outboundMsgs->count < bstate->max_inflight_messages) /* the queue is not full */
{
#if defined(MQTTS)
if (pubclient->protocol == PROTOCOL_MQTTS && strlen(publish->topic) > 2 &&
MQTTSProtocol_getRegisteredTopicId(pubclient, publish->topic) == 0)
{
if (pubclient->pendingRegistration == NULL)
rc = MQTTSProtocol_startRegistration(pubclient, publish->topic);
rc = MQTTProtocol_queuePublish(pubclient, publish, qos, retained, priority, mm);
}
else if (pubclient->protocol == PROTOCOL_MQTTS && qos > 0 && pubclient->outboundMsgs->count > 0)
{
/* can only have 1 qos 1/2 message in flight with MQTT-S */
rc = MQTTProtocol_queuePublish(pubclient, publish, qos, retained, priority, mm);
}
else
{
#endif
rc = MQTTProtocol_startPublish(pubclient, publish, qos, retained, mm);
/* We don't normally queue QoS 0 messages just send them straight through. But in the case when none of the packet
* is written we need to queue it up. If only part of the packet is written, then it is buffered by the socket module.
*/
if (qos == 0 && rc == TCPSOCKET_NOWORK)
rc = MQTTProtocol_queuePublish(pubclient, publish, qos, retained, priority, mm);
#if defined(MQTTS)
}
#endif
}
else if (qos != 0 || (pubclient->connected && pubclient->good)) /* only queue qos 0 if the client is connected in a good way */
rc = MQTTProtocol_queuePublish(pubclient, publish, qos, retained, priority, mm);
FUNC_EXIT_RC(rc);
return rc;
}
/**
* See if any pending writes have been completed, and cleanup if so.
* Cleaning up means removing any publication data that was stored because the write did
* not originally complete.
*/
void MQTTProtocol_checkPendingWrites()
{
FUNC_ENTRY;
if (state.pending_writes.count > 0)
{
ListElement* le = state.pending_writes.first;
while (le)
{
if (Socket_noPendingWrites(((pending_write*)(le->content))->socket))
{
MQTTProtocol_removePublication(((pending_write*)(le->content))->p);
state.pending_writes.current = le;
ListRemove(&(state.pending_writes), le->content); /* does NextElement itself */
le = state.pending_writes.current;
}
else
ListNextElement(&(state.pending_writes), &le);
}
}
FUNC_EXIT;
}
/**
* MQTT protocol keepAlive processing. Sends PINGREQ packets as required.
* @param now current time
*/
void MQTTProtocol_keepalive(time_t now)
{
ListElement* current = NULL;
FUNC_ENTRY;
ListNextElement(bstate->clients, ¤t);
while (current)
{
Clients* client = (Clients*)(current->content);
ListNextElement(bstate->clients, ¤t);
if (client->connected && client->keepAliveInterval > 0 &&
(difftime(now, client->net.lastSent) >= client->keepAliveInterval ||
difftime(now, client->net.lastReceived) >= client->keepAliveInterval))
{
if (client->ping_outstanding == 0)
{
if (Socket_noPendingWrites(client->net.socket))
{
if (MQTTPacket_send_pingreq(&client->net, client->clientID) != TCPSOCKET_COMPLETE)
{
Log(TRACE_PROTOCOL, -1, "Error sending PINGREQ for client %s on socket %d, disconnecting", client->clientID, client->net.socket);
MQTTProtocol_closeSession(client, 1);
}
else
{
client->net.lastSent = now;
client->ping_outstanding = 1;
}
}
}
else
{
Log(TRACE_PROTOCOL, -1, "PINGRESP not received in keepalive interval for client %s on socket %d, disconnecting", client->clientID, client->net.socket);
MQTTProtocol_closeSession(client, 1);
}
}
}
FUNC_EXIT;
}
/**
* MQTT retry processing per client
* @param now current time
* @param client - the client to which to apply the retry processing
*/
void MQTTProtocol_retries(time_t now, Clients* client)
{
ListElement* outcurrent = NULL;
FUNC_ENTRY;
#if defined(MQTTS)
if (client->protocol == PROTOCOL_MQTTS)
{
if (client->pendingRegistration != NULL &&
difftime(now, client->pendingRegistration->sent) > bstate->retry_interval)
{
Registration* reg = client->pendingRegistration->registration;
time(&client->pendingRegistration->sent);
/* NB: no dup bit for these packets */
if (MQTTSPacket_send_register(client, reg->id, reg->topicName, client->pendingRegistration->msgId) == SOCKET_ERROR)
{
client->good = 0;
//TODO: update message
Log(LOG_WARNING, 29, NULL, client->clientID, client->socket);
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
}
#if !defined(NO_BRIDGE)
if (client->pendingSubscription != NULL &&
difftime(now, client->pendingSubscription->sent) > bstate->retry_interval)
{
time(&client->pendingSubscription->sent);
if (MQTTSPacket_send_subscribe(client, client->pendingSubscription->topicName,client->pendingSubscription->qos, client->pendingSubscription->msgId) == SOCKET_ERROR)
{
client->good = 0;
//TODO: update message
Log(LOG_WARNING, 29, NULL, client->clientID, client->socket);
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
}
#endif
}
#endif
while (client && ListNextElement(client->outboundMsgs, &outcurrent) &&
client->connected && client->good && /* client is connected and has no errors */
Socket_noPendingWrites(client->socket)) /* there aren't any previous packets still stacked up on the socket */
{
Messages* m = (Messages*)(outcurrent->content);
if (difftime(now, m->lastTouch) > bstate->retry_interval)
{
if (m->qos == 1 || (m->qos == 2 && m->nextMessageType == PUBREC))
{
Publish publish;
int rc;
Log(LOG_INFO, 28, NULL, client->clientID, client->socket, m->msgid);
publish.msgId = m->msgid;
publish.topic = m->publish->topic;
publish.payload = m->publish->payload;
publish.payloadlen = m->publish->payloadlen;
#if defined(MQTTS)
if (client->protocol == PROTOCOL_MQTTS)
{
if (MQTTSProtocol_startPublishCommon(client, &publish, 1, m->qos, m->retain) == SOCKET_ERROR)
{
client->good = 0;
Log(LOG_WARNING, 29, NULL, client->clientID, client->socket);
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
time(&(m->lastTouch));
}
else
{
#endif
rc = MQTTPacket_send_publish(&publish, 1, m->qos, m->retain, client->socket, client->clientID);
if (rc == SOCKET_ERROR)
{
client->good = 0;
Log(LOG_WARNING, 29, NULL, client->clientID, client->socket);
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
{
if (m->qos == 0 && rc == TCPSOCKET_INTERRUPTED)
MQTTProtocol_storeQoS0(client, &publish);
time(&(m->lastTouch));
}
#if defined(MQTTS)
}
#endif
}
else if (m->qos && m->nextMessageType == PUBCOMP)
{
Log(LOG_WARNING, 30, NULL, client->clientID, m->msgid);
#if defined(MQTTS)
if (client->protocol == PROTOCOL_MQTTS)
{
/* NB: no dup bit for PUBREL */
if (MQTTSPacket_send_pubrel(client, m->msgid) == SOCKET_ERROR)
{
client->good = 0;
Log(LOG_WARNING, 18, NULL, client->clientID, client->socket,
Socket_getpeer(client->socket));
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
time(&(m->lastTouch));
}
else
#endif
if (MQTTPacket_send_pubrel(m->msgid, 1, client->socket, client->clientID) != TCPSOCKET_COMPLETE)
{
client->good = 0;
Log(LOG_WARNING, 18, NULL, client->clientID, client->socket,
Socket_getpeer(client->socket));
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
time(&(m->lastTouch));
}
/* break; why not do all retries at once? */
}
}
FUNC_EXIT;
}
/**
* Start the publish exchange for any queued messages, if possible.
* When the inflight message queue is not at maximum capacity we can start a new
* publication.
* @param client the client to process queued messages for
*/
int MQTTProtocol_processQueued(Clients* client)
{
int rc = 0;
#if defined(QOS0_SEND_LIMIT)
int qos0count = 0;
#endif
int threshold_log_message_issued = 0;
FUNC_ENTRY;
if (Protocol_isClientQuiescing(client))
goto exit; /* don't create new work - just finish in-flight stuff */
Log(TRACE_MAXIMUM, 0, NULL, client->clientID);
while (client->good && Socket_noPendingWrites(client->socket) && /* no point in starting a publish if a write is still pending */
client->outboundMsgs->count < bstate->max_inflight_messages &&
queuedMsgsCount(client) > 0
#if defined(QOS0_SEND_LIMIT)
&& qos0count < bstate->max_inflight_messages /* an arbitrary criterion - but when would we restart? */
#endif
#if defined(MQTTS)
&& (client->protocol == PROTOCOL_MQTT || client->outboundMsgs->count == 0)
#endif
)
{
int pubrc = TCPSOCKET_COMPLETE;
Messages* m = NULL;
int threshold = (THRESHOLD * bstate->max_queued_messages) / 100;
List* queue = NULL;
int i;
for (i = PRIORITY_MAX-1; i >= 0; --i)
{
if (client->queuedMsgs[i]->count > 0)
{
queue = client->queuedMsgs[i];
break;
}
}
m = (Messages*)(queue->first->content);
Log(TRACE_MAXIMUM, 1, NULL, client->clientID);
#if defined(MQTTS)
if (client->protocol == PROTOCOL_MQTTS && strlen(m->publish->topic) > 2 &&
MQTTSProtocol_getRegisteredTopicId(client, m->publish->topic) == 0)
{
if (client->pendingRegistration == NULL)
rc = MQTTSProtocol_startRegistration(client, m->publish->topic);
goto exit;
}
#endif
#if defined(QOS0_SEND_LIMIT)
if (m->qos == 0)
++qos0count;
#endif
pubrc = MQTTProtocol_startQueuedPublish(client, m);
/* regardless of whether the publish packet was sent on the wire (pubrc is good), the
* message has been put onto the outbound queue, so it must be removed from
* the queuedMsgs queue
*/
if (pubrc != TCPSOCKET_COMPLETE && pubrc != TCPSOCKET_INTERRUPTED)
client->good = 0;
if (m->qos == 0)
{
/* This is done primarily for MQTT-S.
* A qos-0 message will be on this queue if its topic
* has to be registered first. Now that the message
* has been sent, it needs to be cleaned up as there
* won't be an ack to trigger it.
*
* For MQTT, there is a scenario in which qos-0 messages
* could be on this list for which the same applies.
*
* Note (IGC): this is also a bug fix I just implemented - applies equally to MQTTs and MQTT!
*/
MQTTProtocol_removePublication(m->publish);
if (!ListRemove(queue, m))
Log(LOG_ERROR, 38, NULL);
}
else if (!ListDetach(queue, m))
Log(LOG_ERROR, 38, NULL);
if (queuedMsgsCount(client) == threshold - 1 && !threshold_log_message_issued)
{
Log(LOG_INFO, 146, NULL, client->clientID, THRESHOLD);
threshold_log_message_issued = 1;
}
}
#if defined(QOS0_SEND_LIMIT)
if (qos0count >= bstate->max_inflight_messages)
rc = 1;
#endif
exit:
FUNC_EXIT_RC(rc);
return rc;
}
/**
* MQTT retry processing per client
* @param now current time
* @param client - the client to which to apply the retry processing
* @param regardless boolean - retry packets regardless of retry interval (used on reconnect)
*/
static void MQTTProtocol_retries(time_t now, Clients* client, int regardless)
{
ListElement* outcurrent = NULL;
FUNC_ENTRY;
if (!regardless && client->retryInterval <= 0) /* 0 or -ive retryInterval turns off retry except on reconnect */
goto exit;
while (client && ListNextElement(client->outboundMsgs, &outcurrent) &&
client->connected && client->good && /* client is connected and has no errors */
Socket_noPendingWrites(client->net.socket)) /* there aren't any previous packets still stacked up on the socket */
{
Messages* m = (Messages*)(outcurrent->content);
if (regardless || difftime(now, m->lastTouch) > max(client->retryInterval, 10))
{
if (m->qos == 1 || (m->qos == 2 && m->nextMessageType == PUBREC))
{
Publish publish;
int rc;
Log(TRACE_MIN, 7, NULL, "PUBLISH", client->clientID, client->net.socket, m->msgid);
publish.msgId = m->msgid;
publish.topic = m->publish->topic;
publish.payload = m->publish->payload;
publish.payloadlen = m->publish->payloadlen;
rc = MQTTPacket_send_publish(&publish, 1, m->qos, m->retain, &client->net, client->clientID);
if (rc == SOCKET_ERROR)
{
client->good = 0;
Log(TRACE_PROTOCOL, 29, NULL, client->clientID, client->net.socket,
Socket_getpeer(client->net.socket));
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
{
if (m->qos == 0 && rc == TCPSOCKET_INTERRUPTED)
MQTTProtocol_storeQoS0(client, &publish);
time(&(m->lastTouch));
}
}
else if (m->qos && m->nextMessageType == PUBCOMP)
{
Log(TRACE_MIN, 7, NULL, "PUBREL", client->clientID, client->net.socket, m->msgid);
if (MQTTPacket_send_pubrel(m->msgid, 0, &client->net, client->clientID) != TCPSOCKET_COMPLETE)
{
client->good = 0;
Log(TRACE_PROTOCOL, 29, NULL, client->clientID, client->net.socket,
Socket_getpeer(client->net.socket));
MQTTProtocol_closeSession(client, 1);
client = NULL;
}
else
time(&(m->lastTouch));
}
/* break; why not do all retries at once? */
}
}
exit:
FUNC_EXIT;
}
int MQTTClient_publish(MQTTClient handle, const char* topicName, int payloadlen, void* payload,
int qos, int retained, MQTTClient_deliveryToken* deliveryToken)
{
int rc = MQTTCLIENT_SUCCESS;
MQTTClients* m = handle;
Messages* msg = NULL;
Publish* p = NULL;
int blocked = 0;
int msgid = 0;
FUNC_ENTRY;
Thread_lock_mutex(mqttclient_mutex);
if (m == NULL || m->c == NULL)
rc = MQTTCLIENT_FAILURE;
else if (m->c->connected == 0)
rc = MQTTCLIENT_DISCONNECTED;
else if (!UTF8_validateString(topicName))
rc = MQTTCLIENT_BAD_UTF8_STRING;
if (rc != MQTTCLIENT_SUCCESS)
goto exit;
/* If outbound queue is full, block until it is not */
while (m->c->outboundMsgs->count >= m->c->maxInflightMessages ||
Socket_noPendingWrites(m->c->net.socket) == 0) /* wait until the socket is free of large packets being written */
{
if (blocked == 0)
{
blocked = 1;
Log(TRACE_MIN, -1, "Blocking publish on queue full for client %s", m->c->clientID);
}
Thread_unlock_mutex(mqttclient_mutex);
MQTTClient_yield();
Thread_lock_mutex(mqttclient_mutex);
if (m->c->connected == 0)
{
rc = MQTTCLIENT_FAILURE;
goto exit;
}
}
if (blocked == 1)
Log(TRACE_MIN, -1, "Resuming publish now queue not full for client %s", m->c->clientID);
if (qos > 0 && (msgid = MQTTProtocol_assignMsgId(m->c)) == 0)
{ /* this should never happen as we've waited for spaces in the queue */
rc = MQTTCLIENT_MAX_MESSAGES_INFLIGHT;
goto exit;
}
p = malloc(sizeof(Publish));
p->payload = payload;
p->payloadlen = payloadlen;
p->topic = (char*)topicName;
p->msgId = msgid;
rc = MQTTProtocol_startPublish(m->c, p, qos, retained, &msg);
/* If the packet was partially written to the socket, wait for it to complete.
* However, if the client is disconnected during this time and qos is not 0, still return success, as
* the packet has already been written to persistence and assigned a message id so will
* be sent when the client next connects.
*/
if (rc == TCPSOCKET_INTERRUPTED)
{
while (m->c->connected == 1 && SocketBuffer_getWrite(m->c->net.socket))
{
Thread_unlock_mutex(mqttclient_mutex);
MQTTClient_yield();
Thread_lock_mutex(mqttclient_mutex);
}
rc = (qos > 0 || m->c->connected == 1) ? MQTTCLIENT_SUCCESS : MQTTCLIENT_FAILURE;
}
if (deliveryToken && qos > 0)
*deliveryToken = msg->msgid;
free(p);
if (rc == SOCKET_ERROR)
{
Thread_unlock_mutex(mqttclient_mutex);
MQTTClient_disconnect_internal(handle, 0);
Thread_lock_mutex(mqttclient_mutex);
/* Return success for qos > 0 as the send will be retried automatically */
rc = (qos > 0) ? MQTTCLIENT_SUCCESS : MQTTCLIENT_FAILURE;
}
exit:
Thread_unlock_mutex(mqttclient_mutex);
FUNC_EXIT_RC(rc);
return rc;
}
