/*
* Send a NOOP message to a message queue
* Returns:
* 0 - if queue is full after the send. This is the normal case
* but various races can change this.
* -1 - if mesq sent successfully but queue not full
* >0 - unexpected error. MQE_xxx returned
*/
static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
void *mesg)
{
const struct message_header noop_header = {
.present = MQS_NOOP, .lines = 1};
unsigned long m;
int substatus, ret;
struct message_header save_mhdr, *mhdr = mesg;
STAT(mesq_noop);
save_mhdr = *mhdr;
*mhdr = noop_header;
gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
ret = gru_wait(cb);
if (ret) {
substatus = gru_get_cb_message_queue_substatus(cb);
switch (substatus) {
case CBSS_NO_ERROR:
STAT(mesq_noop_unexpected_error);
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_LB_OVERFLOWED:
STAT(mesq_noop_lb_overflow);
ret = MQE_CONGESTION;
break;
case CBSS_QLIMIT_REACHED:
STAT(mesq_noop_qlimit_reached);
ret = 0;
break;
case CBSS_AMO_NACKED:
STAT(mesq_noop_amo_nacked);
ret = MQE_CONGESTION;
break;
case CBSS_PUT_NACKED:
STAT(mesq_noop_put_nacked);
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
IMA);
if (gru_wait(cb) == CBS_IDLE)
ret = MQIE_AGAIN;
else
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_PAGE_OVERFLOW:
STAT(mesq_noop_page_overflow);
/* fallthru */
default:
BUG();
}
}
*mhdr = save_mhdr;
return ret;
}
/*
* Handle a gru_mesq full.
*/
static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
union gru_mesqhead mqh;
unsigned int limit, head;
unsigned long avalue;
int half, qlines;
/* Determine if switching to first/second half of q */
avalue = gru_get_amo_value(cb);
head = gru_get_amo_value_head(cb);
limit = gru_get_amo_value_limit(cb);
qlines = mqd->qlines;
half = (limit != qlines);
if (half)
mqh = gru_mesq_head(qlines / 2 + 1, qlines);
else
mqh = gru_mesq_head(2, qlines / 2 + 1);
/* Try to get lock for switching head pointer */
gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
if (!gru_get_amo_value(cb)) {
STAT(mesq_qf_locked);
return MQE_QUEUE_FULL;
}
/* Got the lock. Send optional NOP if queue not full, */
if (head != limit) {
if (send_noop_message(cb, mqd, mesg)) {
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
STAT(mesq_qf_noop_not_full);
return MQIE_AGAIN;
}
avalue++;
}
/* Then flip queuehead to other half of queue. */
gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
/* If not successfully in swapping queue head, clear the hstatus lock */
if (gru_get_amo_value(cb) != avalue) {
STAT(mesq_qf_switch_head_failed);
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
}
return MQIE_AGAIN;
cberr:
STAT(mesq_qf_unexpected_error);
return MQE_UNEXPECTED_CB_ERR;
}
/*
* Handle a PUT failure. Note: if message was a 2-line message, one of the
* lines might have successfully have been written. Before sending the
* message, "present" must be cleared in BOTH lines to prevent the receiver
* from prematurely seeing the full message.
*/
static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
unsigned long m, *val = mesg, gpa, save;
int ret;
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
if (lines == 2) {
gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
}
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
if (!mqd->interrupt_vector)
return MQE_OK;
/*
* Send a cross-partition interrupt to the SSI that contains the target
* message queue. Normally, the interrupt is automatically delivered by
* hardware but some error conditions require explicit delivery.
* Use the GRU to deliver the interrupt. Otherwise partition failures
* could cause unrecovered errors.
*/
gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT);
save = *val;
*val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector,
dest_Fixed);
gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA);
ret = gru_wait(cb);
*val = save;
if (ret != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
return MQE_OK;
}
static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
void *mesg)
{
const struct message_header noop_header = {
.present = MQS_NOOP, .lines = 1};
unsigned long m;
int substatus, ret;
struct message_header save_mhdr, *mhdr = mesg;
STAT(mesq_noop);
save_mhdr = *mhdr;
*mhdr = noop_header;
gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
ret = gru_wait(cb);
if (ret) {
substatus = gru_get_cb_message_queue_substatus(cb);
switch (substatus) {
case CBSS_NO_ERROR:
STAT(mesq_noop_unexpected_error);
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_LB_OVERFLOWED:
STAT(mesq_noop_lb_overflow);
ret = MQE_CONGESTION;
break;
case CBSS_QLIMIT_REACHED:
STAT(mesq_noop_qlimit_reached);
ret = 0;
break;
case CBSS_AMO_NACKED:
STAT(mesq_noop_amo_nacked);
ret = MQE_CONGESTION;
break;
case CBSS_PUT_NACKED:
STAT(mesq_noop_put_nacked);
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
IMA);
if (gru_wait(cb) == CBS_IDLE)
ret = MQIE_AGAIN;
else
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_PAGE_OVERFLOW:
STAT(mesq_noop_page_overflow);
/* */
default:
BUG();
}
}
*mhdr = save_mhdr;
return ret;
}
/*
*/
static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
union gru_mesqhead mqh;
unsigned int limit, head;
unsigned long avalue;
int half, qlines;
/* */
avalue = gru_get_amo_value(cb);
head = gru_get_amo_value_head(cb);
limit = gru_get_amo_value_limit(cb);
qlines = mqd->qlines;
half = (limit != qlines);
if (half)
mqh = gru_mesq_head(qlines / 2 + 1, qlines);
else
mqh = gru_mesq_head(2, qlines / 2 + 1);
/* */
gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
if (!gru_get_amo_value(cb)) {
STAT(mesq_qf_locked);
return MQE_QUEUE_FULL;
}
/* */
if (head != limit) {
if (send_noop_message(cb, mqd, mesg)) {
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
STAT(mesq_qf_noop_not_full);
return MQIE_AGAIN;
}
avalue++;
}
/* */
gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
/* */
if (gru_get_amo_value(cb) != avalue) {
STAT(mesq_qf_switch_head_failed);
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
}
return MQIE_AGAIN;
cberr:
STAT(mesq_qf_unexpected_error);
return MQE_UNEXPECTED_CB_ERR;
}
/*
*/
static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
unsigned long m, *val = mesg, gpa, save;
int ret;
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
if (lines == 2) {
gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
}
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
if (!mqd->interrupt_vector)
return MQE_OK;
/*
*/
gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT);
save = *val;
*val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector,
dest_Fixed);
gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA);
ret = gru_wait(cb);
*val = save;
if (ret != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
return MQE_OK;
}
/*
* Send a NOOP message to a message queue
* Returns:
* 0 - if queue is full after the send. This is the normal case
* but various races can change this.
* -1 - if mesq sent successfully but queue not full
* >0 - unexpected error. MQE_xxx returned
*/
static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
void *mesg)
{
const struct message_header noop_header = {
.present = MQS_NOOP, .lines = 1};
unsigned long m;
int substatus, ret;
struct message_header save_mhdr, *mhdr = mesg;
STAT(mesq_noop);
save_mhdr = *mhdr;
*mhdr = noop_header;
gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
ret = gru_wait(cb);
if (ret) {
substatus = gru_get_cb_message_queue_substatus(cb);
switch (substatus) {
case CBSS_NO_ERROR:
STAT(mesq_noop_unexpected_error);
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_LB_OVERFLOWED:
STAT(mesq_noop_lb_overflow);
ret = MQE_CONGESTION;
break;
case CBSS_QLIMIT_REACHED:
STAT(mesq_noop_qlimit_reached);
ret = 0;
break;
case CBSS_AMO_NACKED:
STAT(mesq_noop_amo_nacked);
ret = MQE_CONGESTION;
break;
case CBSS_PUT_NACKED:
STAT(mesq_noop_put_nacked);
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
IMA);
if (gru_wait(cb) == CBS_IDLE)
ret = MQIE_AGAIN;
else
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_PAGE_OVERFLOW:
STAT(mesq_noop_page_overflow);
/* fall through */
default:
BUG();
}
}
*mhdr = save_mhdr;
return ret;
}
/*
* Handle a gru_mesq full.
*/
static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
union gru_mesqhead mqh;
unsigned int limit, head;
unsigned long avalue;
int half, qlines;
/* Determine if switching to first/second half of q */
avalue = gru_get_amo_value(cb);
head = gru_get_amo_value_head(cb);
limit = gru_get_amo_value_limit(cb);
qlines = mqd->qlines;
half = (limit != qlines);
if (half)
mqh = gru_mesq_head(qlines / 2 + 1, qlines);
else
mqh = gru_mesq_head(2, qlines / 2 + 1);
/* Try to get lock for switching head pointer */
gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
if (!gru_get_amo_value(cb)) {
STAT(mesq_qf_locked);
return MQE_QUEUE_FULL;
}
/* Got the lock. Send optional NOP if queue not full, */
if (head != limit) {
if (send_noop_message(cb, mqd, mesg)) {
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
STAT(mesq_qf_noop_not_full);
return MQIE_AGAIN;
}
avalue++;
}
/* Then flip queuehead to other half of queue. */
gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
/* If not successfully in swapping queue head, clear the hstatus lock */
if (gru_get_amo_value(cb) != avalue) {
STAT(mesq_qf_switch_head_failed);
gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
}
return MQIE_AGAIN;
cberr:
STAT(mesq_qf_unexpected_error);
return MQE_UNEXPECTED_CB_ERR;
}
/*
* Handle a PUT failure. Note: if message was a 2-line message, one of the
* lines might have successfully have been written. Before sending the
* message, "present" must be cleared in BOTH lines to prevent the receiver
* from prematurely seeing the full message.
*/
static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
void *mesg, int lines)
{
unsigned long m;
int ret, loops = 200; /* experimentally determined */
m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
if (lines == 2) {
gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
}
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
return MQE_UNEXPECTED_CB_ERR;
if (!mqd->interrupt_vector)
return MQE_OK;
/*
* Send a noop message in order to deliver a cross-partition interrupt
* to the SSI that contains the target message queue. Normally, the
* interrupt is automatically delivered by hardware following mesq
* operations, but some error conditions require explicit delivery.
* The noop message will trigger delivery. Otherwise partition failures
* could cause unrecovered errors.
*/
do {
ret = send_noop_message(cb, mqd, mesg);
} while ((ret == MQIE_AGAIN || ret == MQE_CONGESTION) && (loops-- > 0));
if (ret == MQIE_AGAIN || ret == MQE_CONGESTION) {
/*
* Don't indicate to the app to resend the message, as it's
* already been successfully sent. We simply send an OK
* (rather than fail the send with MQE_UNEXPECTED_CB_ERR),
* assuming that the other side is receiving enough
* interrupts to get this message processed anyway.
*/
ret = MQE_OK;
}
return ret;
}
/*
* Send a NOOP message to a message queue
* Returns:
* 0 - if queue is full after the send. This is the normal case
* but various races can change this.
* -1 - if mesq sent successfully but queue not full
* >0 - unexpected error. MQE_xxx returned
*/
static int send_noop_message(void *cb,
unsigned long mq, void *mesg)
{
const struct message_header noop_header = {
.present = MQS_NOOP, .lines = 1};
unsigned long m;
int substatus, ret;
struct message_header save_mhdr, *mhdr = mesg;
STAT(mesq_noop);
save_mhdr = *mhdr;
*mhdr = noop_header;
gru_mesq(cb, mq, gru_get_tri(mhdr), 1, IMA);
ret = gru_wait(cb);
if (ret) {
substatus = gru_get_cb_message_queue_substatus(cb);
switch (substatus) {
case CBSS_NO_ERROR:
STAT(mesq_noop_unexpected_error);
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_LB_OVERFLOWED:
STAT(mesq_noop_lb_overflow);
ret = MQE_CONGESTION;
break;
case CBSS_QLIMIT_REACHED:
STAT(mesq_noop_qlimit_reached);
ret = 0;
break;
case CBSS_AMO_NACKED:
STAT(mesq_noop_amo_nacked);
ret = MQE_CONGESTION;
break;
case CBSS_PUT_NACKED:
STAT(mesq_noop_put_nacked);
m = mq + (gru_get_amo_value_head(cb) << 6);
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
IMA);
if (gru_wait(cb) == CBS_IDLE)
ret = MQIE_AGAIN;
else
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_PAGE_OVERFLOW:
default:
BUG();
}
}
*mhdr = save_mhdr;
return ret;
}
/*
* Handle a gru_mesq full.
*/
static int send_message_queue_full(void *cb,
unsigned long mq, void *mesg, int lines)
{
union gru_mesqhead mqh;
unsigned int limit, head;
unsigned long avalue;
int half, qlines, save;
/* Determine if switching to first/second half of q */
avalue = gru_get_amo_value(cb);
head = gru_get_amo_value_head(cb);
limit = gru_get_amo_value_limit(cb);
/*
* Fetch "qlines" from the queue header. Since the queue may be
* in memory that can't be accessed using socket addresses, use
* the GRU to access the data. Use DSR space from the message.
*/
save = *(int *)mesg;
gru_vload(cb, QLINES(mq), gru_get_tri(mesg), XTYPE_W, 1, 1, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
qlines = *(int *)mesg;
*(int *)mesg = save;
half = (limit != qlines);
if (half)
mqh = gru_mesq_head(qlines / 2 + 1, qlines);
else
mqh = gru_mesq_head(2, qlines / 2 + 1);
/* Try to get lock for switching head pointer */
gru_gamir(cb, EOP_IR_CLR, HSTATUS(mq, half), XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
if (!gru_get_amo_value(cb)) {
STAT(mesq_qf_locked);
return MQE_QUEUE_FULL;
}
/* Got the lock. Send optional NOP if queue not full, */
if (head != limit) {
if (send_noop_message(cb, mq, mesg)) {
gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half),
XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
STAT(mesq_qf_noop_not_full);
return MQIE_AGAIN;
}
avalue++;
}
/* Then flip queuehead to other half of queue. */
gru_gamer(cb, EOP_ERR_CSWAP, mq, XTYPE_DW, mqh.val, avalue, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
/* If not successfully in swapping queue head, clear the hstatus lock */
if (gru_get_amo_value(cb) != avalue) {
STAT(mesq_qf_switch_head_failed);
gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), XTYPE_DW, IMA);
if (gru_wait(cb) != CBS_IDLE)
goto cberr;
}
return MQIE_AGAIN;
cberr:
STAT(mesq_qf_unexpected_error);
return MQE_UNEXPECTED_CB_ERR;
}
/*
* Handle a gru_mesq failure. Some of these failures are software recoverable
* or retryable.
*/
static int send_message_failure(void *cb,
unsigned long mq,
void *mesg,
int lines)
{
int substatus, ret = 0;
unsigned long m;
substatus = gru_get_cb_message_queue_substatus(cb);
switch (substatus) {
case CBSS_NO_ERROR:
STAT(mesq_send_unexpected_error);
ret = MQE_UNEXPECTED_CB_ERR;
break;
case CBSS_LB_OVERFLOWED:
STAT(mesq_send_lb_overflow);
ret = MQE_CONGESTION;
break;
case CBSS_QLIMIT_REACHED:
STAT(mesq_send_qlimit_reached);
ret = send_message_queue_full(cb, mq, mesg, lines);
break;
case CBSS_AMO_NACKED:
STAT(mesq_send_amo_nacked);
ret = MQE_CONGESTION;
break;
case CBSS_PUT_NACKED:
STAT(mesq_send_put_nacked);
m =mq + (gru_get_amo_value_head(cb) << 6);
gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
if (gru_wait(cb) == CBS_IDLE)
ret = MQE_OK;
else
ret = MQE_UNEXPECTED_CB_ERR;
break;
default:
BUG();
}
return ret;
}