/* * Send a NOOP message to a message queue * Returns: * 0 - if queue is full after the send. This is the normal case * but various races can change this. * -1 - if mesq sent successfully but queue not full * >0 - unexpected error. MQE_xxx returned */ static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; unsigned long m; int substatus, ret; struct message_header save_mhdr, *mhdr = mesg; STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { substatus = gru_get_cb_message_queue_substatus(cb); switch (substatus) { case CBSS_NO_ERROR: STAT(mesq_noop_unexpected_error); ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_LB_OVERFLOWED: STAT(mesq_noop_lb_overflow); ret = MQE_CONGESTION; break; case CBSS_QLIMIT_REACHED: STAT(mesq_noop_qlimit_reached); ret = 0; break; case CBSS_AMO_NACKED: STAT(mesq_noop_amo_nacked); ret = MQE_CONGESTION; break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQIE_AGAIN; else ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: STAT(mesq_noop_page_overflow); /* fallthru */ default: BUG(); } } *mhdr = save_mhdr; return ret; } /* * Handle a gru_mesq full. */ static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; int half, qlines; /* Determine if switching to first/second half of q */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); qlines = mqd->qlines; half = (limit != qlines); if (half) mqh = gru_mesq_head(qlines / 2 + 1, qlines); else mqh = gru_mesq_head(2, qlines / 2 + 1); /* Try to get lock for switching head pointer */ gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { STAT(mesq_qf_locked); return MQE_QUEUE_FULL; } /* Got the lock. Send optional NOP if queue not full, */ if (head != limit) { if (send_noop_message(cb, mqd, mesg)) { gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; STAT(mesq_qf_noop_not_full); return MQIE_AGAIN; } avalue++; } /* Then flip queuehead to other half of queue. */ gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* If not successfully in swapping queue head, clear the hstatus lock */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } return MQIE_AGAIN; cberr: STAT(mesq_qf_unexpected_error); return MQE_UNEXPECTED_CB_ERR; } /* * Handle a PUT failure. Note: if message was a 2-line message, one of the * lines might have successfully have been written. Before sending the * message, "present" must be cleared in BOTH lines to prevent the receiver * from prematurely seeing the full message. */ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { unsigned long m, *val = mesg, gpa, save; int ret; m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; } gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; if (!mqd->interrupt_vector) return MQE_OK; /* * Send a cross-partition interrupt to the SSI that contains the target * message queue. Normally, the interrupt is automatically delivered by * hardware but some error conditions require explicit delivery. * Use the GRU to deliver the interrupt. Otherwise partition failures * could cause unrecovered errors. */ gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT); save = *val; *val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector, dest_Fixed); gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA); ret = gru_wait(cb); *val = save; if (ret != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; return MQE_OK; }
static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; unsigned long m; int substatus, ret; struct message_header save_mhdr, *mhdr = mesg; STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { substatus = gru_get_cb_message_queue_substatus(cb); switch (substatus) { case CBSS_NO_ERROR: STAT(mesq_noop_unexpected_error); ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_LB_OVERFLOWED: STAT(mesq_noop_lb_overflow); ret = MQE_CONGESTION; break; case CBSS_QLIMIT_REACHED: STAT(mesq_noop_qlimit_reached); ret = 0; break; case CBSS_AMO_NACKED: STAT(mesq_noop_amo_nacked); ret = MQE_CONGESTION; break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQIE_AGAIN; else ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: STAT(mesq_noop_page_overflow); /* */ default: BUG(); } } *mhdr = save_mhdr; return ret; } /* */ static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; int half, qlines; /* */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); qlines = mqd->qlines; half = (limit != qlines); if (half) mqh = gru_mesq_head(qlines / 2 + 1, qlines); else mqh = gru_mesq_head(2, qlines / 2 + 1); /* */ gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { STAT(mesq_qf_locked); return MQE_QUEUE_FULL; } /* */ if (head != limit) { if (send_noop_message(cb, mqd, mesg)) { gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; STAT(mesq_qf_noop_not_full); return MQIE_AGAIN; } avalue++; } /* */ gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } return MQIE_AGAIN; cberr: STAT(mesq_qf_unexpected_error); return MQE_UNEXPECTED_CB_ERR; } /* */ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { unsigned long m, *val = mesg, gpa, save; int ret; m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; } gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; if (!mqd->interrupt_vector) return MQE_OK; /* */ gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT); save = *val; *val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector, dest_Fixed); gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA); ret = gru_wait(cb); *val = save; if (ret != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; return MQE_OK; }