static int quicktest2(unsigned long arg) { static DECLARE_COMPLETION(cmp); unsigned long han; int blade_id = 0; int numcb = 4; int ret = 0; unsigned long *buf; void *cb0, *cb; int i, k, istatus, bytes; bytes = numcb * 4 * 8; buf = kmalloc(bytes, GFP_KERNEL); if (!buf) return -ENOMEM; ret = -EBUSY; han = gru_reserve_async_resources(blade_id, numcb, 0, &cmp); if (!han) goto done; gru_lock_async_resource(han, &cb0, NULL); memset(buf, 0xee, bytes); for (i = 0; i < numcb; i++) gru_vset(cb0 + i * GRU_HANDLE_STRIDE, uv_gpa(&buf[i * 4]), 0, XTYPE_DW, 4, 1, IMA_INTERRUPT); ret = 0; for (k = 0; k < numcb; k++) { gru_wait_async_cbr(han); for (i = 0; i < numcb; i++) { cb = cb0 + i * GRU_HANDLE_STRIDE; istatus = gru_check_status(cb); if (istatus == CBS_ACTIVE) continue; if (istatus == CBS_EXCEPTION) ret = -EFAULT; else if (buf[i] || buf[i + 1] || buf[i + 2] || buf[i + 3]) ret = -EIO; } } BUG_ON(cmp.done); gru_unlock_async_resource(han); gru_release_async_resources(han); done: kfree(buf); return ret; }
/* * Send a NOOP message to a message queue * Returns: * 0 - if queue is full after the send. This is the normal case * but various races can change this. * -1 - if mesq sent successfully but queue not full * >0 - unexpected error. MQE_xxx returned */ static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; unsigned long m; int substatus, ret; struct message_header save_mhdr, *mhdr = mesg; STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { substatus = gru_get_cb_message_queue_substatus(cb); switch (substatus) { case CBSS_NO_ERROR: STAT(mesq_noop_unexpected_error); ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_LB_OVERFLOWED: STAT(mesq_noop_lb_overflow); ret = MQE_CONGESTION; break; case CBSS_QLIMIT_REACHED: STAT(mesq_noop_qlimit_reached); ret = 0; break; case CBSS_AMO_NACKED: STAT(mesq_noop_amo_nacked); ret = MQE_CONGESTION; break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQIE_AGAIN; else ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: STAT(mesq_noop_page_overflow); /* fallthru */ default: BUG(); } } *mhdr = save_mhdr; return ret; } /* * Handle a gru_mesq full. */ static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; int half, qlines; /* Determine if switching to first/second half of q */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); qlines = mqd->qlines; half = (limit != qlines); if (half) mqh = gru_mesq_head(qlines / 2 + 1, qlines); else mqh = gru_mesq_head(2, qlines / 2 + 1); /* Try to get lock for switching head pointer */ gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { STAT(mesq_qf_locked); return MQE_QUEUE_FULL; } /* Got the lock. Send optional NOP if queue not full, */ if (head != limit) { if (send_noop_message(cb, mqd, mesg)) { gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; STAT(mesq_qf_noop_not_full); return MQIE_AGAIN; } avalue++; } /* Then flip queuehead to other half of queue. */ gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* If not successfully in swapping queue head, clear the hstatus lock */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } return MQIE_AGAIN; cberr: STAT(mesq_qf_unexpected_error); return MQE_UNEXPECTED_CB_ERR; } /* * Handle a PUT failure. Note: if message was a 2-line message, one of the * lines might have successfully have been written. Before sending the * message, "present" must be cleared in BOTH lines to prevent the receiver * from prematurely seeing the full message. */ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { unsigned long m, *val = mesg, gpa, save; int ret; m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; } gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; if (!mqd->interrupt_vector) return MQE_OK; /* * Send a cross-partition interrupt to the SSI that contains the target * message queue. Normally, the interrupt is automatically delivered by * hardware but some error conditions require explicit delivery. * Use the GRU to deliver the interrupt. Otherwise partition failures * could cause unrecovered errors. */ gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT); save = *val; *val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector, dest_Fixed); gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA); ret = gru_wait(cb); *val = save; if (ret != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; return MQE_OK; }
static int quicktest2(unsigned long arg) { static DECLARE_COMPLETION(cmp); unsigned long han; int blade_id = 0; int numcb = 4; int ret = 0; unsigned long *buf; void *cb0, *cb; struct gru_control_block_status *gen; int i, k, istatus, bytes; bytes = numcb * 4 * 8; buf = kmalloc(bytes, GFP_KERNEL); if (!buf) return -ENOMEM; ret = -EBUSY; han = gru_reserve_async_resources(blade_id, numcb, 0, &cmp); if (!han) goto done; gru_lock_async_resource(han, &cb0, NULL); memset(buf, 0xee, bytes); for (i = 0; i < numcb; i++) gru_vset(cb0 + i * GRU_HANDLE_STRIDE, uv_gpa(&buf[i * 4]), 0, XTYPE_DW, 4, 1, IMA_INTERRUPT); ret = 0; k = numcb; do { gru_wait_async_cbr(han); for (i = 0; i < numcb; i++) { cb = cb0 + i * GRU_HANDLE_STRIDE; istatus = gru_check_status(cb); if (istatus != CBS_ACTIVE && istatus != CBS_CALL_OS) break; } if (i == numcb) continue; if (istatus != CBS_IDLE) { ; ret = -EFAULT; } else if (buf[4 * i] || buf[4 * i + 1] || buf[4 * i + 2] || buf[4 * i + 3]) { // printk(KERN_DEBUG "GRU:%d quicktest2:cb %d, buf 0x%lx, 0x%lx, 0x%lx, 0x%lx\n", ; ret = -EIO; } k--; gen = cb; gen->istatus = CBS_CALL_OS; /* don't handle this CBR again */ } while (k); BUG_ON(cmp.done); gru_unlock_async_resource(han); gru_release_async_resources(han); done: kfree(buf); return ret; }
/* * Send a NOOP message to a message queue * Returns: * 0 - if queue is full after the send. This is the normal case * but various races can change this. * -1 - if mesq sent successfully but queue not full * >0 - unexpected error. MQE_xxx returned */ static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; unsigned long m; int substatus, ret; struct message_header save_mhdr, *mhdr = mesg; STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { substatus = gru_get_cb_message_queue_substatus(cb); switch (substatus) { case CBSS_NO_ERROR: STAT(mesq_noop_unexpected_error); ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_LB_OVERFLOWED: STAT(mesq_noop_lb_overflow); ret = MQE_CONGESTION; break; case CBSS_QLIMIT_REACHED: STAT(mesq_noop_qlimit_reached); ret = 0; break; case CBSS_AMO_NACKED: STAT(mesq_noop_amo_nacked); ret = MQE_CONGESTION; break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQIE_AGAIN; else ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: STAT(mesq_noop_page_overflow); /* fall through */ default: BUG(); } } *mhdr = save_mhdr; return ret; } /* * Handle a gru_mesq full. */ static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; int half, qlines; /* Determine if switching to first/second half of q */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); qlines = mqd->qlines; half = (limit != qlines); if (half) mqh = gru_mesq_head(qlines / 2 + 1, qlines); else mqh = gru_mesq_head(2, qlines / 2 + 1); /* Try to get lock for switching head pointer */ gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { STAT(mesq_qf_locked); return MQE_QUEUE_FULL; } /* Got the lock. Send optional NOP if queue not full, */ if (head != limit) { if (send_noop_message(cb, mqd, mesg)) { gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; STAT(mesq_qf_noop_not_full); return MQIE_AGAIN; } avalue++; } /* Then flip queuehead to other half of queue. */ gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* If not successfully in swapping queue head, clear the hstatus lock */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } return MQIE_AGAIN; cberr: STAT(mesq_qf_unexpected_error); return MQE_UNEXPECTED_CB_ERR; } /* * Handle a PUT failure. Note: if message was a 2-line message, one of the * lines might have successfully have been written. Before sending the * message, "present" must be cleared in BOTH lines to prevent the receiver * from prematurely seeing the full message. */ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { unsigned long m; int ret, loops = 200; /* experimentally determined */ m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; } gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; if (!mqd->interrupt_vector) return MQE_OK; /* * Send a noop message in order to deliver a cross-partition interrupt * to the SSI that contains the target message queue. Normally, the * interrupt is automatically delivered by hardware following mesq * operations, but some error conditions require explicit delivery. * The noop message will trigger delivery. Otherwise partition failures * could cause unrecovered errors. */ do { ret = send_noop_message(cb, mqd, mesg); } while ((ret == MQIE_AGAIN || ret == MQE_CONGESTION) && (loops-- > 0)); if (ret == MQIE_AGAIN || ret == MQE_CONGESTION) { /* * Don't indicate to the app to resend the message, as it's * already been successfully sent. We simply send an OK * (rather than fail the send with MQE_UNEXPECTED_CB_ERR), * assuming that the other side is receiving enough * interrupts to get this message processed anyway. */ ret = MQE_OK; } return ret; }
static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, void *mesg) { const struct message_header noop_header = { .present = MQS_NOOP, .lines = 1}; unsigned long m; int substatus, ret; struct message_header save_mhdr, *mhdr = mesg; STAT(mesq_noop); save_mhdr = *mhdr; *mhdr = noop_header; gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA); ret = gru_wait(cb); if (ret) { substatus = gru_get_cb_message_queue_substatus(cb); switch (substatus) { case CBSS_NO_ERROR: STAT(mesq_noop_unexpected_error); ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_LB_OVERFLOWED: STAT(mesq_noop_lb_overflow); ret = MQE_CONGESTION; break; case CBSS_QLIMIT_REACHED: STAT(mesq_noop_qlimit_reached); ret = 0; break; case CBSS_AMO_NACKED: STAT(mesq_noop_amo_nacked); ret = MQE_CONGESTION; break; case CBSS_PUT_NACKED: STAT(mesq_noop_put_nacked); m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, IMA); if (gru_wait(cb) == CBS_IDLE) ret = MQIE_AGAIN; else ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: STAT(mesq_noop_page_overflow); /* */ default: BUG(); } } *mhdr = save_mhdr; return ret; } /* */ static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { union gru_mesqhead mqh; unsigned int limit, head; unsigned long avalue; int half, qlines; /* */ avalue = gru_get_amo_value(cb); head = gru_get_amo_value_head(cb); limit = gru_get_amo_value_limit(cb); qlines = mqd->qlines; half = (limit != qlines); if (half) mqh = gru_mesq_head(qlines / 2 + 1, qlines); else mqh = gru_mesq_head(2, qlines / 2 + 1); /* */ gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; if (!gru_get_amo_value(cb)) { STAT(mesq_qf_locked); return MQE_QUEUE_FULL; } /* */ if (head != limit) { if (send_noop_message(cb, mqd, mesg)) { gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; STAT(mesq_qf_noop_not_full); return MQIE_AGAIN; } avalue++; } /* */ gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; /* */ if (gru_get_amo_value(cb) != avalue) { STAT(mesq_qf_switch_head_failed); gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA); if (gru_wait(cb) != CBS_IDLE) goto cberr; } return MQIE_AGAIN; cberr: STAT(mesq_qf_unexpected_error); return MQE_UNEXPECTED_CB_ERR; } /* */ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd, void *mesg, int lines) { unsigned long m, *val = mesg, gpa, save; int ret; m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6); if (lines == 2) { gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; } gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); if (gru_wait(cb) != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; if (!mqd->interrupt_vector) return MQE_OK; /* */ gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT); save = *val; *val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector, dest_Fixed); gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA); ret = gru_wait(cb); *val = save; if (ret != CBS_IDLE) return MQE_UNEXPECTED_CB_ERR; return MQE_OK; }