static long ehca_plpar_hcall9(unsigned long opcode,
unsigned long *outs, /* array of 9 outputs */
unsigned long arg1,
unsigned long arg2,
unsigned long arg3,
unsigned long arg4,
unsigned long arg5,
unsigned long arg6,
unsigned long arg7,
unsigned long arg8,
unsigned long arg9)
{
long ret;
int i, sleep_msecs;
ehca_gen_dbg("opcode=%lx arg1=%lx arg2=%lx arg3=%lx arg4=%lx "
"arg5=%lx arg6=%lx arg7=%lx arg8=%lx arg9=%lx",
opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7,
arg8, arg9);
for (i = 0; i < 5; i++) {
ret = plpar_hcall9(opcode, outs,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9);
if (H_IS_LONG_BUSY(ret)) {
sleep_msecs = get_longbusy_msecs(ret);
msleep_interruptible(sleep_msecs);
continue;
}
if (ret < H_SUCCESS)
ehca_gen_err("opcode=%lx ret=%lx"
" arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
" arg5=%lx arg6=%lx arg7=%lx arg8=%lx"
" arg9=%lx"
" out1=%lx out2=%lx out3=%lx out4=%lx"
" out5=%lx out6=%lx out7=%lx out8=%lx"
" out9=%lx",
opcode, ret,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9,
outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
outs[8]);
ehca_gen_dbg("opcode=%lx ret=%lx out1=%lx out2=%lx out3=%lx "
"out4=%lx out5=%lx out6=%lx out7=%lx out8=%lx "
"out9=%lx",
opcode, ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7], outs[8]);
return ret;
}
return H_BUSY;
}
static long ehca_plpar_hcall9(unsigned long opcode,
unsigned long *outs,
unsigned long arg1,
unsigned long arg2,
unsigned long arg3,
unsigned long arg4,
unsigned long arg5,
unsigned long arg6,
unsigned long arg7,
unsigned long arg8,
unsigned long arg9)
{
long ret;
int i, sleep_msecs;
unsigned long flags = 0;
if (unlikely(ehca_debug_level >= 2))
ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9);
for (i = 0; i < 5; i++) {
if (ehca_lock_hcalls)
spin_lock_irqsave(&hcall_lock, flags);
ret = plpar_hcall9(opcode, outs,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9);
if (ehca_lock_hcalls)
spin_unlock_irqrestore(&hcall_lock, flags);
if (H_IS_LONG_BUSY(ret)) {
sleep_msecs = get_longbusy_msecs(ret);
msleep_interruptible(sleep_msecs);
continue;
}
if (ret < H_SUCCESS) {
ehca_gen_err("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT,
opcode, arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9);
ehca_gen_err("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
outs[8]);
} else if (unlikely(ehca_debug_level >= 2))
ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
outs[8]);
return ret;
}
return H_BUSY;
}
static long ehca_plpar_hcall_norets(unsigned long opcode,
unsigned long arg1,
unsigned long arg2,
unsigned long arg3,
unsigned long arg4,
unsigned long arg5,
unsigned long arg6,
unsigned long arg7)
{
long ret;
int i, sleep_msecs;
unsigned long flags = 0;
if (unlikely(ehca_debug_level >= 2))
ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
for (i = 0; i < 5; i++) {
if (ehca_lock_hcalls)
spin_lock_irqsave(&hcall_lock, flags);
ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
arg5, arg6, arg7);
if (ehca_lock_hcalls)
spin_unlock_irqrestore(&hcall_lock, flags);
if (H_IS_LONG_BUSY(ret)) {
sleep_msecs = get_longbusy_msecs(ret);
msleep_interruptible(sleep_msecs);
continue;
}
if (ret < H_SUCCESS)
ehca_gen_err("opcode=%lx ret=%li " HCALL7_REGS_FORMAT,
opcode, ret, arg1, arg2, arg3,
arg4, arg5, arg6, arg7);
else
if (unlikely(ehca_debug_level >= 2))
ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
return ret;
}
return H_BUSY;
}
static long ehca_plpar_hcall_norets(unsigned long opcode,
unsigned long arg1,
unsigned long arg2,
unsigned long arg3,
unsigned long arg4,
unsigned long arg5,
unsigned long arg6,
unsigned long arg7)
{
long ret;
int i, sleep_msecs;
ehca_gen_dbg("opcode=%lx arg1=%lx arg2=%lx arg3=%lx arg4=%lx "
"arg5=%lx arg6=%lx arg7=%lx",
opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
for (i = 0; i < 5; i++) {
ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
arg5, arg6, arg7);
if (H_IS_LONG_BUSY(ret)) {
sleep_msecs = get_longbusy_msecs(ret);
msleep_interruptible(sleep_msecs);
continue;
}
if (ret < H_SUCCESS)
ehca_gen_err("opcode=%lx ret=%lx"
" arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
" arg5=%lx arg6=%lx arg7=%lx ",
opcode, ret,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7);
ehca_gen_dbg("opcode=%lx ret=%lx", opcode, ret);
return ret;
}
return H_BUSY;
}
u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
const struct ehca_mr *mr,
const u8 pagesize,
const u8 queue_type,
const u64 logical_address_of_page,
const u64 count)
{
u64 ret;
if (unlikely(ehca_debug_level >= 3)) {
if (count > 1) {
u64 *kpage;
int i;
kpage = (u64 *)abs_to_virt(logical_address_of_page);
for (i = 0; i < count; i++)
ehca_gen_dbg("kpage[%d]=%p",
i, (void *)kpage[i]);
} else
ehca_gen_dbg("kpage=%p",
(void *)logical_address_of_page);
}
if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
ehca_gen_err("logical_address_of_page not on a 4k boundary "
"adapter_handle=%llx mr=%p mr_handle=%llx "
"pagesize=%x queue_type=%x "
"logical_address_of_page=%llx count=%llx",
adapter_handle.handle, mr,
mr->ipz_mr_handle.handle, pagesize, queue_type,
logical_address_of_page, count);
ret = H_PARAMETER;
} else
ret = hipz_h_register_rpage(adapter_handle, pagesize,
queue_type,
mr->ipz_mr_handle.handle,
logical_address_of_page, count);
return ret;
}
static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
{
int idx;
int j;
while (send_wr) {
struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
struct ib_sge *sge = send_wr->sg_list;
ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
"send_flags=%x opcode=%x", idx, send_wr->wr_id,
send_wr->num_sge, send_wr->send_flags,
send_wr->opcode);
if (mad_hdr) {
ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
"mgmt_class=%x class_version=%x method=%x "
"status=%x class_specific=%x tid=%lx "
"attr_id=%x resv=%x attr_mod=%x",
idx, mad_hdr->base_version,
mad_hdr->mgmt_class,
mad_hdr->class_version, mad_hdr->method,
mad_hdr->status, mad_hdr->class_specific,
mad_hdr->tid, mad_hdr->attr_id,
mad_hdr->resv,
mad_hdr->attr_mod);
}
for (j = 0; j < send_wr->num_sge; j++) {
u8 *data = (u8 *)abs_to_virt(sge->addr);
ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
"lkey=%x",
idx, j, data, sge->length, sge->lkey);
/* assume length is n*16 */
ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
idx, j);
sge++;
} /* eof for j */
idx++;
send_wr = send_wr->next;
} /* eof while send_wr */
}
int ipz_queue_dtor(struct ipz_queue *queue)
{
int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT;
int g;
int nr_pages;
if (!queue || !queue->queue_pages) {
ehca_gen_dbg("queue or queue_pages is NULL");
return 0;
}
nr_pages = queue->queue_length / queue->pagesize;
for (g = 0; g < nr_pages; g += pages_per_kpage)
free_page((unsigned long)(queue->queue_pages)[g]);
vfree(queue->queue_pages);
return 1;
}
static int __cpuinit comp_pool_callback(struct notifier_block *nfb,
unsigned long action,
void *hcpu)
{
unsigned int cpu = (unsigned long)hcpu;
struct ehca_cpu_comp_task *cct;
switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_PREPARE)", cpu);
if (!create_comp_task(pool, cpu)) {
ehca_gen_err("Can't create comp_task for cpu: %x", cpu);
return NOTIFY_BAD;
}
break;
case CPU_UP_CANCELED:
case CPU_UP_CANCELED_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_CANCELED)", cpu);
cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
kthread_bind(cct->task, cpumask_any(cpu_online_mask));
destroy_comp_task(pool, cpu);
break;
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_ONLINE)", cpu);
cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
kthread_bind(cct->task, cpu);
wake_up_process(cct->task);
break;
case CPU_DOWN_PREPARE:
case CPU_DOWN_PREPARE_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_DOWN_PREPARE)", cpu);
break;
case CPU_DOWN_FAILED:
case CPU_DOWN_FAILED_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_DOWN_FAILED)", cpu);
break;
case CPU_DEAD:
case CPU_DEAD_FROZEN:
ehca_gen_dbg("CPU: %x (CPU_DEAD)", cpu);
destroy_comp_task(pool, cpu);
take_over_work(pool, cpu);
break;
}
return NOTIFY_OK;
}
static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
struct ehca_wqe *wqe_p,
struct ib_recv_wr *recv_wr,
u32 rq_map_idx)
{
u8 cnt_ds;
if (unlikely((recv_wr->num_sge < 0) ||
(recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
ehca_gen_err("Invalid number of WQE SGE. "
"num_sqe=%x max_nr_of_sg=%x",
recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
return -EINVAL; /* invalid SG list length */
}
/* clear wqe header until sglist */
memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
wqe_p->nr_of_data_seg = recv_wr->num_sge;
for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
recv_wr->sg_list[cnt_ds].addr;
wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
recv_wr->sg_list[cnt_ds].lkey;
wqe_p->u.all_rcv.sg_list[cnt_ds].length =
recv_wr->sg_list[cnt_ds].length;
}
if (ehca_debug_level >= 3) {
ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
ipz_rqueue);
ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
}
return 0;
}
static inline int ehca_write_swqe(struct ehca_qp *qp,
struct ehca_wqe *wqe_p,
const struct ib_send_wr *send_wr)
{
u32 idx;
u64 dma_length;
struct ehca_av *my_av;
u32 remote_qkey = send_wr->wr.ud.remote_qkey;
if (unlikely((send_wr->num_sge < 0) ||
(send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
ehca_gen_err("Invalid number of WQE SGE. "
"num_sqe=%x max_nr_of_sg=%x",
send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
return -EINVAL; /* invalid SG list length */
}
/* clear wqe header until sglist */
memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
wqe_p->work_request_id = send_wr->wr_id;
switch (send_wr->opcode) {
case IB_WR_SEND:
case IB_WR_SEND_WITH_IMM:
wqe_p->optype = WQE_OPTYPE_SEND;
break;
case IB_WR_RDMA_WRITE:
case IB_WR_RDMA_WRITE_WITH_IMM:
wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
break;
case IB_WR_RDMA_READ:
wqe_p->optype = WQE_OPTYPE_RDMAREAD;
break;
default:
ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
return -EINVAL; /* invalid opcode */
}
wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
wqe_p->wr_flag = 0;
if (send_wr->send_flags & IB_SEND_SIGNALED)
wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
/* this might not work as long as HW does not support it */
wqe_p->immediate_data = be32_to_cpu(send_wr->imm_data);
wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
}
wqe_p->nr_of_data_seg = send_wr->num_sge;
switch (qp->qp_type) {
case IB_QPT_SMI:
case IB_QPT_GSI:
/* no break is intential here */
case IB_QPT_UD:
/* IB 1.2 spec C10-15 compliance */
if (send_wr->wr.ud.remote_qkey & 0x80000000)
remote_qkey = qp->qkey;
wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
wqe_p->local_ee_context_qkey = remote_qkey;
if (!send_wr->wr.ud.ah) {
ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
return -EINVAL;
}
my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
wqe_p->u.ud_av.ud_av = my_av->av;
/*
* omitted check of IB_SEND_INLINE
* since HW does not support it
*/
for (idx = 0; idx < send_wr->num_sge; idx++) {
wqe_p->u.ud_av.sg_list[idx].vaddr =
send_wr->sg_list[idx].addr;
wqe_p->u.ud_av.sg_list[idx].lkey =
send_wr->sg_list[idx].lkey;
wqe_p->u.ud_av.sg_list[idx].length =
send_wr->sg_list[idx].length;
} /* eof for idx */
if (qp->qp_type == IB_QPT_SMI ||
qp->qp_type == IB_QPT_GSI)
wqe_p->u.ud_av.ud_av.pmtu = 1;
if (qp->qp_type == IB_QPT_GSI) {
wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
#ifdef DEBUG_GSI_SEND_WR
trace_send_wr_ud(send_wr);
#endif /* DEBUG_GSI_SEND_WR */
}
break;
case IB_QPT_UC:
if (send_wr->send_flags & IB_SEND_FENCE)
wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
/* no break is intentional here */
case IB_QPT_RC:
/* TODO: atomic not implemented */
wqe_p->u.nud.remote_virtual_adress =
send_wr->wr.rdma.remote_addr;
wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
/*
* omitted checking of IB_SEND_INLINE
* since HW does not support it
*/
dma_length = 0;
for (idx = 0; idx < send_wr->num_sge; idx++) {
wqe_p->u.nud.sg_list[idx].vaddr =
send_wr->sg_list[idx].addr;
wqe_p->u.nud.sg_list[idx].lkey =
send_wr->sg_list[idx].lkey;
wqe_p->u.nud.sg_list[idx].length =
send_wr->sg_list[idx].length;
dma_length += send_wr->sg_list[idx].length;
} /* eof idx */
wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
break;
default:
ehca_gen_err("Invalid qptype=%x", qp->qp_type);
return -EINVAL;
}
if (ehca_debug_level) {
ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
}
return 0;
}
static long ehca_plpar_hcall9(unsigned long opcode,
unsigned long *outs, /* array of 9 outputs */
unsigned long arg1,
unsigned long arg2,
unsigned long arg3,
unsigned long arg4,
unsigned long arg5,
unsigned long arg6,
unsigned long arg7,
unsigned long arg8,
unsigned long arg9)
{
long ret;
int i, sleep_msecs, lock_is_set = 0;
unsigned long flags;
ehca_gen_dbg("opcode=%lx arg1=%lx arg2=%lx arg3=%lx arg4=%lx "
"arg5=%lx arg6=%lx arg7=%lx arg8=%lx arg9=%lx",
opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7,
arg8, arg9);
for (i = 0; i < 5; i++) {
if ((opcode == H_ALLOC_RESOURCE) && (arg2 == 5)) {
spin_lock_irqsave(&hcall_lock, flags);
lock_is_set = 1;
}
ret = plpar_hcall9(opcode, outs,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9);
if (lock_is_set)
spin_unlock_irqrestore(&hcall_lock, flags);
if (H_IS_LONG_BUSY(ret)) {
sleep_msecs = get_longbusy_msecs(ret);
msleep_interruptible(sleep_msecs);
continue;
}
if (ret < H_SUCCESS)
ehca_gen_err("opcode=%lx ret=%lx"
" arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
" arg5=%lx arg6=%lx arg7=%lx arg8=%lx"
" arg9=%lx"
" out1=%lx out2=%lx out3=%lx out4=%lx"
" out5=%lx out6=%lx out7=%lx out8=%lx"
" out9=%lx",
opcode, ret,
arg1, arg2, arg3, arg4, arg5,
arg6, arg7, arg8, arg9,
outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7],
outs[8]);
ehca_gen_dbg("opcode=%lx ret=%lx out1=%lx out2=%lx out3=%lx "
"out4=%lx out5=%lx out6=%lx out7=%lx out8=%lx "
"out9=%lx",
opcode, ret, outs[0], outs[1], outs[2], outs[3],
outs[4], outs[5], outs[6], outs[7], outs[8]);
return ret;
}
return H_BUSY;
}
