static int s3c24xx_serial_start_tx_dma(struct s3c24xx_uart_port *ourport,
unsigned int count)
{
struct uart_port *port = &ourport->port;
struct circ_buf *xmit = &port->state->xmit;
struct s3c24xx_uart_dma *dma = ourport->dma;
if (ourport->tx_mode != S3C24XX_TX_DMA)
enable_tx_dma(ourport);
dma->tx_size = count & ~(dma_get_cache_alignment() - 1);
dma->tx_transfer_addr = dma->tx_addr + xmit->tail;
dma_sync_single_for_device(ourport->port.dev, dma->tx_transfer_addr,
dma->tx_size, DMA_TO_DEVICE);
dma->tx_desc = dmaengine_prep_slave_single(dma->tx_chan,
dma->tx_transfer_addr, dma->tx_size,
DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT);
if (!dma->tx_desc) {
dev_err(ourport->port.dev, "Unable to get desc for Tx\n");
return -EIO;
}
dma->tx_desc->callback = s3c24xx_serial_tx_dma_complete;
dma->tx_desc->callback_param = ourport;
dma->tx_bytes_requested = dma->tx_size;
ourport->tx_in_progress = S3C24XX_TX_DMA;
dma->tx_cookie = dmaengine_submit(dma->tx_desc);
dma_async_issue_pending(dma->tx_chan);
return 0;
}
static bool spi_qup_can_dma(struct spi_master *master, struct spi_device *spi,
struct spi_transfer *xfer)
{
struct spi_qup *qup = spi_master_get_devdata(master);
size_t dma_align = dma_get_cache_alignment();
u32 mode;
qup->use_dma = 0;
if (xfer->rx_buf && (xfer->len % qup->in_blk_sz ||
IS_ERR_OR_NULL(master->dma_rx) ||
!IS_ALIGNED((size_t)xfer->rx_buf, dma_align)))
return false;
if (xfer->tx_buf && (xfer->len % qup->out_blk_sz ||
IS_ERR_OR_NULL(master->dma_tx) ||
!IS_ALIGNED((size_t)xfer->tx_buf, dma_align)))
return false;
mode = spi_qup_get_mode(master, xfer);
if (mode == QUP_IO_M_MODE_FIFO)
return false;
qup->use_dma = 1;
return true;
}
static void s3c24xx_serial_start_next_tx(struct s3c24xx_uart_port *ourport)
{
struct uart_port *port = &ourport->port;
struct circ_buf *xmit = &port->state->xmit;
unsigned long count;
/* Get data size up to the end of buffer */
count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
if (!count) {
s3c24xx_serial_stop_tx(port);
return;
}
if (!ourport->dma || !ourport->dma->tx_chan ||
count < ourport->min_dma_size ||
xmit->tail & (dma_get_cache_alignment() - 1))
s3c24xx_serial_start_tx_pio(ourport);
else
s3c24xx_serial_start_tx_dma(ourport, count);
}
static void enable_tx_dma(struct s3c24xx_uart_port *ourport)
{
struct uart_port *port = &ourport->port;
u32 ucon;
/* Mask Tx interrupt */
if (s3c24xx_serial_has_interrupt_mask(port))
s3c24xx_set_bit(port, S3C64XX_UINTM_TXD, S3C64XX_UINTM);
else
disable_irq_nosync(ourport->tx_irq);
/* Enable tx dma mode */
ucon = rd_regl(port, S3C2410_UCON);
ucon &= ~(S3C64XX_UCON_TXBURST_MASK | S3C64XX_UCON_TXMODE_MASK);
ucon |= (dma_get_cache_alignment() >= 16) ?
S3C64XX_UCON_TXBURST_16 : S3C64XX_UCON_TXBURST_1;
ucon |= S3C64XX_UCON_TXMODE_DMA;
wr_regl(port, S3C2410_UCON, ucon);
ourport->tx_mode = S3C24XX_TX_DMA;
}
/*
* Read NAND chip to buf. Attempt DMA read for 'large' bufs. Fall-back to
* readsl for small bufs and if FDMA fails to initialise.
*/
static void nand_read_buf_dma(struct mtd_info *mtd, uint8_t *buf, int len)
{
struct nand_chip *chip = mtd->priv;
struct stm_nand_emi *data = chip->priv;
unsigned long dma_align = dma_get_cache_alignment() - 1UL;
int i;
if (len >= 512 &&
(data->dma_chan >= 0 || init_fdma_nand_ratelimit(data) == 0)) {
/* Read up to cache-line boundary */
while ((unsigned long)buf & dma_align) {
*buf++ = readb(chip->IO_ADDR_R);
len--;
}
/* Do DMA transfer, fall-back to readsl if fail */
if (nand_read_dma(mtd, buf, len & ~dma_align, NULL, 0) != 0)
readsl(chip->IO_ADDR_R, buf, (len & ~dma_align)/4);
/* Mop up trailing bytes */
for (i = (len & ~dma_align); i < len; i++)
buf[i] = readb(chip->IO_ADDR_R);
} else {
/* Read buf up to 4-byte boundary */
while ((unsigned int)buf & 0x3) {
*buf++ = readb(chip->IO_ADDR_R);
len--;
}
readsl(chip->IO_ADDR_R, buf, len/4);
/* Mop up trailing bytes */
for (i = (len & ~0x3); i < len; i++)
buf[i] = readb(chip->IO_ADDR_R);
}
}
static int s3c24xx_serial_request_dma(struct s3c24xx_uart_port *p)
{
struct s3c24xx_uart_dma *dma = p->dma;
dma_cap_mask_t mask;
unsigned long flags;
/* Default slave configuration parameters */
dma->rx_conf.direction = DMA_DEV_TO_MEM;
dma->rx_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
dma->rx_conf.src_addr = p->port.mapbase + S3C2410_URXH;
dma->rx_conf.src_maxburst = 16;
dma->tx_conf.direction = DMA_MEM_TO_DEV;
dma->tx_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
dma->tx_conf.dst_addr = p->port.mapbase + S3C2410_UTXH;
if (dma_get_cache_alignment() >= 16)
dma->tx_conf.dst_maxburst = 16;
else
dma->tx_conf.dst_maxburst = 1;
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
dma->rx_chan = dma_request_slave_channel_compat(mask, dma->fn,
dma->rx_param, p->port.dev, "rx");
if (!dma->rx_chan)
return -ENODEV;
dmaengine_slave_config(dma->rx_chan, &dma->rx_conf);
dma->tx_chan = dma_request_slave_channel_compat(mask, dma->fn,
dma->tx_param, p->port.dev, "tx");
if (!dma->tx_chan) {
dma_release_channel(dma->rx_chan);
return -ENODEV;
}
dmaengine_slave_config(dma->tx_chan, &dma->tx_conf);
/* RX buffer */
dma->rx_size = PAGE_SIZE;
dma->rx_buf = kmalloc(dma->rx_size, GFP_KERNEL);
if (!dma->rx_buf) {
dma_release_channel(dma->rx_chan);
dma_release_channel(dma->tx_chan);
return -ENOMEM;
}
dma->rx_addr = dma_map_single(dma->rx_chan->device->dev, dma->rx_buf,
dma->rx_size, DMA_FROM_DEVICE);
spin_lock_irqsave(&p->port.lock, flags);
/* TX buffer */
dma->tx_addr = dma_map_single(dma->tx_chan->device->dev,
p->port.state->xmit.buf,
UART_XMIT_SIZE, DMA_TO_DEVICE);
spin_unlock_irqrestore(&p->port.lock, flags);
return 0;
}
static irqreturn_t s3c24xx_serial_tx_chars(int irq, void *id)
{
struct s3c24xx_uart_port *ourport = id;
struct uart_port *port = &ourport->port;
struct circ_buf *xmit = &port->state->xmit;
unsigned long flags;
int count, dma_count = 0;
spin_lock_irqsave(&port->lock, flags);
count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
if (ourport->dma && ourport->dma->tx_chan &&
count >= ourport->min_dma_size) {
int align = dma_get_cache_alignment() -
(xmit->tail & (dma_get_cache_alignment() - 1));
if (count-align >= ourport->min_dma_size) {
dma_count = count-align;
count = align;
}
}
if (port->x_char) {
wr_regb(port, S3C2410_UTXH, port->x_char);
port->icount.tx++;
port->x_char = 0;
goto out;
}
/* if there isn't anything more to transmit, or the uart is now
* stopped, disable the uart and exit
*/
if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
s3c24xx_serial_stop_tx(port);
goto out;
}
/* try and drain the buffer... */
if (count > port->fifosize) {
count = port->fifosize;
dma_count = 0;
}
while (!uart_circ_empty(xmit) && count > 0) {
if (rd_regl(port, S3C2410_UFSTAT) & ourport->info->tx_fifofull)
break;
wr_regb(port, S3C2410_UTXH, xmit->buf[xmit->tail]);
xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
port->icount.tx++;
count--;
}
if (!count && dma_count) {
s3c24xx_serial_start_tx_dma(ourport, dma_count);
goto out;
}
if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) {
spin_unlock(&port->lock);
uart_write_wakeup(port);
spin_lock(&port->lock);
}
if (uart_circ_empty(xmit))
s3c24xx_serial_stop_tx(port);
out:
spin_unlock_irqrestore(&port->lock, flags);
return IRQ_HANDLED;
}
static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
struct mlx4_init_hca_param *init_hca, u64 icm_size)
{
struct mlx4_priv *priv = mlx4_priv(dev);
u64 aux_pages;
int err;
err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
if (err) {
mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n");
return err;
}
mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n",
(unsigned long long) icm_size >> 10,
(unsigned long long) aux_pages << 2);
priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
GFP_HIGHUSER | __GFP_NOWARN, 0);
if (!priv->fw.aux_icm) {
mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n");
return -ENOMEM;
}
err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
if (err) {
mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n");
goto err_free_aux;
}
err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
if (err) {
mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n");
goto err_unmap_aux;
}
err = mlx4_init_icm_table(dev, &priv->eq_table.table,
init_hca->eqc_base, dev_cap->eqc_entry_sz,
dev->caps.num_eqs, dev->caps.num_eqs,
0, 0);
if (err) {
mlx4_err(dev, "Failed to map EQ context memory, aborting.\n");
goto err_unmap_cmpt;
}
dev->caps.reserved_mtts =
ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
init_hca->mtt_base,
dev->caps.mtt_entry_sz,
dev->caps.num_mtt_segs,
dev->caps.reserved_mtts, 1, 0);
if (err) {
mlx4_err(dev, "Failed to map MTT context memory, aborting.\n");
goto err_unmap_eq;
}
err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
init_hca->dmpt_base,
dev_cap->dmpt_entry_sz,
dev->caps.num_mpts,
dev->caps.reserved_mrws, 1, 1);
if (err) {
mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n");
goto err_unmap_mtt;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
init_hca->qpc_base,
dev_cap->qpc_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
goto err_unmap_dmpt;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
init_hca->auxc_base,
dev_cap->aux_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
goto err_unmap_qp;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
init_hca->altc_base,
dev_cap->altc_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
goto err_unmap_auxc;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
init_hca->rdmarc_base,
dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
goto err_unmap_altc;
}
err = mlx4_init_icm_table(dev, &priv->cq_table.table,
init_hca->cqc_base,
dev_cap->cqc_entry_sz,
dev->caps.num_cqs,
dev->caps.reserved_cqs, 0, 0);
if (err) {
mlx4_err(dev, "Failed to map CQ context memory, aborting.\n");
goto err_unmap_rdmarc;
}
err = mlx4_init_icm_table(dev, &priv->srq_table.table,
init_hca->srqc_base,
dev_cap->srq_entry_sz,
dev->caps.num_srqs,
dev->caps.reserved_srqs, 0, 0);
if (err) {
mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n");
goto err_unmap_cq;
}
err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
init_hca->mc_base, MLX4_MGM_ENTRY_SIZE,
dev->caps.num_mgms + dev->caps.num_amgms,
dev->caps.num_mgms + dev->caps.num_amgms,
0, 0);
if (err) {
mlx4_err(dev, "Failed to map MCG context memory, aborting.\n");
goto err_unmap_srq;
}
return 0;
err_unmap_srq:
mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
err_unmap_cq:
mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
err_unmap_rdmarc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
err_unmap_altc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
err_unmap_auxc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
err_unmap_qp:
mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
err_unmap_dmpt:
mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
err_unmap_mtt:
mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
err_unmap_eq:
mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
err_unmap_cmpt:
mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
err_unmap_aux:
mlx4_UNMAP_ICM_AUX(dev);
err_free_aux:
mlx4_free_icm(dev, priv->fw.aux_icm, 0);
return err;
}
static int spi_qup_probe(struct platform_device *pdev)
{
struct spi_master *master;
struct clk *iclk, *cclk;
struct spi_qup *controller;
struct resource *res;
struct device *dev;
void __iomem *base;
u32 max_freq, iomode, num_cs;
int ret, irq, size;
dev = &pdev->dev;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
base = devm_ioremap_resource(dev, res);
if (IS_ERR(base))
return PTR_ERR(base);
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
cclk = devm_clk_get(dev, "core");
if (IS_ERR(cclk))
return PTR_ERR(cclk);
iclk = devm_clk_get(dev, "iface");
if (IS_ERR(iclk))
return PTR_ERR(iclk);
/* This is optional parameter */
if (of_property_read_u32(dev->of_node, "spi-max-frequency", &max_freq))
max_freq = SPI_MAX_RATE;
if (!max_freq || max_freq > SPI_MAX_RATE) {
dev_err(dev, "invalid clock frequency %d\n", max_freq);
return -ENXIO;
}
ret = clk_prepare_enable(cclk);
if (ret) {
dev_err(dev, "cannot enable core clock\n");
return ret;
}
ret = clk_prepare_enable(iclk);
if (ret) {
clk_disable_unprepare(cclk);
dev_err(dev, "cannot enable iface clock\n");
return ret;
}
master = spi_alloc_master(dev, sizeof(struct spi_qup));
if (!master) {
clk_disable_unprepare(cclk);
clk_disable_unprepare(iclk);
dev_err(dev, "cannot allocate master\n");
return -ENOMEM;
}
/* use num-cs unless not present or out of range */
if (of_property_read_u32(dev->of_node, "num-cs", &num_cs) ||
num_cs > SPI_NUM_CHIPSELECTS)
master->num_chipselect = SPI_NUM_CHIPSELECTS;
else
master->num_chipselect = num_cs;
master->bus_num = pdev->id;
master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LOOP;
master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
master->max_speed_hz = max_freq;
master->transfer_one = spi_qup_transfer_one;
master->dev.of_node = pdev->dev.of_node;
master->auto_runtime_pm = true;
master->dma_alignment = dma_get_cache_alignment();
master->max_dma_len = SPI_MAX_DMA_XFER;
platform_set_drvdata(pdev, master);
controller = spi_master_get_devdata(master);
controller->dev = dev;
controller->base = base;
controller->iclk = iclk;
controller->cclk = cclk;
controller->irq = irq;
ret = spi_qup_init_dma(master, res->start);
if (ret == -EPROBE_DEFER)
goto error;
else if (!ret)
master->can_dma = spi_qup_can_dma;
/* set v1 flag if device is version 1 */
if (of_device_is_compatible(dev->of_node, "qcom,spi-qup-v1.1.1"))
controller->qup_v1 = 1;
spin_lock_init(&controller->lock);
init_completion(&controller->done);
iomode = readl_relaxed(base + QUP_IO_M_MODES);
size = QUP_IO_M_OUTPUT_BLOCK_SIZE(iomode);
if (size)
controller->out_blk_sz = size * 16;
else
controller->out_blk_sz = 4;
size = QUP_IO_M_INPUT_BLOCK_SIZE(iomode);
if (size)
controller->in_blk_sz = size * 16;
else
controller->in_blk_sz = 4;
size = QUP_IO_M_OUTPUT_FIFO_SIZE(iomode);
controller->out_fifo_sz = controller->out_blk_sz * (2 << size);
size = QUP_IO_M_INPUT_FIFO_SIZE(iomode);
controller->in_fifo_sz = controller->in_blk_sz * (2 << size);
dev_info(dev, "IN:block:%d, fifo:%d, OUT:block:%d, fifo:%d\n",
controller->in_blk_sz, controller->in_fifo_sz,
controller->out_blk_sz, controller->out_fifo_sz);
writel_relaxed(1, base + QUP_SW_RESET);
ret = spi_qup_set_state(controller, QUP_STATE_RESET);
if (ret) {
dev_err(dev, "cannot set RESET state\n");
goto error_dma;
}
writel_relaxed(0, base + QUP_OPERATIONAL);
writel_relaxed(0, base + QUP_IO_M_MODES);
if (!controller->qup_v1)
writel_relaxed(0, base + QUP_OPERATIONAL_MASK);
writel_relaxed(SPI_ERROR_CLK_UNDER_RUN | SPI_ERROR_CLK_OVER_RUN,
base + SPI_ERROR_FLAGS_EN);
/* if earlier version of the QUP, disable INPUT_OVERRUN */
if (controller->qup_v1)
writel_relaxed(QUP_ERROR_OUTPUT_OVER_RUN |
QUP_ERROR_INPUT_UNDER_RUN | QUP_ERROR_OUTPUT_UNDER_RUN,
base + QUP_ERROR_FLAGS_EN);
writel_relaxed(0, base + SPI_CONFIG);
writel_relaxed(SPI_IO_C_NO_TRI_STATE, base + SPI_IO_CONTROL);
ret = devm_request_irq(dev, irq, spi_qup_qup_irq,
IRQF_TRIGGER_HIGH, pdev->name, controller);
if (ret)
goto error_dma;
pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC);
pm_runtime_use_autosuspend(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
ret = devm_spi_register_master(dev, master);
if (ret)
goto disable_pm;
return 0;
disable_pm:
pm_runtime_disable(&pdev->dev);
error_dma:
spi_qup_release_dma(master);
error:
clk_disable_unprepare(cclk);
clk_disable_unprepare(iclk);
spi_master_put(master);
return ret;
}
static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
struct mlx4_init_hca_param *init_hca, u64 icm_size)
{
struct mlx4_priv *priv = mlx4_priv(dev);
u64 aux_pages;
int err;
err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
if (err) {
mlx4_err(dev, "SET_ICM_SIZE command failed, aborting.\n");
return err;
}
mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory.\n",
(unsigned long long) icm_size >> 10,
(unsigned long long) aux_pages << 2);
priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
GFP_HIGHUSER | __GFP_NOWARN, 0);
if (!priv->fw.aux_icm) {
mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n");
return -ENOMEM;
}
err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
if (err) {
mlx4_err(dev, "MAP_ICM_AUX command failed, aborting.\n");
goto err_free_aux;
}
err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
if (err) {
mlx4_err(dev, "Failed to map cMPT context memory, aborting.\n");
goto err_unmap_aux;
}
err = mlx4_init_icm_table(dev, &priv->eq_table.table,
init_hca->eqc_base, dev_cap->eqc_entry_sz,
dev->caps.num_eqs, dev->caps.num_eqs,
0, 0);
if (err) {
mlx4_err(dev, "Failed to map EQ context memory, aborting.\n");
goto err_unmap_cmpt;
}
/*
* Reserved MTT entries must be aligned up to a cacheline
* boundary, since the FW will write to them, while the driver
* writes to all other MTT entries. (The variable
* dev->caps.mtt_entry_sz below is really the MTT segment
* size, not the raw entry size)
*/
dev->caps.reserved_mtts =
ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
init_hca->mtt_base,
dev->caps.mtt_entry_sz,
dev->caps.num_mtt_segs,
dev->caps.reserved_mtts, 1, 0);
if (err) {
mlx4_err(dev, "Failed to map MTT context memory, aborting.\n");
goto err_unmap_eq;
}
err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
init_hca->dmpt_base,
dev_cap->dmpt_entry_sz,
dev->caps.num_mpts,
dev->caps.reserved_mrws, 1, 1);
if (err) {
mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n");
goto err_unmap_mtt;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
init_hca->qpc_base,
dev_cap->qpc_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
goto err_unmap_dmpt;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
init_hca->auxc_base,
dev_cap->aux_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
goto err_unmap_qp;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
init_hca->altc_base,
dev_cap->altc_entry_sz,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
goto err_unmap_auxc;
}
err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
init_hca->rdmarc_base,
dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
dev->caps.num_qps,
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
0, 0);
if (err) {
mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
goto err_unmap_altc;
}
err = mlx4_init_icm_table(dev, &priv->cq_table.table,
init_hca->cqc_base,
dev_cap->cqc_entry_sz,
dev->caps.num_cqs,
dev->caps.reserved_cqs, 0, 0);
if (err) {
mlx4_err(dev, "Failed to map CQ context memory, aborting.\n");
goto err_unmap_rdmarc;
}
err = mlx4_init_icm_table(dev, &priv->srq_table.table,
init_hca->srqc_base,
dev_cap->srq_entry_sz,
dev->caps.num_srqs,
dev->caps.reserved_srqs, 0, 0);
if (err) {
mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n");
goto err_unmap_cq;
}
/*
* It's not strictly required, but for simplicity just map the
* whole multicast group table now. The table isn't very big
* and it's a lot easier than trying to track ref counts.
*/
err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
init_hca->mc_base, MLX4_MGM_ENTRY_SIZE,
dev->caps.num_mgms + dev->caps.num_amgms,
dev->caps.num_mgms + dev->caps.num_amgms,
0, 0);
if (err) {
mlx4_err(dev, "Failed to map MCG context memory, aborting.\n");
goto err_unmap_srq;
}
return 0;
err_unmap_srq:
mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
err_unmap_cq:
mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
err_unmap_rdmarc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
err_unmap_altc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
err_unmap_auxc:
mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
err_unmap_qp:
mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
err_unmap_dmpt:
mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
err_unmap_mtt:
mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
err_unmap_eq:
mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
err_unmap_cmpt:
mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
err_unmap_aux:
mlx4_UNMAP_ICM_AUX(dev);
err_free_aux:
mlx4_free_icm(dev, priv->fw.aux_icm, 0);
return err;
}
static int mthca_init_icm(struct mthca_dev *mdev,
struct mthca_dev_lim *dev_lim,
struct mthca_init_hca_param *init_hca,
u64 icm_size)
{
u64 aux_pages;
u8 status;
int err;
err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
if (err) {
mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
return err;
}
if (status) {
mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
"aborting.\n", status);
return -EINVAL;
}
mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
(unsigned long long) icm_size >> 10,
(unsigned long long) aux_pages << 2);
mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
GFP_HIGHUSER | __GFP_NOWARN, 0);
if (!mdev->fw.arbel.aux_icm) {
mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
return -ENOMEM;
}
err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
if (err) {
mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
goto err_free_aux;
}
if (status) {
mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
err = -EINVAL;
goto err_free_aux;
}
err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
if (err) {
mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
goto err_unmap_aux;
}
/* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */
mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * mdev->limits.mtt_seg_size,
dma_get_cache_alignment()) / mdev->limits.mtt_seg_size;
mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
mdev->limits.mtt_seg_size,
mdev->limits.num_mtt_segs,
mdev->limits.reserved_mtts,
1, 0);
if (!mdev->mr_table.mtt_table) {
mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_eq;
}
mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
dev_lim->mpt_entry_sz,
mdev->limits.num_mpts,
mdev->limits.reserved_mrws,
1, 1);
if (!mdev->mr_table.mpt_table) {
mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_mtt;
}
mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
dev_lim->qpc_entry_sz,
mdev->limits.num_qps,
mdev->limits.reserved_qps,
0, 0);
if (!mdev->qp_table.qp_table) {
mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_mpt;
}
mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
dev_lim->eqpc_entry_sz,
mdev->limits.num_qps,
mdev->limits.reserved_qps,
0, 0);
if (!mdev->qp_table.eqp_table) {
mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_qp;
}
mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
MTHCA_RDB_ENTRY_SIZE,
mdev->limits.num_qps <<
mdev->qp_table.rdb_shift, 0,
0, 0);
if (!mdev->qp_table.rdb_table) {
mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
err = -ENOMEM;
goto err_unmap_eqp;
}
mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
dev_lim->cqc_entry_sz,
mdev->limits.num_cqs,
mdev->limits.reserved_cqs,
0, 0);
if (!mdev->cq_table.table) {
mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_rdb;
}
if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
mdev->srq_table.table =
mthca_alloc_icm_table(mdev, init_hca->srqc_base,
dev_lim->srq_entry_sz,
mdev->limits.num_srqs,
mdev->limits.reserved_srqs,
0, 0);
if (!mdev->srq_table.table) {
mthca_err(mdev, "Failed to map SRQ context memory, "
"aborting.\n");
err = -ENOMEM;
goto err_unmap_cq;
}
}
/*
* It's not strictly required, but for simplicity just map the
* whole multicast group table now. The table isn't very big
* and it's a lot easier than trying to track ref counts.
*/
mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
MTHCA_MGM_ENTRY_SIZE,
mdev->limits.num_mgms +
mdev->limits.num_amgms,
mdev->limits.num_mgms +
mdev->limits.num_amgms,
0, 0);
if (!mdev->mcg_table.table) {
mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
err = -ENOMEM;
goto err_unmap_srq;
}
return 0;
err_unmap_srq:
if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
mthca_free_icm_table(mdev, mdev->srq_table.table);
err_unmap_cq:
mthca_free_icm_table(mdev, mdev->cq_table.table);
err_unmap_rdb:
mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
err_unmap_eqp:
mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
err_unmap_qp:
mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
err_unmap_mpt:
mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
err_unmap_mtt:
mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
err_unmap_eq:
mthca_unmap_eq_icm(mdev);
err_unmap_aux:
mthca_UNMAP_ICM_AUX(mdev, &status);
err_free_aux:
mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0);
return err;
}
