diff options
Diffstat (limited to 'drivers/dma')
-rw-r--r-- | drivers/dma/Kconfig | 10 | ||||
-rw-r--r-- | drivers/dma/Makefile | 1 | ||||
-rw-r--r-- | drivers/dma/amba-pl08x.c | 23 | ||||
-rw-r--r-- | drivers/dma/coh901318.c | 26 | ||||
-rw-r--r-- | drivers/dma/cppi41.c | 1059 | ||||
-rw-r--r-- | drivers/dma/dmaengine.c | 81 | ||||
-rw-r--r-- | drivers/dma/dmatest.c | 182 | ||||
-rw-r--r-- | drivers/dma/ioat/dma_v3.c | 26 | ||||
-rw-r--r-- | drivers/dma/mv_xor.c | 53 | ||||
-rw-r--r-- | drivers/dma/mv_xor.h | 28 | ||||
-rw-r--r-- | drivers/dma/sh/shdmac.c | 4 |
11 files changed, 1221 insertions, 272 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 2945ff084a0..526ec77c7ba 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -194,7 +194,7 @@ config SIRF_DMA Enable support for the CSR SiRFprimaII DMA engine. config TI_EDMA - tristate "TI EDMA support" + bool "TI EDMA support" depends on ARCH_DAVINCI || ARCH_OMAP select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -287,6 +287,14 @@ config DMA_OMAP select DMA_ENGINE select DMA_VIRTUAL_CHANNELS +config TI_CPPI41 + tristate "AM33xx CPPI41 DMA support" + depends on ARCH_OMAP + select DMA_ENGINE + help + The Communications Port Programming Interface (CPPI) 4.1 DMA engine + is currently used by the USB driver on AM335x platforms. + config MMP_PDMA bool "MMP PDMA support" depends on (ARCH_MMP || ARCH_PXA) diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 8c97941df27..db89035b362 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -39,4 +39,5 @@ obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o obj-$(CONFIG_DMA_OMAP) += omap-dma.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o +obj-$(CONFIG_TI_CPPI41) += cppi41.o obj-$(CONFIG_K3_DMA) += k3dma.o diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index cd294340c85..fce46c5bf1c 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -131,6 +131,8 @@ struct pl08x_bus_data { u8 buswidth; }; +#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) + /** * struct pl08x_phy_chan - holder for the physical channels * @id: physical index to this channel @@ -961,10 +963,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl); - dev_vdbg(&pl08x->adev->dev, "src=0x%08x%s/%u dst=0x%08x%s/%u len=%zu\n", - bd.srcbus.addr, cctl & PL080_CONTROL_SRC_INCR ? "+" : "", + dev_vdbg(&pl08x->adev->dev, + "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", + (u64)bd.srcbus.addr, + cctl & PL080_CONTROL_SRC_INCR ? "+" : "", bd.srcbus.buswidth, - bd.dstbus.addr, cctl & PL080_CONTROL_DST_INCR ? "+" : "", + (u64)bd.dstbus.addr, + cctl & PL080_CONTROL_DST_INCR ? "+" : "", bd.dstbus.buswidth, bd.remainder); dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", @@ -1002,8 +1007,8 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, return 0; } - if ((bd.srcbus.addr % bd.srcbus.buswidth) || - (bd.dstbus.addr % bd.dstbus.buswidth)) { + if (!IS_BUS_ALIGNED(&bd.srcbus) || + !IS_BUS_ALIGNED(&bd.dstbus)) { dev_err(&pl08x->adev->dev, "%s src & dst address must be aligned to src" " & dst width if peripheral is flow controller", @@ -1025,9 +1030,9 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, */ if (bd.remainder < mbus->buswidth) early_bytes = bd.remainder; - else if ((mbus->addr) % (mbus->buswidth)) { - early_bytes = mbus->buswidth - (mbus->addr) % - (mbus->buswidth); + else if (!IS_BUS_ALIGNED(mbus)) { + early_bytes = mbus->buswidth - + (mbus->addr & (mbus->buswidth - 1)); if ((bd.remainder - early_bytes) < mbus->buswidth) early_bytes = bd.remainder; } @@ -1045,7 +1050,7 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, * Master now aligned * - if slave is not then we must set its width down */ - if (sbus->addr % sbus->buswidth) { + if (!IS_BUS_ALIGNED(sbus)) { dev_dbg(&pl08x->adev->dev, "%s set down bus width to one byte\n", __func__); diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index 9bfaddd57ef..31011d2a26f 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -1339,15 +1339,14 @@ static int coh901318_debugfs_read(struct file *file, char __user *buf, { u64 started_channels = debugfs_dma_base->pm.started_channels; int pool_count = debugfs_dma_base->pool.debugfs_pool_counter; - int i; - int ret = 0; char *dev_buf; char *tmp; - int dev_size; + int ret; + int i; dev_buf = kmalloc(4*1024, GFP_KERNEL); if (dev_buf == NULL) - goto err_kmalloc; + return -ENOMEM; tmp = dev_buf; tmp += sprintf(tmp, "DMA -- enabled dma channels\n"); @@ -1357,26 +1356,11 @@ static int coh901318_debugfs_read(struct file *file, char __user *buf, tmp += sprintf(tmp, "channel %d\n", i); tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count); - dev_size = tmp - dev_buf; - - /* No more to read if offset != 0 */ - if (*f_pos > dev_size) - goto out; - if (count > dev_size - *f_pos) - count = dev_size - *f_pos; - - if (copy_to_user(buf, dev_buf + *f_pos, count)) - ret = -EINVAL; - ret = count; - *f_pos += count; - - out: + ret = simple_read_from_buffer(buf, count, f_pos, dev_buf, + tmp - dev_buf); kfree(dev_buf); return ret; - - err_kmalloc: - return 0; } static const struct file_operations coh901318_debugfs_status_operations = { diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c new file mode 100644 index 00000000000..7c82b92f9b1 --- /dev/null +++ b/drivers/dma/cppi41.c @@ -0,0 +1,1059 @@ +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/slab.h> +#include <linux/of_dma.h> +#include <linux/of_irq.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/of_address.h> +#include <linux/pm_runtime.h> +#include "dmaengine.h" + +#define DESC_TYPE 27 +#define DESC_TYPE_HOST 0x10 +#define DESC_TYPE_TEARD 0x13 + +#define TD_DESC_IS_RX (1 << 16) +#define TD_DESC_DMA_NUM 10 + +#define DESC_LENGTH_BITS_NUM 21 + +#define DESC_TYPE_USB (5 << 26) +#define DESC_PD_COMPLETE (1 << 31) + +/* DMA engine */ +#define DMA_TDFDQ 4 +#define DMA_TXGCR(x) (0x800 + (x) * 0x20) +#define DMA_RXGCR(x) (0x808 + (x) * 0x20) +#define RXHPCRA0 4 + +#define GCR_CHAN_ENABLE (1 << 31) +#define GCR_TEARDOWN (1 << 30) +#define GCR_STARV_RETRY (1 << 24) +#define GCR_DESC_TYPE_HOST (1 << 14) + +/* DMA scheduler */ +#define DMA_SCHED_CTRL 0 +#define DMA_SCHED_CTRL_EN (1 << 31) +#define DMA_SCHED_WORD(x) ((x) * 4 + 0x800) + +#define SCHED_ENTRY0_CHAN(x) ((x) << 0) +#define SCHED_ENTRY0_IS_RX (1 << 7) + +#define SCHED_ENTRY1_CHAN(x) ((x) << 8) +#define SCHED_ENTRY1_IS_RX (1 << 15) + +#define SCHED_ENTRY2_CHAN(x) ((x) << 16) +#define SCHED_ENTRY2_IS_RX (1 << 23) + +#define SCHED_ENTRY3_CHAN(x) ((x) << 24) +#define SCHED_ENTRY3_IS_RX (1 << 31) + +/* Queue manager */ +/* 4 KiB of memory for descriptors, 2 for each endpoint */ +#define ALLOC_DECS_NUM 128 +#define DESCS_AREAS 1 +#define TOTAL_DESCS_NUM (ALLOC_DECS_NUM * DESCS_AREAS) +#define QMGR_SCRATCH_SIZE (TOTAL_DESCS_NUM * 4) + +#define QMGR_LRAM0_BASE 0x80 +#define QMGR_LRAM_SIZE 0x84 +#define QMGR_LRAM1_BASE 0x88 +#define QMGR_MEMBASE(x) (0x1000 + (x) * 0x10) +#define QMGR_MEMCTRL(x) (0x1004 + (x) * 0x10) +#define QMGR_MEMCTRL_IDX_SH 16 +#define QMGR_MEMCTRL_DESC_SH 8 + +#define QMGR_NUM_PEND 5 +#define QMGR_PEND(x) (0x90 + (x) * 4) + +#define QMGR_PENDING_SLOT_Q(x) (x / 32) +#define QMGR_PENDING_BIT_Q(x) (x % 32) + +#define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10) +#define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10) +#define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10) +#define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10) + +/* Glue layer specific */ +/* USBSS / USB AM335x */ +#define USBSS_IRQ_STATUS 0x28 +#define USBSS_IRQ_ENABLER 0x2c +#define USBSS_IRQ_CLEARR 0x30 + +#define USBSS_IRQ_PD_COMP (1 << 2) + +struct cppi41_channel { + struct dma_chan chan; + struct dma_async_tx_descriptor txd; + struct cppi41_dd *cdd; + struct cppi41_desc *desc; + dma_addr_t desc_phys; + void __iomem *gcr_reg; + int is_tx; + u32 residue; + + unsigned int q_num; + unsigned int q_comp_num; + unsigned int port_num; + + unsigned td_retry; + unsigned td_queued:1; + unsigned td_seen:1; + unsigned td_desc_seen:1; +}; + +struct cppi41_desc { + u32 pd0; + u32 pd1; + u32 pd2; + u32 pd3; + u32 pd4; + u32 pd5; + u32 pd6; + u32 pd7; +} __aligned(32); + +struct chan_queues { + u16 submit; + u16 complete; +}; + +struct cppi41_dd { + struct dma_device ddev; + + void *qmgr_scratch; + dma_addr_t scratch_phys; + + struct cppi41_desc *cd; + dma_addr_t descs_phys; + u32 first_td_desc; + struct cppi41_channel *chan_busy[ALLOC_DECS_NUM]; + + void __iomem *usbss_mem; + void __iomem *ctrl_mem; + void __iomem *sched_mem; + void __iomem *qmgr_mem; + unsigned int irq; + const struct chan_queues *queues_rx; + const struct chan_queues *queues_tx; + struct chan_queues td_queue; +}; + +#define FIST_COMPLETION_QUEUE 93 +static struct chan_queues usb_queues_tx[] = { + /* USB0 ENDP 1 */ + [ 0] = { .submit = 32, .complete = 93}, + [ 1] = { .submit = 34, .complete = 94}, + [ 2] = { .submit = 36, .complete = 95}, + [ 3] = { .submit = 38, .complete = 96}, + [ 4] = { .submit = 40, .complete = 97}, + [ 5] = { .submit = 42, .complete = 98}, + [ 6] = { .submit = 44, .complete = 99}, + [ 7] = { .submit = 46, .complete = 100}, + [ 8] = { .submit = 48, .complete = 101}, + [ 9] = { .submit = 50, .complete = 102}, + [10] = { .submit = 52, .complete = 103}, + [11] = { .submit = 54, .complete = 104}, + [12] = { .submit = 56, .complete = 105}, + [13] = { .submit = 58, .complete = 106}, + [14] = { .submit = 60, .complete = 107}, + + /* USB1 ENDP1 */ + [15] = { .submit = 62, .complete = 125}, + [16] = { .submit = 64, .complete = 126}, + [17] = { .submit = 66, .complete = 127}, + [18] = { .submit = 68, .complete = 128}, + [19] = { .submit = 70, .complete = 129}, + [20] = { .submit = 72, .complete = 130}, + [21] = { .submit = 74, .complete = 131}, + [22] = { .submit = 76, .complete = 132}, + [23] = { .submit = 78, .complete = 133}, + [24] = { .submit = 80, .complete = 134}, + [25] = { .submit = 82, .complete = 135}, + [26] = { .submit = 84, .complete = 136}, + [27] = { .submit = 86, .complete = 137}, + [28] = { .submit = 88, .complete = 138}, + [29] = { .submit = 90, .complete = 139}, +}; + +static const struct chan_queues usb_queues_rx[] = { + /* USB0 ENDP 1 */ + [ 0] = { .submit = 1, .complete = 109}, + [ 1] = { .submit = 2, .complete = 110}, + [ 2] = { .submit = 3, .complete = 111}, + [ 3] = { .submit = 4, .complete = 112}, + [ 4] = { .submit = 5, .complete = 113}, + [ 5] = { .submit = 6, .complete = 114}, + [ 6] = { .submit = 7, .complete = 115}, + [ 7] = { .submit = 8, .complete = 116}, + [ 8] = { .submit = 9, .complete = 117}, + [ 9] = { .submit = 10, .complete = 118}, + [10] = { .submit = 11, .complete = 119}, + [11] = { .submit = 12, .complete = 120}, + [12] = { .submit = 13, .complete = 121}, + [13] = { .submit = 14, .complete = 122}, + [14] = { .submit = 15, .complete = 123}, + + /* USB1 ENDP 1 */ + [15] = { .submit = 16, .complete = 141}, + [16] = { .submit = 17, .complete = 142}, + [17] = { .submit = 18, .complete = 143}, + [18] = { .submit = 19, .complete = 144}, + [19] = { .submit = 20, .complete = 145}, + [20] = { .submit = 21, .complete = 146}, + [21] = { .submit = 22, .complete = 147}, + [22] = { .submit = 23, .complete = 148}, + [23] = { .submit = 24, .complete = 149}, + [24] = { .submit = 25, .complete = 150}, + [25] = { .submit = 26, .complete = 151}, + [26] = { .submit = 27, .complete = 152}, + [27] = { .submit = 28, .complete = 153}, + [28] = { .submit = 29, .complete = 154}, + [29] = { .submit = 30, .complete = 155}, +}; + +struct cppi_glue_infos { + irqreturn_t (*isr)(int irq, void *data); + const struct chan_queues *queues_rx; + const struct chan_queues *queues_tx; + struct chan_queues td_queue; +}; + +static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c) +{ + return container_of(c, struct cppi41_channel, chan); +} + +static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc) +{ + struct cppi41_channel *c; + u32 descs_size; + u32 desc_num; + + descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM; + + if (!((desc >= cdd->descs_phys) && + (desc < (cdd->descs_phys + descs_size)))) { + return NULL; + } + + desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc); + BUG_ON(desc_num >= ALLOC_DECS_NUM); + c = cdd->chan_busy[desc_num]; + cdd->chan_busy[desc_num] = NULL; + return c; +} + +static void cppi_writel(u32 val, void *__iomem *mem) +{ + __raw_writel(val, mem); +} + +static u32 cppi_readl(void *__iomem *mem) +{ + return __raw_readl(mem); +} + +static u32 pd_trans_len(u32 val) +{ + return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1); +} + +static irqreturn_t cppi41_irq(int irq, void *data) +{ + struct cppi41_dd *cdd = data; + struct cppi41_channel *c; + u32 status; + int i; + + status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS); + if (!(status & USBSS_IRQ_PD_COMP)) + return IRQ_NONE; + cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS); + + for (i = QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE); i < QMGR_NUM_PEND; + i++) { + u32 val; + u32 q_num; + + val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i)); + if (i == QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE) && val) { + u32 mask; + /* set corresponding bit for completetion Q 93 */ + mask = 1 << QMGR_PENDING_BIT_Q(FIST_COMPLETION_QUEUE); + /* not set all bits for queues less than Q 93 */ + mask--; + /* now invert and keep only Q 93+ set */ + val &= ~mask; + } + + if (val) + __iormb(); + + while (val) { + u32 desc; + + q_num = __fls(val); + val &= ~(1 << q_num); + q_num += 32 * i; + desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(q_num)); + desc &= ~0x1f; + c = desc_to_chan(cdd, desc); + if (WARN_ON(!c)) { + pr_err("%s() q %d desc %08x\n", __func__, + q_num, desc); + continue; + } + c->residue = pd_trans_len(c->desc->pd6) - + pd_trans_len(c->desc->pd0); + + dma_cookie_complete(&c->txd); + c->txd.callback(c->txd.callback_param); + } + } + return IRQ_HANDLED; +} + +static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx) +{ + dma_cookie_t cookie; + + cookie = dma_cookie_assign(tx); + + return cookie; +} + +static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + + dma_cookie_init(chan); + dma_async_tx_descriptor_init(&c->txd, chan); + c->txd.tx_submit = cppi41_tx_submit; + + if (!c->is_tx) + cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0); + + return 0; +} + +static void cppi41_dma_free_chan_resources(struct dma_chan *chan) +{ +} + +static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + enum dma_status ret; + + /* lock */ + ret = dma_cookie_status(chan, cookie, txstate); + if (txstate && ret == DMA_SUCCESS) + txstate->residue = c->residue; + /* unlock */ + + return ret; +} + +static void push_desc_queue(struct cppi41_channel *c) +{ + struct cppi41_dd *cdd = c->cdd; + u32 desc_num; + u32 desc_phys; + u32 reg; + + desc_phys = lower_32_bits(c->desc_phys); + desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); + WARN_ON(cdd->chan_busy[desc_num]); + cdd->chan_busy[desc_num] = c; + + reg = (sizeof(struct cppi41_desc) - 24) / 4; + reg |= desc_phys; + cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); +} + +static void cppi41_dma_issue_pending(struct dma_chan *chan) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + u32 reg; + + c->residue = 0; + + reg = GCR_CHAN_ENABLE; + if (!c->is_tx) { + reg |= GCR_STARV_RETRY; + reg |= GCR_DESC_TYPE_HOST; + reg |= c->q_comp_num; + } + + cppi_writel(reg, c->gcr_reg); + + /* + * We don't use writel() but __raw_writel() so we have to make sure + * that the DMA descriptor in coherent memory made to the main memory + * before starting the dma engine. + */ + __iowmb(); + push_desc_queue(c); +} + +static u32 get_host_pd0(u32 length) +{ + u32 reg; + + reg = DESC_TYPE_HOST << DESC_TYPE; + reg |= length; + + return reg; +} + +static u32 get_host_pd1(struct cppi41_channel *c) +{ + u32 reg; + + reg = 0; + + return reg; +} + +static u32 get_host_pd2(struct cppi41_channel *c) +{ + u32 reg; + + reg = DESC_TYPE_USB; + reg |= c->q_comp_num; + + return reg; +} + +static u32 get_host_pd3(u32 length) +{ + u32 reg; + + /* PD3 = packet size */ + reg = length; + + return reg; +} + +static u32 get_host_pd6(u32 length) +{ + u32 reg; + + /* PD6 buffer size */ + reg = DESC_PD_COMPLETE; + reg |= length; + + return reg; +} + +static u32 get_host_pd4_or_7(u32 addr) +{ + u32 reg; + + reg = addr; + + return reg; +} + +static u32 get_host_pd5(void) +{ + u32 reg; + + reg = 0; + + return reg; +} + +static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len, + enum dma_transfer_direction dir, unsigned long tx_flags, void *context) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + struct cppi41_desc *d; + struct scatterlist *sg; + unsigned int i; + unsigned int num; + + num = 0; + d = c->desc; + for_each_sg(sgl, sg, sg_len, i) { + u32 addr; + u32 len; + + /* We need to use more than one desc once musb supports sg */ + BUG_ON(num > 0); + addr = lower_32_bits(sg_dma_address(sg)); + len = sg_dma_len(sg); + + d->pd0 = get_host_pd0(len); + d->pd1 = get_host_pd1(c); + d->pd2 = get_host_pd2(c); + d->pd3 = get_host_pd3(len); + d->pd4 = get_host_pd4_or_7(addr); + d->pd5 = get_host_pd5(); + d->pd6 = get_host_pd6(len); + d->pd7 = get_host_pd4_or_7(addr); + + d++; + } + + return &c->txd; +} + +static int cpp41_cfg_chan(struct cppi41_channel *c, + struct dma_slave_config *cfg) +{ + return 0; +} + +static void cppi41_compute_td_desc(struct cppi41_desc *d) +{ + d->pd0 = DESC_TYPE_TEARD << DESC_TYPE; +} + +static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num) +{ + u32 desc; + + desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num)); + desc &= ~0x1f; + return desc; +} + +static int cppi41_tear_down_chan(struct cppi41_channel *c) +{ + struct cppi41_dd *cdd = c->cdd; + struct cppi41_desc *td; + u32 reg; + u32 desc_phys; + u32 td_desc_phys; + + td = cdd->cd; + td += cdd->first_td_desc; + + td_desc_phys = cdd->descs_phys; + td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc); + + if (!c->td_queued) { + cppi41_compute_td_desc(td); + __iowmb(); + + reg = (sizeof(struct cppi41_desc) - 24) / 4; + reg |= td_desc_phys; + cppi_writel(reg, cdd->qmgr_mem + + QMGR_QUEUE_D(cdd->td_queue.submit)); + + reg = GCR_CHAN_ENABLE; + if (!c->is_tx) { + reg |= GCR_STARV_RETRY; + reg |= GCR_DESC_TYPE_HOST; + reg |= c->q_comp_num; + } + reg |= GCR_TEARDOWN; + cppi_writel(reg, c->gcr_reg); + c->td_queued = 1; + c->td_retry = 100; + } + + if (!c->td_seen) { + unsigned td_comp_queue; + + if (c->is_tx) + td_comp_queue = cdd->td_queue.complete; + else + td_comp_queue = c->q_comp_num; + + desc_phys = cppi41_pop_desc(cdd, td_comp_queue); + if (desc_phys) { + __iormb(); + + if (desc_phys == td_desc_phys) { + u32 pd0; + pd0 = td->pd0; + WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD); + WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX)); + WARN_ON((pd0 & 0x1f) != c->port_num); + } else { + WARN_ON_ONCE(1); + } + c->td_seen = 1; + } + } + if (!c->td_desc_seen) { + desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); + if (desc_phys) { + __iormb(); + WARN_ON(c->desc_phys != desc_phys); + c->td_desc_seen = 1; + } + } + c->td_retry--; + /* + * If the TX descriptor / channel is in use, the caller needs to poke + * his TD bit multiple times. After that he hardware releases the + * transfer descriptor followed by TD descriptor. Waiting seems not to + * cause any difference. + * RX seems to be thrown out right away. However once the TearDown + * descriptor gets through we are done. If we have seens the transfer + * descriptor before the TD we fetch it from enqueue, it has to be + * there waiting for us. + */ + if (!c->td_seen && c->td_retry) + return -EAGAIN; + + WARN_ON(!c->td_retry); + if (!c->td_desc_seen) { + desc_phys = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); + WARN_ON(!desc_phys); + } + + c->td_queued = 0; + c->td_seen = 0; + c->td_desc_seen = 0; + cppi_writel(0, c->gcr_reg); + return 0; +} + +static int cppi41_stop_chan(struct dma_chan *chan) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + struct cppi41_dd *cdd = c->cdd; + u32 desc_num; + u32 desc_phys; + int ret; + + ret = cppi41_tear_down_chan(c); + if (ret) + return ret; + + desc_phys = lower_32_bits(c->desc_phys); + desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); + WARN_ON(!cdd->chan_busy[desc_num]); + cdd->chan_busy[desc_num] = NULL; + + return 0; +} + +static int cppi41_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct cppi41_channel *c = to_cpp41_chan(chan); + int ret; + + switch (cmd) { + case DMA_SLAVE_CONFIG: + ret = cpp41_cfg_chan(c, (struct dma_slave_config *) arg); + break; + + case DMA_TERMINATE_ALL: + ret = cppi41_stop_chan(chan); + break; + + default: + ret = -ENXIO; + break; + } + return ret; +} + +static void cleanup_chans(struct cppi41_dd *cdd) +{ + while (!list_empty(&cdd->ddev.channels)) { + struct cppi41_channel *cchan; + + cchan = list_first_entry(&cdd->ddev.channels, + struct cppi41_channel, chan.device_node); + list_del(&cchan->chan.device_node); + kfree(cchan); + } +} + +static int cppi41_add_chans(struct platform_device *pdev, struct cppi41_dd *cdd) +{ + struct cppi41_channel *cchan; + int i; + int ret; + u32 n_chans; + + ret = of_property_read_u32(pdev->dev.of_node, "#dma-channels", + &n_chans); + if (ret) + return ret; + /* + * The channels can only be used as TX or as RX. So we add twice + * that much dma channels because USB can only do RX or TX. + */ + n_chans *= 2; + + for (i = 0; i < n_chans; i++) { + cchan = kzalloc(sizeof(*cchan), GFP_KERNEL); + if (!cchan) + goto err; + + cchan->cdd = cdd; + if (i & 1) { + cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1); + cchan->is_tx = 1; + } else { + cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1); + cchan->is_tx = 0; + } + cchan->port_num = i >> 1; + cchan->desc = &cdd->cd[i]; + cchan->desc_phys = cdd->descs_phys; + cchan->desc_phys += i * sizeof(struct cppi41_desc); + cchan->chan.device = &cdd->ddev; + list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels); + } + cdd->first_td_desc = n_chans; + + return 0; +err: + cleanup_chans(cdd); + return -ENOMEM; +} + +static void purge_descs(struct platform_device *pdev, struct cppi41_dd *cdd) +{ + unsigned int mem_decs; + int i; + + mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc); + + for (i = 0; i < DESCS_AREAS; i++) { + + cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i)); + cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i)); + + dma_free_coherent(&pdev->dev, mem_decs, cdd->cd, + cdd->descs_phys); + } +} + +static void disable_sched(struct cppi41_dd *cdd) +{ + cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL); +} + +static void deinit_cpii41(struct platform_device *pdev, struct cppi41_dd *cdd) +{ + disable_sched(cdd); + + purge_descs(pdev, cdd); + + cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); + cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); + dma_free_coherent(&pdev->dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch, + cdd->scratch_phys); +} + +static int init_descs(struct platform_device *pdev, struct cppi41_dd *cdd) +{ + unsigned int desc_size; + unsigned int mem_decs; + int i; + u32 reg; + u32 idx; + + BUILD_BUG_ON(sizeof(struct cppi41_desc) & + (sizeof(struct cppi41_desc) - 1)); + BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32); + BUILD_BUG_ON(ALLOC_DECS_NUM < 32); + + desc_size = sizeof(struct cppi41_desc); + mem_decs = ALLOC_DECS_NUM * desc_size; + + idx = 0; + for (i = 0; i < DESCS_AREAS; i++) { + + reg = idx << QMGR_MEMCTRL_IDX_SH; + reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH; + reg |= ilog2(ALLOC_DECS_NUM) - 5; + + BUILD_BUG_ON(DESCS_AREAS != 1); + cdd->cd = dma_alloc_coherent(&pdev->dev, mem_decs, + &cdd->descs_phys, GFP_KERNEL); + if (!cdd->cd) + return -ENOMEM; + + cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i)); + cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i)); + + idx += ALLOC_DECS_NUM; + } + return 0; +} + +static void init_sched(struct cppi41_dd *cdd) +{ + unsigned ch; + unsigned word; + u32 reg; + + word = 0; + cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL); + for (ch = 0; ch < 15 * 2; ch += 2) { + + reg = SCHED_ENTRY0_CHAN(ch); + reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX; + + reg |= SCHED_ENTRY2_CHAN(ch + 1); + reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX; + cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word)); + word++; + } + reg = 15 * 2 * 2 - 1; + reg |= DMA_SCHED_CTRL_EN; + cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL); +} + +static int init_cppi41(struct platform_device *pdev, struct cppi41_dd *cdd) +{ + int ret; + + BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1)); + cdd->qmgr_scratch = dma_alloc_coherent(&pdev->dev, QMGR_SCRATCH_SIZE, + &cdd->scratch_phys, GFP_KERNEL); + if (!cdd->qmgr_scratch) + return -ENOMEM; + + cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE); + cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE); + cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); + + ret = init_descs(pdev, cdd); + if (ret) + goto err_td; + + cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ); + init_sched(cdd); + return 0; +err_td: + deinit_cpii41(pdev, cdd); + return ret; +} + +static struct platform_driver cpp41_dma_driver; +/* + * The param format is: + * X Y + * X: Port + * Y: 0 = RX else TX + */ +#define INFO_PORT 0 +#define INFO_IS_TX 1 + +static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param) +{ + struct cppi41_channel *cchan; + struct cppi41_dd *cdd; + const struct chan_queues *queues; + u32 *num = param; + + if (chan->device->dev->driver != &cpp41_dma_driver.driver) + return false; + + cchan = to_cpp41_chan(chan); + + if (cchan->port_num != num[INFO_PORT]) + return false; + + if (cchan->is_tx && !num[INFO_IS_TX]) + return false; + cdd = cchan->cdd; + if (cchan->is_tx) + queues = cdd->queues_tx; + else + queues = cdd->queues_rx; + + BUILD_BUG_ON(ARRAY_SIZE(usb_queues_rx) != ARRAY_SIZE(usb_queues_tx)); + if (WARN_ON(cchan->port_num > ARRAY_SIZE(usb_queues_rx))) + return false; + + cchan->q_num = queues[cchan->port_num].submit; + cchan->q_comp_num = queues[cchan->port_num].complete; + return true; +} + +static struct of_dma_filter_info cpp41_dma_info = { + .filter_fn = cpp41_dma_filter_fn, +}; + +static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + int count = dma_spec->args_count; + struct of_dma_filter_info *info = ofdma->of_dma_data; + + if (!info || !info->filter_fn) + return NULL; + + if (count != 2) + return NULL; + + return dma_request_channel(info->dma_cap, info->filter_fn, + &dma_spec->args[0]); +} + +static const struct cppi_glue_infos usb_infos = { + .isr = cppi41_irq, + .queues_rx = usb_queues_rx, + .queues_tx = usb_queues_tx, + .td_queue = { .submit = 31, .complete = 0 }, +}; + +static const struct of_device_id cppi41_dma_ids[] = { + { .compatible = "ti,am3359-cppi41", .data = &usb_infos}, + {}, +}; +MODULE_DEVICE_TABLE(of, cppi41_dma_ids); + +static const struct cppi_glue_infos *get_glue_info(struct platform_device *pdev) +{ + const struct of_device_id *of_id; + + of_id = of_match_node(cppi41_dma_ids, pdev->dev.of_node); + if (!of_id) + return NULL; + return of_id->data; +} + +static int cppi41_dma_probe(struct platform_device *pdev) +{ + struct cppi41_dd *cdd; + const struct cppi_glue_infos *glue_info; + int irq; + int ret; + + glue_info = get_glue_info(pdev); + if (!glue_info) + return -EINVAL; + + cdd = kzalloc(sizeof(*cdd), GFP_KERNEL); + if (!cdd) + return -ENOMEM; + + dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask); + cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources; + cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources; + cdd->ddev.device_tx_status = cppi41_dma_tx_status; + cdd->ddev.device_issue_pending = cppi41_dma_issue_pending; + cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg; + cdd->ddev.device_control = cppi41_dma_control; + cdd->ddev.dev = &pdev->dev; + INIT_LIST_HEAD(&cdd->ddev.channels); + cpp41_dma_info.dma_cap = cdd->ddev.cap_mask; + + cdd->usbss_mem = of_iomap(pdev->dev.of_node, 0); + cdd->ctrl_mem = of_iomap(pdev->dev.of_node, 1); + cdd->sched_mem = of_iomap(pdev->dev.of_node, 2); + cdd->qmgr_mem = of_iomap(pdev->dev.of_node, 3); + + if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem || + !cdd->qmgr_mem) { + ret = -ENXIO; + goto err_remap; + } + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret) + goto err_get_sync; + + cdd->queues_rx = glue_info->queues_rx; + cdd->queues_tx = glue_info->queues_tx; + cdd->td_queue = glue_info->td_queue; + + ret = init_cppi41(pdev, cdd); + if (ret) + goto err_init_cppi; + + ret = cppi41_add_chans(pdev, cdd); + if (ret) + goto err_chans; + + irq = irq_of_parse_and_map(pdev->dev.of_node, 0); + if (!irq) + goto err_irq; + + cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER); + + ret = request_irq(irq, glue_info->isr, IRQF_SHARED, + dev_name(&pdev->dev), cdd); + if (ret) + goto err_irq; + cdd->irq = irq; + + ret = dma_async_device_register(&cdd->ddev); + if (ret) + goto err_dma_reg; + + ret = of_dma_controller_register(pdev->dev.of_node, + cppi41_dma_xlate, &cpp41_dma_info); + if (ret) + goto err_of; + + platform_set_drvdata(pdev, cdd); + return 0; +err_of: + dma_async_device_unregister(&cdd->ddev); +err_dma_reg: + free_irq(irq, cdd); +err_irq: + cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); + cleanup_chans(cdd); +err_chans: + deinit_cpii41(pdev, cdd); +err_init_cppi: + pm_runtime_put(&pdev->dev); +err_get_sync: + pm_runtime_disable(&pdev->dev); + iounmap(cdd->usbss_mem); + iounmap(cdd->ctrl_mem); + iounmap(cdd->sched_mem); + iounmap(cdd->qmgr_mem); +err_remap: + kfree(cdd); + return ret; +} + +static int cppi41_dma_remove(struct platform_device *pdev) +{ + struct cppi41_dd *cdd = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&cdd->ddev); + + cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); + free_irq(cdd->irq, cdd); + cleanup_chans(cdd); + deinit_cpii41(pdev, cdd); + iounmap(cdd->usbss_mem); + iounmap(cdd->ctrl_mem); + iounmap(cdd->sched_mem); + iounmap(cdd->qmgr_mem); + pm_runtime_put(&pdev->dev); + pm_runtime_disable(&pdev->dev); + kfree(cdd); + return 0; +} + +static struct platform_driver cpp41_dma_driver = { + .probe = cppi41_dma_probe, + .remove = cppi41_dma_remove, + .driver = { + .name = "cppi41-dma-engine", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(cppi41_dma_ids), + }, +}; + +module_platform_driver(cpp41_dma_driver); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>"); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index d7d94d21a03..9162ac80c18 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -87,7 +87,8 @@ static struct dma_chan *dev_to_dma_chan(struct device *dev) return chan_dev->chan; } -static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf) +static ssize_t memcpy_count_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct dma_chan *chan; unsigned long count = 0; @@ -106,9 +107,10 @@ static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *at return err; } +static DEVICE_ATTR_RO(memcpy_count); -static ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr, - char *buf) +static ssize_t bytes_transferred_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct dma_chan *chan; unsigned long count = 0; @@ -127,8 +129,10 @@ static ssize_t show_bytes_transferred(struct device *dev, struct device_attribut return err; } +static DEVICE_ATTR_RO(bytes_transferred); -static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) +static ssize_t in_use_show(struct device *dev, struct device_attribute *attr, + char *buf) { struct dma_chan *chan; int err; @@ -143,13 +147,15 @@ static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, ch return err; } +static DEVICE_ATTR_RO(in_use); -static struct device_attribute dma_attrs[] = { - __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL), - __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL), - __ATTR(in_use, S_IRUGO, show_in_use, NULL), - __ATTR_NULL +static struct attribute *dma_dev_attrs[] = { + &dev_attr_memcpy_count.attr, + &dev_attr_bytes_transferred.attr, + &dev_attr_in_use.attr, + NULL, }; +ATTRIBUTE_GROUPS(dma_dev); static void chan_dev_release(struct device *dev) { @@ -167,7 +173,7 @@ static void chan_dev_release(struct device *dev) static struct class dma_devclass = { .name = "dma", - .dev_attrs = dma_attrs, + .dev_groups = dma_dev_groups, .dev_release = chan_dev_release, }; @@ -376,20 +382,30 @@ void dma_issue_pending_all(void) EXPORT_SYMBOL(dma_issue_pending_all); /** - * nth_chan - returns the nth channel of the given capability + * dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu + */ +static bool dma_chan_is_local(struct dma_chan *chan, int cpu) +{ + int node = dev_to_node(chan->device->dev); + return node == -1 || cpumask_test_cpu(cpu, cpumask_of_node(node)); +} + +/** + * min_chan - returns the channel with min count and in the same numa-node as the cpu * @cap: capability to match - * @n: nth channel desired + * @cpu: cpu index which the channel should be close to * - * Defaults to returning the channel with the desired capability and the - * lowest reference count when 'n' cannot be satisfied. Must be called - * under dma_list_mutex. + * If some channels are close to the given cpu, the one with the lowest + * reference count is returned. Otherwise, cpu is ignored and only the + * reference count is taken into account. + * Must be called under dma_list_mutex. */ -static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n) +static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu) { struct dma_device *device; struct dma_chan *chan; - struct dma_chan *ret = NULL; struct dma_chan *min = NULL; + struct dma_chan *localmin = NULL; list_for_each_entry(device, &dma_device_list, global_node) { if (!dma_has_cap(cap, device->cap_mask) || @@ -398,27 +414,22 @@ static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n) list_for_each_entry(chan, &device->channels, device_node) { if (!chan->client_count) continue; - if (!min) - min = chan; - else if (chan->table_count < min->table_count) + if (!min || chan->table_count < min->table_count) min = chan; - if (n-- == 0) { - ret = chan; - break; /* done */ - } + if (dma_chan_is_local(chan, cpu)) + if (!localmin || + chan->table_count < localmin->table_count) + localmin = chan; } - if (ret) - break; /* done */ } - if (!ret) - ret = min; + chan = localmin ? localmin : min; - if (ret) - ret->table_count++; + if (chan) + chan->table_count++; - return ret; + return chan; } /** @@ -435,7 +446,6 @@ static void dma_channel_rebalance(void) struct dma_device *device; int cpu; int cap; - int n; /* undo the last distribution */ for_each_dma_cap_mask(cap, dma_cap_mask_all) @@ -454,14 +464,9 @@ static void dma_channel_rebalance(void) return; /* redistribute available channels */ - n = 0; for_each_dma_cap_mask(cap, dma_cap_mask_all) for_each_online_cpu(cpu) { - if (num_possible_cpus() > 1) - chan = nth_chan(cap, n++); - else - chan = nth_chan(cap, -1); - + chan = min_chan(cap, cpu); per_cpu_ptr(channel_table[cap], cpu)->chan = chan; } } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index e88ded2c8d2..92f796cdc6a 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -25,44 +25,46 @@ #include <linux/seq_file.h> static unsigned int test_buf_size = 16384; -module_param(test_buf_size, uint, S_IRUGO); +module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); static char test_channel[20]; -module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); +module_param_string(channel, test_channel, sizeof(test_channel), + S_IRUGO | S_IWUSR); MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); static char test_device[20]; -module_param_string(device, test_device, sizeof(test_device), S_IRUGO); +module_param_string(device, test_device, sizeof(test_device), + S_IRUGO | S_IWUSR); MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); static unsigned int threads_per_chan = 1; -module_param(threads_per_chan, uint, S_IRUGO); +module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(threads_per_chan, "Number of threads to start per channel (default: 1)"); static unsigned int max_channels; -module_param(max_channels, uint, S_IRUGO); +module_param(max_channels, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(max_channels, "Maximum number of channels to use (default: all)"); static unsigned int iterations; -module_param(iterations, uint, S_IRUGO); +module_param(iterations, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(iterations, "Iterations before stopping test (default: infinite)"); static unsigned int xor_sources = 3; -module_param(xor_sources, uint, S_IRUGO); +module_param(xor_sources, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(xor_sources, "Number of xor source buffers (default: 3)"); static unsigned int pq_sources = 3; -module_param(pq_sources, uint, S_IRUGO); +module_param(pq_sources, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(pq_sources, "Number of p+q source buffers (default: 3)"); static int timeout = 3000; -module_param(timeout, uint, S_IRUGO); +module_param(timeout, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " "Pass -1 for infinite timeout"); @@ -193,7 +195,6 @@ struct dmatest_info { /* debugfs related stuff */ struct dentry *root; - struct dmatest_params dbgfs_params; /* Test results */ struct list_head results; @@ -406,7 +407,11 @@ static int thread_result_add(struct dmatest_info *info, list_add_tail(&tr->node, &r->results); mutex_unlock(&info->results_lock); - pr_warn("%s\n", thread_result_get(r->name, tr)); + if (tr->type == DMATEST_ET_OK) + pr_debug("%s\n", thread_result_get(r->name, tr)); + else + pr_warn("%s\n", thread_result_get(r->name, tr)); + return 0; } @@ -1007,7 +1012,15 @@ static int __restart_threaded_test(struct dmatest_info *info, bool run) result_free(info, NULL); /* Copy test parameters */ - memcpy(params, &info->dbgfs_params, sizeof(*params)); + params->buf_size = test_buf_size; + strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); + strlcpy(params->device, strim(test_device), sizeof(params->device)); + params->threads_per_chan = threads_per_chan; + params->max_channels = max_channels; + params->iterations = iterations; + params->xor_sources = xor_sources; + params->pq_sources = pq_sources; + params->timeout = timeout; /* Run test with new parameters */ return __run_threaded_test(info); @@ -1029,71 +1042,6 @@ static bool __is_threaded_test_run(struct dmatest_info *info) return false; } -static ssize_t dtf_write_string(void *to, size_t available, loff_t *ppos, - const void __user *from, size_t count) -{ - char tmp[20]; - ssize_t len; - - len = simple_write_to_buffer(tmp, sizeof(tmp) - 1, ppos, from, count); - if (len >= 0) { - tmp[len] = '\0'; - strlcpy(to, strim(tmp), available); - } - - return len; -} - -static ssize_t dtf_read_channel(struct file *file, char __user *buf, - size_t count, loff_t *ppos) -{ - struct dmatest_info *info = file->private_data; - return simple_read_from_buffer(buf, count, ppos, - info->dbgfs_params.channel, - strlen(info->dbgfs_params.channel)); -} - -static ssize_t dtf_write_channel(struct file *file, const char __user *buf, - size_t size, loff_t *ppos) -{ - struct dmatest_info *info = file->private_data; - return dtf_write_string(info->dbgfs_params.channel, - sizeof(info->dbgfs_params.channel), - ppos, buf, size); -} - -static const struct file_operations dtf_channel_fops = { - .read = dtf_read_channel, - .write = dtf_write_channel, - .open = simple_open, - .llseek = default_llseek, -}; - -static ssize_t dtf_read_device(struct file *file, char __user *buf, - size_t count, loff_t *ppos) -{ - struct dmatest_info *info = file->private_data; - return simple_read_from_buffer(buf, count, ppos, - info->dbgfs_params.device, - strlen(info->dbgfs_params.device)); -} - -static ssize_t dtf_write_device(struct file *file, const char __user *buf, - size_t size, loff_t *ppos) -{ - struct dmatest_info *info = file->private_data; - return dtf_write_string(info->dbgfs_params.device, - sizeof(info->dbgfs_params.device), - ppos, buf, size); -} - -static const struct file_operations dtf_device_fops = { - .read = dtf_read_device, - .write = dtf_write_device, - .open = simple_open, - .llseek = default_llseek, -}; - static ssize_t dtf_read_run(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { @@ -1187,8 +1135,6 @@ static const struct file_operations dtf_results_fops = { static int dmatest_register_dbgfs(struct dmatest_info *info) { struct dentry *d; - struct dmatest_params *params = &info->dbgfs_params; - int ret = -ENOMEM; d = debugfs_create_dir("dmatest", NULL); if (IS_ERR(d)) @@ -1198,81 +1144,24 @@ static int dmatest_register_dbgfs(struct dmatest_info *info) info->root = d; - /* Copy initial values */ - memcpy(params, &info->params, sizeof(*params)); - - /* Test parameters */ - - d = debugfs_create_u32("test_buf_size", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->buf_size); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_file("channel", S_IRUGO | S_IWUSR, info->root, - info, &dtf_channel_fops); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_file("device", S_IRUGO | S_IWUSR, info->root, - info, &dtf_device_fops); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("threads_per_chan", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->threads_per_chan); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("max_channels", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->max_channels); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("iterations", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->iterations); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("xor_sources", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->xor_sources); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("pq_sources", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->pq_sources); - if (IS_ERR_OR_NULL(d)) - goto err_node; - - d = debugfs_create_u32("timeout", S_IWUSR | S_IRUGO, info->root, - (u32 *)¶ms->timeout); - if (IS_ERR_OR_NULL(d)) - goto err_node; - /* Run or stop threaded test */ - d = debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, - info, &dtf_run_fops); - if (IS_ERR_OR_NULL(d)) - goto err_node; + debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info, + &dtf_run_fops); /* Results of test in progress */ - d = debugfs_create_file("results", S_IRUGO, info->root, info, - &dtf_results_fops); - if (IS_ERR_OR_NULL(d)) - goto err_node; + debugfs_create_file("results", S_IRUGO, info->root, info, + &dtf_results_fops); return 0; -err_node: - debugfs_remove_recursive(info->root); err_root: pr_err("dmatest: Failed to initialize debugfs\n"); - return ret; + return -ENOMEM; } static int __init dmatest_init(void) { struct dmatest_info *info = &test_info; - struct dmatest_params *params = &info->params; int ret; memset(info, 0, sizeof(*info)); @@ -1283,17 +1172,6 @@ static int __init dmatest_init(void) mutex_init(&info->results_lock); INIT_LIST_HEAD(&info->results); - /* Set default parameters */ - params->buf_size = test_buf_size; - strlcpy(params->channel, test_channel, sizeof(params->channel)); - strlcpy(params->device, test_device, sizeof(params->device)); - params->threads_per_chan = threads_per_chan; - params->max_channels = max_channels; - params->iterations = iterations; - params->xor_sources = xor_sources; - params->pq_sources = pq_sources; - params->timeout = timeout; - ret = dmatest_register_dbgfs(info); if (ret) return ret; diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index b642e035579..d8ececaf1b5 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -251,7 +251,7 @@ static bool is_bwd_noraid(struct pci_dev *pdev) } static void pq16_set_src(struct ioat_raw_descriptor *desc[3], - dma_addr_t addr, u32 offset, u8 coef, int idx) + dma_addr_t addr, u32 offset, u8 coef, unsigned idx) { struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0]; struct ioat_pq16a_descriptor *pq16 = @@ -1775,15 +1775,12 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; dma->device_free_chan_resources = ioat2_free_chan_resources; - if (is_xeon_cb32(pdev)) - dma->copy_align = 6; - dma_cap_set(DMA_INTERRUPT, dma->cap_mask); dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; device->cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET); - if (is_bwd_noraid(pdev)) + if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev)) device->cap &= ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS); /* dca is incompatible with raid operations */ @@ -1793,7 +1790,6 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) if (device->cap & IOAT_CAP_XOR) { is_raid_device = true; dma->max_xor = 8; - dma->xor_align = 6; dma_cap_set(DMA_XOR, dma->cap_mask); dma->device_prep_dma_xor = ioat3_prep_xor; @@ -1812,13 +1808,8 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) if (device->cap & IOAT_CAP_RAID16SS) { dma_set_maxpq(dma, 16, 0); - dma->pq_align = 0; } else { dma_set_maxpq(dma, 8, 0); - if (is_xeon_cb32(pdev)) - dma->pq_align = 6; - else - dma->pq_align = 0; } if (!(device->cap & IOAT_CAP_XOR)) { @@ -1829,13 +1820,8 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) if (device->cap & IOAT_CAP_RAID16SS) { dma->max_xor = 16; - dma->xor_align = 0; } else { dma->max_xor = 8; - if (is_xeon_cb32(pdev)) - dma->xor_align = 6; - else - dma->xor_align = 0; } } } @@ -1844,14 +1830,6 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) device->cleanup_fn = ioat3_cleanup_event; device->timer_fn = ioat3_timer_event; - if (is_xeon_cb32(pdev)) { - dma_cap_clear(DMA_XOR_VAL, dma->cap_mask); - dma->device_prep_dma_xor_val = NULL; - - dma_cap_clear(DMA_PQ_VAL, dma->cap_mask); - dma->device_prep_dma_pq_val = NULL; - } - /* starting with CB3.3 super extended descriptors are supported */ if (device->cap & IOAT_CAP_RAID16SS) { char pool_name[14]; diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index d9a26777a1b..536dcb8ba5f 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -64,7 +64,7 @@ static u32 mv_desc_get_src_addr(struct mv_xor_desc_slot *desc, int src_idx) { struct mv_xor_desc *hw_desc = desc->hw_desc; - return hw_desc->phy_src_addr[src_idx]; + return hw_desc->phy_src_addr[mv_phy_src_idx(src_idx)]; } @@ -107,32 +107,32 @@ static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc, int index, dma_addr_t addr) { struct mv_xor_desc *hw_desc = desc->hw_desc; - hw_desc->phy_src_addr[index] = addr; + hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr; if (desc->type == DMA_XOR) hw_desc->desc_command |= (1 << index); } static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan) { - return __raw_readl(XOR_CURR_DESC(chan)); + return readl_relaxed(XOR_CURR_DESC(chan)); } static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan, u32 next_desc_addr) { - __raw_writel(next_desc_addr, XOR_NEXT_DESC(chan)); + writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan)); } static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan) { - u32 val = __raw_readl(XOR_INTR_MASK(chan)); + u32 val = readl_relaxed(XOR_INTR_MASK(chan)); val |= XOR_INTR_MASK_VALUE << (chan->idx * 16); - __raw_writel(val, XOR_INTR_MASK(chan)); + writel_relaxed(val, XOR_INTR_MASK(chan)); } static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan) { - u32 intr_cause = __raw_readl(XOR_INTR_CAUSE(chan)); + u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan)); intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF; return intr_cause; } @@ -149,13 +149,13 @@ static void mv_xor_device_clear_eoc_cause(struct mv_xor_chan *chan) { u32 val = ~(1 << (chan->idx * 16)); dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val); - __raw_writel(val, XOR_INTR_CAUSE(chan)); + writel_relaxed(val, XOR_INTR_CAUSE(chan)); } static void mv_xor_device_clear_err_status(struct mv_xor_chan *chan) { u32 val = 0xFFFF0000 >> (chan->idx * 16); - __raw_writel(val, XOR_INTR_CAUSE(chan)); + writel_relaxed(val, XOR_INTR_CAUSE(chan)); } static int mv_can_chain(struct mv_xor_desc_slot *desc) @@ -173,7 +173,7 @@ static void mv_set_mode(struct mv_xor_chan *chan, enum dma_transaction_type type) { u32 op_mode; - u32 config = __raw_readl(XOR_CONFIG(chan)); + u32 config = readl_relaxed(XOR_CONFIG(chan)); switch (type) { case DMA_XOR: @@ -192,7 +192,14 @@ static void mv_set_mode(struct mv_xor_chan *chan, config &= ~0x7; config |= op_mode; - __raw_writel(config, XOR_CONFIG(chan)); + +#if defined(__BIG_ENDIAN) + config |= XOR_DESCRIPTOR_SWAP; +#else + config &= ~XOR_DESCRIPTOR_SWAP; +#endif + + writel_relaxed(config, XOR_CONFIG(chan)); chan->current_type = type; } @@ -201,14 +208,14 @@ static void mv_chan_activate(struct mv_xor_chan *chan) u32 activation; dev_dbg(mv_chan_to_devp(chan), " activate chan.\n"); - activation = __raw_readl(XOR_ACTIVATION(chan)); + activation = readl_relaxed(XOR_ACTIVATION(chan)); activation |= 0x1; - __raw_writel(activation, XOR_ACTIVATION(chan)); + writel_relaxed(activation, XOR_ACTIVATION(chan)); } static char mv_chan_is_busy(struct mv_xor_chan *chan) { - u32 state = __raw_readl(XOR_ACTIVATION(chan)); + u32 state = readl_relaxed(XOR_ACTIVATION(chan)); state = (state >> 4) & 0x3; @@ -755,22 +762,22 @@ static void mv_dump_xor_regs(struct mv_xor_chan *chan) { u32 val; - val = __raw_readl(XOR_CONFIG(chan)); + val = readl_relaxed(XOR_CONFIG(chan)); dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val); - val = __raw_readl(XOR_ACTIVATION(chan)); + val = readl_relaxed(XOR_ACTIVATION(chan)); dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val); - val = __raw_readl(XOR_INTR_CAUSE(chan)); + val = readl_relaxed(XOR_INTR_CAUSE(chan)); dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val); - val = __raw_readl(XOR_INTR_MASK(chan)); + val = readl_relaxed(XOR_INTR_MASK(chan)); dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val); - val = __raw_readl(XOR_ERROR_CAUSE(chan)); + val = readl_relaxed(XOR_ERROR_CAUSE(chan)); dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val); - val = __raw_readl(XOR_ERROR_ADDR(chan)); + val = readl_relaxed(XOR_ERROR_ADDR(chan)); dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val); } @@ -1029,10 +1036,8 @@ mv_xor_channel_add(struct mv_xor_device *xordev, struct dma_device *dma_dev; mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL); - if (!mv_chan) { - ret = -ENOMEM; - goto err_free_dma; - } + if (!mv_chan) + return ERR_PTR(-ENOMEM); mv_chan->idx = idx; mv_chan->irq = irq; diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index c619359cb7f..06b067f24c9 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -29,8 +29,10 @@ #define MV_XOR_THRESHOLD 1 #define MV_XOR_MAX_CHANNELS 2 +/* Values for the XOR_CONFIG register */ #define XOR_OPERATION_MODE_XOR 0 #define XOR_OPERATION_MODE_MEMCPY 2 +#define XOR_DESCRIPTOR_SWAP BIT(14) #define XOR_CURR_DESC(chan) (chan->mmr_base + 0x210 + (chan->idx * 4)) #define XOR_NEXT_DESC(chan) (chan->mmr_base + 0x200 + (chan->idx * 4)) @@ -143,7 +145,16 @@ struct mv_xor_desc_slot { #endif }; -/* This structure describes XOR descriptor size 64bytes */ +/* + * This structure describes XOR descriptor size 64bytes. The + * mv_phy_src_idx() macro must be used when indexing the values of the + * phy_src_addr[] array. This is due to the fact that the 'descriptor + * swap' feature, used on big endian systems, swaps descriptors data + * within blocks of 8 bytes. So two consecutive values of the + * phy_src_addr[] array are actually swapped in big-endian, which + * explains the different mv_phy_src_idx() implementation. + */ +#if defined(__LITTLE_ENDIAN) struct mv_xor_desc { u32 status; /* descriptor execution status */ u32 crc32_result; /* result of CRC-32 calculation */ @@ -155,6 +166,21 @@ struct mv_xor_desc { u32 reserved0; u32 reserved1; }; +#define mv_phy_src_idx(src_idx) (src_idx) +#else +struct mv_xor_desc { + u32 crc32_result; /* result of CRC-32 calculation */ + u32 status; /* descriptor execution status */ + u32 phy_next_desc; /* next descriptor address pointer */ + u32 desc_command; /* type of operation to be carried out */ + u32 phy_dest_addr; /* destination block address */ + u32 byte_count; /* size of src/dst blocks in bytes */ + u32 phy_src_addr[8]; /* source block addresses */ + u32 reserved1; + u32 reserved0; +}; +#define mv_phy_src_idx(src_idx) (src_idx ^ 1) +#endif #define to_mv_sw_desc(addr_hw_desc) \ container_of(addr_hw_desc, struct mv_xor_desc_slot, hw_desc) diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 3d0472b78f5..1069e8869f2 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -423,8 +423,8 @@ static size_t sh_dmae_get_partial(struct shdma_chan *schan, shdma_chan); struct sh_dmae_desc *sh_desc = container_of(sdesc, struct sh_dmae_desc, shdma_desc); - return (sh_desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << - sh_chan->xmit_shift; + return sh_desc->hw.tcr - + (sh_dmae_readl(sh_chan, TCR) << sh_chan->xmit_shift); } /* Called from error IRQ or NMI */ |