- 根目录:
- arch
- blackfin
- kernel
- bfin_dma.c
/*
* bfin_dma.c - Blackfin DMA implementation
*
* Copyright 2004-2008 Analog Devices Inc.
*
* Licensed under the GPL-2 or later.
*/
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/param.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/spinlock.h>
#include <asm/blackfin.h>
#include <asm/cacheflush.h>
#include <asm/dma.h>
#include <asm/uaccess.h>
#include <asm/early_printk.h>
/*
* To make sure we work around 05000119 - we always check DMA_DONE bit,
* never the DMA_RUN bit
*/
struct dma_channel dma_ch[MAX_DMA_CHANNELS];
EXPORT_SYMBOL(dma_ch);
static int __init blackfin_dma_init(void)
{
int i;
printk(KERN_INFO "Blackfin DMA Controller\n");
#if ANOMALY_05000480
bfin_write_DMAC_TC_PER(0x0111);
#endif
for (i = 0; i < MAX_DMA_CHANNELS; i++) {
atomic_set(&dma_ch[i].chan_status, 0);
dma_ch[i].regs = dma_io_base_addr[i];
}
#if defined(CH_MEM_STREAM3_SRC) && defined(CONFIG_BF60x)
/* Mark MEMDMA Channel 3 as requested since we're using it internally */
request_dma(CH_MEM_STREAM3_DEST, "Blackfin dma_memcpy");
request_dma(CH_MEM_STREAM3_SRC, "Blackfin dma_memcpy");
#else
/* Mark MEMDMA Channel 0 as requested since we're using it internally */
request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
#endif
#if defined(CONFIG_DEB_DMA_URGENT)
bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
| DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
#endif
return 0;
}
arch_initcall(blackfin_dma_init);
#ifdef CONFIG_PROC_FS
static int proc_dma_show(struct seq_file *m, void *v)
{
int i;
for (i = 0; i < MAX_DMA_CHANNELS; ++i)
if (dma_channel_active(i))
seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
return 0;
}
static int proc_dma_open(struct inode *inode, struct file *file)
{
return single_open(file, proc_dma_show, NULL);
}
static const struct file_operations proc_dma_operations = {
.open = proc_dma_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int __init proc_dma_init(void)
{
proc_create("dma", 0, NULL, &proc_dma_operations);
return 0;
}
late_initcall(proc_dma_init);
#endif
static void set_dma_peripheral_map(unsigned int channel, const char *device_id)
{
#ifdef CONFIG_BF54x
unsigned int per_map;
switch (channel) {
case CH_UART2_RX: per_map = 0xC << 12; break;
case CH_UART2_TX: per_map = 0xD << 12; break;
case CH_UART3_RX: per_map = 0xE << 12; break;
case CH_UART3_TX: per_map = 0xF << 12; break;
default: return;
}
if (strncmp(device_id, "BFIN_UART", 9) == 0)
dma_ch[channel].regs->peripheral_map = per_map;
#endif
}
/**
* request_dma - request a DMA channel
*
* Request the specific DMA channel from the system if it's available.
*/
int request_dma(unsigned int channel, const char *device_id)
{
pr_debug("request_dma() : BEGIN\n");
if (device_id == NULL)
printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
#if defined(CONFIG_BF561) && ANOMALY_05000182
if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
if (get_cclk() > 500000000) {
printk(KERN_WARNING
"Request IMDMA failed due to ANOMALY 05000182\n");
return -EFAULT;
}
}
#endif
if (atomic_cmpxchg(&dma_ch[channel].chan_status, 0, 1)) {
pr_debug("DMA CHANNEL IN USE\n");
return -EBUSY;
}
set_dma_peripheral_map(channel, device_id);
dma_ch[channel].device_id = device_id;
dma_ch[channel].irq = 0;
/* This is to be enabled by putting a restriction -
* you have to request DMA, before doing any operations on
* descriptor/channel
*/
pr_debug("request_dma() : END\n");
return 0;
}
EXPORT_SYMBOL(request_dma);
int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
{
int ret;
unsigned int irq;
BUG_ON(channel >= MAX_DMA_CHANNELS || !callback ||
!atomic_read(&dma_ch[channel].chan_status));
irq = channel2irq(channel);
ret = request_irq(irq, callback, 0, dma_ch[channel].device_id, data);
if (ret)
return ret;
dma_ch[channel].irq = irq;
dma_ch[channel].data = data;
return 0;
}
EXPORT_SYMBOL(set_dma_callback);
/**
* clear_dma_buffer - clear DMA fifos for specified channel
*
* Set the Buffer Clear bit in the Configuration register of specific DMA
* channel. This will stop the descriptor based DMA operation.
*/
static void clear_dma_buffer(unsigned int channel)
{
dma_ch[channel].regs->cfg |= RESTART;
SSYNC();
dma_ch[channel].regs->cfg &= ~RESTART;
}
void free_dma(unsigned int channel)
{
pr_debug("freedma() : BEGIN\n");
BUG_ON(channel >= MAX_DMA_CHANNELS ||
!atomic_read(&dma_ch[channel].chan_status));
/* Halt the DMA */
disable_dma(channel);
clear_dma_buffer(channel);
if (dma_ch[channel].irq)
free_irq(dma_ch[channel].irq, dma_ch[channel].data);
/* Clear the DMA Variable in the Channel */
atomic_set(&dma_ch[channel].chan_status, 0);
pr_debug("freedma() : END\n");
}
EXPORT_SYMBOL(free_dma);
#ifdef CONFIG_PM
# ifndef MAX_DMA_SUSPEND_CHANNELS
# define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
# endif
# ifndef CONFIG_BF60x
int blackfin_dma_suspend(void)
{
int i;
for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
if (dma_ch[i].regs->cfg & DMAEN) {
printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
return -EBUSY;
}
if (i < MAX_DMA_SUSPEND_CHANNELS)
dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
}
#if ANOMALY_05000480
bfin_write_DMAC_TC_PER(0x0);
#endif
return 0;
}
void blackfin_dma_resume(void)
{
int i;
for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
dma_ch[i].regs->cfg = 0;
if (i < MAX_DMA_SUSPEND_CHANNELS)
dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
}
#if ANOMALY_05000480
bfin_write_DMAC_TC_PER(0x0111);
#endif
}
# else
int blackfin_dma_suspend(void)
{
return 0;
}
void blackfin_dma_resume(void)
{
}
#endif
#endif
/**
* blackfin_dma_early_init - minimal DMA init
*
* Setup a few DMA registers so we can safely do DMA transfers early on in
* the kernel booting process. Really this just means using dma_memcpy().
*/
void __init blackfin_dma_early_init(void)
{
early_shadow_stamp();
bfin_write_MDMA_S0_CONFIG(0);
bfin_write_MDMA_S1_CONFIG(0);
}
void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size)
{
unsigned long dst = (unsigned long)pdst;
unsigned long src = (unsigned long)psrc;
struct dma_register *dst_ch, *src_ch;
early_shadow_stamp();
/* We assume that everything is 4 byte aligned, so include
* a basic sanity check
*/
BUG_ON(dst % 4);
BUG_ON(src % 4);
BUG_ON(size % 4);
src_ch = 0;
/* Find an avalible memDMA channel */
while (1) {
if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) {
dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR;
src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR;
} else {
dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR;
src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR;
}
if (!DMA_MMR_READ(&src_ch->cfg))
break;
else if (DMA_MMR_READ(&dst_ch->irq_status) & DMA_DONE) {
DMA_MMR_WRITE(&src_ch->cfg, 0);
break;
}
}
/* Force a sync in case a previous config reset on this channel
* occurred. This is needed so subsequent writes to DMA registers
* are not spuriously lost/corrupted.
*/
__builtin_bfin_ssync();
/* Destination */
bfin_write32(&dst_ch->start_addr, dst);
DMA_MMR_WRITE(&dst_ch->x_count, size >> 2);
DMA_MMR_WRITE(&dst_ch->x_modify, 1 << 2);
DMA_MMR_WRITE(&dst_ch->irq_status, DMA_DONE | DMA_ERR);
/* Source */
bfin_write32(&src_ch->start_addr, src);
DMA_MMR_WRITE(&src_ch->x_count, size >> 2);
DMA_MMR_WRITE(&src_ch->x_modify, 1 << 2);
DMA_MMR_WRITE(&src_ch->irq_status, DMA_DONE | DMA_ERR);
/* Enable */
DMA_MMR_WRITE(&src_ch->cfg, DMAEN | WDSIZE_32);
DMA_MMR_WRITE(&dst_ch->cfg, WNR | DI_EN_X | DMAEN | WDSIZE_32);
/* Since we are atomic now, don't use the workaround ssync */
__builtin_bfin_ssync();
#ifdef CONFIG_BF60x
/* Work around a possible MDMA anomaly. Running 2 MDMA channels to
* transfer DDR data to L1 SRAM may corrupt data.
* Should be reverted after this issue is root caused.
*/
while (!(DMA_MMR_READ(&dst_ch->irq_status) & DMA_DONE))
continue;
#endif
}
void __init early_dma_memcpy_done(void)
{
early_shadow_stamp();
while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) ||
(bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE)))
continue;
bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR);
/*
* Now that DMA is done, we would normally flush cache, but
* i/d cache isn't running this early, so we don't bother,
* and just clear out the DMA channel for next time
*/
bfin_write_MDMA_S0_CONFIG(0);
bfin_write_MDMA_S1_CONFIG(0);
bfin_write_MDMA_D0_CONFIG(0);
bfin_write_MDMA_D1_CONFIG(0);
__builtin_bfin_ssync();
}
#if defined(CH_MEM_STREAM3_SRC) && defined(CONFIG_BF60x)
#define bfin_read_MDMA_S_CONFIG bfin_read_MDMA_S3_CONFIG
#define bfin_write_MDMA_S_CONFIG bfin_write_MDMA_S3_CONFIG
#define bfin_write_MDMA_S_START_ADDR bfin_write_MDMA_S3_START_ADDR
#define bfin_write_MDMA_S_IRQ_STATUS bfin_write_MDMA_S3_IRQ_STATUS
#define bfin_write_MDMA_S_X_COUNT bfin_write_MDMA_S3_X_COUNT
#define bfin_write_MDMA_S_X_MODIFY bfin_write_MDMA_S3_X_MODIFY
#define bfin_write_MDMA_S_Y_COUNT bfin_write_MDMA_S3_Y_COUNT
#define bfin_write_MDMA_S_Y_MODIFY bfin_write_MDMA_S3_Y_MODIFY
#define bfin_write_MDMA_D_CONFIG bfin_write_MDMA_D3_CONFIG
#define bfin_write_MDMA_D_START_ADDR bfin_write_MDMA_D3_START_ADDR
#define bfin_read_MDMA_D_IRQ_STATUS bfin_read_MDMA_D3_IRQ_STATUS
#define bfin_write_MDMA_D_IRQ_STATUS bfin_write_MDMA_D3_IRQ_STATUS
#define bfin_write_MDMA_D_X_COUNT bfin_write_MDMA_D3_X_COUNT
#define bfin_write_MDMA_D_X_MODIFY bfin_write_MDMA_D3_X_MODIFY
#define bfin_write_MDMA_D_Y_COUNT bfin_write_MDMA_D3_Y_COUNT
#define bfin_write_MDMA_D_Y_MODIFY bfin_write_MDMA_D3_Y_MODIFY
#else
#define bfin_read_MDMA_S_CONFIG bfin_read_MDMA_S0_CONFIG
#define bfin_write_MDMA_S_CONFIG bfin_write_MDMA_S0_CONFIG
#define bfin_write_MDMA_S_START_ADDR bfin_write_MDMA_S0_START_ADDR
#define bfin_write_MDMA_S_IRQ_STATUS bfin_write_MDMA_S0_IRQ_STATUS
#define bfin_write_MDMA_S_X_COUNT bfin_write_MDMA_S0_X_COUNT
#define bfin_write_MDMA_S_X_MODIFY bfin_write_MDMA_S0_X_MODIFY
#define bfin_write_MDMA_S_Y_COUNT bfin_write_MDMA_S0_Y_COUNT
#define bfin_write_MDMA_S_Y_MODIFY bfin_write_MDMA_S0_Y_MODIFY
#define bfin_write_MDMA_D_CONFIG bfin_write_MDMA_D0_CONFIG
#define bfin_write_MDMA_D_START_ADDR bfin_write_MDMA_D0_START_ADDR
#define bfin_read_MDMA_D_IRQ_STATUS bfin_read_MDMA_D0_IRQ_STATUS
#define bfin_write_MDMA_D_IRQ_STATUS bfin_write_MDMA_D0_IRQ_STATUS
#define bfin_write_MDMA_D_X_COUNT bfin_write_MDMA_D0_X_COUNT
#define bfin_write_MDMA_D_X_MODIFY bfin_write_MDMA_D0_X_MODIFY
#define bfin_write_MDMA_D_Y_COUNT bfin_write_MDMA_D0_Y_COUNT
#define bfin_write_MDMA_D_Y_MODIFY bfin_write_MDMA_D0_Y_MODIFY
#endif
/**
* __dma_memcpy - program the MDMA registers
*
* Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
* while programming registers so that everything is fully configured. Wait
* for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
* check will make sure we don't clobber any existing transfer.
*/
static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
{
static DEFINE_SPINLOCK(mdma_lock);
unsigned long flags;
spin_lock_irqsave(&mdma_lock, flags);
/* Force a sync in case a previous config reset on this channel
* occurred. This is needed so subsequent writes to DMA registers
* are not spuriously lost/corrupted. Do it under irq lock and
* without the anomaly version (because we are atomic already).
*/
__builtin_bfin_ssync();
if (bfin_read_MDMA_S_CONFIG())
while (!(bfin_read_MDMA_D_IRQ_STATUS() & DMA_DONE))
continue;
if (conf & DMA2D) {
/* For larger bit sizes, we've already divided down cnt so it
* is no longer a multiple of 64k. So we have to break down
* the limit here so it is a multiple of the incoming size.
* There is no limitation here in terms of total size other
* than the hardware though as the bits lost in the shift are
* made up by MODIFY (== we can hit the whole address space).
* X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
*/
u32 shift = abs(dmod) >> 1;
size_t ycnt = cnt >> (16 - shift);
cnt = 1 << (16 - shift);
bfin_write_MDMA_D_Y_COUNT(ycnt);
bfin_write_MDMA_S_Y_COUNT(ycnt);
bfin_write_MDMA_D_Y_MODIFY(dmod);
bfin_write_MDMA_S_Y_MODIFY(smod);
}
bfin_write_MDMA_D_START_ADDR(daddr);
bfin_write_MDMA_D_X_COUNT(cnt);
bfin_write_MDMA_D_X_MODIFY(dmod);
bfin_write_MDMA_D_IRQ_STATUS(DMA_DONE | DMA_ERR);
bfin_write_MDMA_S_START_ADDR(saddr);
bfin_write_MDMA_S_X_COUNT(cnt);
bfin_write_MDMA_S_X_MODIFY(smod);
bfin_write_MDMA_S_IRQ_STATUS(DMA_DONE | DMA_ERR);
bfin_write_MDMA_S_CONFIG(DMAEN | conf);
if (conf & DMA2D)
bfin_write_MDMA_D_CONFIG(WNR | DI_EN_Y | DMAEN | conf);
else
bfin_write_MDMA_D_CONFIG(WNR | DI_EN_X | DMAEN | conf);
spin_unlock_irqrestore(&mdma_lock, flags);
SSYNC();
while (!(bfin_read_MDMA_D_IRQ_STATUS() & DMA_DONE))
if (bfin_read_MDMA_S_CONFIG())
continue;
else
return;
bfin_write_MDMA_D_IRQ_STATUS(DMA_DONE | DMA_ERR);
bfin_write_MDMA_S_CONFIG(0);
bfin_write_MDMA_D_CONFIG(0);
}
/**
* _dma_memcpy - translate C memcpy settings into MDMA settings
*
* Handle all the high level steps before we touch the MDMA registers. So
* handle direction, tweaking of sizes, and formatting of addresses.
*/
static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
{
u32 conf, shift;
s16 mod;
unsigned long dst = (unsigned long)pdst;
unsigned long src = (unsigned long)psrc;
if (size == 0)
return NULL;
if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
conf = WDSIZE_32;
shift = 2;
} else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
conf = WDSIZE_16;
shift = 1;
} else {
conf = WDSIZE_8;
shift = 0;
}
/* If the two memory regions have a chance of overlapping, make
* sure the memcpy still works as expected. Do this by having the
* copy run backwards instead.
*/
mod = 1 << shift;
if (src < dst) {
mod *= -1;
dst += size + mod;
src += size + mod;
}
size >>= shift;
#ifndef DMA_MMR_SIZE_32
if (size > 0x10000)
conf |= DMA2D;
#endif
__dma_memcpy(dst, mod, src, mod, size, conf);
return pdst;
}
/**
* dma_memcpy - DMA memcpy under mutex lock
*
* Do not check arguments before starting the DMA memcpy. Break the transfer
* up into two pieces. The first transfer is in multiples of 64k and the
* second transfer is the piece smaller than 64k.
*/
void *dma_memcpy(void *pdst, const void *psrc, size_t size)
{
unsigned long dst = (unsigned long)pdst;
unsigned long src = (unsigned long)psrc;
if (bfin_addr_dcacheable(src))
blackfin_dcache_flush_range(src, src + size);
if (bfin_addr_dcacheable(dst))
blackfin_dcache_invalidate_range(dst, dst + size);
return dma_memcpy_nocache(pdst, psrc, size);
}
EXPORT_SYMBOL(dma_memcpy);
/**
* dma_memcpy_nocache - DMA memcpy under mutex lock
* - No cache flush/invalidate
*
* Do not check arguments before starting the DMA memcpy. Break the transfer
* up into two pieces. The first transfer is in multiples of 64k and the
* second transfer is the piece smaller than 64k.
*/
void *dma_memcpy_nocache(void *pdst, const void *psrc, size_t size)
{
#ifdef DMA_MMR_SIZE_32
_dma_memcpy(pdst, psrc, size);
#else
size_t bulk, rest;
bulk = size & ~0xffff;
rest = size - bulk;
if (bulk)
_dma_memcpy(pdst, psrc, bulk);
_dma_memcpy(pdst + bulk, psrc + bulk, rest);
#endif
return pdst;
}
EXPORT_SYMBOL(dma_memcpy_nocache);
/**
* safe_dma_memcpy - DMA memcpy w/argument checking
*
* Verify arguments are safe before heading to dma_memcpy().
*/
void *safe_dma_memcpy(void *dst, const void *src, size_t size)
{
if (!access_ok(VERIFY_WRITE, dst, size))
return NULL;
if (!access_ok(VERIFY_READ, src, size))
return NULL;
return dma_memcpy(dst, src, size);
}
EXPORT_SYMBOL(safe_dma_memcpy);
static void _dma_out(unsigned long addr, unsigned long buf, unsigned DMA_MMR_SIZE_TYPE len,
u16 size, u16 dma_size)
{
blackfin_dcache_flush_range(buf, buf + len * size);
__dma_memcpy(addr, 0, buf, size, len, dma_size);
}
static void _dma_in(unsigned long addr, unsigned long buf, unsigned DMA_MMR_SIZE_TYPE len,
u16 size, u16 dma_size)
{
blackfin_dcache_invalidate_range(buf, buf + len * size);
__dma_memcpy(buf, size, addr, 0, len, dma_size);
}
#define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned DMA_MMR_SIZE_TYPE len) \
{ \
_dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
} \
EXPORT_SYMBOL(dma_##io##s##bwl)
MAKE_DMA_IO(out, b, 1, 8, const);
MAKE_DMA_IO(in, b, 1, 8, );
MAKE_DMA_IO(out, w, 2, 16, const);
MAKE_DMA_IO(in, w, 2, 16, );
MAKE_DMA_IO(out, l, 4, 32, const);
MAKE_DMA_IO(in, l, 4, 32, );