/*=======================================================================*/
unsigned long dma_copy_to_user(void *to, const void *from, unsigned long n)
{
if(!idma_init)
return __arch_copy_to_user((void *)to, (void *)from, n);
dma_to_user++;
DPRINTK(KERN_CRIT "dma_copy_to_user(%#10x, 0x%#10x, %lu): entering\n", (u32) to, (u32) from, n);
return dma_copy(to, from, n, 1);
}
/*=======================================================================*/
unsigned long xor_copy_to_user(void *to, const void *from, unsigned long n)
{
if(!xor_engine_initialized)
return __arch_copy_to_user((void *)to, (void *)from, n);
dma_to_user++;
DPRINTK(KERN_CRIT "xor_copy_to_user(%#10x, 0x%#10x, %lu): entering\n", (u32) to, (u32) from, n);
return xor_dma_copy(to, from, n, 1);
}
unsigned long ___copy_to_user(void *to, const void *from, unsigned long n)
{
if (get_fs() == KERNEL_DS) {
memcpy(to, from, n);
return 0;
}
if (n < 256)
return __arch_copy_to_user(to, from, n);
down_read(¤t->mm->mmap_sem);
while (n) {
pte_t *pte;
spinlock_t *ptl;
int tocopy;
while (unlikely(!pin_page_for_write(to, &pte, &ptl))) {
up_read(¤t->mm->mmap_sem);
if (put_user(*((u8 *)from), (u8 *)to))
goto out;
down_read(¤t->mm->mmap_sem);
}
tocopy = ((~((unsigned long)to)) & (PAGE_SIZE - 1)) + 1;
if (tocopy > n)
tocopy = n;
memcpy(to, from, tocopy);
to += tocopy;
from += tocopy;
n -= tocopy;
pte_unmap_unlock(pte, ptl);
}
out:
up_read(¤t->mm->mmap_sem);
return n;
}
/*=======================================================================*/
static unsigned long dma_copy(void *to, const void *from, unsigned long n, unsigned int to_user)
{
u32 chunk,i;
u32 k_chunk = 0;
u32 u_chunk = 0;
u32 phys_from, phys_to;
unsigned long flags;
u32 unaligned_to;
u32 index = 0;
u32 temp;
unsigned long uaddr, kaddr;
unsigned char kaddr_kernel_static = 0;
DPRINTK("dma_copy: entering\n");
/*
* The unaligned is taken care seperatly since the dst might be part of a cache line that is changed
* by other process -> we must not invalidate this cache lines and we can't also flush it, since other
* process (or the exception handler) might fetch the cache line before we copied it.
*/
/*
* Ok, start addr is not cache line-aligned, so we need to make it so.
*/
unaligned_to = (u32)to & 31;
if(unaligned_to)
{
DPRINTK("Fixing up starting address %d bytes\n", 32 - unaligned_to);
if(to_user)
{
if(__arch_copy_to_user(to, from, 32 - unaligned_to))
goto exit_dma;
}
else
{
if(__arch_copy_from_user(to, from, 32 - unaligned_to))
goto exit_dma;
}
temp = (u32)to + (32 - unaligned_to);
to = (void *)temp;
temp = (u32)from + (32 - unaligned_to);
from = (void *)temp;
/*it's ok, n supposed to be greater than 32 bytes at this point*/
n -= (32 - unaligned_to);
}
/*
* Ok, we're aligned at the top, now let's check the end
* of the buffer and align that. After this we should have
* a block that is a multiple of cache line size.
*/
unaligned_to = ((u32)to + n) & 31;
if(unaligned_to)
{
u32 tmp_to = (u32)to + (n - unaligned_to);
u32 tmp_from = (u32)from + (n - unaligned_to);
DPRINTK("Fixing ending alignment %d bytes\n", unaligned_to);
if(to_user)
{
if(__arch_copy_to_user((void *)tmp_to, (void *)tmp_from, unaligned_to))
goto exit_dma;
}
else
{
if(__arch_copy_from_user((void *)tmp_to, (void *)tmp_from, unaligned_to))
goto exit_dma;
}
/*it's ok, n supposed to be greater than 32 bytes at this point*/
n -= unaligned_to;
}
if(to_user)
{
uaddr = (unsigned long)to;
kaddr = (unsigned long)from;
}
else
{
uaddr = (unsigned long)from;
kaddr = (unsigned long)to;
}
if(virt_addr_valid(kaddr))
{
kaddr_kernel_static = 1;
k_chunk = n;
}
else
{
DPRINTK("kernel address is not linear, fall back\n");
goto exit_dma;
}
spin_lock_irqsave(¤t->mm->page_table_lock, flags);
if (idma_busy)
{
BUG();
}
idma_busy = 1;
i = 0;
while(n > 0)
{
if(k_chunk == 0)
{
/* virtual address */
k_chunk = page_remainder((u32)kaddr);
DPRINTK("kaddr reminder %d \n",k_chunk);
}
if(u_chunk == 0)
{
u_chunk = page_remainder((u32)uaddr);
DPRINTK("uaddr reminder %d \n", u_chunk);
}
chunk = ((u_chunk < k_chunk) ? u_chunk : k_chunk);
if(n < chunk)
{
chunk = n;
}
if(chunk == 0)
{
break;
}
phys_from = physical_address((u32)from, 0);
phys_to = physical_address((u32)to, 1);
DPRINTK("choose chunk %d \n",chunk);
/* if page doesn't exist go out */
if ((!phys_from) || (!phys_to))
{
/* The requested page isn't available, fall back to */
DPRINTK(" no physical address, fall back: from %p , to %p \n", from, to);
goto wait_for_idmas;
}
/*
* Prepare the IDMA.
*/
if (chunk < IDMA_MIN_COPY_CHUNK)
{
DPRINTK(" chunk %d too small , use memcpy \n",chunk);
/* the "to" address might cross cache line boundary, so part of the line*/
/* may be subject to DMA, so we need to wait to last DMA engine to finish */
if (index > 0)
{
if(wait_for_idma(PREV_CHANNEL(current_dma_channel)))
{
BUG();
}
}
if(to_user)
{
if(__arch_copy_to_user((void *)to, (void *)from, chunk)) {
printk("ERROR: %s %d shouldn't happen\n",__FUNCTION__, __LINE__);
goto wait_for_idmas;
}
}
else
{
if(__arch_copy_from_user((void *)to, (void *)from, chunk)) {
printk("ERROR: %s %d shouldn't happen\n",__FUNCTION__, __LINE__);
goto wait_for_idmas;
}
}
}
else
{
/*
* Ensure that the cache is clean:
* - from range must be cleaned
* - to range must be invalidated
*/
dmac_flush_range(from, from + chunk);
dmac_inv_range(to, to + chunk);
if(index > 1)
{
if(wait_for_idma(current_dma_channel))
{
BUG();
goto unlock_dma;
}
}
/* Start DMA */
DPRINTK(" activate DMA: channel %d from %x to %x len %x\n",
current_dma_channel, phys_from, phys_to, chunk);
mvDmaTransfer(current_dma_channel, phys_from, phys_to, chunk, 0);
current_dma_channel = NEXT_CHANNEL(current_dma_channel);
#ifdef RT_DEBUG
dma_activations++;
#endif
index++;
}
/* go to next chunk */
from += chunk;
to += chunk;
kaddr += chunk;
uaddr += chunk;
n -= chunk;
u_chunk -= chunk;
k_chunk -= chunk;
}
wait_for_idmas:
if (index > 1)
{
if(wait_for_idma(current_dma_channel))
{
BUG();
}
}
if (index > 0)
{
if(wait_for_idma(PREV_CHANNEL(current_dma_channel)))
{
BUG();
}
}
unlock_dma:
idma_busy = 0;
spin_unlock_irqrestore(¤t->mm->page_table_lock, flags);
exit_dma:
DPRINTK("dma_copy(0x%x, 0x%x, %lu): exiting\n", (u32) to,
(u32) from, n);
if(n != 0)
{
if(to_user)
return __arch_copy_to_user((void *)to, (void *)from, n);
else
return __arch_copy_from_user((void *)to, (void *)from, n);
}
return 0;
}
/*
* n must be greater equal than 64.
*/
static unsigned long xor_dma_copy(void *to, const void *from, unsigned long n, unsigned int to_user)
{
u32 chunk,i;
u32 k_chunk = 0;
u32 u_chunk = 0;
u32 phys_from, phys_to;
unsigned long flags;
u32 unaligned_to;
u32 index = 0;
u32 temp;
unsigned long uaddr, kaddr;
int chan1, chan2 = -1;
int current_channel;
struct xor_channel_t *channel;
DPRINTK("xor_dma_copy: entering\n");
chan1 = allocate_channel();
if (chan1 != -1)
{
chan2 = allocate_channel();
if(chan2 == -1)
{
free_channel(&xor_channel[chan1]);
}
}
if((chan1 == -1) || (chan2 == -1))
{
goto exit_dma;
}
current_channel = chan1;
/*
* The unaligned is taken care seperatly since the dst might be part of a cache line that is changed
* by other process -> we must not invalidate this cache lines and we can't also flush it, since other
* process (or the exception handler) might fetch the cache line before we copied it.
*/
/*
* Ok, start addr is not cache line-aligned, so we need to make it so.
*/
unaligned_to = (u32)to & 31;
if(unaligned_to)
{
DPRINTK("Fixing up starting address %d bytes\n", 32 - unaligned_to);
if(to_user)
{
if(__arch_copy_to_user(to, from, 32 - unaligned_to))
goto free_channels;
}
else
{
if(__arch_copy_from_user(to, from, 32 - unaligned_to))
goto free_channels;
}
temp = (u32)to + (32 - unaligned_to);
to = (void *)temp;
temp = (u32)from + (32 - unaligned_to);
from = (void *)temp;
/*it's ok, n supposed to be greater than 32 bytes at this point*/
n -= (32 - unaligned_to);
}
/*
* Ok, we're aligned at the top, now let's check the end
* of the buffer and align that. After this we should have
* a block that is a multiple of cache line size.
*/
unaligned_to = ((u32)to + n) & 31;
if(unaligned_to)
{
u32 tmp_to = (u32)to + (n - unaligned_to);
u32 tmp_from = (u32)from + (n - unaligned_to);
DPRINTK("Fixing ending alignment %d bytes\n", unaligned_to);
if(to_user)
{
if(__arch_copy_to_user((void *)tmp_to, (void *)tmp_from, unaligned_to))
goto free_channels;
}
else
{
if(__arch_copy_from_user((void *)tmp_to, (void *)tmp_from, unaligned_to))
goto free_channels;
}
/*it's ok, n supposed to be greater than 32 bytes at this point*/
n -= unaligned_to;
}
if(to_user)
{
uaddr = (unsigned long)to;
kaddr = (unsigned long)from;
}
else
{
uaddr = (unsigned long)from;
kaddr = (unsigned long)to;
}
if(virt_addr_valid(kaddr))
{
k_chunk = n;
}
else
{
DPRINTK("kernel address is not linear, fall back\n");
goto free_channels;
}
spin_lock_irqsave(¤t->mm->page_table_lock, flags);
i = 0;
while(n > 0)
{
if(k_chunk == 0)
{
/* virtual address */
k_chunk = page_remainder((u32)kaddr);
DPRINTK("kaddr reminder %d \n",k_chunk);
}
if(u_chunk == 0)
{
u_chunk = page_remainder((u32)uaddr);
DPRINTK("uaddr reminder %d \n", u_chunk);
}
chunk = ((u_chunk < k_chunk) ? u_chunk : k_chunk);
if(n < chunk)
{
chunk = n;
}
if(chunk == 0)
{
break;
}
phys_from = physical_address((u32)from, 0);
phys_to = physical_address((u32)to, 1);
DPRINTK("choose chunk %d \n",chunk);
/* if page doesn't exist go out */
if ((!phys_from) || (!phys_to))
{
/* The requested page isn't available, fall back to */
DPRINTK(" no physical address, fall back: from %p , to %p \n", from, to);
goto unlock_dma;
}
/*
* Prepare the IDMA.
*/
if (chunk < XOR_MIN_COPY_CHUNK)
{
int last_chan = chan1;
DPRINTK(" chunk %d too small , use memcpy \n",chunk);
if(current_channel == chan1)
{
last_chan = chan2;
}
/* the "to" address might cross cache line boundary, so part of the line*/
/* may be subject to DMA, so we need to wait to last DMA engine to finish */
if(index > 0)
xor_waiton_eng(last_chan);
if(to_user)
{
if(__arch_copy_to_user((void *)to, (void *)from, chunk)) {
printk("ERROR: %s %d shouldn't happen\n",__FUNCTION__, __LINE__);
goto unlock_dma;
}
}
else
{
if(__arch_copy_from_user((void *)to, (void *)from, chunk)) {
printk("ERROR: %s %d shouldn't happen\n",__FUNCTION__, __LINE__);
goto unlock_dma;
}
}
}
else
{
/*
* Ensure that the cache is clean:
* - from range must be cleaned
* - to range must be invalidated
*/
// mvOsCacheFlush(NULL, (void *)from, chunk);
dmac_flush_range(from, from + chunk);
// mvOsCacheInvalidate(NULL, (void *)to, chunk);
dmac_inv_range(to, to + chunk);
if(index > 0)
{
xor_waiton_eng(current_channel);
}
channel = &xor_channel[current_channel];
/* Start DMA */
DPRINTK(" activate DMA: channel %d from %x to %x len %x\n",
current_channel, phys_from, phys_to, chunk);
channel->pDescriptor->srcAdd0 = phys_from;
channel->pDescriptor->phyDestAdd = phys_to;
channel->pDescriptor->byteCnt = chunk;
channel->pDescriptor->phyNextDescPtr = 0;
channel->pDescriptor->status = BIT31;
channel->chan_active = 1;
if( mvXorTransfer(current_channel, MV_DMA, channel->descPhyAddr) != MV_OK)
{
printk(KERN_ERR "%s: DMA copy operation on channel %d failed!\n", __func__, current_channel);
print_xor_regs(current_channel);
BUG();
}
if(current_channel == chan1)
{
current_channel = chan2;
}
else
{
current_channel = chan1;
}
#ifdef RT_DEBUG
dma_activations++;
#endif
index++;
}
/* go to next chunk */
from += chunk;
to += chunk;
kaddr += chunk;
uaddr += chunk;
n -= chunk;
u_chunk -= chunk;
k_chunk -= chunk;
}
unlock_dma:
xor_waiton_eng(chan1);
xor_waiton_eng(chan2);
spin_unlock_irqrestore(¤t->mm->page_table_lock, flags);
free_channels:
free_channel(&xor_channel[chan1]);
free_channel(&xor_channel[chan2]);
exit_dma:
DPRINTK("xor_dma_copy(0x%x, 0x%x, %lu): exiting\n", (u32) to,
(u32) from, n);
if(n != 0)
{
if(to_user)
return __arch_copy_to_user((void *)to, (void *)from, n);
else
return __arch_copy_from_user((void *)to, (void *)from, n);
}
return 0;
}