BOOL CPCIDisk::SetupDMA( PSG_BUF pSgBuf, DWORD dwSgCount, BOOL fRead ) { DWORD dwAlignMask = m_dwDMAAlign - 1; DWORD dwPageMask = UserKInfo[KINX_PAGESIZE] - 1; DWORD iPage = 0, iPFN, iBuffer; BOOL fUnalign = FALSE; DMA_ADAPTER_OBJECT Adapter; Adapter.ObjectSize = sizeof(DMA_ADAPTER_OBJECT); Adapter.InterfaceType = (INTERFACE_TYPE)m_pPort->m_pController->m_dwi.dwInterfaceType; Adapter.BusNumber = m_pPort->m_pController->m_dwi.dwBusNumber; DEBUGMSG(ZONE_DMA, (_T( "Atapi!CPCIDisk::SetupDMA> Request(%s), SgCount(%d)\r\n" ), fRead ? (_T("Read")) : (_T("Write")), dwSgCount)); #ifndef ST202T_SATA // disable bus master WriteBMCommand(0); #endif if (!m_pPRD) { m_pPRD = (PDMATable)HalAllocateCommonBuffer(&Adapter, UserKInfo[KINX_PAGESIZE], &m_pPRDPhys, FALSE); if (!m_pPRD) { goto ExitFailure; } } // m_pPhysList tracks pages used for DMA buffers when the scatter/gather // buffer is unaligned if (!m_pPhysList) { m_pPhysList = (PPhysTable)VirtualAlloc(m_pStartMemory, UserKInfo[KINX_PAGESIZE], MEM_COMMIT, PAGE_READWRITE); if (!m_pPhysList) { goto ExitFailure; } // allocate the minimum number of fixed pages for (DWORD i = 0; i < MIN_PHYS_PAGES; i++) { PHYSICAL_ADDRESS PhysicalAddress = {0}; m_pPhysList[i].pVirtualAddress = (LPBYTE)HalAllocateCommonBuffer(&Adapter, UserKInfo[KINX_PAGESIZE], &PhysicalAddress, FALSE); m_pPhysList[i].pPhysicalAddress = (LPBYTE)PhysicalAddress.QuadPart; if (!m_pPhysList[i].pVirtualAddress) { goto ExitFailure; } } } m_dwPhysCount = 0; // m_pSGCopy tracks the mapping between scatter/gather buffers and DMA // buffers when the scatter/gather buffer is unaligned and we are reading, // so we can copy the read data back to the scatter/gather buffer; when the // scatter/gather buffer is aligned, m_pSGCopy tracks the scatter/gather // buffers of a particular DMA transfer, so we can unlock the buffers at // completion if (!m_pSGCopy) { m_pSGCopy = (PSGCopyTable)VirtualAlloc( m_pStartMemory + UserKInfo[KINX_PAGESIZE], UserKInfo[KINX_PAGESIZE], MEM_COMMIT, PAGE_READWRITE); if (!m_pSGCopy) { goto ExitFailure; } } m_dwSGCount = 0; if (!m_pPFNs) { m_pPFNs = (PDWORD)VirtualAlloc( m_pStartMemory + 2*UserKInfo[KINX_PAGESIZE], UserKInfo[KINX_PAGESIZE], MEM_COMMIT, PAGE_READWRITE); if (!m_pPFNs) { goto ExitFailure; } } // determine whether the a buffer or the buffer length is unaligned for (iBuffer = 0; iBuffer < dwSgCount; iBuffer++) { if ( ((DWORD)pSgBuf[iBuffer].sb_buf & dwAlignMask) || ((DWORD)pSgBuf[iBuffer].sb_len & dwAlignMask) ) { fUnalign = TRUE; break; } } if (fUnalign) { DWORD dwCurPageOffset = 0; for (iBuffer = 0; iBuffer < dwSgCount; iBuffer++) { LPBYTE pBuffer = (LPBYTE)pSgBuf[iBuffer].sb_buf; DWORD dwBufferLeft = pSgBuf[iBuffer].sb_len; while (dwBufferLeft) { DWORD dwBytesInCurPage = UserKInfo[KINX_PAGESIZE] - dwCurPageOffset; DWORD dwBytesToTransfer = (dwBufferLeft > dwBytesInCurPage) ? dwBytesInCurPage : dwBufferLeft; // allocate a new page, if necessary if ((dwCurPageOffset == 0) && (m_dwPhysCount >= MIN_PHYS_PAGES)) { PHYSICAL_ADDRESS PhysicalAddress = {0}; m_pPhysList[m_dwPhysCount].pVirtualAddress = (LPBYTE)HalAllocateCommonBuffer( &Adapter, UserKInfo[KINX_PAGESIZE], &PhysicalAddress, FALSE); m_pPhysList[m_dwPhysCount].pPhysicalAddress = (LPBYTE)PhysicalAddress.QuadPart; if (!m_pPhysList[m_dwPhysCount].pVirtualAddress) { goto ExitFailure; } } if (fRead) { // prepare a scatter/gather copy entry on read, so we can // copy data from the DMA buffer to the scatter/gather // buffer after this DMA transfer is complete m_pSGCopy[m_dwSGCount].pSrcAddress = m_pPhysList[m_dwPhysCount].pVirtualAddress + dwCurPageOffset; m_pSGCopy[m_dwSGCount].pDstAddress = pBuffer; m_pSGCopy[m_dwSGCount].dwSize = dwBytesToTransfer; m_dwSGCount++; } else { memcpy(m_pPhysList[m_dwPhysCount].pVirtualAddress + dwCurPageOffset, pBuffer, dwBytesToTransfer); } // if this buffer is larger than the space remaining on the page, // then finish processing this page by setting @dwCurPageOffset<-0 if (dwBufferLeft >= dwBytesInCurPage) { dwCurPageOffset = 0; } else { dwCurPageOffset += dwBytesToTransfer; } // have we finished a page? (i.e., offset was reset or this is the last buffer) if ((dwCurPageOffset == 0) || (iBuffer == (dwSgCount - 1))) { // add this to the PRD table m_pPRD[m_dwPhysCount].physAddr = (DWORD)m_pPhysList[m_dwPhysCount].pPhysicalAddress; m_pPRD[m_dwPhysCount].size = dwCurPageOffset ? (USHORT)dwCurPageOffset : (USHORT)UserKInfo[KINX_PAGESIZE]; m_pPRD[m_dwPhysCount].EOTpad = 0; m_dwPhysCount++; } // update transfer dwBufferLeft -= dwBytesToTransfer; pBuffer += dwBytesToTransfer; } } m_pPRD[m_dwPhysCount - 1].EOTpad = 0x8000; } else { DWORD dwTotalBytes = 0; for (iBuffer = 0; iBuffer < dwSgCount; iBuffer++) { LPBYTE pBuffer = (LPBYTE)pSgBuf[iBuffer].sb_buf; // determine the number of bytes remaining to be placed in PRD dwTotalBytes = pSgBuf[iBuffer].sb_len; if (!LockPages ( pBuffer, dwTotalBytes, m_pPFNs, fRead ? LOCKFLAG_WRITE : LOCKFLAG_READ) ) { goto ExitFailure; } // add a scatter/gather copy entry for the area we lock, so that // we can unlock it when we are finished m_pSGCopy[m_dwSGCount].pSrcAddress = pBuffer; m_pSGCopy[m_dwSGCount].pDstAddress = 0; m_pSGCopy[m_dwSGCount].dwSize = dwTotalBytes; m_dwSGCount++; iPFN = 0; while (dwTotalBytes) { DWORD dwBytesToTransfer = UserKInfo[KINX_PAGESIZE]; if ((DWORD)pBuffer & dwPageMask) { // the buffer is not page aligned; use up the next page // boundary dwBytesToTransfer = UserKInfo[KINX_PAGESIZE] - ((DWORD)pBuffer & dwPageMask); } if (dwTotalBytes < dwBytesToTransfer) { // use what remains dwBytesToTransfer = dwTotalBytes; } m_pPRD[iPage].physAddr = (m_pPFNs[iPFN] << UserKInfo[KINX_PFN_SHIFT]) + ((DWORD)pBuffer & dwPageMask); if (!TranslateAddress(&m_pPRD[iPage].physAddr)) { goto ExitFailure; } m_pPRD[iPage].size = (USHORT)dwBytesToTransfer; m_pPRD[iPage].EOTpad = 0; iPage++; iPFN++; // update transfer pBuffer += dwBytesToTransfer; dwTotalBytes -= dwBytesToTransfer; } } m_dwPhysCount = 0; m_pPRD[iPage-1].EOTpad = 0x8000; } return TRUE; ExitFailure: DEBUGCHK(0); // clean up // FreeDMABuffers(); return FALSE; }
// // Perform the actual DMA copy // WARNING!! This function assumes the physical memory is contiguous. // No check is performed for performance improvement. // Buffers passed-in come from CMEM and DISPLAY drivers. // DWORD SdmaPx::Copy( LPVOID pSource, LPVOID pDestination, DWORD dwClientIdx ) { DWORD dwRet = 0; DWORD dwCause, dwStatus; DWORD paSrc, paDst; BYTE ffCached = 0; DmaConfigInfo_t dmaSettings = m_SdmaPxClient[dwClientIdx].GetConfig(); DWORD dwDataLength = m_SdmaPxClient[dwClientIdx].GetDataLength(); DWORD dwElementCount = m_SdmaPxClient[dwClientIdx].GetElementCount(); DWORD dwFrameCount = m_SdmaPxClient[dwClientIdx].GetFrameCount(); ASSERTMSG(L"Client must be configured before Copy I/O control is called !!!\n", m_SdmaPxClient[dwClientIdx].IsClientConfigured()); // Configure SDMA for specific client // Need to call this first for length // memset() on DmaConfigInfo_t not needed as done in the SdmaPxClient::Init() method. // m_SdmaPxClient[dwClientIdx].GetConfig(&dmaSettings, &dwDataLength); if(DmaConfigure(m_hDmaChannel, &dmaSettings, 0, &m_dmaInfo) != TRUE) { ERRORMSG(ZONE_ERROR, (TEXT("ERROR! Unable to configure DMA for client\r\n"))); goto cleanUp; } // flush cache if necessary if (ISUNCACHEDADDRESS(pSource) == FALSE) { ffCached |= SOURCE_CACHED; } if (ISUNCACHEDADDRESS(pDestination) == FALSE) { ffCached |= DESTINATION_CACHED; } if (ffCached & (SOURCE_CACHED | DESTINATION_CACHED)) { FlushCache(pSource, pDestination, dwDataLength, ffCached); } // Retrieve base physical address of buffer to be copied and READ lock the pages on the length. if(LockPages( (LPVOID)pSource, (DWORD)dwDataLength, m_rgPFNsrc, LOCKFLAG_READ) == FALSE) { ERRORMSG(ZONE_ERROR, (TEXT("LockPages call \"src\" failed. (error code=%d)\r\n"), GetLastError())); goto cleanUp; } // Not necessary to do the page shift on ARM platform as always 0. (ref. MSDN) // paSrc = (m_rgPFNsrc[0] << m_pageShift) + ((DWORD)pSource & m_pageMask); paSrc = m_rgPFNsrc[0] + ((DWORD)pSource & m_pageMask); // Retrieve base physical address of destination buffer and WRITE lock the pages on the length. if(LockPages( (LPVOID)pDestination, dwDataLength, m_rgPFNdst, LOCKFLAG_WRITE) == FALSE) { ERRORMSG(ZONE_ERROR, (TEXT("LockPages \"dest\" call failed. (error code=%d)\r\n"), GetLastError())); goto cleanUp; } // Not necessary to do the page shift on ARM platform as always 0. (ref. MSDN) // paDst = (m_rgPFNdst[0] << UserKInfo[KINX_PFN_SHIFT]) + ((DWORD)pDestination & m_pageMask); paDst = m_rgPFNdst[0] + ((DWORD)pDestination & m_pageMask); // Configure Dest and Src buffers for DMA. DmaSetSrcBuffer(&m_dmaInfo, (UINT8 *)pSource, paSrc); DmaSetDstBuffer(&m_dmaInfo, (UINT8 *)pDestination, paDst); // Watch out parameters in DmaSetElemenAndFrameCount. For e.g., shift right element count by 2 if you have bytes in input and you use 32bits element size. DmaSetElementAndFrameCount(&m_dmaInfo, dwElementCount, (UINT16)(dwFrameCount)); // start dma DmaStart(&m_dmaInfo); // wait until we hit the end of buffer // Wait for dma interrupt... dwCause = WaitForSingleObject(m_hEvent, DMA_IRQ_TIMEOUT); switch(dwCause) { case WAIT_OBJECT_0: { // Verify cause of interrupt was because we hit the end of block dwStatus = DmaGetStatus(&m_dmaInfo); if ((dwStatus & (dmaSettings.interrupts)) == 0) { ERRORMSG(ZONE_ERROR, (TEXT("Unexpected cause of interrupt\r\n"))); break; } DmaClearStatus(&m_dmaInfo, dwStatus); if (DmaInterruptDone(m_hDmaChannel) == FALSE) { ERRORMSG(ZONE_ERROR, (TEXT("ERROR! Unable to get status for dma interrupt\r\n"))); break; } // Do the "good" client job DmaStop(&m_dmaInfo); break; } default: RETAILMSG(ZONE_ERROR, (TEXT("ERROR! didn't receive DMA interrupt\r\n"))); break; } #if DEBUG_VERIFY_SDMA_COPY // // Beware!! Bring a lot of overhead and can lead to bad display rendering. // NKDbgPrintfW(L"verify memory\r\n"); if (memcmp(pSource, pDestination, dwDataLength) != 0) { NKDbgPrintfW(L"ERROR! memory doesn't match up\r\n"); DebugBreak(); goto cleanUp; } #endif dwRet = dwDataLength; // everything went fine obviously... cleanUp: UnlockPages((LPVOID)pSource, dwDataLength); UnlockPages((LPVOID)pDestination, dwDataLength); return dwRet; }