diff options
Diffstat (limited to 'drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c')
| -rw-r--r-- | drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c | 3461 |
1 files changed, 3461 insertions, 0 deletions
diff --git a/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c new file mode 100644 index 000000000000..2837a1c7a1e2 --- /dev/null +++ b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c @@ -0,0 +1,3461 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2018 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2018 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include "gc_hal_kernel_context.h" + +#define _GC_OBJ_ZONE gcvZONE_COMMAND + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ + +/******************************************************************************* +** +** _NewQueue +** +** Allocate a new command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL Stalled +** Indicate if hardware is stalled already. +** +** OUTPUT: +** +** gckCOMMAND Command +** gckCOMMAND object has been updated with a new command queue. +*/ +static gceSTATUS +_NewQueue( + IN OUT gckCOMMAND Command, + IN gctBOOL Stalled + ) +{ + gceSTATUS status; + gctINT currentIndex, newIndex; + gctPHYS_ADDR_T physical; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Switch to the next command buffer. */ + currentIndex = Command->index; + newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; + + /* Wait for availability. */ +#if gcdDUMP_COMMAND + gcmkPRINT("@[kernel.waitsignal]"); +#endif + + gcmkONERROR(gckOS_WaitSignal( + Command->os, + Command->queues[newIndex].signal, + gcvFALSE, + gcvINFINITE + )); + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + if (newIndex < currentIndex) + { + Command->wrapCount += 1; + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 2 * 4, + "%s(%d): queue array wrapped around.\n", + __FUNCTION__, __LINE__ + ); + } + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): total queue wrap arounds %d.\n", + __FUNCTION__, __LINE__, Command->wrapCount + ); + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_COMMAND, + 3 * 4, + "%s(%d): switched to queue %d.\n", + __FUNCTION__, __LINE__, newIndex + ); +#endif + + /* Update gckCOMMAND object with new command queue. */ + Command->index = newIndex; + Command->newQueue = gcvTRUE; + Command->virtualMemory = Command->queues[newIndex].physical; + Command->logical = Command->queues[newIndex].logical; + Command->address = Command->queues[newIndex].address; + Command->offset = 0; + + gcmkONERROR(gckOS_GetPhysicalAddress( + Command->os, + Command->logical, + &physical + )); + + gcmkSAFECASTPHYSADDRT(Command->physical, physical); + + if (currentIndex != -1) + { + if (Stalled) + { + gckOS_Signal( + Command->os, + Command->queues[currentIndex].signal, + gcvTRUE + ); + } + else + { + /* Mark the command queue as available. */ + gcmkONERROR(gckEVENT_Signal( + Command->kernel->eventObj, + Command->queues[currentIndex].signal, + gcvKERNEL_COMMAND + )); + } + } + + /* Success. */ + gcmkFOOTER_ARG("Command->index=%d", Command->index); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_IncrementCommitAtom( + IN gckCOMMAND Command, + IN gctBOOL Increment + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctINT32 atomValue; + gctBOOL powerAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Grab the power mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, hardware->powerMutex, gcvINFINITE + )); + powerAcquired = gcvTRUE; + + /* Increment the commit atom. */ + if (Increment) + { + gcmkONERROR(gckOS_AtomIncrement( + Command->os, Command->atomCommit, &atomValue + )); + } + else + { + gcmkONERROR(gckOS_AtomDecrement( + Command->os, Command->atomCommit, &atomValue + )); + } + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + powerAcquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (powerAcquired) + { + /* Release the power mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex( + Command->os, hardware->powerMutex + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if gcdSECURE_USER +static gceSTATUS +_ProcessHints( + IN gckCOMMAND Command, + IN gctUINT32 ProcessID, + IN gcoCMDBUF CommandBuffer + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gckKERNEL kernel; + gctBOOL needCopy = gcvFALSE; + gcskSECURE_CACHE_PTR cache; + gctUINT8_PTR commandBufferLogical; + gctUINT8_PTR hintedData; + gctUINT32_PTR hintArray; + gctUINT i, hintCount; + + gcmkHEADER_ARG( + "Command=0x%08X ProcessID=%d CommandBuffer=0x%08X", + Command, ProcessID, CommandBuffer + ); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Reset state array pointer. */ + hintArray = gcvNULL; + + /* Get the kernel object. */ + kernel = Command->kernel; + + /* Get the cache form the database. */ + gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache)); + + /* Determine the start of the command buffer. */ + commandBufferLogical + = (gctUINT8_PTR) CommandBuffer->logical + + CommandBuffer->startOffset; + + /* Determine the number of records in the state array. */ + hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray; + + /* Check wehther we need to copy the structures or not. */ + gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); + + /* Get access to the state array. */ + if (needCopy) + { + gctUINT copySize; + + if (Command->hintArrayAllocated && + (Command->hintArraySize < CommandBuffer->hintArraySize)) + { + gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); + Command->hintArraySize = gcvFALSE; + } + + if (!Command->hintArrayAllocated) + { + gctPOINTER pointer = gcvNULL; + + gcmkONERROR(gckOS_Allocate( + Command->os, + CommandBuffer->hintArraySize, + &pointer + )); + + Command->hintArray = gcmPTR_TO_UINT64(pointer); + Command->hintArrayAllocated = gcvTRUE; + Command->hintArraySize = CommandBuffer->hintArraySize; + } + + hintArray = gcmUINT64_TO_PTR(Command->hintArray); + copySize = hintCount * gcmSIZEOF(gctUINT32); + + gcmkONERROR(gckOS_CopyFromUserData( + Command->os, + hintArray, + gcmUINT64_TO_PTR(CommandBuffer->hintArray), + copySize + )); + } + else + { + gctPOINTER pointer = gcvNULL; + + gcmkONERROR(gckOS_MapUserPointer( + Command->os, + gcmUINT64_TO_PTR(CommandBuffer->hintArray), + CommandBuffer->hintArraySize, + &pointer + )); + + hintArray = pointer; + } + + /* Scan through the buffer. */ + for (i = 0; i < hintCount; i += 1) + { + /* Determine the location of the hinted data. */ + hintedData = commandBufferLogical + hintArray[i]; + + /* Map handle into physical address. */ + gcmkONERROR(gckKERNEL_MapLogicalToPhysical( + kernel, cache, (gctPOINTER) hintedData + )); + } + +OnError: + /* Get access to the state array. */ + if (!needCopy && (hintArray != gcvNULL)) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + gcmUINT64_TO_PTR(CommandBuffer->hintArray), + CommandBuffer->hintArraySize, + hintArray + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} +#endif + +#if !gcdNULL_DRIVER +static gceSTATUS +_FlushMMU( + IN gckCOMMAND Command + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gctUINT32 oldValue; + gckHARDWARE hardware = Command->kernel->hardware; + gctBOOL pause = gcvFALSE; + + gctUINT8_PTR pointer; + gctUINT32 address; + gctUINT32 eventBytes; + gctUINT32 endBytes; + gctUINT32 bufferSize; + gctUINT32 executeBytes; + gctUINT32 waitLinkBytes; + + gcmkONERROR(gckOS_AtomicExchange(Command->os, + hardware->pageTableDirty[gcvENGINE_RENDER], + 0, + &oldValue)); + + if (oldValue) + { + /* Page Table is upated, flush mmu before commit. */ + gcmkONERROR(gckHARDWARE_FlushMMU(hardware)); + + if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) + && (!hardware->stallFEPrefetch) + ) + { + pause = gcvTRUE; + } + } + + if (pause) + { + /* Query size. */ + gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); + gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, ~0U, &endBytes)); + + executeBytes = eventBytes + endBytes; + + gcmkONERROR(gckHARDWARE_WaitLink( + hardware, + gcvNULL, + ~0U, + Command->offset + executeBytes, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + executeBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + /* Pointer to reserved address. */ + address = Command->address + Command->offset; + + /* Append EVENT(29). */ + gcmkONERROR(gckHARDWARE_Event( + hardware, + pointer, + 29, + gcvKERNEL_PIXEL, + &eventBytes + )); + + /* Append END. */ + pointer += eventBytes; + address += eventBytes; + + gcmkONERROR(gckHARDWARE_End(hardware, pointer, address, &endBytes)); + +#if USE_KERNEL_VIRTUAL_BUFFERS + if (hardware->kernel->virtualCommandBuffer) + { + gcmkONERROR(gckKERNEL_GetGPUAddress( + hardware->kernel, + pointer, + gcvFALSE, + Command->virtualMemory, + &hardware->lastEnd + )); + } +#endif + + gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +static gceSTATUS +_DummyDraw( + IN gckCOMMAND Command + ) +{ +#if gcdSECURITY + return gcvSTATUS_OK; +#else + gceSTATUS status; + gckHARDWARE hardware = Command->kernel->hardware; + + gctUINT8_PTR pointer; + gctUINT32 bufferSize; + + gctUINT32 dummyDrawBytes; + gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) + { + dummyDrawType = gcvDUMMY_DRAW_GC400; + } + + if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && + gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) + { + dummyDrawType = gcvDUMMY_DRAW_V60; + } + + if (dummyDrawType != gcvDUMMY_DRAW_INVALID) + { + gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + /* Reserve space. */ + gcmkONERROR(gckCOMMAND_Reserve( + Command, + dummyDrawBytes, + (gctPOINTER *)&pointer, + &bufferSize + )); + + gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); + + gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); + } + + return gcvSTATUS_OK; +OnError: + return status; +#endif +} + +#endif + +static void +_DumpBuffer( + IN gctPOINTER Buffer, + IN gctUINT32 GpuAddress, + IN gctSIZE_T Size + ) +{ + gctSIZE_T i, line, left; + gctUINT32_PTR data = Buffer; + + line = Size / 32; + left = Size % 32; + + for (i = 0; i < line; i++) + { + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); + data += 8; + GpuAddress += 8 * 4; + } + + switch(left) + { + case 28: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); + break; + case 24: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); + break; + case 20: + gcmkPRINT("%08X : %08X %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3], data[4]); + break; + case 16: + gcmkPRINT("%08X : %08X %08X %08X %08X", + GpuAddress, data[0], data[1], data[2], data[3]); + break; + case 12: + gcmkPRINT("%08X : %08X %08X %08X", + GpuAddress, data[0], data[1], data[2]); + break; + case 8: + gcmkPRINT("%08X : %08X %08X", + GpuAddress, data[0], data[1]); + break; + case 4: + gcmkPRINT("%08X : %08X", + GpuAddress, data[0]); + break; + default: + break; + } +} + +static void +_DumpKernelCommandBuffer( + IN gckCOMMAND Command + ) +{ + gctINT i; + gctUINT64 physical = 0; + gctUINT32 address; + gctPOINTER entry = gcvNULL; + + for (i = 0; i < gcdCOMMAND_QUEUES; i++) + { + entry = Command->queues[i].logical; + + gckOS_GetPhysicalAddress(Command->os, entry, &physical); + + gcmkPRINT("Kernel command buffer %d\n", i); + + gcmkSAFECASTPHYSADDRT(address, physical); + + _DumpBuffer(entry, address, Command->pageSize); + } +} + +#if !gcdNULL_DRIVER +static gceSTATUS +_HandlePatchList( + IN gckCOMMAND Command, + IN gcoCMDBUF CommandBuffer, + IN gctBOOL NeedCopy, + OUT gctUINT64 *AsyncCommandStamp + ) +{ + gceSTATUS status; + gcsPATCH_LIST * uList; + gcsPATCH_LIST * previous; + gcsPATCH_LIST * kList; + gctUINT64 asyncStamp = 0; + + gcmkHEADER_ARG( + "Command=0x%x CommandBuffer=0x%x NeedCopy=%d", + Command, CommandBuffer, NeedCopy + ); + + uList = gcmUINT64_TO_PTR(CommandBuffer->patchHead); + + while (uList) + { + gctUINT i; + + kList = gcvNULL; + previous = uList; + + gcmkONERROR(gckKERNEL_OpenUserData( + Command->kernel, + NeedCopy, + Command->kList, + uList, + gcmSIZEOF(gcsPATCH_LIST), + (gctPOINTER *)&kList + )); + + for (i = 0; i < kList->count; i++) + { + gctUINT64 stamp = 0; + gcsPATCH * patch = &kList->patch[i]; + + /* Touch video memory node. */ + gcmkVERIFY_OK(gckVIDMEM_SetCommitStamp(Command->kernel, gcvENGINE_RENDER, patch->handle, Command->commitStamp)); + + /* Get stamp touched async command buffer. */ + gcmkVERIFY_OK(gckVIDMEM_GetCommitStamp(Command->kernel, gcvENGINE_BLT, patch->handle, &stamp)); + + /* Find latest one. */ + asyncStamp = gcmMAX(asyncStamp, stamp); + } + + uList = kList->next; + + gcmkVERIFY_OK(gckKERNEL_CloseUserData( + Command->kernel, + NeedCopy, + gcvFALSE, + previous, + gcmSIZEOF(gcsPATCH_LIST), + (gctPOINTER *)&kList + )); + } + + if ((Command->asyncCommand != gcvNULL) + && (*(gctUINT64 *)Command->asyncCommand->fence->logical > asyncStamp) + ) + { + /* No need to wait for async command buffer. */ + *AsyncCommandStamp = 0; + } + else + { + /* Need to add a fence wait. */ + *AsyncCommandStamp = asyncStamp; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (kList) + { + gcmkVERIFY_OK(gckKERNEL_CloseUserData( + Command->kernel, + NeedCopy, + gcvFALSE, + previous, + gcmSIZEOF(gcsPATCH_LIST), + (gctPOINTER *)&kList + )); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_WaitForAsyncCommandStamp( + IN gckCOMMAND Command, + IN gctUINT64 Stamp + ) +{ + gctUINT32 bytes; + gceSTATUS status; + gctUINT32 fenceAddress; + gctUINT32 bufferSize; + gctPOINTER pointer; + gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); + + fenceAddress = Command->asyncCommand->fence->address; + + gcmkONERROR(gckHARDWARE_WaitFence(Command->kernel->hardware, + gcvNULL, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Reserve( + Command, + bytes, + &pointer, + &bufferSize + )); + + gcmkONERROR(gckHARDWARE_WaitFence( + Command->kernel->hardware, + pointer, + Stamp, + fenceAddress, + &bytes + )); + + gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + gcmkFOOTER(); + return status; +} + +/******************************************************************************\ +**************** Helper functions for parsing gcoCMDBUF ************************ +\******************************************************************************/ +static void +_GetCMDBUFSize( + IN gcoCMDBUF CommandBuffer, + OUT gctUINT_PTR CommandBufferSize + ) +{ + *CommandBufferSize + = CommandBuffer->offset + + CommandBuffer->reservedTail + - CommandBuffer->startOffset; +} + +static void +_GetCMDBUFTail( + IN gcoCMDBUF CommandBuffer, + OUT gctUINT8_PTR * Tail + ) +{ + gctUINT8_PTR commandBufferLogical; + gctUINT commandBufferSize; + + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset; + + _GetCMDBUFSize(CommandBuffer, &commandBufferSize); + + *Tail + = commandBufferLogical + + commandBufferSize + - CommandBuffer->reservedTail; +} + +static void +_ParseCMDBUFTail( + IN gckHARDWARE Hardware, + IN gcoCMDBUF CommandBuffer, + OUT gctUINT8_PTR * Fence, + OUT gctUINT8_PTR * Link + ) +{ + gctUINT8_PTR tail; + + _GetCMDBUFTail(CommandBuffer, &tail); + + if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FENCE_64BIT)) + { + *Fence = tail; + *Link = tail + gcdRENDER_FENCE_LENGTH; + } + else + { + *Fence = gcvNULL; + *Link = tail; + } +} + +static gceSTATUS +_GetCMDBUFEntry( + IN gckCOMMAND Command, + IN gcoCMDBUF CommandBuffer, + OUT gctUINT32_PTR EntryAddress, + OUT gctUINT32_PTR EntryBytes + ) +{ + gceSTATUS status; + gctUINT8_PTR commandBufferLogical; + gctUINT commandBufferSize; + gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer; + gctUINT32 commandBufferAddress; + gctUINT offset; + + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) + + CommandBuffer->startOffset; + + /* Get the hardware address. */ + if (Command->kernel->virtualCommandBuffer) + { + gckKERNEL kernel = Command->kernel; + + virtualCommandBuffer = gcmNAME_TO_PTR(CommandBuffer->physical); + + if (virtualCommandBuffer == gcvNULL) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + gcmkONERROR(gckKERNEL_GetGPUAddress( + Command->kernel, + commandBufferLogical, + gcvTRUE, + virtualCommandBuffer, + &commandBufferAddress + )); + } + else + { + gcmkONERROR(gckHARDWARE_ConvertLogical( + Command->kernel->hardware, + commandBufferLogical, + gcvTRUE, + &commandBufferAddress + )); + } + + /* Get offset. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, gcvNULL, gcvPIPE_3D, &offset + )); + + _GetCMDBUFSize(CommandBuffer, &commandBufferSize); + + *EntryAddress = commandBufferAddress + offset; + *EntryBytes = commandBufferSize - offset; + + return gcvSTATUS_OK; + +OnError: + return status; +} + +/******************************************************************************* +** +** Link a list of command buffer together to make them atomic. +** Fence will be added in the last command buffer. +*/ +static gceSTATUS +_ProcessUserCommandBufferList( + IN gckCOMMAND Command, + IN gcoCMDBUF CommandBufferListHead, + OUT gcoCMDBUF * CommandBufferListTail + ) +{ + gceSTATUS status; + gctBOOL needCopy; + + struct _gcoCMDBUF _commandBufferObject; + gcoCMDBUF currentCMDBUF; + struct _gcoCMDBUF _nextCMDBUF; + gcoCMDBUF currentCMDBUFUser = CommandBufferListHead; + + gckOS_QueryNeedCopy(Command->os, 0, &needCopy); + + /* Open first gcoCMDBUF object as currentCMDBUF. */ + gcmkONERROR(gckKERNEL_OpenUserData( + Command->kernel, + needCopy, + &_commandBufferObject, + currentCMDBUFUser, + gcmSIZEOF(struct _gcoCMDBUF), + (gctPOINTER *)¤tCMDBUF + )); + + /* Iterate the list. */ + while (currentCMDBUF->nextCMDBUF != 0) + { + gcoCMDBUF nextCMDBUFUser; + gcoCMDBUF nextCMDBUF; + gctUINT8_PTR fenceLogical = gcvNULL; + gctUINT8_PTR linkLogical; + gctUINT32 linkBytes = 8; + gctUINT32 linkLow; + gctUINT32 linkHigh; + + gctUINT32 entryAddress = 0; + gctUINT32 entryBytes = 0; + + nextCMDBUFUser + = gcmUINT64_TO_PTR(currentCMDBUF->nextCMDBUF); + + /* Open next gcoCMDBUF object as nextCMDBUF. */ + gcmkONERROR(gckKERNEL_OpenUserData( + Command->kernel, + needCopy, + &_nextCMDBUF, + nextCMDBUFUser, + gcmSIZEOF(struct _gcoCMDBUF), + (gctPOINTER *)&nextCMDBUF + )); + + /* Get the start hardware address of nextCMDBUF. */ + gcmkONERROR(_GetCMDBUFEntry(Command, + nextCMDBUF, + &entryAddress, + &entryBytes + )); + + /* Process current gcoCMDBUF object. */ + _ParseCMDBUFTail( + Command->kernel->hardware, + currentCMDBUF, + &fenceLogical, + &linkLogical + ); + + /* Don't send fence in the middle of gcoCMDBUF list. */ + if (fenceLogical != gcvNULL) + { + gctUINT i = gcdRENDER_FENCE_LENGTH / gcmSIZEOF(gctUINT32) / 2; + + /* Fill NOPs in space reserved for fence. */ + while (i--) + { + gctSIZE_T nopBytes = 8; + gcmkONERROR(gckHARDWARE_Nop(Command->kernel->hardware, fenceLogical, &nopBytes)); + fenceLogical += nopBytes; + } + } + + /* Generate a LINK from the end of current command buffer + ** to the start of next command buffer. */ + gcmkONERROR(gckHARDWARE_Link( + Command->kernel->hardware, + linkLogical, + entryAddress, + entryBytes, + &linkBytes, + &linkLow, + &linkHigh + )); + + /* Close current gcoCMDBUF object which is processed. */ + gcmkVERIFY_OK(gckKERNEL_CloseUserData( + Command->kernel, + needCopy, + gcvFALSE, + currentCMDBUFUser, + gcmSIZEOF(struct _gcoCMDBUF), + (gctPOINTER *)¤tCMDBUF + )); + + /* Advance to next gcoCMDBUF object. */ + currentCMDBUFUser = nextCMDBUFUser; + currentCMDBUF = nextCMDBUF; + } + + gcmkVERIFY_OK(gckKERNEL_CloseUserData( + Command->kernel, + needCopy, + gcvFALSE, + currentCMDBUFUser, + gcmSIZEOF(struct _gcoCMDBUF), + (gctPOINTER *)¤tCMDBUF + )); + + /* Return the tail of the list. */ + *CommandBufferListTail = currentCMDBUFUser; + + return gcvSTATUS_OK; + +OnError: + return status; +} +#endif + +/******************************************************************************\ +****************************** gckCOMMAND API Code ****************************** +\******************************************************************************/ + +/******************************************************************************* +** +** gckCOMMAND_Construct +** +** Construct a new gckCOMMAND object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckCOMMAND * Command +** Pointer to a variable that will hold the pointer to the gckCOMMAND +** object. +*/ +gceSTATUS +gckCOMMAND_Construct( + IN gckKERNEL Kernel, + OUT gckCOMMAND * Command + ) +{ + gckOS os; + gckCOMMAND command = gcvNULL; + gceSTATUS status; + gctINT i; + gctPOINTER pointer = gcvNULL; + gctSIZE_T pageSize; + + gcmkHEADER_ARG("Kernel=0x%x", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Command != gcvNULL); + + /* Extract the gckOS object. */ + os = Kernel->os; + + /* Allocate the gckCOMMAND structure. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); + command = pointer; + + /* Reset the entire object. */ + gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); + + /* Initialize the gckCOMMAND object.*/ + command->object.type = gcvOBJ_COMMAND; + command->kernel = Kernel; + command->os = os; + + /* Get the command buffer requirements. */ + gcmkONERROR(gckHARDWARE_QueryCommandBuffer( + Kernel->hardware, + gcvENGINE_RENDER, + &command->alignment, + &command->reservedHead, + gcvNULL + )); + + /* Create the command queue mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); + + /* Create the context switching mutex. */ + gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); + + /* Create the power management semaphore. */ + gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); + + /* Create the commit atom. */ + gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); + + /* Get the page size from teh OS. */ + gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); + + gcmkSAFECASTSIZET(command->pageSize, pageSize); + + /* Get process ID. */ + gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); + + /* Set hardware to pipe 0. */ + command->pipeSelect = gcvPIPE_INVALID; + + /* Pre-allocate the command queues. */ + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { +#if USE_KERNEL_VIRTUAL_BUFFERS + if (Kernel->virtualCommandBuffer) + { + gcmkONERROR(gckKERNEL_AllocateVirtualCommandBuffer( + Kernel, + gcvFALSE, + &pageSize, + &command->queues[i].physical, + &command->queues[i].logical + )); + + gcmkONERROR(gckKERNEL_GetGPUAddress( + Kernel, + command->queues[i].logical, + gcvFALSE, + command->queues[i].physical, + &command->queues[i].address + )); + } + else +#endif + { + gcmkONERROR(gckOS_AllocateNonPagedMemory( + os, + gcvFALSE, + &pageSize, + &command->queues[i].physical, + &command->queues[i].logical + )); + + gcmkONERROR(gckHARDWARE_ConvertLogical( + Kernel->hardware, + command->queues[i].logical, + gcvFALSE, + &command->queues[i].address + )); + + gcmkONERROR(gckMMU_FillFlatMapping( + Kernel->mmu, command->queues[i].address, pageSize + )); + } + + gcmkONERROR(gckOS_CreateSignal( + os, gcvFALSE, &command->queues[i].signal + )); + + gcmkONERROR(gckOS_Signal( + os, command->queues[i].signal, gcvTRUE + )); + } + +#if gcdRECORD_COMMAND + gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); +#endif + + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsPATCH_LIST), &command->kList)); + + gcmkONERROR(gckFENCE_Create( + os, Kernel, &command->fence + )); + + /* No command queue in use yet. */ + command->index = -1; + command->logical = gcvNULL; + command->newQueue = gcvFALSE; + + /* Command is not yet running. */ + command->running = gcvFALSE; + + /* Command queue is idle. */ + command->idle = gcvTRUE; + + /* Commit stamp start from 1. */ + command->commitStamp = 1; + + /* END event signal not created. */ + command->endEventSignal = gcvNULL; + + command->dummyDraw = gcvTRUE; + + /* Return pointer to the gckCOMMAND object. */ + *Command = command; + + /* Success. */ + gcmkFOOTER_ARG("*Command=0x%x", *Command); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (command != gcvNULL) + { + gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Destroy +** +** Destroy an gckCOMMAND object. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Destroy( + IN gckCOMMAND Command + ) +{ + gctINT i; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Stop the command queue. */ + gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); + + for (i = 0; i < gcdCOMMAND_QUEUES; ++i) + { + if (Command->queues[i].signal) + { + gcmkVERIFY_OK(gckOS_DestroySignal( + Command->os, Command->queues[i].signal + )); + } + + if (Command->queues[i].logical) + { +#if USE_KERNEL_VIRTUAL_BUFFERS + if (Command->kernel->virtualCommandBuffer) + { + gcmkVERIFY_OK(gckKERNEL_DestroyVirtualCommandBuffer( + Command->kernel, + Command->pageSize, + Command->queues[i].physical, + Command->queues[i].logical + )); + } + else +#endif + { + gcmkVERIFY_OK(gckOS_FreeNonPagedMemory( + Command->os, + Command->pageSize, + Command->queues[i].physical, + Command->queues[i].logical + )); + } + } + } + + /* END event signal. */ + if (Command->endEventSignal != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DestroySignal( + Command->os, Command->endEventSignal + )); + } + + if (Command->mutexContext) + { + /* Delete the context switching mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); + } + + if (Command->mutexContextSeq != gcvNULL) + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); + + if (Command->mutexQueue) + { + /* Delete the command queue mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); + } + + if (Command->powerSemaphore) + { + /* Destroy the power management semaphore. */ + gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); + } + + if (Command->atomCommit) + { + /* Destroy the commit atom. */ + gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); + } + +#if gcdSECURE_USER + /* Free state array. */ + if (Command->hintArrayAllocated) + { + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); + Command->hintArrayAllocated = gcvFALSE; + } +#endif + +#if gcdRECORD_COMMAND + gckRECORDER_Destory(Command->os, Command->recorder); +#endif + + if (Command->stateMap) + { + gcmkOS_SAFE_FREE(Command->os, Command->stateMap); + } + + if (Command->kList) + { + gcmkOS_SAFE_FREE(Command->os, Command->kList); + } + + if (Command->fence) + { + gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); + } + + /* Mark object as unknown. */ + Command->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckCOMMAND object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckCOMMAND_EnterCommit +** +** Acquire command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_EnterCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctBOOL atomIncremented = gcvFALSE; + gctBOOL semaAcquired = gcvFALSE; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (!FromPower) + { + /* Increment COMMIT atom to let power management know that a commit is + ** in progress. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); + atomIncremented = gcvTRUE; + + /* Notify the system the GPU has a commit. */ + gcmkONERROR(gckOS_Broadcast(Command->os, + hardware, + gcvBROADCAST_GPU_COMMIT)); + + /* Acquire the power management semaphore. */ + gcmkONERROR(gckOS_AcquireSemaphore(Command->os, + Command->powerSemaphore)); + semaAcquired = gcvTRUE; + } + + /* Grab the conmmand queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Command->os, + Command->mutexQueue, + gcvINFINITE)); + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (semaAcquired) + { + /* Release the power management semaphore. */ + gcmkVERIFY_OK(gckOS_ReleaseSemaphore( + Command->os, Command->powerSemaphore + )); + } + + if (atomIncremented) + { + /* Decrement the commit atom. */ + gcmkVERIFY_OK(_IncrementCommitAtom( + Command, gcvFALSE + )); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_ExitCommit +** +** Release command queue synchronization objects. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to destroy. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_ExitCommit( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Release the power mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); + + if (!FromPower) + { + /* Release the power management semaphore. */ + gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, + Command->powerSemaphore)); + + /* Decrement the commit atom. */ + gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); + } + + /* Success. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Start +** +** Start up the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to start. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Start( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckHARDWARE hardware; + gctUINT32 waitOffset = 0; + gctUINT32 waitLinkBytes; + gctPOINTER logical; + gctUINT32 physical; + gctUINT32 address; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (Command->running) + { + /* Command queue already running. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckHARDWARE_WaitLink( + hardware, + gcvNULL, + ~0U, + Command->offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + if ((Command->pageSize - Command->offset < waitLinkBytes) + || (Command->logical == gcvNULL) + ) + { + /* Start at beginning of a new queue. */ + gcmkONERROR(_NewQueue(Command, gcvTRUE)); + } + + logical = (gctUINT8_PTR) Command->logical + Command->offset; + physical = Command->physical + Command->offset; + address = Command->address + Command->offset; + + /* Append WAIT/LINK. */ + gcmkONERROR(gckHARDWARE_WaitLink( + hardware, + logical, + address, + 0, + &waitLinkBytes, + &waitOffset, + &Command->waitSize + )); + + Command->waitLogical = (gctUINT8_PTR) logical + waitOffset; + Command->waitPhysical = physical + waitOffset; + Command->waitAddress = address + waitOffset; + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache for the wait/link. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + physical, + logical, + waitLinkBytes + )); +#endif + + /* Adjust offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + +#if gcdSECURITY + /* Start FE by calling security service. */ + gckKERNEL_SecurityStartCommand( + Command->kernel + ); +#else + /* Enable command processor. */ + gcmkONERROR(gckHARDWARE_Execute( + hardware, + address, + waitLinkBytes + )); +#endif + + /* Command queue is running. */ + Command->running = gcvTRUE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stop +** +** Stop the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object to stop. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stop( + IN gckCOMMAND Command + ) +{ + gckHARDWARE hardware; + gceSTATUS status; + gctUINT32 idle; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + if (!Command->running) + { + /* Command queue is not running. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + /* Extract the gckHARDWARE object. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (gckHARDWARE_IsFeatureAvailable(hardware, + gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE) + { + /* Allocate the signal. */ + if (Command->endEventSignal == gcvNULL) + { + gcmkONERROR(gckOS_CreateSignal(Command->os, + gcvTRUE, + &Command->endEventSignal)); + } + + /* Append the END EVENT command to trigger the signal. */ + gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj, + Command->kernelProcessID, + Command->waitPhysical, + Command->waitLogical, + Command->waitAddress, + Command->endEventSignal, + &Command->waitSize)); + } + else + { + /* Replace last WAIT with END. */ + gcmkONERROR(gckHARDWARE_End( + hardware, + Command->waitLogical, + Command->waitAddress, + &Command->waitSize + )); + +#if USE_KERNEL_VIRTUAL_BUFFERS + if (hardware->kernel->virtualCommandBuffer) + { + gcmkONERROR(gckKERNEL_GetGPUAddress( + hardware->kernel, + Command->waitLogical, + gcvFALSE, + Command->virtualMemory, + &hardware->lastEnd + )); + } +#endif + +#if gcdSECURITY + gcmkONERROR(gckKERNEL_SecurityExecute( + Command->kernel, Command->waitLogical, 8 + )); +#endif + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, + Command->logical, + Command->offset)); + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache for the END. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)Command->waitPhysical, + Command->waitLogical, + Command->waitSize + )); +#endif + + /* Wait for idle. */ + gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); + } + + /* Command queue is no longer running. */ + Command->running = gcvFALSE; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Commit +** +** Commit a command buffer to the command queue. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gckCONTEXT Context +** Pointer to a gckCONTEXT object. +** +** gcoCMDBUF CommandBuffer +** Pointer to a gcoCMDBUF object. +** +** gcsSTATE_DELTA_PTR StateDelta +** Pointer to the state delta. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Commit( + IN gckCOMMAND Command, + IN gckCONTEXT Context, + IN gcoCMDBUF CommandBuffer, + IN gcsSTATE_DELTA_PTR StateDelta, + IN gctUINT32 ProcessID, + IN gctBOOL Shared, + IN gctUINT32 Index, + OUT gctUINT64_PTR CommitStamp, + OUT gctBOOL_PTR ContextSwitched + ) +{ + gceSTATUS status; + gctBOOL commitEntered = gcvFALSE; + gctBOOL contextAcquired = gcvFALSE; + gckHARDWARE hardware; + gctBOOL needCopy = gcvFALSE; + gctBOOL commandBufferMapped = gcvFALSE; + gcoCMDBUF commandBufferObject = gcvNULL; + gctBOOL stall = gcvFALSE; + gctBOOL contextSwitched = gcvFALSE; + +#if !gcdNULL_DRIVER + gcsCONTEXT_PTR contextBuffer; + struct _gcoCMDBUF _commandBufferObject; + gctPHYS_ADDR_T commandBufferPhysical; + gctUINT8_PTR commandBufferLogical = gcvNULL; + gctUINT32 commandBufferAddress = 0; + gctUINT8_PTR commandBufferTail = gcvNULL; + gctUINT commandBufferSize; + gctSIZE_T nopBytes; + gctUINT32 pipeBytes; + gctUINT32 linkBytes; + gctSIZE_T bytes; + gctUINT32 offset; +#if gcdNONPAGED_MEMORY_CACHEABLE + gctPHYS_ADDR entryPhysical; +#endif + gctPOINTER entryLogical; + gctUINT32 entryAddress; + gctUINT32 entryBytes; +#if gcdNONPAGED_MEMORY_CACHEABLE + gctPHYS_ADDR exitPhysical; + gctPOINTER exitLogical; +#endif + gctUINT32 exitAddress; + gctUINT32 exitBytes; + gctUINT32 waitLinkPhysical; + gctPOINTER waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkBytes; + gctUINT32 waitOffset; + gctUINT32 waitSize; + +#ifdef __QNXNTO__ + gctPOINTER userCommandBufferLogical = gcvNULL; + gctBOOL userCommandBufferLogicalMapped = gcvFALSE; +#endif + +#if gcdPROCESS_ADDRESS_SPACE + gckMMU mmu; + gctUINT32 oldValue; +#endif + +#if gcdDUMP_COMMAND + gctPOINTER contextDumpLogical = gcvNULL; + gctSIZE_T contextDumpBytes = 0; + gctPOINTER bufferDumpLogical = gcvNULL; + gctSIZE_T bufferDumpBytes = 0; +# endif + gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; + gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; + gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; + + gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL; + gctUINT64 asyncCommandStamp = 0; + gcoCMDBUF lastCommandBuffer = gcvNULL; + gctPOINTER pointer = gcvNULL; + +#endif + + gcmkHEADER_ARG( + "Command=0x%x CommandBuffer=0x%x ProcessID=%d", + Command, CommandBuffer, ProcessID + ); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + +#if !gcdNULL_DRIVER + gcmkONERROR(_ProcessUserCommandBufferList( + Command, + CommandBuffer, + &lastCommandBuffer + )); +#endif + +#if gcdPROCESS_ADDRESS_SPACE + gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu)); + + gcmkONERROR(gckOS_AtomicExchange(Command->os, + mmu->pageTableDirty[Command->kernel->core], + 0, + &oldValue)); +#else +#endif + + /* Acquire the command queue. */ + gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); + commitEntered = gcvTRUE; + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + contextAcquired = gcvTRUE; + + /* Extract the gckHARDWARE and gckEVENT objects. */ + hardware = Command->kernel->hardware; + + /* Check wehther we need to copy the structures or not. */ + gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); + +#if gcdNULL_DRIVER + /* Context switch required? */ + if ((Context != gcvNULL) && (Command->currContext != Context)) + { + /* Yes, merge in the deltas. */ + gckCONTEXT_Update(Context, ProcessID, StateDelta); + + /* Update the current context. */ + Command->currContext = Context; + + contextSwitched = gcvTRUE; + } +#else + if (needCopy) + { + commandBufferObject = &_commandBufferObject; + + gcmkONERROR(gckOS_CopyFromUserData( + Command->os, + commandBufferObject, + CommandBuffer, + gcmSIZEOF(struct _gcoCMDBUF) + )); + + gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); + } + else + { + gcmkONERROR(gckOS_MapUserPointer( + Command->os, + CommandBuffer, + gcmSIZEOF(struct _gcoCMDBUF), + &pointer + )); + + commandBufferObject = pointer; + + gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); + commandBufferMapped = gcvTRUE; + } + + gcmkONERROR(_HandlePatchList(Command, commandBufferObject, needCopy, &asyncCommandStamp)); + + /* Query the size of NOP command. */ + gcmkONERROR(gckHARDWARE_Nop( + hardware, gcvNULL, &nopBytes + )); + + /* Query the size of pipe select command sequence. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + hardware, gcvNULL, gcvPIPE_3D, &pipeBytes + )); + + /* Query the size of LINK command. */ + gcmkONERROR(gckHARDWARE_Link( + hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL + )); + + /* Compute the command buffer entry and the size. */ + commandBufferLogical + = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical) + + commandBufferObject->startOffset; + + /* Get the hardware address. */ + if (Command->kernel->virtualCommandBuffer) + { + gckKERNEL kernel = Command->kernel; + + virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical); + + if (virtualCommandBuffer == gcvNULL) + { + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + gcmkONERROR(gckKERNEL_GetGPUAddress( + Command->kernel, + commandBufferLogical, + gcvTRUE, + virtualCommandBuffer, + &commandBufferAddress + )); + } + else + { + gcmkONERROR(gckHARDWARE_ConvertLogical( + hardware, + commandBufferLogical, + gcvTRUE, + &commandBufferAddress + )); + } + +#ifdef __QNXNTO__ + userCommandBufferLogical = (gctPOINTER) commandBufferLogical; + + gcmkONERROR(gckOS_MapUserPointer( + Command->os, + userCommandBufferLogical, + 0, + &pointer)); + + commandBufferLogical = pointer; + + userCommandBufferLogicalMapped = gcvTRUE; + + gcmkONERROR(gckOS_GetPhysicalAddress( + Command->os, + commandBufferLogical, + &commandBufferPhysical + )); +#else + /* Get the physical address. */ + gcmkONERROR(gckOS_UserLogicalToPhysical( + Command->os, + commandBufferLogical, + &commandBufferPhysical + )); +#endif + + commandBufferSize + = commandBufferObject->offset + + commandBufferObject->reservedTail + - commandBufferObject->startOffset; + + gcmkONERROR(_FlushMMU(Command)); + + if (Command->dummyDraw == gcvTRUE && + Context != gcvNULL) + { + Command->dummyDraw = gcvFALSE; + gcmkONERROR(_DummyDraw(Command)); + } + + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && asyncCommandStamp != 0) + { + gcmkONERROR(_WaitForAsyncCommandStamp(Command, asyncCommandStamp)); + } + + /* Get the current offset. */ + offset = Command->offset; + + /* Compute number of bytes left in current kernel command queue. */ + bytes = Command->pageSize - offset; + + /* Query the size of WAIT/LINK command sequence. */ + gcmkONERROR(gckHARDWARE_WaitLink( + hardware, + gcvNULL, + ~0U, + offset, + &waitLinkBytes, + gcvNULL, + gcvNULL + )); + + /* Is there enough space in the current command queue? */ + if (bytes < waitLinkBytes) + { + /* No, create a new one. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Get the new current offset. */ + offset = Command->offset; + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - offset; + gcmkASSERT(bytes >= waitLinkBytes); + } + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkPhysical = Command->physical + offset; + waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; + waitLinkAddress = Command->address + offset; + + /* Context switch required? */ + if (Context == gcvNULL) + { + /* See if we have to switch pipes for the command buffer. */ + if (commandBufferObject->entryPipe == Command->pipeSelect) + { + /* Skip pipe switching sequence. */ + offset = pipeBytes; + } + else + { + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + commandBufferObject->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + } + + /* Compute the entry. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset; +#endif + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + + Command->currContext = gcvNULL; + } +#if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE + else if (1) +#else + else if (Command->currContext != Context) +#endif + { + /* Get the current context buffer. */ + contextBuffer = Context->buffer; + + /* Yes, merge in the deltas. */ + gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); + + contextSwitched = gcvTRUE; + + /*************************************************************** + ** SWITCHING CONTEXT. + */ + + /* Determine context buffer entry offset. */ + offset = (Command->pipeSelect == gcvPIPE_3D) + + /* Skip pipe switching sequence. */ + ? Context->entryOffset3D + Context->pipeSelectBytes + + /* Do not skip pipe switching sequence. */ + : Context->entryOffset3D; + + /* Compute the entry. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + entryPhysical = (gctUINT8_PTR) contextBuffer->physical + offset; +#endif + entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; + entryAddress = contextBuffer->address + offset; + entryBytes = Context->bufferSize - offset; + + /* See if we have to switch pipes between the context + and command buffers. */ + if (commandBufferObject->entryPipe == gcvPIPE_3D) + { + /* Skip pipe switching sequence. */ + offset = pipeBytes; + } + else + { + /* The current hardware and the initial context pipes are + different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + commandBufferObject->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + } + + /* Generate a LINK from the context buffer to + the command buffer. */ + gcmkONERROR(gckHARDWARE_Link( + hardware, + contextBuffer->link3D, + commandBufferAddress + offset, + commandBufferSize - offset, + &linkBytes, + &commandLinkLow, + &commandLinkHigh + )); + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the context buffer cache. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)entryPhysical, + entryLogical, + entryBytes + )); +#endif + + /* Update the current context. */ + Command->currContext = Context; + +#if gcdDUMP_COMMAND + contextDumpLogical = entryLogical; + contextDumpBytes = entryBytes; +#endif + +#if gcdSECURITY + /* Commit context buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + entryLogical, + entryBytes - 8 + ); +#endif + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + gcvNULL, + 0xFFFFFFFF, + entryLogical, + entryBytes + ); +#endif + } + + /* Same context. */ + else + { + /* See if we have to switch pipes for the command buffer. */ + if (commandBufferObject->entryPipe == Command->pipeSelect) + { + /* Skip pipe switching sequence. */ + offset = pipeBytes; + } + else + { + /* The current hardware and the entry command buffer pipes + ** are different, switch to the correct pipe. */ + gcmkONERROR(gckHARDWARE_PipeSelect( + Command->kernel->hardware, + commandBufferLogical, + commandBufferObject->entryPipe, + &pipeBytes + )); + + /* Do not skip pipe switching sequence. */ + offset = 0; + } + + /* Compute the entry. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + entryPhysical = (gctUINT8_PTR) commandBufferPhysical + offset; +#endif + entryLogical = commandBufferLogical + offset; + entryAddress = commandBufferAddress + offset; + entryBytes = commandBufferSize - offset; + } + +#if gcdDUMP_COMMAND + bufferDumpLogical = commandBufferLogical + offset; + bufferDumpBytes = commandBufferSize - offset; +#endif + +#if gcdSECURE_USER + /* Process user hints. */ + gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject)); +#endif + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + exitPhysical = Command->physical; + exitLogical = Command->logical; +#endif + exitAddress = Command->address; + exitBytes = Command->offset + waitLinkBytes; + } + else + { + /* Still within the preexisting command queue, jump to the new + WAIT/LINK command sequence. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + exitPhysical = waitLinkPhysical; + exitLogical = waitLinkLogical; +#endif + exitAddress = waitLinkAddress; + exitBytes = waitLinkBytes; + } + + /* Add a new WAIT/LINK command sequence. When the command buffer which is + currently being scheduled is fully executed by the GPU, the FE will + jump to this WAIT/LINK sequence. */ + gcmkONERROR(gckHARDWARE_WaitLink( + hardware, + waitLinkLogical, + waitLinkAddress, + offset, + &waitLinkBytes, + &waitOffset, + &waitSize + )); + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the command queue cache. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)exitPhysical, + exitLogical, + exitBytes + )); +#endif + + /* Determine the location of the TAIL in the command buffer. */ + commandBufferTail + = commandBufferLogical + + commandBufferSize + - commandBufferObject->reservedTail; + + /* Generate command which writes out commit stamp. */ + if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) + { + gctUINT32 bytes; + + gcmkONERROR(gckHARDWARE_Fence( + hardware, + gcvENGINE_RENDER, + commandBufferTail, + Command->fence->address, + Command->commitStamp, + &bytes + )); + + commandBufferTail += gcdRENDER_FENCE_LENGTH; + } + + /* Generate a LINK from the end of the command buffer being scheduled + back to the kernel command queue. */ +#if !gcdSECURITY + if (Shared == gcvFALSE) + { + gcmkONERROR(gckHARDWARE_Link( + hardware, + commandBufferTail, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + } + else + { + gctUINT8_PTR link = commandBufferTail + Index * 16; + gctSIZE_T bytes = 8; + + gcmkONERROR(gckHARDWARE_ChipEnable( + hardware, + link, + (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), + &bytes + )); + + link += bytes; + + gcmkONERROR(gckHARDWARE_Link( + hardware, + link, + exitAddress, + exitBytes, + &linkBytes, + &exitLinkLow, + &exitLinkHigh + )); + + link += linkBytes; + } +#endif + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the command buffer cache. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + ProcessID, + gcvNULL, + (gctUINT32)commandBufferPhysical, + commandBufferLogical, + commandBufferSize + )); +#endif + +#if gcdRECORD_COMMAND + gckRECORDER_Record( + Command->recorder, + commandBufferLogical + offset, + commandBufferSize - offset, + gcvNULL, + 0xFFFFFFFF + ); + + gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); +#endif + +#if gcdSECURITY + /* Submit command buffer to trust zone. */ + gckKERNEL_SecurityExecute( + Command->kernel, + commandBufferLogical + offset, + commandBufferSize - offset - 8 + ); +#else + /* Generate a LINK from the previous WAIT/LINK command sequence to the + entry determined above (either the context or the command buffer). + This LINK replaces the WAIT instruction from the previous WAIT/LINK + pair, therefore we use WAIT metrics for generation of this LINK. + This action will execute the entire sequence. */ + gcmkONERROR(gckHARDWARE_Link( + hardware, + Command->waitLogical, + entryAddress, + entryBytes, + &Command->waitSize, + &entryLinkLow, + &entryLinkHigh + )); +#endif + +#if gcdLINK_QUEUE_SIZE + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = entryAddress; + data.linkData.end = entryAddress + entryBytes; + data.linkData.linkLow = entryLinkLow; + data.linkData.linkHigh = entryLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + + if (commandBufferAddress + offset != entryAddress) + { + data.linkData.start = commandBufferAddress + offset; + data.linkData.end = commandBufferAddress + commandBufferSize; + data.linkData.linkLow = commandLinkLow; + data.linkData.linkHigh = commandLinkHigh; + + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + data.linkData.start = exitAddress; + data.linkData.end = exitAddress + exitBytes; + data.linkData.linkLow = exitLinkLow; + data.linkData.linkHigh = exitLinkHigh; + + /* Dump kernel command.*/ + gckQUEUE_Enqueue(&hardware->linkQueue, &data); + } + } +#endif + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache for the link. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)Command->waitPhysical, + Command->waitLogical, + Command->waitSize + )); +#endif + + gcmkDUMPCOMMAND( + Command->os, + Command->waitLogical, + Command->waitSize, + gcvDUMP_BUFFER_LINK, + gcvFALSE + ); + + gcmkDUMPCOMMAND( + Command->os, + contextDumpLogical, + contextDumpBytes, + gcvDUMP_BUFFER_CONTEXT, + gcvFALSE + ); + + gcmkDUMPCOMMAND( + Command->os, + bufferDumpLogical, + bufferDumpBytes, + gcvDUMP_BUFFER_USER, + gcvFALSE + ); + + gcmkDUMPCOMMAND( + Command->os, + waitLinkLogical, + waitLinkBytes, + gcvDUMP_BUFFER_WAITLINK, + gcvFALSE + ); + + /* Update the current pipe. */ + Command->pipeSelect = commandBufferObject->exitPipe; + + /* Update command queue offset. */ + Command->offset += waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update address of last WAIT. */ + Command->waitPhysical = waitLinkPhysical + waitOffset; + Command->waitLogical = (gctUINT8_PTR)waitLinkLogical + waitOffset; + Command->waitAddress = waitLinkAddress + waitOffset; + Command->waitSize = waitSize; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + hardware, Command->logical, Command->offset + )); + +#if gcdDUMP_COMMAND + gcmkPRINT("@[kernel.commit]"); +#endif +#endif /* gcdNULL_DRIVER */ + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + contextAcquired = gcvFALSE; + + *CommitStamp = Command->commitStamp; + *ContextSwitched = contextSwitched; + + Command->commitStamp++; + + stall = gcvFALSE; + +#if gcdLINK_QUEUE_SIZE + if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND) + { + if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0) + { + /* If only context buffer and command buffer is recorded, + ** each commit costs 2 slot in queue, to make sure command + ** causing stuck is recorded, number of pending command buffer + ** is limited to (gckLINK_QUEUE_SIZE/2) + */ + stall = gcvTRUE; + } + } +#endif + + /* Release the command queue. */ + gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + commitEntered = gcvFALSE; + + if (status == gcvSTATUS_INTERRUPTED) + { + gcmkTRACE( + gcvLEVEL_INFO, + "%s(%d): Intterupted in gckEVENT_Submit", + __FUNCTION__, __LINE__ + ); + status = gcvSTATUS_OK; + } + else + { + gcmkONERROR(status); + } + +#ifdef __QNXNTO__ + if (userCommandBufferLogicalMapped) + { + gcmkONERROR(gckOS_UnmapUserPointer( + Command->os, + userCommandBufferLogical, + 0, + commandBufferLogical)); + + userCommandBufferLogicalMapped = gcvFALSE; + } +#endif + + /* Unmap the command buffer pointer. */ + if (commandBufferMapped) + { + gcmkONERROR(gckOS_UnmapUserPointer( + Command->os, + CommandBuffer, + gcmSIZEOF(struct _gcoCMDBUF), + commandBufferObject + )); + + commandBufferMapped = gcvFALSE; + } + + /* Return status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + if (contextAcquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); + } + +#ifdef __QNXNTO__ + if (userCommandBufferLogicalMapped) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + userCommandBufferLogical, + 0, + commandBufferLogical)); + } +#endif + + /* Unmap the command buffer pointer. */ + if (commandBufferMapped) + { + gcmkVERIFY_OK(gckOS_UnmapUserPointer( + Command->os, + CommandBuffer, + gcmSIZEOF(struct _gcoCMDBUF), + commandBufferObject + )); + } + + /* Return status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Reserve +** +** Reserve space in the command queue. Also acquire the command queue mutex. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** gctPOINTER * Buffer +** Pointer to a variable that will receive the address of the reserved +** space. +** +** gctSIZE_T * BufferSize +** Pointer to a variable that will receive the number of bytes +** available in the command queue. +*/ +gceSTATUS +gckCOMMAND_Reserve( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes, + OUT gctPOINTER * Buffer, + OUT gctUINT32 * BufferSize + ) +{ + gceSTATUS status; + gctUINT32 bytes; + gctUINT32 requiredBytes; + gctUINT32 requestedAligned; + + gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Compute aligned number of reuested bytes. */ + requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); + + /* Another WAIT/LINK command sequence will have to be appended after + the requested area being reserved. Compute the number of bytes + required for WAIT/LINK at the location after the reserved area. */ + gcmkONERROR(gckHARDWARE_WaitLink( + Command->kernel->hardware, + gcvNULL, + ~0U, + Command->offset + requestedAligned, + &requiredBytes, + gcvNULL, + gcvNULL + )); + + /* Compute total number of bytes required. */ + requiredBytes += requestedAligned; + + /* Compute number of bytes available in command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Is there enough space in the current command queue? */ + if (bytes < requiredBytes) + { + /* Create a new command queue. */ + gcmkONERROR(_NewQueue(Command, gcvFALSE)); + + /* Recompute the number of bytes in the new kernel command queue. */ + bytes = Command->pageSize - Command->offset; + + /* Still not enough space? */ + if (bytes < requiredBytes) + { + /* Rare case, not enough room in command queue. */ + gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); + } + } + + /* Return pointer to empty slot command queue. */ + *Buffer = (gctUINT8 *) Command->logical + Command->offset; + + /* Return number of bytes left in command queue. */ + *BufferSize = bytes; + + /* Success. */ + gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); + return gcvSTATUS_OK; + +OnError: + /* Return status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Execute +** +** Execute a previously reserved command queue by appending a WAIT/LINK command +** sequence after it and modifying the last WAIT into a LINK command. The +** command FIFO mutex will be released whether this function succeeds or not. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctSIZE_T RequestedBytes +** Number of bytes previously reserved. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Execute( + IN gckCOMMAND Command, + IN gctUINT32 RequestedBytes + ) +{ + gceSTATUS status; + + gctUINT32 waitLinkPhysical; + gctUINT8_PTR waitLinkLogical; + gctUINT32 waitLinkAddress; + gctUINT32 waitLinkOffset; + gctUINT32 waitLinkBytes; + + gctUINT32 waitPhysical; + gctPOINTER waitLogical; + gctUINT32 waitAddress; + gctUINT32 waitOffset; + gctUINT32 waitBytes; + + gctUINT32 linkLow, linkHigh; + +#if gcdNONPAGED_MEMORY_CACHEABLE + gctPHYS_ADDR execPhysical; +#endif + gctPOINTER execLogical; + gctUINT32 execAddress; + gctUINT32 execBytes; + + gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Compute offset for WAIT/LINK. */ + waitLinkOffset = Command->offset + RequestedBytes; + + /* Compute number of bytes left in command queue. */ + waitLinkBytes = Command->pageSize - waitLinkOffset; + + /* Compute the location if WAIT/LINK command sequence. */ + waitLinkPhysical = Command->physical + waitLinkOffset; + waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; + waitLinkAddress = Command->address + waitLinkOffset; + + /* Append WAIT/LINK in command queue. */ + gcmkONERROR(gckHARDWARE_WaitLink( + Command->kernel->hardware, + waitLinkLogical, + waitLinkAddress, + waitLinkOffset, + &waitLinkBytes, + &waitOffset, + &waitBytes + )); + + /* Compute the location if WAIT command. */ + waitPhysical = waitLinkPhysical + waitOffset; + waitLogical = waitLinkLogical + waitOffset; + waitAddress = waitLinkAddress + waitOffset; + + /* Determine the location to jump to for the command buffer being + ** scheduled. */ + if (Command->newQueue) + { + /* New command queue, jump to the beginning of it. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + execPhysical = Command->physical; +#endif + execLogical = Command->logical; + execAddress = Command->address; + execBytes = waitLinkOffset + waitLinkBytes; + } + else + { + /* Still within the preexisting command queue, jump directly to the + reserved area. */ +#if gcdNONPAGED_MEMORY_CACHEABLE + execPhysical = (gctUINT8 *) Command->physical + Command->offset; +#endif + execLogical = (gctUINT8 *) Command->logical + Command->offset; + execAddress = Command->address + Command->offset; + execBytes = RequestedBytes + waitLinkBytes; + } + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)execPhysical, + execLogical, + execBytes + )); +#endif + + /* Convert the last WAIT into a LINK. */ + gcmkONERROR(gckHARDWARE_Link( + Command->kernel->hardware, + Command->waitLogical, + execAddress, + execBytes, + &Command->waitSize, + &linkLow, + &linkHigh + )); + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache. */ + gcmkONERROR(gckOS_CacheClean( + Command->os, + Command->kernelProcessID, + gcvNULL, + (gctUINT32)Command->waitPhysical, + Command->waitLogical, + Command->waitSize + )); +#endif + +#if gcdLINK_QUEUE_SIZE + if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) + { + gcuQUEUEDATA data; + + gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); + + data.linkData.start = execAddress; + data.linkData.end = execAddress + execBytes; + data.linkData.linkLow = linkLow; + data.linkData.linkHigh = linkHigh; + + gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); + } +#endif + + gcmkDUMPCOMMAND( + Command->os, + Command->waitLogical, + Command->waitSize, + gcvDUMP_BUFFER_LINK, + gcvFALSE + ); + + gcmkDUMPCOMMAND( + Command->os, + execLogical, + execBytes, + gcvDUMP_BUFFER_KERNEL, + gcvFALSE + ); + + /* Update the pointer to the last WAIT. */ + Command->waitPhysical = waitPhysical; + Command->waitLogical = waitLogical; + Command->waitAddress = waitAddress; + Command->waitSize = waitBytes; + + /* Update the command queue. */ + Command->offset += RequestedBytes + waitLinkBytes; + Command->newQueue = gcvFALSE; + + /* Update queue tail pointer. */ + gcmkONERROR(gckHARDWARE_UpdateQueueTail( + Command->kernel->hardware, Command->logical, Command->offset + )); + +#if gcdDUMP_COMMAND + gcmkPRINT("@[kernel.execute]"); +#endif + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_Stall +** +** The calling thread will be suspended until the command queue has been +** completed. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to an gckCOMMAND object. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Stall( + IN gckCOMMAND Command, + IN gctBOOL FromPower + ) +{ +#if gcdNULL_DRIVER + /* Do nothing with infinite hardware. */ + return gcvSTATUS_OK; +#else + gckOS os; + gckHARDWARE hardware; + gckEVENT eventObject; + gceSTATUS status; + gctSIGNAL signal = gcvNULL; + gctUINT timer = 0; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Extract the gckOS object pointer. */ + os = Command->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Extract the gckHARDWARE object pointer. */ + hardware = Command->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Extract the gckEVENT object pointer. */ + eventObject = Command->kernel->eventObj; + gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); + + /* Allocate the signal. */ + gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); + + /* Append the EVENT command to trigger the signal. */ + gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); + + /* Submit the event queue. */ + gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); + +#if gcdDUMP_COMMAND + gcmkPRINT("@[kernel.stall]"); +#endif + + if (status == gcvSTATUS_CHIP_NOT_READY) + { + /* Error. */ + goto OnError; + } + + do + { + /* Wait for the signal. */ + status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); + + if (status == gcvSTATUS_TIMEOUT) + { +#if gcmIS_DEBUG(gcdDEBUG_CODE) + gctUINT32 idle; + + /* Read idle register. */ + gcmkVERIFY_OK(gckHARDWARE_GetIdle( + hardware, gcvFALSE, &idle + )); + + gcmkTRACE( + gcvLEVEL_ERROR, + "%s(%d): idle=%08x", + __FUNCTION__, __LINE__, idle + ); + + gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); +#endif + + /* Advance timer. */ + timer += gcdGPU_ADVANCETIMER; + } + else if (status == gcvSTATUS_INTERRUPTED) + { + gcmkONERROR(gcvSTATUS_INTERRUPTED); + } + + } + while (gcmIS_ERROR(status)); + + /* Bail out on timeout. */ + if (gcmIS_ERROR(status)) + { + /* Broadcast the stuck GPU. */ + gcmkONERROR(gckOS_Broadcast( + os, hardware, gcvBROADCAST_GPU_STUCK + )); + } + + /* Delete the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (signal != gcvNULL) + { + /* Free the signal. */ + gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +#endif +} + +/******************************************************************************* +** +** gckCOMMAND_Attach +** +** Attach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gctUINT32 ProcessID +** Current process ID. +** +** OUTPUT: +** +** gckCONTEXT * Context +** Pointer to a variable that will receive a pointer to a new +** gckCONTEXT object. +** +** gctSIZE_T * StateCount +** Pointer to a variable that will receive the number of states +** in the context buffer. +*/ +#if (gcdENABLE_3D || gcdENABLE_2D) +gceSTATUS +gckCOMMAND_Attach( + IN gckCOMMAND Command, + OUT gckCONTEXT * Context, + OUT gctSIZE_T * MaxState, + OUT gctUINT32 * NumStates, + IN gctUINT32 ProcessID + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=0x%x", Command); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Construct( + Command->os, + Command->kernel->hardware, + ProcessID, + Context + )); + + /* Return the number of states in the context. */ + * MaxState = (* Context)->maxState; + * NumStates = (* Context)->numStates; + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Success. */ + gcmkFOOTER_ARG("*Context=0x%x", *Context); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} +#endif + +/******************************************************************************* +** +** gckCOMMAND_Detach +** +** Detach user process. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** gckCONTEXT Context +** Pointer to a gckCONTEXT object to be destroyed. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_Detach( + IN gckCOMMAND Command, + IN gckCONTEXT Context + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); + + /* Acquire the context switching mutex. */ + gcmkONERROR(gckOS_AcquireMutex( + Command->os, Command->mutexContext, gcvINFINITE + )); + acquired = gcvTRUE; + + /* Construct a gckCONTEXT object. */ + gcmkONERROR(gckCONTEXT_Destroy(Context)); + + if (Command->currContext == Context) + { + /* Detach from gckCOMMAND object if the destoryed context is current context. */ + Command->currContext = gcvNULL; + } + + /* Release the context switching mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; + +OnError: + /* Release mutex. */ + if (acquired) + { + /* Release the context switching mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); + acquired = gcvFALSE; + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckCOMMAND_DumpExecutingBuffer +** +** Dump the command buffer which GPU is executing. +** +** INPUT: +** +** gckCOMMAND Command +** Pointer to a gckCOMMAND object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckCOMMAND_DumpExecutingBuffer( + IN gckCOMMAND Command + ) +{ + gceSTATUS status; + gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL; + gctUINT32 gpuAddress; + gctSIZE_T pageCount; + gctPOINTER entry = gcvNULL; + gckOS os = Command->os; + gckKERNEL kernel = Command->kernel; + gctUINT32 i; + gctUINT32 dumpRear; + gckQUEUE queue = &kernel->hardware->linkQueue; + gctSIZE_T bytes; + gckLINKDATA linkData; + gcuQUEUEDATA * queueData; + gctUINT32 offset; + gctPOINTER entryDump; + gctUINT32 pid; + gctUINT8 processName[24] = {0}; + gctPHYS_ADDR_T cpuPhysical; + + gcmkPRINT("**************************\n"); + gcmkPRINT("**** COMMAND BUF DUMP ****\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); + gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); + + gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress); + + /* Search and dump memory around DMA address. */ + if (kernel->virtualCommandBuffer) + { + status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); + } + else + { + status = gcvSTATUS_OK; + } + + if (gcmIS_SUCCESS(status)) + { + if (kernel->virtualCommandBuffer) + { + gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping( + os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, &entry, &pageCount)); + + offset = gpuAddress - buffer->virtualBuffer.gpuAddress; + + entryDump = entry; + + /* Dump one pages. */ + bytes = 4096; + + /* Align to page. */ + offset &= 0xfffff000; + + /* Kernel address of page where stall point stay. */ + entryDump = (gctUINT8_PTR)entryDump + offset; + + /* Align to page. */ + gpuAddress &= 0xfffff000; + } + else + { + gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); + + gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, 4096, &entry)); + + /* Align to page start. */ + entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF); + gpuAddress = gpuAddress & ~0xFFF; + bytes = 4096; + } + + gcmkPRINT("User Command Buffer:\n"); + _DumpBuffer(entryDump, gpuAddress, bytes); + + if (kernel->virtualCommandBuffer) + { + gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping( + os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, entry)); + } + else + { + gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096)); + } + } + else + { + _DumpKernelCommandBuffer(Command); + } + + /* Dump link queue. */ + if (queue->count) + { + gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", + Command->kernel->stuckDump, queue->count); + + dumpRear = queue->count; + + for (i = 0; i < dumpRear; i++) + { + gckQUEUE_GetData(queue, i, &queueData); + + linkData = &queueData->linkData; + + /* Get gpu address of this command buffer. */ + gpuAddress = linkData->start; + bytes = linkData->end - gpuAddress; + + pid = linkData->pid; + + gckOS_GetProcessNameByPid(pid, 16, processName); + + if (kernel->virtualCommandBuffer) + { + buffer = gcvNULL; + + /* Get the whole buffer. */ + status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); + + if (gcmIS_ERROR(status)) + { + /* Get kernel address of kernel command buffer. */ + status = gckCOMMAND_AddressInKernelCommandBuffer( + kernel->command, gpuAddress, &entry); + + if (gcmIS_ERROR(status)) + { + status = gckHARDWARE_AddressInHardwareFuncions( + kernel->hardware, gpuAddress, &entry); + + if (gcmIS_ERROR(status)) + { + gcmkPRINT("Buffer [%08X - %08X] not found, may be freed", + linkData->start, + linkData->end); + continue; + } + } + + offset = 0; + gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh); + } + else + { + /* Get kernel logical for dump. */ + if (buffer->virtualBuffer.kernelLogical) + { + /* Get kernel logical directly if it is a context buffer. */ + entry = buffer->virtualBuffer.kernelLogical; + gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s", + linkData->linkLow, linkData->linkHigh, linkData->pid, processName); + } + else + { + /* Make it accessiable by kernel if it is a user command buffer. */ + gcmkVERIFY_OK( + gckOS_CreateKernelVirtualMapping(os, + buffer->virtualBuffer.physical, + buffer->virtualBuffer.bytes, + &entry, + &pageCount)); + gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s", + linkData->linkLow, linkData->linkHigh, linkData->pid, processName); + } + + offset = gpuAddress - buffer->virtualBuffer.gpuAddress; + } + + /* Dump from the entry. */ + _DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes); + + /* Release kernel logical address if neccessary. */ + if (buffer && !buffer->virtualBuffer.kernelLogical) + { + gcmkVERIFY_OK( + gckOS_DestroyKernelVirtualMapping(os, + buffer->virtualBuffer.physical, + buffer->virtualBuffer.bytes, + entry)); + } + } + else + { + gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); + + gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, bytes, &entry)); + + gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s", + linkData->linkLow, linkData->linkHigh, linkData->pid, processName); + + _DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes); + + gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes)); + } + } + } + + return gcvSTATUS_OK; +} + +gceSTATUS +gckCOMMAND_AddressInKernelCommandBuffer( + IN gckCOMMAND Command, + IN gctUINT32 Address, + OUT gctPOINTER * Pointer + ) +{ + gctINT i; + + for (i = 0; i < gcdCOMMAND_QUEUES; i++) + { + if ((Address >= Command->queues[i].address) + && (Address < (Command->queues[i].address + Command->pageSize)) + ) + { + *Pointer = (gctUINT8_PTR)Command->queues[i].logical + + (Address - Command->queues[i].address) + ; + + return gcvSTATUS_OK; + } + } + + return gcvSTATUS_NOT_FOUND; +} + |