diff options
Diffstat (limited to 'drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c')
| -rw-r--r-- | drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c | 3070 |
1 files changed, 3070 insertions, 0 deletions
diff --git a/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c new file mode 100644 index 000000000000..03bd3c724232 --- /dev/null +++ b/drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c @@ -0,0 +1,3070 @@ +/**************************************************************************** +* +* The MIT License (MIT) +* +* Copyright (c) 2014 - 2018 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +***************************************************************************** +* +* The GPL License (GPL) +* +* Copyright (C) 2014 - 2018 Vivante Corporation +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software Foundation, +* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +* +***************************************************************************** +* +* Note: This software is released under dual MIT and GPL licenses. A +* recipient may use this file under the terms of either the MIT license or +* GPL License. If you wish to use only one license not the other, you can +* indicate your decision by deleting one of the above license notices in your +* version of this file. +* +*****************************************************************************/ + + +#include "gc_hal_kernel_precomp.h" +#include "gc_hal_kernel_buffer.h" + +#ifdef __QNXNTO__ +#include "gc_hal_kernel_qnx.h" +#endif + +#define _GC_OBJ_ZONE gcvZONE_EVENT + +#define gcdEVENT_ALLOCATION_COUNT (4096 / gcmSIZEOF(gcsHAL_INTERFACE)) +#define gcdEVENT_MIN_THRESHOLD 4 + +/******************************************************************************\ +********************************* Support Code ********************************* +\******************************************************************************/ + +static gcmINLINE gceSTATUS +gckEVENT_AllocateQueue( + IN gckEVENT Event, + OUT gcsEVENT_QUEUE_PTR * Queue + ) +{ + gceSTATUS status; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Queue != gcvNULL); + + /* Do we have free queues? */ + if (Event->freeList == gcvNULL) + { + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + /* Move one free queue from the free list. */ + * Queue = Event->freeList; + Event->freeList = Event->freeList->next; + + /* Success. */ + gcmkFOOTER_ARG("*Queue=0x%x", gcmOPT_POINTER(Queue)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckEVENT_FreeQueue( + IN gckEVENT Event, + OUT gcsEVENT_QUEUE_PTR Queue + ) +{ + gceSTATUS status = gcvSTATUS_OK; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Queue != gcvNULL); + + /* Move one free queue from the free list. */ + Queue->next = Event->freeList; + Event->freeList = Queue; + + /* Success. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +gckEVENT_FreeRecord( + IN gckEVENT Event, + IN gcsEVENT_PTR Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->freeEventMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Push the record on the free list. */ + Record->next = Event->freeEventList; + Event->freeEventList = Record; + Event->freeEventCount += 1; + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return gcvSTATUS_OK; +} + +static gceSTATUS +gckEVENT_IsEmpty( + IN gckEVENT Event, + OUT gctBOOL_PTR IsEmpty + ) +{ + gceSTATUS status; + gctSIZE_T i; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(IsEmpty != gcvNULL); + + /* Assume the event queue is empty. */ + *IsEmpty = gcvTRUE; + + /* Walk the event queue. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + /* Check whether this event is in use. */ + if (Event->queues[i].head != gcvNULL) + { + /* The event is in use, hence the queue is not empty. */ + *IsEmpty = gcvFALSE; + break; + } + } + + /* Try acquiring the mutex. */ + status = gckOS_AcquireMutex(Event->os, Event->eventQueueMutex, 0); + if (status == gcvSTATUS_TIMEOUT) + { + /* Timeout - queue is no longer empty. */ + *IsEmpty = gcvFALSE; + } + else + { + /* Bail out on error. */ + gcmkONERROR(status); + + /* Release the mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Success. */ + gcmkFOOTER_ARG("*IsEmpty=%d", gcmOPT_VALUE(IsEmpty)); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static gceSTATUS +_TryToIdleGPU( + IN gckEVENT Event +) +{ + gceSTATUS status; + gctBOOL empty = gcvFALSE, idle = gcvFALSE; + gctBOOL powerLocked = gcvFALSE; + gckHARDWARE hardware; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Grab gckHARDWARE object. */ + hardware = Event->kernel->hardware; + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Check whether the event queue is empty. */ + gcmkONERROR(gckEVENT_IsEmpty(Event, &empty)); + + if (empty) + { + status = gckOS_AcquireMutex(hardware->os, hardware->powerMutex, 0); + if (status == gcvSTATUS_TIMEOUT) + { + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + powerLocked = gcvTRUE; + + /* Query whether the hardware is idle. */ + gcmkONERROR(gckHARDWARE_QueryIdle(Event->kernel->hardware, &idle)); + + gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex)); + powerLocked = gcvFALSE; + + if (idle) + { + /* Inform the system of idle GPU. */ + gcmkONERROR(gckOS_Broadcast(Event->os, + Event->kernel->hardware, + gcvBROADCAST_GPU_IDLE)); + } + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (powerLocked) + { + gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex)); + } + + gcmkFOOTER(); + return status; +} + +static gceSTATUS +__RemoveRecordFromProcessDB( + IN gckEVENT Event, + IN gcsEVENT_PTR Record + ) +{ + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + switch (Record->info.command) + { + case gcvHAL_FREE_NON_PAGED_MEMORY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_NON_PAGED, + gcmUINT64_TO_PTR(Record->info.u.FreeNonPagedMemory.logical))); + break; + + case gcvHAL_FREE_CONTIGUOUS_MEMORY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_CONTIGUOUS, + gcmUINT64_TO_PTR(Record->info.u.FreeContiguousMemory.logical))); + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_VIDEO_MEMORY_LOCKED, + gcmUINT64_TO_PTR(Record->info.u.UnlockVideoMemory.node))); + break; + + case gcvHAL_UNMAP_USER_MEMORY: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_MAP_USER_MEMORY, + gcmINT2PTR(Record->info.u.UnmapUserMemory.info))); + break; + + case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER: + gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB( + Event->kernel, + Record->processID, + gcvDB_COMMAND_BUFFER, + gcmUINT64_TO_PTR(Record->info.u.FreeVirtualCommandBuffer.logical))); + break; + + default: + break; + } + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +static gceSTATUS +_ReleaseVideoMemoryHandle( + IN gckKERNEL Kernel, + IN OUT gcsEVENT_PTR Record, + IN OUT gcsHAL_INTERFACE * Interface + ) +{ + gceSTATUS status; + gckVIDMEM_NODE nodeObject; + gctUINT32 handle; + + switch(Interface->command) + { + case gcvHAL_UNLOCK_VIDEO_MEMORY: + handle = (gctUINT32)Interface->u.UnlockVideoMemory.node; + + gcmkONERROR(gckVIDMEM_HANDLE_Lookup( + Kernel, Record->processID, handle, &nodeObject)); + + Record->info.u.UnlockVideoMemory.node = gcmPTR_TO_UINT64(nodeObject); + + gckVIDMEM_HANDLE_Dereference(Kernel, Record->processID, handle); + break; + + default: + break; + } + + return gcvSTATUS_OK; +OnError: + return status; +} + +/******************************************************************************* +** +** _QueryFlush +** +** Check the type of surfaces which will be released by current event and +** determine the cache needed to flush. +** +*/ +static gceSTATUS +_QueryFlush( + IN gckEVENT Event, + IN gcsEVENT_PTR Record, + OUT gceKERNEL_FLUSH *Flush + ) +{ + gceKERNEL_FLUSH flush = 0; + gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + while (Record != gcvNULL) + { + switch (Record->info.command) + { + case gcvHAL_UNLOCK_VIDEO_MEMORY: + switch(Record->info.u.UnlockVideoMemory.type) + { + case gcvSURF_TILE_STATUS: + flush |= gcvFLUSH_TILE_STATUS; + break; + case gcvSURF_RENDER_TARGET: + flush |= gcvFLUSH_COLOR; + break; + case gcvSURF_DEPTH: + flush |= gcvFLUSH_DEPTH; + break; + case gcvSURF_TEXTURE: + flush |= gcvFLUSH_TEXTURE; + break; + case gcvSURF_ICACHE: + flush |= gcvFLUSH_ICACHE; + break; + case gcvSURF_TXDESC: + flush |= gcvFLUSH_TXDESC; + break; + case gcvSURF_FENCE: + flush |= gcvFLUSH_FENCE; + break; + case gcvSURF_VERTEX: + flush |= gcvFLUSH_VERTEX; + break; + case gcvSURF_TFBHEADER: + flush |= gcvFLUSH_TFBHEADER; + break; + case gcvSURF_TYPE_UNKNOWN: + *Flush = gcvFLUSH_ALL; + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + default: + break; + } + break; + case gcvHAL_UNMAP_USER_MEMORY: + *Flush = gcvFLUSH_ALL; + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + + default: + break; + } + + Record = Record->next; + } + + *Flush = flush; + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +void +_SubmitTimerFunction( + gctPOINTER Data + ) +{ + gckEVENT event = (gckEVENT)Data; + gcmkVERIFY_OK(gckEVENT_Submit(event, gcvTRUE, gcvFALSE)); +} + +/******************************************************************************\ +******************************* gckEVENT API Code ******************************* +\******************************************************************************/ + +/******************************************************************************* +** +** gckEVENT_Construct +** +** Construct a new gckEVENT object. +** +** INPUT: +** +** gckKERNEL Kernel +** Pointer to an gckKERNEL object. +** +** OUTPUT: +** +** gckEVENT * Event +** Pointer to a variable that receives the gckEVENT object pointer. +*/ +gceSTATUS +gckEVENT_Construct( + IN gckKERNEL Kernel, + OUT gckEVENT * Event + ) +{ + gckOS os; + gceSTATUS status; + gckEVENT eventObj = gcvNULL; + int i; + gcsEVENT_PTR record; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Kernel=0x%x", Kernel); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); + gcmkVERIFY_ARGUMENT(Event != gcvNULL); + + /* Extract the pointer to the gckOS object. */ + os = Kernel->os; + gcmkVERIFY_OBJECT(os, gcvOBJ_OS); + + /* Allocate the gckEVENT object. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckEVENT), &pointer)); + + eventObj = pointer; + + /* Reset the object. */ + gcmkVERIFY_OK(gckOS_ZeroMemory(eventObj, gcmSIZEOF(struct _gckEVENT))); + + /* Initialize the gckEVENT object. */ + eventObj->object.type = gcvOBJ_EVENT; + eventObj->kernel = Kernel; + eventObj->os = os; + + /* Create the mutexes. */ + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventQueueMutex)); + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->freeEventMutex)); + gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventListMutex)); + + /* Create a bunch of event reccords. */ + for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1) + { + /* Allocate an event record. */ + gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsEVENT), &pointer)); + + record = pointer; + + /* Push it on the free list. */ + record->next = eventObj->freeEventList; + eventObj->freeEventList = record; + eventObj->freeEventCount += 1; + } + + /* Initialize the free list of event queues. */ + for (i = 0; i < gcdREPO_LIST_COUNT; i += 1) + { + eventObj->repoList[i].next = eventObj->freeList; + eventObj->freeList = &eventObj->repoList[i]; + } + + eventObj->freeQueueCount = gcmCOUNTOF(eventObj->queues); + + gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending)); + + gcmkVERIFY_OK(gckOS_CreateTimer(os, + _SubmitTimerFunction, + (gctPOINTER)eventObj, + &eventObj->submitTimer)); + +#if gcdINTERRUPT_STATISTIC + gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->interruptCount)); + gcmkONERROR(gckOS_AtomSet(os,eventObj->interruptCount, 0)); +#endif + + eventObj->notifyState = -1; + + /* Return pointer to the gckEVENT object. */ + *Event = eventObj; + + /* Success. */ + gcmkFOOTER_ARG("*Event=0x%x", *Event); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (eventObj != gcvNULL) + { + if (eventObj->eventQueueMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventQueueMutex)); + } + + if (eventObj->freeEventMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->freeEventMutex)); + } + + if (eventObj->eventListMutex != gcvNULL) + { + gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventListMutex)); + } + + while (eventObj->freeEventList != gcvNULL) + { + record = eventObj->freeEventList; + eventObj->freeEventList = record->next; + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, record)); + } + + if (eventObj->pending != gcvNULL) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending)); + } + +#if gcdINTERRUPT_STATISTIC + if (eventObj->interruptCount) + { + gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->interruptCount)); + } +#endif + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, eventObj)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Destroy +** +** Destroy an gckEVENT object. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Destroy( + IN gckEVENT Event + ) +{ + gcsEVENT_PTR record; + gcsEVENT_QUEUE_PTR queue; + + gcmkHEADER_ARG("Event=0x%x", Event); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + if (Event->submitTimer != gcvNULL) + { + gcmkVERIFY_OK(gckOS_StopTimer(Event->os, Event->submitTimer)); + gcmkVERIFY_OK(gckOS_DestroyTimer(Event->os, Event->submitTimer)); + } + + /* Delete the queue mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventQueueMutex)); + + /* Free all free events. */ + while (Event->freeEventList != gcvNULL) + { + record = Event->freeEventList; + Event->freeEventList = record->next; + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record)); + } + + /* Delete the free mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->freeEventMutex)); + + /* Free all pending queues. */ + while (Event->queueHead != gcvNULL) + { + /* Get the current queue. */ + queue = Event->queueHead; + + /* Free all pending events. */ + while (queue->head != gcvNULL) + { + record = queue->head; + queue->head = record->next; + + gcmkTRACE_ZONE_N( + gcvLEVEL_WARNING, gcvZONE_EVENT, + gcmSIZEOF(record) + gcmSIZEOF(queue->source), + "Event record 0x%x is still pending for %d.", + record, queue->source + ); + + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record)); + } + + /* Remove the top queue from the list. */ + if (Event->queueHead == Event->queueTail) + { + Event->queueHead = + Event->queueTail = gcvNULL; + } + else + { + Event->queueHead = Event->queueHead->next; + } + + /* Free the queue. */ + gcmkVERIFY_OK(gckEVENT_FreeQueue(Event, queue)); + } + + /* Delete the list mutex. */ + gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventListMutex)); + + gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending)); + +#if gcdINTERRUPT_STATISTIC + gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->interruptCount)); +#endif + + /* Mark the gckEVENT object as unknown. */ + Event->object.type = gcvOBJ_UNKNOWN; + + /* Free the gckEVENT object. */ + gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, Event)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckEVENT_GetEvent +** +** Reserve the next available hardware event. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL Wait +** Set to gcvTRUE to force the function to wait if no events are +** immediately available. +** +** gceKERNEL_WHERE Source +** Source of the event. +** +** OUTPUT: +** +** gctUINT8 * EventID +** Reserved event ID. +*/ +#define gcdINVALID_EVENT_PTR ((gcsEVENT_PTR)gcvMAXUINTPTR_T) + +gceSTATUS +gckEVENT_GetEvent( + IN gckEVENT Event, + IN gctBOOL Wait, + OUT gctUINT8 * EventID, + IN gceKERNEL_WHERE Source + ) +{ + gctINT i, id; + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctINT32 free; + + gcmkHEADER_ARG("Event=0x%x Source=%d", Event, Source); + + while (gcvTRUE) + { + /* Grab the queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Walk through all events. */ + id = Event->lastID; + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + gctINT nextID = id + 1; + + if (nextID == gcmCOUNTOF(Event->queues)) + { + nextID = 0; + } + + if (Event->queues[id].head == gcvNULL) + { + *EventID = (gctUINT8) id; + + Event->lastID = (gctUINT8) nextID; + + /* Save time stamp of event. */ + Event->queues[id].head = gcdINVALID_EVENT_PTR; + Event->queues[id].stamp = ++(Event->stamp); + Event->queues[id].source = Source; + + /* Decrease the number of free events. */ + free = --Event->freeQueueCount; + + /* Make compiler happy. */ + free = free; + +#if gcdDYNAMIC_SPEED + if (free <= gcdDYNAMIC_EVENT_THRESHOLD) + { + gcmkONERROR(gckOS_BroadcastHurry( + Event->os, + Event->kernel->hardware, + gcdDYNAMIC_EVENT_THRESHOLD - free)); + } +#endif + + /* Release the queue mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, + Event->eventQueueMutex)); + + /* Success. */ + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(id), + "Using id=%d", + id + ); + + gcmkFOOTER_ARG("*EventID=%u", *EventID); + return gcvSTATUS_OK; + } + + id = nextID; + } + +#if gcdDYNAMIC_SPEED + /* No free events, speed up the GPU right now! */ + gcmkONERROR(gckOS_BroadcastHurry(Event->os, + Event->kernel->hardware, + gcdDYNAMIC_EVENT_THRESHOLD)); +#endif + + /* Release the queue mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Fail if wait is not requested. */ + if (!Wait) + { + /* Out of resources. */ + gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); + } + + /* Delay a while. */ + gcmkONERROR(gckOS_Delay(Event->os, 1)); + } + +OnError: + if (acquired) + { + /* Release the queue mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_AllocateRecord +** +** Allocate a record for the new event. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL AllocateAllowed +** State for allocation if out of free events. +** +** OUTPUT: +** +** gcsEVENT_PTR * Record +** Allocated event record. +*/ +static gcmINLINE gceSTATUS +gckEVENT_AllocateRecord( + IN gckEVENT Event, + IN gctBOOL AllocateAllowed, + OUT gcsEVENT_PTR * Record + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gctINT i; + gcsEVENT_PTR record; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Event=0x%x AllocateAllowed=%d", Event, AllocateAllowed); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Record != gcvNULL); + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->freeEventMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Test if we are below the allocation threshold. */ + if ( (AllocateAllowed && (Event->freeEventCount < gcdEVENT_MIN_THRESHOLD)) || + (Event->freeEventCount == 0) ) + { + /* Allocate a bunch of records. */ + for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1) + { + /* Allocate an event record. */ + gcmkONERROR(gckOS_Allocate(Event->os, + gcmSIZEOF(gcsEVENT), + &pointer)); + + record = pointer; + + /* Push it on the free list. */ + record->next = Event->freeEventList; + Event->freeEventList = record; + Event->freeEventCount += 1; + } + } + + *Record = Event->freeEventList; + Event->freeEventList = Event->freeEventList->next; + Event->freeEventCount -= 1; + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + + /* Success. */ + gcmkFOOTER_ARG("*Record=0x%x", gcmOPT_POINTER(Record)); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_AddList +** +** Add a new event to the list of events. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gcsHAL_INTERFACE_PTR Interface +** Pointer to the interface for the event to be added. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** gctBOOL AllocateAllowed +** State for allocation if out of free events. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_AddList( + IN gckEVENT Event, + IN gcsHAL_INTERFACE_PTR Interface, + IN gceKERNEL_WHERE FromWhere, + IN gctBOOL AllocateAllowed, + IN gctBOOL FromKernel + ) +{ + gceSTATUS status; + gctBOOL acquired = gcvFALSE; + gcsEVENT_PTR record = gcvNULL; + gcsEVENT_QUEUE_PTR queue; + gckVIRTUAL_COMMAND_BUFFER_PTR buffer; + gckKERNEL kernel = Event->kernel; + + gcmkHEADER_ARG("Event=0x%x Interface=0x%x", + Event, Interface); + + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, _GC_OBJ_ZONE, + "FromWhere=%d AllocateAllowed=%d", + FromWhere, AllocateAllowed); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Interface != gcvNULL); + + /* Verify the event command. */ + gcmkASSERT + ( (Interface->command == gcvHAL_FREE_NON_PAGED_MEMORY) + || (Interface->command == gcvHAL_FREE_CONTIGUOUS_MEMORY) + || (Interface->command == gcvHAL_WRITE_DATA) + || (Interface->command == gcvHAL_UNLOCK_VIDEO_MEMORY) + || (Interface->command == gcvHAL_SIGNAL) + || (Interface->command == gcvHAL_UNMAP_USER_MEMORY) + || (Interface->command == gcvHAL_TIMESTAMP) + || (Interface->command == gcvHAL_COMMIT_DONE) + || (Interface->command == gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER) + || (Interface->command == gcvHAL_DESTROY_MMU) + ); + + /* Validate the source. */ + if ((FromWhere != gcvKERNEL_COMMAND) && (FromWhere != gcvKERNEL_PIXEL)) + { + /* Invalid argument. */ + gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); + } + + /* Allocate a free record. */ + gcmkONERROR(gckEVENT_AllocateRecord(Event, AllocateAllowed, &record)); + + /* Termninate the record. */ + record->next = gcvNULL; + + /* Record the committer. */ + record->fromKernel = FromKernel; + + /* Copy the event interface into the record. */ + gckOS_MemCopy(&record->info, Interface, gcmSIZEOF(record->info)); + + /* Get process ID. */ + gcmkONERROR(gckOS_GetProcessID(&record->processID)); + + if (FromKernel == gcvFALSE) + { + gcmkONERROR(__RemoveRecordFromProcessDB(Event, record)); + + /* Handle is belonged to current process, it must be released now. */ + status = _ReleaseVideoMemoryHandle(Event->kernel, record, Interface); + + if (gcmIS_ERROR(status)) + { + /* Ingore error because there are other events in the queue. */ + status = gcvSTATUS_OK; + goto OnError; + } + } + +#ifdef __QNXNTO__ + record->kernel = Event->kernel; +#endif + + /* Unmap user space logical address. + * Linux kernel does not support unmap the memory of other process any more since 3.5. + * Let's unmap memory of self process before submit the event to gpu. + * */ + switch(Interface->command) + { + case gcvHAL_FREE_NON_PAGED_MEMORY: + gcmkONERROR(gckOS_UnmapUserLogical( + Event->os, + gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physical), + (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes, + gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical))); + break; + case gcvHAL_FREE_CONTIGUOUS_MEMORY: + gcmkONERROR(gckOS_UnmapUserLogical( + Event->os, + gcmNAME_TO_PTR(Interface->u.FreeContiguousMemory.physical), + (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes, + gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical))); + break; + + case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER: + buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)gcmNAME_TO_PTR(Interface->u.FreeVirtualCommandBuffer.physical); + if (buffer != gcvNULL && buffer->virtualBuffer.userLogical) + { + gcmkONERROR(gckOS_DestroyUserVirtualMapping( + Event->os, + buffer->virtualBuffer.physical, + (gctSIZE_T) Interface->u.FreeVirtualCommandBuffer.bytes, + gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical))); + } + break; + + default: + break; + } + + /* Acquire the mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->eventListMutex, gcvINFINITE)); + acquired = gcvTRUE; + + /* Do we need to allocate a new queue? */ + if ((Event->queueTail == gcvNULL) || (Event->queueTail->source < FromWhere)) + { + /* Allocate a new queue. */ + gcmkONERROR(gckEVENT_AllocateQueue(Event, &queue)); + + /* Initialize the queue. */ + queue->source = FromWhere; + queue->head = gcvNULL; + queue->next = gcvNULL; + + /* Attach it to the list of allocated queues. */ + if (Event->queueTail == gcvNULL) + { + Event->queueHead = + Event->queueTail = queue; + } + else + { + Event->queueTail->next = queue; + Event->queueTail = queue; + } + } + else + { + queue = Event->queueTail; + } + + /* Attach the record to the queue. */ + if (queue->head == gcvNULL) + { + queue->head = record; + queue->tail = record; + } + else + { + queue->tail->next = record; + queue->tail = record; + } + + /* Release the mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Roll back. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + } + + if (record != gcvNULL) + { + gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Unlock +** +** Schedule an event to unlock virtual memory. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** gcuVIDMEM_NODE_PTR Node +** Pointer to a gcuVIDMEM_NODE union that specifies the virtual memory +** to unlock. +** +** gceSURF_TYPE Type +** Type of surface to unlock. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Unlock( + IN gckEVENT Event, + IN gceKERNEL_WHERE FromWhere, + IN gctPOINTER Node, + IN gceSURF_TYPE Type + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + + gcmkHEADER_ARG("Event=0x%x FromWhere=%d Node=0x%x Type=%d", + Event, FromWhere, Node, Type); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Node != gcvNULL); + + /* Mark the event as an unlock. */ + iface.command = gcvHAL_UNLOCK_VIDEO_MEMORY; + iface.u.UnlockVideoMemory.node = gcmPTR_TO_UINT64(Node); + iface.u.UnlockVideoMemory.type = Type; + iface.u.UnlockVideoMemory.asynchroneous = 0; + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_FreeNonPagedMemory +** +** Schedule an event to free non-paged memory. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctSIZE_T Bytes +** Number of bytes of non-paged memory to free. +** +** gctPHYS_ADDR Physical +** Physical address of non-paged memory to free. +** +** gctPOINTER Logical +** Logical address of non-paged memory to free. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +*/ +gceSTATUS +gckEVENT_FreeNonPagedMemory( + IN gckEVENT Event, + IN gctSIZE_T Bytes, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + gckKERNEL kernel = Event->kernel; + + gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x " + "FromWhere=%d", + Event, Bytes, Physical, Logical, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Physical != gcvNULL); + gcmkVERIFY_ARGUMENT(Logical != gcvNULL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + + /* Create an event. */ + iface.command = gcvHAL_FREE_NON_PAGED_MEMORY; + iface.u.FreeNonPagedMemory.bytes = Bytes; + iface.u.FreeNonPagedMemory.physical = gcmPTR_TO_NAME(Physical); + iface.u.FreeNonPagedMemory.logical = gcmPTR_TO_UINT64(Logical); + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +gceSTATUS +gckEVENT_DestroyVirtualCommandBuffer( + IN gckEVENT Event, + IN gctSIZE_T Bytes, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + gckKERNEL kernel = Event->kernel; + + gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x " + "FromWhere=%d", + Event, Bytes, Physical, Logical, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Physical != gcvNULL); + gcmkVERIFY_ARGUMENT(Logical != gcvNULL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + + /* Create an event. */ + iface.command = gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER; + iface.u.FreeVirtualCommandBuffer.bytes = Bytes; + iface.u.FreeVirtualCommandBuffer.physical = gcmPTR_TO_NAME(Physical); + iface.u.FreeVirtualCommandBuffer.logical = gcmPTR_TO_UINT64(Logical); + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_FreeContigiuousMemory +** +** Schedule an event to free contiguous memory. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctSIZE_T Bytes +** Number of bytes of contiguous memory to free. +** +** gctPHYS_ADDR Physical +** Physical address of contiguous memory to free. +** +** gctPOINTER Logical +** Logical address of contiguous memory to free. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +*/ +gceSTATUS +gckEVENT_FreeContiguousMemory( + IN gckEVENT Event, + IN gctSIZE_T Bytes, + IN gctPHYS_ADDR Physical, + IN gctPOINTER Logical, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + gckKERNEL kernel = Event->kernel; + + gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x " + "FromWhere=%d", + Event, Bytes, Physical, Logical, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Physical != gcvNULL); + gcmkVERIFY_ARGUMENT(Logical != gcvNULL); + gcmkVERIFY_ARGUMENT(Bytes > 0); + + /* Create an event. */ + iface.command = gcvHAL_FREE_CONTIGUOUS_MEMORY; + iface.u.FreeContiguousMemory.bytes = Bytes; + iface.u.FreeContiguousMemory.physical = gcmPTR_TO_NAME(Physical); + iface.u.FreeContiguousMemory.logical = gcmPTR_TO_UINT64(Logical); + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Signal +** +** Schedule an event to trigger a signal. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctSIGNAL Signal +** Pointer to the signal to trigger. +** +** gceKERNEL_WHERE FromWhere +** Place in the pipe where the event needs to be generated. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Signal( + IN gckEVENT Event, + IN gctSIGNAL Signal, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + + gcmkHEADER_ARG("Event=0x%x Signal=0x%x FromWhere=%d", + Event, Signal, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + gcmkVERIFY_ARGUMENT(Signal != gcvNULL); + + /* Mark the event as a signal. */ + iface.command = gcvHAL_SIGNAL; + iface.u.Signal.signal = gcmPTR_TO_UINT64(Signal); + iface.u.Signal.auxSignal = 0; + iface.u.Signal.process = 0; + +#ifdef __QNXNTO__ + iface.u.Signal.coid = 0; + iface.u.Signal.rcvid = 0; + + gcmkONERROR(gckOS_SignalPending(Event->os, Signal)); +#endif + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +#if gcdPROCESS_ADDRESS_SPACE +gceSTATUS +gckEVENT_DestroyMmu( + IN gckEVENT Event, + IN gckMMU Mmu, + IN gceKERNEL_WHERE FromWhere + ) +{ + gceSTATUS status; + gcsHAL_INTERFACE iface; + + gcmkHEADER_ARG("Event=0x%x FromWhere=%d", Event, FromWhere); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + iface.command = gcvHAL_DESTROY_MMU; + iface.u.DestroyMmu.mmu = gcmPTR_TO_UINT64(Mmu); + + /* Append it to the queue. */ + gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Return the status. */ + gcmkFOOTER(); + return status; +} +#endif + +gceSTATUS +gckEVENT_SubmitAsync( + IN gckEVENT Event, + IN gctBOOL Wait, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gctUINT8 id = 0xFF; + gcsEVENT_QUEUE_PTR queue; + gctBOOL acquired = gcvFALSE; + gctBOOL commitEntered = gcvFALSE; + gctUINT32 start, end; + gctUINT8_PTR startLogical; + gctUINT32 eventBytes; + + gckHARDWARE hardware; + gckASYNC_COMMAND asyncCommand; + + gcmkHEADER_ARG("Event=0x%x Wait=%d", Event, Wait); + + /* Get gckCOMMAND object. */ + hardware = Event->kernel->hardware; + asyncCommand = Event->asyncCommand; + + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + /* Are there event queues? */ + if (Event->queueHead != gcvNULL) + { + /* Acquire the command queue. */ + gcmkONERROR(gckASYNC_COMMAND_EnterCommit(asyncCommand)); + commitEntered = gcvTRUE; + + /* Process all queues. */ + while (Event->queueHead != gcvNULL) + { + /* Acquire the list mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventListMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Get the current queue. */ + queue = Event->queueHead; + + /* Allocate an event ID. */ + gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source)); + + /* Copy event list to event ID queue. */ + Event->queues[id].head = queue->head; + + /* Remove the top queue from the list. */ + if (Event->queueHead == Event->queueTail) + { + Event->queueHead = gcvNULL; + Event->queueTail = gcvNULL; + } + else + { + Event->queueHead = Event->queueHead->next; + } + + /* Free the queue. */ + gcmkONERROR(gckEVENT_FreeQueue(Event, queue)); + + /* Release the list mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + acquired = gcvFALSE; + + gcmkONERROR(gckHARDWARE_Event(Event->kernel->hardware, gcvNULL, id, gcvKERNEL_BLT, &eventBytes)); + + /* Get command sequence. */ + start = hardware->functions[gcvHARDWARE_FUNCTION_BLT_EVENT].address + id * eventBytes; + end = start + 24; + + startLogical = hardware->functions[gcvHARDWARE_FUNCTION_BLT_EVENT].logical + id * eventBytes; + + gcmkDUMPCOMMAND( + Event->os, + startLogical, + end - start, + gcvDUMP_BUFFER_KERNEL, + gcvFALSE + ); + + gcmkONERROR(gckASYNC_COMMAND_Execute(asyncCommand, start, end)); + } + + /* Release the command queue. */ + gcmkONERROR(gckASYNC_COMMAND_ExitCommit(asyncCommand)); + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Need to unroll the mutex acquire. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckASYNC_COMMAND_ExitCommit(asyncCommand)); + } + + if (id != 0xFF) + { + /* Need to unroll the event allocation. */ + Event->queues[id].head = gcvNULL; + } + + if (status == gcvSTATUS_GPU_NOT_RESPONDING) + { + /* Broadcast GPU stuck. */ + status = gckOS_Broadcast(Event->os, + Event->kernel->hardware, + gcvBROADCAST_GPU_STUCK); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Submit +** +** Submit the current event queue to the GPU. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctBOOL Wait +** Submit requires one vacant event; if Wait is set to not zero, +** and there are no vacant events at this time, the function will +** wait until an event becomes vacant so that submission of the +** queue is successful. +** +** gctBOOL FromPower +** Determines whether the call originates from inside the power +** management or not. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Submit( + IN gckEVENT Event, + IN gctBOOL Wait, + IN gctBOOL FromPower + ) +{ + gceSTATUS status; + gctUINT8 id = 0xFF; + gcsEVENT_QUEUE_PTR queue; + gctBOOL acquired = gcvFALSE; + gckCOMMAND command = gcvNULL; + gctBOOL commitEntered = gcvFALSE; +#if !gcdNULL_DRIVER + gctUINT32 bytes; + gctPOINTER buffer; + gctUINT32 executeBytes; + gctUINT32 flushBytes; +#endif + +#if gcdINTERRUPT_STATISTIC + gctINT32 oldValue; +#endif + +#if gcdSECURITY + gctPOINTER reservedBuffer; +#endif + + gckHARDWARE hardware; + + gceKERNEL_FLUSH flush = gcvFALSE; + gctUINT64 commitStamp; + + gcmkHEADER_ARG("Event=0x%x Wait=%d", Event, Wait); + + /* Get gckCOMMAND object. */ + command = Event->kernel->command; + hardware = Event->kernel->hardware; + + gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); + + if (Event->asyncCommand) + { + /* Call async submit path. */ + gcmkONERROR(gckEVENT_SubmitAsync(Event, Wait, FromPower)); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + } + + gckOS_GetTicks(&Event->lastCommitStamp); + + /* Are there event queues? */ + if (Event->queueHead != gcvNULL) + { + /* Acquire the command queue. */ + gcmkONERROR(gckCOMMAND_EnterCommit(command, FromPower)); + commitEntered = gcvTRUE; + + /* Get current commit stamp. */ + commitStamp = Event->kernel->command->commitStamp; + + if (commitStamp) + { + commitStamp -= 1; + } + + /* Process all queues. */ + while (Event->queueHead != gcvNULL) + { + /* Acquire the list mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventListMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Get the current queue. */ + queue = Event->queueHead; + + /* Allocate an event ID. */ + gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source)); + + /* Copy event list to event ID queue. */ + Event->queues[id].head = queue->head; + + /* Update current commit stamp. */ + Event->queues[id].commitStamp = commitStamp; + + /* Remove the top queue from the list. */ + if (Event->queueHead == Event->queueTail) + { + Event->queueHead = gcvNULL; + Event->queueTail = gcvNULL; + } + else + { + Event->queueHead = Event->queueHead->next; + } + + /* Free the queue. */ + gcmkONERROR(gckEVENT_FreeQueue(Event, queue)); + + /* Release the list mutex. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + acquired = gcvFALSE; + + /* Determine cache needed to flush. */ + gcmkVERIFY_OK(_QueryFlush(Event, Event->queues[id].head, &flush)); + +#if gcdNULL_DRIVER +#if gcdINTERRUPT_STATISTIC + gcmkVERIFY_OK(gckOS_AtomIncrement( + Event->os, + Event->interruptCount, + &oldValue + )); +#endif + + /* Notify immediately on infinite hardware. */ + gcmkONERROR(gckEVENT_Interrupt(Event, 1 << id)); + + gcmkONERROR(gckEVENT_Notify(Event, 0)); +#else + /* Get the size of the hardware event. */ + gcmkONERROR(gckHARDWARE_Event( + hardware, + gcvNULL, + id, + Event->queues[id].source, + &bytes + )); + + /* Get the size of flush command. */ + gcmkONERROR(gckHARDWARE_Flush( + hardware, + flush, + gcvNULL, + &flushBytes + )); + + bytes += flushBytes; + + /* Total bytes need to execute. */ + executeBytes = bytes; + + /* Reserve space in the command queue. */ + gcmkONERROR(gckCOMMAND_Reserve(command, bytes, &buffer, &bytes)); +#if gcdSECURITY + reservedBuffer = buffer; +#endif + + /* Set the flush in the command queue. */ + gcmkONERROR(gckHARDWARE_Flush( + hardware, + flush, + buffer, + &flushBytes + )); + + /* Advance to next command. */ + buffer = (gctUINT8_PTR)buffer + flushBytes; + + /* Set the hardware event in the command queue. */ + gcmkONERROR(gckHARDWARE_Event( + hardware, + buffer, + id, + Event->queues[id].source, + &bytes + )); + + /* Advance to next command. */ + buffer = (gctUINT8_PTR)buffer + bytes; + +#if gcdINTERRUPT_STATISTIC + gcmkVERIFY_OK(gckOS_AtomIncrement( + Event->os, + Event->interruptCount, + &oldValue + )); +#endif + +#if gcdSECURITY + gckKERNEL_SecurityExecute( + Event->kernel, + reservedBuffer, + executeBytes + ); +#else + /* Execute the hardware event. */ + gcmkONERROR(gckCOMMAND_Execute(command, executeBytes)); +#endif +#endif + } + + /* Release the command queue. */ + gcmkONERROR(gckCOMMAND_ExitCommit(command, FromPower)); + +#if !gcdNULL_DRIVER + if (!FromPower) + { + gcmkVERIFY_OK(_TryToIdleGPU(Event)); + } +#endif + } + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Need to unroll the mutex acquire. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex)); + } + + if (commitEntered) + { + /* Release the command queue mutex. */ + gcmkVERIFY_OK(gckCOMMAND_ExitCommit(command, FromPower)); + } + + if (id != 0xFF) + { + /* Need to unroll the event allocation. */ + Event->queues[id].head = gcvNULL; + } + + if (status == gcvSTATUS_GPU_NOT_RESPONDING) + { + /* Broadcast GPU stuck. */ + status = gckOS_Broadcast(Event->os, + Event->kernel->hardware, + gcvBROADCAST_GPU_STUCK); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Commit +** +** Commit an event queue from the user. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gcsQUEUE_PTR Queue +** User event queue. +** +** gctBOOL Forced +** Force fire a event. There won't be interrupt if there's no events + queued. Force a event by append a dummy one if this parameter is on. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Commit( + IN gckEVENT Event, + IN gcsQUEUE_PTR Queue, + IN gctBOOL Forced + ) +{ + gceSTATUS status; + gcsQUEUE_PTR record = gcvNULL, next; + gctUINT32 processID; + gctBOOL needCopy = gcvFALSE; + gctPOINTER pointer = gcvNULL; + + gcmkHEADER_ARG("Event=0x%x Queue=0x%x", Event, Queue); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Get the current process ID. */ + gcmkONERROR(gckOS_GetProcessID(&processID)); + + /* Query if we need to copy the client data. */ + gcmkONERROR(gckOS_QueryNeedCopy(Event->os, processID, &needCopy)); + + /* Loop while there are records in the queue. */ + while (Queue != gcvNULL) + { + gcsQUEUE queue; + + if (needCopy) + { + /* Point to stack record. */ + record = &queue; + + /* Copy the data from the client. */ + gcmkONERROR(gckOS_CopyFromUserData(Event->os, + record, + Queue, + gcmSIZEOF(gcsQUEUE))); + } + else + { + + /* Map record into kernel memory. */ + gcmkONERROR(gckOS_MapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + &pointer)); + + record = pointer; + } + + /* Append event record to event queue. */ + gcmkONERROR( + gckEVENT_AddList(Event, &record->iface, gcvKERNEL_PIXEL, gcvTRUE, gcvFALSE)); + + /* Next record in the queue. */ + next = gcmUINT64_TO_PTR(record->next); + + if (!needCopy) + { + /* Unmap record from kernel memory. */ + gcmkONERROR( + gckOS_UnmapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + (gctPOINTER *) record)); + record = gcvNULL; + } + + Queue = next; + } + + if (Forced && Event->queueHead == gcvNULL) + { + gcsHAL_INTERFACE iface; + iface.command = gcvHAL_COMMIT_DONE; + + gcmkONERROR(gckEVENT_AddList(Event, &iface, gcvKERNEL_PIXEL, gcvFALSE, gcvTRUE)); + } + + /* Submit the event list. */ + gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE)); + + /* Success */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (pointer) + { + /* Roll back. */ + gcmkVERIFY_OK(gckOS_UnmapUserPointer(Event->os, + Queue, + gcmSIZEOF(gcsQUEUE), + (gctPOINTER*)pointer)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** +** gckEVENT_Interrupt +** +** Called by the interrupt service routine to store the triggered interrupt +** mask to be later processed by gckEVENT_Notify. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctUINT32 Data +** Mask for the 32 interrupts. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Interrupt( + IN gckEVENT Event, + IN gctUINT32 Data + ) +{ + gcmkHEADER_ARG("Event=0x%x Data=0x%x", Event, Data); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + if (Data & 0x20000000) + { + gctUINT32 resume; + gctUINT32 bytes; + gctUINT32 idle; + gctUINT32 pageSize = Event->kernel->command->pageSize; + Data &= ~0x20000000; + + { + /* Make sure FE is idle. */ + do + { + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Event->os, + Event->kernel->core, + 0x4, + &idle)); + } + while (idle != 0x7FFFFFFF); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Event->os, + Event->kernel->core, + 0x664, + &resume)); + + gcmkVERIFY_OK(gckOS_ReadRegisterEx( + Event->os, + Event->kernel->core, + 0x664, + &resume)); + + gcmkVERIFY_OK(gckHARDWARE_WaitLink( + Event->kernel->hardware, + gcvNULL, + ~0U, + resume & (pageSize - 1), + &bytes, + gcvNULL, + gcvNULL + )); + + /* Start Command Parser. */ + gcmkVERIFY_OK(gckHARDWARE_Execute( + Event->kernel->hardware, + resume, + bytes + )); + } + } + + /* Combine current interrupt status with pending flags. */ + gckOS_AtomSetMask(Event->pending, Data); + +#if gcdINTERRUPT_STATISTIC + { + gctINT j = 0; + gctINT32 oldValue; + + for (j = 0; j < gcmCOUNTOF(Event->queues); j++) + { + if ((Data & (1 << j))) + { + gcmkVERIFY_OK(gckOS_AtomDecrement(Event->os, + Event->interruptCount, + &oldValue)); + } + } + } +#endif + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} + +/******************************************************************************* +** +** gckEVENT_Notify +** +** Process all triggered interrupts. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Notify( + IN gckEVENT Event, + IN gctUINT32 IDs + ) +{ + gceSTATUS status = gcvSTATUS_OK; + gctINT i; + gcsEVENT_QUEUE * queue; + gctUINT mask = 0; + gctBOOL acquired = gcvFALSE; + gctSIGNAL signal; + gctUINT pending = 0; + gckKERNEL kernel = Event->kernel; + +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gctINT eventNumber = 0; +#endif +#if gcdSECURE_USER + gcskSECURE_CACHE_PTR cache; + gcuVIDMEM_NODE_PTR node; +#endif + gckVIDMEM_NODE nodeObject; + + gcmkHEADER_ARG("Event=0x%x IDs=0x%x", Event, IDs); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + gcmDEBUG_ONLY( + if (IDs != 0) + { + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Queue(%d): stamp=%llu source=%d", + i, + Event->queues[i].stamp, + Event->queues[i].source); + } + } + } + ); + + /* Begin of event handling. */ + Event->notifyState = 0; + + for (;;) + { + gcsEVENT_PTR record; + + /* Grab the mutex queue. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + gckOS_AtomGet(Event->os, Event->pending, (gctINT32_PTR)&pending); + + if (pending == 0) + { + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* No more pending interrupts - done. */ + break; + } + + if (pending & 0x80000000) + { + gcmkPRINT("AXI BUS ERROR"); + gckHARDWARE_DumpGPUState(Event->kernel->hardware); + pending &= 0x7FFFFFFF; + } + + if ((pending & 0x40000000) && Event->kernel->hardware->mmuVersion) + { +#if gcdUSE_MMU_EXCEPTION +#if gcdALLOC_ON_FAULT + status = gckHARDWARE_HandleFault(Event->kernel->hardware); +#endif + if (gcmIS_ERROR(status)) + { + /* Dump error is fault can't be handle. */ + gckHARDWARE_DumpMMUException(Event->kernel->hardware); + + gckHARDWARE_DumpGPUState(Event->kernel->hardware); + } +#endif + + pending &= 0xBFFFFFFF; + } + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(pending), + "Pending interrupts 0x%x", + pending + ); + + queue = gcvNULL; + + gcmDEBUG_ONLY( + if (IDs == 0) + { + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Queue(%d): stamp=%llu source=%d", + i, + Event->queues[i].stamp, + Event->queues[i].source); + } + } + } + ); + + /* Find the oldest pending interrupt. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if ((Event->queues[i].head != gcvNULL) + && (pending & (1 << i)) + ) + { + if ((queue == gcvNULL) + || (Event->queues[i].stamp < queue->stamp) + ) + { + queue = &Event->queues[i]; + mask = 1 << i; +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + eventNumber = i; +#endif + } + } + } + + if (queue == gcvNULL) + { + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(pending), + "Interrupts 0x%x are not pending.", + pending + ); + + gckOS_AtomClearMask(Event->pending, pending); + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + break; + } + + /* Check whether there is a missed interrupt. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if ((Event->queues[i].head != gcvNULL) + && (Event->queues[i].stamp < queue->stamp) + && (Event->queues[i].source <= queue->source) + ) + { + gcmkTRACE_N( + gcvLEVEL_ERROR, + gcmSIZEOF(i) + gcmSIZEOF(Event->queues[i].stamp), + "Event %d lost (stamp %llu)", + i, Event->queues[i].stamp + ); + + /* Use this event instead. */ + queue = &Event->queues[i]; + mask = 0; + } + } + + if (mask != 0) + { +#if gcmIS_DEBUG(gcdDEBUG_TRACE) + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(eventNumber), + "Processing interrupt %d", + eventNumber + ); +#endif + } + + gckOS_AtomClearMask(Event->pending, mask); + + if (!gckHARDWARE_IsFeatureAvailable(Event->kernel->hardware, gcvFEATURE_FENCE_64BIT)) + { + /* Write out commit stamp.*/ + *(gctUINT64 *)(Event->kernel->command->fence->logical) = queue->commitStamp; + } + + /* Signal clients waiting for fence. */ + gcmkVERIFY_OK(gckFENCE_Signal( + Event->os, + Event->kernel->command->fence + )); + + /* Grab the event head. */ + record = queue->head; + + /* Now quickly clear its event list. */ + queue->head = gcvNULL; + + /* Increase the number of free events. */ + Event->freeQueueCount++; + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Walk all events for this interrupt. */ + while (record != gcvNULL) + { + gcsEVENT_PTR recordNext; +#ifndef __QNXNTO__ + gctPOINTER logical; +#endif +#if gcdSECURE_USER + gctSIZE_T bytes; +#endif + + /* Grab next record. */ + recordNext = record->next; + +#ifdef __QNXNTO__ + /* + * Assign record->processID as the pid for this galcore thread. + * Used in the OS calls which do not take a pid. + */ + drv_thread_specific_key_assign(record->processID, 0); +#endif + +#if gcdSECURE_USER + /* Get the cache that belongs to this process. */ + gcmkONERROR(gckKERNEL_GetProcessDBCache(Event->kernel, + record->processID, + &cache)); +#endif + + gcmkTRACE_ZONE_N( + gcvLEVEL_INFO, gcvZONE_EVENT, + gcmSIZEOF(record->info.command), + "Processing event type: %d", + record->info.command + ); + + switch (record->info.command) + { + case gcvHAL_FREE_NON_PAGED_MEMORY: + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_FREE_NON_PAGED_MEMORY: 0x%x", + gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical)); + + /* Free non-paged memory. */ + status = gckOS_FreeNonPagedMemory( + Event->os, + (gctSIZE_T) record->info.u.FreeNonPagedMemory.bytes, + gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical), + gcmUINT64_TO_PTR(record->info.u.FreeNonPagedMemory.logical)); + + if (gcmIS_SUCCESS(status)) + { +#if gcdSECURE_USER + gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache( + Event->kernel, + cache, + gcmUINT64_TO_PTR(record->record.u.FreeNonPagedMemory.logical), + (gctSIZE_T) record->record.u.FreeNonPagedMemory.bytes)); +#endif + } + gcmRELEASE_NAME(record->info.u.FreeNonPagedMemory.physical); + break; + + case gcvHAL_FREE_CONTIGUOUS_MEMORY: + gcmkTRACE_ZONE( + gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_FREE_CONTIGUOUS_MEMORY: 0x%x", + gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical)); + + /* Unmap the user memory. */ + status = gckOS_FreeContiguous( + Event->os, + gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical), + gcmUINT64_TO_PTR(record->info.u.FreeContiguousMemory.logical), + (gctSIZE_T) record->info.u.FreeContiguousMemory.bytes); + + if (gcmIS_SUCCESS(status)) + { +#if gcdSECURE_USER + gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache( + Event->kernel, + cache, + gcmUINT64_TO_PTR(event->event.u.FreeContiguousMemory.logical), + (gctSIZE_T) event->event.u.FreeContiguousMemory.bytes)); +#endif + } + gcmRELEASE_NAME(record->info.u.FreeContiguousMemory.physical); + break; + + case gcvHAL_WRITE_DATA: +#ifndef __QNXNTO__ + /* Convert physical into logical address. */ + gcmkERR_BREAK( + gckOS_MapPhysical(Event->os, + record->info.u.WriteData.address, + gcmSIZEOF(gctUINT32), + &logical)); + + /* Write data. */ + gcmkERR_BREAK( + gckOS_WriteMemory(Event->os, + logical, + record->info.u.WriteData.data)); + + /* Unmap the physical memory. */ + gcmkERR_BREAK( + gckOS_UnmapPhysical(Event->os, + logical, + gcmSIZEOF(gctUINT32))); +#else + /* Write data. */ + gcmkERR_BREAK( + gckOS_WriteMemory(Event->os, + gcmUINT64_TO_PTR(record->info.u.WriteData.address), + record->info.u.WriteData.data)); +#endif + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_UNLOCK_VIDEO_MEMORY: 0x%x", + record->info.u.UnlockVideoMemory.node); + + nodeObject = gcmUINT64_TO_PTR(record->info.u.UnlockVideoMemory.node); + +#if gcdSECURE_USER + node = nodeObject->node; + + /* Save node information before it disappears. */ + node = event->event.u.UnlockVideoMemory.node; + if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM) + { + logical = gcvNULL; + bytes = 0; + } + else + { + logical = node->Virtual.logical; + bytes = node->Virtual.bytes; + } +#endif + + /* Unlock. */ + status = gckVIDMEM_Unlock( + Event->kernel, + nodeObject, + record->info.u.UnlockVideoMemory.type, + gcvNULL); + +#if gcdSECURE_USER + if (gcmIS_SUCCESS(status) && (logical != gcvNULL)) + { + gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache( + Event->kernel, + cache, + logical, + bytes)); + } +#endif + +#if gcdPROCESS_ADDRESS_SPACE + gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock( + Event->kernel, + nodeObject, + record->processID + )); +#endif + + status = gckVIDMEM_NODE_Dereference(Event->kernel, nodeObject); + break; + + case gcvHAL_SIGNAL: + signal = gcmUINT64_TO_PTR(record->info.u.Signal.signal); + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_SIGNAL: 0x%x", + signal); + +#ifdef __QNXNTO__ + if ((record->info.u.Signal.coid == 0) + && (record->info.u.Signal.rcvid == 0) + ) + { + /* Kernel signal. */ + gcmkERR_BREAK( + gckOS_SignalPulse(Event->os, + signal)); + } + else + { + /* User signal. */ + gcmkERR_BREAK( + gckOS_UserSignal(Event->os, + signal, + record->info.u.Signal.rcvid, + record->info.u.Signal.coid)); + } +#else + /* Set signal. */ + if (gcmUINT64_TO_PTR(record->info.u.Signal.process) == gcvNULL) + { + /* Kernel signal. */ + gcmkERR_BREAK( + gckOS_Signal(Event->os, + signal, + gcvTRUE)); + } + else + { + /* User signal. */ + gcmkERR_BREAK( + gckOS_UserSignal(Event->os, + signal, + gcmUINT64_TO_PTR(record->info.u.Signal.process))); + } + + gcmkASSERT(record->info.u.Signal.auxSignal == 0); +#endif + break; + + case gcvHAL_TIMESTAMP: + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "gcvHAL_TIMESTAMP: %d %d", + record->info.u.TimeStamp.timer, + record->info.u.TimeStamp.request); + + /* Process the timestamp. */ + switch (record->info.u.TimeStamp.request) + { + case 0: + status = gckOS_GetTime(&Event->kernel->timers[ + record->info.u.TimeStamp.timer]. + stopTime); + break; + + case 1: + status = gckOS_GetTime(&Event->kernel->timers[ + record->info.u.TimeStamp.timer]. + startTime); + break; + + default: + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(record->info.u.TimeStamp.request), + "Invalid timestamp request: %d", + record->info.u.TimeStamp.request + ); + + status = gcvSTATUS_INVALID_ARGUMENT; + break; + } + break; + + case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER: + gcmkVERIFY_OK( + gckKERNEL_DestroyVirtualCommandBuffer(Event->kernel, + (gctSIZE_T) record->info.u.FreeVirtualCommandBuffer.bytes, + gcmNAME_TO_PTR(record->info.u.FreeVirtualCommandBuffer.physical), + gcmUINT64_TO_PTR(record->info.u.FreeVirtualCommandBuffer.logical) + )); + gcmRELEASE_NAME(record->info.u.FreeVirtualCommandBuffer.physical); + break; + +#if gcdPROCESS_ADDRESS_SPACE + case gcvHAL_DESTROY_MMU: + status = gckMMU_Destroy(gcmUINT64_TO_PTR(record->info.u.DestroyMmu.mmu)); + break; +#endif + + case gcvHAL_COMMIT_DONE: + break; + + default: + /* Invalid argument. */ + gcmkTRACE_ZONE_N( + gcvLEVEL_ERROR, gcvZONE_EVENT, + gcmSIZEOF(record->info.command), + "Unknown event type: %d", + record->info.command + ); + + status = gcvSTATUS_INVALID_ARGUMENT; + break; + } + + /* Make sure there are no errors generated. */ + if (gcmIS_ERROR(status)) + { + gcmkTRACE_ZONE_N( + gcvLEVEL_WARNING, gcvZONE_EVENT, + gcmSIZEOF(status), + "Event produced status: %d(%s)", + status, gckOS_DebugStatus2Name(status)); + } + + /* Free the event. */ + gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record)); + + /* Advance to next record. */ + record = recordNext; + } + + gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT, + "Handled interrupt 0x%x", mask); + } + + if (IDs == 0) + { + gcmkONERROR(_TryToIdleGPU(Event)); + } + + /* End of event handling. */ + Event->notifyState = -1; + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + if (acquired) + { + /* Release mutex. */ + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* End of event handling. */ + Event->notifyState = -1; + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckEVENT_FreeProcess +** +** Free all events owned by a particular process ID. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctUINT32 ProcessID +** Process ID of the process to be freed up. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_FreeProcess( + IN gckEVENT Event, + IN gctUINT32 ProcessID + ) +{ + gctSIZE_T i; + gctBOOL acquired = gcvFALSE; + gcsEVENT_PTR record, next; + gceSTATUS status; + gcsEVENT_PTR deleteHead, deleteTail; + + gcmkHEADER_ARG("Event=0x%x ProcessID=%d", Event, ProcessID); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Walk through all queues. */ + for (i = 0; i < gcmCOUNTOF(Event->queues); ++i) + { + if (Event->queues[i].head != gcvNULL) + { + /* Grab the event queue mutex. */ + gcmkONERROR(gckOS_AcquireMutex(Event->os, + Event->eventQueueMutex, + gcvINFINITE)); + acquired = gcvTRUE; + + /* Grab the mutex head. */ + record = Event->queues[i].head; + Event->queues[i].head = gcvNULL; + Event->queues[i].tail = gcvNULL; + deleteHead = gcvNULL; + deleteTail = gcvNULL; + + while (record != gcvNULL) + { + next = record->next; + if (record->processID == ProcessID) + { + if (deleteHead == gcvNULL) + { + deleteHead = record; + } + else + { + deleteTail->next = record; + } + + deleteTail = record; + } + else + { + if (Event->queues[i].head == gcvNULL) + { + Event->queues[i].head = record; + } + else + { + Event->queues[i].tail->next = record; + } + + Event->queues[i].tail = record; + } + + record->next = gcvNULL; + record = next; + } + + /* Release the mutex queue. */ + gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + acquired = gcvFALSE; + + /* Loop through the entire list of events. */ + for (record = deleteHead; record != gcvNULL; record = next) + { + /* Get the next event record. */ + next = record->next; + + /* Free the event record. */ + gcmkONERROR(gckEVENT_FreeRecord(Event, record)); + } + } + } + + gcmkONERROR(_TryToIdleGPU(Event)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + /* Release the event queue mutex. */ + if (acquired) + { + gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex)); + } + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +/******************************************************************************* +** gckEVENT_Stop +** +** Stop the hardware using the End event mechanism. +** +** INPUT: +** +** gckEVENT Event +** Pointer to an gckEVENT object. +** +** gctUINT32 ProcessID +** Process ID Logical belongs. +** +** gctPHYS_ADDR Handle +** Physical address handle. If gcvNULL it is video memory. +** +** gctPOINTER Logical +** Logical address to flush. +** +** gctSIGNAL Signal +** Pointer to the signal to trigger. +** +** OUTPUT: +** +** Nothing. +*/ +gceSTATUS +gckEVENT_Stop( + IN gckEVENT Event, + IN gctUINT32 ProcessID, + IN gctUINT32 Handle, + IN gctPOINTER Logical, + IN gctUINT32 Address, + IN gctSIGNAL Signal, + IN OUT gctUINT32 * waitSize + ) +{ + gceSTATUS status; + /* gctSIZE_T waitSize;*/ + gcsEVENT_PTR record = gcvNULL; + gctUINT8 id = 0xFF; + + gcmkHEADER_ARG("Event=0x%x ProcessID=%u Handle=0x%x Logical=0x%x " + "Address=0x%x Signal=0x%x", + Event, ProcessID, Handle, Logical, Address, Signal); + + /* Verify the arguments. */ + gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT); + + /* Submit the current event queue. */ + gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE)); + gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL)); + + /* Allocate a record. */ + gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &record)); + + /* Initialize the record. */ + record->next = gcvNULL; + record->processID = ProcessID; + record->info.command = gcvHAL_SIGNAL; + record->info.u.Signal.signal = gcmPTR_TO_UINT64(Signal); +#ifdef __QNXNTO__ + record->info.u.Signal.coid = 0; + record->info.u.Signal.rcvid = 0; +#endif + record->info.u.Signal.auxSignal = 0; + record->info.u.Signal.process = 0; + + /* Append the record. */ + Event->queues[id].head = record; + + /* Replace last WAIT with END. */ + gcmkONERROR(gckHARDWARE_End( + Event->kernel->hardware, Logical, Address, waitSize + )); + +#if USE_KERNEL_VIRTUAL_BUFFERS + if (Event->kernel->virtualCommandBuffer) + { + gcmkONERROR(gckKERNEL_GetGPUAddress( + Event->kernel, + Logical, + gcvFALSE, + Event->kernel->command->virtualMemory, + &Event->kernel->hardware->lastEnd + )); + } +#endif + +#if gcdNONPAGED_MEMORY_CACHEABLE + /* Flush the cache for the END. */ + gcmkONERROR(gckOS_CacheClean( + Event->os, + ProcessID, + gcvNULL, + (gctUINT32)Handle, + Logical, + *waitSize + )); +#endif + + /* Wait for the signal. */ + gcmkONERROR(gckOS_WaitSignal(Event->os, Signal, gcvFALSE, gcvINFINITE)); + + /* Success. */ + gcmkFOOTER_NO(); + return gcvSTATUS_OK; + +OnError: + + /* Return the status. */ + gcmkFOOTER(); + return status; +} + +static void +_PrintRecord( + gcsEVENT_PTR record + ) +{ + switch (record->info.command) + { + case gcvHAL_FREE_NON_PAGED_MEMORY: + gcmkPRINT(" gcvHAL_FREE_NON_PAGED_MEMORY"); + break; + + case gcvHAL_FREE_CONTIGUOUS_MEMORY: + gcmkPRINT(" gcvHAL_FREE_CONTIGUOUS_MEMORY"); + break; + + case gcvHAL_WRITE_DATA: + gcmkPRINT(" gcvHAL_WRITE_DATA"); + break; + + case gcvHAL_UNLOCK_VIDEO_MEMORY: + gcmkPRINT(" gcvHAL_UNLOCK_VIDEO_MEMORY"); + break; + + case gcvHAL_SIGNAL: + gcmkPRINT(" gcvHAL_SIGNAL process=%d signal=0x%x", + record->info.u.Signal.process, + record->info.u.Signal.signal); + break; + + case gcvHAL_UNMAP_USER_MEMORY: + gcmkPRINT(" gcvHAL_UNMAP_USER_MEMORY"); + break; + + case gcvHAL_TIMESTAMP: + gcmkPRINT(" gcvHAL_TIMESTAMP"); + break; + + case gcvHAL_COMMIT_DONE: + gcmkPRINT(" gcvHAL_COMMIT_DONE"); + break; + + case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER: + gcmkPRINT(" gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER logical=0x%08x", + record->info.u.FreeVirtualCommandBuffer.logical); + break; + + case gcvHAL_DESTROY_MMU: + gcmkPRINT(" gcvHAL_DESTORY_MMU mmu=0x%08x", + gcmUINT64_TO_PTR(record->info.u.DestroyMmu.mmu)); + + break; + default: + gcmkPRINT(" Illegal Event %d", record->info.command); + break; + } +} + +/******************************************************************************* +** gckEVENT_Dump +** +** Dump record in event queue when stuck happens. +** No protection for the event queue. +**/ +gceSTATUS +gckEVENT_Dump( + IN gckEVENT Event + ) +{ + gcsEVENT_QUEUE_PTR queueHead = Event->queueHead; + gcsEVENT_QUEUE_PTR queue; + gcsEVENT_PTR record = gcvNULL; + gctINT i; +#if gcdINTERRUPT_STATISTIC + gctINT32 pendingInterrupt; + gctUINT32 intrAcknowledge; +#endif + gctINT32 pending; + + gcmkHEADER_ARG("Event=0x%x", Event); + + gcmkPRINT("**************************\n"); + gcmkPRINT("*** EVENT STATE DUMP ***\n"); + gcmkPRINT("**************************\n"); + + gcmkPRINT(" Unsumbitted Event:"); + while(queueHead) + { + queue = queueHead; + record = queueHead->head; + + gcmkPRINT(" [%x]:", queue); + while(record) + { + _PrintRecord(record); + record = record->next; + } + + if (queueHead == Event->queueTail) + { + queueHead = gcvNULL; + } + else + { + queueHead = queueHead->next; + } + } + + gcmkPRINT(" Untriggered Event:"); + for (i = 0; i < gcmCOUNTOF(Event->queues); i++) + { + queue = &Event->queues[i]; + record = queue->head; + + gcmkPRINT(" [%d]:", i); + while(record) + { + _PrintRecord(record); + record = record->next; + } + } + +#if gcdINTERRUPT_STATISTIC + gckOS_AtomGet(Event->os, Event->interruptCount, &pendingInterrupt); + gcmkPRINT(" Number of Pending Interrupt: %d", pendingInterrupt); + + if (Event->kernel->recovery == 0) + { + gckOS_ReadRegisterEx( + Event->os, + Event->kernel->core, + 0x10, + &intrAcknowledge + ); + + gcmkPRINT(" INTR_ACKNOWLEDGE=0x%x", intrAcknowledge); + } +#endif + + gcmkPRINT(" Notify State=%d", Event->notifyState); + + gckOS_AtomGet(Event->os, Event->pending, &pending); + + gcmkPRINT(" Pending=0x%x", pending); + + gcmkFOOTER_NO(); + return gcvSTATUS_OK; +} |