[ros-diffs] [ion] 20557: - Implement Pushlocks. Only Waking,
and Exclusive Acquire/Release + Waits have been tested.
The shared paths and block/unblock are still untested and may
probably contain bugs. Pushlocks are not yet used in the
kernel and they won't be until all paths are tested and validated.
ion at svn.reactos.org
ion at svn.reactos.org
Wed Jan 4 00:36:18 CET 2006
- Implement Pushlocks. Only Waking, and Exclusive Acquire/Release +
Waits have been tested. The shared paths and block/unblock are still
untested and may probably contain bugs. Pushlocks are not yet used in
the kernel and they won't be until all paths are tested and validated.
- Public DDIs Implemented: ExfAcquirePushLockExclusive,
ExfAcquirePushLockShared, ExfTryToWakePushLock, ExfReleasePushLock,
ExfReleasePushLockExclusive, ExfReleasePushLockShared,
ExfUnblockPushLock.
Added: trunk/reactos/ntoskrnl/ex/pushlock.c
Modified: trunk/reactos/ntoskrnl/include/internal/ex.h
Modified: trunk/reactos/ntoskrnl/include/internal/ntoskrnl.h
Modified: trunk/reactos/ntoskrnl/ntoskrnl.xml
_____
Copied: trunk/reactos/ntoskrnl/ex/pushlock.c (from rev 20362,
trunk/reactos/ntoskrnl/ex/synch.c)
--- trunk/reactos/ntoskrnl/ex/synch.c 2005-12-27 01:14:55 UTC (rev
20362)
+++ trunk/reactos/ntoskrnl/ex/pushlock.c 2006-01-03 23:36:05 UTC
(rev 20557)
@@ -0,0 +1,1164 @@
+/*
+ * COPYRIGHT: See COPYING in the top level directory
+ * PROJECT: ReactOS Kernel
+ * FILE: ntoskrnl/ex/pushlock.c
+ * PURPOSE: Pushlock and Cache-Aware Pushlock Implementation
+ * PROGRAMMER: Alex Ionescu (alex.ionescu at reactos.com)
+ */
+
+/* INCLUDES
*****************************************************************/
+
+#include <ntoskrnl.h>
+#define NDEBUG
+#include <internal/debug.h>
+
+/* DATA
**********************************************************************/
+
+ULONG ExPushLockSpinCount;
+
+/* PRIVATE FUNCTIONS
*********************************************************/
+
+/*++
+ * @name ExpInitializePushLocks
+ *
+ * The ExpInitializePushLocks routine initialized Pushlock support.
+ *
+ * @param None.
+ *
+ * @return None.
+ *
+ * @remarks The ExpInitializePushLocks routine sets up the spin on SMP
machines.
+ *
+ *--*/
+VOID
+NTAPI
+ExpInitializePushLocks(VOID)
+{
+ /* Initialize an internal 1024-iteration spin for MP CPUs */
+ ExPushLockSpinCount = (KeNumberProcessors == 1) ? 0 : 1024;
+}
+
+/*++
+ * @name ExfWakePushLock
+ *
+ * The ExfWakePushLock routine wakes a Pushlock that is in the
waiting
+ * state.
+ *
+ * @param PushLock
+ * Pointer to a pushlock that is waiting.
+ *
+ * @param OldValue
+ * Last known value of the pushlock before this routine was
called.
+ *
+ * @return None.
+ *
+ * @remarks This is an internal routine; do not call it manually. Only
the system
+ * can properly know if the pushlock is ready to be awakened
or not.
+ * External callers should use ExfTrytoWakePushLock.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfWakePushLock(PEX_PUSH_LOCK PushLock,
+ EX_PUSH_LOCK OldValue)
+{
+ EX_PUSH_LOCK NewValue;
+ PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock, FirstWaitBlock,
NextWaitBlock;
+ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
+ KIRQL OldIrql;
+
+ /* Start main wake loop */
+ for (;;)
+ {
+ /* Sanity checks */
+ ASSERT(!OldValue.MultipleShared);
+
+ /* Check if it's locked */
+ if (OldValue.Locked)
+ {
+ /* If it's locked we must simply un-wake it*/
+ for (;;)
+ {
+ /* It's not waking anymore */
+ NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
+
+ /* Sanity checks */
+ ASSERT(!NewValue.Waking);
+ ASSERT(NewValue.Locked);
+ ASSERT(NewValue.Waiting);
+
+ /* Write the New Value */
+ NewValue.Ptr =
InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value == OldValue.Value) return;
+
+ /* Someone changed the value behind our back, update
it*/
+ OldValue = NewValue;
+
+ /* Check if it's still locked */
+ if (OldValue.Locked) continue;
+ }
+ }
+
+ /* Save the First Block */
+ FirstWaitBlock =
(PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)OldValue.Ptr &
+ ~EX_PUSH_LOCK_PTR_BITS);
+ NextWaitBlock = FirstWaitBlock;
+ WaitBlock = NextWaitBlock->Last;
+
+ /* Try to find a wait block */
+ while (!WaitBlock)
+ {
+ /* Save the previous block */
+ PreviousWaitBlock = NextWaitBlock;
+
+ /* Move to next block */
+ NextWaitBlock = NextWaitBlock->Next;
+
+ /* Save the previous block */
+ NextWaitBlock->Previous = PreviousWaitBlock;
+
+ /* Move to the next one */
+ WaitBlock = NextWaitBlock->Last;
+ }
+
+ /* Check if the last Wait Block is not Exclusive or if it's the
only one */
+ PreviousWaitBlock = WaitBlock->Previous;
+ if (!(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE) ||
+ !(PreviousWaitBlock))
+ {
+ /* Destroy the pushlock */
+ if (InterlockedCompareExchangePointer(PushLock, 0,
OldValue.Ptr) ==
+ OldValue.Ptr) break;
+ }
+ else
+ {
+ /* Link the wait blocks */
+ FirstWaitBlock->Last = PreviousWaitBlock;
+ WaitBlock->Previous = NULL;
+
+ /* Sanity checks */
+ ASSERT(FirstWaitBlock != WaitBlock);
+ ASSERT(PushLock->Waiting);
+
+ /* Remove waking bit from pushlock */
+ InterlockedAnd((PLONG)PushLock, ~EX_PUSH_LOCK_WAKING);
+ }
+ }
+
+ /* Check if there's a previous block */
+ OldIrql = DISPATCH_LEVEL;
+ if (WaitBlock->Previous)
+ {
+ /* Raise to Dispatch */
+ KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
+ }
+
+ /* Signaling loop */
+ for (;;)
+ {
+ /* Get the previous Wait block */
+ PreviousWaitBlock = WaitBlock->Previous;
+
+ /* Sanity check */
+ ASSERT(!WaitBlock->Signaled);
+
+ /* We are about to get signaled */
+ WaitBlock->Signaled = TRUE;
+
+ /* Set the Wait Bit in the Wait Block */
+ if (!InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
+ {
+ /* Nobody signaled us, so do it */
+ KeSignalGateBoostPriority(&WaitBlock->WakeGate);
+ }
+
+ /* Set the wait block and check if there still is one to loop*/
+ WaitBlock = PreviousWaitBlock;
+ if (!WaitBlock) break;
+ }
+
+ /* Check if we have to lower back the IRQL */
+ if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
+}
+
+/*++
+ * @name ExpOptimizePushLockList
+ *
+ * The ExpOptimizePushLockList routine optimizes the list of
waiters
+ * associated to a pushlock's wait block.
+ *
+ * @param PushLock
+ * Pointer to a pushlock whose waiter list needs to be
optimized.
+ *
+ * @param OldValue
+ * Last known value of the pushlock before this routine was
called.
+ *
+ * @return None.
+ *
+ * @remarks At the end of the optimization, the pushlock will also be
wakened.
+ *
+ *--*/
+VOID
+FASTCALL
+ExpOptimizePushLockList(PEX_PUSH_LOCK PushLock,
+ EX_PUSH_LOCK OldValue)
+{
+ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock,
PreviousWaitBlock;
+ EX_PUSH_LOCK NewValue;
+
+ /* Check if the pushlock is locked */
+ if (OldValue.Locked)
+ {
+ /* Start main loop */
+ for (;;)
+ {
+ /* Get the wait block */
+ WaitBlock =
(PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)OldValue.Ptr &
+ ~EX_PUSH_LOCK_PTR_BITS);
+
+ /* Loop the blocks */
+ LastWaitBlock = WaitBlock->Last;
+ while (LastWaitBlock)
+ {
+ /* Save the block */
+ PreviousWaitBlock = WaitBlock;
+
+ /* Get the next block */
+ WaitBlock = WaitBlock->Next;
+
+ /* Save the previous */
+ WaitBlock->Previous = PreviousWaitBlock;
+
+ /* Move to the next */
+ LastWaitBlock = WaitBlock->Last;
+ }
+
+ /* Remove the wake bit */
+ NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
+
+ /* Sanity checks */
+ ASSERT(NewValue.Locked);
+ ASSERT(!NewValue.Waking);
+
+ /* Update the value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+
+ /* If we updated correctly, leave */
+ if (NewValue.Value == OldValue.Value) return;
+
+ /* If the value is now locked, loop again */
+ if (NewValue.Locked) continue;
+ }
+ }
+
+ /* Wake the push lock */
+ ExfWakePushLock(PushLock, OldValue);
+}
+
+/*++
+ * @name ExTimedWaitForUnblockPushLock
+ *
+ * The ExTimedWaitForUnblockPushLock routine waits for a pushlock
+ * to be unblocked, for a specified internal.
+ *
+ * @param PushLock
+ * Pointer to a pushlock whose waiter list needs to be
optimized.
+ *
+ * @param WaitBlock
+ * Pointer to the pushlock's wait block.
+ *
+ * @param Timeout
+ * Amount of time to wait for this pushlock to be unblocked.
+ *
+ * @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise
the error
+ * code returned by KeWaitForSingleObject.
+ *
+ * @remarks If the wait fails, then a manual unblock is attempted.
+ *
+ *--*/
+NTSTATUS
+FASTCALL
+ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
+ IN PVOID WaitBlock,
+ IN PLARGE_INTEGER Timeout)
+{
+ ULONG i;
+ NTSTATUS Status;
+
+ /* Initialize the wait event */
+
KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->WakeEvent,
+ NotificationEvent,
+ FALSE);
+
+ /* Spin on the push lock if necessary */
+ i = ExPushLockSpinCount;
+ if (i)
+ {
+ /* Spin */
+ while (--i)
+ {
+ /* Check if we got lucky and can leave early */
+ if (!(((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags &
+ EX_PUSH_LOCK_WAITING))
+ {
+ /* This wait block isn't waiting anymore, we can leave
*/
+ return STATUS_SUCCESS;
+ }
+ YieldProcessor();
+ }
+ }
+
+ /* Now try to remove the wait bit */
+ if
(InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flag
s,
+ 1))
+ {
+ /* Nobody removed it already, let's do a full wait */
+ Status =
KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->
+ WakeEvent,
+ WrPushLock,
+ KernelMode,
+ FALSE,
+ Timeout);
+ if (!NT_SUCCESS(Status))
+ {
+ /* Try unblocking the pushlock */
+ ExfUnblockPushLock(PushLock, WaitBlock);
+ }
+ }
+ else
+ {
+ /* Someone beat us to it, no need to wait */
+ Status = STATUS_SUCCESS;
+ }
+
+ /* Return status */
+ return Status;
+}
+
+/*++
+ * @name ExBlockPushLock
+ *
+ * The ExBlockPushLock routine blocks a pushlock.
+ *
+ * @param PushLock
+ * Pointer to a pushlock whose waiter list needs to be
optimized.
+ *
+ * @param WaitBlock
+ * Pointer to the pushlock's wait block.
+ *
+ * @return None.
+ *
+ * @remarks None.
+ *
+ *--*/
+VOID
+FASTCALL
+ExBlockPushLock(PEX_PUSH_LOCK PushLock,
+ PVOID WaitBlock)
+{
+ PVOID NewValue, OldValue;
+
+ /* Set the waiting bit */
+ ((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags |=
EX_PUSH_LOCK_FLAGS_WAIT;
+
+ /* Link the wait blocks */
+ ((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Next = PushLock->Ptr;
+
+ /* Try to set this one as the wait block now */
+ NewValue = PushLock->Ptr;
+ for (;;)
+ {
+ /* Set the new wait block value */
+ OldValue = InterlockedCompareExchangePointer(&PushLock->Ptr,
+ WaitBlock,
+ NewValue);
+ if (OldValue == NewValue) break;
+ NewValue = OldValue;
+ }
+}
+
+/* PUBLIC FUNCTIONS
**********************************************************/
+
+/*++
+ * @name ExAcquirePushLockExclusive
+ * @implemented NT5.1
+ *
+ * The ExAcquirePushLockExclusive macro exclusively acquires a
PushLock.
+ *
+ * @params PushLock
+ * Pointer to the pushlock which is to be acquired.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExAcquirePushLockShared must be running at IRQL
<= APC_LEVEL.
+ * This macro should usually be paired up with
KeAcquireCriticalRegion.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
+{
+ EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
+ EX_PUSH_LOCK OldValue = *PushLock, NewValue, TempValue;
+ BOOLEAN NeedWake;
+ ULONG i;
+
+ /* Start main loop */
+ for (;;)
+ {
+ /* Check if it's unlocked */
+ if (!OldValue.Locked)
+ {
+ /* Lock it */
+ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
+ ASSERT(NewValue.Locked);
+
+ /* Set the new value */
+ if (InterlockedCompareExchangePointer(PushLock,
+ NewValue.Ptr,
+ OldValue.Ptr) !=
OldValue.Ptr)
+ {
+ /* Retry */
+ OldValue = *PushLock;
+ continue;
+ }
+
+ /* Break out of the loop */
+ break;
+ }
+ else
+ {
+ /* We'll have to create a Waitblock */
+ WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_EXCLUSIVE |
+ EX_PUSH_LOCK_FLAGS_WAIT;
+ WaitBlock.Previous = NULL;
+ NeedWake = FALSE;
+
+ /* Check if there is already a waiter */
+ if (OldValue.Waiting)
+ {
+ /* Nobody is the last waiter yet */
+ WaitBlock.Last = NULL;
+
+ /* We are an exclusive waiter */
+ WaitBlock.ShareCount = 0;
+
+ /* Set the current Wait Block pointer */
+ WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)
+ OldValue.Ptr &~
EX_PUSH_LOCK_PTR_BITS);
+
+ /* Point to ours */
+ NewValue.Value = (OldValue.Value &
EX_PUSH_LOCK_MULTIPLE_SHARED) |
+ EX_PUSH_LOCK_LOCK |
+ EX_PUSH_LOCK_WAKING |
+ EX_PUSH_LOCK_WAITING |
+ PtrToUlong(&WaitBlock);
+
+ /* Check if the pushlock was already waking */
+ if (OldValue.Waking) NeedWake = TRUE;
+ }
+ else
+ {
+ /* We are the first waiter, so loop the wait block */
+ WaitBlock.Last = &WaitBlock;
+
+ /* Set the share count */
+ WaitBlock.ShareCount = OldValue.Shared;
+
+ /* Check if someone is sharing this pushlock */
+ if (OldValue.Shared > 1)
+ {
+ /* Point to our wait block */
+ NewValue.Value = EX_PUSH_LOCK_MULTIPLE_SHARED |
+ EX_PUSH_LOCK_LOCK |
+ EX_PUSH_LOCK_WAITING |
+ PtrToUlong(&WaitBlock);
+ }
+ else
+ {
+ /* No shared count */
+ WaitBlock.ShareCount = 0;
+
+ /* Point to our wait block */
+ NewValue.Value = EX_PUSH_LOCK_LOCK |
+ EX_PUSH_LOCK_WAITING |
+ PtrToUlong(&WaitBlock);
+ }
+ }
+
+#if DBG
+ /* Setup the Debug Wait Block */
+ WaitBlock.Signaled = 0;
+ WaitBlock.OldValue = OldValue;
+ WaitBlock.NewValue = NewValue;
+ WaitBlock.PushLock = PushLock;
+#endif
+
+ /* Sanity check */
+ ASSERT(NewValue.Waiting);
+ ASSERT(NewValue.Locked);
+
+ /* Write the new value */
+ TempValue = NewValue;
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value != OldValue.Value)
+ {
+ /* Retry */
+ OldValue = *PushLock;
+ continue;
+ }
+
+ /* Check if the pushlock needed waking */
+ if (NeedWake)
+ {
+ /* Scan the Waiters and Wake PushLocks */
+ ExpOptimizePushLockList(PushLock, TempValue);
+ }
+
+ /* Set up the Wait Gate */
+ KeInitializeGate(&WaitBlock.WakeGate);
+
+ /* Now spin on the push lock if necessary */
+ i = ExPushLockSpinCount;
+ if ((i) && (WaitBlock.Flags & EX_PUSH_LOCK_WAITING))
+ {
+ /* Spin */
+ while (--i) YieldProcessor();
+ }
+
+ /* Now try to remove the wait bit */
+ if (InterlockedBitTestAndReset(&WaitBlock.Flags, 1))
+ {
+ /* Nobody removed it already, let's do a full wait */
+ KeWaitForGate(&WaitBlock.WakeGate, WrPushLock,
KernelMode);
+ ASSERT(WaitBlock.Signaled);
+ }
+
+ /* We shouldn't be shared anymore */
+ ASSERT((WaitBlock.ShareCount == 0));
+
+ /* Loop again */
+ OldValue = NewValue;
+ }
+ }
+}
+
+/*++
+ * @name ExAcquirePushLockExclusive
+ * @implemented NT5.1
+ *
+ * The ExAcquirePushLockShared macro acquires a shared PushLock.
+ *
+ * @params PushLock
+ * Pointer to the pushlock which is to be acquired.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExAcquirePushLockShared must be running at IRQL
<= APC_LEVEL.
+ * This macro should usually be paired up with
KeAcquireCriticalRegion.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
+{
+ EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
+ EX_PUSH_LOCK OldValue = *PushLock, NewValue;
+ BOOLEAN NeedWake;
+ ULONG i;
+
+ /* Start main loop */
+ for (;;)
+ {
+ /* Check if it's unlocked or if it's waiting without any
sharers */
+ if (!(OldValue.Locked) || (OldValue.Waiting && OldValue.Shared
== 0))
+ {
+ /* Check if anyone is waiting on it */
+ if (!OldValue.Waiting)
+ {
+ /* Increase the share count and lock it */
+ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
+ NewValue.Shared++;
+ }
+ else
+ {
+ /* Simply set the lock bit */
+ NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
+ }
+
+ /* Sanity check */
+ ASSERT(NewValue.Locked);
+
+ /* Set the new value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value != OldValue.Value)
+ {
+ /* Retry */
+ OldValue = NewValue;
+ continue;
+ }
+
+ /* Break out of the loop */
+ break;
+ }
+ else
+ {
+ /* We'll have to create a Waitblock */
+ WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_WAIT;
+ WaitBlock.ShareCount = 0;
+ NeedWake = FALSE;
+ WaitBlock.Previous = NULL;
+
+ /* Check if there is already a waiter */
+ if (OldValue.Waiting)
+ {
+ /* Set the current Wait Block pointer */
+ WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)
+ OldValue.Ptr &~
EX_PUSH_LOCK_PTR_BITS);
+
+ /* Nobody is the last waiter yet */
+ WaitBlock.Last = NULL;
+
+ /* Point to ours */
+ NewValue.Value = (OldValue.Value &
(EX_PUSH_LOCK_MULTIPLE_SHARED |
+ EX_PUSH_LOCK_LOCK))
|
+ EX_PUSH_LOCK_WAKING |
+ EX_PUSH_LOCK_WAITING |
+ PtrToUlong(&WaitBlock);
+
+ /* Check if the pushlock was already waking */
+ if (OldValue.Waking) NeedWake = TRUE;
+ }
+ else
+ {
+ /* We are the first waiter, so loop the wait block */
+ WaitBlock.Last = &WaitBlock;
+
+ /* Point to our wait block */
+ NewValue.Value = (OldValue.Value &
(EX_PUSH_LOCK_MULTIPLE_SHARED |
+
EX_PUSH_LOCK_WAKING)) |
+ EX_PUSH_LOCK_WAITING |
+ PtrToUlong(&WaitBlock);
+ }
+
+ /* Sanity check */
+ ASSERT(NewValue.Waiting);
+
+#if DBG
+ /* Setup the Debug Wait Block */
+ WaitBlock.Signaled = 0;
+ WaitBlock.OldValue = OldValue;
+ WaitBlock.NewValue = NewValue;
+ WaitBlock.PushLock = PushLock;
+#endif
+
+ /* Write the new value */
+ if (InterlockedCompareExchangePointer(PushLock,
+ NewValue.Ptr,
+ OldValue.Ptr) !=
OldValue.Ptr)
+ {
+ /* Retry */
+ OldValue = NewValue;
+ continue;
+ }
+
+ /* Update the value now */
+ OldValue = NewValue;
+
+ /* Check if the pushlock needed waking */
+ if (NeedWake)
+ {
+ /* Scan the Waiters and Wake PushLocks */
+ ExpOptimizePushLockList(PushLock, OldValue);
+ }
+
+ /* Set up the Wait Gate */
+ KeInitializeGate(&WaitBlock.WakeGate);
+
+ /* Now spin on the push lock if necessary */
+ i = ExPushLockSpinCount;
+ if ((i) && (WaitBlock.Flags & EX_PUSH_LOCK_WAITING))
+ {
+ /* Spin */
+ while (--i) YieldProcessor();
+ }
+
+ /* Now try to remove the wait bit */
+ if (InterlockedBitTestAndReset(&WaitBlock.Flags, 1))
+ {
+ /* Fast-path did not work, we need to do a full wait */
+ KeWaitForGate(&WaitBlock.WakeGate, WrPushLock,
KernelMode);
+ ASSERT(WaitBlock.Signaled);
+ }
+
+ /* We shouldn't be shared anymore */
+ ASSERT((WaitBlock.ShareCount == 0));
+ }
+ }
+}
+
+/*++
+ * @name ExfReleasePushLock
+ * @implemented NT5.1
+ *
+ * The ExReleasePushLockExclusive routine releases a previously
+ * exclusively acquired PushLock.
+ *
+ * @params PushLock
+ * Pointer to a previously acquired pushlock.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExReleasePushLockExclusive must be running at
IRQL <= APC_LEVEL.
+ * This macro should usually be paired up with
KeLeaveCriticalRegion.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfReleasePushLock(PEX_PUSH_LOCK PushLock)
+{
+ EX_PUSH_LOCK OldValue = *PushLock;
+ EX_PUSH_LOCK NewValue;
+ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
+
+ /* Sanity check */
+ ASSERT(OldValue.Locked);
+
+ /* Check if someone is waiting on the lock */
+ if (!OldValue.Waiting)
+ {
+ /* Nobody is waiting on it, so we'll try a quick release */
+ for (;;)
+ {
+ /* Check if it's shared */
+ if (OldValue.Shared > 1)
+ {
+ /* Write the Old Value but decrease share count */
+ NewValue = OldValue;
+ NewValue.Shared--;
+ }
+ else
+ {
+ /* Simply clear the lock */
+ NewValue.Value = 0;
+ }
+
+ /* Write the New Value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value == OldValue.Value)
+ {
+ /* No waiters left, we're done */
+ goto quit;
+ }
+
+ /* Did it enter a wait state? */
+ OldValue = NewValue;
+ if (NewValue.Waiting) break;
+ }
+ }
+
+ /* Ok, we do know someone is waiting on it. Are there more then
one? */
+ if (OldValue.MultipleShared)
+ {
+ /* Find the last Wait Block */
+ for (WaitBlock =
(PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)OldValue.Ptr &
+
~EX_PUSH_LOCK_PTR_BITS);
+ WaitBlock->Last;
+ WaitBlock = WaitBlock->Next);
+
+ /* Make sure the Share Count is above 0 */
+ if (WaitBlock->ShareCount)
+ {
+ /* This shouldn't be an exclusive wait block */
+ ASSERT(WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
+
+ /* Do the decrease and check if the lock isn't shared
anymore */
+ if (InterlockedExchangeAdd(&WaitBlock->ShareCount, -1))
+ {
+ /* Someone is still holding the lock */
+ goto quit;
+ }
+ }
+ }
+
+ /*
+ * If nobody was waiting on the block, then we possibly reduced the
number
+ * of times the pushlock was shared, and we unlocked it.
+ * If someone was waiting, and more then one person is waiting,
then we
+ * reduced the number of times the pushlock is shared in the wait
block.
+ * Therefore, at this point, we can now 'satisfy' the wait.
+ */
+ for (;;)
+ {
+ /* Now we need to see if it's waking */
+ if (OldValue.Waking)
+ {
+ /* Remove the lock and multiple shared bits */
+ NewValue.Value = OldValue.Value;
+ NewValue.MultipleShared = FALSE;
+ NewValue.Locked = FALSE;
+
+ /* Sanity check */
+ ASSERT(NewValue.Waking && !NewValue.Locked &&
!NewValue.MultipleShared);
+
+ /* Write the new value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value == OldValue.Value) break;
+
+ /* The value changed, try the unlock again */
+ continue;
+ }
+ else
+ {
+ /* Remove the lock and multiple shared bits */
+ NewValue.Value = OldValue.Value;
+ NewValue.MultipleShared = FALSE;
+ NewValue.Locked = FALSE;
+
+ /* It's not already waking, so add the wake bit */
+ NewValue.Waking = TRUE;
+
+ /* Sanity check */
+ ASSERT(NewValue.Waking && !NewValue.Locked &&
!NewValue.MultipleShared);
+
+ /* Write the new value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value != OldValue.Value) continue;
+
+ /* The write was successful. The pushlock is Unlocked and
Waking */
+ ExfWakePushLock(PushLock, NewValue);
+ break;
+ }
+ }
+quit:
+ /* Done! */
+ return;
+}
+
+/*++
+ * @name ExfReleasePushLockShared
+ * @implemented NT5.2
+ *
+ * The ExfReleasePushLockShared macro releases a previously
acquired PushLock.
+ *
+ * @params PushLock
+ * Pointer to a previously acquired pushlock.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExReleasePushLockShared must be running at IRQL
<= APC_LEVEL.
+ * This macro should usually be paired up with
KeLeaveCriticalRegion.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfReleasePushLockShared(PEX_PUSH_LOCK PushLock)
+{
+ EX_PUSH_LOCK OldValue = *PushLock;
+ EX_PUSH_LOCK NewValue;
+ PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
+
+ /* Check if someone is waiting on the lock */
+ if (!OldValue.Waiting)
+ {
+ /* Nobody is waiting on it, so we'll try a quick release */
+ for (;;)
+ {
+ /* Check if it's shared */
+ if (OldValue.Shared > 1)
+ {
+ /* Write the Old Value but decrease share count */
+ NewValue = OldValue;
+ NewValue.Shared--;
+ }
+ else
+ {
+ /* Simply clear the lock */
+ NewValue.Value = 0;
+ }
+
+ /* Write the New Value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value == OldValue.Value)
+ {
+ /* No waiters left, we're done */
+ goto quit;
+ }
+
+ /* Did it enter a wait state? */
+ OldValue = NewValue;
+ if (NewValue.Waiting) break;
+ }
+ }
+
+ /* Ok, we do know someone is waiting on it. Are there more then
one? */
+ if (OldValue.MultipleShared)
+ {
+ /* Find the last Wait Block */
+ for (WaitBlock =
(PEX_PUSH_LOCK_WAIT_BLOCK)((ULONG_PTR)OldValue.Ptr &
+
~EX_PUSH_LOCK_PTR_BITS);
+ WaitBlock->Last;
+ WaitBlock = WaitBlock->Next);
+
+ /* Sanity checks */
+ ASSERT(WaitBlock->ShareCount > 0);
+ ASSERT(WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
+
+ /* Do the decrease and check if the lock isn't shared anymore
*/
+ if (InterlockedExchangeAdd(&WaitBlock->ShareCount, -1)) goto
quit;
+ }
+
+ /*
+ * If nobody was waiting on the block, then we possibly reduced the
number
+ * of times the pushlock was shared, and we unlocked it.
+ * If someone was waiting, and more then one person is waiting,
then we
+ * reduced the number of times the pushlock is shared in the wait
block.
+ * Therefore, at this point, we can now 'satisfy' the wait.
+ */
+ for (;;)
+ {
+ /* Now we need to see if it's waking */
+ if (OldValue.Waking)
+ {
+ /* Remove the lock and multiple shared bits */
+ NewValue.Value = OldValue.Value;
+ NewValue.MultipleShared = FALSE;
+ NewValue.Locked = FALSE;
+
+ /* Sanity check */
+ ASSERT(NewValue.Waking && !NewValue.Locked &&
!NewValue.MultipleShared);
+
+ /* Write the new value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value == OldValue.Value) break;
+
+ /* The value changed, try the unlock again */
+ continue;
+ }
+ else
+ {
+ /* Remove the lock and multiple shared bits */
+ NewValue.Value = OldValue.Value;
+ NewValue.MultipleShared = FALSE;
+ NewValue.Locked = FALSE;
+
+ /* It's not already waking, so add the wake bit */
+ NewValue.Waking = TRUE;
+
+ /* Sanity check */
+ ASSERT(NewValue.Waking && !NewValue.Locked &&
!NewValue.MultipleShared);
+
+ /* Write the new value */
+ NewValue.Ptr = InterlockedCompareExchangePointer(PushLock,
+
NewValue.Ptr,
+
OldValue.Ptr);
+ if (NewValue.Value != OldValue.Value) continue;
+
+ /* The write was successful. The pushlock is Unlocked and
Waking */
+ ExfWakePushLock(PushLock, NewValue);
+ break;
+ }
+ }
+quit:
+ /* Done! */
+ return;
+}
+
+/*++
+ * ExfReleasePushLockExclusive
+ * @implemented NT5.2
+ *
+ * The ExfReleasePushLockExclusive routine releases a previously
+ * exclusively acquired PushLock.
+ *
+ * @params PushLock
+ * Pointer to a previously acquired pushlock.
+ *
+ * @return None.
[truncated at 1000 lines; 463 more skipped]
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://www.reactos.org/pipermail/ros-diffs/attachments/20060104/6527afed/attachment.html
More information about the Ros-diffs
mailing list