diff --git a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs index 423cd22c8..18e60687d 100644 --- a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs +++ b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTree.cs @@ -14,12 +14,13 @@ namespace Ryujinx.Common.Collections /// Adds a new node into the tree. /// /// Node to be added + /// Node to be added under /// is null - public void Add(T node) + public void Add(T node, T parent = null) { ArgumentNullException.ThrowIfNull(node); - Insert(node); + Insert(node, parent); } /// @@ -76,9 +77,11 @@ namespace Ryujinx.Common.Collections /// Inserts a new node into the tree. /// /// Node to be inserted - private void Insert(T node) + /// Node to be inserted under + private void Insert(T node, T parent = null) { - T newNode = BSTInsert(node); + T newNode = parent != null ? InsertWithParent(node, parent) : BSTInsert(node); + RestoreBalanceAfterInsertion(newNode); } @@ -122,10 +125,78 @@ namespace Ryujinx.Common.Collections else if (newNode.CompareTo(parent) < 0) { parent.Left = newNode; + + newNode.Successor = parent; + + if (parent.Predecessor != null) + { + newNode.Predecessor = parent.Predecessor; + parent.Predecessor = newNode; + newNode.Predecessor.Successor = newNode; + } + + parent.Predecessor = newNode; } else { parent.Right = newNode; + + newNode.Predecessor = parent; + + if (parent.Successor != null) + { + newNode.Successor = parent.Successor; + newNode.Successor.Predecessor = newNode; + } + + parent.Successor = newNode; + } + Count++; + return newNode; + } + + /// + /// Insertion Mechanism for a Binary Search Tree (BST). + ///

+ /// Inserts a new node directly under a parent node + /// where all children in the left subtree are less than , + /// and all children in the right subtree are greater than . + /// + /// Node to be inserted + /// Node to be inserted under + /// The inserted Node + private T InsertWithParent(T newNode, T parent) + { + newNode.Parent = parent; + + if (newNode.CompareTo(parent) < 0) + { + parent.Left = newNode; + + newNode.Successor = parent; + + if (parent.Predecessor != null) + { + newNode.Predecessor = parent.Predecessor; + parent.Predecessor = newNode; + newNode.Predecessor.Successor = newNode; + } + + parent.Predecessor = newNode; + } + else + { + parent.Right = newNode; + + newNode.Predecessor = parent; + + if (parent.Successor != null) + { + newNode.Successor = parent.Successor; + newNode.Successor.Predecessor = newNode; + } + + parent.Successor = newNode; } Count++; @@ -159,7 +230,7 @@ namespace Ryujinx.Common.Collections } else { - T element = Minimum(RightOf(nodeToDelete)); + T element = nodeToDelete.Successor; child = RightOf(element); parent = ParentOf(element); @@ -187,6 +258,9 @@ namespace Ryujinx.Common.Collections element.Left = old.Left; element.Right = old.Right; element.Parent = old.Parent; + element.Predecessor = old.Predecessor; + if (element.Predecessor != null) + element.Predecessor.Successor = element; if (ParentOf(old) == null) { @@ -241,6 +315,11 @@ namespace Ryujinx.Common.Collections { RestoreBalanceAfterRemoval(child); } + + if (old.Successor != null) + old.Successor.Predecessor = old.Predecessor; + if (old.Predecessor != null) + old.Predecessor.Successor = old.Successor; return old; } diff --git a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs index 29d2d0c9a..57e0b27c8 100644 --- a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs +++ b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs @@ -9,8 +9,7 @@ namespace Ryujinx.Common.Collections public T Left; public T Right; public T Parent; - - public T Predecessor => IntrusiveRedBlackTreeImpl.PredecessorOf((T)this); - public T Successor => IntrusiveRedBlackTreeImpl.SuccessorOf((T)this); + public T Predecessor; + public T Successor; } } diff --git a/src/Ryujinx.Common/Collections/TreeDictionary.cs b/src/Ryujinx.Common/Collections/TreeDictionary.cs index af104d268..453f128d3 100644 --- a/src/Ryujinx.Common/Collections/TreeDictionary.cs +++ b/src/Ryujinx.Common/Collections/TreeDictionary.cs @@ -109,7 +109,7 @@ namespace Ryujinx.Common.Collections Node node = GetNode(key); if (node != null) { - Node successor = SuccessorOf(node); + Node successor = node.Successor; return successor != null ? successor.Key : default; } @@ -127,7 +127,7 @@ namespace Ryujinx.Common.Collections Node node = GetNode(key); if (node != null) { - Node predecessor = PredecessorOf(node); + Node predecessor = node.Predecessor; return predecessor != null ? predecessor.Key : default; } @@ -136,11 +136,10 @@ namespace Ryujinx.Common.Collections } /// - /// Adds all the nodes in the dictionary as key/value pairs into . + /// Adds all the nodes in the dictionary as key/value pairs into a list. ///

/// The key/value pairs will be added in Level Order. /// - /// List to add the tree pairs into public List> AsLevelOrderList() { List> list = []; @@ -170,7 +169,7 @@ namespace Ryujinx.Common.Collections } /// - /// Adds all the nodes in the dictionary into . + /// Adds all the nodes in the dictionary into a list. /// /// A list of all KeyValuePairs sorted by Key Order public List> AsList() @@ -284,7 +283,7 @@ namespace Ryujinx.Common.Collections } Node newNode = new(key, value, parent); - if (newNode.Parent == null) + if (parent == null) { Root = newNode; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index a2448d76f..293f7fd2c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -15,7 +15,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Buffer, used to store vertex and index data, uniform and storage buffers, and others. /// - class Buffer : IRange, ISyncActionHandler, IDisposable + class Buffer : INonOverlappingRange, ISyncActionHandler, IDisposable { private const ulong GranularBufferThreshold = 4096; @@ -35,7 +35,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Size of the buffer in bytes. /// - public ulong Size { get; } + public ulong Size { get; private set; } /// /// End address of the buffer in guest memory. @@ -60,13 +60,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// This is null until at least one modification occurs. /// - private BufferModifiedRangeList _modifiedRanges = null; + private BufferModifiedRangeList _modifiedRanges; /// /// A structure that is used to flush buffer data back to a host mapped buffer for cached readback. /// Only used if the buffer data is explicitly owned by device local memory. /// - private BufferPreFlush _preFlush = null; + private BufferPreFlush _preFlush; /// /// Usage tracking state that determines what type of backing the buffer should use. @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - IEnumerable baseBuffers = null) + List baseBuffers) { _context = context; _physicalMemory = physicalMemory; @@ -126,21 +126,22 @@ namespace Ryujinx.Graphics.Gpu.Memory _useGranular = size > GranularBufferThreshold; - IEnumerable baseHandles = null; + List baseHandles = null; - if (baseBuffers != null) + if (baseBuffers.Count != 0) { - baseHandles = baseBuffers.SelectMany(buffer => + baseHandles = new List(); + foreach (Buffer buffer in baseBuffers) { if (buffer._useGranular) { - return buffer._memoryTrackingGranular.GetHandles(); + baseHandles.AddRange((buffer._memoryTrackingGranular.GetHandles())); } else { - return Enumerable.Repeat(buffer._memoryTracking, 1); + baseHandles.Add(buffer._memoryTracking); } - }); + } } if (_useGranular) @@ -171,9 +172,9 @@ namespace Ryujinx.Graphics.Gpu.Memory _memoryTracking.RegisterPreciseAction(PreciseAction); } - _externalFlushDelegate = new RegionSignal(ExternalFlush); - _loadDelegate = new Action(LoadRegion); - _modifiedDelegate = new Action(RegionModified); + _externalFlushDelegate = ExternalFlush; + _loadDelegate = LoadRegion; + _modifiedDelegate = RegionModified; _virtualDependenciesLock = new ReaderWriterLockSlim(); } @@ -247,6 +248,11 @@ namespace Ryujinx.Graphics.Gpu.Memory return Address < address + size && address < EndAddress; } + public INonOverlappingRange Split(ulong splitAddress) + { + throw new NotImplementedException(); + } + /// /// Checks if a given range is fully contained in the buffer. /// @@ -435,7 +441,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The buffer to inherit from public void InheritModifiedRanges(Buffer from) { - if (from._modifiedRanges != null && from._modifiedRanges.HasRanges) + if (from._modifiedRanges is { HasRanges: true }) { if (from._syncActionRegistered && !_syncActionRegistered) { @@ -443,7 +449,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _syncActionRegistered = true; } - void registerRangeAction(ulong address, ulong size) + void RegisterRangeAction(ulong address, ulong size) { if (_useGranular) { @@ -457,7 +463,7 @@ namespace Ryujinx.Graphics.Gpu.Memory EnsureRangeList(); - _modifiedRanges.InheritRanges(from._modifiedRanges, registerRangeAction); + _modifiedRanges.InheritRanges(from._modifiedRanges, RegisterRangeAction); } if (from._dirtyStart != ulong.MaxValue) @@ -499,14 +505,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { // Cut off the start. - if (end < _dirtyEnd) - { - _dirtyStart = end; - } - else - { - _dirtyStart = ulong.MaxValue; - } + _dirtyStart = end < _dirtyEnd ? end : ulong.MaxValue; } else if (end >= _dirtyEnd) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index a9b1f50e2..df130bb1d 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -56,7 +56,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable baseBuffers = null) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, List baseBuffers) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -72,7 +72,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.Count == 0) { _desiredType = BufferBackingType.DeviceMemory; } @@ -100,7 +100,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers != null) + if (baseBuffers.Count != 0) { foreach (Buffer buffer in baseBuffers) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index d02efcb29..8d0bf9833 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -2,7 +2,6 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -39,11 +38,9 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Only modified from the GPU thread. Must lock for add/remove. /// Must lock for any access from other threads. /// - private readonly RangeList _buffers; + private readonly NonOverlappingRangeList _buffers; private readonly MultiRangeList _multiRangeBuffers; - private Buffer[] _bufferOverlaps; - private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; private bool _pruneCaches; @@ -64,8 +61,6 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers = []; _multiRangeBuffers = []; - _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; - _dirtyCache = new Dictionary(); // There are a lot more entries on the modified cache, so it is separate from the one for ForceDirty. @@ -79,24 +74,23 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Event arguments public void MemoryUnmappedHandler(object sender, UnmapEventArgs e) { - Buffer[] overlaps = new Buffer[10]; - int overlapCount; - MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size); for (int index = 0; index < range.Count; index++) { MemoryRange subRange = range.GetSubRange(index); + + _buffers.Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = _buffers.FindOverlaps(subRange.Address, subRange.Size); - lock (_buffers) + RangeItem current = first; + while (last != null && current != last.Next) { - overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps); + current.Value.Unmapped(subRange.Address, subRange.Size); + current = current.Next; } - for (int i = 0; i < overlapCount; i++) - { - overlaps[i].Unmapped(subRange.Address, subRange.Size); - } + _buffers.Lock.ExitReadLock(); } } @@ -137,7 +131,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of the buffer, after address translation public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size, BufferStage stage) { - if (gpuVa == 0) + if (gpuVa == 0 || size == 0) { return new MultiRange(MemoryManager.PteUnmapped, size); } @@ -336,7 +330,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; ulong alignedSize = alignedEndAddress - alignedAddress; - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + Buffer buffer = _buffers.FindOverlap(alignedAddress, alignedSize).Value; BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); @@ -403,7 +397,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (subRange.Address != MemoryManager.PteUnmapped) { - Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer buffer = _buffers.FindOverlap(subRange.Address, subRange.Size).Value; virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); physicalBuffers.Add(buffer); @@ -495,10 +489,10 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + _buffers.Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); - if (overlapsCount != 0) + if (first is not null) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -507,9 +501,8 @@ namespace Ryujinx.Graphics.Gpu.Memory // old buffer(s) to the new buffer. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; - if (overlap0.Address > address || overlap0.EndAddress < endAddress) + if (first.Address > address || first.EndAddress < endAddress) { bool anySparseCompatible = false; @@ -522,53 +515,52 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (overlapsCount == 1 && - address >= overlap0.Address && - endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2) + if (first == last && + address >= first.Address && + endAddress - first.EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = overlap0.Size; + ulong existingSize = first.Value.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + (first, last) = _buffers.FindOverlaps(address, size); } + + address = Math.Min(address, first.Address); + endAddress = Math.Max(endAddress, last.EndAddress); - for (int index = 0; index < overlapsCount; index++) + List overlaps = []; + + RangeItem current = first; + while (current != last.Next) { - Buffer buffer = overlaps[index]; - - anySparseCompatible |= buffer.SparseCompatible; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - - lock (_buffers) - { - _buffers.Remove(buffer); - } + anySparseCompatible |= current.Value.SparseCompatible; + overlaps.Add(current.Value); + _buffers.Remove(current.Value); + + current = current.Next; } - + ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, []); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } - - ShrinkOverlapsBufferIfNeeded(); + + _buffers.Lock.ExitWriteLock(); } /// @@ -582,72 +574,68 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Alignment of the start address of the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); bool sparseAligned = alignment >= SparseBufferAlignmentSize; + + _buffers.Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = _buffers.FindOverlaps(address, size); - if (overlapsCount != 0) + if (first is not null) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; - if (overlap0.Address > address || - overlap0.EndAddress < endAddress || - (overlap0.Address & (alignment - 1)) != 0 || - (!overlap0.SparseCompatible && sparseAligned)) + if (first.Address > address || + first.EndAddress < endAddress || + (first.Address & (alignment - 1)) != 0 || + (!first.Value.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it // overlaps more buffers, so try again after each extension // and ensure we cover all overlaps. - int oldOverlapsCount; + RangeItem oldFirst; + endAddress = Math.Max(endAddress, last.EndAddress); do { - for (int index = 0; index < overlapsCount; index++) - { - Buffer buffer = overlaps[index]; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - } + address = Math.Min(address, first.Address); address &= ~(alignment - 1); - oldOverlapsCount = overlapsCount; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); - } - while (oldOverlapsCount != overlapsCount); - - lock (_buffers) - { - for (int index = 0; index < overlapsCount; index++) - { - _buffers.Remove(overlaps[index]); - } + oldFirst = first; + (first, last) = _buffers.FindOverlaps(address, endAddress - address); } + while (oldFirst != first); ulong newSize = endAddress - address; - - CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount); + + List overlaps = []; + + RangeItem current = first; + while (current != last.Next) + { + overlaps.Add(current.Value); + _buffers.Remove(current.Value); + + current = current.Next; + } + + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, []); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } - - ShrinkOverlapsBufferIfNeeded(); + _buffers.Lock.ExitWriteLock(); } /// @@ -660,17 +648,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - /// Total of overlaps - private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount) + private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, List overlaps) { - Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount)); + Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - lock (_buffers) - { - _buffers.Add(newBuffer); - } - - for (int index = 0; index < overlapsCount; index++) + for (int index = 0; index < overlaps.Count; index++) { Buffer buffer = overlaps[index]; @@ -688,6 +670,8 @@ namespace Ryujinx.Graphics.Gpu.Memory NotifyBuffersModified?.Invoke(); RecreateMultiRangeBuffers(address, size); + + return newBuffer; } /// @@ -718,17 +702,6 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - /// - /// Resizes the temporary buffer used for range list intersection results, if it has grown too much. - /// - private void ShrinkOverlapsBufferIfNeeded() - { - if (_bufferOverlaps.Length > OverlapsBufferMaxCapacity) - { - Array.Resize(ref _bufferOverlaps, OverlapsBufferMaxCapacity); - } - } - /// /// Copy a buffer data from a given address to another. /// @@ -909,7 +882,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { MemoryRange subRange = range.GetSubRange(i); - Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size).Value; subBuffer.SynchronizeMemory(subRange.Address, subRange.Size); @@ -957,7 +930,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindFirstOverlap(address, size); + buffer = _buffers.FindOverlapFast(address, size).Value; buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); @@ -969,7 +942,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - buffer = _buffers.FindFirstOverlap(address, 1); + buffer = _buffers.FindOverlapFast(address, 1).Value; } return buffer; @@ -1007,7 +980,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (size != 0) { - Buffer buffer = _buffers.FindFirstOverlap(address, size); + Buffer buffer = _buffers.FindOverlapFast(address, size).Value; if (copyBackVirtual) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index cb99b455b..73647bef5 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -258,7 +258,7 @@ namespace Ryujinx.Graphics.Gpu.Memory RecordStorageAlignment(_cpStorageBuffers, index, gpuVa); - gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); + gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size, BufferStageUtils.ComputeStorage(flags)); @@ -282,7 +282,7 @@ namespace Ryujinx.Graphics.Gpu.Memory RecordStorageAlignment(buffers, index, gpuVa); - gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); + gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size, BufferStageUtils.GraphicsStorage(stage, flags)); @@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, bufferStage); @@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute); @@ -817,7 +817,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Bind respective buffer bindings on the host API. /// /// Host buffers to bind, with their offsets and sizes - /// First binding point /// Number of bindings /// Indicates if the buffers are storage or uniform buffers [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -866,7 +865,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Buffer texture /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture - /// Format of the buffer texture /// Whether the binding is for an image or a sampler public void SetBufferTextureStorage( ShaderStage stage, @@ -889,7 +887,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture /// Index of the binding on the array - /// Format of the buffer texture public void SetBufferTextureStorage( ShaderStage stage, ITextureArray array, @@ -912,7 +909,6 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical ranges of memory where the buffer texture data is located /// Binding info for the buffer texture /// Index of the binding on the array - /// Format of the buffer texture public void SetBufferTextureStorage( ShaderStage stage, IImageArray array, diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index bccbdfd31..8c1132d9b 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -1,25 +1,24 @@ -using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System; +using System.Collections.Generic; using System.Linq; -using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { /// /// A range within a buffer that has been modified by the GPU. /// - class BufferModifiedRange : IRange + class BufferModifiedRange : INonOverlappingRange { /// /// Start address of the range in guest memory. /// - public ulong Address { get; } + public ulong Address { get; internal set; } /// /// Size of the range in bytes. /// - public ulong Size { get; } + public ulong Size { get; internal set; } /// /// End address of the range in guest memory. @@ -61,14 +60,19 @@ namespace Ryujinx.Graphics.Gpu.Memory { return Address < address + size && address < EndAddress; } + + public INonOverlappingRange Split(ulong splitAddress) + { + throw new NotImplementedException(); + } } /// /// A structure used to track GPU modified ranges within a buffer. /// - class BufferModifiedRangeList : RangeList + class BufferModifiedRangeList : NonOverlappingRangeList { - private const int BackingInitialSize = 8; + private new const int BackingInitialSize = 8; private readonly GpuContext _context; private readonly Buffer _parent; @@ -77,8 +81,6 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; - private readonly Lock _lock = new(); - /// /// Whether the modified range list has any entries or not. /// @@ -86,10 +88,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { get { - lock (_lock) - { - return Count > 0; - } + Lock.EnterReadLock(); + bool result = Count > 0; + Lock.ExitReadLock(); + return result; } } @@ -114,33 +116,41 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Action to perform for each remaining sub-range of the input range public void ExcludeModifiedRegions(ulong address, ulong size, Action action) { - lock (_lock) + // Slices a given region using the modified regions in the list. Calls the action for the new slices. + bool lockOwner = Lock.IsReadLockHeld; + if (!lockOwner) { - // Slices a given region using the modified regions in the list. Calls the action for the new slices. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Lock.EnterReadLock(); + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); - for (int i = 0; i < count; i++) + RangeItem current = first; + while (last != null && current != last.Next) + { + BufferModifiedRange overlap = current.Value; + + if (overlap.Address > address) { - BufferModifiedRange overlap = overlaps[i]; - - if (overlap.Address > address) - { - // The start of the remaining region is uncovered by this overlap. Call the action for it. - action(address, overlap.Address - address); - } - - // Remaining region is after this overlap. - size -= overlap.EndAddress - address; - address = overlap.EndAddress; + // The start of the remaining region is uncovered by this overlap. Call the action for it. + action(address, overlap.Address - address); } - if ((long)size > 0) - { - // If there is any region left after removing the overlaps, signal it. - action(address, size); - } + // Remaining region is after this overlap. + size -= overlap.EndAddress - address; + address = overlap.EndAddress; + current = current.Next; + } + + if (!lockOwner) + { + Lock.ExitReadLock(); + } + + if ((long)size > 0) + { + // If there is any region left after removing the overlaps, signal it. + action(address, size); } } @@ -152,51 +162,101 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // Must lock, as this can affect flushes from the background thread. - lock (_lock) + // We may overlap with some existing modified regions. They must be cut into by the new entry. + Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); + + ulong endAddress = address + size; + ulong syncNumber = _context.SyncNumber; + + if (first is null) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + return; + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; - ulong endAddress = address + size; - ulong syncNumber = _context.SyncNumber; - - for (int i = 0; i < count; i++) + if (first == last) + { + if (first.Address == address && first.EndAddress == endAddress) { - // The overlaps must be removed or split. + first.Value.SyncNumber = syncNumber; + first.Value.Parent = this; + Lock.ExitWriteLock(); + return; + } - BufferModifiedRange overlap = overlaps[i]; + if (first.Address < address) + { + first.Value.Size = address - first.Address; - if (overlap.Address == address && overlap.Size == size) + extendsPre = true; + + if (first.EndAddress > endAddress) { - // Region already exists. Just update the existing sync number. - overlap.SyncNumber = syncNumber; - overlap.Parent = this; - - return; + buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, + first.Value.SyncNumber, first.Value.Parent); + extendsPost = true; } - - Remove(overlap); - - if (overlap.Address < address && overlap.EndAddress > address) + } + else + { + if (first.EndAddress > endAddress) { - // A split item must be created behind this overlap. - - Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); + first.Value.Size = first.EndAddress - endAddress; + first.Value.Address = endAddress; } - - if (overlap.Address < endAddress && overlap.EndAddress > endAddress) + else { - // A split item must be created after this overlap. - - Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); + Remove(first.Value); } } + if (extendsPre && extendsPost) + { + Add(buffPost); + } + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + + return; } + + BufferModifiedRange buffPre = null; + + if (first.Address < address) + { + buffPre = new BufferModifiedRange(first.Address, address - first.Address, + first.Value.SyncNumber, first.Value.Parent); + extendsPre = true; + } + + if (last.EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, + last.Value.SyncNumber, last.Value.Parent); + extendsPost = true; + } + + RemoveRange(first, last); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); } /// @@ -208,25 +268,23 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { - int count = 0; + Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + RangeItem current = first; + while (last != null && current != last.Next) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); - } - - for (int i = 0; i < count; i++) - { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = current.Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } + + current = current.Next; } + + Lock.ExitReadLock(); } /// @@ -237,19 +295,23 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRanges(ulong address, ulong size, Action rangeAction) { - int count = 0; - - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + List> overlaps = []; + + // We use the non-span method here because keeping the lock will cause a deadlock. + Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); + + RangeItem current = first; + while (last != null && current != last.Next) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); + overlaps.Add(current); + current = current.Next; } + Lock.ExitReadLock(); - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Count; i++) { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } @@ -262,11 +324,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if a range exists in the specified region, false otherwise public bool HasRange(ulong address, ulong size) { - // Range list must be consistent for this operation. - lock (_lock) - { - return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray.Get()) > 0; - } + Lock.EnterReadLock(); + (RangeItem first, RangeItem _) = FindOverlaps(address, size); + bool result = first is not null; + Lock.ExitReadLock(); + return result; } /// @@ -298,38 +360,37 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The start address of the flush range /// The end address of the flush range private void RemoveRangesAndFlush( - BufferModifiedRange[] overlaps, + RangeItem[] overlaps, int rangeCount, long highestDiff, ulong currentSync, ulong address, ulong endAddress) { - lock (_lock) + if (_migrationTarget == null) { - if (_migrationTarget == null) + ulong waitSync = currentSync + (ulong)highestDiff; + + for (int i = 0; i < rangeCount; i++) { - ulong waitSync = currentSync + (ulong)highestDiff; + BufferModifiedRange overlap = overlaps[i].Value; - for (int i = 0; i < rangeCount; i++) + long diff = (long)(overlap.SyncNumber - currentSync); + + if (diff <= highestDiff) { - BufferModifiedRange overlap = overlaps[i]; + ulong clampAddress = Math.Max(address, overlap.Address); + ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - long diff = (long)(overlap.SyncNumber - currentSync); + Lock.EnterWriteLock(); + ClearPart(overlap, clampAddress, clampEnd); + Lock.ExitWriteLock(); - if (diff <= highestDiff) - { - ulong clampAddress = Math.Max(address, overlap.Address); - ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - - ClearPart(overlap, clampAddress, clampEnd); - - RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); - } + RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); } - - return; } + + return; } // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. @@ -355,28 +416,37 @@ namespace Ryujinx.Graphics.Gpu.Memory int rangeCount = 0; - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + List> overlaps = []; // Range list must be consistent for this operation - lock (_lock) + Lock.EnterReadLock(); + if (_migrationTarget != null) { - if (_migrationTarget != null) + rangeCount = -1; + } + else + { + // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. + (RangeItem first, RangeItem last) = FindOverlaps(address, size); + + RangeItem current = first; + while (last != null && current != last.Next) { - rangeCount = -1; - } - else - { - rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); + rangeCount++; + overlaps.Add(current); + current = current.Next; } } + Lock.ExitReadLock(); if (rangeCount == -1) { - _migrationTarget.WaitForAndFlushRanges(address, size); + _migrationTarget!.WaitForAndFlushRanges(address, size); return; } - else if (rangeCount == 0) + + if (rangeCount == 0) { return; } @@ -388,7 +458,7 @@ namespace Ryujinx.Graphics.Gpu.Memory for (int i = 0; i < rangeCount; i++) { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; long diff = (long)(overlap.SyncNumber - currentSync); @@ -406,7 +476,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // Wait for the syncpoint. _context.Renderer.WaitSync(currentSync + (ulong)highestDiff); - RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress); + RemoveRangesAndFlush(overlaps.ToArray(), rangeCount, highestDiff, currentSync, address, endAddress); } /// @@ -419,42 +489,39 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { - BufferModifiedRange[] inheritRanges; + ranges.Lock.EnterReadLock(); + BufferModifiedRange[] inheritRanges = ranges.ToArray(); + ranges.Lock.ExitReadLock(); - lock (ranges._lock) + // Copy over the migration from the previous range list + + BufferMigration oldMigration = ranges._source; + + BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); + ranges._parent.IncrementReferenceCount(); + + if (_source == null) { - inheritRanges = ranges.ToArray(); + // Create a new migration. + _source = new BufferMigration([span], this, _context.SyncNumber); - lock (_lock) - { - // Copy over the migration from the previous range list - - BufferMigration oldMigration = ranges._source; - - BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); - ranges._parent.IncrementReferenceCount(); - - if (_source == null) - { - // Create a new migration. - _source = new BufferMigration([span], this, _context.SyncNumber); - - _context.RegisterBufferMigration(_source); - } - else - { - // Extend the migration - _source.AddSpanToEnd(span); - } - - ranges._migrationTarget = this; - - foreach (BufferModifiedRange range in inheritRanges) - { - Add(range); - } - } + _context.RegisterBufferMigration(_source); } + else + { + // Extend the migration + _source.AddSpanToEnd(span); + } + + ranges._migrationTarget = this; + + Lock.EnterWriteLock(); + foreach (BufferModifiedRange range in inheritRanges) + { + Add(range); + } + + Lock.ExitWriteLock(); ulong currentSync = _context.SyncNumber; foreach (BufferModifiedRange range in inheritRanges) @@ -473,18 +540,18 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void SelfMigration() { - lock (_lock) - { - BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); - BufferMigration migration = new([span], this, _context.SyncNumber); + BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), + _parent.GetSnapshotFlushAction(), _source); + BufferMigration migration = new([span], this, _context.SyncNumber); - // Migration target is used to redirect flush actions to the latest range list, - // so we don't need to set it here. (this range list is still the latest) + // Migration target is used to redirect flush actions to the latest range list, + // so we don't need to set it here. (this range list is still the latest) - _context.RegisterBufferMigration(migration); + _context.RegisterBufferMigration(migration); - _source = migration; - } + Lock.EnterWriteLock(); + _source = migration; + Lock.ExitWriteLock(); } /// @@ -493,13 +560,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The migration to remove public void RemoveMigration(BufferMigration migration) { - lock (_lock) + Lock.EnterWriteLock(); + if (_source == migration) { - if (_source == migration) - { - _source = null; - } + _source = null; } + + Lock.ExitWriteLock(); } private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) @@ -526,33 +593,85 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size to clear public void Clear(ulong address, ulong size) { - lock (_lock) + ulong endAddress = address + size; + Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); + + if (first is null) { - // This function can be called from any thread, so it cannot use the arrays for background or foreground. - BufferModifiedRange[] toClear = new BufferModifiedRange[1]; + Lock.ExitWriteLock(); + return; + } - int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear); + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; - ulong endAddress = address + size; - - for (int i = 0; i < rangeCount; i++) + if (first == last) + { + if (first.Address < address) { - BufferModifiedRange overlap = toClear[i]; + first.Value.Size = address - first.Address; + extendsPre = true; - ClearPart(overlap, address, endAddress); + if (first.EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, + first.Value.SyncNumber, first.Value.Parent); + extendsPost = true; + } + } + else + { + if (first.EndAddress > endAddress) + { + first.Value.Size = first.EndAddress - endAddress; + first.Value.Address = endAddress; + } + else + { + Remove(first.Value); + } } - } - } - /// - /// Clear all modified ranges. - /// - public void Clear() - { - lock (_lock) - { - Count = 0; + if (extendsPre && extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); + return; } + + BufferModifiedRange buffPre = null; + + if (first.Address < address) + { + buffPre = new BufferModifiedRange(first.Address, address - first.Address, + first.Value.SyncNumber, first.Value.Parent); + extendsPre = true; + } + + if (last.EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, + last.Value.SyncNumber, last.Value.Parent); + extendsPost = true; + } + + RemoveRange(first, last); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs index 8a9f37658..c299731f8 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferStage.cs @@ -1,4 +1,5 @@ using Ryujinx.Graphics.Shader; +using System; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -7,6 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Pipeline stages that can modify buffer data, as well as flags indicating storage usage. /// Must match ShaderStage for the shader stages, though anything after that can be in any order. /// + [Flags] internal enum BufferStage : byte { Compute, diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 6efb7f334..95e43e341 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -690,11 +690,8 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_pageTable[l0] == null) { _pageTable[l0] = new ulong[PtLvl1Size]; - - for (ulong index = 0; index < PtLvl1Size; index++) - { - _pageTable[l0][index] = PteUnmapped; - } + + Array.Fill(_pageTable[l0], PteUnmapped); } _pageTable[l0][l1] = pte; diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index ac25b3e5d..06253cb30 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -15,7 +15,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Represents a GPU virtual memory range. /// - private readonly struct VirtualRange : IRange + private class VirtualRange : INonOverlappingRange { /// /// GPU virtual address where the range starts. @@ -25,7 +25,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Size of the range in bytes. /// - public ulong Size { get; } + public ulong Size { get; private set; } /// /// GPU virtual address where the range ends. @@ -35,7 +35,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Physical regions where the GPU virtual region is mapped. /// - public MultiRange Range { get; } + public MultiRange Range { get; private set; } /// /// Creates a new virtual memory range. @@ -60,10 +60,14 @@ namespace Ryujinx.Graphics.Gpu.Memory { return Address < address + size && address < EndAddress; } + + public INonOverlappingRange Split(ulong splitAddress) + { + throw new NotImplementedException(); + } } - private readonly RangeList _virtualRanges; - private VirtualRange[] _virtualRangeOverlaps; + private readonly NonOverlappingRangeList _virtualRanges; private readonly ConcurrentQueue _deferredUnmaps; private int _hasDeferredUnmaps; @@ -75,7 +79,6 @@ namespace Ryujinx.Graphics.Gpu.Memory { _memoryManager = memoryManager; _virtualRanges = []; - _virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity]; _deferredUnmaps = new ConcurrentQueue(); } @@ -106,19 +109,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if the range already existed, false if a new one was created and added public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { - VirtualRange[] overlaps = _virtualRangeOverlaps; - int overlapsCount; - if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0) { while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange)) { - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps); - - for (int index = 0; index < overlapsCount; index++) - { - _virtualRanges.Remove(overlaps[index]); - } + _virtualRanges.RemoveRange(unmappedRange.Address, unmappedRange.Size); } } @@ -126,27 +121,22 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps); - - if (overlapsCount != 0) + _virtualRanges.Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = _virtualRanges.FindOverlaps(gpuVa, size); + + if (first is not null) { // The virtual range already exists. We just need to check if our range fits inside // the existing one, and if not, we must extend the existing one. ulong endAddress = gpuVa + size; - VirtualRange overlap0 = overlaps[0]; - if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress) + if (first.Address > gpuVa || first.EndAddress < endAddress) { - for (int index = 0; index < overlapsCount; index++) - { - VirtualRange virtualRange = overlaps[index]; - - gpuVa = Math.Min(gpuVa, virtualRange.Address); - endAddress = Math.Max(endAddress, virtualRange.EndAddress); - - _virtualRanges.Remove(virtualRange); - } + gpuVa = Math.Min(gpuVa, first.Address); + endAddress = Math.Max(endAddress, last.EndAddress); + + _virtualRanges.RemoveRange(first, last); ulong newSize = endAddress - gpuVa; MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize); @@ -157,8 +147,8 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range); - range = overlap0.Range.Slice(gpuVa - overlap0.Address, size); + found = first.Value.Range.Count == 1 || IsSparseAligned(first.Value.Range); + range = first.Value.Range.Slice(gpuVa - first.Address, size); } } else @@ -170,8 +160,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _virtualRanges.Add(virtualRange); } - - ShrinkOverlapsBufferIfNeeded(); + _virtualRanges.Lock.ExitWriteLock(); // If the range is not properly aligned for sparse mapping, // let's just force it to a single range. @@ -221,16 +210,5 @@ namespace Ryujinx.Graphics.Gpu.Memory return true; } - - /// - /// Resizes the temporary buffer used for range list intersection results, if it has grown too much. - /// - private void ShrinkOverlapsBufferIfNeeded() - { - if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity) - { - Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity); - } - } } } diff --git a/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs b/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs index f13a3554a..6a57aa3d1 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs @@ -89,13 +89,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); currBlock = newBlock; } @@ -143,13 +149,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); currBlock = newBlock; } @@ -199,13 +211,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress > currBaseAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); } if (endAddr < currEndAddr) { KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); + if (currBlock.Left == null) + _blockTree.Add(newBlock, currBlock); + else + _blockTree.Add(newBlock, currBlock.Predecessor); currBlock = newBlock; } diff --git a/src/Ryujinx.Memory/Range/INonOverlappingRange.cs b/src/Ryujinx.Memory/Range/INonOverlappingRange.cs index 23194e0f2..c6a0197d4 100644 --- a/src/Ryujinx.Memory/Range/INonOverlappingRange.cs +++ b/src/Ryujinx.Memory/Range/INonOverlappingRange.cs @@ -3,7 +3,7 @@ namespace Ryujinx.Memory.Range /// /// Range of memory that can be split in two. /// - interface INonOverlappingRange : IRange + public interface INonOverlappingRange : IRange { /// /// Split this region into two, around the specified address. diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 894078aee..7803b03d1 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -1,5 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Memory.Range { @@ -7,8 +10,284 @@ namespace Ryujinx.Memory.Range /// A range list that assumes ranges are non-overlapping, with list items that can be split in two to avoid overlaps. /// /// Type of the range. - class NonOverlappingRangeList : RangeList where T : INonOverlappingRange + public class NonOverlappingRangeList : RangeListBase where T : INonOverlappingRange { + private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); + private readonly Dictionary> _fastQuickAccess = new(AddressEqualityComparer.Comparer); + + public readonly ReaderWriterLockSlim Lock = new(); + + /// + /// Creates a new non-overlapping range list. + /// + public NonOverlappingRangeList() { } + + /// + /// Creates a new non-overlapping range list. + /// + /// The initial size of the backing array + public NonOverlappingRangeList(int backingInitialSize) : base(backingInitialSize) { } + + /// + /// Adds a new item to the list. + /// + /// The item to be added + public override void Add(T item) + { + int index = BinarySearch(item.Address); + + if (index < 0) + { + index = ~index; + } + + RangeItem rangeItem = new(item); + + Insert(index, rangeItem); + + _quickAccess.Add(item.Address, rangeItem); + } + + /// + /// Updates an item's end address on the list. Address must be the same. + /// + /// The item to be updated + /// True if the item was located and updated, false otherwise + protected override bool Update(T item) + { + int index = BinarySearch(item.Address); + + if (index >= 0 && Items[index].Value.Equals(item)) + { + RangeItem rangeItem = new(item) { Previous = Items[index].Previous, Next = Items[index].Next }; + + if (index > 0) + { + Items[index - 1].Next = rangeItem; + } + + if (index < Count - 1) + { + Items[index + 1].Previous = rangeItem; + } + + foreach (ulong addr in Items[index].QuickAccessAddresses) + { + _quickAccess.Remove(addr); + _fastQuickAccess.Remove(addr); + } + + Items[index] = rangeItem; + + _quickAccess[item.Address] = rangeItem; + + return true; + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void Insert(int index, RangeItem item) + { + Debug.Assert(item.Address != item.EndAddress); + + if (Count + 1 > Items.Length) + { + Array.Resize(ref Items, Items.Length + BackingGrowthSize); + } + + if (index >= Count) + { + if (index == Count) + { + if (index != 0) + { + item.Previous = Items[index - 1]; + Items[index - 1].Next = item; + } + Items[index] = item; + Count++; + } + } + else + { + Array.Copy(Items, index, Items, index + 1, Count - index); + + Items[index] = item; + if (index != 0) + { + item.Previous = Items[index - 1]; + Items[index - 1].Next = item; + } + + item.Next = Items[index + 1]; + Items[index + 1].Previous = item; + + Count++; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void RemoveAt(int index) + { + if (index < Count - 1) + { + Items[index + 1].Previous = index > 0 ? Items[index - 1] : null; + } + + if (index > 0) + { + Items[index - 1].Next = index < Count - 1 ? Items[index + 1] : null; + } + + if (index < --Count) + { + Array.Copy(Items, index + 1, Items, index, Count - index); + } + } + + /// + /// Removes an item from the list. + /// + /// The item to be removed + /// True if the item was removed, or false if it was not found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override bool Remove(T item) + { + int index = BinarySearch(item.Address); + + if (index >= 0 && Items[index].Value.Equals(item)) + { + _quickAccess.Remove(item.Address); + + foreach (ulong addr in Items[index].QuickAccessAddresses) + { + _quickAccess.Remove(addr); + _fastQuickAccess.Remove(addr); + } + + RemoveAt(index); + + return true; + } + + return false; + } + + /// + /// Removes a range of items from the item list + /// + /// The first item in the range of items to be removed + /// The last item in the range of items to be removed + public override void RemoveRange(RangeItem startItem, RangeItem endItem) + { + if (startItem is null) + { + return; + } + + if (startItem == endItem) + { + Remove(startItem.Value); + return; + } + + int startIndex = BinarySearch(startItem.Address); + int endIndex = BinarySearch(endItem.Address); + + if (endIndex < Count - 1) + { + Items[endIndex + 1].Previous = startIndex > 0 ? Items[startIndex - 1] : null; + } + + if (startIndex > 0) + { + Items[startIndex - 1].Next = endIndex < Count - 1 ? Items[endIndex + 1] : null; + } + + + if (endIndex < Count - 1) + { + Array.Copy(Items, endIndex + 1, Items, startIndex, Count - endIndex - 1); + } + + Count -= endIndex - startIndex + 1; + + while (startItem != endItem.Next) + { + _quickAccess.Remove(startItem.Address); + foreach (ulong addr in startItem.QuickAccessAddresses) + { + _quickAccess.Remove(addr); + _fastQuickAccess.Remove(addr); + } + startItem = startItem.Next; + } + } + + /// + /// Removes a range of items from the item list + /// + /// Start address of the range + /// Size of the range + public void RemoveRange(ulong address, ulong size) + { + int startIndex = BinarySearchLeftEdge(address, address + size); + + if (startIndex < 0) + { + return; + } + + RangeItem startItem = Items[startIndex]; + + int endIndex = startIndex; + + while (startItem is not null && startItem.Address < address + size) + { + _quickAccess.Remove(startItem.Address); + foreach (ulong addr in startItem.QuickAccessAddresses) + { + _quickAccess.Remove(addr); + _fastQuickAccess.Remove(addr); + } + startItem = startItem.Next; + endIndex++; + } + + if (endIndex < Count - 1) + { + Items[endIndex + 1].Previous = startIndex > 0 ? Items[startIndex - 1] : null; + } + + if (startIndex > 0) + { + Items[startIndex - 1].Next = endIndex < Count - 1 ? Items[endIndex + 1] : null; + } + + + if (endIndex < Count - 1) + { + Array.Copy(Items, endIndex + 1, Items, startIndex, Count - endIndex - 1); + } + + Count -= endIndex - startIndex + 1; + } + + /// + /// Clear all ranges. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear() + { + Lock.EnterWriteLock(); + Count = 0; + _quickAccess.Clear(); + _fastQuickAccess.Clear(); + Lock.ExitWriteLock(); + } + /// /// Finds a list of regions that cover the desired (address, size) range. /// If this range starts or ends in the middle of an existing region, it is split and only the relevant part is added. @@ -19,17 +298,18 @@ namespace Ryujinx.Memory.Range /// Start address of the search region /// Size of the search region /// Factory for creating new ranges - public void GetOrAddRegions(List list, ulong address, ulong size, Func factory) + public void GetOrAddRegions(out List list, ulong address, ulong size, Func factory) { // (regarding the specific case this generalized function is used for) // A new region may be split into multiple parts if multiple virtual regions have mapped to it. // For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved... // So we need to return both the split 0-1 and 1-2 ranges. - - T[] results = new T[1]; - int count = FindOverlapsNonOverlapping(address, size, ref results); - - if (count == 0) + + Lock.EnterWriteLock(); + (RangeItem first, RangeItem last) = FindOverlaps(address, size); + list = new List(); + + if (first is null) { // The region is fully unmapped. Create and add it to the range list. T region = factory(address, size); @@ -41,13 +321,15 @@ namespace Ryujinx.Memory.Range ulong lastAddress = address; ulong endAddress = address + size; - for (int i = 0; i < count; i++) + RangeItem current = first; + while (last is not null && current is not null && current.Address < endAddress) { - T region = results[i]; - if (count == 1 && region.Address == address && region.Size == size) + T region = current.Value; + if (first == last && region.Address == address && region.Size == size) { // Exact match, no splitting required. list.Add(region); + Lock.ExitWriteLock(); return; } @@ -75,6 +357,7 @@ namespace Ryujinx.Memory.Range list.Add(region); lastAddress = region.EndAddress; + current = current.Next; } if (lastAddress < endAddress) @@ -85,6 +368,8 @@ namespace Ryujinx.Memory.Range Add(fillRegion); } } + + Lock.ExitWriteLock(); } /// @@ -95,6 +380,7 @@ namespace Ryujinx.Memory.Range /// The region to split /// The address to split with /// The new region (high part) + [MethodImpl(MethodImplOptions.AggressiveInlining)] private T Split(T region, ulong splitAddress) { T newRegion = (T)region.Split(splitAddress); @@ -102,5 +388,113 @@ namespace Ryujinx.Memory.Range Add(newRegion); return newRegion; } + + /// + /// Gets an item on the list overlapping the specified memory range. + /// + /// Start address of the range + /// Size in bytes of the range + /// The leftmost overlapping item, or null if none is found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override RangeItem FindOverlap(ulong address, ulong size) + { + if (_quickAccess.TryGetValue(address, out RangeItem overlap)) + { + return overlap; + } + + int index = BinarySearchLeftEdge(address, address + size); + + if (index < 0) + { + return null; + } + + if (Items[index].Address < address) + { + _quickAccess.TryAdd(address, Items[index]); + Items[index].QuickAccessAddresses.Add(address); + } + + return Items[index]; + } + + /// + /// Gets an item on the list overlapping the specified memory range. + /// + /// Start address of the range + /// Size in bytes of the range + /// The overlapping item, or null if none is found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override RangeItem FindOverlapFast(ulong address, ulong size) + { + if (_quickAccess.TryGetValue(address, out RangeItem overlap) || _fastQuickAccess.TryGetValue(address, out overlap)) + { + return overlap; + } + + int index = BinarySearch(address, address + size); + + if (index < 0) + { + return null; + } + + if (Items[index].Address < address) + { + _quickAccess.TryAdd(address, Items[index]); + } + else + { + _fastQuickAccess.TryAdd(address, Items[index]); + } + + Items[index].QuickAccessAddresses.Add(address); + + return Items[index]; + } + + /// + /// Gets all items on the list overlapping the specified memory range. + /// + /// Start address of the range + /// Size in bytes of the range + /// The first and last overlapping items, or null if none are found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (RangeItem, RangeItem) FindOverlaps(ulong address, ulong size) + { + if (_quickAccess.TryGetValue(address, out RangeItem overlap)) + { + if (overlap.Next is null || overlap.Next.Address >= address + size) + { + return (overlap, overlap); + } + + return (overlap, Items[BinarySearchRightEdge(address, address + size)]); + } + + (int index, int endIndex) = BinarySearchEdges(address, address + size); + + if (index < 0) + { + return (null, null); + } + + if (Items[index].Address < address) + { + _quickAccess.TryAdd(address, Items[index]); + Items[index].QuickAccessAddresses.Add(address); + } + + return (Items[index], Items[endIndex - 1]); + } + + public override IEnumerator GetEnumerator() + { + for (int i = 0; i < Count; i++) + { + yield return Items[i].Value; + } + } } } diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 72cef1de0..600ff748e 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -1,61 +1,91 @@ using System; -using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Memory.Range { + public class RangeItem(TValue value) where TValue : IRange + { + public RangeItem Next; + public RangeItem Previous; + + public readonly ulong Address = value.Address; + public readonly ulong EndAddress = value.Address + value.Size; + + public readonly TValue Value = value; + + public readonly List QuickAccessAddresses = []; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool OverlapsWith(ulong address, ulong endAddress) + { + return Address < endAddress && address < EndAddress; + } + } + + class AddressEqualityComparer : IEqualityComparer + { + public bool Equals(ulong u1, ulong u2) + { + return u1 == u2; + } + + public int GetHashCode(ulong value) => (int)(value >> 5); + + public static readonly AddressEqualityComparer Comparer = new(); + } + + /// + /// Result of an Overlaps Finder function. WARNING: if the result is from the optimized + /// Overlaps Finder, the StartIndex will be -1 even when the result isn't empty + /// + /// + /// startIndex is inclusive. + /// endIndex is exclusive. + /// + public readonly struct OverlapResult where T : IRange + { + public readonly int StartIndex = -1; + public readonly int EndIndex = -1; + public readonly RangeItem QuickResult; + public int Count => EndIndex - StartIndex; + + public OverlapResult(int startIndex, int endIndex, RangeItem quickResult = null) + { + this.StartIndex = startIndex; + this.EndIndex = endIndex; + this.QuickResult = quickResult; + } + } + /// /// Sorted list of ranges that supports binary search. /// /// Type of the range. - public class RangeList : IEnumerable where T : IRange + public class RangeList : RangeListBase where T : IRange { - private readonly struct RangeItem where TValue : IRange - { - public readonly ulong Address; - public readonly ulong EndAddress; - - public readonly TValue Value; - - public RangeItem(TValue value) - { - Value = value; - - Address = value.Address; - EndAddress = value.Address + value.Size; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool OverlapsWith(ulong address, ulong endAddress) - { - return Address < endAddress && address < EndAddress; - } - } - - private const int BackingInitialSize = 1024; - private const int ArrayGrowthSize = 32; - - private RangeItem[] _items; - private readonly int _backingGrowthSize; - - public int Count { get; protected set; } + public readonly ReaderWriterLockSlim Lock = new(); + + private readonly Dictionary> _quickAccess = new(AddressEqualityComparer.Comparer); + /// + /// Creates a new range list. + /// + public RangeList() { } + /// /// Creates a new range list. /// /// The initial size of the backing array - public RangeList(int backingInitialSize = BackingInitialSize) - { - _backingGrowthSize = backingInitialSize; - _items = new RangeItem[backingInitialSize]; - } + public RangeList(int backingInitialSize) : base(backingInitialSize) { } /// /// Adds a new item to the list. /// /// The item to be added - public void Add(T item) + public override void Add(T item) { int index = BinarySearch(item.Address); @@ -72,27 +102,27 @@ namespace Ryujinx.Memory.Range /// /// The item to be updated /// True if the item was located and updated, false otherwise - public bool Update(T item) + protected override bool Update(T item) { int index = BinarySearch(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { - if (_items[index].Value.Equals(item)) + if (Items[index].Value.Equals(item)) { - _items[index] = new RangeItem(item); + foreach (ulong address in Items[index].QuickAccessAddresses) + { + _quickAccess.Remove(address); + } + + Items[index] = new RangeItem(item); return true; } - if (_items[index].Address > item.Address) + if (Items[index].Address > item.Address) { break; } @@ -107,23 +137,42 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] private void Insert(int index, RangeItem item) { - if (Count + 1 > _items.Length) + Debug.Assert(item.Address != item.EndAddress); + + Debug.Assert(item.Address % 32 == 0); + + if (Count + 1 > Items.Length) { - Array.Resize(ref _items, _items.Length + _backingGrowthSize); + Array.Resize(ref Items, Items.Length + BackingGrowthSize); } if (index >= Count) { if (index == Count) { - _items[Count++] = item; + if (index != 0) + { + item.Previous = Items[index - 1]; + Items[index - 1].Next = item; + } + Items[index] = item; + Count++; } } else { - Array.Copy(_items, index, _items, index + 1, Count - index); + Array.Copy(Items, index, Items, index + 1, Count - index); - _items[index] = item; + Items[index] = item; + if (index != 0) + { + item.Previous = Items[index - 1]; + Items[index - 1].Next = item; + } + + item.Next = Items[index + 1]; + Items[index + 1].Previous = item; + Count++; } } @@ -131,9 +180,71 @@ namespace Ryujinx.Memory.Range [MethodImpl(MethodImplOptions.AggressiveInlining)] private void RemoveAt(int index) { + foreach (ulong address in Items[index].QuickAccessAddresses) + { + _quickAccess.Remove(address); + } + + if (index < Count - 1) + { + Items[index + 1].Previous = index > 0 ? Items[index - 1] : null; + } + + if (index > 0) + { + Items[index - 1].Next = index < Count - 1 ? Items[index + 1] : null; + } + if (index < --Count) { - Array.Copy(_items, index + 1, _items, index, Count - index); + Array.Copy(Items, index + 1, Items, index, Count - index); + } + } + + /// + /// Removes a range of items from the item list + /// + /// The first item in the range of items to be removed + /// The last item in the range of items to be removed + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void RemoveRange(RangeItem startItem, RangeItem endItem) + { + if (endItem.Next is not null) + { + endItem.Next.Previous = startItem.Previous; + } + + if (startItem.Previous is not null) + { + startItem.Previous.Next = endItem.Next; + } + + RangeItem current = startItem; + while (current != endItem.Next) + { + foreach (ulong address in current.QuickAccessAddresses) + { + _quickAccess.Remove(address); + } + + current = current.Next; + } + + RangeItem[] array = []; + OverlapResult overlapResult = FindOverlaps(startItem.Address, endItem.EndAddress, ref array); + + if (overlapResult.EndIndex < Count) + { + Array.Copy(Items, overlapResult.EndIndex, Items, overlapResult.StartIndex, Count - overlapResult.EndIndex); + Count -= overlapResult.Count; + } + else if (overlapResult.EndIndex == Count) + { + Count = overlapResult.StartIndex; + } + else + { + Debug.Assert(false); } } @@ -142,27 +253,22 @@ namespace Ryujinx.Memory.Range /// /// The item to be removed /// True if the item was removed, or false if it was not found - public bool Remove(T item) + public override bool Remove(T item) { int index = BinarySearch(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { - if (_items[index].Value.Equals(item)) + if (Items[index].Value.Equals(item)) { RemoveAt(index); return true; } - if (_items[index].Address > item.Address) + if (Items[index].Address > item.Address) { break; } @@ -173,310 +279,130 @@ namespace Ryujinx.Memory.Range return false; } - + /// - /// Updates an item's end address. - /// - /// The item to be updated - public void UpdateEndAddress(T item) - { - int index = BinarySearch(item.Address); - - if (index >= 0) - { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - - while (index < Count) - { - if (_items[index].Value.Equals(item)) - { - _items[index] = new RangeItem(item); - - return; - } - - if (_items[index].Address > item.Address) - { - break; - } - - index++; - } - } - } - - /// - /// Gets the first item on the list overlapping in memory with the specified item. + /// Gets an item on the list overlapping the specified memory range. /// /// - /// Despite the name, this has no ordering guarantees of the returned item. - /// It only ensures that the item returned overlaps the specified item. - /// - /// Item to check for overlaps - /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(T item) - { - return FindFirstOverlap(item.Address, item.Size); - } - - /// - /// Gets the first item on the list overlapping the specified memory range. - /// - /// - /// Despite the name, this has no ordering guarantees of the returned item. + /// This has no ordering guarantees of the returned item. /// It only ensures that the item returned overlaps the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(ulong address, ulong size) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override RangeItem FindOverlap(ulong address, ulong size) { + int index = BinarySearchLeftEdge(address, address + size); + + if (index < 0) + { + return null; + } + + return Items[index]; + } + + /// + /// Gets an item on the list overlapping the specified memory range. + /// + /// + /// This has no ordering guarantees of the returned item. + /// It only ensures that the item returned overlaps the specified memory range. + /// + /// Start address of the range + /// Size in bytes of the range + /// The overlapping item, or the default value for the type if none found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override RangeItem FindOverlapFast(ulong address, ulong size) + { + if (_quickAccess.TryGetValue(address, out RangeItem quickResult)) + { + return quickResult; + } + int index = BinarySearch(address, address + size); if (index < 0) { - return default; + return null; } - return _items[index].Value; - } + if (Items[index].OverlapsWith(address, address + 1)) + { + _quickAccess.Add(address, Items[index]); + Items[index].QuickAccessAddresses.Add(address); + } - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlaps(T item, ref T[] output) - { - return FindOverlaps(item.Address, item.Size, ref output); + return Items[index]; } - + /// /// Gets all items on the list overlapping the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlaps(ulong address, ulong size, ref T[] output) + /// Range information of overlapping items found + private OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem[] output) { - int outputIndex = 0; + int outputCount = 0; ulong endAddress = address + size; + + int startIndex = BinarySearch(address, endAddress); + if (startIndex < 0) + startIndex = ~startIndex; + int endIndex = -1; - for (int i = 0; i < Count; i++) + for (int i = startIndex; i < Count; i++) { - ref RangeItem item = ref _items[i]; + ref RangeItem item = ref Items[i]; if (item.Address >= endAddress) { + endIndex = i; break; } if (item.OverlapsWith(address, endAddress)) { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = item.Value; + outputCount++; } } - return outputIndex; - } - - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// - /// This method only returns correct results if none of the items on the list overlaps with - /// each other. If that is not the case, this method should not be used. - /// This method is faster than the regular method to find all overlaps. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlapsNonOverlapping(T item, ref T[] output) - { - return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); - } - - /// - /// Gets all items on the list overlapping the specified memory range. - /// - /// - /// This method only returns correct results if none of the items on the list overlaps with - /// each other. If that is not the case, this method should not be used. - /// This method is faster than the regular method to find all overlaps. - /// - /// Start address of the range - /// Size in bytes of the range - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output) - { - // This is a bit faster than FindOverlaps, but only works - // when none of the items on the list overlaps with each other. - int outputIndex = 0; - - ulong endAddress = address + size; - - int index = BinarySearch(address, endAddress); - - if (index >= 0) + if (endIndex == -1 && outputCount > 0) { - while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress)) - { - index--; - } - - do - { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = _items[index++].Value; - } - while (index < Count && _items[index].OverlapsWith(address, endAddress)); + endIndex = Count; } - return outputIndex; - } - - /// - /// Gets all items on the list with the specified memory address. - /// - /// Address to find - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of matches found - public int FindOverlaps(ulong address, ref T[] output) - { - int index = BinarySearch(address); - - int outputIndex = 0; - - if (index >= 0) + if (outputCount > 0 && outputCount == endIndex - startIndex) { - while (index > 0 && _items[index - 1].Address == address) - { - index--; - } - - while (index < Count) - { - ref RangeItem overlap = ref _items[index++]; - - if (overlap.Address != address) - { - break; - } - - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = overlap.Value; - } + Array.Resize(ref output, outputCount); + Array.Copy(Items, endIndex - outputCount, output, 0, outputCount); + + return new OverlapResult(startIndex, endIndex); } - - return outputIndex; - } - - /// - /// Performs binary search on the internal list of items. - /// - /// Address to find - /// List index of the item, or complement index of nearest item with lower value on the list - private int BinarySearch(ulong address) - { - int left = 0; - int right = Count - 1; - - while (left <= right) + else if (outputCount > 0) { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref _items[middle]; - - if (item.Address == address) + Array.Resize(ref output, outputCount); + int arrIndex = 0; + for (int i = startIndex; i < endIndex; i++) { - return middle; - } - - if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; + output[arrIndex++] = Items[i]; } + + return new OverlapResult(endIndex - outputCount, endIndex); } - - return ~left; + + return new OverlapResult(); } - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// List index of the item, or complement index of nearest item with lower value on the list - private int BinarySearch(ulong address, ulong endAddress) - { - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref _items[middle]; - - if (item.OverlapsWith(address, endAddress)) - { - return middle; - } - - if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - public IEnumerator GetEnumerator() + public override IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) { - yield return _items[i].Value; - } - } - - IEnumerator IEnumerable.GetEnumerator() - { - for (int i = 0; i < Count; i++) - { - yield return _items[i].Value; + yield return Items[i].Value; } } } diff --git a/src/Ryujinx.Memory/Range/RangeListBase.cs b/src/Ryujinx.Memory/Range/RangeListBase.cs new file mode 100644 index 000000000..7e26442f0 --- /dev/null +++ b/src/Ryujinx.Memory/Range/RangeListBase.cs @@ -0,0 +1,359 @@ +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Ryujinx.Memory.Range +{ + public abstract class RangeListBase : IEnumerable where T : IRange + { + protected const int BackingInitialSize = 1024; + + protected RangeItem[] Items; + protected readonly int BackingGrowthSize; + + public int Count { get; protected set; } + + /// + /// Creates a new range list. + /// + /// The initial size of the backing array + protected RangeListBase(int backingInitialSize = BackingInitialSize) + { + BackingGrowthSize = backingInitialSize; + Items = new RangeItem[backingInitialSize]; + } + + public abstract void Add(T item); + + /// + /// Updates an item's end address on the list. Address must be the same. + /// + /// The item to be updated + /// True if the item was located and updated, false otherwise + protected abstract bool Update(T item); + + public abstract bool Remove(T item); + + public abstract void RemoveRange(RangeItem startItem, RangeItem endItem); + + public abstract RangeItem FindOverlap(ulong address, ulong size); + + public abstract RangeItem FindOverlapFast(ulong address, ulong size); + + /// + /// Performs binary search on the internal list of items. + /// + /// Address to find + /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected int BinarySearch(ulong address) + { + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref Items[middle]; + + if (item.Address == address) + { + return middle; + } + + if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected int BinarySearch(ulong address, ulong endAddress) + { + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref Items[middle]; + + if (item.OverlapsWith(address, endAddress)) + { + return middle; + } + + if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected int BinarySearchLeftEdge(ulong address, ulong endAddress) + { + if (Count == 0) + return ~0; + + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref Items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return middle; + else if (address < item.Address) + return ~(right); + else + return ~(right + 1); + } + + if (match) + { + right = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected int BinarySearchRightEdge(ulong address, ulong endAddress) + { + if (Count == 0) + return ~0; + + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = right - (range >> 1); + + ref RangeItem item = ref Items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return middle; + else if (endAddress > item.EndAddress) + return ~(left + 1); + else + return ~(left); + } + + if (match) + { + left = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected (int, int) BinarySearchEdges(ulong address, ulong endAddress) + { + if (Count == 0) + return (~0, ~0); + + if (Count == 1) + { + ref RangeItem item = ref Items[0]; + + if (item.OverlapsWith(address, endAddress)) + { + return (0, 1); + } + + if (address < item.Address) + { + return (~0, ~0); + } + else + { + return (~1, ~1); + } + } + + int left = 0; + int right = Count - 1; + + int leftEdge = -1; + int rightEdgeMatch = -1; + int rightEdgeNoMatch = -1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref Items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + { + leftEdge = middle; + break; + } + else if (address < item.Address) + { + return (~right, ~right); + } + else + { + return (~(right + 1), ~(right + 1)); + } + } + + if (match) + { + right = middle; + if (rightEdgeMatch == -1) + rightEdgeMatch = middle; + } + else if (address < item.Address) + { + right = middle - 1; + rightEdgeNoMatch = middle; + } + else + { + left = middle + 1; + } + } + + if (left > right) + { + return (~left, ~left); + } + + if (rightEdgeMatch == -1) + { + return (leftEdge, leftEdge + 1); + } + + left = rightEdgeMatch; + right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = right - (range >> 1); + + ref RangeItem item = ref Items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return (leftEdge, middle + 1); + else + return (leftEdge, middle); + } + + if (match) + { + left = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return (leftEdge, right + 1); + } + + public abstract IEnumerator GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index e7791fec3..8f7ef0be2 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,4 +1,3 @@ -using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System.Collections.Generic; @@ -76,17 +75,16 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + regions.Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); + + RangeItem current = first; + while (last != null && current != last.Next) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = current.Value; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -96,7 +94,9 @@ namespace Ryujinx.Memory.Tracking } region.UpdateProtection(); + current = current.Next; } + regions.Lock.ExitReadLock(); } } } @@ -114,20 +114,21 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + regions.Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = regions.FindOverlaps(va, size); + + RangeItem current = first; + while (last != null && current != last.Next) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = current.Value; region.SignalMappingChanged(false); + current = current.Next; } + regions.Lock.ExitReadLock(); } } } @@ -165,10 +166,11 @@ namespace Ryujinx.Memory.Tracking /// A list of virtual regions within the given range internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { - List result = []; NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); - + regions.Lock.EnterUpgradeableReadLock(); + regions.GetOrAddRegions(out List result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); + regions.Lock.ExitUpgradeableReadLock(); + return result; } @@ -296,25 +298,33 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; + List> overlaps = []; + + // We use the non-span method here because keeping the lock will cause a deadlock. + regions.Lock.EnterReadLock(); + (RangeItem first, RangeItem last) = regions.FindOverlaps(address, size); + + RangeItem current = first; + while (last != null && current != last.Next) + { + overlaps.Add(current); + current = current.Next; + } + regions.Lock.ExitReadLock(); - int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); - - if (count == 0 && !precise) + if (first is null && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); + return true; // This memory _should_ be mapped, so we need to try again. } - else - { - shouldThrow = true; - } + + shouldThrow = true; } else { @@ -324,9 +334,9 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Count; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; if (precise) {