using Ryujinx.Memory.Range; using System; using System.Collections.Generic; using System.Linq; namespace Ryujinx.Graphics.Gpu.Memory { /// /// A range within a buffer that has been modified by the GPU. /// class BufferModifiedRange : INonOverlappingRange { /// /// Start address of the range in guest memory. /// public ulong Address { get; internal set; } /// /// Size of the range in bytes. /// public ulong Size { get; internal set; } /// /// End address of the range in guest memory. /// public ulong EndAddress => Address + Size; /// /// The GPU sync number at the time of the last modification. /// public ulong SyncNumber { get; internal set; } /// /// The range list that originally owned this range. /// public BufferModifiedRangeList Parent { get; internal set; } /// /// Creates a new instance of a modified range. /// /// Start address of the range /// Size of the range in bytes /// The GPU sync number at the time of creation /// The range list that owns this range public BufferModifiedRange(ulong address, ulong size, ulong syncNumber, BufferModifiedRangeList parent) { Address = address; Size = size; SyncNumber = syncNumber; Parent = parent; } /// /// Checks if a given range overlaps with the modified range. /// /// Start address of the range /// Size in bytes of the range /// True if the range overlaps, false otherwise public bool OverlapsWith(ulong address, ulong size) { return Address < address + size && address < EndAddress; } public INonOverlappingRange Split(ulong splitAddress) { throw new NotImplementedException(); } } /// /// A structure used to track GPU modified ranges within a buffer. /// class BufferModifiedRangeList : NonOverlappingRangeList { private new const int BackingInitialSize = 8; private readonly GpuContext _context; private readonly Buffer _parent; private readonly BufferFlushAction _flushAction; private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; /// /// Whether the modified range list has any entries or not. /// public bool HasRanges { get { Lock.EnterReadLock(); bool result = Count > 0; Lock.ExitReadLock(); return result; } } /// /// Creates a new instance of a modified range list. /// /// GPU context that the buffer range list belongs to /// The parent buffer that owns this range list /// The flush action for the parent buffer public BufferModifiedRangeList(GpuContext context, Buffer parent, BufferFlushAction flushAction) : base(BackingInitialSize) { _context = context; _parent = parent; _flushAction = flushAction; } /// /// Given an input range, calls the given action with sub-ranges which exclude any of the modified regions. /// /// Start address of the query range /// Size of the query range in bytes /// Action to perform for each remaining sub-range of the input range public void ExcludeModifiedRegions(ulong address, ulong size, Action action) { // Slices a given region using the modified regions in the list. Calls the action for the new slices. bool lockOwner = Lock.IsReadLockHeld; if (!lockOwner) { Lock.EnterReadLock(); } (RangeItem first, RangeItem last) = FindOverlaps(address, size); RangeItem current = first; while (last != null && current != last.Next) { BufferModifiedRange overlap = current.Value; if (overlap.Address > address) { // The start of the remaining region is uncovered by this overlap. Call the action for it. action(address, overlap.Address - address); } // Remaining region is after this overlap. size -= overlap.EndAddress - address; address = overlap.EndAddress; current = current.Next; } if (!lockOwner) { Lock.ExitReadLock(); } if ((long)size > 0) { // If there is any region left after removing the overlaps, signal it. action(address, size); } } /// /// Signal that a region of the buffer has been modified, and add the new region to the range list. /// Any overlapping ranges will be (partially) removed. /// /// Start address of the modified region /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { // We may overlap with some existing modified regions. They must be cut into by the new entry. Lock.EnterWriteLock(); (RangeItem first, RangeItem last) = FindOverlaps(address, size); ulong endAddress = address + size; ulong syncNumber = _context.SyncNumber; if (first is null) { Add(new BufferModifiedRange(address, size, syncNumber, this)); Lock.ExitWriteLock(); return; } BufferModifiedRange buffPost = null; bool extendsPost = false; bool extendsPre = false; if (first == last) { if (first.Address == address && first.EndAddress == endAddress) { first.Value.SyncNumber = syncNumber; first.Value.Parent = this; Lock.ExitWriteLock(); return; } if (first.Address < address) { first.Value.Size = address - first.Address; extendsPre = true; if (first.EndAddress > endAddress) { buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, first.Value.SyncNumber, first.Value.Parent); extendsPost = true; } } else { if (first.EndAddress > endAddress) { first.Value.Size = first.EndAddress - endAddress; first.Value.Address = endAddress; } else { Remove(first.Value); } } if (extendsPre && extendsPost) { Add(buffPost); } Add(new BufferModifiedRange(address, size, syncNumber, this)); Lock.ExitWriteLock(); return; } BufferModifiedRange buffPre = null; if (first.Address < address) { buffPre = new BufferModifiedRange(first.Address, address - first.Address, first.Value.SyncNumber, first.Value.Parent); extendsPre = true; } if (last.EndAddress > endAddress) { buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, last.Value.SyncNumber, last.Value.Parent); extendsPost = true; } RemoveRange(first, last); if (extendsPre) { Add(buffPre); } if (extendsPost) { Add(buffPost); } Add(new BufferModifiedRange(address, size, syncNumber, this)); Lock.ExitWriteLock(); } /// /// Gets modified ranges within the specified region, and then fires the given action for each range individually. /// /// Start address to query /// Size to query /// Sync number required for a range to be signalled /// The action to call for each modified range public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { Lock.EnterReadLock(); (RangeItem first, RangeItem last) = FindOverlaps(address, size); RangeItem current = first; while (last != null && current != last.Next) { BufferModifiedRange overlap = current.Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } current = current.Next; } Lock.ExitReadLock(); } /// /// Gets modified ranges within the specified region, and then fires the given action for each range individually. /// /// Start address to query /// Size to query /// The action to call for each modified range public void GetRanges(ulong address, ulong size, Action rangeAction) { List> overlaps = []; // We use the non-span method here because keeping the lock will cause a deadlock. Lock.EnterReadLock(); (RangeItem first, RangeItem last) = FindOverlaps(address, size); RangeItem current = first; while (last != null && current != last.Next) { overlaps.Add(current); current = current.Next; } Lock.ExitReadLock(); for (int i = 0; i < overlaps.Count; i++) { BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } /// /// Queries if a range exists within the specified region. /// /// Start address to query /// Size to query /// True if a range exists in the specified region, false otherwise public bool HasRange(ulong address, ulong size) { Lock.EnterReadLock(); (RangeItem first, RangeItem _) = FindOverlaps(address, size); bool result = first is not null; Lock.ExitReadLock(); return result; } /// /// Performs the given range action, or one from a migration that overlaps and has not synced yet. /// /// The offset to pass to the action /// The size to pass to the action /// The sync number that has been reached /// The action to perform public void RangeActionWithMigration(ulong offset, ulong size, ulong syncNumber, BufferFlushAction rangeAction) { if (_source != null) { _source.RangeActionWithMigration(offset, size, syncNumber, rangeAction); } else { rangeAction(offset, size, syncNumber); } } /// /// Removes modified ranges ready by the sync number from the list, and flushes their buffer data within a given address range. /// /// Overlapping ranges to check /// Number of overlapping ranges /// The highest difference between an overlapping range's sync number and the current one /// The current sync number /// The start address of the flush range /// The end address of the flush range private void RemoveRangesAndFlush( RangeItem[] overlaps, int rangeCount, long highestDiff, ulong currentSync, ulong address, ulong endAddress) { if (_migrationTarget == null) { ulong waitSync = currentSync + (ulong)highestDiff; for (int i = 0; i < rangeCount; i++) { BufferModifiedRange overlap = overlaps[i].Value; long diff = (long)(overlap.SyncNumber - currentSync); if (diff <= highestDiff) { ulong clampAddress = Math.Max(address, overlap.Address); ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); Lock.EnterWriteLock(); ClearPart(overlap, clampAddress, clampEnd); Lock.ExitWriteLock(); RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); } } return; } // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. _migrationTarget.RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress); } /// /// Gets modified ranges within the specified region, waits on ones from a previous sync number, /// and then fires the flush action for each range individually. /// /// /// This function assumes it is called from the background thread. /// Modifications from the current sync number are ignored because the guest should not expect them to be available yet. /// They will remain reserved, so that any data sync prioritizes the data in the GPU. /// /// Start address to query /// Size to query public void WaitForAndFlushRanges(ulong address, ulong size) { ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; int rangeCount = 0; List> overlaps = []; // Range list must be consistent for this operation Lock.EnterReadLock(); if (_migrationTarget != null) { rangeCount = -1; } else { // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. (RangeItem first, RangeItem last) = FindOverlaps(address, size); RangeItem current = first; while (last != null && current != last.Next) { rangeCount++; overlaps.Add(current); current = current.Next; } } Lock.ExitReadLock(); if (rangeCount == -1) { _migrationTarget!.WaitForAndFlushRanges(address, size); return; } if (rangeCount == 0) { return; } // First, determine which syncpoint to wait on. // This is the latest syncpoint that is not equal to the current sync. long highestDiff = long.MinValue; for (int i = 0; i < rangeCount; i++) { BufferModifiedRange overlap = overlaps[i].Value; long diff = (long)(overlap.SyncNumber - currentSync); if (diff < 0 && diff > highestDiff) { highestDiff = diff; } } if (highestDiff == long.MinValue) { return; } // Wait for the syncpoint. _context.Renderer.WaitSync(currentSync + (ulong)highestDiff); RemoveRangesAndFlush(overlaps.ToArray(), rangeCount, highestDiff, currentSync, address, endAddress); } /// /// Inherit ranges from another modified range list. /// /// /// Assumes that ranges will be inherited in address ascending order. /// /// The range list to inherit from /// The action to call for each modified range public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { ranges.Lock.EnterReadLock(); BufferModifiedRange[] inheritRanges = ranges.ToArray(); ranges.Lock.ExitReadLock(); // Copy over the migration from the previous range list BufferMigration oldMigration = ranges._source; BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); ranges._parent.IncrementReferenceCount(); if (_source == null) { // Create a new migration. _source = new BufferMigration([span], this, _context.SyncNumber); _context.RegisterBufferMigration(_source); } else { // Extend the migration _source.AddSpanToEnd(span); } ranges._migrationTarget = this; Lock.EnterWriteLock(); foreach (BufferModifiedRange range in inheritRanges) { Add(range); } Lock.ExitWriteLock(); ulong currentSync = _context.SyncNumber; foreach (BufferModifiedRange range in inheritRanges) { if (range.SyncNumber != currentSync) { registerRangeAction(range.Address, range.Size); } } } /// /// Register a migration from previous buffer storage. This migration is from a snapshot of the buffer's /// current handle to its handle in the future, and is assumed to be complete when the sync action completes. /// When the migration completes, the handle is disposed. /// public void SelfMigration() { BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); BufferMigration migration = new([span], this, _context.SyncNumber); // Migration target is used to redirect flush actions to the latest range list, // so we don't need to set it here. (this range list is still the latest) _context.RegisterBufferMigration(migration); Lock.EnterWriteLock(); _source = migration; Lock.ExitWriteLock(); } /// /// Removes a source buffer migration, indicating its copy has completed. /// /// The migration to remove public void RemoveMigration(BufferMigration migration) { Lock.EnterWriteLock(); if (_source == migration) { _source = null; } Lock.ExitWriteLock(); } private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) { Remove(overlap); // If the overlap extends outside of the clear range, make sure those parts still exist. if (overlap.Address < address) { Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); } if (overlap.EndAddress > endAddress) { Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); } } /// /// Clear modified ranges within the specified area. /// /// Start address to clear /// Size to clear public void Clear(ulong address, ulong size) { ulong endAddress = address + size; Lock.EnterWriteLock(); (RangeItem first, RangeItem last) = FindOverlaps(address, size); if (first is null) { Lock.ExitWriteLock(); return; } BufferModifiedRange buffPost = null; bool extendsPost = false; bool extendsPre = false; if (first == last) { if (first.Address < address) { first.Value.Size = address - first.Address; extendsPre = true; if (first.EndAddress > endAddress) { buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress, first.Value.SyncNumber, first.Value.Parent); extendsPost = true; } } else { if (first.EndAddress > endAddress) { first.Value.Size = first.EndAddress - endAddress; first.Value.Address = endAddress; } else { Remove(first.Value); } } if (extendsPre && extendsPost) { Add(buffPost); } Lock.ExitWriteLock(); return; } BufferModifiedRange buffPre = null; if (first.Address < address) { buffPre = new BufferModifiedRange(first.Address, address - first.Address, first.Value.SyncNumber, first.Value.Parent); extendsPre = true; } if (last.EndAddress > endAddress) { buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress, last.Value.SyncNumber, last.Value.Parent); extendsPost = true; } RemoveRange(first, last); if (extendsPre) { Add(buffPre); } if (extendsPost) { Add(buffPost); } Lock.ExitWriteLock(); } } }