using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Ryujinx.Graphics.Gpu.Memory
{
///
/// A range within a buffer that has been modified by the GPU.
///
class BufferModifiedRange : INonOverlappingRange
{
///
/// Start address of the range in guest memory.
///
public ulong Address { get; internal set; }
///
/// Size of the range in bytes.
///
public ulong Size { get; internal set; }
///
/// End address of the range in guest memory.
///
public ulong EndAddress => Address + Size;
///
/// The GPU sync number at the time of the last modification.
///
public ulong SyncNumber { get; internal set; }
///
/// The range list that originally owned this range.
///
public BufferModifiedRangeList Parent { get; internal set; }
///
/// Creates a new instance of a modified range.
///
/// Start address of the range
/// Size of the range in bytes
/// The GPU sync number at the time of creation
/// The range list that owns this range
public BufferModifiedRange(ulong address, ulong size, ulong syncNumber, BufferModifiedRangeList parent)
{
Address = address;
Size = size;
SyncNumber = syncNumber;
Parent = parent;
}
///
/// Checks if a given range overlaps with the modified range.
///
/// Start address of the range
/// Size in bytes of the range
/// True if the range overlaps, false otherwise
public bool OverlapsWith(ulong address, ulong size)
{
return Address < address + size && address < EndAddress;
}
public INonOverlappingRange Split(ulong splitAddress)
{
throw new NotImplementedException();
}
}
///
/// A structure used to track GPU modified ranges within a buffer.
///
class BufferModifiedRangeList : NonOverlappingRangeList
{
private new const int BackingInitialSize = 8;
private readonly GpuContext _context;
private readonly Buffer _parent;
private readonly BufferFlushAction _flushAction;
private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget;
///
/// Whether the modified range list has any entries or not.
///
public bool HasRanges
{
get
{
Lock.EnterReadLock();
bool result = Count > 0;
Lock.ExitReadLock();
return result;
}
}
///
/// Creates a new instance of a modified range list.
///
/// GPU context that the buffer range list belongs to
/// The parent buffer that owns this range list
/// The flush action for the parent buffer
public BufferModifiedRangeList(GpuContext context, Buffer parent, BufferFlushAction flushAction) : base(BackingInitialSize)
{
_context = context;
_parent = parent;
_flushAction = flushAction;
}
///
/// Given an input range, calls the given action with sub-ranges which exclude any of the modified regions.
///
/// Start address of the query range
/// Size of the query range in bytes
/// Action to perform for each remaining sub-range of the input range
public void ExcludeModifiedRegions(ulong address, ulong size, Action action)
{
// Slices a given region using the modified regions in the list. Calls the action for the new slices.
bool lockOwner = Lock.IsReadLockHeld;
if (!lockOwner)
{
Lock.EnterReadLock();
}
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
RangeItem current = first;
while (last != null && current != last.Next)
{
BufferModifiedRange overlap = current.Value;
if (overlap.Address > address)
{
// The start of the remaining region is uncovered by this overlap. Call the action for it.
action(address, overlap.Address - address);
}
// Remaining region is after this overlap.
size -= overlap.EndAddress - address;
address = overlap.EndAddress;
current = current.Next;
}
if (!lockOwner)
{
Lock.ExitReadLock();
}
if ((long)size > 0)
{
// If there is any region left after removing the overlaps, signal it.
action(address, size);
}
}
///
/// Signal that a region of the buffer has been modified, and add the new region to the range list.
/// Any overlapping ranges will be (partially) removed.
///
/// Start address of the modified region
/// Size of the modified region in bytes
public void SignalModified(ulong address, ulong size)
{
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber;
if (first is null)
{
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
if (first == last)
{
if (first.Address == address && first.EndAddress == endAddress)
{
first.Value.SyncNumber = syncNumber;
first.Value.Parent = this;
Lock.ExitWriteLock();
return;
}
if (first.Address < address)
{
first.Value.Size = address - first.Address;
extendsPre = true;
if (first.EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress,
first.Value.SyncNumber, first.Value.Parent);
extendsPost = true;
}
}
else
{
if (first.EndAddress > endAddress)
{
first.Value.Size = first.EndAddress - endAddress;
first.Value.Address = endAddress;
}
else
{
Remove(first.Value);
}
}
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPre = null;
if (first.Address < address)
{
buffPre = new BufferModifiedRange(first.Address, address - first.Address,
first.Value.SyncNumber, first.Value.Parent);
extendsPre = true;
}
if (last.EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress,
last.Value.SyncNumber, last.Value.Parent);
extendsPost = true;
}
RemoveRange(first, last);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
}
///
/// Gets modified ranges within the specified region, and then fires the given action for each range individually.
///
/// Start address to query
/// Size to query
/// Sync number required for a range to be signalled
/// The action to call for each modified range
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction)
{
Lock.EnterReadLock();
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
RangeItem current = first;
while (last != null && current != last.Next)
{
BufferModifiedRange overlap = current.Value;
if (overlap.SyncNumber == syncNumber)
{
rangeAction(overlap.Address, overlap.Size);
}
current = current.Next;
}
Lock.ExitReadLock();
}
///
/// Gets modified ranges within the specified region, and then fires the given action for each range individually.
///
/// Start address to query
/// Size to query
/// The action to call for each modified range
public void GetRanges(ulong address, ulong size, Action rangeAction)
{
List> overlaps = [];
// We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock();
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
RangeItem current = first;
while (last != null && current != last.Next)
{
overlaps.Add(current);
current = current.Next;
}
Lock.ExitReadLock();
for (int i = 0; i < overlaps.Count; i++)
{
BufferModifiedRange overlap = overlaps[i].Value;
rangeAction(overlap.Address, overlap.Size);
}
}
///
/// Queries if a range exists within the specified region.
///
/// Start address to query
/// Size to query
/// True if a range exists in the specified region, false otherwise
public bool HasRange(ulong address, ulong size)
{
Lock.EnterReadLock();
(RangeItem first, RangeItem _) = FindOverlaps(address, size);
bool result = first is not null;
Lock.ExitReadLock();
return result;
}
///
/// Performs the given range action, or one from a migration that overlaps and has not synced yet.
///
/// The offset to pass to the action
/// The size to pass to the action
/// The sync number that has been reached
/// The action to perform
public void RangeActionWithMigration(ulong offset, ulong size, ulong syncNumber, BufferFlushAction rangeAction)
{
if (_source != null)
{
_source.RangeActionWithMigration(offset, size, syncNumber, rangeAction);
}
else
{
rangeAction(offset, size, syncNumber);
}
}
///
/// Removes modified ranges ready by the sync number from the list, and flushes their buffer data within a given address range.
///
/// Overlapping ranges to check
/// Number of overlapping ranges
/// The highest difference between an overlapping range's sync number and the current one
/// The current sync number
/// The start address of the flush range
/// The end address of the flush range
private void RemoveRangesAndFlush(
RangeItem[] overlaps,
int rangeCount,
long highestDiff,
ulong currentSync,
ulong address,
ulong endAddress)
{
if (_migrationTarget == null)
{
ulong waitSync = currentSync + (ulong)highestDiff;
for (int i = 0; i < rangeCount; i++)
{
BufferModifiedRange overlap = overlaps[i].Value;
long diff = (long)(overlap.SyncNumber - currentSync);
if (diff <= highestDiff)
{
ulong clampAddress = Math.Max(address, overlap.Address);
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
Lock.EnterWriteLock();
ClearPart(overlap, clampAddress, clampEnd);
Lock.ExitWriteLock();
RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction);
}
}
return;
}
// There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine.
_migrationTarget.RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress);
}
///
/// Gets modified ranges within the specified region, waits on ones from a previous sync number,
/// and then fires the flush action for each range individually.
///
///
/// This function assumes it is called from the background thread.
/// Modifications from the current sync number are ignored because the guest should not expect them to be available yet.
/// They will remain reserved, so that any data sync prioritizes the data in the GPU.
///
/// Start address to query
/// Size to query
public void WaitForAndFlushRanges(ulong address, ulong size)
{
ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber;
int rangeCount = 0;
List> overlaps = [];
// Range list must be consistent for this operation
Lock.EnterReadLock();
if (_migrationTarget != null)
{
rangeCount = -1;
}
else
{
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
RangeItem current = first;
while (last != null && current != last.Next)
{
rangeCount++;
overlaps.Add(current);
current = current.Next;
}
}
Lock.ExitReadLock();
if (rangeCount == -1)
{
_migrationTarget!.WaitForAndFlushRanges(address, size);
return;
}
if (rangeCount == 0)
{
return;
}
// First, determine which syncpoint to wait on.
// This is the latest syncpoint that is not equal to the current sync.
long highestDiff = long.MinValue;
for (int i = 0; i < rangeCount; i++)
{
BufferModifiedRange overlap = overlaps[i].Value;
long diff = (long)(overlap.SyncNumber - currentSync);
if (diff < 0 && diff > highestDiff)
{
highestDiff = diff;
}
}
if (highestDiff == long.MinValue)
{
return;
}
// Wait for the syncpoint.
_context.Renderer.WaitSync(currentSync + (ulong)highestDiff);
RemoveRangesAndFlush(overlaps.ToArray(), rangeCount, highestDiff, currentSync, address, endAddress);
}
///
/// Inherit ranges from another modified range list.
///
///
/// Assumes that ranges will be inherited in address ascending order.
///
/// The range list to inherit from
/// The action to call for each modified range
public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction)
{
ranges.Lock.EnterReadLock();
BufferModifiedRange[] inheritRanges = ranges.ToArray();
ranges.Lock.ExitReadLock();
// Copy over the migration from the previous range list
BufferMigration oldMigration = ranges._source;
BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration);
ranges._parent.IncrementReferenceCount();
if (_source == null)
{
// Create a new migration.
_source = new BufferMigration([span], this, _context.SyncNumber);
_context.RegisterBufferMigration(_source);
}
else
{
// Extend the migration
_source.AddSpanToEnd(span);
}
ranges._migrationTarget = this;
Lock.EnterWriteLock();
foreach (BufferModifiedRange range in inheritRanges)
{
Add(range);
}
Lock.ExitWriteLock();
ulong currentSync = _context.SyncNumber;
foreach (BufferModifiedRange range in inheritRanges)
{
if (range.SyncNumber != currentSync)
{
registerRangeAction(range.Address, range.Size);
}
}
}
///
/// Register a migration from previous buffer storage. This migration is from a snapshot of the buffer's
/// current handle to its handle in the future, and is assumed to be complete when the sync action completes.
/// When the migration completes, the handle is disposed.
///
public void SelfMigration()
{
BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(),
_parent.GetSnapshotFlushAction(), _source);
BufferMigration migration = new([span], this, _context.SyncNumber);
// Migration target is used to redirect flush actions to the latest range list,
// so we don't need to set it here. (this range list is still the latest)
_context.RegisterBufferMigration(migration);
Lock.EnterWriteLock();
_source = migration;
Lock.ExitWriteLock();
}
///
/// Removes a source buffer migration, indicating its copy has completed.
///
/// The migration to remove
public void RemoveMigration(BufferMigration migration)
{
Lock.EnterWriteLock();
if (_source == migration)
{
_source = null;
}
Lock.ExitWriteLock();
}
private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress)
{
Remove(overlap);
// If the overlap extends outside of the clear range, make sure those parts still exist.
if (overlap.Address < address)
{
Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent));
}
if (overlap.EndAddress > endAddress)
{
Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent));
}
}
///
/// Clear modified ranges within the specified area.
///
/// Start address to clear
/// Size to clear
public void Clear(ulong address, ulong size)
{
ulong endAddress = address + size;
Lock.EnterWriteLock();
(RangeItem first, RangeItem last) = FindOverlaps(address, size);
if (first is null)
{
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
if (first == last)
{
if (first.Address < address)
{
first.Value.Size = address - first.Address;
extendsPre = true;
if (first.EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, first.EndAddress - endAddress,
first.Value.SyncNumber, first.Value.Parent);
extendsPost = true;
}
}
else
{
if (first.EndAddress > endAddress)
{
first.Value.Size = first.EndAddress - endAddress;
first.Value.Address = endAddress;
}
else
{
Remove(first.Value);
}
}
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPre = null;
if (first.Address < address)
{
buffPre = new BufferModifiedRange(first.Address, address - first.Address,
first.Value.SyncNumber, first.Value.Parent);
extendsPre = true;
}
if (last.EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, last.EndAddress - endAddress,
last.Value.SyncNumber, last.Value.Parent);
extendsPost = true;
}
RemoveRange(first, last);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
}
}
}