marcel/Ryujinx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support memory aliasing (#2954)

Browse Source 
* Back to the origins: Make memory manager take guest PA rather than host address once again

* Direct mapping with alias support on Windows

* Fixes and remove more of the emulated shared memory

* Linux support

* Make shared and transfer memory not depend on SharedMemoryStorage

* More efficient view mapping on Windows (no more restricted to 4KB pages at a time)

* Handle potential access violations caused by partial unmap

* Implement host mapping using shared memory on Linux

* Add new GetPhysicalAddressChecked method, used to ensure the virtual address is mapped before address translation

Also align GetRef behaviour with software memory manager

* We don't need a mirrorable memory block for software memory manager mode

* Disable memory aliasing tests while we don't have shared memory support on Mac

* Shared memory & SIGBUS handler for macOS

* Fix typo + nits + re-enable memory tests

* Set MAP_JIT_DARWIN on x86 Mac too

* Add back the address space mirror

* Only set MAP_JIT_DARWIN if we are mapping as executable

* Disable aliasing tests again (still fails on Mac)

* Fix UnmapView4KB (by not casting size to int)

* Use ref counting on memory blocks to delay closing the shared memory handle until all blocks using it are disposed

* Address PR feedback

* Make RO hold a reference to the guest process memory manager to avoid early disposal

Co-authored-by: nastys <nastys@users.noreply.github.com>
This commit is contained in:
gdkchan
2022-05-02 20:30:02 -03:00
committed by GitHub
parent 4a892fbdc9
commit 95017b8c66
41 changed files with 2373 additions and 2155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

703
Ryujinx.Memory/WindowsShared/EmulatedSharedMemoryWindows.cs
View File

@@ -1,703 +0,0 @@
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
namespace Ryujinx.Memory.WindowsShared
{
class EmulatedSharedMemoryWindows : IDisposable
{
private static readonly IntPtr InvalidHandleValue = new IntPtr(-1);
private static readonly IntPtr CurrentProcessHandle = new IntPtr(-1);
public const int MappingBits = 16; // Windows 64kb granularity.
public const ulong MappingGranularity = 1 << MappingBits;
public const ulong MappingMask = MappingGranularity - 1;
public const ulong BackingSize32GB = 32UL * 1024UL * 1024UL * 1024UL; // Reasonable max size of 32GB.
private class SharedMemoryMapping : INonOverlappingRange
{
public ulong Address { get; }
public ulong Size { get; private set; }
public ulong EndAddress { get; private set; }
public List<int> Blocks;
public SharedMemoryMapping(ulong address, ulong size, List<int> blocks = null)
{
Address = address;
Size = size;
EndAddress = address + size;
Blocks = blocks ?? new List<int>();
}
public bool OverlapsWith(ulong address, ulong size)
{
return Address < address + size && address < EndAddress;
}
public void ExtendTo(ulong endAddress, RangeList<SharedMemoryMapping> list)
{
EndAddress = endAddress;
Size = endAddress - Address;
list.UpdateEndAddress(this);
}
public void AddBlocks(IEnumerable<int> blocks)
{
if (Blocks.Count > 0 && blocks.Count() > 0 && Blocks.Last() == blocks.First())
{
Blocks.AddRange(blocks.Skip(1));
}
else
{
Blocks.AddRange(blocks);
}
}
public INonOverlappingRange Split(ulong splitAddress)
{
SharedMemoryMapping newRegion = new SharedMemoryMapping(splitAddress, EndAddress - splitAddress);
int end = (int)((EndAddress + MappingMask) >> MappingBits);
int start = (int)(Address >> MappingBits);
Size = splitAddress - Address;
EndAddress = splitAddress;
int splitEndBlock = (int)((splitAddress + MappingMask) >> MappingBits);
int splitStartBlock = (int)(splitAddress >> MappingBits);
newRegion.AddBlocks(Blocks.Skip(splitStartBlock - start));
Blocks.RemoveRange(splitEndBlock - start, end - splitEndBlock);
return newRegion;
}
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern IntPtr CreateFileMapping(
IntPtr hFile,
IntPtr lpFileMappingAttributes,
FileMapProtection flProtect,
uint dwMaximumSizeHigh,
uint dwMaximumSizeLow,
[MarshalAs(UnmanagedType.LPWStr)] string lpName);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool CloseHandle(IntPtr hObject);
[DllImport("KernelBase.dll", SetLastError = true)]
private static extern IntPtr VirtualAlloc2(
IntPtr process,
IntPtr lpAddress,
IntPtr dwSize,
AllocationType flAllocationType,
MemoryProtection flProtect,
IntPtr extendedParameters,
ulong parameterCount);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool VirtualFree(IntPtr lpAddress, IntPtr dwSize, AllocationType dwFreeType);
[DllImport("KernelBase.dll", SetLastError = true)]
private static extern IntPtr MapViewOfFile3(
IntPtr hFileMappingObject,
IntPtr process,
IntPtr baseAddress,
ulong offset,
IntPtr dwNumberOfBytesToMap,
ulong allocationType,
MemoryProtection dwDesiredAccess,
IntPtr extendedParameters,
ulong parameterCount);
[DllImport("KernelBase.dll", SetLastError = true)]
private static extern bool UnmapViewOfFile2(IntPtr process, IntPtr lpBaseAddress, ulong unmapFlags);
private ulong _size;
private object _lock = new object();
private ulong _backingSize;
private IntPtr _backingMemHandle;
private int _backingEnd;
private int _backingAllocated;
private Queue<int> _backingFreeList;
private List<ulong> _mappedBases;
private RangeList<SharedMemoryMapping> _mappings;
private SharedMemoryMapping[] _foundMappings = new SharedMemoryMapping[32];
private PlaceholderList _placeholders;
public EmulatedSharedMemoryWindows(ulong size)
{
ulong backingSize = BackingSize32GB;
_size = size;
_backingSize = backingSize;
_backingMemHandle = CreateFileMapping(
InvalidHandleValue,
IntPtr.Zero,
FileMapProtection.PageReadWrite | FileMapProtection.SectionReserve,
(uint)(backingSize >> 32),
(uint)backingSize,
null);
if (_backingMemHandle == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
_backingFreeList = new Queue<int>();
_mappings = new RangeList<SharedMemoryMapping>();
_mappedBases = new List<ulong>();
_placeholders = new PlaceholderList(size >> MappingBits);
}
private (ulong granularStart, ulong granularEnd) GetAlignedRange(ulong address, ulong size)
{
return (address & (~MappingMask), (address + size + MappingMask) & (~MappingMask));
}
private void Commit(ulong address, ulong size)
{
(ulong granularStart, ulong granularEnd) = GetAlignedRange(address, size);
ulong endAddress = address + size;
lock (_lock)
{
// Search a bit before and after the new mapping.
// When adding our new mapping, we may need to join an existing mapping into our new mapping (or in some cases, to the other side!)
ulong searchStart = granularStart == 0 ? 0 : (granularStart - 1);
int mappingCount = _mappings.FindOverlapsNonOverlapping(searchStart, (granularEnd - searchStart) + 1, ref _foundMappings);
int first = -1;
int last = -1;
SharedMemoryMapping startOverlap = null;
SharedMemoryMapping endOverlap = null;
int lastIndex = (int)(address >> MappingBits);
int endIndex = (int)((endAddress + MappingMask) >> MappingBits);
int firstBlock = -1;
int endBlock = -1;
for (int i = 0; i < mappingCount; i++)
{
SharedMemoryMapping mapping = _foundMappings[i];
if (mapping.Address < address)
{
if (mapping.EndAddress >= address)
{
startOverlap = mapping;
}
if ((int)((mapping.EndAddress - 1) >> MappingBits) == lastIndex)
{
lastIndex = (int)((mapping.EndAddress + MappingMask) >> MappingBits);
firstBlock = mapping.Blocks.Last();
}
}
if (mapping.EndAddress > endAddress)
{
if (mapping.Address <= endAddress)
{
endOverlap = mapping;
}
if ((int)((mapping.Address) >> MappingBits) + 1 == endIndex)
{
endIndex = (int)((mapping.Address) >> MappingBits);
endBlock = mapping.Blocks.First();
}
}
if (mapping.OverlapsWith(address, size))
{
if (first == -1)
{
first = i;
}
last = i;
}
}
if (startOverlap == endOverlap && startOverlap != null)
{
// Already fully committed.
return;
}
var blocks = new List<int>();
int lastBlock = -1;
if (firstBlock != -1)
{
blocks.Add(firstBlock);
lastBlock = firstBlock;
}
bool hasMapped = false;
Action map = () =>
{
if (!hasMapped)
{
_placeholders.EnsurePlaceholders(address >> MappingBits, (granularEnd - granularStart) >> MappingBits, SplitPlaceholder);
hasMapped = true;
}
// There's a gap between this index and the last. Allocate blocks to fill it.
blocks.Add(MapBackingBlock(MappingGranularity * (ulong)lastIndex++));
};
if (first != -1)
{
for (int i = first; i <= last; i++)
{
SharedMemoryMapping mapping = _foundMappings[i];
int mapIndex = (int)(mapping.Address >> MappingBits);
while (lastIndex < mapIndex)
{
map();
}
if (lastBlock == mapping.Blocks[0])
{
blocks.AddRange(mapping.Blocks.Skip(1));
}
else
{
blocks.AddRange(mapping.Blocks);
}
lastIndex = (int)((mapping.EndAddress - 1) >> MappingBits) + 1;
}
}
while (lastIndex < endIndex)
{
map();
}
if (endBlock != -1 && endBlock != lastBlock)
{
blocks.Add(endBlock);
}
if (startOverlap != null && endOverlap != null)
{
// Both sides should be coalesced. Extend the start overlap to contain the end overlap, and add together their blocks.
_mappings.Remove(endOverlap);
startOverlap.ExtendTo(endOverlap.EndAddress, _mappings);
startOverlap.AddBlocks(blocks);
startOverlap.AddBlocks(endOverlap.Blocks);
}
else if (startOverlap != null)
{
startOverlap.ExtendTo(endAddress, _mappings);
startOverlap.AddBlocks(blocks);
}
else
{
var mapping = new SharedMemoryMapping(address, size, blocks);
if (endOverlap != null)
{
mapping.ExtendTo(endOverlap.EndAddress, _mappings);
mapping.AddBlocks(endOverlap.Blocks);
_mappings.Remove(endOverlap);
}
_mappings.Add(mapping);
}
}
}
private void Decommit(ulong address, ulong size)
{
(ulong granularStart, ulong granularEnd) = GetAlignedRange(address, size);
ulong endAddress = address + size;
lock (_lock)
{
int mappingCount = _mappings.FindOverlapsNonOverlapping(granularStart, granularEnd - granularStart, ref _foundMappings);
int first = -1;
int last = -1;
for (int i = 0; i < mappingCount; i++)
{
SharedMemoryMapping mapping = _foundMappings[i];
if (mapping.OverlapsWith(address, size))
{
if (first == -1)
{
first = i;
}
last = i;
}
}
if (first == -1)
{
return; // Could not find any regions to decommit.
}
int lastReleasedBlock = -1;
bool releasedFirst = false;
bool releasedLast = false;
for (int i = last; i >= first; i--)
{
SharedMemoryMapping mapping = _foundMappings[i];
bool releaseEnd = true;
bool releaseStart = true;
if (i == last)
{
// If this is the last region, do not release the block if there is a page ahead of us, or the block continues after us. (it is keeping the block alive)
releaseEnd = last == mappingCount - 1;
// If the end region starts after the decommit end address, split and readd it after modifying its base address.
if (mapping.EndAddress > endAddress)
{
var newMapping = (SharedMemoryMapping)mapping.Split(endAddress);
_mappings.UpdateEndAddress(mapping);
_mappings.Add(newMapping);
if ((endAddress & MappingMask) != 0)
{
releaseEnd = false;
}
}
releasedLast = releaseEnd;
}
if (i == first)
{
// If this is the first region, do not release the block if there is a region behind us. (it is keeping the block alive)
releaseStart = first == 0;
// If the first region starts before the decommit address, split it by modifying its end address.
if (mapping.Address < address)
{
var oldMapping = mapping;
mapping = (SharedMemoryMapping)mapping.Split(address);
_mappings.UpdateEndAddress(oldMapping);
if ((address & MappingMask) != 0)
{
releaseStart = false;
}
}
releasedFirst = releaseStart;
}
_mappings.Remove(mapping);
ulong releasePointer = (mapping.EndAddress + MappingMask) & (~MappingMask);
for (int j = mapping.Blocks.Count - 1; j >= 0; j--)
{
int blockId = mapping.Blocks[j];
releasePointer -= MappingGranularity;
if (lastReleasedBlock == blockId)
{
// When committed regions are fragmented, multiple will have the same block id for their start/end granular block.
// Avoid releasing these blocks twice.
continue;
}
if ((j != 0 || releaseStart) && (j != mapping.Blocks.Count - 1 || releaseEnd))
{
ReleaseBackingBlock(releasePointer, blockId);
}
lastReleasedBlock = blockId;
}
}
ulong placeholderStart = (granularStart >> MappingBits) + (releasedFirst ? 0UL : 1UL);
ulong placeholderEnd = (granularEnd >> MappingBits) - (releasedLast ? 0UL : 1UL);
if (placeholderEnd > placeholderStart)
{
_placeholders.RemovePlaceholders(placeholderStart, placeholderEnd - placeholderStart, CoalescePlaceholder);
}
}
}
public bool CommitMap(IntPtr address, IntPtr size)
{
lock (_lock)
{
foreach (ulong mapping in _mappedBases)
{
ulong offset = (ulong)address - mapping;
if (offset < _size)
{
Commit(offset, (ulong)size);
return true;
}
}
}
return false;
}
public bool DecommitMap(IntPtr address, IntPtr size)
{
lock (_lock)
{
foreach (ulong mapping in _mappedBases)
{
ulong offset = (ulong)address - mapping;
if (offset < _size)
{
Decommit(offset, (ulong)size);
return true;
}
}
}
return false;
}
private int MapBackingBlock(ulong offset)
{
bool allocate = false;
int backing;
if (_backingFreeList.Count > 0)
{
backing = _backingFreeList.Dequeue();
}
else
{
if (_backingAllocated == _backingEnd)
{
// Allocate the backing.
_backingAllocated++;
allocate = true;
}
backing = _backingEnd++;
}
ulong backingOffset = MappingGranularity * (ulong)backing;
foreach (ulong baseAddress in _mappedBases)
{
CommitToMap(baseAddress, offset, MappingGranularity, backingOffset, allocate);
allocate = false;
}
return backing;
}
private void ReleaseBackingBlock(ulong offset, int id)
{
foreach (ulong baseAddress in _mappedBases)
{
DecommitFromMap(baseAddress, offset);
}
if (_backingEnd - 1 == id)
{
_backingEnd = id;
}
else
{
_backingFreeList.Enqueue(id);
}
}
public IntPtr Map()
{
IntPtr newMapping = VirtualAlloc2(
CurrentProcessHandle,
IntPtr.Zero,
(IntPtr)_size,
AllocationType.Reserve | AllocationType.ReservePlaceholder,
MemoryProtection.NoAccess,
IntPtr.Zero,
0);
if (newMapping == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
// Apply all existing mappings to the new mapping
lock (_lock)
{
int lastBlock = -1;
foreach (SharedMemoryMapping mapping in _mappings)
{
ulong blockAddress = mapping.Address & (~MappingMask);
foreach (int block in mapping.Blocks)
{
if (block != lastBlock)
{
ulong backingOffset = MappingGranularity * (ulong)block;
CommitToMap((ulong)newMapping, blockAddress, MappingGranularity, backingOffset, false);
lastBlock = block;
}
blockAddress += MappingGranularity;
}
}
_mappedBases.Add((ulong)newMapping);
}
return newMapping;
}
private void SplitPlaceholder(ulong address, ulong size)
{
ulong byteAddress = address << MappingBits;
IntPtr byteSize = (IntPtr)(size << MappingBits);
foreach (ulong mapAddress in _mappedBases)
{
bool result = VirtualFree((IntPtr)(mapAddress + byteAddress), byteSize, AllocationType.PreservePlaceholder | AllocationType.Release);
if (!result)
{
throw new InvalidOperationException("Placeholder could not be split.");
}
}
}
private void CoalescePlaceholder(ulong address, ulong size)
{
ulong byteAddress = address << MappingBits;
IntPtr byteSize = (IntPtr)(size << MappingBits);
foreach (ulong mapAddress in _mappedBases)
{
bool result = VirtualFree((IntPtr)(mapAddress + byteAddress), byteSize, AllocationType.CoalescePlaceholders | AllocationType.Release);
if (!result)
{
throw new InvalidOperationException("Placeholder could not be coalesced.");
}
}
}
private void CommitToMap(ulong mapAddress, ulong address, ulong size, ulong backingOffset, bool allocate)
{
IntPtr targetAddress = (IntPtr)(mapAddress + address);
// Assume the placeholder worked (or already exists)
// Map the backing memory into the mapped location.
IntPtr mapped = MapViewOfFile3(
_backingMemHandle,
CurrentProcessHandle,
targetAddress,
backingOffset,
(IntPtr)MappingGranularity,
0x4000, // REPLACE_PLACEHOLDER
MemoryProtection.ReadWrite,
IntPtr.Zero,
0);
if (mapped == IntPtr.Zero)
{
throw new InvalidOperationException($"Could not map view of backing memory. (va=0x{address:X16} size=0x{size:X16}, error code {Marshal.GetLastWin32Error()})");
}
if (allocate)
{
// Commit this part of the shared memory.
VirtualAlloc2(CurrentProcessHandle, targetAddress, (IntPtr)MappingGranularity, AllocationType.Commit, MemoryProtection.ReadWrite, IntPtr.Zero, 0);
}
}
private void DecommitFromMap(ulong baseAddress, ulong address)
{
UnmapViewOfFile2(CurrentProcessHandle, (IntPtr)(baseAddress + address), 2);
}
public bool Unmap(ulong baseAddress)
{
lock (_lock)
{
if (_mappedBases.Remove(baseAddress))
{
int lastBlock = -1;
foreach (SharedMemoryMapping mapping in _mappings)
{
ulong blockAddress = mapping.Address & (~MappingMask);
foreach (int block in mapping.Blocks)
{
if (block != lastBlock)
{
DecommitFromMap(baseAddress, blockAddress);
lastBlock = block;
}
blockAddress += MappingGranularity;
}
}
if (!VirtualFree((IntPtr)baseAddress, (IntPtr)0, AllocationType.Release))
{
throw new InvalidOperationException("Couldn't free mapping placeholder.");
}
return true;
}
return false;
}
}
public void Dispose()
{
// Remove all file mappings
lock (_lock)
{
foreach (ulong baseAddress in _mappedBases.ToArray())
{
Unmap(baseAddress);
}
}
// Finally, delete the file mapping.
CloseHandle(_backingMemHandle);
}
}
}

740
Ryujinx.Memory/WindowsShared/IntervalTree.cs Normal file
View File

@@ -0,0 +1,740 @@
using System;
using System.Collections.Generic;
namespace Ryujinx.Memory.WindowsShared
{
/// <summary>
/// An Augmented Interval Tree based off of the "TreeDictionary"'s Red-Black Tree. Allows fast overlap checking of ranges.
/// </summary>
/// <typeparam name="K">Key</typeparam>
/// <typeparam name="V">Value</typeparam>
class IntervalTree<K, V> where K : IComparable<K>
{
private const int ArrayGrowthSize = 32;
private const bool Black = true;
private const bool Red = false;
private IntervalTreeNode<K, V> _root = null;
private int _count = 0;
public int Count => _count;
public IntervalTree() { }
#region Public Methods
/// <summary>
/// Gets the values of the interval whose key is <paramref name="key"/>.
/// </summary>
/// <param name="key">Key of the node value to get</param>
/// <param name="value">Value with the given <paramref name="key"/></param>
/// <returns>True if the key is on the dictionary, false otherwise</returns>
public bool TryGet(K key, out V value)
{
IntervalTreeNode<K, V> node = GetNode(key);
if (node == null)
{
value = default;
return false;
}
value = node.Value;
return true;
}
/// <summary>
/// Returns the start addresses of the intervals whose start and end keys overlap the given range.
/// </summary>
/// <param name="start">Start of the range</param>
/// <param name="end">End of the range</param>
/// <param name="overlaps">Overlaps array to place results in</param>
/// <param name="overlapCount">Index to start writing results into the array. Defaults to 0</param>
/// <returns>Number of intervals found</returns>
public int Get(K start, K end, ref IntervalTreeNode<K, V>[] overlaps, int overlapCount = 0)
{
GetNodes(_root, start, end, ref overlaps, ref overlapCount);
return overlapCount;
}
/// <summary>
/// Adds a new interval into the tree whose start is <paramref name="start"/>, end is <paramref name="end"/> and value is <paramref name="value"/>.
/// </summary>
/// <param name="start">Start of the range to add</param>
/// <param name="end">End of the range to insert</param>
/// <param name="value">Value to add</param>
/// <exception cref="ArgumentNullException"><paramref name="value"/> is null</exception>
public void Add(K start, K end, V value)
{
if (value == null)
{
throw new ArgumentNullException(nameof(value));
}
BSTInsert(start, end, value, null, out _);
}
/// <summary>
/// Removes a value from the tree, searching for it with <paramref name="key"/>.
/// </summary>
/// <param name="key">Key of the node to remove</param>
/// <returns>Number of deleted values</returns>
public int Remove(K key)
{
return Remove(GetNode(key));
}
/// <summary>
/// Removes a value from the tree, searching for it with <paramref name="key"/>.
/// </summary>
/// <param name="nodeToDelete">Node to be removed</param>
/// <returns>Number of deleted values</returns>
public int Remove(IntervalTreeNode<K, V> nodeToDelete)
{
if (nodeToDelete == null)
{
return 0;
}
Delete(nodeToDelete);
_count--;
return 1;
}
/// <summary>
/// Adds all the nodes in the dictionary into <paramref name="list"/>.
/// </summary>
/// <returns>A list of all values sorted by Key Order</returns>
public List<V> AsList()
{
List<V> list = new List<V>();
AddToList(_root, list);
return list;
}
#endregion
#region Private Methods (BST)
/// <summary>
/// Adds all values that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order.
/// </summary>
/// <param name="node">The node to search for values within</param>
/// <param name="list">The list to add values to</param>
private void AddToList(IntervalTreeNode<K, V> node, List<V> list)
{
if (node == null)
{
return;
}
AddToList(node.Left, list);
list.Add(node.Value);
AddToList(node.Right, list);
}
/// <summary>
/// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists.
/// </summary>
/// <param name="key">Key of the node to get</param>
/// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception>
/// <returns>Node reference in the tree</returns>
private IntervalTreeNode<K, V> GetNode(K key)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
IntervalTreeNode<K, V> node = _root;
while (node != null)
{
int cmp = key.CompareTo(node.Start);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
return node;
}
}
return null;
}
/// <summary>
/// Retrieve all nodes that overlap the given start and end keys.
/// </summary>
/// <param name="start">Start of the range</param>
/// <param name="end">End of the range</param>
/// <param name="overlaps">Overlaps array to place results in</param>
/// <param name="overlapCount">Overlaps count to update</param>
private void GetNodes(IntervalTreeNode<K, V> node, K start, K end, ref IntervalTreeNode<K, V>[] overlaps, ref int overlapCount)
{
if (node == null || start.CompareTo(node.Max) >= 0)
{
return;
}
GetNodes(node.Left, start, end, ref overlaps, ref overlapCount);
bool endsOnRight = end.CompareTo(node.Start) > 0;
if (endsOnRight)
{
if (start.CompareTo(node.End) < 0)
{
if (overlaps.Length >= overlapCount)
{
Array.Resize(ref overlaps, overlapCount + ArrayGrowthSize);
}
overlaps[overlapCount++] = node;
}
GetNodes(node.Right, start, end, ref overlaps, ref overlapCount);
}
}
/// <summary>
/// Propagate an increase in max value starting at the given node, heading up the tree.
/// This should only be called if the max increases - not for rebalancing or removals.
/// </summary>
/// <param name="node">The node to start propagating from</param>
private void PropagateIncrease(IntervalTreeNode<K, V> node)
{
K max = node.Max;
IntervalTreeNode<K, V> ptr = node;
while ((ptr = ptr.Parent) != null)
{
if (max.CompareTo(ptr.Max) > 0)
{
ptr.Max = max;
}
else
{
break;
}
}
}
/// <summary>
/// Propagate recalculating max value starting at the given node, heading up the tree.
/// This fully recalculates the max value from all children when there is potential for it to decrease.
/// </summary>
/// <param name="node">The node to start propagating from</param>
private void PropagateFull(IntervalTreeNode<K, V> node)
{
IntervalTreeNode<K, V> ptr = node;
do
{
K max = ptr.End;
if (ptr.Left != null && ptr.Left.Max.CompareTo(max) > 0)
{
max = ptr.Left.Max;
}
if (ptr.Right != null && ptr.Right.Max.CompareTo(max) > 0)
{
max = ptr.Right.Max;
}
ptr.Max = max;
} while ((ptr = ptr.Parent) != null);
}
/// <summary>
/// Insertion Mechanism for the interval tree. Similar to a BST insert, with the start of the range as the key.
/// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="start"/>, and all children in the right subtree are greater than <paramref name="start"/>.
/// Each node can contain multiple values, and has an end address which is the maximum of all those values.
/// Post insertion, the "max" value of the node and all parents are updated.
/// </summary>
/// <param name="start">Start of the range to insert</param>
/// <param name="end">End of the range to insert</param>
/// <param name="value">Value to insert</param>
/// <param name="updateFactoryCallback">Optional factory used to create a new value if <paramref name="start"/> is already on the tree</param>
/// <param name="outNode">Node that was inserted or modified</param>
/// <returns>True if <paramref name="start"/> was not yet on the tree, false otherwise</returns>
private bool BSTInsert(K start, K end, V value, Func<K, V, V> updateFactoryCallback, out IntervalTreeNode<K, V> outNode)
{
IntervalTreeNode<K, V> parent = null;
IntervalTreeNode<K, V> node = _root;
while (node != null)
{
parent = node;
int cmp = start.CompareTo(node.Start);
if (cmp < 0)
{
node = node.Left;
}
else if (cmp > 0)
{
node = node.Right;
}
else
{
outNode = node;
if (updateFactoryCallback != null)
{
// Replace
node.Value = updateFactoryCallback(start, node.Value);
int endCmp = end.CompareTo(node.End);
if (endCmp > 0)
{
node.End = end;
if (end.CompareTo(node.Max) > 0)
{
node.Max = end;
PropagateIncrease(node);
RestoreBalanceAfterInsertion(node);
}
}
else if (endCmp < 0)
{
node.End = end;
PropagateFull(node);
}
}
return false;
}
}
IntervalTreeNode<K, V> newNode = new IntervalTreeNode<K, V>(start, end, value, parent);
if (newNode.Parent == null)
{
_root = newNode;
}
else if (start.CompareTo(parent.Start) < 0)
{
parent.Left = newNode;
}
else
{
parent.Right = newNode;
}
PropagateIncrease(newNode);
_count++;
RestoreBalanceAfterInsertion(newNode);
outNode = newNode;
return true;
}
/// <summary>
/// Removes the value from the dictionary after searching for it with <paramref name="key">.
/// </summary>
/// <param name="key">Tree node to be removed</param>
private void Delete(IntervalTreeNode<K, V> nodeToDelete)
{
IntervalTreeNode<K, V> replacementNode;
if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null)
{
replacementNode = nodeToDelete;
}
else
{
replacementNode = PredecessorOf(nodeToDelete);
}
IntervalTreeNode<K, V> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode);
if (tmp != null)
{
tmp.Parent = ParentOf(replacementNode);
}
if (ParentOf(replacementNode) == null)
{
_root = tmp;
}
else if (replacementNode == LeftOf(ParentOf(replacementNode)))
{
ParentOf(replacementNode).Left = tmp;
}
else
{
ParentOf(replacementNode).Right = tmp;
}
if (replacementNode != nodeToDelete)
{
nodeToDelete.Start = replacementNode.Start;
nodeToDelete.Value = replacementNode.Value;
nodeToDelete.End = replacementNode.End;
nodeToDelete.Max = replacementNode.Max;
}
PropagateFull(replacementNode);
if (tmp != null && ColorOf(replacementNode) == Black)
{
RestoreBalanceAfterRemoval(tmp);
}
}
/// <summary>
/// Returns the node with the largest key where <paramref name="node"/> is considered the root node.
/// </summary>
/// <param name="node">Root Node</param>
/// <returns>Node with the maximum key in the tree of <paramref name="node"/></returns>
private static IntervalTreeNode<K, V> Maximum(IntervalTreeNode<K, V> node)
{
IntervalTreeNode<K, V> tmp = node;
while (tmp.Right != null)
{
tmp = tmp.Right;
}
return tmp;
}
/// <summary>
/// Finds the node whose key is immediately less than <paramref name="node"/>.
/// </summary>
/// <param name="node">Node to find the predecessor of</param>
/// <returns>Predecessor of <paramref name="node"/></returns>
private static IntervalTreeNode<K, V> PredecessorOf(IntervalTreeNode<K, V> node)
{
if (node.Left != null)
{
return Maximum(node.Left);
}
IntervalTreeNode<K, V> parent = node.Parent;
while (parent != null && node == parent.Left)
{
node = parent;
parent = parent.Parent;
}
return parent;
}
#endregion
#region Private Methods (RBL)
private void RestoreBalanceAfterRemoval(IntervalTreeNode<K, V> balanceNode)
{
IntervalTreeNode<K, V> ptr = balanceNode;
while (ptr != _root && ColorOf(ptr) == Black)
{
if (ptr == LeftOf(ParentOf(ptr)))
{
IntervalTreeNode<K, V> sibling = RightOf(ParentOf(ptr));
if (ColorOf(sibling) == Red)
{
SetColor(sibling, Black);
SetColor(ParentOf(ptr), Red);
RotateLeft(ParentOf(ptr));
sibling = RightOf(ParentOf(ptr));
}
if (ColorOf(LeftOf(sibling)) == Black && ColorOf(RightOf(sibling)) == Black)
{
SetColor(sibling, Red);
ptr = ParentOf(ptr);
}
else
{
if (ColorOf(RightOf(sibling)) == Black)
{
SetColor(LeftOf(sibling), Black);
SetColor(sibling, Red);
RotateRight(sibling);
sibling = RightOf(ParentOf(ptr));
}
SetColor(sibling, ColorOf(ParentOf(ptr)));
SetColor(ParentOf(ptr), Black);
SetColor(RightOf(sibling), Black);
RotateLeft(ParentOf(ptr));
ptr = _root;
}
}
else
{
IntervalTreeNode<K, V> sibling = LeftOf(ParentOf(ptr));
if (ColorOf(sibling) == Red)
{
SetColor(sibling, Black);
SetColor(ParentOf(ptr), Red);
RotateRight(ParentOf(ptr));
sibling = LeftOf(ParentOf(ptr));
}
if (ColorOf(RightOf(sibling)) == Black && ColorOf(LeftOf(sibling)) == Black)
{
SetColor(sibling, Red);
ptr = ParentOf(ptr);
}
else
{
if (ColorOf(LeftOf(sibling)) == Black)
{
SetColor(RightOf(sibling), Black);
SetColor(sibling, Red);
RotateLeft(sibling);
sibling = LeftOf(ParentOf(ptr));
}
SetColor(sibling, ColorOf(ParentOf(ptr)));
SetColor(ParentOf(ptr), Black);
SetColor(LeftOf(sibling), Black);
RotateRight(ParentOf(ptr));
ptr = _root;
}
}
}
SetColor(ptr, Black);
}
private void RestoreBalanceAfterInsertion(IntervalTreeNode<K, V> balanceNode)
{
SetColor(balanceNode, Red);
while (balanceNode != null && balanceNode != _root && ColorOf(ParentOf(balanceNode)) == Red)
{
if (ParentOf(balanceNode) == LeftOf(ParentOf(ParentOf(balanceNode))))
{
IntervalTreeNode<K, V> sibling = RightOf(ParentOf(ParentOf(balanceNode)));
if (ColorOf(sibling) == Red)
{
SetColor(ParentOf(balanceNode), Black);
SetColor(sibling, Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
balanceNode = ParentOf(ParentOf(balanceNode));
}
else
{
if (balanceNode == RightOf(ParentOf(balanceNode)))
{
balanceNode = ParentOf(balanceNode);
RotateLeft(balanceNode);
}
SetColor(ParentOf(balanceNode), Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
RotateRight(ParentOf(ParentOf(balanceNode)));
}
}
else
{
IntervalTreeNode<K, V> sibling = LeftOf(ParentOf(ParentOf(balanceNode)));
if (ColorOf(sibling) == Red)
{
SetColor(ParentOf(balanceNode), Black);
SetColor(sibling, Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
balanceNode = ParentOf(ParentOf(balanceNode));
}
else
{
if (balanceNode == LeftOf(ParentOf(balanceNode)))
{
balanceNode = ParentOf(balanceNode);
RotateRight(balanceNode);
}
SetColor(ParentOf(balanceNode), Black);
SetColor(ParentOf(ParentOf(balanceNode)), Red);
RotateLeft(ParentOf(ParentOf(balanceNode)));
}
}
}
SetColor(_root, Black);
}
private void RotateLeft(IntervalTreeNode<K, V> node)
{
if (node != null)
{
IntervalTreeNode<K, V> right = RightOf(node);
node.Right = LeftOf(right);
if (node.Right != null)
{
node.Right.Parent = node;
}
IntervalTreeNode<K, V> nodeParent = ParentOf(node);
right.Parent = nodeParent;
if (nodeParent == null)
{
_root = right;
}
else if (node == LeftOf(nodeParent))
{
nodeParent.Left = right;
}
else
{
nodeParent.Right = right;
}
right.Left = node;
node.Parent = right;
PropagateFull(node);
}
}
private void RotateRight(IntervalTreeNode<K, V> node)
{
if (node != null)
{
IntervalTreeNode<K, V> left = LeftOf(node);
node.Left = RightOf(left);
if (node.Left != null)
{
node.Left.Parent = node;
}
IntervalTreeNode<K, V> nodeParent = ParentOf(node);
left.Parent = nodeParent;
if (nodeParent == null)
{
_root = left;
}
else if (node == RightOf(nodeParent))
{
nodeParent.Right = left;
}
else
{
nodeParent.Left = left;
}
left.Right = node;
node.Parent = left;
PropagateFull(node);
}
}
#endregion
#region Safety-Methods
// These methods save memory by allowing us to forego sentinel nil nodes, as well as serve as protection against NullReferenceExceptions.
/// <summary>
/// Returns the color of <paramref name="node"/>, or Black if it is null.
/// </summary>
/// <param name="node">Node</param>
/// <returns>The boolean color of <paramref name="node"/>, or black if null</returns>
private static bool ColorOf(IntervalTreeNode<K, V> node)
{
return node == null || node.Color;
}
/// <summary>
/// Sets the color of <paramref name="node"/> node to <paramref name="color"/>.
/// <br></br>
/// This method does nothing if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to set the color of</param>
/// <param name="color">Color (Boolean)</param>
private static void SetColor(IntervalTreeNode<K, V> node, bool color)
{
if (node != null)
{
node.Color = color;
}
}
/// <summary>
/// This method returns the left node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the left child from</param>
/// <returns>Left child of <paramref name="node"/></returns>
private static IntervalTreeNode<K, V> LeftOf(IntervalTreeNode<K, V> node)
{
return node?.Left;
}
/// <summary>
/// This method returns the right node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the right child from</param>
/// <returns>Right child of <paramref name="node"/></returns>
private static IntervalTreeNode<K, V> RightOf(IntervalTreeNode<K, V> node)
{
return node?.Right;
}
/// <summary>
/// Returns the parent node of <paramref name="node"/>, or null if <paramref name="node"/> is null.
/// </summary>
/// <param name="node">Node to retrieve the parent from</param>
/// <returns>Parent of <paramref name="node"/></returns>
private static IntervalTreeNode<K, V> ParentOf(IntervalTreeNode<K, V> node)
{
return node?.Parent;
}
#endregion
public bool ContainsKey(K key)
{
return GetNode(key) != null;
}
public void Clear()
{
_root = null;
_count = 0;
}
}
/// <summary>
/// Represents a node in the IntervalTree which contains start and end keys of type K, and a value of generic type V.
/// </summary>
/// <typeparam name="K">Key type of the node</typeparam>
/// <typeparam name="V">Value type of the node</typeparam>
class IntervalTreeNode<K, V>
{
public bool Color = true;
public IntervalTreeNode<K, V> Left = null;
public IntervalTreeNode<K, V> Right = null;
public IntervalTreeNode<K, V> Parent = null;
/// <summary>
/// The start of the range.
/// </summary>
public K Start;
/// <summary>
/// The end of the range.
/// </summary>
public K End;
/// <summary>
/// The maximum end value of this node and all its children.
/// </summary>
public K Max;
/// <summary>
/// Value stored on this node.
/// </summary>
public V Value;
public IntervalTreeNode(K start, K end, V value, IntervalTreeNode<K, V> parent)
{
Start = start;
End = end;
Max = end;
Value = value;
Parent = parent;
}
}
}

293
Ryujinx.Memory/WindowsShared/PlaceholderList.cs
View File

@@ -1,293 +0,0 @@
using Ryujinx.Memory.Range;
using System;
using System.Diagnostics;
namespace Ryujinx.Memory.WindowsShared
{
/// <summary>
/// A specialized list used for keeping track of Windows 10's memory placeholders.
/// This is used to make splitting a large placeholder into equally small
/// granular chunks much easier, while avoiding slowdown due to a large number of
/// placeholders by coalescing adjacent granular placeholders after they are unused.
/// </summary>
class PlaceholderList
{
private class PlaceholderBlock : IRange
{
public ulong Address { get; }
public ulong Size { get; private set; }
public ulong EndAddress { get; private set; }
public bool IsGranular { get; set; }
public PlaceholderBlock(ulong id, ulong size, bool isGranular)
{
Address = id;
Size = size;
EndAddress = id + size;
IsGranular = isGranular;
}
public bool OverlapsWith(ulong address, ulong size)
{
return Address < address + size && address < EndAddress;
}
public void ExtendTo(ulong end, RangeList<PlaceholderBlock> list)
{
EndAddress = end;
Size = end - Address;
list.UpdateEndAddress(this);
}
}
private RangeList<PlaceholderBlock> _placeholders;
private PlaceholderBlock[] _foundBlocks = new PlaceholderBlock[32];
/// <summary>
/// Create a new list to manage placeholders.
/// Note that a size is measured in granular placeholders.
/// If the placeholder granularity is 65536 bytes, then a 65536 region will be covered by 1 placeholder granularity.
/// </summary>
/// <param name="size">Size measured in granular placeholders</param>
public PlaceholderList(ulong size)
{
_placeholders = new RangeList<PlaceholderBlock>();
_placeholders.Add(new PlaceholderBlock(0, size, false));
}
/// <summary>
/// Ensure that the given range of placeholders is granular.
/// </summary>
/// <param name="id">Start of the range, measured in granular placeholders</param>
/// <param name="size">Size of the range, measured in granular placeholders</param>
/// <param name="splitPlaceholderCallback">Callback function to run when splitting placeholders, calls with (start, middle)</param>
public void EnsurePlaceholders(ulong id, ulong size, Action<ulong, ulong> splitPlaceholderCallback)
{
// Search 1 before and after the placeholders, as we may need to expand/join granular regions surrounding the requested area.
ulong endId = id + size;
ulong searchStartId = id == 0 ? 0 : (id - 1);
int blockCount = _placeholders.FindOverlapsNonOverlapping(searchStartId, (endId - searchStartId) + 1, ref _foundBlocks);
PlaceholderBlock first = _foundBlocks[0];
PlaceholderBlock last = _foundBlocks[blockCount - 1];
bool overlapStart = first.EndAddress >= id && id != 0;
bool overlapEnd = last.Address <= endId;
for (int i = 0; i < blockCount; i++)
{
// Go through all non-granular blocks in the range and create placeholders.
PlaceholderBlock block = _foundBlocks[i];
if (block.Address <= id && block.EndAddress >= endId && block.IsGranular)
{
return; // The region we're searching for is already granular.
}
if (!block.IsGranular)
{
ulong placeholderStart = Math.Max(block.Address, id);
ulong placeholderEnd = Math.Min(block.EndAddress - 1, endId);
if (placeholderStart != block.Address && placeholderStart != block.EndAddress)
{
splitPlaceholderCallback(block.Address, placeholderStart - block.Address);
}
for (ulong j = placeholderStart; j < placeholderEnd; j++)
{
splitPlaceholderCallback(j, 1);
}
}
if (!((block == first && overlapStart) || (block == last && overlapEnd)))
{
// Remove blocks that will be replaced
_placeholders.Remove(block);
}
}
if (overlapEnd)
{
if (!(first == last && overlapStart))
{
_placeholders.Remove(last);
}
if (last.IsGranular)
{
endId = last.EndAddress;
}
else if (last.EndAddress != endId)
{
_placeholders.Add(new PlaceholderBlock(endId, last.EndAddress - endId, false));
}
}
if (overlapStart && first.IsGranular)
{
first.ExtendTo(endId, _placeholders);
}
else
{
if (overlapStart)
{
first.ExtendTo(id, _placeholders);
}
_placeholders.Add(new PlaceholderBlock(id, endId - id, true));
}
ValidateList();
}
/// <summary>
/// Coalesces placeholders in a given region, as they are not being used.
/// This assumes that the region only contains placeholders - all views and allocations must have been replaced with placeholders.
/// </summary>
/// <param name="id">Start of the range, measured in granular placeholders</param>
/// <param name="size">Size of the range, measured in granular placeholders</param>
/// <param name="coalescePlaceholderCallback">Callback function to run when coalescing two placeholders, calls with (start, end)</param>
public void RemovePlaceholders(ulong id, ulong size, Action<ulong, ulong> coalescePlaceholderCallback)
{
ulong endId = id + size;
int blockCount = _placeholders.FindOverlapsNonOverlapping(id, size, ref _foundBlocks);
PlaceholderBlock first = _foundBlocks[0];
PlaceholderBlock last = _foundBlocks[blockCount - 1];
// All granular blocks must have non-granular blocks surrounding them, unless they start at 0.
// We must extend the non-granular blocks into the granular ones. This does mean that we need to search twice.
if (first.IsGranular || last.IsGranular)
{
ulong surroundStart = Math.Max(0, (first.IsGranular && first.Address != 0) ? first.Address - 1 : id);
blockCount = _placeholders.FindOverlapsNonOverlapping(
surroundStart,
(last.IsGranular ? last.EndAddress + 1 : endId) - surroundStart,
ref _foundBlocks);
first = _foundBlocks[0];
last = _foundBlocks[blockCount - 1];
}
if (first == last)
{
return; // Already coalesced.
}
PlaceholderBlock extendBlock = id == 0 ? null : first;
bool newBlock = false;
for (int i = extendBlock == null ? 0 : 1; i < blockCount; i++)
{
// Go through all granular blocks in the range and extend placeholders.
PlaceholderBlock block = _foundBlocks[i];
ulong blockEnd = block.EndAddress;
ulong extendFrom;
ulong extent = Math.Min(blockEnd, endId);
if (block.Address < id && blockEnd > id)
{
block.ExtendTo(id, _placeholders);
extendBlock = null;
}
else
{
_placeholders.Remove(block);
}
if (extendBlock == null)
{
extendFrom = id;
extendBlock = new PlaceholderBlock(id, extent - id, false);
_placeholders.Add(extendBlock);
if (blockEnd > extent)
{
_placeholders.Add(new PlaceholderBlock(extent, blockEnd - extent, true));
// Skip the next non-granular block, and extend from that into the granular block afterwards.
// (assuming that one is still in the requested range)
if (i + 1 < blockCount)
{
extendBlock = _foundBlocks[i + 1];
}
i++;
}
newBlock = true;
}
else
{
extendFrom = extendBlock.Address;
extendBlock.ExtendTo(block.IsGranular ? extent : block.EndAddress, _placeholders);
}
if (block.IsGranular)
{
ulong placeholderStart = Math.Max(block.Address, id);
ulong placeholderEnd = extent;
if (newBlock)
{
placeholderStart++;
newBlock = false;
}
for (ulong j = placeholderStart; j < placeholderEnd; j++)
{
coalescePlaceholderCallback(extendFrom, (j + 1) - extendFrom);
}
if (extent < block.EndAddress)
{
_placeholders.Add(new PlaceholderBlock(placeholderEnd, block.EndAddress - placeholderEnd, true));
ValidateList();
return;
}
}
else
{
coalescePlaceholderCallback(extendFrom, block.EndAddress - extendFrom);
}
}
ValidateList();
}
/// <summary>
/// Ensure that the placeholder list is valid.
/// A valid list should not have any gaps between the placeholders,
/// and there may be no placehonders with the same IsGranular value next to each other.
/// </summary>
[Conditional("DEBUG")]
private void ValidateList()
{
bool isGranular = false;
bool first = true;
ulong lastAddress = 0;
foreach (var placeholder in _placeholders)
{
if (placeholder.Address != lastAddress)
{
throw new InvalidOperationException("Gap in placeholder list.");
}
if (isGranular == placeholder.IsGranular && !first)
{
throw new InvalidOperationException("Placeholder list not alternating.");
}
first = false;
isGranular = placeholder.IsGranular;
lastAddress = placeholder.EndAddress;
}
}
}
}

633
Ryujinx.Memory/WindowsShared/PlaceholderManager.cs Normal file
View File

@@ -0,0 +1,633 @@
using System;
using System.Diagnostics;
using System.Threading;
namespace Ryujinx.Memory.WindowsShared
{
/// <summary>
/// Windows memory placeholder manager.
/// </summary>
class PlaceholderManager
{
private const ulong MinimumPageSize = 0x1000;
[ThreadStatic]
private static int _threadLocalPartialUnmapsCount;
private readonly IntervalTree<ulong, ulong> _mappings;
private readonly IntervalTree<ulong, MemoryPermission> _protections;
private readonly ReaderWriterLock _partialUnmapLock;
private int _partialUnmapsCount;
/// <summary>
/// Creates a new instance of the Windows memory placeholder manager.
/// </summary>
public PlaceholderManager()
{
_mappings = new IntervalTree<ulong, ulong>();
_protections = new IntervalTree<ulong, MemoryPermission>();
_partialUnmapLock = new ReaderWriterLock();
}
/// <summary>
/// Reserves a range of the address space to be later mapped as shared memory views.
/// </summary>
/// <param name="address">Start address of the region to reserve</param>
/// <param name="size">Size in bytes of the region to reserve</param>
public void ReserveRange(ulong address, ulong size)
{
lock (_mappings)
{
_mappings.Add(address, address + size, ulong.MaxValue);
}
}
/// <summary>
/// Maps a shared memory view on a previously reserved memory region.
/// </summary>
/// <param name="sharedMemory">Shared memory that will be the backing storage for the view</param>
/// <param name="srcOffset">Offset in the shared memory to map</param>
/// <param name="location">Address to map the view into</param>
/// <param name="size">Size of the view in bytes</param>
public void MapView(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size)
{
_partialUnmapLock.AcquireReaderLock(Timeout.Infinite);
try
{
UnmapViewInternal(sharedMemory, location, size);
MapViewInternal(sharedMemory, srcOffset, location, size);
}
finally
{
_partialUnmapLock.ReleaseReaderLock();
}
}
/// <summary>
/// Maps a shared memory view on a previously reserved memory region.
/// </summary>
/// <param name="sharedMemory">Shared memory that will be the backing storage for the view</param>
/// <param name="srcOffset">Offset in the shared memory to map</param>
/// <param name="location">Address to map the view into</param>
/// <param name="size">Size of the view in bytes</param>
/// <exception cref="WindowsApiException">Thrown when the Windows API returns an error mapping the memory</exception>
private void MapViewInternal(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size)
{
SplitForMap((ulong)location, (ulong)size, srcOffset);
var ptr = WindowsApi.MapViewOfFile3(
sharedMemory,
WindowsApi.CurrentProcessHandle,
location,
srcOffset,
size,
0x4000,
MemoryProtection.ReadWrite,
IntPtr.Zero,
0);
if (ptr == IntPtr.Zero)
{
throw new WindowsApiException("MapViewOfFile3");
}
}
/// <summary>
/// Splits a larger placeholder, slicing at the start and end address, for a new memory mapping.
/// </summary>
/// <param name="address">Address to split</param>
/// <param name="size">Size of the new region</param>
/// <param name="backingOffset">Offset in the shared memory that will be mapped</param>
private void SplitForMap(ulong address, ulong size, ulong backingOffset)
{
ulong endAddress = address + size;
var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>();
lock (_mappings)
{
int count = _mappings.Get(address, endAddress, ref overlaps);
Debug.Assert(count == 1);
Debug.Assert(!IsMapped(overlaps[0].Value));
var overlap = overlaps[0];
// Tree operations might modify the node start/end values, so save a copy before we modify the tree.
ulong overlapStart = overlap.Start;
ulong overlapEnd = overlap.End;
ulong overlapValue = overlap.Value;
_mappings.Remove(overlap);
bool overlapStartsBefore = overlapStart < address;
bool overlapEndsAfter = overlapEnd > endAddress;
if (overlapStartsBefore && overlapEndsAfter)
{
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)size,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(overlapStart, address, overlapValue);
_mappings.Add(endAddress, overlapEnd, AddBackingOffset(overlapValue, endAddress - overlapStart));
}
else if (overlapStartsBefore)
{
ulong overlappedSize = overlapEnd - address;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)overlappedSize,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(overlapStart, address, overlapValue);
}
else if (overlapEndsAfter)
{
ulong overlappedSize = endAddress - overlapStart;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)overlapStart,
(IntPtr)overlappedSize,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(endAddress, overlapEnd, AddBackingOffset(overlapValue, overlappedSize));
}
_mappings.Add(address, endAddress, backingOffset);
}
}
/// <summary>
/// Unmaps a view that has been previously mapped with <see cref="MapView"/>.
/// </summary>
/// <remarks>
/// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be
/// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped.
/// </remarks>
/// <param name="sharedMemory">Shared memory that the view being unmapped belongs to</param>
/// <param name="location">Address to unmap</param>
/// <param name="size">Size of the region to unmap in bytes</param>
public void UnmapView(IntPtr sharedMemory, IntPtr location, IntPtr size)
{
_partialUnmapLock.AcquireReaderLock(Timeout.Infinite);
try
{
UnmapViewInternal(sharedMemory, location, size);
}
finally
{
_partialUnmapLock.ReleaseReaderLock();
}
}
/// <summary>
/// Unmaps a view that has been previously mapped with <see cref="MapView"/>.
/// </summary>
/// <remarks>
/// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be
/// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped.
/// </remarks>
/// <param name="sharedMemory">Shared memory that the view being unmapped belongs to</param>
/// <param name="location">Address to unmap</param>
/// <param name="size">Size of the region to unmap in bytes</param>
/// <exception cref="WindowsApiException">Thrown when the Windows API returns an error unmapping or remapping the memory</exception>
private void UnmapViewInternal(IntPtr sharedMemory, IntPtr location, IntPtr size)
{
ulong startAddress = (ulong)location;
ulong unmapSize = (ulong)size;
ulong endAddress = startAddress + unmapSize;
var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>();
int count = 0;
lock (_mappings)
{
count = _mappings.Get(startAddress, endAddress, ref overlaps);
}
for (int index = 0; index < count; index++)
{
var overlap = overlaps[index];
if (IsMapped(overlap.Value))
{
if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2))
{
throw new WindowsApiException("UnmapViewOfFile2");
}
// Tree operations might modify the node start/end values, so save a copy before we modify the tree.
ulong overlapStart = overlap.Start;
ulong overlapEnd = overlap.End;
ulong overlapValue = overlap.Value;
_mappings.Remove(overlap);
_mappings.Add(overlapStart, overlapEnd, ulong.MaxValue);
bool overlapStartsBefore = overlapStart < startAddress;
bool overlapEndsAfter = overlapEnd > endAddress;
if (overlapStartsBefore || overlapEndsAfter)
{
// If the overlap extends beyond the region we are unmapping,
// then we need to re-map the regions that are supposed to remain mapped.
// This is necessary because Windows does not support partial view unmaps.
// That is, you can only fully unmap a view that was previously mapped, you can't just unmap a chunck of it.
LockCookie lockCookie = _partialUnmapLock.UpgradeToWriterLock(Timeout.Infinite);
_partialUnmapsCount++;
if (overlapStartsBefore)
{
ulong remapSize = startAddress - overlapStart;
MapViewInternal(sharedMemory, overlapValue, (IntPtr)overlapStart, (IntPtr)remapSize);
RestoreRangeProtection(overlapStart, remapSize);
}
if (overlapEndsAfter)
{
ulong overlappedSize = endAddress - overlapStart;
ulong remapBackingOffset = overlapValue + overlappedSize;
ulong remapAddress = overlapStart + overlappedSize;
ulong remapSize = overlapEnd - endAddress;
MapViewInternal(sharedMemory, remapBackingOffset, (IntPtr)remapAddress, (IntPtr)remapSize);
RestoreRangeProtection(remapAddress, remapSize);
}
_partialUnmapLock.DowngradeFromWriterLock(ref lockCookie);
}
}
}
CoalesceForUnmap(startAddress, unmapSize);
RemoveProtection(startAddress, unmapSize);
}
/// <summary>
/// Coalesces adjacent placeholders after unmap.
/// </summary>
/// <param name="address">Address of the region that was unmapped</param>
/// <param name="size">Size of the region that was unmapped in bytes</param>
private void CoalesceForUnmap(ulong address, ulong size)
{
ulong endAddress = address + size;
var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>();
int unmappedCount = 0;
lock (_mappings)
{
int count = _mappings.Get(address - MinimumPageSize, endAddress + MinimumPageSize, ref overlaps);
if (count < 2)
{
// Nothing to coalesce if we only have 1 or no overlaps.
return;
}
for (int index = 0; index < count; index++)
{
var overlap = overlaps[index];
if (!IsMapped(overlap.Value))
{
if (address > overlap.Start)
{
address = overlap.Start;
}
if (endAddress < overlap.End)
{
endAddress = overlap.End;
}
_mappings.Remove(overlap);
unmappedCount++;
}
}
_mappings.Add(address, endAddress, ulong.MaxValue);
}
if (unmappedCount > 1)
{
size = endAddress - address;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)size,
AllocationType.Release | AllocationType.CoalescePlaceholders));
}
}
/// <summary>
/// Reprotects a region of memory that has been mapped.
/// </summary>
/// <param name="address">Address of the region to reprotect</param>
/// <param name="size">Size of the region to reprotect in bytes</param>
/// <param name="permission">New permissions</param>
/// <returns>True if the reprotection was successful, false otherwise</returns>
public bool ReprotectView(IntPtr address, IntPtr size, MemoryPermission permission)
{
_partialUnmapLock.AcquireReaderLock(Timeout.Infinite);
try
{
return ReprotectViewInternal(address, size, permission, false);
}
finally
{
_partialUnmapLock.ReleaseReaderLock();
}
}
/// <summary>
/// Reprotects a region of memory that has been mapped.
/// </summary>
/// <param name="address">Address of the region to reprotect</param>
/// <param name="size">Size of the region to reprotect in bytes</param>
/// <param name="permission">New permissions</param>
/// <param name="throwOnError">Throw an exception instead of returning an error if the operation fails</param>
/// <returns>True if the reprotection was successful or if <paramref name="throwOnError"/> is true, false otherwise</returns>
/// <exception cref="WindowsApiException">If <paramref name="throwOnError"/> is true, it is thrown when the Windows API returns an error reprotecting the memory</exception>
private bool ReprotectViewInternal(IntPtr address, IntPtr size, MemoryPermission permission, bool throwOnError)
{
ulong reprotectAddress = (ulong)address;
ulong reprotectSize = (ulong)size;
ulong endAddress = reprotectAddress + reprotectSize;
var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>();
int count = 0;
lock (_mappings)
{
count = _mappings.Get(reprotectAddress, endAddress, ref overlaps);
}
bool success = true;
for (int index = 0; index < count; index++)
{
var overlap = overlaps[index];
ulong mappedAddress = overlap.Start;
ulong mappedSize = overlap.End - overlap.Start;
if (mappedAddress < reprotectAddress)
{
ulong delta = reprotectAddress - mappedAddress;
mappedAddress = reprotectAddress;
mappedSize -= delta;
}
ulong mappedEndAddress = mappedAddress + mappedSize;
if (mappedEndAddress > endAddress)
{
ulong delta = mappedEndAddress - endAddress;
mappedSize -= delta;
}
if (!WindowsApi.VirtualProtect((IntPtr)mappedAddress, (IntPtr)mappedSize, WindowsApi.GetProtection(permission), out _))
{
if (throwOnError)
{
throw new WindowsApiException("VirtualProtect");
}
success = false;
}
// We only keep track of "non-standard" protections,
// that is, everything that is not just RW (which is the default when views are mapped).
if (permission == MemoryPermission.ReadAndWrite)
{
RemoveProtection(mappedAddress, mappedSize);
}
else
{
AddProtection(mappedAddress, mappedSize, permission);
}
}
return success;
}
/// <summary>
/// Checks the result of a VirtualFree operation, throwing if needed.
/// </summary>
/// <param name="success">Operation result</param>
/// <exception cref="WindowsApiException">Thrown if <paramref name="success"/> is false</exception>
private static void CheckFreeResult(bool success)
{
if (!success)
{
throw new WindowsApiException("VirtualFree");
}
}
/// <summary>
/// Adds an offset to a backing offset. This will do nothing if the backing offset is the special "unmapped" value.
/// </summary>
/// <param name="backingOffset">Backing offset</param>
/// <param name="offset">Offset to be added</param>
/// <returns>Added offset or just <paramref name="backingOffset"/> if the region is unmapped</returns>
private static ulong AddBackingOffset(ulong backingOffset, ulong offset)
{
if (backingOffset == ulong.MaxValue)
{
return backingOffset;
}
return backingOffset + offset;
}
/// <summary>
/// Checks if a region is unmapped.
/// </summary>
/// <param name="backingOffset">Backing offset to check</param>
/// <returns>True if the backing offset is the special "unmapped" value, false otherwise</returns>
private static bool IsMapped(ulong backingOffset)
{
return backingOffset != ulong.MaxValue;
}
/// <summary>
/// Adds a protection to the list of protections.
/// </summary>
/// <param name="address">Address of the protected region</param>
/// <param name="size">Size of the protected region in bytes</param>
/// <param name="permission">Memory permissions of the region</param>
private void AddProtection(ulong address, ulong size, MemoryPermission permission)
{
ulong endAddress = address + size;
var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>();
int count = 0;
lock (_protections)
{
count = _protections.Get(address, endAddress, ref overlaps);
Debug.Assert(count > 0);
if (count == 1 &&
overlaps[0].Start <= address &&
overlaps[0].End >= endAddress &&
overlaps[0].Value == permission)
{
return;
}
ulong startAddress = address;
for (int index = 0; index < count; index++)
{
var protection = overlaps[index];
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
MemoryPermission protPermission = protection.Value;
_protections.Remove(protection);
if (protection.Value == permission)
{
if (startAddress > protAddress)
{
startAddress = protAddress;
}
if (endAddress < protEndAddress)
{
endAddress = protEndAddress;
}
}
else
{
if (startAddress > protAddress)
{
_protections.Add(protAddress, startAddress, protPermission);
}
if (endAddress < protEndAddress)
{
_protections.Add(endAddress, protEndAddress, protPermission);
}
}
}
_protections.Add(startAddress, endAddress, permission);
}
}
/// <summary>
/// Removes protection from the list of protections.
/// </summary>
/// <param name="address">Address of the protected region</param>
/// <param name="size">Size of the protected region in bytes</param>
private void RemoveProtection(ulong address, ulong size)
{
ulong endAddress = address + size;
var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>();
int count = 0;
lock (_protections)
{
count = _protections.Get(address, endAddress, ref overlaps);
for (int index = 0; index < count; index++)
{
var protection = overlaps[index];
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
MemoryPermission protPermission = protection.Value;
_protections.Remove(protection);
if (address > protAddress)
{
_protections.Add(protAddress, address, protPermission);
}
if (endAddress < protEndAddress)
{
_protections.Add(endAddress, protEndAddress, protPermission);
}
}
}
}
/// <summary>
/// Restores the protection of a given memory region that was remapped, using the protections list.
/// </summary>
/// <param name="address">Address of the remapped region</param>
/// <param name="size">Size of the remapped region in bytes</param>
private void RestoreRangeProtection(ulong address, ulong size)
{
ulong endAddress = address + size;
var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>();
int count = 0;
lock (_protections)
{
count = _protections.Get(address, endAddress, ref overlaps);
}
ulong startAddress = address;
for (int index = 0; index < count; index++)
{
var protection = overlaps[index];
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
if (protAddress < address)
{
protAddress = address;
}
if (protEndAddress > endAddress)
{
protEndAddress = endAddress;
}
ReprotectViewInternal((IntPtr)protAddress, (IntPtr)(protEndAddress - protAddress), protection.Value, true);
}
}
/// <summary>
/// Checks if an access violation handler should retry execution due to a fault caused by partial unmap.
/// </summary>
/// <remarks>
/// Due to Windows limitations, <see cref="UnmapView"/> might need to unmap more memory than requested.
/// The additional memory that was unmapped is later remapped, however this leaves a time gap where the
/// memory might be accessed but is unmapped. Users of the API must compensate for that by catching the
/// access violation and retrying if it happened between the unmap and remap operation.
/// This method can be used to decide if retrying in such cases is necessary or not.
/// </remarks>
/// <returns>True if execution should be retried, false otherwise</returns>
public bool RetryFromAccessViolation()
{
_partialUnmapLock.AcquireReaderLock(Timeout.Infinite);
bool retry = _threadLocalPartialUnmapsCount != _partialUnmapsCount;
if (retry)
{
_threadLocalPartialUnmapsCount = _partialUnmapsCount;
}
_partialUnmapLock.ReleaseReaderLock();
return retry;
}
}
}

93
Ryujinx.Memory/WindowsShared/WindowsApi.cs Normal file
View File

@@ -0,0 +1,93 @@
using System;
using System.Runtime.InteropServices;
namespace Ryujinx.Memory.WindowsShared
{
static class WindowsApi
{
public static readonly IntPtr InvalidHandleValue = new IntPtr(-1);
public static readonly IntPtr CurrentProcessHandle = new IntPtr(-1);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern IntPtr VirtualAlloc(
IntPtr lpAddress,
IntPtr dwSize,
AllocationType flAllocationType,
MemoryProtection flProtect);
[DllImport("KernelBase.dll", SetLastError = true)]
public static extern IntPtr VirtualAlloc2(
IntPtr process,
IntPtr lpAddress,
IntPtr dwSize,
AllocationType flAllocationType,
MemoryProtection flProtect,
IntPtr extendedParameters,
ulong parameterCount);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool VirtualProtect(
IntPtr lpAddress,
IntPtr dwSize,
MemoryProtection flNewProtect,
out MemoryProtection lpflOldProtect);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool VirtualFree(IntPtr lpAddress, IntPtr dwSize, AllocationType dwFreeType);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern IntPtr CreateFileMapping(
IntPtr hFile,
IntPtr lpFileMappingAttributes,
FileMapProtection flProtect,
uint dwMaximumSizeHigh,
uint dwMaximumSizeLow,
[MarshalAs(UnmanagedType.LPWStr)] string lpName);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool CloseHandle(IntPtr hObject);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern IntPtr MapViewOfFile(
IntPtr hFileMappingObject,
uint dwDesiredAccess,
uint dwFileOffsetHigh,
uint dwFileOffsetLow,
IntPtr dwNumberOfBytesToMap);
[DllImport("KernelBase.dll", SetLastError = true)]
public static extern IntPtr MapViewOfFile3(
IntPtr hFileMappingObject,
IntPtr process,
IntPtr baseAddress,
ulong offset,
IntPtr dwNumberOfBytesToMap,
ulong allocationType,
MemoryProtection dwDesiredAccess,
IntPtr extendedParameters,
ulong parameterCount);
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool UnmapViewOfFile(IntPtr lpBaseAddress);
[DllImport("KernelBase.dll", SetLastError = true)]
public static extern bool UnmapViewOfFile2(IntPtr process, IntPtr lpBaseAddress, ulong unmapFlags);
[DllImport("kernel32.dll")]
public static extern uint GetLastError();
public static MemoryProtection GetProtection(MemoryPermission permission)
{
return permission switch
{
MemoryPermission.None => MemoryProtection.NoAccess,
MemoryPermission.Read => MemoryProtection.ReadOnly,
MemoryPermission.ReadAndWrite => MemoryProtection.ReadWrite,
MemoryPermission.ReadAndExecute => MemoryProtection.ExecuteRead,
MemoryPermission.ReadWriteExecute => MemoryProtection.ExecuteReadWrite,
MemoryPermission.Execute => MemoryProtection.Execute,
_ => throw new MemoryProtectionException(permission)
};
}
}
}

24
Ryujinx.Memory/WindowsShared/WindowsApiException.cs Normal file
View File

@@ -0,0 +1,24 @@
using System;
namespace Ryujinx.Memory.WindowsShared
{
class WindowsApiException : Exception
{
public WindowsApiException()
{
}
public WindowsApiException(string functionName) : base(CreateMessage(functionName))
{
}
public WindowsApiException(string functionName, Exception inner) : base(CreateMessage(functionName), inner)
{
}
private static string CreateMessage(string functionName)
{
return $"{functionName} returned error code 0x{WindowsApi.GetLastError():X}.";
}
}
}