SabreTools/SabreTools.IO
1
0
Fork 0
mirror of https://github.com/SabreTools/SabreTools.IO.git synced 2026-07-08 17:57:02 +00:00
Code Issues 1 Packages Projects Releases 20 Wiki Activity

Port RVZ-Pack compression code from Serialization

Browse Source
This commit is contained in:
Matt Nadareski
2026-05-12 21:15:05 -04:00
parent 0174e9f2cd
commit e7c75e62b2
7 changed files with 698 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
SabreTools.IO.Compression/RVZPack/ChunkResult.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Result of packing a single chunk: compressed payload and its logical size.
/// </summary>
public struct ChunkResult
{
/// <summary>
/// Packed payload, or null if the chunk contains no junk.
/// </summary>
public byte[]? Packed;
/// <summary>
/// Number of bytes the decompressor needs to consume from <see cref="Packed"/>.
/// </summary>
public uint RvzPackedSize;
}
}

280
SabreTools.IO.Compression/RVZPack/Compressor.cs Normal file
View File

@@ -0,0 +1,280 @@
using System;
using System.Collections.Generic;
using System.IO;
using SabreTools.Numerics.Extensions;
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Encodes disc data into RVZ-Pack format by replacing predictable LFG
/// (Lagged Fibonacci Generator) junk regions with compact seed descriptors.
///
/// This is the exact inverse of <see cref="Decompressor"/> and mirrors
/// Dolphin's RVZPack() in WIABlob.cpp.
///
/// Two-phase algorithm:
/// <list type="number">
/// <item>Phase 1 (<see cref="ScanForJunk"/>): walk the buffer, identify LFG
/// junk regions, build a map keyed by end-offset.</item>
/// <item>Phase 2 (<see cref="EmitChunk"/>): for each chunk, use the map to
/// emit alternating real-data and junk-seed segments.</item>
/// </list>
/// </summary>
public static class Compressor
{
/// <remarks>
/// 17 u32s × 4 bytes = 68 bytes — minimum size to record a seed
/// </remarks>
private const int SeedSizeBytes = LaggedFibonacciGenerator.SEED_SIZE * 4;
/// <summary>
/// RVZ-pack a single chunk.
/// Returns null if the chunk contains no junk (write raw instead).
/// <paramref name="rvzPackedSize"/> is the number of bytes actually needed
/// by the decompressor (may be &lt; packed.Length due to alignment).
/// </summary>
/// <param name="data">Source buffer</param>
/// <param name="dataOffset">Start of data within <paramref name="data"/></param>
public static byte[]? Pack(byte[] data,
int dataOffset,
int size,
long discDataOffset,
out uint rvzPackedSize,
FileSystemTableReader? fst = null)
{
rvzPackedSize = 0;
if (size <= 0)
return null;
var junkInfo = ScanForJunk(data, dataOffset, size, discDataOffset, fst);
if (junkInfo.Count == 0)
return null;
ChunkResult result = EmitChunk(data, dataOffset, 0L, size, junkInfo);
rvzPackedSize = result.RvzPackedSize;
return result.Packed;
}
/// <summary>
/// RVZ-pack a multi-chunk buffer (e.g. a full 2 MiB Wii group).
/// Performs one Phase-1 scan over the entire buffer, then calls
/// <see cref="EmitChunk"/> per chunk.
/// </summary>
/// <param name="data">Source buffer</param>
/// <param name="dataOffset">Start of data within <paramref name="data"/></param>
/// <param name="totalSize">Total number of bytes to process</param>
/// <param name="bytesPerChunk">Size of each individual chunk</param>
/// <param name="numChunks">Number of chunks</param>
/// <param name="discDataOffset">Disc-partition byte offset of the first byte</param>
/// <param name="fst">Optional FST for file-boundary optimisation</param>
/// <returns>
/// One <see cref="ChunkResult"/> per chunk;
/// Packed == null means the chunk has no junk and should be written raw.
/// </returns>
public static ChunkResult[] PackGroup(byte[] data,
int dataOffset,
int totalSize,
int bytesPerChunk,
int numChunks,
long discDataOffset,
FileSystemTableReader? fst = null)
{
var junkInfo = ScanForJunk(data, dataOffset, totalSize, discDataOffset, fst);
var result = new ChunkResult[numChunks];
for (int i = 0; i < numChunks; i++)
{
long chunkStart = (long)i * bytesPerChunk;
long chunkEnd = Math.Min(chunkStart + bytesPerChunk, totalSize);
result[i] = EmitChunk(data, dataOffset, chunkStart, chunkEnd, junkInfo);
}
return result;
}
/// <summary>
/// Scan buffer for junk regions
/// </summary>
/// <param name="data">Source buffer</param>
/// <param name="dataOffset">Start of data within <paramref name="data"/></param>
/// <param name="totalSize">Total number of bytes to process</param>
/// <param name="discDataOffset">Disc-partition byte offset of the first byte</param>
/// <param name="fst">Optional FST for file-boundary optimisation</param>
/// <returns></returns>
private static SortedDictionary<long, JunkRegion> ScanForJunk(
byte[] data,
int dataOffset,
int totalSize,
long discDataOffset,
FileSystemTableReader? fst)
{
var junkInfo = new SortedDictionary<long, JunkRegion>();
long position = 0;
long dataOff = discDataOffset;
while (position < totalSize)
{
// Step 1: count and advance past leading zeros
long zeroes = 0;
while ((position + zeroes) < totalSize && data[dataOffset + position + zeroes] == 0)
{
zeroes++;
}
if (zeroes > SeedSizeBytes)
{
junkInfo[position + zeroes] = new JunkRegion
{
StartOffset = position,
Seed = new uint[LaggedFibonacciGenerator.SEED_SIZE]
};
}
position += zeroes;
dataOff += zeroes;
if (position >= totalSize)
break;
// Step 2: compute aligned read window (next 0x8000 boundary)
long nextBoundary = AlignUp(dataOff + 1, 0x8000);
long bytesToRead = Math.Min(nextBoundary - dataOff, totalSize - position);
int dataOffMod = (int)(dataOff % 0x8000);
// Step 3: ALWAYS call GetSeed unconditionally — no FST pre-check
var seed = new uint[LaggedFibonacciGenerator.SEED_SIZE];
int reconstructed = LaggedFibonacciGenerator.GetSeed(data, (int)(dataOffset + position), (int)bytesToRead, dataOffMod, seed);
if (reconstructed > 0)
{
junkInfo[position + reconstructed] = new JunkRegion
{
StartOffset = position,
Seed = seed
};
}
// Step 4: FST skip AFTER GetSeed
if (fst is not null)
{
long queryOff = dataOff + reconstructed;
FileEntry? fileInfo = fst.FindFileInfo(queryOff);
if (fileInfo is not null)
{
long fileEnd = fileInfo.Value.FileEnd;
if (fileEnd < (dataOff + bytesToRead))
{
position += fileEnd - dataOff;
dataOff = fileEnd;
continue;
}
}
}
// Step 5: normal advance by block window
position += bytesToRead;
dataOff += bytesToRead;
}
return junkInfo;
}
/// <summary>
/// Emit packed segments for a single chunk
/// </summary>
/// <param name="data">Source buffer</param>
/// <param name="dataOffset">Start of data within <paramref name="data"/></param>
/// <param name="chunkStart"></param>
/// <param name="chunkEnd"></param>
/// <param name="junkInfo"></param>
/// <returns></returns>
private static ChunkResult EmitChunk(
byte[] data,
int dataOffset,
long chunkStart,
long chunkEnd,
SortedDictionary<long, JunkRegion> junkInfo)
{
long currentOffset = chunkStart;
bool firstIteration = true;
var output = new MemoryStream((int)(chunkEnd - chunkStart));
uint packedSize = 0;
while (currentOffset < chunkEnd)
{
long remaining = chunkEnd - currentOffset;
long nextJunkStart = chunkEnd;
long nextJunkEnd = chunkEnd;
uint[]? junkSeed = null;
if (remaining > SeedSizeBytes)
{
foreach (var kvp in junkInfo)
{
// Dolphin Phase-2 condition:
// key > currentOffset + SEED_SIZE_BYTES AND
// startOffset + SEED_SIZE_BYTES < chunkEnd
if ((kvp.Key > (currentOffset + SeedSizeBytes)) && ((kvp.Value.StartOffset + SeedSizeBytes) < chunkEnd))
{
nextJunkStart = Math.Max(currentOffset, kvp.Value.StartOffset);
nextJunkEnd = Math.Min(chunkEnd, kvp.Key);
junkSeed = kvp.Value.Seed;
break;
}
}
}
// On the first iteration, bail out if there is no junk in this chunk
if (firstIteration)
{
if (nextJunkStart == chunkEnd)
return new ChunkResult { Packed = null, RvzPackedSize = 0 };
firstIteration = false;
}
// Emit real-data segment before the junk region
long nonJunkBytes = nextJunkStart - currentOffset;
if (nonJunkBytes > 0)
{
output.WriteBigEndian((uint)nonJunkBytes);
output.Write(data, (int)(dataOffset + currentOffset), (int)nonJunkBytes);
packedSize += 4 + (uint)nonJunkBytes;
currentOffset += nonJunkBytes;
}
// Emit junk-seed segment
long junkBytes = nextJunkEnd - currentOffset;
if (junkBytes > 0 && junkSeed is not null)
{
output.WriteBigEndian(0x80000000u | (uint)junkBytes);
byte[] seedBytes = new byte[SeedSizeBytes];
Buffer.BlockCopy(junkSeed, 0, seedBytes, 0, SeedSizeBytes);
output.Write(seedBytes, 0, SeedSizeBytes);
packedSize += 4 + (uint)SeedSizeBytes;
currentOffset += junkBytes;
}
if (junkSeed == null)
break;
}
return new ChunkResult { Packed = output.ToArray(), RvzPackedSize = packedSize };
}
#region Helpers
/// <summary>
/// Align a value to a boundary
/// </summary>
/// TODO: Figure out how to use buffer alignment helpers here
private static long AlignUp(long value, long alignment) => (value + alignment - 1) & ~(alignment - 1);
#endregion
}
}

142
SabreTools.IO.Compression/RVZPack/Decompressor.cs Normal file
View File

@@ -0,0 +1,142 @@
using System;
using SabreTools.Numerics.Extensions;
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Decompressor for RVZ packed format.
/// RVZ uses run-length encoding to store real data and junk data efficiently:
/// - Real data: size (4 bytes) + data bytes
/// - Junk data: size with high bit set (4 bytes) + 68-byte seed -> regenerate using LFG
/// </summary>
public class Decompressor
{
/// <summary>
/// Packed RVZ data to decompress
/// </summary>
private readonly byte[] _packedData;
/// <summary>
/// Expected size of packed data (for validation)
/// </summary>
private readonly uint _rvzPackedSize;
/// <summary>
/// Offset in the virtual disc, used by <see cref="_lfg"/>
/// </summary>
private long _dataOffset;
/// <summary>
/// Lagged Fibonacci generator for junk processing
/// </summary>
private readonly LaggedFibonacciGenerator _lfg;
/// <summary>
/// Current position into <see cref="_packedData"/>
/// </summary>
private int _inPosition = 0;
/// <summary>
/// The number of writable bytes in the current segment
/// </summary>
private uint _currentSize = 0;
/// <summary>
/// Indicates if the current segment is junk data
/// </summary>
private bool _currentIsJunk = false;
/// <summary>
/// Creates a new RVZ pack decompressor.
/// </summary>
/// <param name="packedData">The packed RVZ data</param>
/// <param name="rvzPackedSize">Expected size of packed data (for validation)</param>
/// <param name="dataOffset">Offset in the virtual disc (for LFG alignment)</param>
public Decompressor(byte[] packedData, uint rvzPackedSize, long dataOffset)
{
_packedData = packedData;
_rvzPackedSize = rvzPackedSize;
_dataOffset = dataOffset;
_lfg = new LaggedFibonacciGenerator();
}
/// <summary>
/// Decompresses the packed data into the output buffer.
/// </summary>
/// <param name="output">Destination buffer</param>
/// <param name="outputOffset">Offset in destination buffer</param>
/// <param name="count">Number of bytes to decompress</param>
/// <returns>Number of bytes actually decompressed</returns>
public int Decompress(byte[] output, int outputOffset, int count)
{
int totalWritten = 0;
while (totalWritten < count && !IsDone())
{
if (_currentSize == 0)
{
if (!ReadNextSegment())
break;
}
int bytesToWrite = Math.Min((int)_currentSize, count - totalWritten);
if (_currentIsJunk)
{
_lfg.GetBytes(bytesToWrite, output, outputOffset + totalWritten);
}
else
{
Array.Copy(_packedData, _inPosition, output, outputOffset + totalWritten, bytesToWrite);
_inPosition += bytesToWrite;
}
_currentSize -= (uint)bytesToWrite;
totalWritten += bytesToWrite;
_dataOffset += bytesToWrite;
}
return totalWritten;
}
/// <summary>
/// Checks if decompression is complete.
/// </summary>
public bool IsDone() => _currentSize == 0 && _inPosition >= _rvzPackedSize;
/// <summary>
/// Read the next segment of packed data
/// </summary>
/// <returns>True if the segment was read and cached, false otherwise</returns>
private bool ReadNextSegment()
{
if (_inPosition + 4 > _packedData.Length)
return false;
// Size field is big-endian u32; high bit signals junk data
uint sizeField = _packedData.ReadUInt32BigEndian(ref _inPosition);
_currentIsJunk = (sizeField & 0x80000000) != 0;
_currentSize = sizeField & 0x7FFFFFFF;
if (_currentIsJunk)
{
if (_inPosition + (LaggedFibonacciGenerator.SEED_SIZE * 4) > _packedData.Length)
return false;
byte[] seed = new byte[LaggedFibonacciGenerator.SEED_SIZE * 4];
Array.Copy(_packedData, _inPosition, seed, 0, seed.Length);
_inPosition += seed.Length;
_lfg.SetSeed(seed);
// Advance LFG to the correct position within the buffer.
// Dolphin: lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32))
int offsetInBuffer = (int)(_dataOffset % LaggedFibonacciGenerator.BUFFER_BYTES);
if (offsetInBuffer > 0)
_lfg.Forward(offsetInBuffer);
}
return true;
}
}
}

23
SabreTools.IO.Compression/RVZPack/FileEntry.cs Normal file
View File

@@ -0,0 +1,23 @@
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// File entry with start and end byte offsets on disc
/// </summary>
public struct FileEntry
{
/// <summary>
/// Indicates if the entry respresents a directory
/// </summary>
public bool IsDirectory;
/// <summary>
/// Starting data offset for the entry
/// </summary>
public long FileStart;
/// <summary>
/// Ending data offset for the entry
/// </summary>
public long FileEnd;
}
}

216
SabreTools.IO.Compression/RVZPack/FileSystemTableReader.cs Normal file
View File

@@ -0,0 +1,216 @@
using System.Collections.Generic;
using SabreTools.Numerics.Extensions;
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Lightweight GameCube / Wii File-System Table (FST) reader used by
/// <see cref="Compressor"/> to distinguish real-file regions from junk.
///
/// Mirrors Dolphin's FileSystemGCWii offset-to-file-info cache
/// (m_offset_file_info_cache).
/// </summary>
public sealed class FileSystemTableReader
{
/// <remarks>
/// Sorted ascending by FileEnd for O(log n) upper_bound queries.
/// </remarks>
private readonly List<FileEntry> _files;
private FileSystemTableReader(List<FileEntry> files)
{
_files = files;
}
/// <summary>
/// Parses a raw FST binary blob and returns a <see cref="FileSystemTableReader"/>,
/// or null if the data is too short or structurally invalid.
/// </summary>
/// <param name="fstData">
/// Raw FST bytes exactly as stored on disc (GameCube) or in decrypted
/// Wii partition data.
/// </param>
/// <param name="offsetShift">
/// Bit-shift to convert raw file-offset fields to byte addresses.
/// 0 for GameCube (direct bytes); 2 for Wii (offset × 4).
/// </param>
public static FileSystemTableReader? TryParse(byte[] fstData, int offsetShift)
{
// Read the file system table
var table = ParseFileSystemTable(fstData, offsetShift);
if (table is null)
return null;
// Filter out the entries to non-empty files only
var filtered = new List<FileEntry>();
foreach (var entry in table)
{
// Directory entry
if (entry.IsDirectory)
continue;
// Empty file
if (entry.FileStart == entry.FileEnd)
continue;
filtered.Add(entry);
}
// Sort ascending by FileEnd so binary-search upper_bound works correctly.
filtered.Sort(delegate (FileEntry a, FileEntry b)
{
return a.FileEnd.CompareTo(b.FileEnd);
});
return new FileSystemTableReader(filtered);
}
/// <summary>
/// Returns the file entry whose byte range contains <paramref name="discOffset"/>,
/// or null if no file does.
/// </summary>
/// TODO: Determine how to use List<T>.BinarySearch here
public FileEntry? FindFileInfo(long discOffset)
{
if (_files.Count == 0)
return null;
// Binary search: first index where _files[i].FileEnd > discOffset
int lo = 0, hi = _files.Count;
while (lo < hi)
{
int mid = (lo + hi) >> 1;
if (_files[mid].FileEnd <= discOffset)
lo = mid + 1;
else
hi = mid;
}
if (lo >= _files.Count)
return null;
var e = _files[lo];
if (e.FileStart <= discOffset)
return e;
return null;
}
/// <summary>
/// Returns the smallest FileEnd value strictly greater than
/// <paramref name="discOffset"/>, or null if there is none.
/// </summary>
public long? FindNextFileEnd(long discOffset)
{
if (_files.Count == 0)
return null;
int lo = 0, hi = _files.Count;
while (lo < hi)
{
int mid = (lo + hi) >> 1;
if (_files[mid].FileEnd <= discOffset)
lo = mid + 1;
else
hi = mid;
}
return lo < _files.Count ? _files[lo].FileEnd : null;
}
/// <summary>
/// Returns the smallest FileStart value strictly greater than
/// <paramref name="discOffset"/>, or null if there is none.
/// </summary>
public long? FindNextFileStart(long discOffset)
{
if (_files.Count == 0)
return null;
// Sort is by FileEnd; scan all entries whose FileEnd > discOffset
int lo = 0, hi = _files.Count;
while (lo < hi)
{
int mid = (lo + hi) >> 1;
if (_files[mid].FileEnd <= discOffset)
lo = mid + 1;
else
hi = mid;
}
long? best = null;
for (int i = lo; i < _files.Count; i++)
{
long start = _files[i].FileStart;
if (start <= discOffset)
continue;
if (best == null || start < best.Value)
best = start;
}
return best;
}
/// <summary>
/// Parse a byte array into a list of file entries
/// </summary>
/// <param name="data">Byte array to parse</param>
/// <param name="offsetShift">
/// Bit-shift to convert raw file-offset fields to byte addresses.
/// 0 for GameCube (direct bytes); 2 for Wii (offset × 4).
/// </param>
/// <returns>Filled file entry list on success, null on error</returns>
/// <remarks>Adapted from Serialization</remarks>
public static List<FileEntry>? ParseFileSystemTable(byte[] data, int offsetShift)
{
// Check that the root entry exists
if (data.Length < 12)
return null;
// Read the root entry first
int offset = 0;
_ = data.ReadBytes(ref offset, 8);
uint entryCount = data.ReadUInt32BigEndian(ref offset);
if (entryCount < 1 || (entryCount * 12) > data.Length)
return null;
// Read all entries
offset = 0;
var obj = new List<FileEntry>();
for (int i = 0; i < entryCount; i++)
{
var entry = ParseFileSystemTableEntry(data, ref offset, offsetShift);
obj.Add(entry);
}
return obj;
}
/// <summary>
/// Parse a byte array into a FileSystemTableEntry
/// </summary>
/// <param name="data">Byte array to parse</param>
/// <param name="offsetShift">
/// Bit-shift to convert raw file-offset fields to byte addresses.
/// 0 for GameCube (direct bytes); 2 for Wii (offset × 4).
/// </param>
/// <returns>Filled FileSystemTableEntry on success, null on error</returns>
/// <remarks>Adapted from Serialization</remarks>
public static FileEntry ParseFileSystemTableEntry(byte[] data, ref int offset, int offsetShift)
{
var obj = new FileEntry();
uint nameOffset = data.ReadUInt32BigEndian(ref offset);
obj.IsDirectory = (nameOffset & 0xFF000000) != 0;
uint fileOffset = data.ReadUInt32BigEndian(ref offset);
uint fileSize = data.ReadUInt32BigEndian(ref offset);
obj.FileStart = fileOffset << offsetShift;
obj.FileEnd = obj.FileStart + fileSize;
return obj;
}
}
}

18
SabreTools.IO.Compression/RVZPack/JunkRegion.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Junk region information for recreation
/// </summary>
internal struct JunkRegion
{
/// <summary>
/// Starting offset of the junk region
/// </summary>
public long StartOffset;
/// <summary>
/// Seed used to recreate the junk
/// </summary>
public uint[]? Seed;
}
}

2
SabreTools.Security.Cryptography/LaggedFibonacciGenerator.cs → SabreTools.IO.Compression/RVZPack/LaggedFibonacciGenerator.cs
View File

@@ -1,7 +1,7 @@
using System;
using SabreTools.Numerics.Extensions;
namespace SabreTools.Security.Cryptography
namespace SabreTools.IO.Compression.RVZPack
{
/// <summary>
/// Lagged Fibonacci Generator matching Dolphin's LaggedFibonacciGenerator exactly.