From 89a67d1bd286aa8a590ace9cb36f603dc695aae5 Mon Sep 17 00:00:00 2001 From: HeroponRikiBestest <50224630+HeroponRikiBestest@users.noreply.github.com> Date: Wed, 28 Jan 2026 21:30:08 -0500 Subject: [PATCH] Implement model, reader, wrapper for valve SKU sis files. (#62) * Try again * Fix import alphebetization * Fixes. * first part of first attempt at a model * Reimplement Sku Sis parsing * First round of fixes * Make sure stream isn't closed * Missed this newline --- .../Readers/SkuSisTests.cs | 73 ++++++++ .../Wrappers/SkuSisTests.cs | 61 +++++++ .../Models/VDF/Constants.cs | 19 +++ SabreTools.Serialization/Models/VDF/Sku.cs | 158 ++++++++++++++++++ SabreTools.Serialization/Models/VDF/SkuSis.cs | 30 ++++ SabreTools.Serialization/Readers/SkuSis.cs | 146 ++++++++++++++++ SabreTools.Serialization/WrapperFactory.cs | 11 ++ SabreTools.Serialization/Wrappers/SkuSis.cs | 140 ++++++++++++++++ .../Wrappers/WrapperType.cs | 5 + 9 files changed, 643 insertions(+) create mode 100644 SabreTools.Serialization.Test/Readers/SkuSisTests.cs create mode 100644 SabreTools.Serialization.Test/Wrappers/SkuSisTests.cs create mode 100644 SabreTools.Serialization/Models/VDF/Constants.cs create mode 100644 SabreTools.Serialization/Models/VDF/Sku.cs create mode 100644 SabreTools.Serialization/Models/VDF/SkuSis.cs create mode 100644 SabreTools.Serialization/Readers/SkuSis.cs create mode 100644 SabreTools.Serialization/Wrappers/SkuSis.cs diff --git a/SabreTools.Serialization.Test/Readers/SkuSisTests.cs b/SabreTools.Serialization.Test/Readers/SkuSisTests.cs new file mode 100644 index 00000000..3067f6db --- /dev/null +++ b/SabreTools.Serialization.Test/Readers/SkuSisTests.cs @@ -0,0 +1,73 @@ +using System.IO; +using System.Linq; +using SabreTools.Serialization.Readers; +using Xunit; + +namespace SabreTools.Serialization.Test.Readers +{ + public class SkuSisTests + { + [Fact] + public void NullArray_Null() + { + byte[]? data = null; + int offset = 0; + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void EmptyArray_Null() + { + byte[]? data = []; + int offset = 0; + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void InvalidArray_Null() + { + byte[]? data = [.. Enumerable.Repeat(0xFF, 1024)]; + int offset = 0; + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data, offset); + Assert.Null(actual); + } + + [Fact] + public void NullStream_Null() + { + Stream? data = null; + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + + [Fact] + public void EmptyStream_Null() + { + Stream? data = new MemoryStream([]); + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + + [Fact] + public void InvalidStream_Null() + { + Stream? data = new MemoryStream([.. Enumerable.Repeat(0xFF, 1024)]); + var deserializer = new SkuSis(); + + var actual = deserializer.Deserialize(data); + Assert.Null(actual); + } + } +} diff --git a/SabreTools.Serialization.Test/Wrappers/SkuSisTests.cs b/SabreTools.Serialization.Test/Wrappers/SkuSisTests.cs new file mode 100644 index 00000000..d90ce3c8 --- /dev/null +++ b/SabreTools.Serialization.Test/Wrappers/SkuSisTests.cs @@ -0,0 +1,61 @@ +using System.IO; +using System.Linq; +using SabreTools.Serialization.Wrappers; +using Xunit; + +namespace SabreTools.Serialization.Test.Wrappers +{ + public class SkuSisTests + { + [Fact] + public void NullArray_Null() + { + byte[]? data = null; + int offset = 0; + var actual = SkuSis.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void EmptyArray_Null() + { + byte[]? data = []; + int offset = 0; + var actual = SkuSis.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void InvalidArray_Null() + { + byte[]? data = [.. Enumerable.Repeat(0xFF, 1024)]; + int offset = 0; + var actual = SkuSis.Create(data, offset); + Assert.Null(actual); + } + + [Fact] + public void NullStream_Null() + { + Stream? data = null; + var actual = SkuSis.Create(data); + Assert.Null(actual); + } + + [Fact] + public void EmptyStream_Null() + { + Stream? data = new MemoryStream([]); + var actual = SkuSis.Create(data); + Assert.Null(actual); + } + + [Fact] + public void InvalidStream_Null() + { + Stream? data = new MemoryStream([.. Enumerable.Repeat(0xFF, 1024)]); + var actual = SkuSis.Create(data); + Assert.Null(actual); + } + } +} diff --git a/SabreTools.Serialization/Models/VDF/Constants.cs b/SabreTools.Serialization/Models/VDF/Constants.cs new file mode 100644 index 00000000..5da9a46c --- /dev/null +++ b/SabreTools.Serialization/Models/VDF/Constants.cs @@ -0,0 +1,19 @@ +namespace SabreTools.Data.Models.VDF +{ + public static class Constants + { + /// + /// Top-level item (and thus also first 5 bytes) of Steam2 (sis/sim/sid) retail installers + /// + public static readonly byte[] SteamSimSidSisSignatureBytes = [0x22, 0x53, 0x4B, 0x55, 0x22]; // "SKU" + + public static readonly string SteamSimSidSisSignatureString = "\"SKU\""; + + /// + /// Top-level item (and thus also first 5 bytes) of Steam3 (sis/csm/csd) retail installers + /// + public static readonly byte[] SteamCsmCsdSisSignatureBytes = [0x22, 0x73, 0x6B, 0x75, 0x22]; // "sku" + + public static readonly string SteamCsmCsdSisSignatureString = "\"sku\""; + } +} diff --git a/SabreTools.Serialization/Models/VDF/Sku.cs b/SabreTools.Serialization/Models/VDF/Sku.cs new file mode 100644 index 00000000..f339353c --- /dev/null +++ b/SabreTools.Serialization/Models/VDF/Sku.cs @@ -0,0 +1,158 @@ +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace SabreTools.Data.Models.VDF +{ + /// + /// Contains metadata information about retail Steam discs + /// Stored in a VDF file on the disc + /// + /// Stored in the order it appears in the sku sis file, as it is always the same order. + [JsonObject] + public class Sku + { + // At the moment, the only keys that matter for anything in SabreTools are sku, apps, depots, and manifests + // TODO: check case sensitivity + #region Non-Arrays + + /// + /// "name" + /// Name of the disc/app + /// Known values: Arbitrary string + /// + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string? Name { get; set; } + + /// + /// "productname" + /// productname of the retail installer + /// Known values: Arbitrary string + /// + /// sim/sid only + [JsonProperty("productname", NullValueHandling = NullValueHandling.Ignore)] + public string? ProductName { get; set; } + + /// + /// "subscriptionID" + /// subscriptionID of the retail installer + /// Known values: Arbitrary number + /// + /// sim/sid only + [JsonProperty("subscriptionID", NullValueHandling = NullValueHandling.Ignore)] + public long? SubscriptionId { get; set; } + + /// + /// "appID" + /// AppID of the retail installer + /// Known values: Arbitrary number + /// + /// sim/sid only. Both appID and AppID seem to be used in practice. + [JsonProperty("appID", NullValueHandling = NullValueHandling.Ignore)] + public long? AppId { get; set; } + + /// + /// "disks" + /// Number of discs of the retail installer set + /// Known values: 1-5? 10? Unsure what the most discs in a steam retail installer is currently known to be + /// + [JsonProperty("disks", NullValueHandling = NullValueHandling.Ignore)] + public uint? Disks { get; set; } + + /// + /// "language" + /// language of the retail installer + /// Known values: english, russian + /// + /// sim/sid only + [JsonProperty("language", NullValueHandling = NullValueHandling.Ignore)] + public string? Language { get; set; } + + /// + /// "disk" + /// Numbered disk of the retail installer set + /// Known values: 1-5? 10? Unsure what the most discs in a steam retail installer is currently known to be + /// + /// csm/csd only + [JsonProperty("disk", NullValueHandling = NullValueHandling.Ignore)] + public uint? Disk { get; set; } + + /// + /// "backup" + /// Unknown. This is probably a boolean? + /// Known values: 0 + /// + [JsonProperty("backup", NullValueHandling = NullValueHandling.Ignore)] + public uint? Backup { get; set; } + + /// + /// "contenttype" + /// Unknown. + /// Known values: 3 + /// + [JsonProperty("contenttype", NullValueHandling = NullValueHandling.Ignore)] + public uint? ContentType { get; set; } + + #endregion + + // When VDF has an array, they represent it like this, with the left numbers being indexes: + /// "1" "1056577072" + /// "2" "1056702256" + /// "3" "1056203136" + /// etc. + /// The following format is also used like this, although this isn't one that needs to be parsed right now. + /// Currently unsure what the first number means. Maybe this is a two dimensional array? + /// "1 0" "1493324560" + /// "1 1" "1492884912" + /// "1 2" "1492755784" + /// "1 3" "28749920" + #region Arrays + + /// + /// "apps" + /// AppIDs contained on the disc. + /// Known values: arbitrary + /// + /// On csm/csd discs, both are used interchangeably, but never at the same time. It's usually still lowercase though. + /// It always seems to be lowercase on sim/sid discs + [JsonProperty("apps", NullValueHandling = NullValueHandling.Ignore)] + public Dictionary? Apps { get; set; } + + /// + /// "depots" + /// DepotIDs contained on the disc. + /// Known values: arbitrary + /// + [JsonProperty("depots", NullValueHandling = NullValueHandling.Ignore)] + public Dictionary? Depots { get; set; } + + // The "packages" property should go here, but it uses the second array format mentioned above, so it's more + // difficult to adapt. Since it's not needed at the moment anyways, it's left out for now. + + /// + /// "manifests" + /// DepotIDs contained on the disc. + /// Known values: arbitrary pairs of DepotID - Manifest + /// + [JsonProperty("manifests", NullValueHandling = NullValueHandling.Ignore)] + public Dictionary? Manifests { get; set; } + + /// + /// "chunkstores" + /// chunkstores contained on the disc. + /// Known values: DepotIDs containing arrays of chunkstores. + /// + /// These are indexed from 1 instead of 0 for some reason. + /// TODO: not that it really matters, but will this parse the internal values recursively properly? + [JsonProperty("chunkstores", NullValueHandling = NullValueHandling.Ignore)] + public Dictionary?>? Chunkstores { get; set; } + + /// + /// All remaining data not matched above. + /// + [JsonExtensionData] + public IDictionary? EverythingElse { get; set; } + + #endregion + } +} diff --git a/SabreTools.Serialization/Models/VDF/SkuSis.cs b/SabreTools.Serialization/Models/VDF/SkuSis.cs new file mode 100644 index 00000000..29079104 --- /dev/null +++ b/SabreTools.Serialization/Models/VDF/SkuSis.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace SabreTools.Data.Models.VDF +{ + /// + /// Contains metadata information about retail Steam discs + /// Stored in a VDF file on the disc + /// + /// Stored in the order it appears in the sku sis file, as it is always the same order. + [JsonObject] + public class SkuSis + { + // At the moment, the only keys that matter for anything in SabreTools are sku, apps, depots, and manifests + // TODO: check case sensitivity + #region Non-Arrays + + /// + /// "sku" + /// Top-level value for sku.sis files. + /// Known values: the entire sku.sis object + /// + /// capital SKU on sim/sid, lowercase sku on csm/csd + [JsonProperty("sku", NullValueHandling = NullValueHandling.Ignore)] + public Sku? Sku { get; set; } + + #endregion + } +} diff --git a/SabreTools.Serialization/Readers/SkuSis.cs b/SabreTools.Serialization/Readers/SkuSis.cs new file mode 100644 index 00000000..273426cc --- /dev/null +++ b/SabreTools.Serialization/Readers/SkuSis.cs @@ -0,0 +1,146 @@ +using System; +using System.IO; +using System.Text; +using SabreTools.IO.Extensions; +using static SabreTools.Data.Models.VDF.Constants; + +namespace SabreTools.Serialization.Readers +{ + /// + /// The VDF file format was used for a very wide scope of functions on steam. At the moment, VDF file support is + /// only needed when it comes to parsing retail sku sis files, so the current parser is only aimed at supporting + /// these files, as they're overall very consistent, and trying to test every usage of VDF files would be extremely + /// time-consuming for little benefit. If parsing other usages of VDF files ever becomes necessary, this should be + /// replaced with a general-purpose VDF parser. + /// Most observations about sku sis files described here probably also apply to VDF files. + /// + public class SkuSis : BaseBinaryReader + { + /// + public override Data.Models.VDF.SkuSis? Deserialize(Stream? data) + { + // If the data is invalid + if (data == null || !data.CanRead) + return null; + + try + { + // Cache the current offset + long initialOffset = data.Position; + + // Check if file contains the top level sku value, otherwise return null + var signatureBytes = data.ReadBytes(5); + if (!signatureBytes.EqualsExactly(SteamSimSidSisSignatureBytes) + && !signatureBytes.EqualsExactly(SteamCsmCsdSisSignatureBytes)) + { + return null; + } + + data.SeekIfPossible(initialOffset, SeekOrigin.Begin); + + var jsonBytes = ParseSkuSis(data); + if (jsonBytes == null) + return null; + + var deserializer = new SkuSisJson(); + var skuSisJson = deserializer.Deserialize(jsonBytes, 0); + if (skuSisJson == null) + return null; + + return skuSisJson; + } + catch + { + // Ignore the actual error + return null; + } + } + + /// + /// Handles deserialization of the json-modified VDF string into a json. + /// + /// Requires VDF-to-JSON conversion, should not be public. + private class SkuSisJson : JsonFile + { + #region IByteReader + + /// All known sku sis files are observed to be ASCII + public override Data.Models.VDF.SkuSis? Deserialize(byte[]? data, int offset) + => Deserialize(data, offset, new ASCIIEncoding()); + + #endregion + + #region IFileReader + + /// All known sku sis files are observed to be ASCII + public override Data.Models.VDF.SkuSis? Deserialize(string? path) + => Deserialize(path, new ASCIIEncoding()); + + #endregion + + #region IStreamReader + + /// All known sku sis files are observed to be ASCII + public override Data.Models.VDF.SkuSis? Deserialize(Stream? data) + => Deserialize(data, new ASCIIEncoding()); + + #endregion + } + + /// + /// Parse a Stream into a Header + /// + /// Stream to parse + /// Filled Header on success, null on error + public static byte[]? ParseSkuSis(Stream data) + { + string json = "{\n"; // Sku sis files have no surrounding curly braces, which json doesn't allow + const string delimiter = "\"\t\t\""; // KVPs are always quoted, and are delimited by two tabs + + // This closes the stream, but can't be easily avoided on earlier versions of dotnet +#if NET20 || NET35 || NET40 + var reader = new StreamReader(data, Encoding.ASCII); +#else + var reader = new StreamReader(data, Encoding.ASCII, false, -1, true); +#endif + while (!reader.EndOfStream) + { + string? line = reader.ReadLine(); + if (line == null) + continue; + + // Curly braces are always on their own lines + if (line.Contains("{")) + { + json += "{\n"; + continue; + } + else if (line.Contains("}")) + { + json += line; + json += ",\n"; + continue; + } + + int index = line.IndexOf(delimiter, StringComparison.Ordinal); + + // If the delimiter isn't found, this is the start of an object with multiple KVPs and the next line + // will be an opening curly brace line. + if (index <= -1) + { + json += line; + json += ": "; + } + else // If the delimiter is found, it's just a normal KVP + { + json += line.Replace(delimiter, "\": \""); + json += ",\n"; + } + } + + json += "\n}"; + byte[] jsonBytes = Encoding.ASCII.GetBytes(json); + return jsonBytes; + } + } +} diff --git a/SabreTools.Serialization/WrapperFactory.cs b/SabreTools.Serialization/WrapperFactory.cs index 68f427a8..83ebb823 100644 --- a/SabreTools.Serialization/WrapperFactory.cs +++ b/SabreTools.Serialization/WrapperFactory.cs @@ -53,6 +53,7 @@ namespace SabreTools.Serialization WrapperType.SecuROMDFA => SecuROMDFA.Create(data), WrapperType.SevenZip => SevenZip.Create(data), WrapperType.Skeleton => Skeleton.Create(data), + WrapperType.SkuSis => SkuSis.Create(data), WrapperType.SFFS => SFFS.Create(data), WrapperType.SGA => SGA.Create(data), WrapperType.TapeArchive => TapeArchive.Create(data), @@ -677,6 +678,16 @@ namespace SabreTools.Serialization #endregion + #region SkuSis + + if (magic.StartsWith(Data.Models.VDF.Constants.SteamSimSidSisSignatureBytes)) + return WrapperType.SkuSis; + + if (magic.StartsWith(Data.Models.VDF.Constants.SteamCsmCsdSisSignatureBytes)) + return WrapperType.SkuSis; + + #endregion + #region SGA if (magic.StartsWith(Data.Models.SGA.Constants.SignatureBytes)) diff --git a/SabreTools.Serialization/Wrappers/SkuSis.cs b/SabreTools.Serialization/Wrappers/SkuSis.cs new file mode 100644 index 00000000..b030ee1f --- /dev/null +++ b/SabreTools.Serialization/Wrappers/SkuSis.cs @@ -0,0 +1,140 @@ +using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json.Linq; +using SabreTools.Data.Models.VDF; + +namespace SabreTools.Serialization.Wrappers +{ + public partial class SkuSis : WrapperBase + { + #region Descriptive Properties + + /// + public override string DescriptionString => "Valve Data File"; + + #endregion + + #region Extension Properties + + /// + public Sku? Sku => Model.Sku; + + /// + public string? Name => Sku?.Name; + + /// + public string? ProductName => Sku?.ProductName; + + /// + public long? SubscriptionId => Sku?.SubscriptionId; + + /// + public long? AppId => Sku?.AppId; + + /// + public uint? Disks => Sku?.Disks; + + /// + public string? Language => Sku?.Language; + + /// + public uint? Disk => Sku?.Disk; + + /// + public uint? Backup => Sku?.Backup; + + /// + public uint? ContentType => Sku?.ContentType; + + /// + public Dictionary? Apps => Sku?.Apps; + + /// + public Dictionary? Depots => Sku?.Depots; + + /// + public Dictionary? Manifests => Sku?.Manifests; + + /// + public Dictionary?>? Chunkstores => Sku?.Chunkstores; + + /// + public IDictionary? EverythingElse => Sku?.EverythingElse; + + #endregion + + #region Constructors + + public SkuSis(Data.Models.VDF.SkuSis model, byte[] data) : base(model, data) { } + + /// + public SkuSis(Data.Models.VDF.SkuSis model, byte[] data, int offset) : base(model, data, offset) { } + + /// + public SkuSis(Data.Models.VDF.SkuSis model, byte[] data, int offset, int length) : base(model, data, offset, length) { } + + /// + public SkuSis(Data.Models.VDF.SkuSis model, Stream data) : base(model, data) { } + + /// + public SkuSis(Data.Models.VDF.SkuSis model, Stream data, long offset) : base(model, data, offset) { } + + /// + public SkuSis(Data.Models.VDF.SkuSis model, Stream data, long offset, long length) : base(model, data, offset, length) { } + + #endregion + + #region Static Constructors + + /// + /// Create an SKU sis from a byte array and offset + /// + /// Byte array representing the SKU sis + /// Offset within the array to parse + /// An SKU sis wrapper on success, null on failure + public static SkuSis? Create(byte[]? data, int offset) + { + // If the data is invalid + if (data == null || data.Length == 0) + return null; + + // If the offset is out of bounds + if (offset < 0 || offset >= data.Length) + return null; + + // Create a memory stream and use that + var dataStream = new MemoryStream(data, offset, data.Length - offset); + return Create(dataStream); + } + + /// + /// Create an SKU sis from a Stream + /// + /// Stream representing the SKU sis + /// An SKU sis wrapper on success, null on failure + public static SkuSis? Create(Stream? data) + { + // If the data is invalid + if (data == null || !data.CanRead) + return null; + + try + { + // Cache the current offset + long currentOffset = data.Position; + + var model = new Readers.SkuSis().Deserialize(data); + if (model == null) + return null; + + return new SkuSis(model, data, currentOffset); + } + catch + { + return null; + } + } + + #endregion + } +} diff --git a/SabreTools.Serialization/Wrappers/WrapperType.cs b/SabreTools.Serialization/Wrappers/WrapperType.cs index aff782b0..3f71a39e 100644 --- a/SabreTools.Serialization/Wrappers/WrapperType.cs +++ b/SabreTools.Serialization/Wrappers/WrapperType.cs @@ -217,6 +217,11 @@ namespace SabreTools.Serialization.Wrappers /// Skeleton, + /// + /// Steam SKU sis file + /// + SkuSis, + /// /// Tape archive ///