Implement model, reader, wrapper for valve SKU sis files. (#62)

* Try again

* Fix import alphebetization

* Fixes.

* first part of first attempt at a model

* Reimplement Sku Sis parsing

* First round of fixes

* Make sure stream isn't closed

* Missed this newline

SabreTools.Serialization.Test/Readers/SkuSisTests.cs

@@ -0,0 +1,73 @@
+using System.IO;
+using System.Linq;
+using SabreTools.Serialization.Readers;
+using Xunit;
+
+namespace SabreTools.Serialization.Test.Readers
+{
+    public class SkuSisTests
+    {
+        [Fact]
+        public void NullArray_Null()
+        {
+            byte[]? data = null;
+            int offset = 0;
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void EmptyArray_Null()
+        {
+            byte[]? data = [];
+            int offset = 0;
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void InvalidArray_Null()
+        {
+            byte[]? data = [.. Enumerable.Repeat<byte>(0xFF, 1024)];
+            int offset = 0;
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void NullStream_Null()
+        {
+            Stream? data = null;
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void EmptyStream_Null()
+        {
+            Stream? data = new MemoryStream([]);
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void InvalidStream_Null()
+        {
+            Stream? data = new MemoryStream([.. Enumerable.Repeat<byte>(0xFF, 1024)]);
+            var deserializer = new SkuSis();
+
+            var actual = deserializer.Deserialize(data);
+            Assert.Null(actual);
+        }
+    }
+}

SabreTools.Serialization.Test/Wrappers/SkuSisTests.cs

@@ -0,0 +1,61 @@
+using System.IO;
+using System.Linq;
+using SabreTools.Serialization.Wrappers;
+using Xunit;
+
+namespace SabreTools.Serialization.Test.Wrappers
+{
+    public class SkuSisTests
+    {
+        [Fact]
+        public void NullArray_Null()
+        {
+            byte[]? data = null;
+            int offset = 0;
+            var actual = SkuSis.Create(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void EmptyArray_Null()
+        {
+            byte[]? data = [];
+            int offset = 0;
+            var actual = SkuSis.Create(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void InvalidArray_Null()
+        {
+            byte[]? data = [.. Enumerable.Repeat<byte>(0xFF, 1024)];
+            int offset = 0;
+            var actual = SkuSis.Create(data, offset);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void NullStream_Null()
+        {
+            Stream? data = null;
+            var actual = SkuSis.Create(data);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void EmptyStream_Null()
+        {
+            Stream? data = new MemoryStream([]);
+            var actual = SkuSis.Create(data);
+            Assert.Null(actual);
+        }
+
+        [Fact]
+        public void InvalidStream_Null()
+        {
+            Stream? data = new MemoryStream([.. Enumerable.Repeat<byte>(0xFF, 1024)]);
+            var actual = SkuSis.Create(data);
+            Assert.Null(actual);
+        }
+    }
+}

SabreTools.Serialization/Models/VDF/Constants.cs

@@ -0,0 +1,19 @@
+namespace SabreTools.Data.Models.VDF
+{
+    public static class Constants
+    {
+        /// <summary>
+        /// Top-level item (and thus also first 5 bytes) of Steam2 (sis/sim/sid) retail installers
+        /// </summary>
+        public static readonly byte[] SteamSimSidSisSignatureBytes = [0x22, 0x53, 0x4B, 0x55, 0x22]; // "SKU"
+
+        public static readonly string SteamSimSidSisSignatureString = "\"SKU\"";
+
+        /// <summary>
+        /// Top-level item (and thus also first 5 bytes) of Steam3 (sis/csm/csd) retail installers
+        /// </summary>
+        public static readonly byte[] SteamCsmCsdSisSignatureBytes = [0x22, 0x73, 0x6B, 0x75, 0x22]; // "sku"
+
+        public static readonly string SteamCsmCsdSisSignatureString = "\"sku\"";
+    }
+}

SabreTools.Serialization/Models/VDF/Sku.cs

@@ -0,0 +1,158 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace SabreTools.Data.Models.VDF
+{
+    /// <summary>
+    /// Contains metadata information about retail Steam discs
+    /// Stored in a VDF file on the disc
+    /// </summary>
+    /// <remarks>Stored in the order it appears in the sku sis file, as it is always the same order.</remarks>
+    [JsonObject]
+    public class Sku
+    {
+        // At the moment, the only keys that matter for anything in SabreTools are sku, apps, depots, and manifests
+        // TODO: check case sensitivity
+        #region Non-Arrays
+
+        /// <summary>
+        /// "name"
+        /// Name of the disc/app
+        /// Known values: Arbitrary string
+        /// </summary>
+        [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
+        public string? Name { get; set; }
+
+        /// <summary>
+        /// "productname"
+        /// productname of the retail installer
+        /// Known values: Arbitrary string
+        /// </summary>
+        /// <remarks>sim/sid only</remarks>
+        [JsonProperty("productname", NullValueHandling = NullValueHandling.Ignore)]
+        public string? ProductName { get; set; }
+
+        /// <summary>
+        /// "subscriptionID"
+        /// subscriptionID of the retail installer
+        /// Known values: Arbitrary number
+        /// </summary>
+        /// <remarks>sim/sid only</remarks>
+        [JsonProperty("subscriptionID", NullValueHandling = NullValueHandling.Ignore)]
+        public long? SubscriptionId { get; set; }
+
+        /// <summary>
+        /// "appID"
+        /// AppID of the retail installer
+        /// Known values: Arbitrary number
+        /// </summary>
+        /// <remarks>sim/sid only. Both appID and AppID seem to be used in practice.</remarks>
+        [JsonProperty("appID", NullValueHandling = NullValueHandling.Ignore)]
+        public long? AppId { get; set; }
+
+        /// <summary>
+        /// "disks"
+        /// Number of discs of the retail installer set
+        /// Known values: 1-5? 10? Unsure what the most discs in a steam retail installer is currently known to be
+        /// </summary>
+        [JsonProperty("disks", NullValueHandling = NullValueHandling.Ignore)]
+        public uint? Disks { get; set; }
+
+        /// <summary>
+        /// "language"
+        /// language of the retail installer
+        /// Known values: english, russian
+        /// </summary>
+        /// <remarks>sim/sid only</remarks>
+        [JsonProperty("language", NullValueHandling = NullValueHandling.Ignore)]
+        public string? Language { get; set; }
+
+        /// <summary>
+        /// "disk"
+        /// Numbered disk of the retail installer set
+        /// Known values: 1-5? 10? Unsure what the most discs in a steam retail installer is currently known to be
+        /// </summary>
+        /// <remarks>csm/csd only</remarks>
+        [JsonProperty("disk", NullValueHandling = NullValueHandling.Ignore)]
+        public uint? Disk { get; set; }
+
+        /// <summary>
+        /// "backup"
+        /// Unknown. This is probably a boolean?
+        /// Known values: 0
+        /// </summary>
+        [JsonProperty("backup", NullValueHandling = NullValueHandling.Ignore)]
+        public uint? Backup { get; set; }
+
+        /// <summary>
+        /// "contenttype"
+        /// Unknown.
+        /// Known values: 3
+        /// </summary>
+        [JsonProperty("contenttype", NullValueHandling = NullValueHandling.Ignore)]
+        public uint? ContentType { get; set; }
+
+        #endregion
+
+        // When VDF has an array, they represent it like this, with the left numbers being indexes:
+        /// "1"		"1056577072"
+        /// "2"		"1056702256"
+        /// "3"		"1056203136"
+        /// etc.
+        /// The following format is also used like this, although this isn't one that needs to be parsed right now.
+        /// Currently unsure what the first number means. Maybe this is a two dimensional array?
+        /// "1 0"		"1493324560"
+        /// "1 1"		"1492884912"
+        /// "1 2"		"1492755784"
+        /// "1 3"		"28749920"
+        #region Arrays
+
+        /// <summary>
+        /// "apps"
+        /// AppIDs contained on the disc.
+        /// Known values: arbitrary
+        /// </summary>
+        /// <remarks>On csm/csd discs, both are used interchangeably, but never at the same time. It's usually still lowercase though.
+        /// It always seems to be lowercase on sim/sid discs</remarks>
+        [JsonProperty("apps", NullValueHandling = NullValueHandling.Ignore)]
+        public Dictionary<long, long>? Apps { get; set; }
+
+        /// <summary>
+        /// "depots"
+        /// DepotIDs contained on the disc.
+        /// Known values: arbitrary
+        /// </summary>
+        [JsonProperty("depots", NullValueHandling = NullValueHandling.Ignore)]
+        public Dictionary<long, long>? Depots { get; set; }
+
+        // The "packages" property should go here, but it uses the second array format mentioned above, so it's more
+        // difficult to adapt. Since it's not needed at the moment anyways, it's left out for now.
+
+        /// <summary>
+        /// "manifests"
+        /// DepotIDs contained on the disc.
+        /// Known values: arbitrary pairs of DepotID - Manifest
+        /// </summary>
+        [JsonProperty("manifests", NullValueHandling = NullValueHandling.Ignore)]
+        public Dictionary<long, long>? Manifests { get; set; }
+
+        /// <summary>
+        /// "chunkstores"
+        /// chunkstores contained on the disc.
+        /// Known values: DepotIDs containing arrays of chunkstores.
+        /// </summary>
+        /// <remarks>These are indexed from 1 instead of 0 for some reason.</remarks>
+        /// TODO: not that it really matters, but will this parse the internal values recursively properly?
+        [JsonProperty("chunkstores", NullValueHandling = NullValueHandling.Ignore)]
+        public Dictionary<long, Dictionary<long, long>?>? Chunkstores { get; set; }
+
+        /// <summary>
+        /// All remaining data not matched above.
+        /// </summary>
+        [JsonExtensionData]
+        public IDictionary<string, JToken>? EverythingElse { get; set; }
+
+        #endregion
+    }
+}

SabreTools.Serialization/Models/VDF/SkuSis.cs

@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace SabreTools.Data.Models.VDF
+{
+    /// <summary>
+    /// Contains metadata information about retail Steam discs
+    /// Stored in a VDF file on the disc
+    /// </summary>
+    /// <remarks>Stored in the order it appears in the sku sis file, as it is always the same order.</remarks>
+    [JsonObject]
+    public class SkuSis
+    {
+        // At the moment, the only keys that matter for anything in SabreTools are sku, apps, depots, and manifests
+        // TODO: check case sensitivity
+        #region Non-Arrays
+
+        /// <summary>
+        /// "sku"
+        /// Top-level value for sku.sis files.
+        /// Known values: the entire sku.sis object
+        /// </summary>
+        /// <remarks>capital SKU on sim/sid, lowercase sku on csm/csd</remarks>
+        [JsonProperty("sku", NullValueHandling = NullValueHandling.Ignore)]
+        public Sku? Sku { get; set; }
+
+        #endregion
+    }
+}

SabreTools.Serialization/Readers/SkuSis.cs

@@ -0,0 +1,146 @@
+using System;
+using System.IO;
+using System.Text;
+using SabreTools.IO.Extensions;
+using static SabreTools.Data.Models.VDF.Constants;
+
+namespace SabreTools.Serialization.Readers
+{
+    /// <remarks>
+    /// The VDF file format was used for a very wide scope of functions on steam. At the moment, VDF file support is
+    /// only needed when it comes to parsing retail sku sis files, so the current parser is only aimed at supporting
+    /// these files, as they're overall very consistent, and trying to test every usage of VDF files would be extremely
+    /// time-consuming for little benefit. If parsing other usages of VDF files ever becomes necessary, this should be
+    /// replaced with a general-purpose VDF parser.
+    /// Most observations about sku sis files described here probably also apply to VDF files.
+    /// </remarks>
+    public class SkuSis : BaseBinaryReader<Data.Models.VDF.SkuSis>
+    {
+        /// <inheritdoc/>
+        public override Data.Models.VDF.SkuSis? Deserialize(Stream? data)
+        {
+            // If the data is invalid
+            if (data == null || !data.CanRead)
+                return null;
+
+            try
+            {
+                // Cache the current offset
+                long initialOffset = data.Position;
+
+                // Check if file contains the top level sku value, otherwise return null
+                var signatureBytes = data.ReadBytes(5);
+                if (!signatureBytes.EqualsExactly(SteamSimSidSisSignatureBytes)
+                    && !signatureBytes.EqualsExactly(SteamCsmCsdSisSignatureBytes))
+                {
+                    return null;
+                }
+
+                data.SeekIfPossible(initialOffset, SeekOrigin.Begin);
+
+                var jsonBytes = ParseSkuSis(data);
+                if (jsonBytes == null)
+                    return null;
+
+                var deserializer = new SkuSisJson();
+                var skuSisJson = deserializer.Deserialize(jsonBytes, 0);
+                if (skuSisJson == null)
+                    return null;
+
+                return skuSisJson;
+            }
+            catch
+            {
+                // Ignore the actual error
+                return null;
+            }
+        }
+
+        /// <summary>
+        ///  Handles deserialization of the json-modified VDF string into a json.
+        /// </summary>
+        /// <remarks>Requires VDF-to-JSON conversion, should not be public.</remarks>
+        private class SkuSisJson : JsonFile<Data.Models.VDF.SkuSis>
+        {
+            #region IByteReader
+
+            /// <remarks>All known sku sis files are observed to be ASCII</remarks>
+            public override Data.Models.VDF.SkuSis? Deserialize(byte[]? data, int offset)
+                => Deserialize(data, offset, new ASCIIEncoding());
+
+            #endregion
+
+            #region IFileReader
+
+            /// <remarks>All known sku sis files are observed to be ASCII</remarks>
+            public override Data.Models.VDF.SkuSis? Deserialize(string? path)
+                => Deserialize(path, new ASCIIEncoding());
+
+            #endregion
+
+            #region IStreamReader
+
+            /// <remarks>All known sku sis files are observed to be ASCII</remarks>
+            public override Data.Models.VDF.SkuSis? Deserialize(Stream? data)
+                => Deserialize(data, new ASCIIEncoding());
+
+            #endregion
+        }
+
+        /// <summary>
+        /// Parse a Stream into a Header
+        /// </summary>
+        /// <param name="data">Stream to parse</param>
+        /// <returns>Filled Header on success, null on error</returns>
+        public static byte[]? ParseSkuSis(Stream data)
+        {
+            string json = "{\n"; // Sku sis files have no surrounding curly braces, which json doesn't allow
+            const string delimiter = "\"\t\t\""; // KVPs are always quoted, and are delimited by two tabs
+
+            // This closes the stream, but can't be easily avoided on earlier versions of dotnet
+#if NET20 || NET35 || NET40
+            var reader = new StreamReader(data, Encoding.ASCII);
+#else
+            var reader = new StreamReader(data, Encoding.ASCII, false, -1, true);
+#endif
+            while (!reader.EndOfStream)
+            {
+                string? line = reader.ReadLine();
+                if (line == null)
+                    continue;
+
+                // Curly braces are always on their own lines
+                if (line.Contains("{"))
+                {
+                    json += "{\n";
+                    continue;
+                }
+                else if (line.Contains("}"))
+                {
+                    json += line;
+                    json += ",\n";
+                    continue;
+                }
+
+                int index = line.IndexOf(delimiter, StringComparison.Ordinal);
+
+                // If the delimiter isn't found, this is the start of an object with multiple KVPs and the next line
+                // will be an opening curly brace line.
+                if (index <= -1)
+                {
+                    json += line;
+                    json += ": ";
+                }
+                else // If the delimiter is found, it's just a normal KVP
+                {
+                    json += line.Replace(delimiter, "\": \"");
+                    json += ",\n";
+                }
+            }
+
+            json += "\n}";
+            byte[] jsonBytes = Encoding.ASCII.GetBytes(json);
+            return jsonBytes;
+        }
+    }
+}

SabreTools.Serialization/WrapperFactory.cs

@@ -53,6 +53,7 @@ namespace SabreTools.Serialization
                 WrapperType.SecuROMDFA => SecuROMDFA.Create(data),
                 WrapperType.SevenZip => SevenZip.Create(data),
                 WrapperType.Skeleton => Skeleton.Create(data),
+                WrapperType.SkuSis => SkuSis.Create(data),
                 WrapperType.SFFS => SFFS.Create(data),
                 WrapperType.SGA => SGA.Create(data),
                 WrapperType.TapeArchive => TapeArchive.Create(data),
@@ -677,6 +678,16 @@ namespace SabreTools.Serialization
 
             #endregion
 
+            #region SkuSis
+
+            if (magic.StartsWith(Data.Models.VDF.Constants.SteamSimSidSisSignatureBytes))
+                return WrapperType.SkuSis;
+
+            if (magic.StartsWith(Data.Models.VDF.Constants.SteamCsmCsdSisSignatureBytes))
+                return WrapperType.SkuSis;
+
+            #endregion
+
             #region SGA
 
             if (magic.StartsWith(Data.Models.SGA.Constants.SignatureBytes))

SabreTools.Serialization/Wrappers/SkuSis.cs

@@ -0,0 +1,140 @@
+using System.Collections.Generic;
+using System.IO;
+using Newtonsoft.Json.Linq;
+using SabreTools.Data.Models.VDF;
+
+namespace SabreTools.Serialization.Wrappers
+{
+    public partial class SkuSis : WrapperBase<Data.Models.VDF.SkuSis>
+    {
+        #region Descriptive Properties
+
+        /// <inheritdoc/>
+        public override string DescriptionString => "Valve Data File";
+
+        #endregion
+
+        #region Extension Properties
+
+        /// <inheritdoc cref="Models.VDF.SkuSis.Sku"/>
+        public Sku? Sku => Model.Sku;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Name"/>
+        public string? Name => Sku?.Name;
+
+        /// <inheritdoc cref="Models.VDF.Sku.ProductName"/>
+        public string? ProductName => Sku?.ProductName;
+
+        /// <inheritdoc cref="Models.VDF.Sku.SubscriptionId"/>
+        public long? SubscriptionId => Sku?.SubscriptionId;
+
+        /// <inheritdoc cref="Models.VDF.Sku.AppId"/>
+        public long? AppId => Sku?.AppId;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Disks"/>
+        public uint? Disks => Sku?.Disks;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Language"/>
+        public string? Language => Sku?.Language;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Disk"/>
+        public uint? Disk => Sku?.Disk;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Backup"/>
+        public uint? Backup => Sku?.Backup;
+
+        /// <inheritdoc cref="Models.VDF.Sku.contenttype"/>
+        public uint? ContentType => Sku?.ContentType;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Apps"/>
+        public Dictionary<long, long>? Apps => Sku?.Apps;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Depots"/>
+        public Dictionary<long, long>? Depots => Sku?.Depots;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Manifests"/>
+        public Dictionary<long, long>? Manifests => Sku?.Manifests;
+
+        /// <inheritdoc cref="Models.VDF.Sku.Chunkstores"/>
+        public Dictionary<long, Dictionary<long, long>?>? Chunkstores => Sku?.Chunkstores;
+
+        /// <inheritdoc cref="Models.VDF.Sku.EverythingElse"/>
+        public IDictionary<string, JToken>? EverythingElse => Sku?.EverythingElse;
+
+        #endregion
+
+        #region Constructors
+
+        public SkuSis(Data.Models.VDF.SkuSis model, byte[] data) : base(model, data) { }
+
+        /// <inheritdoc/>
+        public SkuSis(Data.Models.VDF.SkuSis model, byte[] data, int offset) : base(model, data, offset) { }
+
+        /// <inheritdoc/>
+        public SkuSis(Data.Models.VDF.SkuSis model, byte[] data, int offset, int length) : base(model, data, offset, length) { }
+
+        /// <inheritdoc/>
+        public SkuSis(Data.Models.VDF.SkuSis model, Stream data) : base(model, data) { }
+
+        /// <inheritdoc/>
+        public SkuSis(Data.Models.VDF.SkuSis model, Stream data, long offset) : base(model, data, offset) { }
+
+        /// <inheritdoc/>
+        public SkuSis(Data.Models.VDF.SkuSis model, Stream data, long offset, long length) : base(model, data, offset, length) { }
+
+        #endregion
+
+        #region Static Constructors
+
+        /// <summary>
+        /// Create an SKU sis from a byte array and offset
+        /// </summary>
+        /// <param name="data">Byte array representing the SKU sis</param>
+        /// <param name="offset">Offset within the array to parse</param>
+        /// <returns>An SKU sis wrapper on success, null on failure</returns>
+        public static SkuSis? Create(byte[]? data, int offset)
+        {
+            // If the data is invalid
+            if (data == null || data.Length == 0)
+                return null;
+
+            // If the offset is out of bounds
+            if (offset < 0 || offset >= data.Length)
+                return null;
+
+            // Create a memory stream and use that
+            var dataStream = new MemoryStream(data, offset, data.Length - offset);
+            return Create(dataStream);
+        }
+
+        /// <summary>
+        /// Create an SKU sis from a Stream
+        /// </summary>
+        /// <param name="data">Stream representing the SKU sis</param>
+        /// <returns>An SKU sis wrapper on success, null on failure</returns>
+        public static SkuSis? Create(Stream? data)
+        {
+            // If the data is invalid
+            if (data == null || !data.CanRead)
+                return null;
+
+            try
+            {
+                // Cache the current offset
+                long currentOffset = data.Position;
+
+                var model = new Readers.SkuSis().Deserialize(data);
+                if (model == null)
+                    return null;
+
+                return new SkuSis(model, data, currentOffset);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        #endregion
+    }
+}

SabreTools.Serialization/Wrappers/WrapperType.cs

@@ -217,6 +217,11 @@ namespace SabreTools.Serialization.Wrappers
         /// </summary>
         Skeleton,
 
+        /// <summary>
+        /// Steam SKU sis file
+        /// </summary>
+        SkuSis,
+
         /// <summary>
         /// Tape archive
         /// </summary>