Fix Sort

SabreTools.Library/DatFiles/DatFile.cs

@@ -1908,7 +1908,7 @@ namespace SabreTools.Library.DatFiles
                 });
 
                 // Now find all folders that are empty, if we are supposed to
-                if (!(Header.OutputDepot?.IsActive ?? false) && addBlanks)
+                if (Header.OutputDepot?.IsActive != true && addBlanks)
                 {
                     List<string> empties = DirectoryExtensions.ListEmpty(basePath);
                     Parallel.ForEach(empties, Globals.ParallelOptions, dir =>
@@ -1979,7 +1979,7 @@ namespace SabreTools.Library.DatFiles
             bool copyFiles)
         {
             // Special case for if we are in Depot mode (all names are supposed to be SHA-1 hashes)
-            if (Header.OutputDepot?.IsActive ?? false)
+            if (Header.OutputDepot?.IsActive == true)
             {
                 GZipArchive gzarc = new GZipArchive(item);
                 BaseFile baseFile = gzarc.GetTorrentGZFileInfo();
@@ -2980,11 +2980,7 @@ namespace SabreTools.Library.DatFiles
         /// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
         /// <param name="asFiles">TreatAsFiles representing CHD and Archive scanning</param>
         /// <returns>True if verification was a success, false otherwise</returns>
-        public bool VerifyGeneric(
+        public bool VerifyGeneric(List<string> inputs, bool hashOnly, bool quickScan, TreatAsFiles asFiles = 0x00)
-            List<string> inputs,
-            bool hashOnly = false,
-            bool quickScan = false,
-            TreatAsFiles asFiles = 0x00)
         {
             bool success = true;
 
@@ -2996,45 +2992,27 @@ namespace SabreTools.Library.DatFiles
                 PopulateFromDir(input, quickScan ? Hash.SecureHashes : Hash.DeepHashes, asFiles: asFiles);
             }
 
-            // If we are checking hashes only, essentially diff the inputs
+            // Force bucketing according to the flags
+            Items.SetBucketedBy(Field.NULL);
             if (hashOnly)
-            {
-                // First we need to bucket and dedupe by hash to get duplicates
                 Items.BucketBy(Field.DatItem_CRC, DedupeType.Full);
-
-                var keys = Items.SortedKeys.ToList();
-                foreach (string key in keys)
-                {
-                    List<DatItem> items = Items[key];
-                    for (int i = 0; i < items.Count; i++)
-                    {
-                        // Unmatched items will have a source ID of 99, remove all others
-                        if (items[i].Source.Index != 99)
-                            items[i].Remove = true;
-                    }
-
-                    // Set the list back, just in case
-                    Items[key] = items;
-                }
-            }
-            // If we are checking full names, get only files found in directory
             else
-            {
+                Items.BucketBy(Field.Machine_Name, DedupeType.Full);
-                var keys = Items.SortedKeys.ToList();
-                foreach (string key in keys)
-                {
-                    List<DatItem> items = Items[key];
-                    List<DatItem> newItems = DatItem.Merge(items);
-                    for (int i = 0; i < newItems.Count; i++)
-                    {
-                        // Unmatched items will have a source ID of 99, remove all others
-                        if (newItems[i].Source.Index != 99)
-                            newItems[i].Remove = true;
-                    }
 
-                    // Set the list back, just in case
+            // Then mark items for removal
-                    Items[key] = newItems;
+            var keys = Items.SortedKeys.ToList();
+            foreach (string key in keys)
+            {
+                List<DatItem> items = Items[key];
+                for (int i = 0; i < items.Count; i++)
+                {
+                    // Unmatched items will have a source ID of 99, remove all others
+                    if (items[i].Source.Index != 99)
+                        items[i].Remove = true;
                 }
+
+                // Set the list back, just in case
+                Items[key] = items;
             }
 
             // Set fixdat headers in case of writing out
@@ -3618,7 +3596,7 @@ namespace SabreTools.Library.DatFiles
             string post = CreatePrefixPostfix(item, false);
 
             // If we're in Depot mode, take care of that instead
-            if (Header.OutputDepot?.IsActive ?? false)
+            if (Header.OutputDepot?.IsActive == true)
             {
                 if (item.ItemType == ItemType.Disk)
                 {

SabreTools.Library/DatFiles/ItemDictionary.cs

@@ -373,6 +373,27 @@ namespace SabreTools.Library.DatFiles
             return items[key].Remove(value);
         }
 
+        /// <summary>
+        /// Reset a key from the file dictionary if it exists
+        /// </summary>
+        /// <param name="key">Key in the dictionary to reset</param>
+        public bool Reset(string key)
+        {
+            // If the key doesn't exist, return
+            if (!ContainsKey(key))
+                return false;
+
+            // Remove the statistics first
+            foreach (DatItem item in items[key])
+            {
+                RemoveItemStatistics(item);
+            }
+
+            // Remove the key from the dictionary
+            items[key] = new List<DatItem>();
+            return true;
+        }
+
         /// <summary>
         /// Override the internal Field value
         /// </summary>
@@ -658,10 +679,11 @@ namespace SabreTools.Library.DatFiles
                 // Set the sorted type
                 mergedBy = dedupeType;
 
-                Parallel.ForEach(Keys, Globals.ParallelOptions, key =>
+                List<string> keys = Keys.ToList();
+                Parallel.ForEach(keys, Globals.ParallelOptions, key =>
                 {
                     // Get the possibly unsorted list
-                    List<DatItem> sortedlist = this[key];
+                    List<DatItem> sortedlist = this[key].ToList();
 
                     // Sort the list of items to be consistent
                     DatItem.Sort(ref sortedlist, false);
@@ -671,14 +693,15 @@ namespace SabreTools.Library.DatFiles
                         sortedlist = DatItem.Merge(sortedlist);
 
                     // Add the list back to the dictionary
-                    Remove(key);
+                    Reset(key);
                     AddRange(key, sortedlist);
                 });
             }
             // If the merge type is the same, we want to sort the dictionary to be consistent
             else
             {
-                Parallel.ForEach(Keys, Globals.ParallelOptions, key =>
+                List<string> keys = Keys.ToList();
+                Parallel.ForEach(keys, Globals.ParallelOptions, key =>
                 {
                     // Get the possibly unsorted list
                     List<DatItem> sortedlist = this[key];
@@ -811,9 +834,12 @@ namespace SabreTools.Library.DatFiles
         /// </summary>
         private void ClearEmpty()
         {
-            var keys = items.Keys.ToList();
+            var keys = items.Keys.Where(k => k != null).ToList();
             foreach (string key in keys)
             {
+                if (!items.ContainsKey(key))
+                    continue;
+
                 if (items[key] == null || items[key].Count == 0)
                     items.Remove(key);
             }

SabreTools.Library/DatFiles/Missfile.cs

@@ -136,7 +136,7 @@ namespace SabreTools.Library.DatFiles
                 ProcessItemName(datItem, false, forceRomName: false);
 
                 // Romba mode automatically uses item name
-                if ((Header.OutputDepot?.IsActive ?? false) || Header.UseRomName)
+                if (Header.OutputDepot?.IsActive == true || Header.UseRomName)
                 {
                     sw.Write($"{datItem.Name}\n");
                 }

SabreTools.Library/DatItems/DatItem.cs

@@ -1211,7 +1211,7 @@ namespace SabreTools.Library.DatItems
         }
 
         /// <summary>
-        /// Sort a list of File objects by SystemID, SourceID, Game, and Name (in order)
+        /// Sort a list of File objects by SourceID, Game, and Name (in order)
         /// </summary>
         /// <param name="roms">List of File objects representing the roms to be sorted</param>
         /// <param name="norename">True if files are not renamed, false otherwise</param>

SabreTools.Library/External/Compress/ZipFile/ZLib/ZlibBaseStream.cs

@@ -321,8 +321,7 @@ namespace Compress.ZipFile.ZLib
 
         public override System.Int64 Seek(System.Int64 offset, System.IO.SeekOrigin origin)
         {
-            throw new NotImplementedException();
+            return _stream.Seek(offset, origin);
-            //_outStream.Seek(offset, origin);
         }
         public override void SetLength(System.Int64 value)
         {

SabreTools.Library/FileTypes/AaruFormat.cs

@@ -70,7 +70,7 @@ namespace SabreTools.Library.FileTypes
             {
                 // Validate that this is actually a valid AaruFormat (by magic string alone)
                 bool validated = ValidateHeader(aarustream);
-                aarustream.Seek(-8, SeekOrigin.Current); // Seek back to start
+                aarustream.SeekIfPossible(); // Seek back to start
                 if (!validated)
                     return null;
 

SabreTools.Library/FileTypes/CHDFile.cs

@@ -46,7 +46,7 @@ namespace SabreTools.Library.FileTypes
             {
                 // Read the standard CHD headers
                 (char[] tag, uint length, uint version) = GetHeaderValues(chdstream);
-                chdstream.Seek(-16, SeekOrigin.Current); // Seek back to start
+                chdstream.SeekIfPossible(); // Seek back to start
 
                 // Validate that this is actually a valid CHD
                 uint validatedVersion = ValidateHeader(tag, length, version);

SabreTools.Library/FileTypes/GZipArchive.cs

@@ -231,7 +231,7 @@ namespace SabreTools.Library.FileTypes
                             var gz = new gZip();
                             ZipReturn ret = gz.ZipFileOpen(this.Filename);
                             ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize);
-                            BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan);
+                            BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                             gzipEntryRom.Filename = gz.Filename(0);
                             gzipEntryRom.Parent = gamename;
                             gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null);

SabreTools.Library/FileTypes/RarArchive.cs

@@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes
                     else
                     {
                         Stream entryStream = entry.OpenEntryStream();
-                        BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan);
+                        BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                         rarEntryRom.Filename = entry.Key;
                         rarEntryRom.Parent = gamename;
                         rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss");

SabreTools.Library/FileTypes/SevenZipArchive.cs

@@ -316,7 +316,7 @@ namespace SabreTools.Library.FileTypes
                     // Otherwise, use the stream directly
                     else
                     {
-                        BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true);
+                        BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                         zipEntryRom.Filename = zf.Filename(i);
                         zipEntryRom.Parent = gamename;
                         found.Add(zipEntryRom);

SabreTools.Library/FileTypes/TapeArchive.cs

@@ -209,7 +209,7 @@ namespace SabreTools.Library.FileTypes
                     else
                     {
                         Stream entryStream = entry.OpenEntryStream();
-                        BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan);
+                        BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                         tarEntryRom.Filename = entry.Key;
                         tarEntryRom.Parent = gamename;
                         tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss");

SabreTools.Library/FileTypes/XZArchive.cs

@@ -221,7 +221,7 @@ namespace SabreTools.Library.FileTypes
                         else
                         {
                             var xzStream = new XZStream(File.OpenRead(this.Filename));
-                            BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan);
+                            BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                             xzEntryRom.Filename = gamename;
                             xzEntryRom.Parent = gamename;
                             _children.Add(xzEntryRom);

SabreTools.Library/FileTypes/ZipArchive.cs

@@ -319,7 +319,7 @@ namespace SabreTools.Library.FileTypes
                     // Otherwise, use the stream directly
                     else
                     {
-                        BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true);
+                        BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs);
                         zipEntryRom.Filename = zf.Filename(i);
                         zipEntryRom.Parent = gamename;
                         string convertedDate = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss");

SabreTools.Library/IO/StreamExtensions.cs

@@ -209,28 +209,30 @@ namespace SabreTools.Library.IO
         /// </summary>
         /// <param name="input">Input stream to try seeking on</param>
         /// <param name="offset">Optional offset to seek to</param>
-        private static long SeekIfPossible(this Stream input, long offset = 0)
+        public static long SeekIfPossible(this Stream input, long offset = 0)
         {
-            if (input.CanSeek)
+            try
             {
-                try
+                if (input.CanSeek)
                 {
                     if (offset < 0)
                         return input.Seek(offset, SeekOrigin.End);
                     else if (offset >= 0)
                         return input.Seek(offset, SeekOrigin.Begin);
                 }
-                catch (NotSupportedException)
+
-                {
+                return input.Position;
-                    Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed");
+            }
-                }
+            catch (NotSupportedException)
-                catch (NotImplementedException)
+            {
-                {
+                Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed");
-                    Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed");
+            }
-                }
+            catch (NotImplementedException)
+            {
+                Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed");
             }
 
-            return input.Position;
+            return -1;
         }
     }
 }

SabreTools/Features/Sort.cs

@@ -68,13 +68,17 @@ namespace SabreTools.Features
             bool updateDat = GetBoolean(features, UpdateDatValue);
             var outputFormat = GetOutputFormat(features);
 
-            // If we have TorrentGzip output and the romba flag, update
+            // If we have the romba flag
-            if ((Header.OutputDepot?.IsActive ?? false) && outputFormat == OutputFormat.TorrentGzip)
+            if (Header.OutputDepot?.IsActive == true)
-                outputFormat = OutputFormat.TorrentGzipRomba;
+            {
+                // Update TorrentGzip output
+                if (outputFormat == OutputFormat.TorrentGzip)
+                    outputFormat = OutputFormat.TorrentGzipRomba;
 
-            // If we hae TorrentXZ output and the romba flag, update
+                // Update TorrentXz output
-            if ((Header.OutputDepot?.IsActive ?? false) && outputFormat == OutputFormat.TorrentXZ)
+                else if (outputFormat == OutputFormat.TorrentXZ)
-                outputFormat = OutputFormat.TorrentXZRomba;
+                    outputFormat = OutputFormat.TorrentXZRomba;
+            }
 
             // Get a list of files from the input datfiles
             var datfiles = GetList(features, DatListValue);