From 4bf5a835e7c095be587ee1f3ca7517aaea300a9f Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Fri, 28 Aug 2020 01:13:55 -0700 Subject: [PATCH] Fix Sort --- SabreTools.Library/DatFiles/DatFile.cs | 62 ++++++------------- SabreTools.Library/DatFiles/ItemDictionary.cs | 36 +++++++++-- SabreTools.Library/DatFiles/Missfile.cs | 2 +- SabreTools.Library/DatItems/DatItem.cs | 2 +- .../Compress/ZipFile/ZLib/ZlibBaseStream.cs | 3 +- SabreTools.Library/FileTypes/AaruFormat.cs | 2 +- SabreTools.Library/FileTypes/CHDFile.cs | 2 +- SabreTools.Library/FileTypes/GZipArchive.cs | 2 +- SabreTools.Library/FileTypes/RarArchive.cs | 2 +- .../FileTypes/SevenZipArchive.cs | 2 +- SabreTools.Library/FileTypes/TapeArchive.cs | 2 +- SabreTools.Library/FileTypes/XZArchive.cs | 2 +- SabreTools.Library/FileTypes/ZipArchive.cs | 2 +- SabreTools.Library/IO/StreamExtensions.cs | 26 ++++---- SabreTools/Features/Sort.cs | 16 +++-- 15 files changed, 86 insertions(+), 77 deletions(-) diff --git a/SabreTools.Library/DatFiles/DatFile.cs b/SabreTools.Library/DatFiles/DatFile.cs index 260c955d..a4038d9b 100644 --- a/SabreTools.Library/DatFiles/DatFile.cs +++ b/SabreTools.Library/DatFiles/DatFile.cs @@ -1908,7 +1908,7 @@ namespace SabreTools.Library.DatFiles }); // Now find all folders that are empty, if we are supposed to - if (!(Header.OutputDepot?.IsActive ?? false) && addBlanks) + if (Header.OutputDepot?.IsActive != true && addBlanks) { List empties = DirectoryExtensions.ListEmpty(basePath); Parallel.ForEach(empties, Globals.ParallelOptions, dir => @@ -1979,7 +1979,7 @@ namespace SabreTools.Library.DatFiles bool copyFiles) { // Special case for if we are in Depot mode (all names are supposed to be SHA-1 hashes) - if (Header.OutputDepot?.IsActive ?? false) + if (Header.OutputDepot?.IsActive == true) { GZipArchive gzarc = new GZipArchive(item); BaseFile baseFile = gzarc.GetTorrentGZFileInfo(); @@ -2980,11 +2980,7 @@ namespace SabreTools.Library.DatFiles /// True to enable external scanning of archives, false otherwise /// TreatAsFiles representing CHD and Archive scanning /// True if verification was a success, false otherwise - public bool VerifyGeneric( - List inputs, - bool hashOnly = false, - bool quickScan = false, - TreatAsFiles asFiles = 0x00) + public bool VerifyGeneric(List inputs, bool hashOnly, bool quickScan, TreatAsFiles asFiles = 0x00) { bool success = true; @@ -2996,45 +2992,27 @@ namespace SabreTools.Library.DatFiles PopulateFromDir(input, quickScan ? Hash.SecureHashes : Hash.DeepHashes, asFiles: asFiles); } - // If we are checking hashes only, essentially diff the inputs + // Force bucketing according to the flags + Items.SetBucketedBy(Field.NULL); if (hashOnly) - { - // First we need to bucket and dedupe by hash to get duplicates Items.BucketBy(Field.DatItem_CRC, DedupeType.Full); - - var keys = Items.SortedKeys.ToList(); - foreach (string key in keys) - { - List items = Items[key]; - for (int i = 0; i < items.Count; i++) - { - // Unmatched items will have a source ID of 99, remove all others - if (items[i].Source.Index != 99) - items[i].Remove = true; - } - - // Set the list back, just in case - Items[key] = items; - } - } - // If we are checking full names, get only files found in directory else - { - var keys = Items.SortedKeys.ToList(); - foreach (string key in keys) - { - List items = Items[key]; - List newItems = DatItem.Merge(items); - for (int i = 0; i < newItems.Count; i++) - { - // Unmatched items will have a source ID of 99, remove all others - if (newItems[i].Source.Index != 99) - newItems[i].Remove = true; - } + Items.BucketBy(Field.Machine_Name, DedupeType.Full); - // Set the list back, just in case - Items[key] = newItems; + // Then mark items for removal + var keys = Items.SortedKeys.ToList(); + foreach (string key in keys) + { + List items = Items[key]; + for (int i = 0; i < items.Count; i++) + { + // Unmatched items will have a source ID of 99, remove all others + if (items[i].Source.Index != 99) + items[i].Remove = true; } + + // Set the list back, just in case + Items[key] = items; } // Set fixdat headers in case of writing out @@ -3618,7 +3596,7 @@ namespace SabreTools.Library.DatFiles string post = CreatePrefixPostfix(item, false); // If we're in Depot mode, take care of that instead - if (Header.OutputDepot?.IsActive ?? false) + if (Header.OutputDepot?.IsActive == true) { if (item.ItemType == ItemType.Disk) { diff --git a/SabreTools.Library/DatFiles/ItemDictionary.cs b/SabreTools.Library/DatFiles/ItemDictionary.cs index cb919091..58a5972e 100644 --- a/SabreTools.Library/DatFiles/ItemDictionary.cs +++ b/SabreTools.Library/DatFiles/ItemDictionary.cs @@ -373,6 +373,27 @@ namespace SabreTools.Library.DatFiles return items[key].Remove(value); } + /// + /// Reset a key from the file dictionary if it exists + /// + /// Key in the dictionary to reset + public bool Reset(string key) + { + // If the key doesn't exist, return + if (!ContainsKey(key)) + return false; + + // Remove the statistics first + foreach (DatItem item in items[key]) + { + RemoveItemStatistics(item); + } + + // Remove the key from the dictionary + items[key] = new List(); + return true; + } + /// /// Override the internal Field value /// @@ -658,10 +679,11 @@ namespace SabreTools.Library.DatFiles // Set the sorted type mergedBy = dedupeType; - Parallel.ForEach(Keys, Globals.ParallelOptions, key => + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Get the possibly unsorted list - List sortedlist = this[key]; + List sortedlist = this[key].ToList(); // Sort the list of items to be consistent DatItem.Sort(ref sortedlist, false); @@ -671,14 +693,15 @@ namespace SabreTools.Library.DatFiles sortedlist = DatItem.Merge(sortedlist); // Add the list back to the dictionary - Remove(key); + Reset(key); AddRange(key, sortedlist); }); } // If the merge type is the same, we want to sort the dictionary to be consistent else { - Parallel.ForEach(Keys, Globals.ParallelOptions, key => + List keys = Keys.ToList(); + Parallel.ForEach(keys, Globals.ParallelOptions, key => { // Get the possibly unsorted list List sortedlist = this[key]; @@ -811,9 +834,12 @@ namespace SabreTools.Library.DatFiles /// private void ClearEmpty() { - var keys = items.Keys.ToList(); + var keys = items.Keys.Where(k => k != null).ToList(); foreach (string key in keys) { + if (!items.ContainsKey(key)) + continue; + if (items[key] == null || items[key].Count == 0) items.Remove(key); } diff --git a/SabreTools.Library/DatFiles/Missfile.cs b/SabreTools.Library/DatFiles/Missfile.cs index 6ef017c3..4a507cb1 100644 --- a/SabreTools.Library/DatFiles/Missfile.cs +++ b/SabreTools.Library/DatFiles/Missfile.cs @@ -136,7 +136,7 @@ namespace SabreTools.Library.DatFiles ProcessItemName(datItem, false, forceRomName: false); // Romba mode automatically uses item name - if ((Header.OutputDepot?.IsActive ?? false) || Header.UseRomName) + if (Header.OutputDepot?.IsActive == true || Header.UseRomName) { sw.Write($"{datItem.Name}\n"); } diff --git a/SabreTools.Library/DatItems/DatItem.cs b/SabreTools.Library/DatItems/DatItem.cs index 1194df64..e5058522 100644 --- a/SabreTools.Library/DatItems/DatItem.cs +++ b/SabreTools.Library/DatItems/DatItem.cs @@ -1211,7 +1211,7 @@ namespace SabreTools.Library.DatItems } /// - /// Sort a list of File objects by SystemID, SourceID, Game, and Name (in order) + /// Sort a list of File objects by SourceID, Game, and Name (in order) /// /// List of File objects representing the roms to be sorted /// True if files are not renamed, false otherwise diff --git a/SabreTools.Library/External/Compress/ZipFile/ZLib/ZlibBaseStream.cs b/SabreTools.Library/External/Compress/ZipFile/ZLib/ZlibBaseStream.cs index 8d80b07e..1eba6572 100644 --- a/SabreTools.Library/External/Compress/ZipFile/ZLib/ZlibBaseStream.cs +++ b/SabreTools.Library/External/Compress/ZipFile/ZLib/ZlibBaseStream.cs @@ -321,8 +321,7 @@ namespace Compress.ZipFile.ZLib public override System.Int64 Seek(System.Int64 offset, System.IO.SeekOrigin origin) { - throw new NotImplementedException(); - //_outStream.Seek(offset, origin); + return _stream.Seek(offset, origin); } public override void SetLength(System.Int64 value) { diff --git a/SabreTools.Library/FileTypes/AaruFormat.cs b/SabreTools.Library/FileTypes/AaruFormat.cs index 9e68d724..c9383086 100644 --- a/SabreTools.Library/FileTypes/AaruFormat.cs +++ b/SabreTools.Library/FileTypes/AaruFormat.cs @@ -70,7 +70,7 @@ namespace SabreTools.Library.FileTypes { // Validate that this is actually a valid AaruFormat (by magic string alone) bool validated = ValidateHeader(aarustream); - aarustream.Seek(-8, SeekOrigin.Current); // Seek back to start + aarustream.SeekIfPossible(); // Seek back to start if (!validated) return null; diff --git a/SabreTools.Library/FileTypes/CHDFile.cs b/SabreTools.Library/FileTypes/CHDFile.cs index 228457f1..82b1d4ad 100644 --- a/SabreTools.Library/FileTypes/CHDFile.cs +++ b/SabreTools.Library/FileTypes/CHDFile.cs @@ -46,7 +46,7 @@ namespace SabreTools.Library.FileTypes { // Read the standard CHD headers (char[] tag, uint length, uint version) = GetHeaderValues(chdstream); - chdstream.Seek(-16, SeekOrigin.Current); // Seek back to start + chdstream.SeekIfPossible(); // Seek back to start // Validate that this is actually a valid CHD uint validatedVersion = ValidateHeader(tag, length, version); diff --git a/SabreTools.Library/FileTypes/GZipArchive.cs b/SabreTools.Library/FileTypes/GZipArchive.cs index 20311972..c6b6e6ff 100644 --- a/SabreTools.Library/FileTypes/GZipArchive.cs +++ b/SabreTools.Library/FileTypes/GZipArchive.cs @@ -231,7 +231,7 @@ namespace SabreTools.Library.FileTypes var gz = new gZip(); ZipReturn ret = gz.ZipFileOpen(this.Filename); ret = gz.ZipFileOpenReadStream(0, out Stream gzstream, out ulong streamSize); - BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan); + BaseFile gzipEntryRom = gzstream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); gzipEntryRom.Filename = gz.Filename(0); gzipEntryRom.Parent = gamename; gzipEntryRom.Date = (date && gz.TimeStamp > 0 ? gz.TimeStamp.ToString() : null); diff --git a/SabreTools.Library/FileTypes/RarArchive.cs b/SabreTools.Library/FileTypes/RarArchive.cs index f16c6f45..53cd6ba1 100644 --- a/SabreTools.Library/FileTypes/RarArchive.cs +++ b/SabreTools.Library/FileTypes/RarArchive.cs @@ -204,7 +204,7 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan); + BaseFile rarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); rarEntryRom.Filename = entry.Key; rarEntryRom.Parent = gamename; rarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/SevenZipArchive.cs b/SabreTools.Library/FileTypes/SevenZipArchive.cs index 2e41a20f..38e86c55 100644 --- a/SabreTools.Library/FileTypes/SevenZipArchive.cs +++ b/SabreTools.Library/FileTypes/SevenZipArchive.cs @@ -316,7 +316,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true); + BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; found.Add(zipEntryRom); diff --git a/SabreTools.Library/FileTypes/TapeArchive.cs b/SabreTools.Library/FileTypes/TapeArchive.cs index 2bdbc3cb..35856882 100644 --- a/SabreTools.Library/FileTypes/TapeArchive.cs +++ b/SabreTools.Library/FileTypes/TapeArchive.cs @@ -209,7 +209,7 @@ namespace SabreTools.Library.FileTypes else { Stream entryStream = entry.OpenEntryStream(); - BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan); + BaseFile tarEntryRom = entryStream.GetInfo(size: entry.Size, omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); tarEntryRom.Filename = entry.Key; tarEntryRom.Parent = gamename; tarEntryRom.Date = entry.LastModifiedTime?.ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/FileTypes/XZArchive.cs b/SabreTools.Library/FileTypes/XZArchive.cs index a44a3aef..009b6ac5 100644 --- a/SabreTools.Library/FileTypes/XZArchive.cs +++ b/SabreTools.Library/FileTypes/XZArchive.cs @@ -221,7 +221,7 @@ namespace SabreTools.Library.FileTypes else { var xzStream = new XZStream(File.OpenRead(this.Filename)); - BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan); + BaseFile xzEntryRom = xzStream.GetInfo(omitFromScan: omitFromScan, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); xzEntryRom.Filename = gamename; xzEntryRom.Parent = gamename; _children.Add(xzEntryRom); diff --git a/SabreTools.Library/FileTypes/ZipArchive.cs b/SabreTools.Library/FileTypes/ZipArchive.cs index 3f39ac44..65443e54 100644 --- a/SabreTools.Library/FileTypes/ZipArchive.cs +++ b/SabreTools.Library/FileTypes/ZipArchive.cs @@ -319,7 +319,7 @@ namespace SabreTools.Library.FileTypes // Otherwise, use the stream directly else { - BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true); + BaseFile zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), omitFromScan: omitFromScan, keepReadOpen: true, asFiles: TreatAsFiles.AaruFormats | TreatAsFiles.CHDs); zipEntryRom.Filename = zf.Filename(i); zipEntryRom.Parent = gamename; string convertedDate = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss"); diff --git a/SabreTools.Library/IO/StreamExtensions.cs b/SabreTools.Library/IO/StreamExtensions.cs index 1e01c54f..e1cb41c0 100644 --- a/SabreTools.Library/IO/StreamExtensions.cs +++ b/SabreTools.Library/IO/StreamExtensions.cs @@ -209,28 +209,30 @@ namespace SabreTools.Library.IO /// /// Input stream to try seeking on /// Optional offset to seek to - private static long SeekIfPossible(this Stream input, long offset = 0) + public static long SeekIfPossible(this Stream input, long offset = 0) { - if (input.CanSeek) + try { - try + if (input.CanSeek) { if (offset < 0) return input.Seek(offset, SeekOrigin.End); else if (offset >= 0) return input.Seek(offset, SeekOrigin.Begin); } - catch (NotSupportedException) - { - Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); - } - catch (NotImplementedException) - { - Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed"); - } + + return input.Position; + } + catch (NotSupportedException) + { + Globals.Logger.Verbose("Stream does not support seeking to starting offset. Stream position not changed"); + } + catch (NotImplementedException) + { + Globals.Logger.Warning("Stream does not support seeking to starting offset. Stream position not changed"); } - return input.Position; + return -1; } } } diff --git a/SabreTools/Features/Sort.cs b/SabreTools/Features/Sort.cs index 56ad0ae1..afeee244 100644 --- a/SabreTools/Features/Sort.cs +++ b/SabreTools/Features/Sort.cs @@ -68,13 +68,17 @@ namespace SabreTools.Features bool updateDat = GetBoolean(features, UpdateDatValue); var outputFormat = GetOutputFormat(features); - // If we have TorrentGzip output and the romba flag, update - if ((Header.OutputDepot?.IsActive ?? false) && outputFormat == OutputFormat.TorrentGzip) - outputFormat = OutputFormat.TorrentGzipRomba; + // If we have the romba flag + if (Header.OutputDepot?.IsActive == true) + { + // Update TorrentGzip output + if (outputFormat == OutputFormat.TorrentGzip) + outputFormat = OutputFormat.TorrentGzipRomba; - // If we hae TorrentXZ output and the romba flag, update - if ((Header.OutputDepot?.IsActive ?? false) && outputFormat == OutputFormat.TorrentXZ) - outputFormat = OutputFormat.TorrentXZRomba; + // Update TorrentXz output + else if (outputFormat == OutputFormat.TorrentXZ) + outputFormat = OutputFormat.TorrentXZRomba; + } // Get a list of files from the input datfiles var datfiles = GetList(features, DatListValue);