From b415ff5c4e3dc9818c03b91528f62f6b1d7d4cce Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sat, 13 Nov 2021 19:27:46 +0000 Subject: [PATCH] Enable calculating entropy of byte addressable images. --- Aaru.Core/Entropy.cs | 373 +++++++++++++++++++-------------- Aaru/Commands/Image/Entropy.cs | 8 +- 2 files changed, 216 insertions(+), 165 deletions(-) diff --git a/Aaru.Core/Entropy.cs b/Aaru.Core/Entropy.cs index 45a1f841f..1b94f0889 100644 --- a/Aaru.Core/Entropy.cs +++ b/Aaru.Core/Entropy.cs @@ -40,197 +40,244 @@ using Aaru.CommonTypes.Interfaces; using Aaru.CommonTypes.Structs; using Aaru.Console; -namespace Aaru.Core +namespace Aaru.Core; + +/// Media image entropy operations +public sealed class Entropy { - /// Media image entropy operations - public sealed class Entropy + readonly bool _debug; + readonly IBaseImage _inputFormat; + + /// Initializes an instance with the specified parameters + /// Debug enabled + /// Media image + public Entropy(bool debug, IBaseImage inputFormat) { - readonly bool _debug; - readonly IMediaImage _inputFormat; + _debug = debug; + _inputFormat = inputFormat; + } - /// Initializes an instance with the specified parameters - /// Debug enabled - /// Media image - public Entropy(bool debug, IMediaImage inputFormat) + /// Event raised when a progress bar is needed + public event InitProgressHandler InitProgressEvent; + /// Event raised to update the values of a determinate progress bar + public event UpdateProgressHandler UpdateProgressEvent; + /// Event raised when the progress bar is not longer needed + public event EndProgressHandler EndProgressEvent; + /// Event raised when a progress bar is needed + public event InitProgressHandler InitProgress2Event; + /// Event raised to update the values of a determinate progress bar + public event UpdateProgressHandler UpdateProgress2Event; + /// Event raised when the progress bar is not longer needed + public event EndProgressHandler EndProgress2Event; + + /// Calculates the tracks entropy + /// Checks for duplicated sectors + /// Calculated entropy + public EntropyResults[] CalculateTracksEntropy(bool duplicatedSectors) + { + List entropyResults = new(); + + if(_inputFormat is not IOpticalMediaImage opticalMediaImage) { - _debug = debug; - _inputFormat = inputFormat; - } - - /// Event raised when a progress bar is needed - public event InitProgressHandler InitProgressEvent; - /// Event raised to update the values of a determinate progress bar - public event UpdateProgressHandler UpdateProgressEvent; - /// Event raised when the progress bar is not longer needed - public event EndProgressHandler EndProgressEvent; - /// Event raised when a progress bar is needed - public event InitProgressHandler InitProgress2Event; - /// Event raised to update the values of a determinate progress bar - public event UpdateProgressHandler UpdateProgress2Event; - /// Event raised when the progress bar is not longer needed - public event EndProgressHandler EndProgress2Event; - - /// Calculates the tracks entropy - /// Checks for duplicated sectors - /// Calculated entropy - public EntropyResults[] CalculateTracksEntropy(bool duplicatedSectors) - { - List entropyResults = new(); - - if(!(_inputFormat is IOpticalMediaImage opticalMediaImage)) - { - AaruConsole.ErrorWriteLine("The selected image does not support tracks."); - - return entropyResults.ToArray(); - } - - try - { - List inputTracks = opticalMediaImage.Tracks; - - InitProgressEvent?.Invoke(); - - foreach(Track currentTrack in inputTracks) - { - var trackEntropy = new EntropyResults - { - Track = currentTrack.Sequence, - Entropy = 0 - }; - - UpdateProgressEvent?. - Invoke($"Entropying track {currentTrack.Sequence} of {inputTracks.Max(t => t.Sequence)}", - currentTrack.Sequence, inputTracks.Max(t => t.Sequence)); - - ulong[] entTable = new ulong[256]; - ulong trackSize = 0; - List uniqueSectorsPerTrack = new(); - - trackEntropy.Sectors = currentTrack.EndSector - currentTrack.StartSector + 1; - - AaruConsole.VerboseWriteLine("Track {0} has {1} sectors", currentTrack.Sequence, - trackEntropy.Sectors); - - InitProgress2Event?.Invoke(); - - for(ulong i = 0; i < trackEntropy.Sectors; i++) - { - UpdateProgress2Event?.Invoke($"Entropying sector {i + 1} of track {currentTrack.Sequence}", - (long)(i + 1), (long)currentTrack.EndSector); - - ErrorNumber errno = opticalMediaImage.ReadSector(i, currentTrack.Sequence, out byte[] sector); - - if(errno != ErrorNumber.NoError) - { - AaruConsole.ErrorWriteLine($"Error {errno} while reading sector {i}, continuing..."); - - continue; - } - - if(duplicatedSectors) - { - string sectorHash = Sha1Context.Data(sector, out _); - - if(!uniqueSectorsPerTrack.Contains(sectorHash)) - uniqueSectorsPerTrack.Add(sectorHash); - } - - foreach(byte b in sector) - entTable[b]++; - - trackSize += (ulong)sector.LongLength; - } - - EndProgress2Event?.Invoke(); - - trackEntropy.Entropy += entTable.Select(l => l / (double)trackSize). - Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum(); - - if(duplicatedSectors) - trackEntropy.UniqueSectors = uniqueSectorsPerTrack.Count; - - entropyResults.Add(trackEntropy); - } - - EndProgressEvent?.Invoke(); - } - catch(Exception ex) - { - if(_debug) - AaruConsole.DebugWriteLine("Could not get tracks because {0}", ex.Message); - else - AaruConsole.ErrorWriteLine("Unable to get separate tracks, not calculating their entropy"); - } + AaruConsole.ErrorWriteLine("The selected image does not support tracks."); return entropyResults.ToArray(); } - /// Calculates the media entropy - /// Checks for duplicated sectors - /// Calculated entropy - public EntropyResults CalculateMediaEntropy(bool duplicatedSectors) + try { - var entropy = new EntropyResults - { - Entropy = 0 - }; + List inputTracks = opticalMediaImage.Tracks; - ulong[] entTable = new ulong[256]; - ulong diskSize = 0; - List uniqueSectors = new(); - - entropy.Sectors = _inputFormat.Info.Sectors; - AaruConsole.WriteLine("Sectors {0}", entropy.Sectors); InitProgressEvent?.Invoke(); - for(ulong i = 0; i < entropy.Sectors; i++) + foreach(Track currentTrack in inputTracks) { - UpdateProgressEvent?.Invoke($"Entropying sector {i + 1}", (long)(i + 1), (long)entropy.Sectors); - ErrorNumber errno = _inputFormat.ReadSector(i, out byte[] sector); - - if(errno != ErrorNumber.NoError) + var trackEntropy = new EntropyResults { - AaruConsole.ErrorWriteLine($"Error {errno} while reading sector {i}, continuing..."); + Track = currentTrack.Sequence, + Entropy = 0 + }; - continue; + UpdateProgressEvent?. + Invoke($"Entropying track {currentTrack.Sequence} of {inputTracks.Max(t => t.Sequence)}", + currentTrack.Sequence, inputTracks.Max(t => t.Sequence)); + + ulong[] entTable = new ulong[256]; + ulong trackSize = 0; + List uniqueSectorsPerTrack = new(); + + trackEntropy.Sectors = currentTrack.EndSector - currentTrack.StartSector + 1; + + AaruConsole.VerboseWriteLine("Track {0} has {1} sectors", currentTrack.Sequence, trackEntropy.Sectors); + + InitProgress2Event?.Invoke(); + + for(ulong i = 0; i < trackEntropy.Sectors; i++) + { + UpdateProgress2Event?.Invoke($"Entropying sector {i + 1} of track {currentTrack.Sequence}", + (long)(i + 1), (long)currentTrack.EndSector); + + ErrorNumber errno = opticalMediaImage.ReadSector(i, currentTrack.Sequence, out byte[] sector); + + if(errno != ErrorNumber.NoError) + { + AaruConsole.ErrorWriteLine($"Error {errno} while reading sector {i}, continuing..."); + + continue; + } + + if(duplicatedSectors) + { + string sectorHash = Sha1Context.Data(sector, out _); + + if(!uniqueSectorsPerTrack.Contains(sectorHash)) + uniqueSectorsPerTrack.Add(sectorHash); + } + + foreach(byte b in sector) + entTable[b]++; + + trackSize += (ulong)sector.LongLength; } + EndProgress2Event?.Invoke(); + + trackEntropy.Entropy += entTable.Select(l => l / (double)trackSize). + Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum(); + if(duplicatedSectors) - { - string sectorHash = Sha1Context.Data(sector, out _); + trackEntropy.UniqueSectors = uniqueSectorsPerTrack.Count; - if(!uniqueSectors.Contains(sectorHash)) - uniqueSectors.Add(sectorHash); - } - - foreach(byte b in sector) - entTable[b]++; - - diskSize += (ulong)sector.LongLength; + entropyResults.Add(trackEntropy); } EndProgressEvent?.Invoke(); + } + catch(Exception ex) + { + if(_debug) + AaruConsole.DebugWriteLine("Could not get tracks because {0}", ex.Message); + else + AaruConsole.ErrorWriteLine("Unable to get separate tracks, not calculating their entropy"); + } - entropy.Entropy += entTable.Select(l => l / (double)diskSize). - Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum(); + return entropyResults.ToArray(); + } + + /// Calculates the media entropy for block addressable media + /// Checks for duplicated sectors + /// Calculated entropy + public EntropyResults CalculateMediaEntropy(bool duplicatedSectors) + { + var entropy = new EntropyResults + { + Entropy = 0 + }; + + if(_inputFormat is not IMediaImage mediaImage) + return entropy; + + ulong[] entTable = new ulong[256]; + ulong diskSize = 0; + List uniqueSectors = new(); + + entropy.Sectors = mediaImage.Info.Sectors; + AaruConsole.WriteLine("Sectors {0}", entropy.Sectors); + InitProgressEvent?.Invoke(); + + for(ulong i = 0; i < entropy.Sectors; i++) + { + UpdateProgressEvent?.Invoke($"Entropying sector {i + 1}", (long)(i + 1), (long)entropy.Sectors); + ErrorNumber errno = mediaImage.ReadSector(i, out byte[] sector); + + if(errno != ErrorNumber.NoError) + { + AaruConsole.ErrorWriteLine($"Error {errno} while reading sector {i}, continuing..."); + + continue; + } if(duplicatedSectors) - entropy.UniqueSectors = uniqueSectors.Count; + { + string sectorHash = Sha1Context.Data(sector, out _); + + if(!uniqueSectors.Contains(sectorHash)) + uniqueSectors.Add(sectorHash); + } + + foreach(byte b in sector) + entTable[b]++; + + diskSize += (ulong)sector.LongLength; + } + + EndProgressEvent?.Invoke(); + + entropy.Entropy += entTable.Select(l => l / (double)diskSize). + Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum(); + + if(duplicatedSectors) + entropy.UniqueSectors = uniqueSectors.Count; + + return entropy; + } + + /// Calculates the media entropy for byte addressable media + /// Calculated entropy + public EntropyResults CalculateLinearMediaEntropy() + { + var entropy = new EntropyResults + { + Entropy = 0 + }; + + if(_inputFormat is not IByteAddressableImage byteAddressableImage) + return entropy; + + ulong[] entTable = new ulong[256]; + byte[] data = new byte[byteAddressableImage.Info.Sectors]; + + entropy.Sectors = _inputFormat.Info.Sectors; + AaruConsole.WriteLine("{0} bytes", entropy.Sectors); + InitProgressEvent?.Invoke(); + + ErrorNumber errno = byteAddressableImage.ReadBytes(data, 0, data.Length, out int bytesRead); + + if(errno != ErrorNumber.NoError) + { + AaruConsole.ErrorWriteLine($"Error {errno} while reading data, not continuing..."); return entropy; } - } - /// Entropy results - public struct EntropyResults - { - /// Track number, if applicable - public uint Track; - /// Entropy - public double Entropy; - /// Number of unique sectors - public int? UniqueSectors; - /// Number of total sectors - public ulong Sectors; + if(bytesRead != data.Length) + { + byte[] tmp = new byte[bytesRead]; + Array.Copy(data, 0, tmp, 0, bytesRead); + data = tmp; + } + + foreach(byte b in data) + entTable[b]++; + + EndProgressEvent?.Invoke(); + + entropy.Entropy += entTable.Select(l => l / (double)data.Length). + Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum(); + + return entropy; } +} + +/// Entropy results +public struct EntropyResults +{ + /// Track number, if applicable + public uint Track; + /// Entropy + public double Entropy; + /// Number of unique sectors + public int? UniqueSectors; + /// Number of total sectors + public ulong Sectors; } \ No newline at end of file diff --git a/Aaru/Commands/Image/Entropy.cs b/Aaru/Commands/Image/Entropy.cs index 3c46d1810..7c8ca264b 100644 --- a/Aaru/Commands/Image/Entropy.cs +++ b/Aaru/Commands/Image/Entropy.cs @@ -140,7 +140,7 @@ namespace Aaru.Commands.Image return (int)ErrorNumber.CannotOpenFile; } - IMediaImage inputFormat = null; + IBaseImage inputFormat = null; Core.Spectre.ProgressSingleSpinner(ctx => { @@ -253,7 +253,11 @@ namespace Aaru.Commands.Image if(!wholeDisc) return; - EntropyResults entropy = entropyCalculator.CalculateMediaEntropy(duplicatedSectors); + EntropyResults entropy; + + entropy = inputFormat.Info.XmlMediaType == XmlMediaType.LinearMedia + ? entropyCalculator.CalculateLinearMediaEntropy() + : entropyCalculator.CalculateMediaEntropy(duplicatedSectors); AaruConsole.WriteLine("Entropy for disk is {0:F4}.", entropy.Entropy);