aaru-dps/Aaru
1
0
Fork 0
mirror of https://github.com/aaru-dps/Aaru.git synced 2025-12-16 19:24:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move entropy calculation to Core.

Browse Source
This commit is contained in:
Natalia Portillo
2018-10-13 14:39:26 +01:00
parent 084e689ff0
commit b693a2e1aa
4 changed files with 201 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/.idea.DiscImageChef/.idea/contentModel.xml generated
View File

@@ -246,6 +246,7 @@
</e>
<e p="DiscImageChef.Core.csproj" t="IncludeRecursive" />
<e p="DiscImageChef.Core.csproj.DotSettings" t="Include" />
<e p="Entropy.cs" t="Include" />
<e p="Filesystems.cs" t="Include" />
<e p="GetPluginBase.cs" t="Include" />
<e p="ImageFormat.cs" t="Include" />

1
DiscImageChef.Core/DiscImageChef.Core.csproj
View File

@@ -49,6 +49,7 @@
<Compile Include="Devices\Info\DeviceInfo.cs" />
<Compile Include="Devices\Info\Plextor.cs" />
<Compile Include="Devices\Info\Properties.cs" />
<Compile Include="Entropy.cs" />
<Compile Include="GetPluginBase.cs" />
<Compile Include="ImageInfo.cs" />
<Compile Include="Media\Detection\MMC.cs" />

174
DiscImageChef.Core/Entropy.cs Normal file
View File

@@ -0,0 +1,174 @@
// /***************************************************************************
// The Disc Image Chef
// ----------------------------------------------------------------------------
//
// Filename : Entropy.cs
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// Component : Core algorithms.
//
// --[ Description ] ----------------------------------------------------------
//
// Calculates the entropy of an image
//
// --[ License ] --------------------------------------------------------------
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2011-2018 Natalia Portillo
// ****************************************************************************/
using System;
using System.Collections.Generic;
using System.Linq;
using DiscImageChef.Checksums;
using DiscImageChef.CommonTypes.Interfaces;
using DiscImageChef.CommonTypes.Structs;
using DiscImageChef.Console;
namespace DiscImageChef.Core
{
public class Entropy
{
bool debug;
IMediaImage inputFormat;
bool verbose;
public Entropy(bool debug, bool verbose, IMediaImage inputFormat)
{
this.debug = debug;
this.verbose = verbose;
this.inputFormat = inputFormat;
}
public event InitProgressHandler InitProgressEvent;
public event UpdateProgressHandler UpdateProgressEvent;
public event EndProgressHandler EndProgressEvent;
public event InitProgressHandler InitProgress2Event;
public event UpdateProgressHandler UpdateProgress2Event;
public event EndProgressHandler EndProgress2Event;
public EntropyResults[] CalculateTracksEntropy(bool duplicatedSectors)
{
List<EntropyResults> entropyResultses = new List<EntropyResults>();
try
{
List<Track> inputTracks = inputFormat.Tracks;
InitProgressEvent?.Invoke();
foreach(Track currentTrack in inputTracks)
{
EntropyResults trackEntropy = new EntropyResults {Track = currentTrack.TrackSequence, Entropy = 0};
UpdateProgressEvent
?.Invoke($"Entropying track {currentTrack.TrackSequence} of {inputTracks.Max(t => t.TrackSequence)}",
currentTrack.TrackSequence, inputTracks.Max(t => t.TrackSequence));
ulong[] entTable = new ulong[256];
ulong trackSize = 0;
List<string> uniqueSectorsPerTrack = new List<string>();
trackEntropy.Sectors = currentTrack.TrackEndSector - currentTrack.TrackStartSector + 1;
DicConsole.VerboseWriteLine("Track {0} has {1} sectors", currentTrack.TrackSequence,
trackEntropy.Sectors);
InitProgress2Event?.Invoke();
for(ulong i = currentTrack.TrackStartSector; i <= currentTrack.TrackEndSector; i++)
{
UpdateProgress2Event
?.Invoke($"Entropying sector {i + 1} of track {currentTrack.TrackSequence}",
(long)(currentTrack.TrackEndSector - (i + 1)),
(long)trackEntropy.Sectors);
byte[] sector = inputFormat.ReadSector(i, currentTrack.TrackSequence);
if(duplicatedSectors)
{
string sectorHash = Sha1Context.Data(sector, out _);
if(!uniqueSectorsPerTrack.Contains(sectorHash)) uniqueSectorsPerTrack.Add(sectorHash);
}
foreach(byte b in sector) entTable[b]++;
trackSize += (ulong)sector.LongLength;
}
EndProgress2Event?.Invoke();
trackEntropy.Entropy += entTable.Select(l => (double)l / (double)trackSize)
.Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum();
if(duplicatedSectors) trackEntropy.UniqueSectors = uniqueSectorsPerTrack.Count;
entropyResultses.Add(trackEntropy);
}
EndProgressEvent?.Invoke();
}
catch(Exception ex)
{
if(debug) DicConsole.DebugWriteLine("Could not get tracks because {0}", ex.Message);
else DicConsole.ErrorWriteLine("Unable to get separate tracks, not calculating their entropy");
}
return entropyResultses.ToArray();
}
public EntropyResults CalculateMediaEntropy(bool duplicatedSectors)
{
EntropyResults entropy = new EntropyResults {Entropy = 0};
ulong[] entTable = new ulong[256];
ulong diskSize = 0;
List<string> uniqueSectors = new List<string>();
entropy.Sectors = inputFormat.Info.Sectors;
DicConsole.WriteLine("Sectors {0}", entropy.Sectors);
InitProgressEvent?.Invoke();
for(ulong i = 0; i < entropy.Sectors; i++)
{
UpdateProgressEvent?.Invoke($"Entropying sector {i + 1}", (long)(i + 1), (long)entropy.Sectors);
byte[] sector = inputFormat.ReadSector(i);
if(duplicatedSectors)
{
string sectorHash = Sha1Context.Data(sector, out _);
if(!uniqueSectors.Contains(sectorHash)) uniqueSectors.Add(sectorHash);
}
foreach(byte b in sector) entTable[b]++;
diskSize += (ulong)sector.LongLength;
}
EndProgressEvent?.Invoke();
entropy.Entropy += entTable.Select(l => (double)l / (double)diskSize)
.Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum();
if(duplicatedSectors) entropy.UniqueSectors = uniqueSectors.Count;
return entropy;
}
}
public struct EntropyResults
{
public uint Track;
public double Entropy;
public int? UniqueSectors;
public ulong Sectors;
}
}

110
DiscImageChef/Commands/Entropy.cs
View File

@@ -30,16 +30,10 @@
// Copyright © 2011-2018 Natalia Portillo
// ****************************************************************************/
using System;
using System.Collections.Generic;
using System.Linq;
using DiscImageChef.Checksums;
using DiscImageChef.CommonTypes;
using DiscImageChef.CommonTypes.Interfaces;
using DiscImageChef.CommonTypes.Structs;
using DiscImageChef.Console;
using DiscImageChef.Core;
using DiscImageChef.Filters;
namespace DiscImageChef.Commands
{
@@ -75,94 +69,40 @@ namespace DiscImageChef.Commands
Core.Statistics.AddMediaFormat(inputFormat.Format);
Core.Statistics.AddMedia(inputFormat.Info.MediaType, false);
Core.Statistics.AddFilter(inputFilter.Name);
double entropy = 0;
ulong[] entTable;
ulong sectors;
Core.Entropy entropyCalculator = new Core.Entropy(options.Debug, options.Verbose, inputFormat);
entropyCalculator.InitProgressEvent += Progress.InitProgress;
entropyCalculator.InitProgress2Event += Progress.InitProgress2;
entropyCalculator.UpdateProgressEvent += Progress.UpdateProgress;
entropyCalculator.UpdateProgress2Event += Progress.UpdateProgress2;
entropyCalculator.EndProgressEvent += Progress.EndProgress;
entropyCalculator.EndProgress2Event += Progress.EndProgress2;
if(options.SeparatedTracks)
try
{
List<Track> inputTracks = inputFormat.Tracks;
foreach(Track currentTrack in inputTracks)
{
entTable = new ulong[256];
ulong trackSize = 0;
List<string> uniqueSectorsPerTrack = new List<string>();
sectors = currentTrack.TrackEndSector - currentTrack.TrackStartSector + 1;
DicConsole.WriteLine("Track {0} has {1} sectors", currentTrack.TrackSequence, sectors);
for(ulong i = currentTrack.TrackStartSector; i <= currentTrack.TrackEndSector; i++)
{
DicConsole.Write("\rEntropying sector {0} of track {1}", i + 1, currentTrack.TrackSequence);
byte[] sector = inputFormat.ReadSector(i, currentTrack.TrackSequence);
if(options.DuplicatedSectors)
{
string sectorHash = Sha1Context.Data(sector, out _);
if(!uniqueSectorsPerTrack.Contains(sectorHash)) uniqueSectorsPerTrack.Add(sectorHash);
}
foreach(byte b in sector) entTable[b]++;
trackSize += (ulong)sector.LongLength;
}
entropy += entTable.Select(l => (double)l / (double)trackSize)
.Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum();
DicConsole.WriteLine("Entropy for track {0} is {1:F4}.", currentTrack.TrackSequence, entropy);
if(options.DuplicatedSectors)
DicConsole.WriteLine("Track {0} has {1} unique sectors ({1:P3})",
currentTrack.TrackSequence, uniqueSectorsPerTrack.Count,
(double)uniqueSectorsPerTrack.Count / (double)sectors);
DicConsole.WriteLine();
}
}
catch(Exception ex)
{
if(options.Debug) DicConsole.DebugWriteLine("Could not get tracks because {0}", ex.Message);
else DicConsole.ErrorWriteLine("Unable to get separate tracks, not calculating their entropy");
}
if(!options.WholeDisc) return;
entTable = new ulong[256];
ulong diskSize = 0;
List<string> uniqueSectors = new List<string>();
sectors = inputFormat.Info.Sectors;
DicConsole.WriteLine("Sectors {0}", sectors);
for(ulong i = 0; i < sectors; i++)
{
DicConsole.Write("\rEntropying sector {0}", i + 1);
byte[] sector = inputFormat.ReadSector(i);
if(options.DuplicatedSectors)
EntropyResults[] tracksEntropy = entropyCalculator.CalculateTracksEntropy(options.DuplicatedSectors);
foreach(EntropyResults trackEntropy in tracksEntropy)
{
string sectorHash = Sha1Context.Data(sector, out _);
if(!uniqueSectors.Contains(sectorHash)) uniqueSectors.Add(sectorHash);
DicConsole.WriteLine("Entropy for track {0} is {1:F4}.", trackEntropy.Track, trackEntropy.Entropy);
if(trackEntropy.UniqueSectors != null)
DicConsole.WriteLine("Track {0} has {1} unique sectors ({1:P3})", trackEntropy.Track,
trackEntropy.UniqueSectors,
(double)trackEntropy.UniqueSectors / (double)trackEntropy.Sectors);
}
foreach(byte b in sector) entTable[b]++;
diskSize += (ulong)sector.LongLength;
}
entropy += entTable.Select(l => (double)l / (double)diskSize)
.Select(frequency => -(frequency * Math.Log(frequency, 2))).Sum();
if(!options.WholeDisc)
{
Core.Statistics.AddCommand("entropy");
return;
}
DicConsole.WriteLine();
EntropyResults entropy = entropyCalculator.CalculateMediaEntropy(options.DuplicatedSectors);
DicConsole.WriteLine("Entropy for disk is {0:F4}.", entropy);
if(options.DuplicatedSectors)
DicConsole.WriteLine("Disk has {0} unique sectors ({1:P3})", uniqueSectors.Count,
(double)uniqueSectors.Count / (double)sectors);
DicConsole.WriteLine("Entropy for disk is {0:F4}.", entropy.Entropy);
if(entropy.UniqueSectors != null)
DicConsole.WriteLine("Disk has {0} unique sectors ({1:P3})", entropy.UniqueSectors,
(double)entropy.UniqueSectors / (double)entropy.Sectors);
Core.Statistics.AddCommand("entropy");
}