From 09ca223f02da117f7aa14a1970652eedc9dd1283 Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Mon, 24 Aug 2020 23:27:03 +0100 Subject: [PATCH] Add option to export ROMs as ZIP files. --- RomRepoMgr.Core/RomRepoMgr.Core.csproj | 1 + RomRepoMgr.Core/StreamWithLength.cs | 43 +++ RomRepoMgr.Core/Workers/FileExporter.cs | 277 +++++++++++++++++++ RomRepoMgr/ViewModels/ExportRomsViewModel.cs | 269 ++++++++++++++++++ RomRepoMgr/ViewModels/MainWindowViewModel.cs | 20 ++ RomRepoMgr/Views/ExportRoms.xaml | 66 +++++ RomRepoMgr/Views/ExportRoms.xaml.cs | 45 +++ RomRepoMgr/Views/MainWindow.xaml | 1 + 8 files changed, 722 insertions(+) create mode 100644 RomRepoMgr.Core/StreamWithLength.cs create mode 100644 RomRepoMgr.Core/Workers/FileExporter.cs create mode 100644 RomRepoMgr/ViewModels/ExportRomsViewModel.cs create mode 100644 RomRepoMgr/Views/ExportRoms.xaml create mode 100644 RomRepoMgr/Views/ExportRoms.xaml.cs diff --git a/RomRepoMgr.Core/RomRepoMgr.Core.csproj b/RomRepoMgr.Core/RomRepoMgr.Core.csproj index 2914fc9..e9ad386 100644 --- a/RomRepoMgr.Core/RomRepoMgr.Core.csproj +++ b/RomRepoMgr.Core/RomRepoMgr.Core.csproj @@ -5,6 +5,7 @@ + diff --git a/RomRepoMgr.Core/StreamWithLength.cs b/RomRepoMgr.Core/StreamWithLength.cs new file mode 100644 index 0000000..dbae417 --- /dev/null +++ b/RomRepoMgr.Core/StreamWithLength.cs @@ -0,0 +1,43 @@ +using System; +using System.IO; + +namespace RomRepoMgr.Core +{ + internal sealed class StreamWithLength : Stream + { + readonly Stream _baseStream; + + public StreamWithLength(Stream baseStream, long length) + { + _baseStream = baseStream; + Length = length; + } + + public override bool CanRead => _baseStream.CanRead; + public override bool CanSeek => _baseStream.CanSeek; + public override bool CanWrite => _baseStream.CanWrite; + public override long Length { get; } + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() => _baseStream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => _baseStream.Read(buffer, offset, count); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void Close() + { + _baseStream.Close(); + base.Close(); + } + } +} \ No newline at end of file diff --git a/RomRepoMgr.Core/Workers/FileExporter.cs b/RomRepoMgr.Core/Workers/FileExporter.cs new file mode 100644 index 0000000..9769a4d --- /dev/null +++ b/RomRepoMgr.Core/Workers/FileExporter.cs @@ -0,0 +1,277 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Ionic.Zip; +using Ionic.Zlib; +using RomRepoMgr.Core.EventArgs; +using RomRepoMgr.Database; +using RomRepoMgr.Database.Models; +using SharpCompress.Compressors.LZMA; +using CompressionMode = SharpCompress.Compressors.CompressionMode; + +namespace RomRepoMgr.Core.Workers +{ + public class FileExporter + { + readonly string _outPath; + readonly long _romSetId; + long _filePosition; + Dictionary _filesByMachine; + long _machinePosition; + Machine[] _machines; + string _zipCurrentEntryName; + + public FileExporter(long romSetId, string outPath) + { + _romSetId = romSetId; + _outPath = outPath; + } + + public event EventHandler WorkFinished; + public event EventHandler SetProgressBounds; + public event EventHandler SetProgress; + public event EventHandler SetMessage; + public event EventHandler SetProgress2Bounds; + public event EventHandler SetProgress2; + public event EventHandler SetMessage2; + public event EventHandler SetProgress3Bounds; + public event EventHandler SetProgress3; + public event EventHandler SetMessage3; + + public void Export() + { + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Retrieving ROM set from database." + }); + + RomSet romSet = Context.Singleton.RomSets.Find(_romSetId); + + if(romSet == null) + { + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Could not ROM set in database." + }); + + WorkFinished?.Invoke(this, System.EventArgs.Empty); + + return; + } + + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Exporting ROMs..." + }); + + _machines = Context.Singleton.Machines.Where(m => m.RomSet.Id == _romSetId).ToArray(); + + SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = _machines.Length + }); + + _machinePosition = 0; + CompressNextMachine(); + } + + void CompressNextMachine() + { + SetProgress?.Invoke(this, new ProgressEventArgs + { + Value = _machinePosition + }); + + if(_machinePosition >= _machines.Length) + { + SetMessage?.Invoke(this, new MessageEventArgs + { + Message = "Finished!" + }); + + WorkFinished?.Invoke(this, System.EventArgs.Empty); + + return; + } + + Machine machine = _machines[_machinePosition]; + + SetMessage2?.Invoke(this, new MessageEventArgs + { + Message = machine.Name + }); + + _filesByMachine = Context.Singleton.FilesByMachines. + Where(f => f.Machine.Id == machine.Id && f.File.IsInRepo). + ToDictionary(f => f.Name); + + if(_filesByMachine.Count == 0) + { + _machinePosition++; + Task.Run(CompressNextMachine); + + return; + } + + SetProgress2Bounds?.Invoke(this, new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = _filesByMachine.Count + }); + + string machineName = machine.Name; + + if(!machineName.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase)) + machineName += ".zip"; + + var zf = new ZipFile(Path.Combine(_outPath, machineName), Encoding.UTF8) + { + CompressionLevel = CompressionLevel.BestCompression, + CompressionMethod = CompressionMethod.Deflate, + EmitTimesInUnixFormatWhenSaving = true, + EmitTimesInWindowsFormatWhenSaving = true, + UseZip64WhenSaving = Zip64Option.AsNecessary, + SortEntriesBeforeSaving = true + }; + + zf.SaveProgress += Zf_SaveProgress; + + foreach(KeyValuePair fileByMachine in _filesByMachine) + { + // Is a directory + if((fileByMachine.Key.EndsWith("/", StringComparison.InvariantCultureIgnoreCase) || + fileByMachine.Key.EndsWith("\\", StringComparison.InvariantCultureIgnoreCase)) && + fileByMachine.Value.File.Size == 0) + { + ZipEntry zd = zf.AddDirectoryByName(fileByMachine.Key.Replace('/', '\\')); + zd.Attributes = FileAttributes.Normal; + zd.CreationTime = DateTime.UtcNow; + zd.AccessedTime = DateTime.UtcNow; + zd.LastModified = DateTime.UtcNow; + zd.ModifiedTime = DateTime.UtcNow; + + continue; + } + + ZipEntry zi = zf.AddEntry(fileByMachine.Key, Zf_HandleOpen, Zf_HandleClose); + zi.Attributes = FileAttributes.Normal; + zi.CreationTime = DateTime.UtcNow; + zi.AccessedTime = DateTime.UtcNow; + zi.LastModified = DateTime.UtcNow; + zi.ModifiedTime = DateTime.UtcNow; + } + + zf.Save(); + } + + Stream Zf_HandleOpen(string entryName) + { + if(!_filesByMachine.TryGetValue(entryName, out FileByMachine fileByMachine)) + if(!_filesByMachine.TryGetValue(entryName.Replace('/', '\\'), out fileByMachine)) + throw new ArgumentException("Cannot find requested zip entry in hashes dictionary"); + + DbFile file = fileByMachine.File; + + // Special case for empty file, as it seems to crash when SharpCompress tries to unLZMA it. + if(file.Size == 0) + return new MemoryStream(); + + byte[] sha384Bytes = new byte[48]; + string sha384 = file.Sha384; + + for(int i = 0; i < 48; i++) + { + if(sha384[i * 2] >= 0x30 && + sha384[i * 2] <= 0x39) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10); + else if(sha384[i * 2] >= 0x41 && + sha384[i * 2] <= 0x46) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x37) * 0x10); + else if(sha384[i * 2] >= 0x61 && + sha384[i * 2] <= 0x66) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x57) * 0x10); + + if(sha384[(i * 2) + 1] >= 0x30 && + sha384[(i * 2) + 1] <= 0x39) + sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x30); + else if(sha384[(i * 2) + 1] >= 0x41 && + sha384[(i * 2) + 1] <= 0x46) + sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x37); + else if(sha384[(i * 2) + 1] >= 0x61 && + sha384[(i * 2) + 1] <= 0x66) + sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x57); + } + + string sha384B32 = Base32.ToBase32String(sha384Bytes); + + string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, "files", sha384B32[0].ToString(), + sha384B32[1].ToString(), sha384B32[2].ToString(), sha384B32[3].ToString(), + sha384B32[4].ToString(), sha384B32 + ".lz"); + + if(!File.Exists(repoPath)) + throw new ArgumentException($"Cannot find file with hash {file.Sha256} in the repository"); + + var inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); + + return new StreamWithLength(new LZipStream(inFs, CompressionMode.Decompress), (long)file.Size); + } + + void Zf_HandleClose(string entryName, Stream stream) => stream.Close(); + + void Zf_SaveProgress(object sender, SaveProgressEventArgs e) + { + if(e.CurrentEntry != null && + e.CurrentEntry.FileName != _zipCurrentEntryName) + { + _zipCurrentEntryName = e.CurrentEntry.FileName; + _filePosition++; + + SetProgress2?.Invoke(this, new ProgressEventArgs + { + Value = _filePosition + }); + + if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName, out FileByMachine fileByMachine)) + if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName.Replace('/', '\\'), out fileByMachine)) + throw new ArgumentException("Cannot find requested zip entry in hashes dictionary"); + + DbFile currentFile = fileByMachine.File; + + SetMessage3?.Invoke(this, new MessageEventArgs + { + Message = string.Format("Compressing {0}...", e.CurrentEntry.FileName) + }); + + SetProgress3Bounds?.Invoke(this, new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = currentFile.Size + }); + } + + SetProgress3?.Invoke(this, new ProgressEventArgs + { + Value = e.BytesTransferred + }); + + switch(e.EventType) + { + case ZipProgressEventType.Error_Saving: + #if DEBUG + throw new Exception(); + #endif + + break; + case ZipProgressEventType.Saving_Completed: + _machinePosition++; + CompressNextMachine(); + + break; + } + } + } +} \ No newline at end of file diff --git a/RomRepoMgr/ViewModels/ExportRomsViewModel.cs b/RomRepoMgr/ViewModels/ExportRomsViewModel.cs new file mode 100644 index 0000000..331cdba --- /dev/null +++ b/RomRepoMgr/ViewModels/ExportRomsViewModel.cs @@ -0,0 +1,269 @@ +/****************************************************************************** +// RomRepoMgr - ROM repository manager +// ---------------------------------------------------------------------------- +// +// Author(s) : Natalia Portillo +// +// --[ License ] -------------------------------------------------------------- +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2020 Natalia Portillo +*******************************************************************************/ + +using System; +using System.Reactive; +using System.Threading.Tasks; +using Avalonia.Threading; +using JetBrains.Annotations; +using ReactiveUI; +using RomRepoMgr.Core.EventArgs; +using RomRepoMgr.Core.Workers; +using RomRepoMgr.Views; + +namespace RomRepoMgr.ViewModels +{ + public sealed class ExportRomsViewModel : ViewModelBase + { + readonly long _romSetId; + readonly ExportRoms _view; + bool _canClose; + bool _progress2IsIndeterminate; + double _progress2Maximum; + double _progress2Minimum; + double _progress2Value; + bool _progress2Visible; + bool _progress3IsIndeterminate; + double _progress3Maximum; + double _progress3Minimum; + double _progress3Value; + bool _progress3Visible; + bool _progressIsIndeterminate; + double _progressMaximum; + double _progressMinimum; + double _progressValue; + bool _progressVisible; + string _status2Message; + string _status3Message; + string _statusMessage; + + public ExportRomsViewModel(ExportRoms view, string folderPath, long romSetId) + { + _view = view; + _romSetId = romSetId; + FolderPath = folderPath; + CloseCommand = ReactiveCommand.Create(ExecuteCloseCommand); + CanClose = false; + } + + [NotNull] + public string PathLabel => "Path:"; + public string FolderPath { get; } + + public bool ProgressVisible + { + get => _progressVisible; + set => this.RaiseAndSetIfChanged(ref _progressVisible, value); + } + + public string StatusMessage + { + get => _statusMessage; + set => this.RaiseAndSetIfChanged(ref _statusMessage, value); + } + + public double ProgressMinimum + { + get => _progressMinimum; + set => this.RaiseAndSetIfChanged(ref _progressMinimum, value); + } + + public double ProgressMaximum + { + get => _progressMaximum; + set => this.RaiseAndSetIfChanged(ref _progressMaximum, value); + } + + public double ProgressValue + { + get => _progressValue; + set => this.RaiseAndSetIfChanged(ref _progressValue, value); + } + + public bool ProgressIsIndeterminate + { + get => _progressIsIndeterminate; + set => this.RaiseAndSetIfChanged(ref _progressIsIndeterminate, value); + } + + public bool Progress2Visible + { + get => _progress2Visible; + set => this.RaiseAndSetIfChanged(ref _progress2Visible, value); + } + + public string Status2Message + { + get => _status2Message; + set => this.RaiseAndSetIfChanged(ref _status2Message, value); + } + + public double Progress2Minimum + { + get => _progress2Minimum; + set => this.RaiseAndSetIfChanged(ref _progress2Minimum, value); + } + + public double Progress2Maximum + { + get => _progress2Maximum; + set => this.RaiseAndSetIfChanged(ref _progress2Maximum, value); + } + + public double Progress2Value + { + get => _progress2Value; + set => this.RaiseAndSetIfChanged(ref _progress2Value, value); + } + + public bool Progress2IsIndeterminate + { + get => _progress2IsIndeterminate; + set => this.RaiseAndSetIfChanged(ref _progress2IsIndeterminate, value); + } + + public bool Progress3Visible + { + get => _progress3Visible; + set => this.RaiseAndSetIfChanged(ref _progress3Visible, value); + } + + public string Status3Message + { + get => _status3Message; + set => this.RaiseAndSetIfChanged(ref _status3Message, value); + } + + public double Progress3Minimum + { + get => _progress3Minimum; + set => this.RaiseAndSetIfChanged(ref _progress3Minimum, value); + } + + public double Progress3Maximum + { + get => _progress3Maximum; + set => this.RaiseAndSetIfChanged(ref _progress3Maximum, value); + } + + public double Progress3Value + { + get => _progress3Value; + set => this.RaiseAndSetIfChanged(ref _progress3Value, value); + } + + public bool Progress3IsIndeterminate + { + get => _progress3IsIndeterminate; + set => this.RaiseAndSetIfChanged(ref _progress3IsIndeterminate, value); + } + + [NotNull] + public string Title => "Exporting ROM files to folder..."; + [NotNull] + public string CloseLabel => "Close"; + + public bool CanClose + { + get => _canClose; + set => this.RaiseAndSetIfChanged(ref _canClose, value); + } + + public ReactiveCommand CloseCommand { get; } + + void ExecuteCloseCommand() => _view.Close(); + + void OnWorkerOnFinished(object sender, EventArgs args) => Dispatcher.UIThread.Post(() => + { + ProgressVisible = false; + CanClose = true; + Progress2Visible = false; + Progress3Visible = false; + }); + + void OnWorkerOnSetProgressBounds(object sender, ProgressBoundsEventArgs args) => Dispatcher.UIThread.Post(() => + { + ProgressIsIndeterminate = false; + ProgressMaximum = args.Maximum; + ProgressMinimum = args.Minimum; + }); + + void OnWorkerOnSetProgress(object sender, ProgressEventArgs args) => + Dispatcher.UIThread.Post(() => ProgressValue = args.Value); + + void OnWorkerOnSetMessage(object sender, MessageEventArgs args) => + Dispatcher.UIThread.Post(() => StatusMessage = args.Message); + + void OnWorkerOnSetIndeterminateProgress(object sender, EventArgs args) => + Dispatcher.UIThread.Post(() => ProgressIsIndeterminate = true); + + void OnWorkerOnSetProgressBounds2(object sender, ProgressBoundsEventArgs args) => Dispatcher.UIThread.Post(() => + { + Progress2Visible = true; + Progress2IsIndeterminate = false; + Progress2Maximum = args.Maximum; + Progress2Minimum = args.Minimum; + }); + + void OnWorkerOnSetProgress2(object sender, ProgressEventArgs args) => + Dispatcher.UIThread.Post(() => Progress2Value = args.Value); + + void OnWorkerOnSetMessage2(object sender, MessageEventArgs args) => + Dispatcher.UIThread.Post(() => Status2Message = args.Message); + + void OnWorkerOnSetProgressBounds3(object sender, ProgressBoundsEventArgs args) => Dispatcher.UIThread.Post(() => + { + Progress3Visible = true; + Progress3IsIndeterminate = false; + Progress3Maximum = args.Maximum; + Progress3Minimum = args.Minimum; + }); + + void OnWorkerOnSetProgress3(object sender, ProgressEventArgs args) => + Dispatcher.UIThread.Post(() => Progress3Value = args.Value); + + void OnWorkerOnSetMessage3(object sender, MessageEventArgs args) => + Dispatcher.UIThread.Post(() => Status3Message = args.Message); + + public void OnOpened() + { + var worker = new FileExporter(_romSetId, FolderPath); + worker.SetMessage += OnWorkerOnSetMessage; + worker.SetProgress += OnWorkerOnSetProgress; + worker.SetProgressBounds += OnWorkerOnSetProgressBounds; + worker.SetMessage2 += OnWorkerOnSetMessage2; + worker.SetProgress2 += OnWorkerOnSetProgress2; + worker.SetProgress2Bounds += OnWorkerOnSetProgressBounds2; + worker.SetMessage3 += OnWorkerOnSetMessage3; + worker.SetProgress3 += OnWorkerOnSetProgress3; + worker.SetProgress3Bounds += OnWorkerOnSetProgressBounds3; + worker.WorkFinished += OnWorkerOnFinished; + + ProgressVisible = true; + + Task.Run(worker.Export); + } + } +} \ No newline at end of file diff --git a/RomRepoMgr/ViewModels/MainWindowViewModel.cs b/RomRepoMgr/ViewModels/MainWindowViewModel.cs index c834a96..f42f324 100644 --- a/RomRepoMgr/ViewModels/MainWindowViewModel.cs +++ b/RomRepoMgr/ViewModels/MainWindowViewModel.cs @@ -58,6 +58,7 @@ namespace RomRepoMgr.ViewModels DeleteRomSetCommand = ReactiveCommand.Create(ExecuteDeleteRomSetCommand); EditRomSetCommand = ReactiveCommand.Create(ExecuteEditRomSetCommand); ExportDatCommand = ReactiveCommand.Create(ExecuteExportDatCommand); + ExportRomsCommand = ReactiveCommand.Create(ExecuteExportRomsCommand); RomSets = new ObservableCollection(romSets); } @@ -89,6 +90,7 @@ namespace RomRepoMgr.ViewModels public ReactiveCommand DeleteRomSetCommand { get; } public ReactiveCommand EditRomSetCommand { get; } public ReactiveCommand ExportDatCommand { get; } + public ReactiveCommand ExportRomsCommand { get; } public RomSetModel SelectedRomSet { @@ -262,5 +264,23 @@ namespace RomRepoMgr.ViewModels dialog.DataContext = viewModel; await dialog.ShowDialog(_view); } + + async void ExecuteExportRomsCommand() + { + var dlgOpen = new OpenFolderDialog + { + Title = "Export ROMs to folder..." + }; + + string result = await dlgOpen.ShowAsync(_view); + + if(result == null) + return; + + var dialog = new ExportRoms(); + var viewModel = new ExportRomsViewModel(dialog, result, SelectedRomSet.Id); + dialog.DataContext = viewModel; + await dialog.ShowDialog(_view); + } } } \ No newline at end of file diff --git a/RomRepoMgr/Views/ExportRoms.xaml b/RomRepoMgr/Views/ExportRoms.xaml new file mode 100644 index 0000000..521c21d --- /dev/null +++ b/RomRepoMgr/Views/ExportRoms.xaml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/RomRepoMgr/Views/ExportRoms.xaml.cs b/RomRepoMgr/Views/ExportRoms.xaml.cs new file mode 100644 index 0000000..04c4c7f --- /dev/null +++ b/RomRepoMgr/Views/ExportRoms.xaml.cs @@ -0,0 +1,45 @@ +/****************************************************************************** +// RomRepoMgr - ROM repository manager +// ---------------------------------------------------------------------------- +// +// Author(s) : Natalia Portillo +// +// --[ License ] -------------------------------------------------------------- +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2020 Natalia Portillo +*******************************************************************************/ + +using System; +using Avalonia.Controls; +using Avalonia.Markup.Xaml; +using RomRepoMgr.ViewModels; + +namespace RomRepoMgr.Views +{ + public sealed class ExportRoms : Window + { + public ExportRoms() => InitializeComponent(); + + void InitializeComponent() => AvaloniaXamlLoader.Load(this); + + protected override void OnOpened(EventArgs e) + { + base.OnOpened(e); + (DataContext as ExportRomsViewModel)?.OnOpened(); + } + } +} \ No newline at end of file diff --git a/RomRepoMgr/Views/MainWindow.xaml b/RomRepoMgr/Views/MainWindow.xaml index 0a9fb3e..ff9420d 100644 --- a/RomRepoMgr/Views/MainWindow.xaml +++ b/RomRepoMgr/Views/MainWindow.xaml @@ -21,6 +21,7 @@ +