From 5621483a9c64eead9dd47be684cd33caaf761fbe Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Fri, 19 May 2017 16:57:28 +0100 Subject: [PATCH] Added code to clean database and repository. --- osrepodbmgr.Core/ChangeLog | 6 + osrepodbmgr.Core/DBOps.cs | 15 ++ osrepodbmgr.Core/Workers/Files.cs | 175 +++++++++++++++++++++ osrepodbmgr.Eto/ChangeLog | 6 + osrepodbmgr.Eto/frmMain.xeto | 1 + osrepodbmgr.Eto/frmMain.xeto.cs | 72 +++++++++ osrepodbmgr/ChangeLog | 8 + osrepodbmgr/frmMain.cs | 78 +++++++++ osrepodbmgr/gtk-gui/generated.cs | 8 +- osrepodbmgr/gtk-gui/gui.stetic | 22 ++- osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs | 42 +++-- 11 files changed, 413 insertions(+), 20 deletions(-) diff --git a/osrepodbmgr.Core/ChangeLog b/osrepodbmgr.Core/ChangeLog index 848ced5..20ec660 100644 --- a/osrepodbmgr.Core/ChangeLog +++ b/osrepodbmgr.Core/ChangeLog @@ -1,3 +1,9 @@ +2017-05-19 Natalia Portillo + + * DBOps.cs: + * Workers/Files.cs: + Added code to clean database and repository. + 2017-05-19 Natalia Portillo * DBOps.cs: diff --git a/osrepodbmgr.Core/DBOps.cs b/osrepodbmgr.Core/DBOps.cs index a462d93..f80669b 100644 --- a/osrepodbmgr.Core/DBOps.cs +++ b/osrepodbmgr.Core/DBOps.cs @@ -851,6 +851,21 @@ namespace osrepodbmgr.Core return true; } + + public bool DeleteFile(string hash) + { + IDbCommand dbcmd = dbCon.CreateCommand(); + IDbDataParameter param1 = dbcmd.CreateParameter(); + + param1.ParameterName = "@sha256"; + param1.DbType = DbType.String; + param1.Value = hash; + dbcmd.Parameters.Add(param1); + dbcmd.CommandText = "DELETE FROM `files` WHERE sha256 = @sha256"; + dbcmd.ExecuteNonQuery(); + + return true; + } } } diff --git a/osrepodbmgr.Core/Workers/Files.cs b/osrepodbmgr.Core/Workers/Files.cs index 118e6f9..299ae83 100644 --- a/osrepodbmgr.Core/Workers/Files.cs +++ b/osrepodbmgr.Core/Workers/Files.cs @@ -774,5 +774,180 @@ namespace osrepodbmgr.Core Failed(string.Format("Exception {0}\n{1}", ex.Message, ex.InnerException)); } } + + public static void CleanFiles() + { + ulong count = dbCore.DBOps.GetFilesCount(); + const ulong page = 2500; + ulong offset = 0; + + List filesPage, allFiles; + allFiles = new List(); + +#if DEBUG + stopwatch.Restart(); +#endif + while(dbCore.DBOps.GetFiles(out filesPage, offset, page)) + { + if(filesPage.Count == 0) + break; + + if(UpdateProgress != null) + UpdateProgress(null, string.Format("Loaded file {0} of {1}", offset, count), (long)offset, (long)count); + + allFiles.AddRange(filesPage); + + offset += page; + } +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to get all files from the database", stopwatch.Elapsed.TotalSeconds); +#endif + + filesPage = null; + + if(UpdateProgress != null) + UpdateProgress(null, "Getting OSes from the database", 0, 0); +#if DEBUG + stopwatch.Restart(); +#endif + List oses; + dbCore.DBOps.GetAllOSes(out oses); +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to get OSes from database", stopwatch.Elapsed.TotalSeconds); +#endif + + List orphanFiles = new List(); + +#if DEBUG + stopwatch.Restart(); + Stopwatch stopwatch2 = new Stopwatch(); +#endif + int counterF = 0; + foreach(DBFile file in allFiles) + { + if(UpdateProgress != null) + UpdateProgress(null, string.Format("Checking file {0} of {1}", counterF, allFiles.Count), counterF, allFiles.Count); + + bool fileExists = false; + int counterO = 0; +#if DEBUG + stopwatch2.Restart(); +#endif + foreach(DBEntry os in oses) + { + if(UpdateProgress2 != null) + UpdateProgress2(null, string.Format("Checking OS {0} of {1}", counterO, oses.Count), counterO, oses.Count); + + if(dbCore.DBOps.ExistsFileInOS(file.Sha256, os.id)) + { + fileExists = true; + break; + } + + counterO++; + } +#if DEBUG + stopwatch2.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check file in all OSes", stopwatch2.Elapsed.TotalSeconds); +#endif + + if(!fileExists) + orphanFiles.Add(file.Sha256); + + counterF++; + } +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check all files", stopwatch.Elapsed.TotalSeconds); +#endif + + if(UpdateProgress2 != null) + UpdateProgress2(null, null, 0, 0); + +#if DEBUG + stopwatch.Restart(); +#endif + counterF = 0; + foreach(string hash in orphanFiles) + { + if(UpdateProgress != null) + UpdateProgress(null, string.Format("Deleting file {0} of {1} from database", counterF, orphanFiles.Count), counterF, orphanFiles.Count); + + dbCore.DBOps.DeleteFile(hash); + counterF++; + } +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to remove all orphan files from database", stopwatch.Elapsed.TotalSeconds); +#endif + + if(UpdateProgress != null) + UpdateProgress(null, "Listing files in repository", 0, 0); + +#if DEBUG + stopwatch.Restart(); +#endif + List repoFiles = new List(Directory.EnumerateFiles(Settings.Current.RepositoryPath, "*", SearchOption.AllDirectories)); + repoFiles.Sort(); +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to find all files", stopwatch.Elapsed.TotalSeconds); + stopwatch.Restart(); +#endif + counterF = 0; + List filesToDelete = new List(); + foreach(string file in repoFiles) + { + if(UpdateProgress != null) + UpdateProgress(null, string.Format("Checking file {0} of {1} from repository", counterF, repoFiles.Count), counterF, repoFiles.Count); + + // Allow database to be inside repo + if(file == Settings.Current.DatabasePath) + continue; + + if(Path.GetExtension(file).ToLowerInvariant() == ".xml" || + Path.GetExtension(file).ToLowerInvariant() == ".json") + { + if(!dbCore.DBOps.ExistsOS(Path.GetFileNameWithoutExtension(file))) + filesToDelete.Add(file); + } + else if(!dbCore.DBOps.ExistsFile(Path.GetFileNameWithoutExtension(file))) + filesToDelete.Add(file); + + counterF++; + } +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check all repository files", stopwatch.Elapsed.TotalSeconds); + stopwatch.Restart(); +#endif + counterF = 0; + foreach(string file in filesToDelete) + { + if(UpdateProgress != null) + UpdateProgress(null, string.Format("Deleting file {0} of {1} from repository", counterF, filesToDelete.Count), counterF, filesToDelete.Count); + + try + { + File.Delete(file); + } +#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body + catch +#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body { + // Do not crash + } + + counterF++; + } +#if DEBUG + stopwatch.Stop(); + Console.WriteLine("Core.CleanFiles(): Took {0} seconds to delete all orphan files", stopwatch.Elapsed.TotalSeconds); +#endif + + if(Finished != null) + Finished(); + } } } diff --git a/osrepodbmgr.Eto/ChangeLog b/osrepodbmgr.Eto/ChangeLog index 02af6e5..7f49fb9 100644 --- a/osrepodbmgr.Eto/ChangeLog +++ b/osrepodbmgr.Eto/ChangeLog @@ -1,3 +1,9 @@ +2017-05-19 Natalia Portillo + + * frmMain.xeto: + * frmMain.xeto.cs: + Added code to clean database and repository. + 2017-05-19 Natalia Portillo * frmMain.xeto.cs: diff --git a/osrepodbmgr.Eto/frmMain.xeto b/osrepodbmgr.Eto/frmMain.xeto index b47ebd2..776e34f 100644 --- a/osrepodbmgr.Eto/frmMain.xeto +++ b/osrepodbmgr.Eto/frmMain.xeto @@ -49,6 +49,7 @@ + diff --git a/osrepodbmgr.Eto/frmMain.xeto.cs b/osrepodbmgr.Eto/frmMain.xeto.cs index a187c90..08dc55d 100644 --- a/osrepodbmgr.Eto/frmMain.xeto.cs +++ b/osrepodbmgr.Eto/frmMain.xeto.cs @@ -46,6 +46,7 @@ namespace osrepodbmgr.Eto Thread thdScanFile; DBFile outIter; bool scanningFiles; + Thread thdCleanFiles; #region XAML UI elements #pragma warning disable 0649 @@ -74,6 +75,9 @@ namespace osrepodbmgr.Eto Button btnPopulateFiles; TabPage tabOSes; Button btnScanAllPending; + Button btnCleanFiles; + ButtonMenuItem btnQuit; + ButtonMenuItem mnuFile; #pragma warning restore 0649 #endregion XAML UI elements @@ -937,6 +941,7 @@ namespace osrepodbmgr.Eto treeFiles.Enabled = true; tabOSes.Enabled = true; btnScanAllPending.Visible = true; + btnCleanFiles.Visible = true; }); } @@ -1008,5 +1013,72 @@ namespace osrepodbmgr.Eto lblProgressFiles2.Text = text; }); } + + protected void OnBtnCleanFilesClicked(object sender, EventArgs e) + { + DialogResult result = MessageBox.Show("This option will search the database for any known file that doesn't\n" + + "belong to any OS and remove it from the database.\n\n" + + "It will then search the repository for any file not on the database and remove it.\n\n" + + "THIS OPERATION MAY VERY LONG, CANNOT BE CANCELED AND REMOVES DATA ON DISK.\n\n" + + "Are you sure to continue?", MessageBoxButtons.YesNo, MessageBoxType.Question); + if(result == DialogResult.Yes) + { + btnCleanFiles.Visible = false; + btnToggleCrack.Visible = false; + btnScanWithClamd.Visible = false; + btnScanAllPending.Visible = false; + btnCheckInVirusTotal.Visible = false; + tabOSes.Enabled = false; + treeFiles.Enabled = false; + lblProgressFiles1.Visible = true; + lblProgressFiles2.Visible = true; + Workers.Finished += CleanFilesFinished; + Workers.UpdateProgress += UpdateFileProgress; + Workers.UpdateProgress2 += UpdateFileProgress2; + lblProgressFiles1.Text = ""; + prgProgressFiles1.Visible = true; + lblProgressFiles2.Text = ""; + prgProgressFiles2.Visible = true; + btnStopFiles.Visible = false; + prgProgress2.Indeterminate = true; + btnSettings.Enabled = false; + btnHelp.Enabled = false; + mnuCompress.Enabled = false; + btnQuit.Enabled = false; + mnuFile.Enabled = false; + thdCleanFiles = new Thread(Workers.CleanFiles); + thdCleanFiles.Start(); + } + } + + void CleanFilesFinished() + { + Application.Instance.Invoke(delegate + { + btnCleanFiles.Visible = true; + btnToggleCrack.Visible = true; + btnScanWithClamd.Visible = true; + btnScanAllPending.Visible = true; + btnCheckInVirusTotal.Visible = true; + tabOSes.Enabled = true; + treeFiles.Enabled = true; + Workers.Finished -= CleanFilesFinished; + Workers.UpdateProgress -= UpdateFileProgress; + Workers.UpdateProgress2 -= UpdateFileProgress2; + lblProgressFiles1.Text = ""; + prgProgressFiles1.Visible = false; + lblProgressFiles2.Text = ""; + prgProgressFiles2.Visible = false; + btnSettings.Enabled = true; + btnHelp.Enabled = true; + mnuCompress.Enabled = true; + btnQuit.Enabled = true; + mnuFile.Enabled = true; + if(thdCleanFiles != null) + thdCleanFiles = null; + + OnBtnPopulateFilesClicked(null, new EventArgs()); + }); + } } } diff --git a/osrepodbmgr/ChangeLog b/osrepodbmgr/ChangeLog index 51cceed..67df9f3 100644 --- a/osrepodbmgr/ChangeLog +++ b/osrepodbmgr/ChangeLog @@ -1,3 +1,11 @@ +2017-05-19 Natalia Portillo + + * frmMain.cs: + * gtk-gui/gui.stetic: + * gtk-gui/generated.cs: + * gtk-gui/osrepodbmgr.frmMain.cs: + Added code to clean database and repository. + 2017-05-19 Natalia Portillo * frmMain.cs: diff --git a/osrepodbmgr/frmMain.cs b/osrepodbmgr/frmMain.cs index 2f7757b..ed7c27b 100644 --- a/osrepodbmgr/frmMain.cs +++ b/osrepodbmgr/frmMain.cs @@ -47,6 +47,7 @@ namespace osrepodbmgr bool scanningFiles; Thread thdScanFile; TreeIter outIter; + Thread thdCleanFiles; public frmMain() : base(WindowType.Toplevel) @@ -1036,6 +1037,7 @@ namespace osrepodbmgr populatingFiles = false; treeFiles.Sensitive = true; notebook1.GetNthPage(0).Sensitive = true; + btnCleanFiles.Visible = true; }); } @@ -1121,5 +1123,81 @@ namespace osrepodbmgr prgProgressFiles2.Text = text; }); } + + protected void OnBtnCleanFilesClicked(object sender, EventArgs e) + { + MessageDialog dlgMsg = new MessageDialog(this, DialogFlags.Modal, MessageType.Warning, ButtonsType.YesNo, + "This option will search the database for any known file that doesn't\n" + + "belong to any OS and remove it from the database.\n\n" + + "It will then search the repository for any file not on the database and remove it.\n\n" + + "THIS OPERATION MAY VERY LONG, CANNOT BE CANCELED AND REMOVES DATA ON DISK.\n\n" + + "Are you sure to continue?"); + + if(dlgMsg.Run() == (int)ResponseType.Yes) + { + dlgMsg.Destroy(); + btnCleanFiles.Visible = false; + btnToggleCrack.Visible = false; + btnScanWithClamd.Visible = false; + btnScanAllPending.Visible = false; + btnCheckInVirusTotal.Visible = false; + notebook1.GetNthPage(0).Sensitive = false; + treeFiles.Sensitive = false; + Workers.Finished += CleanFilesFinished; + Workers.UpdateProgress += UpdateFileProgress; + Workers.UpdateProgress2 += UpdateFileProgress2; + prgProgressFiles1.Text = ""; + prgProgressFiles1.Visible = true; + prgProgressFiles2.Text = ""; + prgProgressFiles2.Visible = true; + btnStopFiles.Visible = false; + thdPulseProgress = new Thread(() => + { + while(true) + { + Application.Invoke(delegate + { + prgProgressFiles2.Pulse(); + }); + Thread.Sleep(66); + } + }); + thdPulseProgress.Start(); + thdCleanFiles = new Thread(Workers.CleanFiles); + thdCleanFiles.Start(); + } + else + dlgMsg.Destroy(); + } + + void CleanFilesFinished() + { + Application.Invoke(delegate + { + btnCleanFiles.Visible = true; + btnToggleCrack.Visible = true; + btnScanWithClamd.Visible = true; + btnScanAllPending.Visible = true; + btnCheckInVirusTotal.Visible = true; + notebook1.GetNthPage(0).Sensitive = true; + treeFiles.Sensitive = true; + Workers.Finished -= CleanFilesFinished; + Workers.UpdateProgress -= UpdateFileProgress; + Workers.UpdateProgress2 -= UpdateFileProgress2; + prgProgressFiles1.Text = ""; + prgProgressFiles1.Visible = false; + prgProgressFiles2.Text = ""; + prgProgressFiles2.Visible = false; + if(thdPulseProgress != null) + { + thdPulseProgress.Abort(); + thdPulseProgress = null; + } + if(thdCleanFiles != null) + thdCleanFiles = null; + + OnBtnPopulateFilesClicked(null, new EventArgs()); + }); + } } } \ No newline at end of file diff --git a/osrepodbmgr/gtk-gui/generated.cs b/osrepodbmgr/gtk-gui/generated.cs index f916e7a..50283a9 100644 --- a/osrepodbmgr/gtk-gui/generated.cs +++ b/osrepodbmgr/gtk-gui/generated.cs @@ -8,7 +8,7 @@ namespace Stetic internal static void Initialize(Gtk.Widget iconRenderer) { - if((Stetic.Gui.initialized == false)) + if ((Stetic.Gui.initialized == false)) { Stetic.Gui.initialized = true; } @@ -20,7 +20,7 @@ namespace Stetic public static Gdk.Pixbuf LoadIcon(Gtk.Widget widget, string name, Gtk.IconSize size) { Gdk.Pixbuf res = widget.RenderIcon(name, size, null); - if((res != null)) + if ((res != null)) { return res; } @@ -32,9 +32,9 @@ namespace Stetic { return Gtk.IconTheme.Default.LoadIcon(name, sz, 0); } - catch(System.Exception) + catch (System.Exception) { - if((name != "gtk-missing-image")) + if ((name != "gtk-missing-image")) { return Stetic.IconLoader.LoadIcon(widget, "gtk-missing-image", size); } diff --git a/osrepodbmgr/gtk-gui/gui.stetic b/osrepodbmgr/gtk-gui/gui.stetic index f5f4995..445f50f 100644 --- a/osrepodbmgr/gtk-gui/gui.stetic +++ b/osrepodbmgr/gtk-gui/gui.stetic @@ -12702,7 +12702,7 @@ QNX/QNX/20090229/source.zip - + OS Repository DB Manager CenterOnParent @@ -13162,6 +13162,24 @@ QNX/QNX/20090229/source.zip False + + + + False + True + TextOnly + Clean files + True + + + + End + 5 + True + False + False + + @@ -13175,7 +13193,7 @@ QNX/QNX/20090229/source.zip End - 5 + 6 True False False diff --git a/osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs b/osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs index 4ba9a52..dc2db2b 100644 --- a/osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs +++ b/osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs @@ -76,6 +76,8 @@ namespace osrepodbmgr private global::Gtk.Button btnToggleCrack; + private global::Gtk.Button btnCleanFiles; + private global::Gtk.Button btnStopFiles; private global::Gtk.Label label3; @@ -415,6 +417,16 @@ namespace osrepodbmgr w34.Expand = false; w34.Fill = false; // Container child hbox6.Gtk.Box+BoxChild + this.btnCleanFiles = new global::Gtk.Button(); + this.btnCleanFiles.CanFocus = true; + this.btnCleanFiles.Name = "btnCleanFiles"; + this.btnCleanFiles.UseUnderline = true; + this.btnCleanFiles.Label = global::Mono.Unix.Catalog.GetString("Clean files"); + this.hbox6.Add(this.btnCleanFiles); + global::Gtk.Box.BoxChild w35 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnCleanFiles])); + w35.PackType = ((global::Gtk.PackType)(1)); + w35.Position = 5; + // Container child hbox6.Gtk.Box+BoxChild this.btnStopFiles = new global::Gtk.Button(); this.btnStopFiles.CanFocus = true; this.btnStopFiles.Name = "btnStopFiles"; @@ -422,19 +434,19 @@ namespace osrepodbmgr this.btnStopFiles.UseUnderline = true; this.btnStopFiles.Label = "gtk-stop"; this.hbox6.Add(this.btnStopFiles); - global::Gtk.Box.BoxChild w35 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnStopFiles])); - w35.PackType = ((global::Gtk.PackType)(1)); - w35.Position = 5; - w35.Expand = false; - w35.Fill = false; - this.vbox4.Add(this.hbox6); - global::Gtk.Box.BoxChild w36 = ((global::Gtk.Box.BoxChild)(this.vbox4[this.hbox6])); - w36.Position = 3; + global::Gtk.Box.BoxChild w36 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnStopFiles])); + w36.PackType = ((global::Gtk.PackType)(1)); + w36.Position = 6; w36.Expand = false; w36.Fill = false; + this.vbox4.Add(this.hbox6); + global::Gtk.Box.BoxChild w37 = ((global::Gtk.Box.BoxChild)(this.vbox4[this.hbox6])); + w37.Position = 3; + w37.Expand = false; + w37.Fill = false; this.notebook1.Add(this.vbox4); - global::Gtk.Notebook.NotebookChild w37 = ((global::Gtk.Notebook.NotebookChild)(this.notebook1[this.vbox4])); - w37.Position = 1; + global::Gtk.Notebook.NotebookChild w38 = ((global::Gtk.Notebook.NotebookChild)(this.notebook1[this.vbox4])); + w38.Position = 1; // Notebook tab this.label3 = new global::Gtk.Label(); this.label3.Name = "label3"; @@ -442,14 +454,14 @@ namespace osrepodbmgr this.notebook1.SetTabLabel(this.vbox4, this.label3); this.label3.ShowAll(); this.vbox2.Add(this.notebook1); - global::Gtk.Box.BoxChild w38 = ((global::Gtk.Box.BoxChild)(this.vbox2[this.notebook1])); - w38.Position = 0; + global::Gtk.Box.BoxChild w39 = ((global::Gtk.Box.BoxChild)(this.vbox2[this.notebook1])); + w39.Position = 0; this.Add(this.vbox2); - if((this.Child != null)) + if ((this.Child != null)) { this.Child.ShowAll(); } - this.DefaultWidth = 612; + this.DefaultWidth = 686; this.DefaultHeight = 365; this.lblProgress2.Hide(); this.prgProgress2.Hide(); @@ -467,6 +479,7 @@ namespace osrepodbmgr this.btnScanWithClamd.Hide(); this.btnScanAllPending.Hide(); this.btnToggleCrack.Hide(); + this.btnCleanFiles.Hide(); this.btnStopFiles.Hide(); this.Show(); this.DeleteEvent += new global::Gtk.DeleteEventHandler(this.OnDeleteEvent); @@ -479,6 +492,7 @@ namespace osrepodbmgr this.btnSettings.Clicked += new global::System.EventHandler(this.OnBtnSettingsClicked); this.btnQuit.Clicked += new global::System.EventHandler(this.OnBtnQuitClicked); this.btnStopFiles.Clicked += new global::System.EventHandler(this.OnBtnStopFilesClicked); + this.btnCleanFiles.Clicked += new global::System.EventHandler(this.OnBtnCleanFilesClicked); this.btnToggleCrack.Clicked += new global::System.EventHandler(this.OnBtnToggleCrackClicked); this.btnScanAllPending.Clicked += new global::System.EventHandler(this.OnBtnScanAllPendingClicked); this.btnScanWithClamd.Clicked += new global::System.EventHandler(this.OnBtnScanWithClamdClicked);