claunia/osrepodbmgr
1
0
Fork 0
mirror of https://github.com/claunia/osrepodbmgr.git synced 2025-12-16 19:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added code to clean database and repository.

Browse Source
This commit is contained in:
Natalia Portillo
2017-05-19 16:57:28 +01:00
parent 835b6e44f6
commit 5621483a9c
11 changed files with 413 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
osrepodbmgr.Core/ChangeLog
View File

@@ -1,3 +1,9 @@
2017-05-19 Natalia Portillo <claunia@claunia.com>
* DBOps.cs:
* Workers/Files.cs:
Added code to clean database and repository.
2017-05-19 Natalia Portillo <claunia@claunia.com>
* DBOps.cs:

15
osrepodbmgr.Core/DBOps.cs
View File

@@ -851,6 +851,21 @@ namespace osrepodbmgr.Core
return true;
}
public bool DeleteFile(string hash)
{
IDbCommand dbcmd = dbCon.CreateCommand();
IDbDataParameter param1 = dbcmd.CreateParameter();
param1.ParameterName = "@sha256";
param1.DbType = DbType.String;
param1.Value = hash;
dbcmd.Parameters.Add(param1);
dbcmd.CommandText = "DELETE FROM `files` WHERE sha256 = @sha256";
dbcmd.ExecuteNonQuery();
return true;
}
}
}

175
osrepodbmgr.Core/Workers/Files.cs
View File

@@ -774,5 +774,180 @@ namespace osrepodbmgr.Core
Failed(string.Format("Exception {0}\n{1}", ex.Message, ex.InnerException));
}
}
public static void CleanFiles()
{
ulong count = dbCore.DBOps.GetFilesCount();
const ulong page = 2500;
ulong offset = 0;
List<DBFile> filesPage, allFiles;
allFiles = new List<DBFile>();
#if DEBUG
stopwatch.Restart();
#endif
while(dbCore.DBOps.GetFiles(out filesPage, offset, page))
{
if(filesPage.Count == 0)
break;
if(UpdateProgress != null)
UpdateProgress(null, string.Format("Loaded file {0} of {1}", offset, count), (long)offset, (long)count);
allFiles.AddRange(filesPage);
offset += page;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to get all files from the database", stopwatch.Elapsed.TotalSeconds);
#endif
filesPage = null;
if(UpdateProgress != null)
UpdateProgress(null, "Getting OSes from the database", 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
List<DBEntry> oses;
dbCore.DBOps.GetAllOSes(out oses);
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to get OSes from database", stopwatch.Elapsed.TotalSeconds);
#endif
List<string> orphanFiles = new List<string>();
#if DEBUG
stopwatch.Restart();
Stopwatch stopwatch2 = new Stopwatch();
#endif
int counterF = 0;
foreach(DBFile file in allFiles)
{
if(UpdateProgress != null)
UpdateProgress(null, string.Format("Checking file {0} of {1}", counterF, allFiles.Count), counterF, allFiles.Count);
bool fileExists = false;
int counterO = 0;
#if DEBUG
stopwatch2.Restart();
#endif
foreach(DBEntry os in oses)
{
if(UpdateProgress2 != null)
UpdateProgress2(null, string.Format("Checking OS {0} of {1}", counterO, oses.Count), counterO, oses.Count);
if(dbCore.DBOps.ExistsFileInOS(file.Sha256, os.id))
{
fileExists = true;
break;
}
counterO++;
}
#if DEBUG
stopwatch2.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check file in all OSes", stopwatch2.Elapsed.TotalSeconds);
#endif
if(!fileExists)
orphanFiles.Add(file.Sha256);
counterF++;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check all files", stopwatch.Elapsed.TotalSeconds);
#endif
if(UpdateProgress2 != null)
UpdateProgress2(null, null, 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
counterF = 0;
foreach(string hash in orphanFiles)
{
if(UpdateProgress != null)
UpdateProgress(null, string.Format("Deleting file {0} of {1} from database", counterF, orphanFiles.Count), counterF, orphanFiles.Count);
dbCore.DBOps.DeleteFile(hash);
counterF++;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to remove all orphan files from database", stopwatch.Elapsed.TotalSeconds);
#endif
if(UpdateProgress != null)
UpdateProgress(null, "Listing files in repository", 0, 0);
#if DEBUG
stopwatch.Restart();
#endif
List<string> repoFiles = new List<string>(Directory.EnumerateFiles(Settings.Current.RepositoryPath, "*", SearchOption.AllDirectories));
repoFiles.Sort();
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to find all files", stopwatch.Elapsed.TotalSeconds);
stopwatch.Restart();
#endif
counterF = 0;
List<string> filesToDelete = new List<string>();
foreach(string file in repoFiles)
{
if(UpdateProgress != null)
UpdateProgress(null, string.Format("Checking file {0} of {1} from repository", counterF, repoFiles.Count), counterF, repoFiles.Count);
// Allow database to be inside repo
if(file == Settings.Current.DatabasePath)
continue;
if(Path.GetExtension(file).ToLowerInvariant() == ".xml" ||
Path.GetExtension(file).ToLowerInvariant() == ".json")
{
if(!dbCore.DBOps.ExistsOS(Path.GetFileNameWithoutExtension(file)))
filesToDelete.Add(file);
}
else if(!dbCore.DBOps.ExistsFile(Path.GetFileNameWithoutExtension(file)))
filesToDelete.Add(file);
counterF++;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to check all repository files", stopwatch.Elapsed.TotalSeconds);
stopwatch.Restart();
#endif
counterF = 0;
foreach(string file in filesToDelete)
{
if(UpdateProgress != null)
UpdateProgress(null, string.Format("Deleting file {0} of {1} from repository", counterF, filesToDelete.Count), counterF, filesToDelete.Count);
try
{
File.Delete(file);
}
#pragma warning disable RECS0022 // A catch clause that catches System.Exception and has an empty body
catch
#pragma warning restore RECS0022 // A catch clause that catches System.Exception and has an empty body
{
// Do not crash
}
counterF++;
}
#if DEBUG
stopwatch.Stop();
Console.WriteLine("Core.CleanFiles(): Took {0} seconds to delete all orphan files", stopwatch.Elapsed.TotalSeconds);
#endif
if(Finished != null)
Finished();
}
}

6
osrepodbmgr.Eto/ChangeLog
View File

@@ -1,3 +1,9 @@
2017-05-19 Natalia Portillo <claunia@claunia.com>
* frmMain.xeto:
* frmMain.xeto.cs:
Added code to clean database and repository.
2017-05-19 Natalia Portillo <claunia@claunia.com>
* frmMain.xeto.cs:

1
osrepodbmgr.Eto/frmMain.xeto
View File

@@ -49,6 +49,7 @@
<StackLayoutItem HorizontalAlignment="Center">
<StackLayout Orientation="Horizontal">
<Button x:Name="btnStopFiles" Click="OnBtnStopFilesClicked" Visible="False">Stop</Button>
<Button x:Name="btnCleanFiles" Click="OnBtnCleanFilesClicked" Visible="False">Clean files</Button>
<Button x:Name="btnToggleCrack" Click="OnBtnToggleCrackClicked" Visible="False">Mark as crack</Button>
<Button x:Name="btnScanAllPending" Click="OnBtnScanAllPendingClicked" Visible="False">Scan all with clamd</Button>
<Button x:Name="btnScanWithClamd" Click="OnBtnScanWithClamdClicked" Visible="False">Scan with clamd</Button>

72
osrepodbmgr.Eto/frmMain.xeto.cs
View File

@@ -46,6 +46,7 @@ namespace osrepodbmgr.Eto
Thread thdScanFile;
DBFile outIter;
bool scanningFiles;
Thread thdCleanFiles;
#region XAML UI elements
#pragma warning disable 0649
@@ -74,6 +75,9 @@ namespace osrepodbmgr.Eto
Button btnPopulateFiles;
TabPage tabOSes;
Button btnScanAllPending;
Button btnCleanFiles;
ButtonMenuItem btnQuit;
ButtonMenuItem mnuFile;
#pragma warning restore 0649
#endregion XAML UI elements
@@ -937,6 +941,7 @@ namespace osrepodbmgr.Eto
treeFiles.Enabled = true;
tabOSes.Enabled = true;
btnScanAllPending.Visible = true;
btnCleanFiles.Visible = true;
});
}
@@ -1008,5 +1013,72 @@ namespace osrepodbmgr.Eto
lblProgressFiles2.Text = text;
});
}
protected void OnBtnCleanFilesClicked(object sender, EventArgs e)
{
DialogResult result = MessageBox.Show("This option will search the database for any known file that doesn't\n" +
"belong to any OS and remove it from the database.\n\n" +
"It will then search the repository for any file not on the database and remove it.\n\n" +
"THIS OPERATION MAY VERY LONG, CANNOT BE CANCELED AND REMOVES DATA ON DISK.\n\n" +
"Are you sure to continue?", MessageBoxButtons.YesNo, MessageBoxType.Question);
if(result == DialogResult.Yes)
{
btnCleanFiles.Visible = false;
btnToggleCrack.Visible = false;
btnScanWithClamd.Visible = false;
btnScanAllPending.Visible = false;
btnCheckInVirusTotal.Visible = false;
tabOSes.Enabled = false;
treeFiles.Enabled = false;
lblProgressFiles1.Visible = true;
lblProgressFiles2.Visible = true;
Workers.Finished += CleanFilesFinished;
Workers.UpdateProgress += UpdateFileProgress;
Workers.UpdateProgress2 += UpdateFileProgress2;
lblProgressFiles1.Text = "";
prgProgressFiles1.Visible = true;
lblProgressFiles2.Text = "";
prgProgressFiles2.Visible = true;
btnStopFiles.Visible = false;
prgProgress2.Indeterminate = true;
btnSettings.Enabled = false;
btnHelp.Enabled = false;
mnuCompress.Enabled = false;
btnQuit.Enabled = false;
mnuFile.Enabled = false;
thdCleanFiles = new Thread(Workers.CleanFiles);
thdCleanFiles.Start();
}
}
void CleanFilesFinished()
{
Application.Instance.Invoke(delegate
{
btnCleanFiles.Visible = true;
btnToggleCrack.Visible = true;
btnScanWithClamd.Visible = true;
btnScanAllPending.Visible = true;
btnCheckInVirusTotal.Visible = true;
tabOSes.Enabled = true;
treeFiles.Enabled = true;
Workers.Finished -= CleanFilesFinished;
Workers.UpdateProgress -= UpdateFileProgress;
Workers.UpdateProgress2 -= UpdateFileProgress2;
lblProgressFiles1.Text = "";
prgProgressFiles1.Visible = false;
lblProgressFiles2.Text = "";
prgProgressFiles2.Visible = false;
btnSettings.Enabled = true;
btnHelp.Enabled = true;
mnuCompress.Enabled = true;
btnQuit.Enabled = true;
mnuFile.Enabled = true;
if(thdCleanFiles != null)
thdCleanFiles = null;
OnBtnPopulateFilesClicked(null, new EventArgs());
});
}
}
}

8
osrepodbmgr/ChangeLog
View File

@@ -1,3 +1,11 @@
2017-05-19 Natalia Portillo <claunia@claunia.com>
* frmMain.cs:
* gtk-gui/gui.stetic:
* gtk-gui/generated.cs:
* gtk-gui/osrepodbmgr.frmMain.cs:
Added code to clean database and repository.
2017-05-19 Natalia Portillo <claunia@claunia.com>
* frmMain.cs:

78
osrepodbmgr/frmMain.cs
View File

@@ -47,6 +47,7 @@ namespace osrepodbmgr
bool scanningFiles;
Thread thdScanFile;
TreeIter outIter;
Thread thdCleanFiles;
public frmMain() :
base(WindowType.Toplevel)
@@ -1036,6 +1037,7 @@ namespace osrepodbmgr
populatingFiles = false;
treeFiles.Sensitive = true;
notebook1.GetNthPage(0).Sensitive = true;
btnCleanFiles.Visible = true;
});
}
@@ -1121,5 +1123,81 @@ namespace osrepodbmgr
prgProgressFiles2.Text = text;
});
}
protected void OnBtnCleanFilesClicked(object sender, EventArgs e)
{
MessageDialog dlgMsg = new MessageDialog(this, DialogFlags.Modal, MessageType.Warning, ButtonsType.YesNo,
"This option will search the database for any known file that doesn't\n" +
"belong to any OS and remove it from the database.\n\n" +
"It will then search the repository for any file not on the database and remove it.\n\n" +
"THIS OPERATION MAY VERY LONG, CANNOT BE CANCELED AND REMOVES DATA ON DISK.\n\n" +
"Are you sure to continue?");
if(dlgMsg.Run() == (int)ResponseType.Yes)
{
dlgMsg.Destroy();
btnCleanFiles.Visible = false;
btnToggleCrack.Visible = false;
btnScanWithClamd.Visible = false;
btnScanAllPending.Visible = false;
btnCheckInVirusTotal.Visible = false;
notebook1.GetNthPage(0).Sensitive = false;
treeFiles.Sensitive = false;
Workers.Finished += CleanFilesFinished;
Workers.UpdateProgress += UpdateFileProgress;
Workers.UpdateProgress2 += UpdateFileProgress2;
prgProgressFiles1.Text = "";
prgProgressFiles1.Visible = true;
prgProgressFiles2.Text = "";
prgProgressFiles2.Visible = true;
btnStopFiles.Visible = false;
thdPulseProgress = new Thread(() =>
{
while(true)
{
Application.Invoke(delegate
{
prgProgressFiles2.Pulse();
});
Thread.Sleep(66);
}
});
thdPulseProgress.Start();
thdCleanFiles = new Thread(Workers.CleanFiles);
thdCleanFiles.Start();
}
else
dlgMsg.Destroy();
}
void CleanFilesFinished()
{
Application.Invoke(delegate
{
btnCleanFiles.Visible = true;
btnToggleCrack.Visible = true;
btnScanWithClamd.Visible = true;
btnScanAllPending.Visible = true;
btnCheckInVirusTotal.Visible = true;
notebook1.GetNthPage(0).Sensitive = true;
treeFiles.Sensitive = true;
Workers.Finished -= CleanFilesFinished;
Workers.UpdateProgress -= UpdateFileProgress;
Workers.UpdateProgress2 -= UpdateFileProgress2;
prgProgressFiles1.Text = "";
prgProgressFiles1.Visible = false;
prgProgressFiles2.Text = "";
prgProgressFiles2.Visible = false;
if(thdPulseProgress != null)
{
thdPulseProgress.Abort();
thdPulseProgress = null;
}
if(thdCleanFiles != null)
thdCleanFiles = null;
OnBtnPopulateFilesClicked(null, new EventArgs());
});
}
}
}

8
osrepodbmgr/gtk-gui/generated.cs
View File

@@ -8,7 +8,7 @@ namespace Stetic
internal static void Initialize(Gtk.Widget iconRenderer)
{
if((Stetic.Gui.initialized == false))
if ((Stetic.Gui.initialized == false))
{
Stetic.Gui.initialized = true;
}
@@ -20,7 +20,7 @@ namespace Stetic
public static Gdk.Pixbuf LoadIcon(Gtk.Widget widget, string name, Gtk.IconSize size)
{
Gdk.Pixbuf res = widget.RenderIcon(name, size, null);
if((res != null))
if ((res != null))
{
return res;
}
@@ -32,9 +32,9 @@ namespace Stetic
{
return Gtk.IconTheme.Default.LoadIcon(name, sz, 0);
}
catch(System.Exception)
catch (System.Exception)
{
if((name != "gtk-missing-image"))
if ((name != "gtk-missing-image"))
{
return Stetic.IconLoader.LoadIcon(widget, "gtk-missing-image", size);
}

22
osrepodbmgr/gtk-gui/gui.stetic
View File

@@ -12702,7 +12702,7 @@ QNX/QNX/20090229/source.zip</property>
</widget>
</child>
</widget>
<widget class="Gtk.Window" id="osrepodbmgr.frmMain" design-size="612 365">
<widget class="Gtk.Window" id="osrepodbmgr.frmMain" design-size="686 365">
<property name="MemberName" />
<property name="Title" translatable="yes">OS Repository DB Manager</property>
<property name="WindowPosition">CenterOnParent</property>
@@ -13162,6 +13162,24 @@ QNX/QNX/20090229/source.zip</property>
<property name="Fill">False</property>
</packing>
</child>
<child>
<widget class="Gtk.Button" id="btnCleanFiles">
<property name="MemberName" />
<property name="Visible">False</property>
<property name="CanFocus">True</property>
<property name="Type">TextOnly</property>
<property name="Label" translatable="yes">Clean files</property>
<property name="UseUnderline">True</property>
<signal name="Clicked" handler="OnBtnCleanFilesClicked" />
</widget>
<packing>
<property name="PackType">End</property>
<property name="Position">5</property>
<property name="AutoSize">True</property>
<property name="Expand">False</property>
<property name="Fill">False</property>
</packing>
</child>
<child>
<widget class="Gtk.Button" id="btnStopFiles">
<property name="MemberName" />
@@ -13175,7 +13193,7 @@ QNX/QNX/20090229/source.zip</property>
</widget>
<packing>
<property name="PackType">End</property>
<property name="Position">5</property>
<property name="Position">6</property>
<property name="AutoSize">True</property>
<property name="Expand">False</property>
<property name="Fill">False</property>

42
osrepodbmgr/gtk-gui/osrepodbmgr.frmMain.cs
View File

@@ -76,6 +76,8 @@ namespace osrepodbmgr
private global::Gtk.Button btnToggleCrack;
private global::Gtk.Button btnCleanFiles;
private global::Gtk.Button btnStopFiles;
private global::Gtk.Label label3;
@@ -415,6 +417,16 @@ namespace osrepodbmgr
w34.Expand = false;
w34.Fill = false;
// Container child hbox6.Gtk.Box+BoxChild
this.btnCleanFiles = new global::Gtk.Button();
this.btnCleanFiles.CanFocus = true;
this.btnCleanFiles.Name = "btnCleanFiles";
this.btnCleanFiles.UseUnderline = true;
this.btnCleanFiles.Label = global::Mono.Unix.Catalog.GetString("Clean files");
this.hbox6.Add(this.btnCleanFiles);
global::Gtk.Box.BoxChild w35 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnCleanFiles]));
w35.PackType = ((global::Gtk.PackType)(1));
w35.Position = 5;
// Container child hbox6.Gtk.Box+BoxChild
this.btnStopFiles = new global::Gtk.Button();
this.btnStopFiles.CanFocus = true;
this.btnStopFiles.Name = "btnStopFiles";
@@ -422,19 +434,19 @@ namespace osrepodbmgr
this.btnStopFiles.UseUnderline = true;
this.btnStopFiles.Label = "gtk-stop";
this.hbox6.Add(this.btnStopFiles);
global::Gtk.Box.BoxChild w35 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnStopFiles]));
w35.PackType = ((global::Gtk.PackType)(1));
w35.Position = 5;
w35.Expand = false;
w35.Fill = false;
this.vbox4.Add(this.hbox6);
global::Gtk.Box.BoxChild w36 = ((global::Gtk.Box.BoxChild)(this.vbox4[this.hbox6]));
w36.Position = 3;
global::Gtk.Box.BoxChild w36 = ((global::Gtk.Box.BoxChild)(this.hbox6[this.btnStopFiles]));
w36.PackType = ((global::Gtk.PackType)(1));
w36.Position = 6;
w36.Expand = false;
w36.Fill = false;
this.vbox4.Add(this.hbox6);
global::Gtk.Box.BoxChild w37 = ((global::Gtk.Box.BoxChild)(this.vbox4[this.hbox6]));
w37.Position = 3;
w37.Expand = false;
w37.Fill = false;
this.notebook1.Add(this.vbox4);
global::Gtk.Notebook.NotebookChild w37 = ((global::Gtk.Notebook.NotebookChild)(this.notebook1[this.vbox4]));
w37.Position = 1;
global::Gtk.Notebook.NotebookChild w38 = ((global::Gtk.Notebook.NotebookChild)(this.notebook1[this.vbox4]));
w38.Position = 1;
// Notebook tab
this.label3 = new global::Gtk.Label();
this.label3.Name = "label3";
@@ -442,14 +454,14 @@ namespace osrepodbmgr
this.notebook1.SetTabLabel(this.vbox4, this.label3);
this.label3.ShowAll();
this.vbox2.Add(this.notebook1);
global::Gtk.Box.BoxChild w38 = ((global::Gtk.Box.BoxChild)(this.vbox2[this.notebook1]));
w38.Position = 0;
global::Gtk.Box.BoxChild w39 = ((global::Gtk.Box.BoxChild)(this.vbox2[this.notebook1]));
w39.Position = 0;
this.Add(this.vbox2);
if((this.Child != null))
if ((this.Child != null))
{
this.Child.ShowAll();
}
this.DefaultWidth = 612;
this.DefaultWidth = 686;
this.DefaultHeight = 365;
this.lblProgress2.Hide();
this.prgProgress2.Hide();
@@ -467,6 +479,7 @@ namespace osrepodbmgr
this.btnScanWithClamd.Hide();
this.btnScanAllPending.Hide();
this.btnToggleCrack.Hide();
this.btnCleanFiles.Hide();
this.btnStopFiles.Hide();
this.Show();
this.DeleteEvent += new global::Gtk.DeleteEventHandler(this.OnDeleteEvent);
@@ -479,6 +492,7 @@ namespace osrepodbmgr
this.btnSettings.Clicked += new global::System.EventHandler(this.OnBtnSettingsClicked);
this.btnQuit.Clicked += new global::System.EventHandler(this.OnBtnQuitClicked);
this.btnStopFiles.Clicked += new global::System.EventHandler(this.OnBtnStopFilesClicked);
this.btnCleanFiles.Clicked += new global::System.EventHandler(this.OnBtnCleanFilesClicked);
this.btnToggleCrack.Clicked += new global::System.EventHandler(this.OnBtnToggleCrackClicked);
this.btnScanAllPending.Clicked += new global::System.EventHandler(this.OnBtnScanAllPendingClicked);
this.btnScanWithClamd.Clicked += new global::System.EventHandler(this.OnBtnScanWithClamdClicked);