From 3387d90b6c057fa3f9007e5765df107169fbc206 Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Thu, 11 May 2017 04:36:45 +0100 Subject: [PATCH] Change to opaque repo, where the created files are a hash of the OS data (name, developer, etc). --- osrepodbmgr.Core/ChangeLog | 8 ++ osrepodbmgr.Core/DBOps.cs | 35 ++++++- osrepodbmgr.Core/Schema.cs | 2 + osrepodbmgr.Core/Workers.cs | 181 +++++++++++++++++++++++++----------- 4 files changed, 168 insertions(+), 58 deletions(-) diff --git a/osrepodbmgr.Core/ChangeLog b/osrepodbmgr.Core/ChangeLog index 2e524f8..667b6f7 100644 --- a/osrepodbmgr.Core/ChangeLog +++ b/osrepodbmgr.Core/ChangeLog @@ -1,3 +1,11 @@ +2017-05-11 Natalia Portillo + + * DBOps.cs: + * Schema.cs: + * Workers.cs: + Change to opaque repo, where the created files are a hash of + the OS data (name, developer, etc). + 2017-05-11 Natalia Portillo * Workers.cs: diff --git a/osrepodbmgr.Core/DBOps.cs b/osrepodbmgr.Core/DBOps.cs index 10a015c..f8a723d 100644 --- a/osrepodbmgr.Core/DBOps.cs +++ b/osrepodbmgr.Core/DBOps.cs @@ -51,6 +51,7 @@ namespace osrepodbmgr.Core public bool netinstall; public byte[] xml; public byte[] json; + public string mdid; } public struct DBFile @@ -108,6 +109,7 @@ namespace osrepodbmgr.Core fEntry.source = bool.Parse(dRow["source"].ToString()); fEntry.files = bool.Parse(dRow["files"].ToString()); fEntry.netinstall = bool.Parse(dRow["netinstall"].ToString()); + fEntry.mdid = dRow["mdid"].ToString(); if(dRow["xml"] != DBNull.Value) fEntry.xml = (byte[])dRow["xml"]; @@ -139,6 +141,7 @@ namespace osrepodbmgr.Core IDbDataParameter param14 = dbcmd.CreateParameter(); IDbDataParameter param15 = dbcmd.CreateParameter(); IDbDataParameter param16 = dbcmd.CreateParameter(); + IDbDataParameter param17 = dbcmd.CreateParameter(); param1.ParameterName = "@developer"; param2.ParameterName = "@product"; @@ -156,6 +159,7 @@ namespace osrepodbmgr.Core param14.ParameterName = "@netinstall"; param15.ParameterName = "@xml"; param16.ParameterName = "@json"; + param17.ParameterName = "@mdid"; param1.DbType = DbType.String; param2.DbType = DbType.String; @@ -172,6 +176,7 @@ namespace osrepodbmgr.Core param14.DbType = DbType.Boolean; param15.DbType = DbType.Object; param16.DbType = DbType.Object; + param17.DbType = DbType.String; param1.Value = entry.developer; param2.Value = entry.product; @@ -189,6 +194,7 @@ namespace osrepodbmgr.Core param14.Value = entry.netinstall; param15.Value = entry.xml; param16.Value = entry.json; + param17.Value = entry.mdid; dbcmd.Parameters.Add(param1); dbcmd.Parameters.Add(param2); @@ -206,6 +212,7 @@ namespace osrepodbmgr.Core dbcmd.Parameters.Add(param14); dbcmd.Parameters.Add(param15); dbcmd.Parameters.Add(param16); + dbcmd.Parameters.Add(param17); return dbcmd; } @@ -216,8 +223,8 @@ namespace osrepodbmgr.Core IDbTransaction trans = dbCon.BeginTransaction(); dbcmd.Transaction = trans; - const string sql = "INSERT INTO oses (developer, product, version, languages, architecture, machine, format, description, oem, upgrade, `update`, source, files, netinstall, xml, json)" + - " VALUES (@developer, @product, @version, @languages, @architecture, @machine, @format, @description, @oem, @upgrade, @update, @source, @files, @netinstall, @xml, @json)"; + const string sql = "INSERT INTO oses (developer, product, version, languages, architecture, machine, format, description, oem, upgrade, `update`, source, files, netinstall, xml, json, mdid)" + + " VALUES (@developer, @product, @version, @languages, @architecture, @machine, @format, @description, @oem, @upgrade, @update, @source, @files, @netinstall, @xml, @json, @mdid)"; dbcmd.CommandText = sql; @@ -399,6 +406,30 @@ namespace osrepodbmgr.Core return false; } + + public bool ExistsOS(string mdid) + { + IDbCommand dbcmd = dbCon.CreateCommand(); + IDbDataParameter param1 = dbcmd.CreateParameter(); + + param1.ParameterName = "@mdid"; + param1.DbType = DbType.String; + param1.Value = mdid; + dbcmd.Parameters.Add(param1); + dbcmd.CommandText = "SELECT * FROM `oses` WHERE mdid = @mdid"; + DataSet dataSet = new DataSet(); + IDbDataAdapter dataAdapter = dbCore.GetNewDataAdapter(); + dataAdapter.SelectCommand = dbcmd; + dataAdapter.Fill(dataSet); + DataTable dataTable = dataSet.Tables[0]; + + foreach(DataRow dRow in dataTable.Rows) + { + return true; + } + + return false; + } } } diff --git a/osrepodbmgr.Core/Schema.cs b/osrepodbmgr.Core/Schema.cs index 3bcea9b..fc0234b 100644 --- a/osrepodbmgr.Core/Schema.cs +++ b/osrepodbmgr.Core/Schema.cs @@ -45,6 +45,7 @@ namespace osrepodbmgr.Core "DROP TABLE IF EXISTS `oses` ;\n\n" + "CREATE TABLE IF NOT EXISTS `oses` (\n" + " `id` INTEGER PRIMARY KEY AUTOINCREMENT,\n" + + " `mdid` CHAR(40) NOT NULL,\n" + " `developer` VARCHAR(45) NOT NULL,\n" + " `product` VARCHAR(45) NOT NULL,\n" + " `version` VARCHAR(45) NULL,\n" + @@ -62,6 +63,7 @@ namespace osrepodbmgr.Core " `xml` BLOB NULL,\n" + " `json` BLOB NULL);\n\n" + "CREATE UNIQUE INDEX `oses_id_UNIQUE` ON `oses` (`id` ASC);\n\n" + + "CREATE UNIQUE INDEX `oses_mdid_UNIQUE` ON `oses` (`mdid` ASC);\n\n" + "CREATE INDEX `oses_developer_idx` ON `oses` (`developer` ASC);\n\n" + "CREATE INDEX `oses_product_idx` ON `oses` (`product` ASC);\n\n" + "CREATE INDEX `oses_version_idx` ON `oses` (`version` ASC);\n\n" + diff --git a/osrepodbmgr.Core/Workers.cs b/osrepodbmgr.Core/Workers.cs index eb068d8..8067ece 100644 --- a/osrepodbmgr.Core/Workers.cs +++ b/osrepodbmgr.Core/Workers.cs @@ -732,49 +732,25 @@ namespace osrepodbmgr.Core return; } - // Check if repository folder exists - string destinationFolder = Settings.Current.RepositoryPath; - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if developer folder exists + string destinationFolder = ""; destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.developer); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if product folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.product); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if version folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.version); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); if(!string.IsNullOrWhiteSpace(Context.dbInfo.languages)) { - // Check if languages folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.languages); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(!string.IsNullOrWhiteSpace(Context.dbInfo.architecture)) { - // Check if architecture folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.architecture); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(Context.dbInfo.oem) { - // Check if oem folder exists destinationFolder = Path.Combine(destinationFolder, "oem"); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(!string.IsNullOrWhiteSpace(Context.dbInfo.machine)) { - // Check if architecture folder exists destinationFolder = Path.Combine(destinationFolder, "for " + Context.dbInfo.machine); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } string destinationFile = ""; @@ -822,17 +798,63 @@ namespace osrepodbmgr.Core } string destination = Path.Combine(destinationFolder, destinationFile) + ".zip"; + + MD5Context md5 = new MD5Context(); + md5.Init(); + byte[] tmp; + string mdid = md5.Data(Encoding.UTF8.GetBytes(destination), out tmp); + Console.WriteLine("MDID: {0}", mdid); + + destinationFolder = Settings.Current.RepositoryPath; + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[0].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[1].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[2].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[3].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[4].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + + destination = Path.Combine(destinationFolder, mdid) + ".zip"; + + if(dbCore.DBOps.ExistsOS(mdid)) + { + if(File.Exists(destination)) + { + if(Failed != null) + Failed("OS already exists."); + return; + } + + if(Failed != null) + Failed("OS already exists in the database but not in the repository, check for inconsistencies."); + return; + } + if(File.Exists(destination)) { if(Failed != null) - Failed("File already exists"); + Failed("OS already exists in the repository but not in the database, check for inconsistencies."); return; } + Context.dbInfo.mdid = mdid; + ZipFile zf = new ZipFile(destination); zf.CompressionLevel = Ionic.Zlib.CompressionLevel.BestCompression; zf.CompressionMethod = Settings.Current.CompressionAlgorithm; zf.UseZip64WhenSaving = Zip64Option.AsNecessary; + zf.AlternateEncoding = Encoding.UTF8; + zf.AlternateEncodingUsage = ZipOption.Always; string filesPath; @@ -892,10 +914,10 @@ namespace osrepodbmgr.Core zj.LastModified = zx.AccessedTime; zj.FileName = "metadata.json"; - FileStream xfs = new FileStream(Path.Combine(destinationFolder, destinationFile + ".xml"), FileMode.CreateNew, FileAccess.Write); + FileStream xfs = new FileStream(Path.Combine(destinationFolder, mdid + ".xml"), FileMode.CreateNew, FileAccess.Write); xms.CopyTo(xfs); xfs.Close(); - FileStream jfs = new FileStream(Path.Combine(destinationFolder, destinationFile + ".json"), FileMode.CreateNew, FileAccess.Write); + FileStream jfs = new FileStream(Path.Combine(destinationFolder, mdid + ".json"), FileMode.CreateNew, FileAccess.Write); jms.CopyTo(jfs); jfs.Close(); @@ -1235,7 +1257,7 @@ namespace osrepodbmgr.Core if(UpdateProgress != null && e.CurrentEntry != null && e.EntriesTotal > 0) UpdateProgress("Extracting...", e.CurrentEntry.FileName, zipCounter, e.EntriesTotal); - if(UpdateProgress2 != null) + if(UpdateProgress2 != null && e.TotalBytesToTransfer > 0) UpdateProgress2(string.Format("{0:P}", e.BytesTransferred / (double)e.TotalBytesToTransfer), string.Format("{0} / {1}", e.BytesTransferred, e.TotalBytesToTransfer), e.BytesTransferred, e.TotalBytesToTransfer); @@ -1270,49 +1292,25 @@ namespace osrepodbmgr.Core return; } - // Check if repository folder exists - string destinationFolder = Settings.Current.RepositoryPath; - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if developer folder exists + string destinationFolder = ""; destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.developer); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if product folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.product); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); - // Check if version folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.version); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); if(!string.IsNullOrWhiteSpace(Context.dbInfo.languages)) { - // Check if languages folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.languages); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(!string.IsNullOrWhiteSpace(Context.dbInfo.architecture)) { - // Check if architecture folder exists destinationFolder = Path.Combine(destinationFolder, Context.dbInfo.architecture); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(Context.dbInfo.oem) { - // Check if oem folder exists destinationFolder = Path.Combine(destinationFolder, "oem"); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } if(!string.IsNullOrWhiteSpace(Context.dbInfo.machine)) { - // Check if architecture folder exists destinationFolder = Path.Combine(destinationFolder, "for " + Context.dbInfo.machine); - if(!Directory.Exists(destinationFolder)) - Directory.CreateDirectory(destinationFolder); } string destinationFile = ""; @@ -1360,14 +1358,85 @@ namespace osrepodbmgr.Core } string destination = Path.Combine(destinationFolder, destinationFile) + ".zip"; - if(File.Exists(destination)) + + MD5Context md5 = new MD5Context(); + md5.Init(); + byte[] tmp; + string mdid = md5.Data(Encoding.UTF8.GetBytes(destination), out tmp); + Console.WriteLine("MDID: {0}", mdid); + + destinationFolder = Settings.Current.RepositoryPath; + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[0].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[1].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[2].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[3].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + destinationFolder = Path.Combine(destinationFolder, mdid[4].ToString()); + if(!Directory.Exists(destinationFolder)) + Directory.CreateDirectory(destinationFolder); + + destination = Path.Combine(destinationFolder, mdid) + ".zip"; + + if(dbCore.DBOps.ExistsOS(mdid)) { + if(File.Exists(destination)) + { + if(Failed != null) + Failed("OS already exists."); + return; + } + if(Failed != null) - Failed("File already exists"); + Failed("OS already exists in the database but not in the repository, check for inconsistencies."); return; } + if(File.Exists(destination)) + { + if(Failed != null) + Failed("OS already exists in the repository but not in the database, check for inconsistencies."); + return; + } + + Context.dbInfo.mdid = mdid; + File.Copy(Context.path, destination); + + if(Context.metadata != null) + { + MemoryStream xms = new MemoryStream(); + XmlSerializer xs = new XmlSerializer(typeof(CICMMetadataType)); + xs.Serialize(xms, Context.metadata); + xms.Position = 0; + + JsonSerializer js = new JsonSerializer(); + js.Formatting = Newtonsoft.Json.Formatting.Indented; + js.NullValueHandling = NullValueHandling.Ignore; + MemoryStream jms = new MemoryStream(); + StreamWriter sw = new StreamWriter(jms, Encoding.UTF8, 1048576, true); + js.Serialize(sw, Context.metadata, typeof(CICMMetadataType)); + sw.Close(); + jms.Position = 0; + + FileStream xfs = new FileStream(Path.Combine(destinationFolder, mdid + ".xml"), FileMode.CreateNew, FileAccess.Write); + xms.CopyTo(xfs); + xfs.Close(); + FileStream jfs = new FileStream(Path.Combine(destinationFolder, mdid + ".json"), FileMode.CreateNew, FileAccess.Write); + jms.CopyTo(jfs); + jfs.Close(); + + xms.Position = 0; + jms.Position = 0; + } } catch(Exception ex) {