claunia/marechai
1
0
Fork 0
mirror of https://github.com/claunia/marechai.git synced 2025-12-16 11:04:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move seeders outside of database context.

Browse Source
This commit is contained in:
Natalia Portillo
2020-05-23 18:28:56 +01:00
parent dd1df010d7
commit 87d200b715
12 changed files with 10546 additions and 4948 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3649
Marechai.Database/Migrations/20200523172033_MoveSeedOutOfBuilder.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

3587
Marechai.Database/Migrations/20200523172033_MoveSeedOutOfBuilder.cs Normal file
View File

File diff suppressed because it is too large Load Diff

2333
Marechai.Database/Migrations/MarechaiContextModelSnapshot.cs
View File

File diff suppressed because it is too large Load Diff

3
Marechai.Database/Models/MarechaiContext.cs
View File

@@ -1463,9 +1463,6 @@ namespace Marechai.Database.Models
entity.HasIndex(e => e.FsfApproved);
entity.HasIndex(e => e.OsiApproved);
});
Seeders.License.Seed(modelBuilder);
Seeders.DocumentRoles.Seed(modelBuilder);
}
}
}

56
Marechai.Database/Seeders/All.cs Normal file
View File

@@ -0,0 +1,56 @@
using System;
using Marechai.Database.Models;
namespace Marechai.Database.Seeders
{
public static class All
{
public static void Seed(MarechaiContext context)
{
DateTime start, end;
start = DateTime.Now;
Console.WriteLine("\u001b[31;1mImporting ISO-639 codes...\u001b[0m");
try
{
Iso639.Seed(context);
}
catch(Exception e)
{
Console.WriteLine("Exception {0} importing ISO-639 codes, saving changes and continuing...", e);
}
context.SaveChanges();
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
start = DateTime.Now;
Console.WriteLine("\u001b[31;1mSeeding document roles...\u001b[0m");
DocumentRoles.Seed(context);
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
start = DateTime.Now;
Console.WriteLine("\u001b[31;1mSeeding licenses...\u001b[0m");
License.Seed(context);
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
start = DateTime.Now;
Console.WriteLine("\u001b[31;1mSaving changes...\u001b[0m");
context.SaveChanges();
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
}
}
}

1944
Marechai.Database/Seeders/DocumentRoles.cs
View File

File diff suppressed because it is too large Load Diff

158
Marechai.Database/Seeders/Iso639.cs Normal file
View File

@@ -0,0 +1,158 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Marechai.Database.Models;
namespace Marechai.Database.Seeders
{
public static class Iso639
{
// Data files can be found in https://iso639-3.sil.org/code_tables/download_tables
public static void Seed(MarechaiContext context)
{
if(!Directory.Exists("iso639"))
return;
IEnumerable<string> files = Directory.EnumerateFiles("iso639", "iso-639-3_*.tab");
long modified = 0;
List<Models.Iso639> codes = new List<Models.Iso639>();
foreach(string file in files)
{
Console.WriteLine("Importing ISO-639 codes from {0}", file);
using var fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None);
using var sr = new StreamReader(fs, Encoding.UTF8);
string line = sr.ReadLine();
if(line is null)
{
Console.WriteLine("Invalid data, not proceeding");
continue;
}
string[] pieces = line.Split('\t');
if(pieces.Length != 8 ||
pieces[0] != "Id" ||
pieces[1] != "Part2B" ||
pieces[2] != "Part2T" ||
pieces[3] != "Part1" ||
pieces[4] != "Scope" ||
pieces[5] != "Language_Type" ||
pieces[6] != "Ref_Name" ||
pieces[7] != "Comment")
{
Console.WriteLine("Invalid data, not proceeding");
continue;
}
line = sr.ReadLine();
while(line != null)
{
pieces = line.Split('\t');
if(pieces.Length != 8)
{
Console.WriteLine("Invalid data, continuing with next line");
line = sr.ReadLine();
continue;
}
for(int p = 1; p < 8; p++)
if(pieces[p] == "")
pieces[p] = null;
codes.Add(new Models.Iso639
{
Id = pieces[0], Part2B = pieces[1], Part2T = pieces[2], Part1 = pieces[3],
Scope = pieces[4], Type = pieces[5], ReferenceName = pieces[6], Comment = pieces[7]
});
line = sr.ReadLine();
}
}
if(codes.Count == 0)
{
Console.WriteLine("No codes found");
return;
}
List<Models.Iso639> existingCodes = context.Iso639.ToList();
List<Models.Iso639> newCodes = new List<Models.Iso639>();
foreach(Models.Iso639 code in codes)
{
Models.Iso639 existingCode = existingCodes.FirstOrDefault(c => c.Id == code.Id);
if(existingCode is null)
newCodes.Add(code);
else
{
bool changed = false;
if(code.Part2B != existingCode.Part2B)
{
existingCode.Part2B = code.Part2B;
changed = true;
}
if(code.Part2T != existingCode.Part2T)
{
existingCode.Part2T = code.Part2T;
changed = true;
}
if(code.Part1 != existingCode.Part1)
{
existingCode.Part1 = code.Part1;
changed = true;
}
if(code.Scope != existingCode.Scope)
{
existingCode.Scope = code.Scope;
changed = true;
}
if(code.Type != existingCode.Type)
{
existingCode.Type = code.Type;
changed = true;
}
if(code.ReferenceName != existingCode.ReferenceName)
{
existingCode.ReferenceName = code.ReferenceName;
changed = true;
}
if(code.Comment != existingCode.Comment)
{
existingCode.Comment = code.Comment;
changed = true;
}
if(changed)
modified++;
}
}
context.Iso639.AddRange(newCodes);
Console.WriteLine("{0} language codes added", newCodes.Count);
Console.WriteLine("{0} language codes modified", modified);
}
}
}

3592
Marechai.Database/Seeders/License.cs
View File

File diff suppressed because it is too large Load Diff

125
Marechai/Helpers/Iso639.cs
View File

@@ -1,125 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Marechai.Database.Models;
namespace Marechai.Helpers
{
public static class Iso639
{
// TODO: Changes file
// Data files can be found in https://iso639-3.sil.org/code_tables/download_tables
internal static void Import(MarechaiContext context)
{
if(!Directory.Exists("iso639"))
return;
IEnumerable<string> files = Directory.EnumerateFiles("iso639", "iso-639-3_*.tab");
long added = 0;
long modified = 0;
foreach(string file in files)
{
Console.WriteLine("Importing ISO-639 codes from {0}", file);
using(var fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.None))
{
using(var sr = new StreamReader(fs, Encoding.UTF8))
{
string line;
string[] pieces;
line = sr.ReadLine();
if(line is null)
{
Console.WriteLine("Invalid data, not proceeding");
continue;
}
pieces = line.Split('\t');
if(pieces.Length != 8 ||
pieces[0] != "Id" ||
pieces[1] != "Part2B" ||
pieces[2] != "Part2T" ||
pieces[3] != "Part1" ||
pieces[4] != "Scope" ||
pieces[5] != "Language_Type" ||
pieces[6] != "Ref_Name" ||
pieces[7] != "Comment")
{
Console.WriteLine("Invalid data, not proceeding");
continue;
}
line = sr.ReadLine();
while(line != null)
{
pieces = line.Split('\t');
if(pieces.Length != 8)
{
Console.WriteLine("Invalid data, continuing with next line");
line = sr.ReadLine();
continue;
}
for(int p = 1; p < 8; p++)
if(pieces[p] == "")
pieces[p] = null;
Database.Models.Iso639 lang = context.Iso639.FirstOrDefault(i => i.Id == pieces[0]);
if(lang is null)
{
context.Iso639.Add(new Database.Models.Iso639
{
Id = pieces[0], Part2B = pieces[1], Part2T = pieces[2],
Part1 = pieces[3],
Scope = pieces[4], Type = pieces[5], ReferenceName = pieces[6],
Comment = pieces[7]
});
line = sr.ReadLine();
added++;
continue;
}
if(pieces[1] != lang.Part2B ||
pieces[2] != lang.Part2T ||
pieces[3] != lang.Part1 ||
pieces[4] != lang.Scope ||
pieces[5] != lang.Type ||
pieces[6] != lang.ReferenceName ||
pieces[7] != lang.Comment)
{
lang.Part2B = pieces[1];
lang.Part2T = pieces[2];
lang.Part1 = pieces[3];
lang.Scope = pieces[4];
lang.Type = pieces[5];
lang.ReferenceName = pieces[6];
lang.Comment = pieces[7];
context.Iso639.Update(lang);
modified++;
}
line = sr.ReadLine();
}
}
}
}
Console.WriteLine("{0} language codes added", added);
Console.WriteLine("{0} language codes modified", modified);
}
}
}

10
Marechai/Marechai.csproj
View File

@@ -2,7 +2,7 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Version>3.0.99.990</Version>
<Version>3.0.99.1007</Version>
<Company>Canary Islands Computer Museum</Company>
<Copyright>Copyright © 2003-2020 Natalia Portillo</Copyright>
<Product>Canary Islands Computer Museum Website</Product>
@@ -34,14 +34,6 @@
<PackageReference Include="SkiaSharp.Svg" Version="1.60.0" />
<PackageReference Include="Unclassified.NetRevisionTask" Version="0.3.0" />
</ItemGroup>
<ItemGroup>
<DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="2.0.3" />
</ItemGroup>
<ItemGroup>
<Reference Include="Microsoft.AspNetCore.Authorization, Version=2.2.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60">
<HintPath>..\..\..\.nuget\packages\microsoft.aspnetcore.authorization\2.2.0\lib\netstandard2.0\Microsoft.AspNetCore.Authorization.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<None Update="iso639\iso-639-3_20190408.tab">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

22
Marechai/Program.cs
View File

@@ -39,7 +39,6 @@ using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Iso639 = Marechai.Helpers.Iso639;
using Version = DiscImageChef.Interop.Version;
namespace Marechai
@@ -192,27 +191,6 @@ namespace Marechai
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
start = DateTime.Now;
Console.WriteLine("\u001b[31;1mImporting ISO-639 codes...\u001b[0m");
try
{
Iso639.Import(context);
}
catch(Exception e)
{
Console.WriteLine("Exception {0} importing ISO-639 codes, saving changes and continuing...", e);
throw;
}
context.SaveChanges();
end = DateTime.Now;
Console.WriteLine("\u001b[31;1mTook \u001b[32;1m{0} seconds\u001b[31;1m...\u001b[0m",
(end - start).TotalSeconds);
}

15
Marechai/Startup.cs
View File

@@ -34,6 +34,7 @@ using Blazorise.Bootstrap;
using Blazorise.Icons.FontAwesome;
using Marechai.Areas.Identity;
using Marechai.Database.Models;
using Marechai.Database.Seeders;
using Marechai.Services;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Components.Authorization;
@@ -62,10 +63,8 @@ namespace Marechai
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services
.AddBlazorise( options => options.ChangeTextOnKeyPress = true)
.AddBootstrapProviders()
.AddFontAwesomeIcons();
services.AddBlazorise(options => options.ChangeTextOnKeyPress = true).AddBootstrapProviders().
AddFontAwesomeIcons();
services.AddDbContext<MarechaiContext>(options => options.
UseLazyLoadingProxies().
@@ -87,7 +86,7 @@ namespace Marechai
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
public void Configure(IApplicationBuilder app, IWebHostEnvironment env, MarechaiContext context)
{
if(env.IsDevelopment())
{
@@ -118,13 +117,13 @@ namespace Marechai
app.UseRouting();
app.ApplicationServices
.UseBootstrapProviders()
.UseFontAwesomeIcons();
app.ApplicationServices.UseBootstrapProviders().UseFontAwesomeIcons();
app.UseAuthentication();
app.UseAuthorization();
All.Seed(context);
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();