feat: begin working on database maintainer

src/SanctionsSearch.Worker.Tests/Faker/OfacFileServiceOptionsFaker.cs

@@ -0,0 +1,17 @@
+namespace SanctionsSearch.Worker.Tests.Faker;
+
+class OfacFileServiceOptionsFaker : Faker<OfacFileServiceOptions>
+{
+  public OfacFileServiceOptionsFaker()
+  {
+    RuleFor(x => x.Url, f => f.Internet.Url());
+    RuleFor(x => x.SdnFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.AddressFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.AltNamesFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.CommentsFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.ConPrimaryNameFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.ConAddressesFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.ConAltNamesFileName, f => f.System.FileName("csv"));
+    RuleFor(x => x.ConCommentsFileName, f => f.System.FileName("csv"));
+  }
+}

src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs

@@ -0,0 +1,86 @@
+namespace SanctionsSearch.Worker.Tests.Integration;
+
+public class DatabaseMaintainerTests : DatabaseTest
+{
+  private readonly MockHttpMessageHandler _mockHttp = new();
+  private readonly OfacFileServiceOptionsFaker _ofacFileServiceOptionsFaker = new();
+  private readonly OfacFileServiceOptions _ofacFileServiceOptions;
+  private readonly DatabaseMaintainer _databaseMaintainer;
+  private static MemoryStream CreateCsvStream(string csv) => new(Encoding.UTF8.GetBytes(csv));
+
+  public DatabaseMaintainerTests()
+  {
+    _ofacFileServiceOptions = _ofacFileServiceOptionsFaker.Generate();
+
+    var ofacFileService = new OfacFileService(
+      _mockHttp.ToHttpClient(),
+      _loggerFactory.CreateLogger<OfacFileService>(),
+      _ofacFileServiceOptions
+    );
+
+    var uow = new EfUnitOfWork(_context, _loggerFactory);
+    var logger = _loggerFactory.CreateLogger<DatabaseMaintainer>();
+    _databaseMaintainer = new DatabaseMaintainer(uow, ofacFileService, logger);
+  }
+
+  [Fact]
+  public async Task BuildSdnTableAsync_WhenCalled_ItShouldAddSdnCsvRecordsToDatabase()
+  {
+    var testCsv = """
+      6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male."
+      6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt."
+      6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt."
+    """;
+
+    var testStream = CreateCsvStream(testCsv);
+
+    _mockHttp
+      .When(_ofacFileServiceOptions.GetSdnFileUri().ToString())
+      .Respond("text/csv", testStream);
+
+    var now = DateTimeOffset.UtcNow;
+
+    _timeProviderMock
+      .Setup(x => x.GetUtcNow())
+      .Returns(now);
+
+    await _databaseMaintainer.BuildSdnTableAsync();
+
+    var sdns = await _context.Set<Sdn>().ToListAsync();
+
+    sdns.Should().HaveCount(3);
+
+    sdns[0].Should().BeEquivalentTo(new Sdn()
+    {
+      Id = 6906,
+      Name = "AL-IRAQI, Abd al-Hadi",
+      Type = "individual",
+      Program = "SDGT",
+      Remarks = "DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male.",
+      CreatedAt = now.DateTime,
+      UpdatedAt = now.DateTime
+    });
+
+    sdns[1].Should().BeEquivalentTo(new Sdn()
+    {
+      Id = 6907,
+      Name = "SHIHATA, Thirwat Salah",
+      Type = "individual",
+      Program = "SDGT",
+      Remarks = "DOB 29 Jun 1960; POB Egypt.",
+      CreatedAt = now.DateTime,
+      UpdatedAt = now.DateTime
+    });
+
+    sdns[2].Should().BeEquivalentTo(new Sdn()
+    {
+      Id = 6908,
+      Name = "AHMAD, Tariq Anwar al-Sayyid",
+      Type = "individual",
+      Program = "SDGT",
+      Remarks = "DOB 15 Mar 1963; POB Alexandria, Egypt.",
+      CreatedAt = now.DateTime,
+      UpdatedAt = now.DateTime
+    });
+  }
+}

src/SanctionsSearch.Worker.Tests/Unit/OfacFileServiceTests.cs

@@ -2,35 +2,22 @@ namespace SanctionsSearch.Worker.Tests.Unit;
 
 public class OfacFileServiceTests
 {
-  private const string Url = "https://example.com";
   private const string TestCsv = "Name,Address\nJohn Doe,123 Main St\n";
   private static Stream TestStream => new MemoryStream(Encoding.UTF8.GetBytes(TestCsv));
   private readonly MockHttpMessageHandler _mockHttp = new();
   private readonly Mock<ILogger<OfacFileService>> _mockLogger = new();
-  private readonly Mock<IOptionsSnapshot<OfacFileServiceOptions>> _mockOptions = new();
+  private readonly OfacFileServiceOptionsFaker _optionsFaker = new();
+  private readonly OfacFileServiceOptions _options;
   private readonly OfacFileService _service;
 
   public OfacFileServiceTests()
   {
-    _mockOptions
+    _options = _optionsFaker.Generate();
-      .Setup(x => x.Value)
-      .Returns(new OfacFileServiceOptions
-      {
-        Url = Url,
-        SdnFileName = "sdn.csv",
-        AddressFileName = "address.csv",
-        AltNamesFileName = "alt_names.csv",
-        CommentsFileName = "comments.csv",
-        ConPrimaryNameFileName = "con_primary_name.csv",
-        ConAddressesFileName = "con_addresses.csv",
-        ConAltNamesFileName = "con_alt_names.csv",
-        ConCommentsFileName = "con_comments.csv"
-      });
 
     _service = new OfacFileService(
       _mockHttp.ToHttpClient(),
       _mockLogger.Object,
-      _mockOptions.Object
+      _options
     );
   }
 
@@ -46,7 +33,7 @@ public class OfacFileServiceTests
     var act = () => new OfacFileService(
       null!,
       _mockLogger.Object,
-      _mockOptions.Object
+      _options
     );
 
     act.Should().Throw<ArgumentNullException>();
@@ -58,7 +45,7 @@ public class OfacFileServiceTests
     var act = () => new OfacFileService(
       _mockHttp.ToHttpClient(),
       null!,
-      _mockOptions.Object
+      _options
     );
 
     act.Should().Throw<ArgumentNullException>();
@@ -79,12 +66,10 @@ public class OfacFileServiceTests
   [Fact]
   public void Constructor_WhenCalledWithNullOptionsValue_ThrowsArgumentNullException()
   {
-    _mockOptions.Setup(x => x.Value).Returns(() => null!);
-
     var act = () => new OfacFileService(
       _mockHttp.ToHttpClient(),
       _mockLogger.Object,
-      _mockOptions.Object
+      null!
     );
 
     act.Should().Throw<ArgumentNullException>();
@@ -94,7 +79,7 @@ public class OfacFileServiceTests
   public async Task GetSdnFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/sdn.csv")
+      .When(_options.GetSdnFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetSdnFileAsync();
@@ -107,7 +92,7 @@ public class OfacFileServiceTests
   public async Task GetSdnFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/sdn.csv")
+      .When(_options.GetSdnFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetSdnFileAsync();
@@ -119,7 +104,7 @@ public class OfacFileServiceTests
   public async Task GetSdnFileAsync_WhenExceptionThrown_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/sdn.csv")
+      .When(_options.GetSdnFileUri().ToString())
       .Throw(new Exception("Test exception"));
 
     var result = await _service.GetSdnFileAsync();
@@ -131,7 +116,7 @@ public class OfacFileServiceTests
   public async Task GetAddressFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/address.csv")
+      .When(_options.GetAddressFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetAddressFileAsync();
@@ -144,7 +129,7 @@ public class OfacFileServiceTests
   public async Task GetAddressFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/address.csv")
+      .When(_options.GetAddressFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetAddressFileAsync();
@@ -156,7 +141,7 @@ public class OfacFileServiceTests
   public async Task GetAltNamesFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/alt_names.csv")
+      .When(_options.GetAltNamesFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetAltNamesFileAsync();
@@ -169,7 +154,7 @@ public class OfacFileServiceTests
   public async Task GetAltNamesFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/alt_names.csv")
+      .When(_options.GetAltNamesFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetAltNamesFileAsync();
@@ -181,7 +166,7 @@ public class OfacFileServiceTests
   public async Task GetCommentsFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/comments.csv")
+      .When(_options.GetCommentsFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetCommentsFileAsync();
@@ -194,7 +179,7 @@ public class OfacFileServiceTests
   public async Task GetCommentsFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/comments.csv")
+      .When(_options.GetCommentsFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetCommentsFileAsync();
@@ -206,7 +191,7 @@ public class OfacFileServiceTests
   public async Task GetConPrimaryNameFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/con_primary_name.csv")
+      .When(_options.GetConPrimaryNameFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetConPrimaryNameFileAsync();
@@ -219,7 +204,7 @@ public class OfacFileServiceTests
   public async Task GetConPrimaryNameFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/con_primary_name.csv")
+      .When(_options.GetConPrimaryNameFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetConPrimaryNameFileAsync();
@@ -231,7 +216,7 @@ public class OfacFileServiceTests
   public async Task GetConAddressesFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/con_addresses.csv")
+      .When(_options.GetConAddressesFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetConAddressesFileAsync();
@@ -244,7 +229,7 @@ public class OfacFileServiceTests
   public async Task GetConAddressesFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/con_addresses.csv")
+      .When(_options.GetConAddressesFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetConAddressesFileAsync();
@@ -256,7 +241,7 @@ public class OfacFileServiceTests
   public async Task GetConAltNamesFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/con_alt_names.csv")
+      .When(_options.GetConAltNamesFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetConAltNamesFileAsync();
@@ -269,7 +254,7 @@ public class OfacFileServiceTests
   public async Task GetConAltNamesFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/con_alt_names.csv")
+      .When(_options.GetConAltNamesFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetConAltNamesFileAsync();
@@ -281,7 +266,7 @@ public class OfacFileServiceTests
   public async Task GetConCommentsFileAsync_WhenCalled_ItShouldReturnStream()
   {
     _mockHttp
-      .When("https://example.com/con_comments.csv")
+      .When(_options.GetConCommentsFileUri().ToString())
       .Respond("text/csv", TestStream);
 
     var result = await _service.GetConCommentsFileAsync();
@@ -294,7 +279,7 @@ public class OfacFileServiceTests
   public async Task GetConCommentsFileAsync_WhenRequestFails_ItShouldReturnFailure()
   {
     _mockHttp
-      .When("https://example.com/con_comments.csv")
+      .When(_options.GetConCommentsFileUri().ToString())
       .Respond(HttpStatusCode.InternalServerError);
 
     var result = await _service.GetConCommentsFileAsync();

src/SanctionsSearch.Worker/Interfaces/IDatabaseMaintainer.cs

@@ -0,0 +1,9 @@
+namespace SanctionsSearch.Worker.Interfaces;
+
+interface IDatabaseMaintainer
+{
+  Task BuildSdnTableAsync();
+  Task BuildAddressTableAsync();
+  Task BuiltAliasTableAsync();
+  Task BuildCommentTableAsync();
+}

src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs

@@ -0,0 +1,52 @@
+namespace SanctionsSearch.Worker.Models;
+
+class DatabaseMaintainer(
+  IUnitOfWork unitOfWork,
+  IOfacFileService ofacFileService,
+  ILogger<DatabaseMaintainer> logger
+) : IDatabaseMaintainer
+{
+  private readonly IUnitOfWork _unitOfWork = unitOfWork;
+  private readonly IOfacFileService _ofacFileService = ofacFileService;
+  private readonly ILogger<DatabaseMaintainer> _logger = logger;
+
+  public async Task BuildSdnTableAsync()
+  {
+    var result = await _ofacFileService.GetSdnFileAsync();
+
+    if (result.IsFailed)
+    {
+      _logger.LogError("Failed to get SDN file from OFAC.");
+      return;
+    }
+
+    using var stream = result.Value;
+    using var reader = new StreamReader(stream);
+    var config = new CsvConfiguration(CultureInfo.InvariantCulture) { HasHeaderRecord = false };
+    using var csv = new CsvReader(reader, config);
+    csv.Context.RegisterClassMap<SdnMap>();
+    var records = csv.GetRecords<Sdn>();
+
+    foreach (var record in records)
+    {
+      await _unitOfWork.Sdns.Upsert(record);
+    }
+
+    await _unitOfWork.SaveChangesAsync();
+  }
+
+  public Task BuildAddressTableAsync()
+  {
+    throw new NotImplementedException();
+  }
+
+  public Task BuildCommentTableAsync()
+  {
+    throw new NotImplementedException();
+  }
+
+  public Task BuiltAliasTableAsync()
+  {
+    throw new NotImplementedException();
+  }
+}

src/SanctionsSearch.Worker/Models/NullCharacterConverter.cs

@@ -0,0 +1,15 @@
+using CsvHelper.TypeConversion;
+
+namespace SanctionsSearch.Worker.Models;
+
+class NullCharacterConverter : DefaultTypeConverter
+{
+  private const string NullCharacter = "-0-";
+
+  public override object ConvertFromString(string? text, IReaderRow row, MemberMapData memberMapData)
+  {
+    return text is null || text.Trim() is NullCharacter
+      ? string.Empty
+      : text;
+  }
+}

src/SanctionsSearch.Worker/Models/SdnMap.cs

@@ -0,0 +1,20 @@
+namespace SanctionsSearch.Worker.Models;
+
+class SdnMap : ClassMap<Sdn>
+{
+  public SdnMap()
+  {
+    Map(m => m.Id).Index(0);
+    Map(m => m.Name).Index(1).TypeConverter<NullCharacterConverter>();
+    Map(m => m.Type).Index(2).TypeConverter<NullCharacterConverter>();
+    Map(m => m.Program).Index(3).TypeConverter<NullCharacterConverter>();
+    Map(m => m.Title).Index(4).TypeConverter<NullCharacterConverter>();
+    Map(m => m.CallSign).Index(5).TypeConverter<NullCharacterConverter>();
+    Map(m => m.VesselType).Index(6).TypeConverter<NullCharacterConverter>();
+    Map(m => m.Tonnage).Index(7).TypeConverter<NullCharacterConverter>();
+    Map(m => m.GrossRegisteredTonnage).Index(8).TypeConverter<NullCharacterConverter>();
+    Map(m => m.VesselFlag).Index(9).TypeConverter<NullCharacterConverter>();
+    Map(m => m.VesselOwner).Index(10).TypeConverter<NullCharacterConverter>();
+    Map(m => m.Remarks).Index(11).TypeConverter<NullCharacterConverter>();
+  }
+}

src/SanctionsSearch.Worker/Program.cs

@@ -59,6 +59,8 @@ class Program
     builder.Logging.AddSerilog();
 
     builder.Services.ConfigureOptions<OfacFileServiceOptionsSetup>();
+    builder.Services.AddScoped(rs => rs.GetRequiredService<IOptionsSnapshot<OfacFileServiceOptions>>().Value);
+
     builder.Services.ConfigureOptions<DbOptionsSetup>();
     builder.Services.AddScoped(rs => rs.GetRequiredService<IOptionsSnapshot<DbOptions>>().Value);
 

src/SanctionsSearch.Worker/Services/OfacFileService.cs

@@ -3,12 +3,12 @@ namespace SanctionsSearch.Worker.Services;
 class OfacFileService(
   HttpClient client,
   ILogger<OfacFileService> logger,
-  IOptionsSnapshot<OfacFileServiceOptions> options
+  OfacFileServiceOptions options
 ) : IOfacFileService
 {
   private readonly HttpClient _client = client ?? throw new ArgumentNullException(nameof(client));
   private readonly ILogger<OfacFileService> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
-  private readonly OfacFileServiceOptions _options = options?.Value ?? throw new ArgumentNullException(nameof(options));
+  private readonly OfacFileServiceOptions _options = options ?? throw new ArgumentNullException(nameof(options));
   private async Task<Result<Stream>> GetFileAsync(Uri fileUri)
   {
     try
@@ -19,7 +19,7 @@ class OfacFileService(
 
       if (response.IsSuccessStatusCode is false)
       {
-        _logger.LogError("Failed to download file from {FileUri}", fileUri);
+        _logger.LogError("Failed to download file from {FileUri} with Status Code: {StatusCode}", fileUri, response.StatusCode);
         return Result.Fail($"Failed to download file from {fileUri}");
       }
 

src/SanctionsSearch.Worker/Usings.cs

@@ -17,3 +17,7 @@ global using SanctionsSearch.Worker.Setup;
 global using Serilog;
 global using Serilog.Exceptions;
 global using Serilog.Formatting.Compact;
+
+global using System.Globalization;
+global using CsvHelper;
+global using CsvHelper.Configuration;