feat: add database worker with timer for building database at start up and refreshing once every hour.

src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs

@@ -9,14 +9,24 @@ class DatabaseMaintainer(
   private readonly IUnitOfWork _unitOfWork = unitOfWork;
   private readonly IOfacFileService _ofacFileService = ofacFileService;
   private readonly ILogger<DatabaseMaintainer> _logger = logger;
-  private readonly CsvConfiguration _csvConfig = new(CultureInfo.InvariantCulture) { HasHeaderRecord = false };
+
   private readonly List<CsvReader> _csvReaders = [];
   private readonly List<StreamReader> _streamReaders = [];
 
+  private bool HandleReadingException(ReadingExceptionOccurredArgs args)
+  {
+    _logger.LogError(args.Exception, "Error reading CSV record at row {Row}", args.Exception.Context?.Reader?.Parser.Row);
+    return false;
+  }
+
   private IEnumerable<T> GetRecordsFromStream<T>(Stream stream)
   {
     var reader = new StreamReader(stream);
-    var csv = new CsvReader(reader, _csvConfig);
+    var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture)
+    {
+      HasHeaderRecord = false,
+      ReadingExceptionOccurred = HandleReadingException
+    });
 
     csv.Context.RegisterClassMap<SdnMap>();
     csv.Context.RegisterClassMap<AddressMap>();
@@ -31,6 +41,8 @@ class DatabaseMaintainer(
 
   public async Task BuildSdnTableAsync()
   {
+    _logger.LogInformation("Building SDN table");
+
     var result = await _ofacFileService.GetSdnFileAsync();
 
     if (result.IsFailed)
@@ -44,14 +56,19 @@ class DatabaseMaintainer(
 
     foreach (var record in records)
     {
+      _logger.LogDebug("Upserting SDN record with ID {Id}", record.Id);
       await _unitOfWork.Sdns.Upsert(record);
     }
 
     await _unitOfWork.SaveChangesAsync();
+
+    _logger.LogInformation("SDN table built");
   }
 
   public async Task BuildAddressTableAsync()
   {
+    _logger.LogInformation("Building Address table");
+
     var result = await _ofacFileService.GetAddressFileAsync();
 
     if (result.IsFailed)
@@ -73,14 +90,19 @@ class DatabaseMaintainer(
         continue;
       }
 
+      _logger.LogDebug("Upserting Address record with ID {Id}", record.Id);
       await _unitOfWork.Addresses.Upsert(record);
     }
 
     await _unitOfWork.SaveChangesAsync();
+
+    _logger.LogInformation("Address table built");
   }
 
   public async Task BuiltAliasTableAsync()
   {
+    _logger.LogInformation("Building Alias table");
+
     var result = await _ofacFileService.GetAltNamesFileAsync();
 
     if (result.IsFailed)
@@ -102,14 +124,19 @@ class DatabaseMaintainer(
         continue;
       }
 
+      _logger.LogDebug("Upserting Alias record with ID {Id}", record.Id);
       await _unitOfWork.Aliases.Upsert(record);
     }
 
     await _unitOfWork.SaveChangesAsync();
+
+    _logger.LogInformation("Alias table built");
   }
 
   public async Task BuildCommentTableAsync()
   {
+    _logger.LogInformation("Building Comment table");
+
     var result = await _ofacFileService.GetCommentsFileAsync();
 
     if (result.IsFailed)
@@ -131,10 +158,13 @@ class DatabaseMaintainer(
         continue;
       }
 
+      _logger.LogDebug("Upserting Comment record with ID {Id}", record.Id);
       await _unitOfWork.Comments.Upsert(record);
     }
 
     await _unitOfWork.SaveChangesAsync();
+
+    _logger.LogInformation("Comment table built");
   }
 
   public void Dispose()

src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs

@@ -1,6 +1,6 @@
 namespace SanctionsSearch.Worker.Persistence;
 
-class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWork, IAsyncDisposable
+class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWork, IAsyncDisposable, IDisposable
 {
   private readonly DbContext _context = context;
   private readonly ILogger<EfUnitOfWork> _logger = loggerFactory.CreateLogger<EfUnitOfWork>();
@@ -32,4 +32,16 @@ class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWor
       _logger.LogError(ex, "Failed to dispose of the database context.");
     }
   }
+
+  public void Dispose()
+  {
+    try
+    {
+      _context.Dispose();
+    }
+    catch (Exception ex)
+    {
+      _logger.LogError(ex, "Failed to dispose of the database context.");
+    }
+  }
 }

src/SanctionsSearch.Worker/Program.cs

@@ -14,7 +14,7 @@ class Program
       .Enrich.WithThreadId()
       .Enrich.WithExceptionDetails()
       .Enrich.FromLogContext()
-      .MinimumLevel.Debug()
+      .MinimumLevel.Information()
       .WriteTo.Console()
       .WriteTo.File(new CompactJsonFormatter(), "logs/log.json", rollingInterval: RollingInterval.Day)
       .CreateLogger();
@@ -65,7 +65,10 @@ class Program
     builder.Services.AddScoped(rs => rs.GetRequiredService<IOptionsSnapshot<DbOptions>>().Value);
 
     builder.Services.AddSingleton(TimeProvider.System);
-    // builder.Services.AddScoped<IOfacFileService, OfacFileService>();
+
+    builder.Services
+      .AddHttpClient<IOfacFileService, OfacFileService>()
+      .AddStandardResilienceHandler();
 
     builder.Services.AddScoped<ISdnRepository, SdnRepository>();
     builder.Services.AddScoped<IAddressRepository, AddressRepository>();
@@ -74,7 +77,8 @@ class Program
     builder.Services.AddScoped<IUnitOfWork, EfUnitOfWork>();
     builder.Services.AddDbContext<DbContext, AppDbContext>();
 
-    builder.Services.AddHostedService<Worker>();
+    builder.Services.AddScoped<IDatabaseMaintainer, DatabaseMaintainer>();
+    builder.Services.AddHostedService<DatabaseWorker>();
 
     return builder;
   }

src/SanctionsSearch.Worker/SanctionsSearch.Worker.csproj

@@ -16,6 +16,8 @@
     </PackageReference>
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.8.0" />
     <PackageReference Include="Serilog" Version="4.0.1" />
     <PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />
     <PackageReference Include="Serilog.Enrichers.Process" Version="3.0.0" />

src/SanctionsSearch.Worker/Services/OfacFileService.cs

@@ -6,6 +6,7 @@ class OfacFileService(
   OfacFileServiceOptions options
 ) : IOfacFileService
 {
+  private const string UserAgent = "SanctionsSearch.Worker";
   private readonly HttpClient _client = client ?? throw new ArgumentNullException(nameof(client));
   private readonly ILogger<OfacFileService> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
   private readonly OfacFileServiceOptions _options = options ?? throw new ArgumentNullException(nameof(options));
@@ -15,6 +16,8 @@ class OfacFileService(
     {
       _logger.LogInformation("Downloading file from {FileUri}", fileUri);
 
+      _client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
+
       var response = await _client.GetAsync(fileUri);
 
       if (response.IsSuccessStatusCode is false)

src/SanctionsSearch.Worker/Usings.cs

@@ -13,6 +13,7 @@ global using SanctionsSearch.Worker.Options;
 global using SanctionsSearch.Worker.Persistence;
 global using SanctionsSearch.Worker.Services;
 global using SanctionsSearch.Worker.Setup;
+global using SanctionsSearch.Worker.Workers;
 
 global using Serilog;
 global using Serilog.Exceptions;

src/SanctionsSearch.Worker/Worker.cs

@@ -1,23 +0,0 @@
-namespace SanctionsSearch.Worker;
-
-public class Worker : BackgroundService
-{
-  private readonly ILogger<Worker> _logger;
-
-  public Worker(ILogger<Worker> logger)
-  {
-    _logger = logger;
-  }
-
-  protected override async Task ExecuteAsync(CancellationToken stoppingToken)
-  {
-    while (!stoppingToken.IsCancellationRequested)
-    {
-      if (_logger.IsEnabled(LogLevel.Information))
-      {
-        _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
-      }
-      await Task.Delay(1000, stoppingToken);
-    }
-  }
-}

src/SanctionsSearch.Worker/Workers/DatabaseWorker.cs

@@ -0,0 +1,63 @@
+namespace SanctionsSearch.Worker.Workers;
+
+public class DatabaseWorker(
+  ILogger<DatabaseWorker> logger,
+  TimeProvider timeProvider,
+  IServiceScopeFactory serviceScopeFactory
+) : IHostedService, IDisposable
+{
+  private readonly TimeProvider _timeProvider = timeProvider;
+  private readonly ILogger<DatabaseWorker> _logger = logger;
+  private readonly IServiceScopeFactory _serviceScopeFactory = serviceScopeFactory;
+  private ITimer? _timer;
+
+  public async Task StartAsync(CancellationToken cancellationToken)
+  {
+    _logger.LogInformation("Database worker started");
+
+    await UpdateDatabase();
+
+    _timer = _timeProvider.CreateTimer(
+      callback: async _ => await UpdateDatabase(),
+      state: null,
+      dueTime: TimeSpan.FromHours(1),
+      period: TimeSpan.FromHours(1)
+    );
+  }
+
+  public Task StopAsync(CancellationToken cancellationToken)
+  {
+    _logger.LogInformation("Database worker stopped");
+
+    _timer?.Change(Timeout.InfiniteTimeSpan, TimeSpan.Zero);
+
+    return Task.CompletedTask;
+  }
+
+  public void Dispose()
+  {
+    _timer?.Dispose();
+  }
+
+  private async Task UpdateDatabase()
+  {
+    _logger.LogInformation("Updating database");
+
+    try
+    {
+      using var scope = _serviceScopeFactory.CreateAsyncScope();
+      var databaseMaintainer = scope.ServiceProvider.GetRequiredService<IDatabaseMaintainer>();
+
+      await databaseMaintainer.BuildSdnTableAsync();
+      await databaseMaintainer.BuildAddressTableAsync();
+      await databaseMaintainer.BuiltAliasTableAsync();
+      await databaseMaintainer.BuildCommentTableAsync();
+
+      _logger.LogInformation("Database updated");
+    }
+    catch (Exception ex)
+    {
+      _logger.LogError(ex, "Error updating database");
+    }
+  }
+}