diff --git a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs index ce0e88a..5b31c29 100644 --- a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs +++ b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs @@ -9,14 +9,24 @@ class DatabaseMaintainer( private readonly IUnitOfWork _unitOfWork = unitOfWork; private readonly IOfacFileService _ofacFileService = ofacFileService; private readonly ILogger _logger = logger; - private readonly CsvConfiguration _csvConfig = new(CultureInfo.InvariantCulture) { HasHeaderRecord = false }; + private readonly List _csvReaders = []; private readonly List _streamReaders = []; + private bool HandleReadingException(ReadingExceptionOccurredArgs args) + { + _logger.LogError(args.Exception, "Error reading CSV record at row {Row}", args.Exception.Context?.Reader?.Parser.Row); + return false; + } + private IEnumerable GetRecordsFromStream(Stream stream) { var reader = new StreamReader(stream); - var csv = new CsvReader(reader, _csvConfig); + var csv = new CsvReader(reader, new CsvConfiguration(CultureInfo.InvariantCulture) + { + HasHeaderRecord = false, + ReadingExceptionOccurred = HandleReadingException + }); csv.Context.RegisterClassMap(); csv.Context.RegisterClassMap(); @@ -31,6 +41,8 @@ class DatabaseMaintainer( public async Task BuildSdnTableAsync() { + _logger.LogInformation("Building SDN table"); + var result = await _ofacFileService.GetSdnFileAsync(); if (result.IsFailed) @@ -44,14 +56,19 @@ class DatabaseMaintainer( foreach (var record in records) { + _logger.LogDebug("Upserting SDN record with ID {Id}", record.Id); await _unitOfWork.Sdns.Upsert(record); } await _unitOfWork.SaveChangesAsync(); + + _logger.LogInformation("SDN table built"); } public async Task BuildAddressTableAsync() { + _logger.LogInformation("Building Address table"); + var result = await _ofacFileService.GetAddressFileAsync(); if (result.IsFailed) @@ -73,14 +90,19 @@ class DatabaseMaintainer( continue; } + _logger.LogDebug("Upserting Address record with ID {Id}", record.Id); await _unitOfWork.Addresses.Upsert(record); } await _unitOfWork.SaveChangesAsync(); + + _logger.LogInformation("Address table built"); } public async Task BuiltAliasTableAsync() { + _logger.LogInformation("Building Alias table"); + var result = await _ofacFileService.GetAltNamesFileAsync(); if (result.IsFailed) @@ -102,14 +124,19 @@ class DatabaseMaintainer( continue; } + _logger.LogDebug("Upserting Alias record with ID {Id}", record.Id); await _unitOfWork.Aliases.Upsert(record); } await _unitOfWork.SaveChangesAsync(); + + _logger.LogInformation("Alias table built"); } public async Task BuildCommentTableAsync() { + _logger.LogInformation("Building Comment table"); + var result = await _ofacFileService.GetCommentsFileAsync(); if (result.IsFailed) @@ -131,10 +158,13 @@ class DatabaseMaintainer( continue; } + _logger.LogDebug("Upserting Comment record with ID {Id}", record.Id); await _unitOfWork.Comments.Upsert(record); } await _unitOfWork.SaveChangesAsync(); + + _logger.LogInformation("Comment table built"); } public void Dispose() diff --git a/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs b/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs index d02c857..0603caf 100644 --- a/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs +++ b/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs @@ -1,6 +1,6 @@ namespace SanctionsSearch.Worker.Persistence; -class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWork, IAsyncDisposable +class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWork, IAsyncDisposable, IDisposable { private readonly DbContext _context = context; private readonly ILogger _logger = loggerFactory.CreateLogger(); @@ -32,4 +32,16 @@ class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWor _logger.LogError(ex, "Failed to dispose of the database context."); } } + + public void Dispose() + { + try + { + _context.Dispose(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to dispose of the database context."); + } + } } \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Program.cs b/src/SanctionsSearch.Worker/Program.cs index 7a1740d..3883e8e 100644 --- a/src/SanctionsSearch.Worker/Program.cs +++ b/src/SanctionsSearch.Worker/Program.cs @@ -14,7 +14,7 @@ class Program .Enrich.WithThreadId() .Enrich.WithExceptionDetails() .Enrich.FromLogContext() - .MinimumLevel.Debug() + .MinimumLevel.Information() .WriteTo.Console() .WriteTo.File(new CompactJsonFormatter(), "logs/log.json", rollingInterval: RollingInterval.Day) .CreateLogger(); @@ -65,7 +65,10 @@ class Program builder.Services.AddScoped(rs => rs.GetRequiredService>().Value); builder.Services.AddSingleton(TimeProvider.System); - // builder.Services.AddScoped(); + + builder.Services + .AddHttpClient() + .AddStandardResilienceHandler(); builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -74,7 +77,8 @@ class Program builder.Services.AddScoped(); builder.Services.AddDbContext(); - builder.Services.AddHostedService(); + builder.Services.AddScoped(); + builder.Services.AddHostedService(); return builder; } diff --git a/src/SanctionsSearch.Worker/SanctionsSearch.Worker.csproj b/src/SanctionsSearch.Worker/SanctionsSearch.Worker.csproj index 9143c2f..2e18cbb 100644 --- a/src/SanctionsSearch.Worker/SanctionsSearch.Worker.csproj +++ b/src/SanctionsSearch.Worker/SanctionsSearch.Worker.csproj @@ -16,6 +16,8 @@ + + diff --git a/src/SanctionsSearch.Worker/Services/OfacFileService.cs b/src/SanctionsSearch.Worker/Services/OfacFileService.cs index 6205c37..aaff995 100644 --- a/src/SanctionsSearch.Worker/Services/OfacFileService.cs +++ b/src/SanctionsSearch.Worker/Services/OfacFileService.cs @@ -6,6 +6,7 @@ class OfacFileService( OfacFileServiceOptions options ) : IOfacFileService { + private const string UserAgent = "SanctionsSearch.Worker"; private readonly HttpClient _client = client ?? throw new ArgumentNullException(nameof(client)); private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); private readonly OfacFileServiceOptions _options = options ?? throw new ArgumentNullException(nameof(options)); @@ -15,6 +16,8 @@ class OfacFileService( { _logger.LogInformation("Downloading file from {FileUri}", fileUri); + _client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + var response = await _client.GetAsync(fileUri); if (response.IsSuccessStatusCode is false) diff --git a/src/SanctionsSearch.Worker/Usings.cs b/src/SanctionsSearch.Worker/Usings.cs index 3bcd6d2..b600fcc 100644 --- a/src/SanctionsSearch.Worker/Usings.cs +++ b/src/SanctionsSearch.Worker/Usings.cs @@ -13,6 +13,7 @@ global using SanctionsSearch.Worker.Options; global using SanctionsSearch.Worker.Persistence; global using SanctionsSearch.Worker.Services; global using SanctionsSearch.Worker.Setup; +global using SanctionsSearch.Worker.Workers; global using Serilog; global using Serilog.Exceptions; diff --git a/src/SanctionsSearch.Worker/Worker.cs b/src/SanctionsSearch.Worker/Worker.cs deleted file mode 100644 index 29470e6..0000000 --- a/src/SanctionsSearch.Worker/Worker.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace SanctionsSearch.Worker; - -public class Worker : BackgroundService -{ - private readonly ILogger _logger; - - public Worker(ILogger logger) - { - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - while (!stoppingToken.IsCancellationRequested) - { - if (_logger.IsEnabled(LogLevel.Information)) - { - _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); - } - await Task.Delay(1000, stoppingToken); - } - } -} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Workers/DatabaseWorker.cs b/src/SanctionsSearch.Worker/Workers/DatabaseWorker.cs new file mode 100644 index 0000000..79a5d0e --- /dev/null +++ b/src/SanctionsSearch.Worker/Workers/DatabaseWorker.cs @@ -0,0 +1,63 @@ +namespace SanctionsSearch.Worker.Workers; + +public class DatabaseWorker( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory +) : IHostedService, IDisposable +{ + private readonly TimeProvider _timeProvider = timeProvider; + private readonly ILogger _logger = logger; + private readonly IServiceScopeFactory _serviceScopeFactory = serviceScopeFactory; + private ITimer? _timer; + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Database worker started"); + + await UpdateDatabase(); + + _timer = _timeProvider.CreateTimer( + callback: async _ => await UpdateDatabase(), + state: null, + dueTime: TimeSpan.FromHours(1), + period: TimeSpan.FromHours(1) + ); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Database worker stopped"); + + _timer?.Change(Timeout.InfiniteTimeSpan, TimeSpan.Zero); + + return Task.CompletedTask; + } + + public void Dispose() + { + _timer?.Dispose(); + } + + private async Task UpdateDatabase() + { + _logger.LogInformation("Updating database"); + + try + { + using var scope = _serviceScopeFactory.CreateAsyncScope(); + var databaseMaintainer = scope.ServiceProvider.GetRequiredService(); + + await databaseMaintainer.BuildSdnTableAsync(); + await databaseMaintainer.BuildAddressTableAsync(); + await databaseMaintainer.BuiltAliasTableAsync(); + await databaseMaintainer.BuildCommentTableAsync(); + + _logger.LogInformation("Database updated"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error updating database"); + } + } +} \ No newline at end of file