diff --git a/src/SanctionsSearch.Worker.Tests/Faker/OfacFileServiceOptionsFaker.cs b/src/SanctionsSearch.Worker.Tests/Faker/OfacFileServiceOptionsFaker.cs new file mode 100644 index 0000000..8c53832 --- /dev/null +++ b/src/SanctionsSearch.Worker.Tests/Faker/OfacFileServiceOptionsFaker.cs @@ -0,0 +1,17 @@ +namespace SanctionsSearch.Worker.Tests.Faker; + +class OfacFileServiceOptionsFaker : Faker +{ + public OfacFileServiceOptionsFaker() + { + RuleFor(x => x.Url, f => f.Internet.Url()); + RuleFor(x => x.SdnFileName, f => f.System.FileName("csv")); + RuleFor(x => x.AddressFileName, f => f.System.FileName("csv")); + RuleFor(x => x.AltNamesFileName, f => f.System.FileName("csv")); + RuleFor(x => x.CommentsFileName, f => f.System.FileName("csv")); + RuleFor(x => x.ConPrimaryNameFileName, f => f.System.FileName("csv")); + RuleFor(x => x.ConAddressesFileName, f => f.System.FileName("csv")); + RuleFor(x => x.ConAltNamesFileName, f => f.System.FileName("csv")); + RuleFor(x => x.ConCommentsFileName, f => f.System.FileName("csv")); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs new file mode 100644 index 0000000..ba5f283 --- /dev/null +++ b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs @@ -0,0 +1,86 @@ +namespace SanctionsSearch.Worker.Tests.Integration; + +public class DatabaseMaintainerTests : DatabaseTest +{ + private readonly MockHttpMessageHandler _mockHttp = new(); + private readonly OfacFileServiceOptionsFaker _ofacFileServiceOptionsFaker = new(); + private readonly OfacFileServiceOptions _ofacFileServiceOptions; + private readonly DatabaseMaintainer _databaseMaintainer; + private static MemoryStream CreateCsvStream(string csv) => new(Encoding.UTF8.GetBytes(csv)); + + public DatabaseMaintainerTests() + { + _ofacFileServiceOptions = _ofacFileServiceOptionsFaker.Generate(); + + var ofacFileService = new OfacFileService( + _mockHttp.ToHttpClient(), + _loggerFactory.CreateLogger(), + _ofacFileServiceOptions + ); + + var uow = new EfUnitOfWork(_context, _loggerFactory); + var logger = _loggerFactory.CreateLogger(); + _databaseMaintainer = new DatabaseMaintainer(uow, ofacFileService, logger); + } + + [Fact] + public async Task BuildSdnTableAsync_WhenCalled_ItShouldAddSdnCsvRecordsToDatabase() + { + var testCsv = """ + 6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male." + 6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt." + 6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt." + """; + + var testStream = CreateCsvStream(testCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetSdnFileUri().ToString()) + .Respond("text/csv", testStream); + + var now = DateTimeOffset.UtcNow; + + _timeProviderMock + .Setup(x => x.GetUtcNow()) + .Returns(now); + + await _databaseMaintainer.BuildSdnTableAsync(); + + var sdns = await _context.Set().ToListAsync(); + + sdns.Should().HaveCount(3); + + sdns[0].Should().BeEquivalentTo(new Sdn() + { + Id = 6906, + Name = "AL-IRAQI, Abd al-Hadi", + Type = "individual", + Program = "SDGT", + Remarks = "DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male.", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime + }); + + sdns[1].Should().BeEquivalentTo(new Sdn() + { + Id = 6907, + Name = "SHIHATA, Thirwat Salah", + Type = "individual", + Program = "SDGT", + Remarks = "DOB 29 Jun 1960; POB Egypt.", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime + }); + + sdns[2].Should().BeEquivalentTo(new Sdn() + { + Id = 6908, + Name = "AHMAD, Tariq Anwar al-Sayyid", + Type = "individual", + Program = "SDGT", + Remarks = "DOB 15 Mar 1963; POB Alexandria, Egypt.", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime + }); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker.Tests/Unit/OfacFileServiceTests.cs b/src/SanctionsSearch.Worker.Tests/Unit/OfacFileServiceTests.cs index 0166c38..dd24bac 100644 --- a/src/SanctionsSearch.Worker.Tests/Unit/OfacFileServiceTests.cs +++ b/src/SanctionsSearch.Worker.Tests/Unit/OfacFileServiceTests.cs @@ -2,35 +2,22 @@ namespace SanctionsSearch.Worker.Tests.Unit; public class OfacFileServiceTests { - private const string Url = "https://example.com"; private const string TestCsv = "Name,Address\nJohn Doe,123 Main St\n"; private static Stream TestStream => new MemoryStream(Encoding.UTF8.GetBytes(TestCsv)); private readonly MockHttpMessageHandler _mockHttp = new(); private readonly Mock> _mockLogger = new(); - private readonly Mock> _mockOptions = new(); + private readonly OfacFileServiceOptionsFaker _optionsFaker = new(); + private readonly OfacFileServiceOptions _options; private readonly OfacFileService _service; public OfacFileServiceTests() { - _mockOptions - .Setup(x => x.Value) - .Returns(new OfacFileServiceOptions - { - Url = Url, - SdnFileName = "sdn.csv", - AddressFileName = "address.csv", - AltNamesFileName = "alt_names.csv", - CommentsFileName = "comments.csv", - ConPrimaryNameFileName = "con_primary_name.csv", - ConAddressesFileName = "con_addresses.csv", - ConAltNamesFileName = "con_alt_names.csv", - ConCommentsFileName = "con_comments.csv" - }); + _options = _optionsFaker.Generate(); _service = new OfacFileService( _mockHttp.ToHttpClient(), _mockLogger.Object, - _mockOptions.Object + _options ); } @@ -46,7 +33,7 @@ public class OfacFileServiceTests var act = () => new OfacFileService( null!, _mockLogger.Object, - _mockOptions.Object + _options ); act.Should().Throw(); @@ -58,7 +45,7 @@ public class OfacFileServiceTests var act = () => new OfacFileService( _mockHttp.ToHttpClient(), null!, - _mockOptions.Object + _options ); act.Should().Throw(); @@ -79,12 +66,10 @@ public class OfacFileServiceTests [Fact] public void Constructor_WhenCalledWithNullOptionsValue_ThrowsArgumentNullException() { - _mockOptions.Setup(x => x.Value).Returns(() => null!); - var act = () => new OfacFileService( _mockHttp.ToHttpClient(), _mockLogger.Object, - _mockOptions.Object + null! ); act.Should().Throw(); @@ -94,7 +79,7 @@ public class OfacFileServiceTests public async Task GetSdnFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/sdn.csv") + .When(_options.GetSdnFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetSdnFileAsync(); @@ -107,7 +92,7 @@ public class OfacFileServiceTests public async Task GetSdnFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/sdn.csv") + .When(_options.GetSdnFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetSdnFileAsync(); @@ -119,7 +104,7 @@ public class OfacFileServiceTests public async Task GetSdnFileAsync_WhenExceptionThrown_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/sdn.csv") + .When(_options.GetSdnFileUri().ToString()) .Throw(new Exception("Test exception")); var result = await _service.GetSdnFileAsync(); @@ -131,7 +116,7 @@ public class OfacFileServiceTests public async Task GetAddressFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/address.csv") + .When(_options.GetAddressFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetAddressFileAsync(); @@ -144,7 +129,7 @@ public class OfacFileServiceTests public async Task GetAddressFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/address.csv") + .When(_options.GetAddressFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetAddressFileAsync(); @@ -156,7 +141,7 @@ public class OfacFileServiceTests public async Task GetAltNamesFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/alt_names.csv") + .When(_options.GetAltNamesFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetAltNamesFileAsync(); @@ -169,7 +154,7 @@ public class OfacFileServiceTests public async Task GetAltNamesFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/alt_names.csv") + .When(_options.GetAltNamesFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetAltNamesFileAsync(); @@ -181,7 +166,7 @@ public class OfacFileServiceTests public async Task GetCommentsFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/comments.csv") + .When(_options.GetCommentsFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetCommentsFileAsync(); @@ -194,7 +179,7 @@ public class OfacFileServiceTests public async Task GetCommentsFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/comments.csv") + .When(_options.GetCommentsFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetCommentsFileAsync(); @@ -206,7 +191,7 @@ public class OfacFileServiceTests public async Task GetConPrimaryNameFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/con_primary_name.csv") + .When(_options.GetConPrimaryNameFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetConPrimaryNameFileAsync(); @@ -219,7 +204,7 @@ public class OfacFileServiceTests public async Task GetConPrimaryNameFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/con_primary_name.csv") + .When(_options.GetConPrimaryNameFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetConPrimaryNameFileAsync(); @@ -231,7 +216,7 @@ public class OfacFileServiceTests public async Task GetConAddressesFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/con_addresses.csv") + .When(_options.GetConAddressesFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetConAddressesFileAsync(); @@ -244,7 +229,7 @@ public class OfacFileServiceTests public async Task GetConAddressesFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/con_addresses.csv") + .When(_options.GetConAddressesFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetConAddressesFileAsync(); @@ -256,7 +241,7 @@ public class OfacFileServiceTests public async Task GetConAltNamesFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/con_alt_names.csv") + .When(_options.GetConAltNamesFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetConAltNamesFileAsync(); @@ -269,7 +254,7 @@ public class OfacFileServiceTests public async Task GetConAltNamesFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/con_alt_names.csv") + .When(_options.GetConAltNamesFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetConAltNamesFileAsync(); @@ -281,7 +266,7 @@ public class OfacFileServiceTests public async Task GetConCommentsFileAsync_WhenCalled_ItShouldReturnStream() { _mockHttp - .When("https://example.com/con_comments.csv") + .When(_options.GetConCommentsFileUri().ToString()) .Respond("text/csv", TestStream); var result = await _service.GetConCommentsFileAsync(); @@ -294,7 +279,7 @@ public class OfacFileServiceTests public async Task GetConCommentsFileAsync_WhenRequestFails_ItShouldReturnFailure() { _mockHttp - .When("https://example.com/con_comments.csv") + .When(_options.GetConCommentsFileUri().ToString()) .Respond(HttpStatusCode.InternalServerError); var result = await _service.GetConCommentsFileAsync(); diff --git a/src/SanctionsSearch.Worker/Interfaces/IDatabaseMaintainer.cs b/src/SanctionsSearch.Worker/Interfaces/IDatabaseMaintainer.cs new file mode 100644 index 0000000..1b031cf --- /dev/null +++ b/src/SanctionsSearch.Worker/Interfaces/IDatabaseMaintainer.cs @@ -0,0 +1,9 @@ +namespace SanctionsSearch.Worker.Interfaces; + +interface IDatabaseMaintainer +{ + Task BuildSdnTableAsync(); + Task BuildAddressTableAsync(); + Task BuiltAliasTableAsync(); + Task BuildCommentTableAsync(); +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs new file mode 100644 index 0000000..d8931a7 --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs @@ -0,0 +1,52 @@ +namespace SanctionsSearch.Worker.Models; + +class DatabaseMaintainer( + IUnitOfWork unitOfWork, + IOfacFileService ofacFileService, + ILogger logger +) : IDatabaseMaintainer +{ + private readonly IUnitOfWork _unitOfWork = unitOfWork; + private readonly IOfacFileService _ofacFileService = ofacFileService; + private readonly ILogger _logger = logger; + + public async Task BuildSdnTableAsync() + { + var result = await _ofacFileService.GetSdnFileAsync(); + + if (result.IsFailed) + { + _logger.LogError("Failed to get SDN file from OFAC."); + return; + } + + using var stream = result.Value; + using var reader = new StreamReader(stream); + var config = new CsvConfiguration(CultureInfo.InvariantCulture) { HasHeaderRecord = false }; + using var csv = new CsvReader(reader, config); + csv.Context.RegisterClassMap(); + var records = csv.GetRecords(); + + foreach (var record in records) + { + await _unitOfWork.Sdns.Upsert(record); + } + + await _unitOfWork.SaveChangesAsync(); + } + + public Task BuildAddressTableAsync() + { + throw new NotImplementedException(); + } + + public Task BuildCommentTableAsync() + { + throw new NotImplementedException(); + } + + public Task BuiltAliasTableAsync() + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/NullCharacterConverter.cs b/src/SanctionsSearch.Worker/Models/NullCharacterConverter.cs new file mode 100644 index 0000000..47da7da --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/NullCharacterConverter.cs @@ -0,0 +1,15 @@ +using CsvHelper.TypeConversion; + +namespace SanctionsSearch.Worker.Models; + +class NullCharacterConverter : DefaultTypeConverter +{ + private const string NullCharacter = "-0-"; + + public override object ConvertFromString(string? text, IReaderRow row, MemberMapData memberMapData) + { + return text is null || text.Trim() is NullCharacter + ? string.Empty + : text; + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/SdnMap.cs b/src/SanctionsSearch.Worker/Models/SdnMap.cs new file mode 100644 index 0000000..6c5cbf5 --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/SdnMap.cs @@ -0,0 +1,20 @@ +namespace SanctionsSearch.Worker.Models; + +class SdnMap : ClassMap +{ + public SdnMap() + { + Map(m => m.Id).Index(0); + Map(m => m.Name).Index(1).TypeConverter(); + Map(m => m.Type).Index(2).TypeConverter(); + Map(m => m.Program).Index(3).TypeConverter(); + Map(m => m.Title).Index(4).TypeConverter(); + Map(m => m.CallSign).Index(5).TypeConverter(); + Map(m => m.VesselType).Index(6).TypeConverter(); + Map(m => m.Tonnage).Index(7).TypeConverter(); + Map(m => m.GrossRegisteredTonnage).Index(8).TypeConverter(); + Map(m => m.VesselFlag).Index(9).TypeConverter(); + Map(m => m.VesselOwner).Index(10).TypeConverter(); + Map(m => m.Remarks).Index(11).TypeConverter(); + } +} diff --git a/src/SanctionsSearch.Worker/Program.cs b/src/SanctionsSearch.Worker/Program.cs index 6de4e60..7a1740d 100644 --- a/src/SanctionsSearch.Worker/Program.cs +++ b/src/SanctionsSearch.Worker/Program.cs @@ -59,6 +59,8 @@ class Program builder.Logging.AddSerilog(); builder.Services.ConfigureOptions(); + builder.Services.AddScoped(rs => rs.GetRequiredService>().Value); + builder.Services.ConfigureOptions(); builder.Services.AddScoped(rs => rs.GetRequiredService>().Value); diff --git a/src/SanctionsSearch.Worker/Services/OfacFileService.cs b/src/SanctionsSearch.Worker/Services/OfacFileService.cs index 7d51402..6205c37 100644 --- a/src/SanctionsSearch.Worker/Services/OfacFileService.cs +++ b/src/SanctionsSearch.Worker/Services/OfacFileService.cs @@ -3,12 +3,12 @@ namespace SanctionsSearch.Worker.Services; class OfacFileService( HttpClient client, ILogger logger, - IOptionsSnapshot options + OfacFileServiceOptions options ) : IOfacFileService { private readonly HttpClient _client = client ?? throw new ArgumentNullException(nameof(client)); private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - private readonly OfacFileServiceOptions _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + private readonly OfacFileServiceOptions _options = options ?? throw new ArgumentNullException(nameof(options)); private async Task> GetFileAsync(Uri fileUri) { try @@ -19,7 +19,7 @@ class OfacFileService( if (response.IsSuccessStatusCode is false) { - _logger.LogError("Failed to download file from {FileUri}", fileUri); + _logger.LogError("Failed to download file from {FileUri} with Status Code: {StatusCode}", fileUri, response.StatusCode); return Result.Fail($"Failed to download file from {fileUri}"); } diff --git a/src/SanctionsSearch.Worker/Usings.cs b/src/SanctionsSearch.Worker/Usings.cs index 400fb68..3bcd6d2 100644 --- a/src/SanctionsSearch.Worker/Usings.cs +++ b/src/SanctionsSearch.Worker/Usings.cs @@ -17,3 +17,7 @@ global using SanctionsSearch.Worker.Setup; global using Serilog; global using Serilog.Exceptions; global using Serilog.Formatting.Compact; + +global using System.Globalization; +global using CsvHelper; +global using CsvHelper.Configuration; \ No newline at end of file