From 0b9a83619d7b984651ddf262719fa9ddd70066b9 Mon Sep 17 00:00:00 2001 From: Stevan Freeborn <65925598+StevanFreeborn@users.noreply.github.com> Date: Sat, 24 Aug 2024 16:06:44 -0500 Subject: [PATCH] feat: implement building address table from csv --- .../Integration/DatabaseMaintainerTests.cs | 71 +++++++++++++++++++ .../Models/AddressMap.cs | 14 ++++ .../Models/DatabaseMaintainer.cs | 62 +++++++++++++--- .../Persistence/EfUnitOfWork.cs | 19 ++++- 4 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 src/SanctionsSearch.Worker/Models/AddressMap.cs diff --git a/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs index ba5f283..aab3b51 100644 --- a/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs +++ b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs @@ -2,6 +2,10 @@ namespace SanctionsSearch.Worker.Tests.Integration; public class DatabaseMaintainerTests : DatabaseTest { + private const string TestAddressCsv = """ + 306,201,"Dai-Ichi Bldg. 6th Floor, 10-2 Nihombashi, 2-chome, Chuo-ku","Tokyo 103","Japan",-0- + 306,202,"Federico Boyd Avenue & 51 Street","Panama City","Panama",-0- + """; private readonly MockHttpMessageHandler _mockHttp = new(); private readonly OfacFileServiceOptionsFaker _ofacFileServiceOptionsFaker = new(); private readonly OfacFileServiceOptions _ofacFileServiceOptions; @@ -83,4 +87,71 @@ public class DatabaseMaintainerTests : DatabaseTest UpdatedAt = now.DateTime }); } + + [Fact] + public async Task BuildAddressTableAsync_WhenCalledAndNoSdnRecordFound_ItShouldNotAddAddressCsvRecordsToDatabase() + { + var testStream = CreateCsvStream(TestAddressCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetAddressFileUri().ToString()) + .Respond("text/csv", testStream); + + await _databaseMaintainer.BuildAddressTableAsync(); + + var addresses = await _context.Set
().ToListAsync(); + + addresses.Should().BeEmpty(); + } + + [Fact] + public async Task BuildAddressTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddAddressCsvRecordsToDatabase() + { + var testStream = CreateCsvStream(TestAddressCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetAddressFileUri().ToString()) + .Respond("text/csv", testStream); + + var now = DateTimeOffset.UtcNow; + + _timeProviderMock + .Setup(x => x.GetUtcNow()) + .Returns(now); + + var sdn = new Sdn() { Id = 306 }; + + await _context.Set().AddAsync(sdn); + await _context.SaveChangesAsync(); + + await _databaseMaintainer.BuildAddressTableAsync(); + + var addresses = await _context.Set
().ToListAsync(); + + addresses.Should().BeEquivalentTo(new[] + { + new Address() + { + SdnId = sdn.Id, + Id = 201, + StreetAddress = "Dai-Ichi Bldg. 6th Floor, 10-2 Nihombashi, 2-chome, Chuo-ku", + CityProvincePostal = "Tokyo 103", + Country = "Japan", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime, + Sdn = sdn + }, + new Address() + { + SdnId = sdn.Id, + Id = 202, + StreetAddress = "Federico Boyd Avenue & 51 Street", + CityProvincePostal = "Panama City", + Country = "Panama", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime, + Sdn = sdn + } + }); + } } \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/AddressMap.cs b/src/SanctionsSearch.Worker/Models/AddressMap.cs new file mode 100644 index 0000000..9010c67 --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/AddressMap.cs @@ -0,0 +1,14 @@ +namespace SanctionsSearch.Worker.Models; + +class AddressMap : ClassMap
+{ + public AddressMap() + { + Map(m => m.SdnId).Index(0); + Map(m => m.Id).Index(1); + Map(m => m.StreetAddress).Index(2).TypeConverter(); + Map(m => m.CityProvincePostal).Index(3).TypeConverter(); + Map(m => m.Country).Index(4).TypeConverter(); + Map(m => m.Remarks).Index(5).TypeConverter(); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs index a3a087b..a5786f6 100644 --- a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs +++ b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs @@ -4,11 +4,28 @@ class DatabaseMaintainer( IUnitOfWork unitOfWork, IOfacFileService ofacFileService, ILogger logger -) : IDatabaseMaintainer +) : IDatabaseMaintainer, IDisposable { private readonly IUnitOfWork _unitOfWork = unitOfWork; private readonly IOfacFileService _ofacFileService = ofacFileService; private readonly ILogger _logger = logger; + private readonly CsvConfiguration _csvConfig = new(CultureInfo.InvariantCulture) { HasHeaderRecord = false }; + private readonly List _csvReaders = []; + private readonly List _streamReaders = []; + + private IEnumerable GetRecordsFromStream(Stream stream) + { + var reader = new StreamReader(stream); + var csv = new CsvReader(reader, _csvConfig); + + csv.Context.RegisterClassMap(); + csv.Context.RegisterClassMap(); + + _streamReaders.Add(reader); + _csvReaders.Add(csv); + + return csv.GetRecords(); + } public async Task BuildSdnTableAsync() { @@ -19,14 +36,9 @@ class DatabaseMaintainer( _logger.LogError("Failed to get SDN file from OFAC."); return; } - using var stream = result.Value; - // TODO: Abstract this logic to wrapper around CsvHelper - using var reader = new StreamReader(stream); - var config = new CsvConfiguration(CultureInfo.InvariantCulture) { HasHeaderRecord = false }; - using var csv = new CsvReader(reader, config); - csv.Context.RegisterClassMap(); - var records = csv.GetRecords(); + using var stream = result.Value; + var records = GetRecordsFromStream(stream); foreach (var record in records) { @@ -36,9 +48,33 @@ class DatabaseMaintainer( await _unitOfWork.SaveChangesAsync(); } - public Task BuildAddressTableAsync() + public async Task BuildAddressTableAsync() { - throw new NotImplementedException(); + var result = await _ofacFileService.GetAddressFileAsync(); + + if (result.IsFailed) + { + _logger.LogError("Failed to get Address file from OFAC."); + return; + } + + using var stream = result.Value; + var records = GetRecordsFromStream
(stream); + + foreach (var record in records) + { + var sdn = await _unitOfWork.Sdns.Find(s => s.Id == record.SdnId); + + if (sdn.Count() is 0) + { + _logger.LogWarning("Address's SDN with ID {Id} not found. Skipping address.", record.SdnId); + continue; + } + + await _unitOfWork.Addresses.Upsert(record); + } + + await _unitOfWork.SaveChangesAsync(); } public Task BuildCommentTableAsync() @@ -50,4 +86,10 @@ class DatabaseMaintainer( { throw new NotImplementedException(); } + + public void Dispose() + { + _csvReaders.ForEach(csv => csv.Dispose()); + _streamReaders.ForEach(reader => reader.Dispose()); + } } \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs b/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs index 65e632c..d02c857 100644 --- a/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs +++ b/src/SanctionsSearch.Worker/Persistence/EfUnitOfWork.cs @@ -3,6 +3,7 @@ namespace SanctionsSearch.Worker.Persistence; class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWork, IAsyncDisposable { private readonly DbContext _context = context; + private readonly ILogger _logger = loggerFactory.CreateLogger(); public ISdnRepository Sdns { get; } = new SdnRepository(context, loggerFactory.CreateLogger()); public IAddressRepository Addresses { get; } = new AddressRepository(context, loggerFactory.CreateLogger()); public IAliasRepository Aliases { get; } = new AliasRepository(context, loggerFactory.CreateLogger()); @@ -10,11 +11,25 @@ class EfUnitOfWork(DbContext context, ILoggerFactory loggerFactory) : IUnitOfWor public async Task SaveChangesAsync() { - await _context.SaveChangesAsync(); + try + { + await _context.SaveChangesAsync(); + } + catch (DbUpdateException ex) + { + _logger.LogError(ex, "Failed to save changes to the database."); + } } public async ValueTask DisposeAsync() { - await _context.DisposeAsync(); + try + { + await _context.DisposeAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to dispose of the database context."); + } } } \ No newline at end of file