From 14e35670a5d87edb1ca280b8129016cf64665eb8 Mon Sep 17 00:00:00 2001 From: Stevan Freeborn <65925598+StevanFreeborn@users.noreply.github.com> Date: Sun, 25 Aug 2024 11:48:10 -0500 Subject: [PATCH] feat: implement building alias and comment tables --- .../Integration/DatabaseMaintainerTests.cs | 143 ++++++++++++++++-- src/SanctionsSearch.Worker/Models/AliasMap.cs | 13 ++ .../Models/CommentMap.cs | 10 ++ .../Models/DatabaseMaintainer.cs | 58 ++++++- 4 files changed, 211 insertions(+), 13 deletions(-) create mode 100644 src/SanctionsSearch.Worker/Models/AliasMap.cs create mode 100644 src/SanctionsSearch.Worker/Models/CommentMap.cs diff --git a/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs index aab3b51..a1f653a 100644 --- a/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs +++ b/src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs @@ -2,10 +2,24 @@ namespace SanctionsSearch.Worker.Tests.Integration; public class DatabaseMaintainerTests : DatabaseTest { + private const string TestSdnCsv = """ + 6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male." + 6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt." + 6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt." + """; private const string TestAddressCsv = """ 306,201,"Dai-Ichi Bldg. 6th Floor, 10-2 Nihombashi, 2-chome, Chuo-ku","Tokyo 103","Japan",-0- 306,202,"Federico Boyd Avenue & 51 Street","Panama City","Panama",-0- """; + private const string TestAliasCsv = """ + 555,477,"aka","COPROVA",-0- + 555,478,"aka","COPROVA SARL",-0- + """; + private const string TestCommentCsv = """ + 27307,"G TEAM'; a.k.a. 'RED DOT'; a.k.a. 'TEMP.HERMIT'; a.k.a. 'GROUP 77'; a.k.a. 'ZINC'; a.k.a. 'APT-C-26'; a.k.a. 'APPLEWORM'." + 28263,"hn'; Linked To: LAZARUS GROUP." + """; + private readonly MockHttpMessageHandler _mockHttp = new(); private readonly OfacFileServiceOptionsFaker _ofacFileServiceOptionsFaker = new(); private readonly OfacFileServiceOptions _ofacFileServiceOptions; @@ -30,13 +44,7 @@ public class DatabaseMaintainerTests : DatabaseTest [Fact] public async Task BuildSdnTableAsync_WhenCalled_ItShouldAddSdnCsvRecordsToDatabase() { - var testCsv = """ - 6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male." - 6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt." - 6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt." - """; - - var testStream = CreateCsvStream(testCsv); + using var testStream = CreateCsvStream(TestSdnCsv); _mockHttp .When(_ofacFileServiceOptions.GetSdnFileUri().ToString()) @@ -91,7 +99,7 @@ public class DatabaseMaintainerTests : DatabaseTest [Fact] public async Task BuildAddressTableAsync_WhenCalledAndNoSdnRecordFound_ItShouldNotAddAddressCsvRecordsToDatabase() { - var testStream = CreateCsvStream(TestAddressCsv); + using var testStream = CreateCsvStream(TestAddressCsv); _mockHttp .When(_ofacFileServiceOptions.GetAddressFileUri().ToString()) @@ -107,7 +115,7 @@ public class DatabaseMaintainerTests : DatabaseTest [Fact] public async Task BuildAddressTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddAddressCsvRecordsToDatabase() { - var testStream = CreateCsvStream(TestAddressCsv); + using var testStream = CreateCsvStream(TestAddressCsv); _mockHttp .When(_ofacFileServiceOptions.GetAddressFileUri().ToString()) @@ -154,4 +162,121 @@ public class DatabaseMaintainerTests : DatabaseTest } }); } + + [Fact] + public async Task BuildAliasTableAsync_WhenCalledAndSdnRecordNotFound_ItShouldNotAddAliasCsvRecordsToDatabase() + { + using var testStream = CreateCsvStream(TestAliasCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetAltNamesFileUri().ToString()) + .Respond("text/csv", testStream); + + await _databaseMaintainer.BuiltAliasTableAsync(); + + var aliases = await _context.Set().ToListAsync(); + + aliases.Should().BeEmpty(); + } + + [Fact] + public async Task BuildAliasTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddAliasCsvRecordsToDatabase() + { + using var testStream = CreateCsvStream(TestAliasCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetAltNamesFileUri().ToString()) + .Respond("text/csv", testStream); + + var now = DateTimeOffset.UtcNow; + + _timeProviderMock + .Setup(x => x.GetUtcNow()) + .Returns(now); + + var sdn = new Sdn() { Id = 555 }; + await _context.Set().AddAsync(sdn); + await _context.SaveChangesAsync(); + + await _databaseMaintainer.BuiltAliasTableAsync(); + + var aliases = await _context.Set().ToListAsync(); + + aliases.Should().BeEquivalentTo(new[] + { + new Alias() + { + SdnId = sdn.Id, + Id = 477, + Name = "COPROVA", + Type = "aka", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime, + Sdn = sdn + }, + new Alias() + { + SdnId = sdn.Id, + Id = 478, + Name = "COPROVA SARL", + Type = "aka", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime, + Sdn = sdn + } + }); + } + + [Fact] + public async Task BuildCommentTableAsync_WhenCalledAndSdnRecordNotFound_ItShouldNotAddCommentCsvRecordsToDatabase() + { + using var testStream = CreateCsvStream(TestCommentCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetCommentsFileUri().ToString()) + .Respond("text/csv", testStream); + + await _databaseMaintainer.BuildCommentTableAsync(); + + var comments = await _context.Set().ToListAsync(); + + comments.Should().BeEmpty(); + } + + [Fact] + public async Task BuildCommentTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddCommentCsvRecordsToDatabase() + { + using var testStream = CreateCsvStream(TestCommentCsv); + + _mockHttp + .When(_ofacFileServiceOptions.GetCommentsFileUri().ToString()) + .Respond("text/csv", testStream); + + var now = DateTimeOffset.UtcNow; + + _timeProviderMock + .Setup(x => x.GetUtcNow()) + .Returns(now); + + var sdn = new Sdn() { Id = 27307 }; + await _context.Set().AddAsync(sdn); + await _context.SaveChangesAsync(); + + await _databaseMaintainer.BuildCommentTableAsync(); + + var comments = await _context.Set().ToListAsync(); + + comments.Should().BeEquivalentTo(new[] + { + new Comment() + { + SdnId = sdn.Id, + Id = 1, + Remarks = "G TEAM'; a.k.a. 'RED DOT'; a.k.a. 'TEMP.HERMIT'; a.k.a. 'GROUP 77'; a.k.a. 'ZINC'; a.k.a. 'APT-C-26'; a.k.a. 'APPLEWORM'.", + CreatedAt = now.DateTime, + UpdatedAt = now.DateTime, + Sdn = sdn + } + }); + } } \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/AliasMap.cs b/src/SanctionsSearch.Worker/Models/AliasMap.cs new file mode 100644 index 0000000..8f93da4 --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/AliasMap.cs @@ -0,0 +1,13 @@ +namespace SanctionsSearch.Worker.Models; + +class AliasMap : ClassMap +{ + public AliasMap() + { + Map(m => m.SdnId).Index(0); + Map(m => m.Id).Index(1); + Map(m => m.Type).Index(2).TypeConverter(); + Map(m => m.Name).Index(3).TypeConverter(); + Map(m => m.Remarks).Index(4).TypeConverter(); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/CommentMap.cs b/src/SanctionsSearch.Worker/Models/CommentMap.cs new file mode 100644 index 0000000..f829dde --- /dev/null +++ b/src/SanctionsSearch.Worker/Models/CommentMap.cs @@ -0,0 +1,10 @@ +namespace SanctionsSearch.Worker.Models; + +class CommentMap : ClassMap +{ + public CommentMap() + { + Map(m => m.SdnId).Index(0); + Map(m => m.Remarks).Index(1).TypeConverter(); + } +} \ No newline at end of file diff --git a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs index a5786f6..ce0e88a 100644 --- a/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs +++ b/src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs @@ -20,6 +20,8 @@ class DatabaseMaintainer( csv.Context.RegisterClassMap(); csv.Context.RegisterClassMap(); + csv.Context.RegisterClassMap(); + csv.Context.RegisterClassMap(); _streamReaders.Add(reader); _csvReaders.Add(csv); @@ -77,14 +79,62 @@ class DatabaseMaintainer( await _unitOfWork.SaveChangesAsync(); } - public Task BuildCommentTableAsync() + public async Task BuiltAliasTableAsync() { - throw new NotImplementedException(); + var result = await _ofacFileService.GetAltNamesFileAsync(); + + if (result.IsFailed) + { + _logger.LogError("Failed to get Alias file from OFAC."); + return; + } + + using var stream = result.Value; + var records = GetRecordsFromStream(stream); + + foreach (var record in records) + { + var sdn = await _unitOfWork.Sdns.Find(s => s.Id == record.SdnId); + + if (sdn.Count() is 0) + { + _logger.LogWarning("Alias's SDN with ID {Id} not found. Skipping alias.", record.SdnId); + continue; + } + + await _unitOfWork.Aliases.Upsert(record); + } + + await _unitOfWork.SaveChangesAsync(); } - public Task BuiltAliasTableAsync() + public async Task BuildCommentTableAsync() { - throw new NotImplementedException(); + var result = await _ofacFileService.GetCommentsFileAsync(); + + if (result.IsFailed) + { + _logger.LogError("Failed to get Comment file from OFAC."); + return; + } + + using var stream = result.Value; + var records = GetRecordsFromStream(stream); + + foreach (var record in records) + { + var sdn = await _unitOfWork.Sdns.Find(s => s.Id == record.SdnId); + + if (sdn.Count() is 0) + { + _logger.LogWarning("Comment's SDN with ID {Id} not found. Skipping comment.", record.SdnId); + continue; + } + + await _unitOfWork.Comments.Upsert(record); + } + + await _unitOfWork.SaveChangesAsync(); } public void Dispose()