Stevan/sanctions-search
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement building alias and comment tables

Browse Source
This commit is contained in:
Stevan Freeborn
2024-08-25 11:48:10 -05:00
parent 0b9a83619d
commit 14e35670a5
4 changed files with 211 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/SanctionsSearch.Worker.Tests/Integration/DatabaseMaintainerTests.cs
+134 -9
View File
@@ -2,10 +2,24 @@ namespace SanctionsSearch.Worker.Tests.Integration;
public class DatabaseMaintainerTests : DatabaseTest
{
private const string TestSdnCsv = """
6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male."
6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt."
6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt."
""";
private const string TestAddressCsv = """
306,201,"Dai-Ichi Bldg. 6th Floor, 10-2 Nihombashi, 2-chome, Chuo-ku","Tokyo 103","Japan",-0-
306,202,"Federico Boyd Avenue & 51 Street","Panama City","Panama",-0-
""";
private const string TestAliasCsv = """
555,477,"aka","COPROVA",-0-
555,478,"aka","COPROVA SARL",-0-
""";
private const string TestCommentCsv = """
27307,"G TEAM'; a.k.a. 'RED DOT'; a.k.a. 'TEMP.HERMIT'; a.k.a. 'GROUP 77'; a.k.a. 'ZINC'; a.k.a. 'APT-C-26'; a.k.a. 'APPLEWORM'."
28263,"hn'; Linked To: LAZARUS GROUP."
""";
private readonly MockHttpMessageHandler _mockHttp = new();
private readonly OfacFileServiceOptionsFaker _ofacFileServiceOptionsFaker = new();
private readonly OfacFileServiceOptions _ofacFileServiceOptions;
@@ -30,13 +44,7 @@ public class DatabaseMaintainerTests : DatabaseTest
[Fact]
public async Task BuildSdnTableAsync_WhenCalled_ItShouldAddSdnCsvRecordsToDatabase()
{
var testCsv = """
6906,"AL-IRAQI, Abd al-Hadi","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 1961; POB Mosul, Iraq; nationality Iraq; Gender Male."
6907,"SHIHATA, Thirwat Salah","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 29 Jun 1960; POB Egypt."
6908,"AHMAD, Tariq Anwar al-Sayyid","individual","SDGT",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,"DOB 15 Mar 1963; POB Alexandria, Egypt."
""";
var testStream = CreateCsvStream(testCsv);
using var testStream = CreateCsvStream(TestSdnCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetSdnFileUri().ToString())
@@ -91,7 +99,7 @@ public class DatabaseMaintainerTests : DatabaseTest
[Fact]
public async Task BuildAddressTableAsync_WhenCalledAndNoSdnRecordFound_ItShouldNotAddAddressCsvRecordsToDatabase()
{
var testStream = CreateCsvStream(TestAddressCsv);
using var testStream = CreateCsvStream(TestAddressCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetAddressFileUri().ToString())
@@ -107,7 +115,7 @@ public class DatabaseMaintainerTests : DatabaseTest
[Fact]
public async Task BuildAddressTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddAddressCsvRecordsToDatabase()
{
var testStream = CreateCsvStream(TestAddressCsv);
using var testStream = CreateCsvStream(TestAddressCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetAddressFileUri().ToString())
@@ -154,4 +162,121 @@ public class DatabaseMaintainerTests : DatabaseTest
}
});
}
[Fact]
public async Task BuildAliasTableAsync_WhenCalledAndSdnRecordNotFound_ItShouldNotAddAliasCsvRecordsToDatabase()
{
using var testStream = CreateCsvStream(TestAliasCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetAltNamesFileUri().ToString())
.Respond("text/csv", testStream);
await _databaseMaintainer.BuiltAliasTableAsync();
var aliases = await _context.Set<Alias>().ToListAsync();
aliases.Should().BeEmpty();
}
[Fact]
public async Task BuildAliasTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddAliasCsvRecordsToDatabase()
{
using var testStream = CreateCsvStream(TestAliasCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetAltNamesFileUri().ToString())
.Respond("text/csv", testStream);
var now = DateTimeOffset.UtcNow;
_timeProviderMock
.Setup(x => x.GetUtcNow())
.Returns(now);
var sdn = new Sdn() { Id = 555 };
await _context.Set<Sdn>().AddAsync(sdn);
await _context.SaveChangesAsync();
await _databaseMaintainer.BuiltAliasTableAsync();
var aliases = await _context.Set<Alias>().ToListAsync();
aliases.Should().BeEquivalentTo(new[]
{
new Alias()
{
SdnId = sdn.Id,
Id = 477,
Name = "COPROVA",
Type = "aka",
CreatedAt = now.DateTime,
UpdatedAt = now.DateTime,
Sdn = sdn
},
new Alias()
{
SdnId = sdn.Id,
Id = 478,
Name = "COPROVA SARL",
Type = "aka",
CreatedAt = now.DateTime,
UpdatedAt = now.DateTime,
Sdn = sdn
}
});
}
[Fact]
public async Task BuildCommentTableAsync_WhenCalledAndSdnRecordNotFound_ItShouldNotAddCommentCsvRecordsToDatabase()
{
using var testStream = CreateCsvStream(TestCommentCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetCommentsFileUri().ToString())
.Respond("text/csv", testStream);
await _databaseMaintainer.BuildCommentTableAsync();
var comments = await _context.Set<Comment>().ToListAsync();
comments.Should().BeEmpty();
}
[Fact]
public async Task BuildCommentTableAsync_WhenCalledAndSdnRecordFound_ItShouldAddCommentCsvRecordsToDatabase()
{
using var testStream = CreateCsvStream(TestCommentCsv);
_mockHttp
.When(_ofacFileServiceOptions.GetCommentsFileUri().ToString())
.Respond("text/csv", testStream);
var now = DateTimeOffset.UtcNow;
_timeProviderMock
.Setup(x => x.GetUtcNow())
.Returns(now);
var sdn = new Sdn() { Id = 27307 };
await _context.Set<Sdn>().AddAsync(sdn);
await _context.SaveChangesAsync();
await _databaseMaintainer.BuildCommentTableAsync();
var comments = await _context.Set<Comment>().ToListAsync();
comments.Should().BeEquivalentTo(new[]
{
new Comment()
{
SdnId = sdn.Id,
Id = 1,
Remarks = "G TEAM'; a.k.a. 'RED DOT'; a.k.a. 'TEMP.HERMIT'; a.k.a. 'GROUP 77'; a.k.a. 'ZINC'; a.k.a. 'APT-C-26'; a.k.a. 'APPLEWORM'.",
CreatedAt = now.DateTime,
UpdatedAt = now.DateTime,
Sdn = sdn
}
});
}
}
src/SanctionsSearch.Worker/Models/AliasMap.cs
+13
View File
@@ -0,0 +1,13 @@
namespace SanctionsSearch.Worker.Models;
class AliasMap : ClassMap<Alias>
{
public AliasMap()
{
Map(m => m.SdnId).Index(0);
Map(m => m.Id).Index(1);
Map(m => m.Type).Index(2).TypeConverter<NullCharacterConverter>();
Map(m => m.Name).Index(3).TypeConverter<NullCharacterConverter>();
Map(m => m.Remarks).Index(4).TypeConverter<NullCharacterConverter>();
}
}
src/SanctionsSearch.Worker/Models/CommentMap.cs
+10
View File
@@ -0,0 +1,10 @@
namespace SanctionsSearch.Worker.Models;
class CommentMap : ClassMap<Comment>
{
public CommentMap()
{
Map(m => m.SdnId).Index(0);
Map(m => m.Remarks).Index(1).TypeConverter<NullCharacterConverter>();
}
}
src/SanctionsSearch.Worker/Models/DatabaseMaintainer.cs
+54 -4
View File
@@ -20,6 +20,8 @@ class DatabaseMaintainer(
csv.Context.RegisterClassMap<SdnMap>();
csv.Context.RegisterClassMap<AddressMap>();
csv.Context.RegisterClassMap<AliasMap>();
csv.Context.RegisterClassMap<CommentMap>();
_streamReaders.Add(reader);
_csvReaders.Add(csv);
@@ -77,14 +79,62 @@ class DatabaseMaintainer(
await _unitOfWork.SaveChangesAsync();
}
public Task BuildCommentTableAsync()
public async Task BuiltAliasTableAsync()
{
throw new NotImplementedException();
var result = await _ofacFileService.GetAltNamesFileAsync();
if (result.IsFailed)
{
_logger.LogError("Failed to get Alias file from OFAC.");
return;
}
public Task BuiltAliasTableAsync()
using var stream = result.Value;
var records = GetRecordsFromStream<Alias>(stream);
foreach (var record in records)
{
throw new NotImplementedException();
var sdn = await _unitOfWork.Sdns.Find(s => s.Id == record.SdnId);
if (sdn.Count() is 0)
{
_logger.LogWarning("Alias's SDN with ID {Id} not found. Skipping alias.", record.SdnId);
continue;
}
await _unitOfWork.Aliases.Upsert(record);
}
await _unitOfWork.SaveChangesAsync();
}
public async Task BuildCommentTableAsync()
{
var result = await _ofacFileService.GetCommentsFileAsync();
if (result.IsFailed)
{
_logger.LogError("Failed to get Comment file from OFAC.");
return;
}
using var stream = result.Value;
var records = GetRecordsFromStream<Comment>(stream);
foreach (var record in records)
{
var sdn = await _unitOfWork.Sdns.Find(s => s.Id == record.SdnId);
if (sdn.Count() is 0)
{
_logger.LogWarning("Comment's SDN with ID {Id} not found. Skipping comment.", record.SdnId);
continue;
}
await _unitOfWork.Comments.Upsert(record);
}
await _unitOfWork.SaveChangesAsync();
}
public void Dispose()