feat: add Plaid transaction handling and async queue processing
- introduced new migration to add Cursor column to AccountMetadata. - updated AppDbContext model snapshot to reflect changes in the database schema. - implemented async queue processing with ChannelAsyncQueue and AsyncQueueHostedService. - created Plaid transaction handlers for added, modified, and removed transactions. - developed PlaidTransactionService and PlaidTransactionSyncer for syncing transactions. - added SyncUpdatesProcessor to handle sync updates from Plaid. - updated tests to accommodate changes in institution and account metadata handling.
This commit is contained in:
Stevan Freeborn
@@ -0,0 +1,54 @@
|
||||
using Account = FiscalOS.Core.Accounts.Account;
|
||||
using Transaction = FiscalOS.Core.Transactions.Transaction;
|
||||
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidAddedTransactionHandler
|
||||
{
|
||||
void Handle(Account account, IEnumerable<Going.Plaid.Entity.Transaction> added);
|
||||
}
|
||||
|
||||
internal sealed class PlaidAddedTransactionHandler : IPlaidAddedTransactionHandler
|
||||
{
|
||||
private readonly ILogger<PlaidAddedTransactionHandler> _logger;
|
||||
|
||||
private PlaidAddedTransactionHandler(ILogger<PlaidAddedTransactionHandler> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public static PlaidAddedTransactionHandler From(IServiceProvider serviceProvider)
|
||||
{
|
||||
return new(
|
||||
serviceProvider.GetRequiredService<ILogger<PlaidAddedTransactionHandler>>()
|
||||
);
|
||||
}
|
||||
|
||||
public void Handle(Account account, IEnumerable<Going.Plaid.Entity.Transaction> added)
|
||||
{
|
||||
foreach (var addedTransaction in added)
|
||||
{
|
||||
if (addedTransaction.Pending.GetValueOrDefault())
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Skipping pending transaction {TransactionId} for account {AccountId} as it has not been posted yet",
|
||||
addedTransaction.TransactionId,
|
||||
account.Id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
var transactionMetadata = PlaidTransactionMetadata.From(addedTransaction.TransactionId);
|
||||
var transaction = Transaction.From(
|
||||
account.UserId,
|
||||
account.Id,
|
||||
addedTransaction.MerchantName,
|
||||
addedTransaction.OriginalDescription,
|
||||
addedTransaction.Amount,
|
||||
addedTransaction.Datetime,
|
||||
transactionMetadata
|
||||
);
|
||||
account.AddTransaction(transaction);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
using Account = FiscalOS.Core.Accounts.Account;
|
||||
using Transaction = FiscalOS.Core.Transactions.Transaction;
|
||||
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidModifiedTransactionHandler
|
||||
{
|
||||
void Handle(Account account, IEnumerable<Going.Plaid.Entity.Transaction> modified);
|
||||
}
|
||||
|
||||
internal sealed class PlaidModifiedTransactionHandler : IPlaidModifiedTransactionHandler
|
||||
{
|
||||
private readonly ILogger<PlaidModifiedTransactionHandler> _logger;
|
||||
private readonly AppDbContext _appDbContext;
|
||||
|
||||
private PlaidModifiedTransactionHandler(
|
||||
ILogger<PlaidModifiedTransactionHandler> logger,
|
||||
AppDbContext appDbContext
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_appDbContext = appDbContext;
|
||||
}
|
||||
|
||||
public static PlaidModifiedTransactionHandler From(IServiceProvider serviceProvider)
|
||||
{
|
||||
return new(
|
||||
serviceProvider.GetRequiredService<ILogger<PlaidModifiedTransactionHandler>>(),
|
||||
serviceProvider.GetRequiredService<AppDbContext>()
|
||||
);
|
||||
}
|
||||
|
||||
public void Handle(Account account, IEnumerable<Going.Plaid.Entity.Transaction> modified)
|
||||
{
|
||||
var modifiedTransactionIds = modified.Select(t => t.TransactionId);
|
||||
var existingModifiedTransactions = _appDbContext.Transactions
|
||||
.Where(t => t.Metadata is PlaidTransactionMetadata && modifiedTransactionIds.Contains(((PlaidTransactionMetadata)t.Metadata).PlaidId))
|
||||
.ToList();
|
||||
|
||||
foreach (var existing in existingModifiedTransactions)
|
||||
{
|
||||
if (existing.Metadata is not PlaidTransactionMetadata plaidMetadata)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Existing transaction {TransactionId} has non-Plaid metadata. Skipping update for this transaction.",
|
||||
existing.Id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
var plaidModifiedTransaction = modified.FirstOrDefault(t => t.TransactionId == plaidMetadata.PlaidId);
|
||||
|
||||
if (plaidModifiedTransaction is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No corresponding modified transaction found in Plaid response for existing transaction {TransactionId}. Skipping update for this transaction.",
|
||||
existing.Id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
var newTransactionData = Transaction.From(
|
||||
existing.UserId,
|
||||
existing.AccountId,
|
||||
plaidModifiedTransaction.MerchantName,
|
||||
plaidModifiedTransaction.OriginalDescription,
|
||||
plaidModifiedTransaction.Amount,
|
||||
plaidModifiedTransaction.Datetime,
|
||||
plaidMetadata
|
||||
);
|
||||
|
||||
_appDbContext.Entry(existing).CurrentValues.SetValues(newTransactionData);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidRemovedTransactionHandler
|
||||
{
|
||||
Task HandleAsync(IEnumerable<RemovedTransaction> removed, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class PlaidRemovedTransactionHandler : IPlaidRemovedTransactionHandler
|
||||
{
|
||||
private readonly AppDbContext _appDbContext;
|
||||
|
||||
private PlaidRemovedTransactionHandler(AppDbContext appDbContext)
|
||||
{
|
||||
_appDbContext = appDbContext;
|
||||
}
|
||||
|
||||
public static PlaidRemovedTransactionHandler From(IServiceProvider serviceProvider)
|
||||
{
|
||||
return new(
|
||||
serviceProvider.GetRequiredService<AppDbContext>()
|
||||
);
|
||||
}
|
||||
|
||||
public async Task HandleAsync(IEnumerable<RemovedTransaction> removed, CancellationToken cancellationToken)
|
||||
{
|
||||
var removedIdsList = removed.Select(t => t.TransactionId);
|
||||
|
||||
await _appDbContext.Transactions
|
||||
.Where(t => t.Metadata is PlaidTransactionMetadata && removedIdsList.Contains(((PlaidTransactionMetadata)t.Metadata).PlaidId))
|
||||
.ExecuteDeleteAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using FiscalOS.Core.Transactions;
|
||||
using FiscalOS.Infra.Common;
|
||||
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
public sealed class PlaidTransactionMetadata : TransactionMetadata
|
||||
{
|
||||
public const string TypeValue = Providers.Plaid;
|
||||
public string PlaidId { get; init; } = string.Empty;
|
||||
|
||||
private PlaidTransactionMetadata() : base(TypeValue)
|
||||
{
|
||||
}
|
||||
|
||||
public static PlaidTransactionMetadata From(string? plaidId)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plaidId);
|
||||
|
||||
return new()
|
||||
{
|
||||
PlaidId = plaidId,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using Account = FiscalOS.Core.Accounts.Account;
|
||||
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidTransactionProcessor
|
||||
{
|
||||
Task ProcessAsync(
|
||||
Account account,
|
||||
TransactionsSyncResponse syncResponse,
|
||||
CancellationToken cancellationToken
|
||||
);
|
||||
}
|
||||
|
||||
internal sealed class PlaidTransactionProcessor : IPlaidTransactionProcessor
|
||||
{
|
||||
private readonly IPlaidAddedTransactionHandler _addedHandler;
|
||||
private readonly IPlaidModifiedTransactionHandler _modifiedHandler;
|
||||
private readonly IPlaidRemovedTransactionHandler _removedHandler;
|
||||
|
||||
private PlaidTransactionProcessor(
|
||||
IPlaidAddedTransactionHandler addedHandler,
|
||||
IPlaidModifiedTransactionHandler modifiedHandler,
|
||||
IPlaidRemovedTransactionHandler removedHandler
|
||||
)
|
||||
{
|
||||
_addedHandler = addedHandler;
|
||||
_modifiedHandler = modifiedHandler;
|
||||
_removedHandler = removedHandler;
|
||||
}
|
||||
|
||||
internal static PlaidTransactionProcessor From(IServiceProvider provider)
|
||||
{
|
||||
return new(
|
||||
provider.GetRequiredService<IPlaidAddedTransactionHandler>(),
|
||||
provider.GetRequiredService<IPlaidModifiedTransactionHandler>(),
|
||||
provider.GetRequiredService<IPlaidRemovedTransactionHandler>()
|
||||
);
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(Account account, TransactionsSyncResponse syncResponse, CancellationToken cancellationToken)
|
||||
{
|
||||
_addedHandler.Handle(account, syncResponse.Added);
|
||||
_modifiedHandler.Handle(account, syncResponse.Modified);
|
||||
await _removedHandler.HandleAsync(syncResponse.Removed, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidTransactionService
|
||||
{
|
||||
IAsyncEnumerable<TransactionsSyncResponse> SyncTransactionsForAccountAsync(
|
||||
string accessToken,
|
||||
string accountId,
|
||||
string? initialCursor = null
|
||||
);
|
||||
}
|
||||
|
||||
internal sealed class PlaidTransactionService : IPlaidTransactionService
|
||||
{
|
||||
private readonly PlaidClient _client;
|
||||
|
||||
private PlaidTransactionService(PlaidClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public static PlaidTransactionService From(IServiceProvider sp)
|
||||
{
|
||||
var client = sp.GetRequiredService<PlaidClient>();
|
||||
return new(client);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TransactionsSyncResponse> SyncTransactionsForAccountAsync(
|
||||
string accessToken,
|
||||
string accountId,
|
||||
string? initialCursor = null
|
||||
)
|
||||
{
|
||||
TransactionsSyncResponse? response;
|
||||
var cursor = initialCursor;
|
||||
|
||||
do
|
||||
{
|
||||
response = await _client.TransactionsSyncAsync(new()
|
||||
{
|
||||
AccessToken = accessToken,
|
||||
Cursor = cursor,
|
||||
Count = 500,
|
||||
Options = new()
|
||||
{
|
||||
AccountId = accountId,
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
if (response.Error?.ErrorCode is "TRANSACTIONS_SYNC_MUTATION_DURING_PAGINATION")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
cursor = response.NextCursor;
|
||||
yield return response;
|
||||
} while (response is not null && response.HasMore);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using Account = FiscalOS.Core.Accounts.Account;
|
||||
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal interface IPlaidTransactionSyncer
|
||||
{
|
||||
Task SyncTransactionsForAccountAsync(
|
||||
Account account,
|
||||
string decryptedAccessToken,
|
||||
CancellationToken cancellationToken
|
||||
);
|
||||
}
|
||||
|
||||
internal sealed class PlaidTransactionSyncer : IPlaidTransactionSyncer
|
||||
{
|
||||
private readonly ILogger<PlaidTransactionSyncer> _logger;
|
||||
private readonly IPlaidTransactionService _plaidTransactionService;
|
||||
private readonly IPlaidTransactionProcessor _transactionProcessor;
|
||||
|
||||
private PlaidTransactionSyncer(
|
||||
ILogger<PlaidTransactionSyncer> logger,
|
||||
IPlaidTransactionService plaidTransactionService,
|
||||
IPlaidTransactionProcessor plaidTransactionProcessor
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_plaidTransactionService = plaidTransactionService;
|
||||
_transactionProcessor = plaidTransactionProcessor;
|
||||
}
|
||||
|
||||
public static PlaidTransactionSyncer From(IServiceProvider serviceProvider)
|
||||
{
|
||||
return new(
|
||||
serviceProvider.GetRequiredService<ILogger<PlaidTransactionSyncer>>(),
|
||||
serviceProvider.GetRequiredService<IPlaidTransactionService>(),
|
||||
serviceProvider.GetRequiredService<IPlaidTransactionProcessor>()
|
||||
);
|
||||
}
|
||||
|
||||
public async Task SyncTransactionsForAccountAsync(Account account, string decryptedAccessToken, CancellationToken cancellationToken)
|
||||
{
|
||||
if (account.Metadata is not PlaidAccountMetadata plaidAccountMetadata)
|
||||
{
|
||||
_logger.LogWarning("Unable to sync transactions for account {AccountId} as it has no Plaid metadata", account.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
var responses = _plaidTransactionService.SyncTransactionsForAccountAsync(
|
||||
decryptedAccessToken,
|
||||
plaidAccountMetadata.PlaidId,
|
||||
plaidAccountMetadata.Cursor
|
||||
).ConfigureAwait(false);
|
||||
|
||||
await foreach (var response in responses)
|
||||
{
|
||||
if (response.IsSuccessStatusCode is false)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Failed to sync transactions for request {RequestId} for account {AccountId}: {ErrorMessage}",
|
||||
response.RequestId,
|
||||
account.Id,
|
||||
response.Error?.ErrorMessage
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
if (response.Accounts.Any() is false)
|
||||
{
|
||||
_logger.LogInformation("No transactions to sync for request {RequestId} for account {AccountId}", response.RequestId, account.Id);
|
||||
break;
|
||||
}
|
||||
|
||||
var plaidAccount = response.Accounts
|
||||
.FirstOrDefault(pa => pa.AccountId == plaidAccountMetadata.PlaidId);
|
||||
|
||||
if (plaidAccount is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Failed to sync transactions for request {RequestId} for account {AccountId}: No corresponding account received in Plaid response",
|
||||
response.RequestId,
|
||||
account.Id
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
plaidAccountMetadata.SetCursor(response.NextCursor);
|
||||
var balance = Balance.From(plaidAccount.Current, plaidAccount.Available, plaidAccount.CurrencyCode);
|
||||
account.AddBalance(balance);
|
||||
|
||||
await _transactionProcessor.ProcessAsync(account, response, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
internal sealed class SyncUpdatesProcessor : IAsyncQueueProcessor<SyncUpdatesQueueItem>
|
||||
{
|
||||
private readonly ILogger<SyncUpdatesProcessor> _logger;
|
||||
private readonly AppDbContext _appDbContext;
|
||||
private readonly IEncryptor _encryptor;
|
||||
private readonly IPlaidTransactionSyncer _plaidTransactionSyncer;
|
||||
|
||||
private SyncUpdatesProcessor(
|
||||
ILogger<SyncUpdatesProcessor> logger,
|
||||
AppDbContext appDbContext,
|
||||
IEncryptor encryptor,
|
||||
IPlaidTransactionSyncer plaidTransactionSyncer
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_appDbContext = appDbContext;
|
||||
_encryptor = encryptor;
|
||||
_plaidTransactionSyncer = plaidTransactionSyncer;
|
||||
}
|
||||
|
||||
public static SyncUpdatesProcessor From(IServiceProvider serviceProvider)
|
||||
{
|
||||
return new(
|
||||
serviceProvider.GetRequiredService<ILogger<SyncUpdatesProcessor>>(),
|
||||
serviceProvider.GetRequiredService<AppDbContext>(),
|
||||
serviceProvider.GetRequiredService<IEncryptor>(),
|
||||
serviceProvider.GetRequiredService<IPlaidTransactionSyncer>()
|
||||
);
|
||||
}
|
||||
|
||||
public async Task ProcessAsync(SyncUpdatesQueueItem item, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_logger.LogInformation("Processing sync updates for item {ItemId}", item.InstitutionItemId);
|
||||
|
||||
var institution = await _appDbContext.Institutions
|
||||
.Include(i => i.User)
|
||||
.Include(i => i.Accounts)
|
||||
.ThenInclude(a => a.Metadata)
|
||||
.Include(i => i.Metadata)
|
||||
.Where(i => i.Metadata is PlaidInstitutionMetadata && ((PlaidInstitutionMetadata)i.Metadata).ItemId == item.InstitutionItemId)
|
||||
.AsSplitQuery()
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (institution is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Unable to process sync updates for item {ItemId} as no institution with matching Plaid item ID was found.",
|
||||
item.InstitutionItemId
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (institution.User is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Unable to process sync updates for item {ItemId} as the associated institution {InstitutionId} does not have a user.",
|
||||
item.InstitutionItemId,
|
||||
institution.Id
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (institution.Metadata is not PlaidInstitutionMetadata plaidMetadata)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Unable to process sync updates for item {ItemId} as the associated institution {InstitutionId} does not have Plaid metadata.",
|
||||
item.InstitutionItemId,
|
||||
institution.Id
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
var decryptedAccessToken = await _encryptor.DecryptAsyncFor(
|
||||
institution.User!,
|
||||
plaidMetadata.EncryptedAccessToken,
|
||||
cancellationToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var account in institution.Accounts)
|
||||
{
|
||||
await _plaidTransactionSyncer.SyncTransactionsForAccountAsync(account, decryptedAccessToken, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _appDbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Successfully processed sync updates for item {ItemId}", item.InstitutionItemId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace FiscalOS.Infra.Transactions.Plaid;
|
||||
|
||||
public sealed record SyncUpdatesQueueItem(string InstitutionItemId);
|
||||
Reference in New Issue
Block a user