feat: add Plaid transaction handling and async queue processing

- introduced new migration to add Cursor column to AccountMetadata.
- updated AppDbContext model snapshot to reflect changes in the database schema.
- implemented async queue processing with ChannelAsyncQueue and AsyncQueueHostedService.
- created Plaid transaction handlers for added, modified, and removed transactions.
- developed PlaidTransactionService and PlaidTransactionSyncer for syncing transactions.
- added SyncUpdatesProcessor to handle sync updates from Plaid.
- updated tests to accommodate changes in institution and account metadata handling.

src/FiscalOS.API/Accounts/Add/Endpoint.cs

@@ -13,15 +13,20 @@ internal static class Endpoint
     HttpContext httpContext,
     [FromBody] Request request,
     [FromServices] AppDbContext appDbContext,
+    [FromServices] IEncryptor encryptor,
+    [FromServices] IPlaidAccountService plaidAccountService,
+    [FromServices] IAsyncQueue<SyncUpdatesQueueItem> queue,
     CancellationToken ct
   )
   {
     var userId = httpContext.GetUserId();
 
     var user = await appDbContext.Users
-      .Include(u => u.Institutions.Where(i => i.Metadata is PlaidMetadata && ((PlaidMetadata)i.Metadata).PlaidId == request.PlaidInstitutionId))
+      .Include(u => u.Institutions.Where(i => i.Metadata is PlaidInstitutionMetadata && ((PlaidInstitutionMetadata)i.Metadata).PlaidId == request.PlaidInstitutionId))
+      .ThenInclude(i => i.Metadata)
       .Include(u => u.Accounts.Where(a => a.Metadata is PlaidAccountMetadata && ((PlaidAccountMetadata)a.Metadata).PlaidId == request.PlaidAccountId))
       .ThenInclude(a => a.Metadata)
+      .AsSplitQuery()
       .SingleOrDefaultAsync(u => u.Id == userId, ct);
 
     if (user is null)
@@ -37,19 +42,28 @@ internal static class Endpoint
       });
     }
 
+    var plaidInstitutionMetadata = (PlaidInstitutionMetadata?)user.Institutions.First().Metadata;
+
+    if (plaidInstitutionMetadata is null)
+    {
+      return Results.ValidationProblem(new Dictionary<string, string[]>
+      {
+        ["PlaidInstitutionId"] = ["The PlaidInstitutionId field is invalid. The connected institution has no plaid metadata"],
+      });
+    }
+
     if (user.Accounts.Any())
     {
       return Results.Conflict();
     }
 
+    var decryptedAccessToken = await encryptor.DecryptAsyncFor(user, plaidInstitutionMetadata.EncryptedAccessToken, ct);
     var accountMetadata = PlaidAccountMetadata.From(request.PlaidAccountId, request.PlaidAccountName);
     var account = Account.From(user.Institutions.First().Id, request.PlaidAccountName, accountMetadata);
-    var balance = Balance.From(request.AccountCurrentBalance, request.AccountAvailableBalance, request.AccountCurrencyCode);
-
-    account.AddBalance(balance);
     user.AddAccount(account);
 
     await appDbContext.SaveChangesAsync(ct);
+    await queue.EnqueueAsync(new(plaidInstitutionMetadata.ItemId), ct);
 
     return Results.Ok();
   }

src/FiscalOS.API/Accounts/Add/Request.cs

@@ -5,9 +5,6 @@ public record Request : IValidatableObject
   public string PlaidInstitutionId { get; init; } = string.Empty;
   public string PlaidAccountId { get; init; } = string.Empty;
   public string PlaidAccountName { get; init; } = string.Empty;
-  public decimal AccountCurrentBalance { get; init; }
-  public decimal AccountAvailableBalance { get; init; }
-  public string AccountCurrencyCode { get; init; } = string.Empty;
 
   public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
   {
@@ -28,11 +25,5 @@ public record Request : IValidatableObject
       var fieldName = nameof(PlaidAccountName);
       yield return new($"The {fieldName} field is required.", [fieldName]);
     }
-
-    if (string.IsNullOrWhiteSpace(AccountCurrencyCode))
-    {
-      var fieldName = nameof(AccountCurrencyCode);
-      yield return new($"The {fieldName} field is required.", [fieldName]);
-    }
   }
 }

src/FiscalOS.API/Institutions/Connect/Endpoint.cs

@@ -13,7 +13,7 @@ internal static class Endpoint
     HttpContext httpContext,
     [FromBody] Request request,
     [FromServices] AppDbContext appDbContext,
-    [FromServices] PlaidService plaidService,
+    [FromServices] IPlaidAccountService plaidAccountService,
     [FromServices] IEncryptor encryptor,
     CancellationToken ct
   )
@@ -23,8 +23,8 @@ internal static class Endpoint
     var user = await appDbContext.Users
       .Include(u => u.Institutions
         .Where(
-          i => i.Metadata is PlaidMetadata &&
-            ((PlaidMetadata)i.Metadata).PlaidId == request.PlaidInstitutionId
+          i => i.Metadata is PlaidInstitutionMetadata &&
+            ((PlaidInstitutionMetadata)i.Metadata).PlaidId == request.PlaidInstitutionId
         )
       )
       .ThenInclude(i => i.Metadata)
@@ -40,12 +40,11 @@ internal static class Endpoint
       return Results.Conflict();
     }
 
-    var (itemId, accessToken) = await plaidService.ExchangeTokenAsync(request.PublicToken);
-    var item = await plaidService.GetItemAsync(accessToken);
-
+    var (itemId, accessToken) = await plaidAccountService.ExchangeTokenAsync(request.PublicToken);
+    var item = await plaidAccountService.GetItemAsync(accessToken);
     var encryptedAccessToken = await encryptor.EncryptAsyncFor(user, accessToken, ct);
 
-    var plaidMetadata = PlaidMetadata.From(item.InstitutionId, item.InstitutionName, encryptedAccessToken);
+    var plaidMetadata = PlaidInstitutionMetadata.From(item.InstitutionId, item.InstitutionName, encryptedAccessToken, itemId);
     var institution = Institution.From(item.InstitutionName, plaidMetadata);
 
     user.AddInstitution(institution);

src/FiscalOS.API/Institutions/GetAvailable/Endpoint.cs

@@ -13,7 +13,7 @@ internal static class Endpoint
     HttpContext httpContext,
     [FromRoute] Guid id,
     [FromServices] AppDbContext appDbContext,
-    [FromServices] PlaidService plaidService,
+    [FromServices] IPlaidAccountService plaidAccountService,
     [FromServices] IEncryptor encryptor,
     CancellationToken ct
   )
@@ -37,13 +37,13 @@ internal static class Endpoint
 
     var institution = user.Institutions.First();
 
-    if (institution.Metadata is not PlaidMetadata plaidMetadata)
+    if (institution.Metadata is not PlaidInstitutionMetadata plaidMetadata)
     {
       return Results.BadRequest();
     }
 
     var accessToken = await encryptor.DecryptAsyncFor(user, plaidMetadata.EncryptedAccessToken, ct);
-    var accounts = await plaidService.GetAccountsAsync(accessToken);
+    var accounts = await plaidAccountService.GetAccountsAsync(accessToken);
     var accountsDtos = accounts.Select(a => AvailableAccountDto.From(plaidMetadata, a));
 
     return Results.Ok(Response.From(accountsDtos));

src/FiscalOS.API/Institutions/GetAvailable/Response.cs

@@ -33,7 +33,7 @@ internal sealed record AvailableAccountDto
   }
 
   public static AvailableAccountDto From(
-    PlaidMetadata plaidMetadata,
+    PlaidInstitutionMetadata plaidMetadata,
     Going.Plaid.Entity.Account plaidAccount
   )
   {

src/FiscalOS.API/Institutions/Link/Endpoint.cs

@@ -12,7 +12,7 @@ internal static class Endpoint
   private static async Task<IResult> HandleAsync(
     HttpContext httpContext,
     [FromServices] AppDbContext appDbContext,
-    [FromServices] PlaidService plaidService
+    [FromServices] IPlaidAccountService plaidAccountService
   )
   {
     var userId = httpContext.GetUserId();
@@ -24,7 +24,7 @@ internal static class Endpoint
       return Results.Unauthorized();
     }
 
-    var linkToken = await plaidService.CreateLinkTokenAsync(user.Id.ToString());
+    var linkToken = await plaidAccountService.CreateLinkTokenAsync(user.Id.ToString());
 
     return Results.Ok(Response.From(linkToken));
   }

src/FiscalOS.API/Program.cs

@@ -1,4 +1,4 @@
-using FiscalOS.ServiceDefaults;
+using Going.Plaid.Converters;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -7,6 +7,12 @@ builder.AddServiceDefaults();
 builder.Services.AddValidation();
 builder.Services.AddProblemDetails();
 
+builder.Services.ConfigureHttpJsonOptions(static options =>
+{
+  options.SerializerOptions.Converters.Add(new EnumConverterFactory());
+  options.SerializerOptions.Converters.Add(new WebhookBaseConverter());
+});
+
 builder.Services.AddInfrastructure();
 
 builder.Services.AddAuthentication(static o =>
@@ -41,5 +47,6 @@ app.MapDefaultEndpoints();
 app.MapAuthEndpoints();
 app.MapAccountsEndpoints();
 app.MapInstitutionsEndpoints();
+app.MapTransactionsGroup();
 
 app.Run();

src/FiscalOS.API/Transactions/FireWebhook/Endpoint.cs

@@ -0,0 +1,65 @@
+using Going.Plaid;
+using Going.Plaid.Entity;
+
+namespace FiscalOS.API.Transactions.FireWebhook;
+
+internal static class Endpoint
+{
+  private const string Route = "/fire-webhook";
+
+  public static RouteHandlerBuilder MapFireWebhookEndpoint(this RouteGroupBuilder groupBuilder)
+  {
+    return groupBuilder.MapPost(Route, HandleAsync);
+  }
+
+  private static async Task<IResult> HandleAsync(
+    HttpContext httpContext,
+    [FromBody] Request request,
+    [FromServices] AppDbContext appDbContext,
+    [FromServices] IEncryptor encryptor,
+    [FromServices] PlaidClient plaidClient,
+    CancellationToken ct
+  )
+  {
+    var userId = httpContext.GetUserId();
+
+    var user = await appDbContext.Users
+      .Include(u => u.Accounts.Where(a => a.Id.ToString() == request.AccountId))
+      .ThenInclude(static a => a.Institution!)
+      .ThenInclude(static i => i.Metadata)
+      .FirstOrDefaultAsync(u => u.Id == userId, ct);
+
+    if (user is null)
+    {
+      return Results.Unauthorized();
+    }
+
+    if (user.Accounts.Any() is false)
+    {
+      return Results.NotFound();
+    }
+
+    var plaidMetadata = (PlaidInstitutionMetadata?)user.Accounts.First().Institution?.Metadata;
+
+    if (plaidMetadata is null)
+    {
+      return Results.InternalServerError("No metadata associated with account");
+    }
+
+    var accessToken = await encryptor.DecryptAsyncFor(user, plaidMetadata.EncryptedAccessToken, ct);
+
+    var fireEventResponse = await plaidClient.SandboxItemFireWebhookAsync(new()
+    {
+      AccessToken = accessToken,
+      WebhookType = SandboxItemFireWebhookRequestWebhookTypeEnum.Transactions,
+      WebhookCode = SandboxItemFireWebhookRequestWebhookCodeEnum.SyncUpdatesAvailable,
+    });
+
+    if (fireEventResponse.IsSuccessStatusCode is false)
+    {
+      return Results.InternalServerError(fireEventResponse.Error?.ErrorMessage);
+    }
+
+    return Results.Ok(new { fireEventResponse.RequestId, fireEventResponse.WebhookFired });
+  }
+}

src/FiscalOS.API/Transactions/FireWebhook/Request.cs

@@ -0,0 +1,7 @@
+namespace FiscalOS.API.Transactions.FireWebhook;
+
+public sealed record Request(
+  string AccountId,
+  string WebhookType,
+  string WebhookCode
+);

src/FiscalOS.API/Transactions/TransactionsExtensions.cs

@@ -0,0 +1,21 @@
+namespace FiscalOS.API.Transactions;
+
+internal static class TransactionsExtensions
+{
+  private const string RouteGroupPrefix = "/transactions";
+
+  public static RouteGroupBuilder MapTransactionsGroup(this WebApplication app)
+  {
+    var transactionsGroup = app.MapGroup(RouteGroupPrefix)
+      .RequireAuthorization();
+
+    if (app.Environment.IsProduction() is false)
+    {
+      transactionsGroup.MapFireWebhookEndpoint();
+    }
+
+    transactionsGroup.MapWebhookEndpoint().AllowAnonymous();
+
+    return transactionsGroup;
+  }
+}

src/FiscalOS.API/Transactions/Webhook/Endpoint.cs

@@ -0,0 +1,41 @@
+namespace FiscalOS.API.Transactions.Webhook;
+
+internal static class Endpoint
+{
+  private const string Route = "/webhook";
+
+  public static RouteHandlerBuilder MapWebhookEndpoint(this RouteGroupBuilder groupBuilder)
+  {
+    return groupBuilder.MapPost(Route, HandleAsync);
+  }
+
+  private static async Task<IResult> HandleAsync(
+    [FromBody] WebhookBase request,
+    [FromServices] ILogger<Program> logger,
+    [FromServices] IAsyncQueue<SyncUpdatesQueueItem> queue,
+    CancellationToken ct
+  )
+  {
+    logger.LogInformation(
+      "Received {WebhookType} webhook with {WebhookCode} payload",
+      request.WebhookType,
+      request.WebhookCode
+    );
+
+    switch (request)
+    {
+      case SyncUpdatesAvailableWebhook webhook:
+        await queue.EnqueueAsync(new(webhook.ItemId), ct);
+        logger.LogInformation(
+          "Enqueued {WebhookType} webhook with {WebhookCode} payload",
+          request.WebhookType,
+          request.WebhookCode
+        );
+        break;
+      default:
+        break;
+    }
+
+    return Results.Ok();
+  }
+}

src/FiscalOS.API/Usings.cs

@@ -2,7 +2,6 @@ global using System.ComponentModel.DataAnnotations;
 global using System.Security.Claims;
 global using System.Text.Json.Serialization;
 
-global using FiscalOS.API.Institutions.Get;
 global using FiscalOS.API.Accounts;
 global using FiscalOS.API.Accounts.Add;
 global using FiscalOS.API.Auth;
@@ -11,16 +10,27 @@ global using FiscalOS.API.Auth.Refresh;
 global using FiscalOS.API.Http;
 global using FiscalOS.API.Institutions;
 global using FiscalOS.API.Institutions.Connect;
+global using FiscalOS.API.Institutions.Get;
 global using FiscalOS.API.Institutions.GetAvailable;
 global using FiscalOS.API.Institutions.Link;
+global using FiscalOS.API.Transactions;
+global using FiscalOS.API.Transactions.FireWebhook;
+global using FiscalOS.API.Transactions.Webhook;
 global using FiscalOS.Core.Accounts;
 global using FiscalOS.Core.Authentication;
 global using FiscalOS.Core.Identity;
+global using FiscalOS.Core.Queuing;
 global using FiscalOS.Core.Security;
+global using FiscalOS.Core.Transactions;
 global using FiscalOS.Infra.Accounts.Plaid;
 global using FiscalOS.Infra.Authentication;
 global using FiscalOS.Infra.Data;
 global using FiscalOS.Infra.DependencyInjection;
+global using FiscalOS.Infra.Transactions.Plaid;
+global using FiscalOS.ServiceDefaults;
+
+global using Going.Plaid;
+global using Going.Plaid.Webhook;
 
 global using Microsoft.AspNetCore.Mvc;
 global using Microsoft.EntityFrameworkCore;

src/FiscalOS.API/appsettings.Example.json

@@ -21,6 +21,7 @@
   },
   "PlaidClientOptions": {
     "ClientId": "ClientId",
-    "Secret": "Secret"
+    "Secret": "Secret",
+    "Webhook": "Webhook"
   }
 }