CCM-17047: Add client , campaign and supplier name to Grafana dashboard

.env.template

@@ -1,9 +1,12 @@
-ENVIRONMENT=$ENV_NAME
-API_KEY=
-HEADERAUTH=
 PR_NUMBER=prxx # remove if needs to run against main
-NHSD_APIM_TOKEN=
-PROXY_NAME=
+GITHUB_TOKEN= # Your github Personal Access Token (PAT)
+
+
+# The variables below are used for End to End tests
+PROXY_NAME= # information about the proxy name can be found in the tests/e2e-tests/README.md
+
+
+
 # * nhs-notify-supplier--internal-dev--nhs-notify-supplier
 # * nhs-notify-supplier--internal-dev--nhs-notify-supplier-PR-XX
 # * nhs-notify-supplier--ref--nhs-notify-supplier -- ref env

README.md

@@ -1,3 +1,4 @@
+<!-- vale off -->
 # NHS Notify Supplier API
 
 [![1. CI/CD pull request](https://github.com/NHSDigital/nhs-notify-supplier-api/actions/workflows/cicd-1-pull-request.yaml/badge.svg)](https://github.com/NHSDigital/nhs-notify-supplier-api/actions/workflows/cicd-1-pull-request.yaml)
@@ -75,9 +76,50 @@ New developers of the NHS Notify Supplier API should understand the below.
 
 #### Prerequisites and Configuration
 
-- Utilised the devcontainer, for pre reqs and configuration.
-- You should open in a devcontainer or a Github workspaces.
-- By default it will run `make config` when the container is first setup
+- create the file `~/.aws/config` with the following contents:
+
+  ```dsconfig
+  [profile ]
+  region = eu-west-2
+  output = json
+
+  [profile supplier-dev]
+  sso_start_url = https://d-9c67018f89.awsapps.com/start#/
+  sso_region = eu-west-2
+  sso_account_id = 820178564574
+  sso_role_name = nhs-notify-bc-developer
+  region = eu-west-2
+  output = json
+
+  [profile supplier-nonprod]
+  sso_start_url = https://d-9c67018f89.awsapps.com/start#/
+  sso_region = eu-west-2
+  sso_account_id = 885964308133
+  sso_role_name = nhs-notify-bc-developer
+  region = eu-west-2
+  output = json
+  ```
+
+- in your `~/.bashrc` or `~/.zshrc` add the export `export AWS_PROFILE=supplier-dev`, or whichever profile you need
+- In the project's root directory create an `.env` file based on the `.env.template` file and fille variables as needed.
+- create the file `~/.npmrc` with the contents:
+
+   ```dsconfig
+    # Authenticate to GitHub Packages for github.com
+    //npm.pkg.github.com/:_authToken=<Insert your Github PAT (Personal Access Token)>
+
+
+    # Package is scoped under @org, set registry for that scope
+    @nhsdigital:registry=https://npm.pkg.github.com
+   ```
+
+- Install `node` (to run `npm install` and build the project)
+- install `aws cli` to be able to connect to AWS (needed for some tests)
+- If AWS CLI calls are blocked by a firewall (e.g. Zscaler), you need to add the custom certificates in the location `/scripts/devcontainer/custom-ca-certs`
+- install `docker` or `Rancher` for containerisation
+  - Utilised the devcontainer, for pre reqs and configuration.
+  - You should open in a devcontainer or a Github workspaces. (In VSCode -> open control palet -> "Dev containers: rebuild without cache and reopen in container")
+  - By default it will run `make config` when the container is first setup
 
 ##### SDKs
 
@@ -151,3 +193,5 @@ Import the files into postman
 Select a target environment in postman
 Run the collection
 The collections must be kept in sync manually
+
+<!-- vale on -->

lambdas/supplier-allocator/src/handler/__tests__/allocate-handler.test.ts

@@ -69,8 +69,8 @@ function createPreparedV1Event(
       requestItemId: "requestItem1",
       requestItemPlanId: "requestItemPlan1",
       clientId: "client1",
-      campaignId: "campaign1",
-      templateId: "template1",
+      campaignId: overrides.campaignId ?? "campaign1",
+      templateId: overrides.templateId ?? "template1",
       url: overrides.url ?? "s3://letterDataBucket/letter1.pdf",
       sha256Hash:
         "3a7bd3e2360a3d29eea436fcfb7e44c735d117c8f2f1d2d1e4f6e8f7e6e8f7e6",
@@ -230,6 +230,40 @@ describe("createSupplierAllocatorHandler", () => {
     });
   });
 
+  test("parses SNS notification and sends message to SQS queue for v2 event without a campaignId and templateId", async () => {
+    const preparedEvent = createPreparedV2Event({
+      campaignId: "",
+      templateId: "",
+    });
+    const evt: SQSEvent = createSQSEvent([
+      createSqsRecord("msg1", JSON.stringify(preparedEvent)),
+    ]);
+
+    setupDefaultMocks();
+    process.env.UPSERT_LETTERS_QUEUE_URL = "https://sqs.test.queue";
+
+    const handler = createSupplierAllocatorHandler(mockedDeps);
+    const result = await handler(evt, {} as any, {} as any);
+
+    expect(result).toBeDefined();
+    if (!result) throw new Error("expected BatchResponse, got void");
+
+    expect(result.batchItemFailures).toHaveLength(0);
+
+    expect(mockSqsClient.send).toHaveBeenCalledTimes(1);
+    const sendCall = (mockSqsClient.send as jest.Mock).mock.calls[0][0];
+    expect(sendCall).toBeInstanceOf(SendMessageCommand);
+
+    const messageBody = JSON.parse(sendCall.input.MessageBody);
+    expect(messageBody.letterEvent).toEqual(preparedEvent);
+    expect(messageBody.supplierSpec).toEqual({
+      supplierId: "supplier1",
+      specId: "spec1",
+      priority: 1,
+      billingId: "billing1",
+    });
+  });
+
   test("parses SNS notification and sends message to SQS queue for v1 event", async () => {
     const preparedEvent = createPreparedV1Event();
 

lambdas/supplier-allocator/src/handler/allocate-handler.ts

@@ -144,6 +144,33 @@ function emitMetrics(
   }
 }
 
+/**
+ * NOTE: `groupId` needs to match the groupId in the function {@link upsert-handler#mapToInsertLetter}
+ * so the value always needs to be updated in both places
+ */
+function emitDataMetrics(
+  letterEvent: PreparedEvents,
+  supplier: string,
+  metricKey: string,
+  deps: Deps,
+) {
+  const namespace = "supplier-allocator";
+  const { campaignId, clientId, templateId } = letterEvent.data;
+  const dimensions: Record<string, string> = {
+    Supplier: supplier,
+    ClientId: clientId,
+    CampaignId: campaignId || "unknown",
+    TemplateId: templateId || "unknown",
+    GroupId: `${clientId}_${campaignId}_${templateId}`,
+  };
+  const metric: MetricEntry = {
+    key: metricKey,
+    value: 1,
+    unit: Unit.Count,
+  };
+  deps.logger.info(buildEMFObject(namespace, dimensions, metric));
+}
+
 export default function createSupplierAllocatorHandler(deps: Deps): SQSHandler {
   return async (event: SQSEvent) => {
     const batchItemFailures: SQSBatchItemFailure[] = [];
@@ -153,8 +180,9 @@ export default function createSupplierAllocatorHandler(deps: Deps): SQSHandler {
     const tasks = event.Records.map(async (record) => {
       let supplier = "unknown";
       let priority = "unknown";
+      let letterEvent: PreparedEvents | undefined;
       try {
-        const letterEvent: unknown = JSON.parse(record.body);
+        letterEvent = JSON.parse(record.body) as PreparedEvents;
 
         deps.logger.info({
           description: "Extracted letter event",
@@ -163,8 +191,8 @@ export default function createSupplierAllocatorHandler(deps: Deps): SQSHandler {
 
         validateType(letterEvent);
 
-        const supplierSpec = getSupplier(letterEvent as PreparedEvents, deps);
-        await getSupplierFromConfig(letterEvent as PreparedEvents, deps);
+        const supplierSpec = getSupplier(letterEvent, deps);
+        await getSupplierFromConfig(letterEvent, deps);
 
         supplier = supplierSpec.supplierId;
         priority = String(supplierSpec.priority);
@@ -199,6 +227,7 @@ export default function createSupplierAllocatorHandler(deps: Deps): SQSHandler {
         );
 
         incrementMetric(perAllocationSuccess, supplier, priority);
+        emitDataMetrics(letterEvent, supplier, "extra_data_dimensions", deps);
       } catch (error) {
         deps.logger.error({
           description: "Error processing allocation of record",

lambdas/upsert-letter/src/handler/__tests__/upsert-handler.test.ts

@@ -11,6 +11,8 @@ import {
   $LetterStatusChangeEvent,
   LetterStatusChangeEvent,
 } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
+import { MetricStatus, buildEMFObject } from "@internal/helpers";
+import { Unit } from "aws-embedded-metrics";
 import createUpsertLetterHandler from "../upsert-handler";
 import { Deps } from "../../config/deps";
 import { EnvVars } from "../../config/env";
@@ -24,6 +26,15 @@ jest.mock("@aws-lambda-powertools/idempotency", () => {
   };
 });
 
+// mock the buildEMFObject function to just return its input so we can assert on it
+jest.mock("@internal/helpers", () => {
+  const original = jest.requireActual("@internal/helpers");
+  return {
+    ...original,
+    buildEMFObject: jest.fn(original.buildEMFObject),
+  };
+});
+
 const renderingSchemaVersion: string =
   packageJson.dependencies[
     "@nhsdigital/nhs-notify-event-schemas-letter-rendering"
@@ -190,26 +201,6 @@ function createSupplierStatusChangeEvent(
   });
 }
 
-// Mock aws-embedded-metrics
-let mockMetrics: any;
-jest.mock("aws-embedded-metrics", () => ({
-  metricScope: (
-    handler: (metrics: any) => (event: SQSEvent) => Promise<any>,
-  ) => {
-    return async (event: SQSEvent) => {
-      mockMetrics = {
-        setNamespace: jest.fn(),
-        putDimensions: jest.fn(),
-        putMetric: jest.fn(),
-      };
-      return handler(mockMetrics)(event);
-    };
-  },
-  Unit: {
-    Count: "Count",
-  },
-}));
-
 describe("createUpsertLetterHandler", () => {
   const mockedDeps: jest.Mocked<Deps> = {
     letterRepo: {
@@ -280,7 +271,7 @@ describe("createUpsertLetterHandler", () => {
     expect(insertedV2Letter.billingRef).toBe("spec1");
     expect(insertedV2Letter.url).toBe("s3://letterDataBucket/letter1.pdf");
     expect(insertedV2Letter.status).toBe("PENDING");
-    expect(insertedV2Letter.groupId).toBe("client1campaign1template1");
+    expect(insertedV2Letter.groupId).toBe("client1_campaign1_template1");
     expect(insertedV2Letter.source).toBe("/data-plane/letter-rendering/test");
     expect(insertedV2Letter.specificationBillingId).toBe("billing1");
     expect(insertedV2Letter.priority).toBe(10);
@@ -293,7 +284,7 @@ describe("createUpsertLetterHandler", () => {
     expect(insertedV1Letter.billingRef).toBe("spec2");
     expect(insertedV1Letter.url).toBe("s3://letterDataBucket/letter1.pdf");
     expect(insertedV1Letter.status).toBe("PENDING");
-    expect(insertedV1Letter.groupId).toBe("client1campaign1template1");
+    expect(insertedV1Letter.groupId).toBe("client1_campaign1_template1");
     expect(insertedV1Letter.source).toBe("/data-plane/letter-rendering/test");
     expect(insertedV1Letter.specificationBillingId).toBe("billing2");
     expect(insertedV1Letter.priority).toBe(10);
@@ -306,19 +297,17 @@ describe("createUpsertLetterHandler", () => {
     expect(updatedLetter.reasonCode).toBe("R07");
     expect(updatedLetter.reasonText).toBe("No such address");
     expect(updatedLetter.supplierId).toBe("supplier1");
-    expect(mockMetrics.setNamespace).toHaveBeenCalledWith("upsertLetter");
-    expect(mockMetrics.putDimensions).toHaveBeenCalledWith({
-      Supplier: "supplier1",
-    });
-    expect(mockMetrics.putMetric).toHaveBeenCalledWith(
-      "MessagesProcessed",
-      2,
-      "Count",
-    );
-    expect(mockMetrics.putMetric).toHaveBeenCalledWith(
-      "MessagesProcessed",
-      1,
-      "Count",
+    expect(buildEMFObject as jest.Mock).toHaveBeenCalledWith(
+      "upsertLetter",
+      {
+        Supplier: "supplier1",
+        GroupId: "client1_campaign1_template1",
+      },
+      expect.objectContaining({
+        key: MetricStatus.Success,
+        value: 1,
+        unit: Unit.Count,
+      }),
     );
   });
 
@@ -376,14 +365,17 @@ describe("createUpsertLetterHandler", () => {
 
     await createUpsertLetterHandler(mockedDeps)(evt, {} as any, {} as any);
 
-    expect(mockMetrics.setNamespace).toHaveBeenCalledWith("upsertLetter");
-    expect(mockMetrics.putDimensions).toHaveBeenCalledWith({
-      Supplier: "unknown",
-    });
-    expect(mockMetrics.putMetric).toHaveBeenCalledWith(
-      "MessagesProcessed",
-      1,
-      "Count",
+    expect(buildEMFObject as jest.Mock).toHaveBeenCalledWith(
+      "upsertLetter",
+      {
+        Supplier: "unknown",
+        GroupId: "unknown",
+      },
+      expect.objectContaining({
+        key: MetricStatus.Success,
+        value: 1,
+        unit: Unit.Count,
+      }),
     );
   });
 
@@ -410,14 +402,18 @@ describe("createUpsertLetterHandler", () => {
       }),
     );
     expect(mockedDeps.letterRepo.putLetter).not.toHaveBeenCalled();
-    expect(mockMetrics.setNamespace).toHaveBeenCalledWith("upsertLetter");
-    expect(mockMetrics.putDimensions).toHaveBeenCalledWith({
-      Supplier: "unknown",
-    });
-    expect(mockMetrics.putMetric).toHaveBeenCalledWith(
-      "MessageFailed",
-      1,
-      "Count",
+    // replace these
+    expect(buildEMFObject as jest.Mock).toHaveBeenCalledWith(
+      "upsertLetter",
+      {
+        Supplier: "unknown",
+        GroupId: "unknown",
+      },
+      expect.objectContaining({
+        key: MetricStatus.Failure,
+        value: 1,
+        unit: Unit.Count,
+      }),
     );
   });
 
@@ -506,14 +502,18 @@ describe("createUpsertLetterHandler", () => {
         messageId: "bad-event",
       }),
     );
-    expect(mockMetrics.setNamespace).toHaveBeenCalledWith("upsertLetter");
-    expect(mockMetrics.putDimensions).toHaveBeenCalledWith({
-      Supplier: "unknown",
-    });
-    expect(mockMetrics.putMetric).toHaveBeenCalledWith(
-      "MessageFailed",
-      1,
-      "Count",
+    // replace these
+    expect(buildEMFObject as jest.Mock).toHaveBeenCalledWith(
+      "upsertLetter",
+      {
+        Supplier: "unknown",
+        GroupId: "unknown",
+      },
+      expect.objectContaining({
+        key: MetricStatus.Failure,
+        value: 1,
+        unit: Unit.Count,
+      }),
     );
   });