Feature/safo6 nrl 721 seed sandbox data

.github/workflows/deploy-account-wide-infra.yml

@@ -51,13 +51,27 @@ jobs:
           echo "${HOME}/.asdf/bin" >> $GITHUB_PATH
           poetry install --no-root
 
+      - name: Build Lambda Layers
+        run: |
+          make build-layers
+          make build-dependency-layer
+
+      - name: Build Seed Sandbox Lambda
+        run: make build-seed-sandbox-lambda
+
       - name: Configure Management Credentials
         uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a #v4.3.1
         with:
           aws-region: eu-west-2
           role-to-assume: ${{ secrets.MGMT_ROLE_ARN }}
           role-session-name: github-actions-ci-${{ inputs.environment }}-${{ github.run_id }}
 
+      - name: Add S3 Permissions to Lambda Layer
+        env:
+          ACCOUNT_NAME: ${{ vars.ACCOUNT_NAME }}
+        run: |
+          make get-s3-perms ENV=${ACCOUNT_NAME}
+
       - name: Retrieve Server Certificates
         env:
           ACCOUNT_NAME: ${{ vars.ACCOUNT_NAME }}
@@ -92,6 +106,11 @@ jobs:
           aws s3 cp terraform/account-wide-infrastructure/$ACCOUNT_NAME/tfplan.txt s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/tfplan.txt
           aws s3 cp terraform/account-wide-infrastructure/modules/glue/files/src.zip s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/glue-src.zip
 
+          aws s3 cp dist/nrlf.zip s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/nrlf.zip
+          aws s3 cp dist/dependency_layer.zip s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/dependency_layer.zip
+          aws s3 cp dist/nrlf_permissions.zip s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/nrlf_permissions.zip
+          aws s3 cp dist/seed_sandbox.zip s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/seed_sandbox.zip
+
   terraform-apply:
     name: Terraform Apply - ${{ inputs.environment }}
     needs: [terraform-plan]
@@ -126,6 +145,12 @@ jobs:
           mkdir -p terraform/account-wide-infrastructure/modules/glue/files
           aws s3 cp s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/glue-src.zip terraform/account-wide-infrastructure/modules/glue/files/src.zip
 
+          mkdir -p dist
+          aws s3 cp s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/nrlf.zip dist/nrlf.zip
+          aws s3 cp s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/dependency_layer.zip dist/dependency_layer.zip
+          aws s3 cp s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/nrlf_permissions.zip dist/nrlf_permissions.zip
+          aws s3 cp s3://nhsd-nrlf--mgmt--ci-data/acc-$ACCOUNT_NAME/${{ github.run_id }}/seed_sandbox.zip dist/seed_sandbox.zip
+
       - name: Retrieve Server Certificates
         env:
           ACCOUNT_NAME: ${{ vars.ACCOUNT_NAME }}

Makefile

@@ -58,7 +58,11 @@ check-deploy: ## check the deploy environment is setup correctly
 check-deploy-warn:
 	@SHOULD_WARN_ONLY=true ./scripts/check-deploy-environment.sh
 
-build: check-warn build-api-packages build-layers build-dependency-layer ## Build the project
+build: check-warn build-api-packages build-layers build-dependency-layer build-seed-sandbox-lambda ## Build the project
+
+build-seed-sandbox-lambda:
+	@echo "Building seed_sandbox Lambda"
+	@cd lambdas/seed_sandbox && make build
 
 build-dependency-layer:
 	@echo "Building Lambda dependency layer"

README.md

@@ -375,8 +375,7 @@ In order to deploy to a sandbox environment (`dev-sandbox`, `qa-sandbox`, `int-s
 
 ### Sandbox database clear and reseed
 
-Any workspace suffixed with `-sandbox` has a small amount of additional infrastructure deployed to clear and reseed the DynamoDB tables (auth and document pointers) using a Lambda running
-on a cron schedule that can be found in the `cron/seed_sandbox` directory in the root of this project. The data used to seed the DynamoDB tables can found in the `cron/seed_sandbox/data` directory.
+The dev and test environments have a small amount of additional infrastructure deployed to clear and reseed specified DynamoDB sandbox tables with realistic data using a Lambda running on an Eventbridge schedule. You can specify the tables to be reseeded in `terraform/account-wide-infrastructure/{env}/lambda\__seed-sandbox.tf.` If you want to perform this manually on an adhoc basis, you can use `./scripts/reset_sandbox_table.py`.
 
 ### Sandbox authorisation
 

lambdas/seed_sandbox/Makefile

@@ -0,0 +1,29 @@
+.PHONY: build clean
+
+build: clean
+	@echo "Building Lambda deployment package..."
+	mkdir -p build
+
+	# Copy the handler
+	cp index.py build/
+
+	# Copy the required scripts
+	mkdir -p build/scripts
+	cp ../../scripts/delete_all_table_items.py build/scripts/
+	cp ../../scripts/seed_sandbox_table.py build/scripts/
+	cp ../../scripts/seed_utils.py build/scripts/
+
+	# Copy the pointer template data
+	mkdir -p build/tests/data/samples
+	cp -r ../../tests/data/samples/*.json build/tests/data/samples/
+
+	# Create the zip file in root dist
+	mkdir -p ../../dist
+	cd build && zip -r ../../../dist/seed_sandbox.zip . -x "*.pyc" -x "__pycache__/*" -x ".DS_Store"
+
+	@echo "✓ Lambda package created: ../../dist/seed_sandbox.zip"
+
+clean:
+	@echo "Cleaning build artifacts..."
+	rm -rf build
+	@echo "✓ Clean complete"

lambdas/seed_sandbox/index.py

@@ -0,0 +1,108 @@
+# flake8: noqa: T201
+
+import json
+import os
+
+from scripts.delete_all_table_items import delete_all_table_items
+from scripts.seed_sandbox_table import seed_sandbox_table
+
+
+def handler(event, context):
+    """
+    Lambda handler that orchestrates the reset of specified pointer tables in the dev & test accounts, deleting all items and reseeding with fresh data .
+
+    The tables to be reset and number of pointers per type can be specified in `terraform/account-wide-infrastructure/{env}/lambda__seed-sandbox.tf`
+
+    """
+    table_names_str = os.environ.get("TABLE_NAMES", "")
+    pointers_per_type = int(os.environ.get("POINTERS_PER_TYPE", "2"))
+
+    if not table_names_str:
+        error_msg = "TABLE_NAMES environment variable is required"
+        print(f"ERROR: {error_msg}")
+        return {"statusCode": 500, "body": json.dumps({"error": error_msg})}
+
+    table_names = [name.strip() for name in table_names_str.split(",") if name.strip()]
+
+    if not table_names:
+        error_msg = "No valid table names provided in TABLE_NAMES"
+        print(f"ERROR: {error_msg}")
+        return {"statusCode": 500, "body": json.dumps({"error": error_msg})}
+
+    print(
+        f"Starting table reset for {len(table_names)} table(s): {', '.join(table_names)}"
+    )
+    print(f"Pointers per type: {pointers_per_type}")
+
+    results = []
+    failed_tables = []
+
+    for table_name in table_names:
+        print(f"\n{'='*60}")
+        print(f"Processing table: {table_name}")
+        print(f"{'='*60}")
+
+        try:
+            print("Step 1: Deleting all items from table...")
+            pointers_deleted_count = delete_all_table_items(table_name=table_name)
+            print(f"✓ Deleted {pointers_deleted_count} items")
+
+            print("Step 2: Seeding table with fresh data...")
+            seed_result = seed_sandbox_table(
+                table_name=table_name,
+                pointers_per_type=pointers_per_type,
+                force=True,
+                write_csv=False,
+            )
+            print(f"✓ Created {seed_result['successful']} pointers")
+
+            results.append(
+                {
+                    "table_name": table_name,
+                    "status": "success",
+                    "pointers_deleted": pointers_deleted_count,
+                    "pointers_created": seed_result["successful"],
+                    "pointers_attempted": seed_result["attempted"],
+                    "pointers_failed": seed_result["failed"],
+                }
+            )
+
+        except Exception as e:
+            error_msg = f"Failed to reset table {table_name}: {str(e)}"
+            print(f"ERROR: {error_msg}")
+            failed_tables.append(table_name)
+            results.append(
+                {
+                    "table_name": table_name,
+                    "status": "failed",
+                    "error": str(e),
+                }
+            )
+
+    if failed_tables:
+        status_code = 500 if len(failed_tables) == len(table_names) else 207
+        message = (
+            f"Failed to reset {len(failed_tables)} table(s): {', '.join(failed_tables)}"
+        )
+    else:
+        status_code = 200
+        message = f"Successfully reset {len(table_names)} table(s)"
+
+    result = {
+        "statusCode": status_code,
+        "body": json.dumps(
+            {
+                "message": message,
+                "tables_processed": len(table_names),
+                "tables_succeeded": len(table_names) - len(failed_tables),
+                "tables_failed": len(failed_tables),
+                "results": results,
+                "pointers_per_type": pointers_per_type,
+            }
+        ),
+    }
+
+    print(f"\n{'='*60}")
+    print(f"RESULT: {message}")
+    print(f"{'='*60}")
+    return result

lambdas/seed_sandbox/tests/test_index.py

@@ -0,0 +1,177 @@
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from lambdas.seed_sandbox.index import handler
+
+
+@pytest.fixture
+def mock_lambda_context():
+    mock_context = MagicMock()
+    mock_context.function_name = "test-function"
+    mock_context.invoked_function_arn = (
+        "arn:aws:lambda:eu-west-2:123456789012:function:test-function"
+    )
+    return mock_context
+
+
+@pytest.fixture
+def mock_env_vars():
+    with patch.dict(
+        "os.environ", {"TABLE_NAMES": "test-table", "POINTERS_PER_TYPE": "2"}
+    ):
+        yield
+
+
+class TestHandler:
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_single_table_reset_success(
+        self, mock_delete, mock_seed, mock_lambda_context, mock_env_vars
+    ):
+        mock_delete.return_value = 10
+        mock_seed.return_value = {"successful": 8, "attempted": 8, "failed": 0}
+
+        result = handler({}, mock_lambda_context)
+
+        assert result["statusCode"] == 200
+        body = json.loads(result["body"])
+        assert body["message"] == "Successfully reset 1 table(s)"
+        assert body["tables_processed"] == 1
+        assert body["tables_succeeded"] == 1
+        assert body["tables_failed"] == 0
+        assert len(body["results"]) == 1
+        assert body["results"][0]["table_name"] == "test-table"
+        assert body["results"][0]["status"] == "success"
+        assert body["results"][0]["pointers_deleted"] == 10
+        assert body["results"][0]["pointers_created"] == 8
+
+        mock_delete.assert_called_once_with(table_name="test-table")
+        mock_seed.assert_called_once_with(
+            table_name="test-table", pointers_per_type=2, force=True, write_csv=False
+        )
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_multiple_table_reset_success(
+        self, mock_delete, mock_seed, mock_lambda_context, mock_env_vars
+    ):
+        with patch.dict(
+            "os.environ",
+            {"TABLE_NAMES": "table1,table2,table3", "POINTERS_PER_TYPE": "5"},
+        ):
+            mock_delete.return_value = 15
+            mock_seed.return_value = {"successful": 20, "attempted": 20, "failed": 0}
+
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 200
+            body = json.loads(result["body"])
+            assert body["message"] == "Successfully reset 3 table(s)"
+            assert body["tables_processed"] == 3
+            assert body["tables_succeeded"] == 3
+            assert body["tables_failed"] == 0
+            assert len(body["results"]) == 3
+            assert all(r["status"] == "success" for r in body["results"])
+
+            assert mock_delete.call_count == 3
+            assert mock_seed.call_count == 3
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_partial_failure(self, mock_delete, mock_seed, mock_lambda_context):
+        with patch.dict(
+            "os.environ",
+            {"TABLE_NAMES": "table1,table2,table3", "POINTERS_PER_TYPE": "2"},
+        ):
+            # First and third tables succeed, second fails during delete
+            mock_delete.side_effect = [10, Exception("Access denied"), 5]
+            mock_seed.side_effect = [
+                {"successful": 8, "attempted": 8, "failed": 0},
+                {"successful": 8, "attempted": 8, "failed": 0},
+            ]
+
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 207
+            body = json.loads(result["body"])
+            assert "Failed to reset 1 table(s): table2" in body["message"]
+            assert body["tables_processed"] == 3
+            assert body["tables_succeeded"] == 2
+            assert body["tables_failed"] == 1
+            assert len(body["results"]) == 3
+            assert body["results"][0]["status"] == "success"
+            assert body["results"][1]["status"] == "failed"
+            assert body["results"][1]["error"] == "Access denied"
+            assert body["results"][2]["status"] == "success"
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_complete_failure(self, mock_delete, mock_seed, mock_lambda_context):
+        with patch.dict(
+            "os.environ", {"TABLE_NAMES": "table1,table2", "POINTERS_PER_TYPE": "2"}
+        ):
+            mock_delete.side_effect = Exception("Database error")
+
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 500
+            body = json.loads(result["body"])
+            assert "Failed to reset 2 table(s)" in body["message"]
+            assert body["tables_processed"] == 2
+            assert body["tables_succeeded"] == 0
+            assert body["tables_failed"] == 2
+
+    def test_missing_table_names_env_var(self, mock_lambda_context):
+        with patch.dict("os.environ", {}, clear=True):
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 500
+            body = json.loads(result["body"])
+            assert body["error"] == "TABLE_NAMES environment variable is required"
+
+    def test_empty_table_names(self, mock_lambda_context):
+        with patch.dict("os.environ", {"TABLE_NAMES": "   ", "POINTERS_PER_TYPE": "2"}):
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 500
+            body = json.loads(result["body"])
+            assert body["error"] == "No valid table names provided in TABLE_NAMES"
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_table_names_with_whitespace(
+        self, mock_delete, mock_seed, mock_lambda_context
+    ):
+        with patch.dict(
+            "os.environ",
+            {"TABLE_NAMES": " table1 , table2 , ", "POINTERS_PER_TYPE": "2"},
+        ):
+            mock_delete.return_value = 5
+            mock_seed.return_value = {"successful": 4, "attempted": 4, "failed": 0}
+
+            result = handler({}, mock_lambda_context)
+
+            assert result["statusCode"] == 200
+            body = json.loads(result["body"])
+            assert body["tables_processed"] == 2
+            assert body["results"][0]["table_name"] == "table1"
+            assert body["results"][1]["table_name"] == "table2"
+
+    @patch("lambdas.seed_sandbox.index.seed_sandbox_table")
+    @patch("lambdas.seed_sandbox.index.delete_all_table_items")
+    def test_seed_with_failures(
+        self, mock_delete, mock_seed, mock_lambda_context, mock_env_vars
+    ):
+        mock_delete.return_value = 5
+        mock_seed.return_value = {"successful": 6, "attempted": 8, "failed": 2}
+
+        result = handler({}, mock_lambda_context)
+
+        assert result["statusCode"] == 200
+        body = json.loads(result["body"])
+        assert body["results"][0]["pointers_created"] == 6
+        assert body["results"][0]["pointers_attempted"] == 8
+        assert body["results"][0]["pointers_failed"] == 2