prettier

iac/main/resources/raw.yml

@@ -168,3 +168,55 @@ SplunkPerformanceIndexSecret:
     Description: 'a secret to store username and password for access the performance index'
     Name: !Sub ${Environment}-SplunkPerformanceIndexSecret
     SecretString: '{"username":"xxx","password":"xxx"}'
+
+RawLayerTestCrawler:
+  Type: AWS::Glue::Crawler
+  Properties:
+    Name: txma_raw_layer_test
+    Role: !GetAtt RawGlueCrawlerRole.Arn
+    Targets:
+      S3Targets:
+        - Path: !Sub 's3://${RawLayerBucket}/txma/'
+    DatabaseName: !Ref RawGlueDatabase
+    CrawlerSecurityConfiguration: !Ref GlueSecurityConfig
+    RecrawlPolicy:
+      RecrawlBehavior: CRAWL_EVERYTHING
+    SchemaChangePolicy:
+      UpdateBehavior: UPDATE_IN_DATABASE
+      DeleteBehavior: DELETE_FROM_DATABASE
+
+TxmaCrawlerInvoke:
+  # checkov:skip=CKV_AWS_116: DLQ not needed as this lambda failing will cause state machine to fail
+  Type: AWS::Serverless::Function
+  Properties:
+    FunctionName: !Sub trigger-txma-crawler-${Environment}
+    Handler: trigger-txma-crawler.handler
+    Policies:
+      - AWSLambdaBasicExecutionRole
+      - Statement:
+          - Effect: Allow
+            Action:
+              - glue:StartCrawler
+              - glue:UpdateCrawler
+              - glue:GetCrawler
+            Resource: '*'
+          - Effect: Allow
+            Action:
+              - logs:CreateLogGroup
+              - logs:CreateLogStream
+              - logs:PutLogStream
+            Resource: 'arn:aws:logs:*:*:*'
+    ReservedConcurrentExecutions: 10
+    Environment:
+      # checkov:skip=CKV_AWS_173: These environment variables do not require encryption
+      Variables:
+        ENVIRONMENT: !Ref Environment
+    Tags:
+      Environment: !Ref Environment
+    VpcConfig:
+      SecurityGroupIds:
+        - !Ref LambdaSecurityGroup
+      SubnetIds:
+        - !Ref SubnetForDAP1
+        - !Ref SubnetForDAP2
+        - !Ref SubnetForDAP3

package.json

@@ -16,6 +16,7 @@
     "@aws-sdk/client-quicksight": "^3.696.0",
     "@aws-sdk/client-redshift-data": "^3.696.0",
     "@aws-sdk/client-s3": "^3.697.0",
+    "@aws-sdk/client-glue": "3.696.0",
     "@aws-sdk/client-secrets-manager": "^3.696.0",
     "@aws-sdk/client-sfn": "^3.696.0",
     "@aws-sdk/client-sqs": "^3.696.0",

src/handlers/trigger-txma-crawler/handler.ts

@@ -0,0 +1,22 @@
+import { StartCrawlerCommand, UpdateCrawlerCommand } from '@aws-sdk/client-glue';
+import { glueClient } from '../../shared/clients';
+import { getLoggerAndMetrics } from '../../shared/powertools';
+
+export const { logger, metrics } = getLoggerAndMetrics('lambda/trigger-txma-crawler');
+
+export const handler = async (): Promise<unknown> => {
+  const updateCrawlerCommand = new UpdateCrawlerCommand({
+    Name: 'some-crawler-name',
+    Targets: {
+      S3Targets: [{ Path: 'somePath' }],
+    },
+  });
+  await glueClient.send(updateCrawlerCommand);
+
+  const startCrawlerCommand = new StartCrawlerCommand({
+    Name: 'some-crawler-name',
+  });
+
+  await glueClient.send(startCrawlerCommand);
+  return null;
+};

src/shared/clients.ts

@@ -1,4 +1,5 @@
 import { S3Client } from '@aws-sdk/client-s3';
+import { GlueClient } from '@aws-sdk/client-glue';
 import { CloudWatchLogsClient } from '@aws-sdk/client-cloudwatch-logs';
 import { AWS_CLIENT_BASE_CONFIG } from './constants';
 import { LambdaClient } from '@aws-sdk/client-lambda';
@@ -39,3 +40,5 @@ export const secretsManagerClient = new SecretsManagerClient(AWS_CLIENT_BASE_CON
 export const sqsClient = new SQSClient(AWS_CLIENT_BASE_CONFIG);
 
 export const sfnClient = new SFNClient(AWS_CLIENT_BASE_CONFIG);
+
+export const glueClient = new GlueClient(AWS_CLIENT_BASE_CONFIG);