From 8e6a2d317d370ee724f0ad6a04a479c6ff14b83d Mon Sep 17 00:00:00 2001 From: Adnan Hassan Date: Tue, 7 Jan 2025 17:40:58 +0000 Subject: [PATCH] prettier --- iac/main/resources/raw.yml | 52 +++++++++++++++++++ package-lock.json | 53 ++++++++++++++++++++ package.json | 1 + src/handlers/trigger-txma-crawler/handler.ts | 22 ++++++++ src/shared/clients.ts | 3 ++ 5 files changed, 131 insertions(+) create mode 100644 src/handlers/trigger-txma-crawler/handler.ts diff --git a/iac/main/resources/raw.yml b/iac/main/resources/raw.yml index 7cbc8adff..da7a853ed 100644 --- a/iac/main/resources/raw.yml +++ b/iac/main/resources/raw.yml @@ -168,3 +168,55 @@ SplunkPerformanceIndexSecret: Description: 'a secret to store username and password for access the performance index' Name: !Sub ${Environment}-SplunkPerformanceIndexSecret SecretString: '{"username":"xxx","password":"xxx"}' + +RawLayerTestCrawler: + Type: AWS::Glue::Crawler + Properties: + Name: txma_raw_layer_test + Role: !GetAtt RawGlueCrawlerRole.Arn + Targets: + S3Targets: + - Path: !Sub 's3://${RawLayerBucket}/txma/' + DatabaseName: !Ref RawGlueDatabase + CrawlerSecurityConfiguration: !Ref GlueSecurityConfig + RecrawlPolicy: + RecrawlBehavior: CRAWL_EVERYTHING + SchemaChangePolicy: + UpdateBehavior: UPDATE_IN_DATABASE + DeleteBehavior: DELETE_FROM_DATABASE + +TxmaCrawlerInvoke: + # checkov:skip=CKV_AWS_116: DLQ not needed as this lambda failing will cause state machine to fail + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub trigger-txma-crawler-${Environment} + Handler: trigger-txma-crawler.handler + Policies: + - AWSLambdaBasicExecutionRole + - Statement: + - Effect: Allow + Action: + - glue:StartCrawler + - glue:UpdateCrawler + - glue:GetCrawler + Resource: '*' + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogStream + Resource: 'arn:aws:logs:*:*:*' + ReservedConcurrentExecutions: 10 + Environment: + # checkov:skip=CKV_AWS_173: These environment variables do not require encryption + Variables: + ENVIRONMENT: !Ref Environment + Tags: + Environment: !Ref Environment + VpcConfig: + SecurityGroupIds: + - !Ref LambdaSecurityGroup + SubnetIds: + - !Ref SubnetForDAP1 + - !Ref SubnetForDAP2 + - !Ref SubnetForDAP3 diff --git a/package-lock.json b/package-lock.json index 495976cd0..df12d3037 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@aws-sdk/client-cognito-identity-provider": "^3.696.0", "@aws-sdk/client-eventbridge": "^3.696.0", "@aws-sdk/client-firehose": "^3.696.0", + "@aws-sdk/client-glue": "3.696.0", "@aws-sdk/client-lambda": "^3.697.0", "@aws-sdk/client-quicksight": "^3.696.0", "@aws-sdk/client-redshift-data": "^3.696.0", @@ -621,6 +622,58 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-glue": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-glue/-/client-glue-3.696.0.tgz", + "integrity": "sha512-gAEjwbOST4UoHvXihAmEvOlsvfE29CKlBkC/ZpjZt+G4T9g4krtQrOk3tazOg9KXOtt+vYLkaqflKUn1+Ex4sQ==", + "dev": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.696.0", + "@aws-sdk/client-sts": "3.696.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@aws-sdk/client-lambda": { "version": "3.697.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.697.0.tgz", diff --git a/package.json b/package.json index 443f16359..630eb3ae6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@aws-sdk/client-quicksight": "^3.696.0", "@aws-sdk/client-redshift-data": "^3.696.0", "@aws-sdk/client-s3": "^3.697.0", + "@aws-sdk/client-glue": "3.696.0", "@aws-sdk/client-secrets-manager": "^3.696.0", "@aws-sdk/client-sfn": "^3.696.0", "@aws-sdk/client-sqs": "^3.696.0", diff --git a/src/handlers/trigger-txma-crawler/handler.ts b/src/handlers/trigger-txma-crawler/handler.ts new file mode 100644 index 000000000..eb5cb11bd --- /dev/null +++ b/src/handlers/trigger-txma-crawler/handler.ts @@ -0,0 +1,22 @@ +import { StartCrawlerCommand, UpdateCrawlerCommand } from '@aws-sdk/client-glue'; +import { glueClient } from '../../shared/clients'; +import { getLoggerAndMetrics } from '../../shared/powertools'; + +export const { logger, metrics } = getLoggerAndMetrics('lambda/trigger-txma-crawler'); + +export const handler = async (): Promise => { + const updateCrawlerCommand = new UpdateCrawlerCommand({ + Name: 'some-crawler-name', + Targets: { + S3Targets: [{ Path: 'somePath' }], + }, + }); + await glueClient.send(updateCrawlerCommand); + + const startCrawlerCommand = new StartCrawlerCommand({ + Name: 'some-crawler-name', + }); + + await glueClient.send(startCrawlerCommand); + return null; +}; diff --git a/src/shared/clients.ts b/src/shared/clients.ts index 3a31b579c..f15e9326e 100644 --- a/src/shared/clients.ts +++ b/src/shared/clients.ts @@ -1,4 +1,5 @@ import { S3Client } from '@aws-sdk/client-s3'; +import { GlueClient } from '@aws-sdk/client-glue'; import { CloudWatchLogsClient } from '@aws-sdk/client-cloudwatch-logs'; import { AWS_CLIENT_BASE_CONFIG } from './constants'; import { LambdaClient } from '@aws-sdk/client-lambda'; @@ -39,3 +40,5 @@ export const secretsManagerClient = new SecretsManagerClient(AWS_CLIENT_BASE_CON export const sqsClient = new SQSClient(AWS_CLIENT_BASE_CONFIG); export const sfnClient = new SFNClient(AWS_CLIENT_BASE_CONFIG); + +export const glueClient = new GlueClient(AWS_CLIENT_BASE_CONFIG);