Data Lake 3 - mlockwood/aws GitHub Wiki
AWSTemplateFormatVersion: "2010-09-09"
Description: "The AWS CloudFormation template for deployment of the AWS Data Lake resources"
Parameters: UserPoolId: Type: String Description: Id of data lake Amazon Cognito User Pool SourceS3Bucket: Type: String SourceKeyPrefix: Type: String
Resources:
DataLakeKmsKey:
Type: "AWS::KMS::Key"
Properties:
Description: "Data Lake KMS Key"
KeyPolicy:
Version: "2012-10-17"
Id: "data-lake-key-default-1"
Statement:
-
Sid: "Allow administration of the key"
Effect: "Allow"
Principal:
AWS: !Join ["", ["arn:aws:iam::", Ref: "AWS::AccountId", ":root"]]
Action:
- "kms:*"
Resource: "*"
DataLakeKmsKeyAlias:
Type: AWS::KMS::Alias
Properties:
AliasName: !Join ["", ["alias/datalake-", Ref: "AWS::Region" ]]
TargetKeyId:
Ref: DataLakeKmsKey
DataLakeLoggingRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-logging-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeLoggingPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the logging microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
- !Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/datalake/audit-log:*"]]
- !Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-logging-service:*"]]
-
Effect: "Allow"
Action:
- "dynamodb:GetItem"
- "dynamodb:PutItem"
- "dynamodb:Query"
- "dynamodb:Scan"
- "dynamodb:UpdateItem"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
Roles:
-
Ref: "DataLakeLoggingRole"
DataLakeLoggingService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-logging-service.zip"]]
Description: "A data lake microservice function for logging functionality"
FunctionName: "data-lake-logging-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeLoggingRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "60"
DataLakeAdminRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-admin-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeAdminPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the admin microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-admin-service:*"]]
-
Effect: "Allow"
Action:
- "*"
Resource:
!Join ["", ["arn:aws:cognito-idp:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":userpool/", Ref: UserPoolId]]
-
Effect: "Allow"
Action:
- "dynamodb:BatchGetItem"
- "dynamodb:BatchWriteItem"
- "dynamodb:DeleteItem"
- "dynamodb:GetItem"
- "dynamodb:PutItem"
- "dynamodb:Query"
- "dynamodb:Scan"
- "dynamodb:UpdateItem"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-keys"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeLoggingService"
- "Arn"
-
Effect: "Allow"
Action:
- "kms:Encrypt"
- "kms:Decrypt"
- "kms:ReEncrypt*"
- "kms:GenerateDataKey*"
- "kms:DescribeKey"
Resource:
Fn::GetAtt:
- "DataLakeKmsKey"
- "Arn"
Roles:
-
Ref: "DataLakeAdminRole"
DataLakeAdminService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-admin-service.zip"]]
Description: "A data lake microservice function for admin functionality"
FunctionName: "data-lake-admin-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeAdminRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "60"
DataLakeSearchRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-search-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeSearchPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the search microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-search-service:*"]]
-
Effect: "Allow"
Action:
- "es:ESHttpPost"
- "es:ESHttpDelete"
- "es:ESHttpPut"
- "es:ESHttpGet"
Resource:
!Join ["", ["arn:aws:es:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":domain/data-lake/*"]]
-
Effect: "Allow"
Action:
- "dynamodb:GetItem"
- "dynamodb:Query"
- "dynamodb:Scan"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeAdminService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeLoggingService"
- "Arn"
Roles:
-
Ref: "DataLakeSearchRole"
DataLakeSearchService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-search-service.zip"]]
Description: "A data lake microservice function for interacting with the elasticsearch cluster"
FunctionName: "data-lake-search-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeSearchRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "60"
DataLakeManifestRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-manifest-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeManifestPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the manifest microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-manifest-service:*"]]
-
Effect: "Allow"
Action:
- "dynamodb:BatchGetItem"
- "dynamodb:BatchWriteItem"
- "dynamodb:DeleteItem"
- "dynamodb:GetItem"
- "dynamodb:PutItem"
- "dynamodb:Query"
- "dynamodb:Scan"
- "dynamodb:UpdateItem"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-cart"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-datasets"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeAdminService"
- "Arn"
-
Effect: "Allow"
Action:
- "s3:GetObject"
Resource:
- "arn:aws:s3:::*"
-
Effect: "Allow"
Action:
- "s3:PutObject"
Resource:
- !Join ["", ["arn:aws:s3:::data-lake-", Ref: "AWS::Region", "-", Ref: "AWS::AccountId", "/*" ]]
-
Effect: "Allow"
Action:
- "kms:Encrypt"
- "kms:Decrypt"
- "kms:ReEncrypt*"
- "kms:GenerateDataKey*"
- "kms:DescribeKey"
Resource:
Fn::GetAtt:
- "DataLakeKmsKey"
- "Arn"
Roles:
-
Ref: "DataLakeManifestRole"
DataLakeManifestService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-manifest-service.zip"]]
Description: "A data lake microservice function for manifest functionality"
FunctionName: "data-lake-manifest-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeManifestRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "300"
DataLakeCartRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-cart-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeCartPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the cart microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-cart-service:*"]]
-
Effect: "Allow"
Action:
- "dynamodb:BatchGetItem"
- "dynamodb:BatchWriteItem"
- "dynamodb:DeleteItem"
- "dynamodb:GetItem"
- "dynamodb:PutItem"
- "dynamodb:Query"
- "dynamodb:Scan"
- "dynamodb:UpdateItem"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-cart"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-packages"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeAdminService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeLoggingService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeManifestService"
- "Arn"
Roles:
-
Ref: "DataLakeCartRole"
DataLakeCartService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-cart-service.zip"]]
Description: "A data lake microservice function for cart functionality"
FunctionName: "data-lake-cart-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeCartRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "60"
DataLakePackagesRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-packages-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakePackagesPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the package microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-package-service:*"]]
-
Effect: "Allow"
Action:
- "dynamodb:BatchGetItem"
- "dynamodb:BatchWriteItem"
- "dynamodb:DeleteItem"
- "dynamodb:GetItem"
- "dynamodb:PutItem"
- "dynamodb:Query"
- "dynamodb:Scan"
- "dynamodb:UpdateItem"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-packages"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-datasets"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-metadata"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeAdminService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeSearchService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeManifestService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeLoggingService"
- "Arn"
-
Effect: "Allow"
Action:
- "s3:GetObject"
- "s3:PutObject"
Resource:
- !Join ["", ["arn:aws:s3:::data-lake-", Ref: "AWS::Region", "-", Ref: "AWS::AccountId", "/*" ]]
-
Effect: "Allow"
Action:
- "kms:Encrypt"
- "kms:Decrypt"
- "kms:ReEncrypt*"
- "kms:GenerateDataKey*"
- "kms:DescribeKey"
Resource:
Fn::GetAtt:
- "DataLakeKmsKey"
- "Arn"
Roles:
-
Ref: "DataLakePackagesRole"
DataLakePackagesService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-package-service.zip"]]
Description: "A data lake microservice function for package functionality"
FunctionName: "data-lake-package-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakePackagesRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "300"
DataLakeProfileRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-profile-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeProfilePolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the profile microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-profile-service:*"]]
-
Effect: "Allow"
Action:
- "dynamodb:GetItem"
- "dynamodb:Query"
- "dynamodb:Scan"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
-
Effect: "Allow"
Action:
- "lambda:InvokeFunction"
Resource:
-
Fn::GetAtt:
- "DataLakeAdminService"
- "Arn"
-
Fn::GetAtt:
- "DataLakeLoggingService"
- "Arn"
-
Effect: "Allow"
Action:
- "*"
Resource:
!Join ["", ["arn:aws:cognito-idp:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":userpool/", Ref: UserPoolId]]
-
Effect: "Allow"
Action:
- "kms:Encrypt"
- "kms:Decrypt"
- "kms:ReEncrypt*"
- "kms:GenerateDataKey*"
- "kms:DescribeKey"
Resource:
Fn::GetAtt:
- "DataLakeKmsKey"
- "Arn"
Roles:
-
Ref: "DataLakeProfileRole"
DataLakeProfileService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-profile-service.zip"]]
Description: "A data lake microservice function for profile functionality"
FunctionName: "data-lake-profile-service"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeProfileRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "300"
DataLakeAuthorizerRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: !Join ["-", ["data-lake-authorizer-role", Ref: "AWS::Region" ]]
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "lambda.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
DataLakeAuthorizerPolicy:
Type: "AWS::IAM::ManagedPolicy"
Properties:
Description: "Data Lake policy for the authorizer microservice Lambda function."
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Action:
- "logs:CreateLogGroup"
- "logs:CreateLogStream"
- "logs:PutLogEvents"
Resource:
!Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/data-lake-authorizer:*"]]
-
Effect: "Allow"
Action:
- "*"
Resource:
!Join ["", ["arn:aws:cognito-idp:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":userpool/", Ref: UserPoolId]]
-
Effect: "Allow"
Action:
- "dynamodb:GetItem"
- "dynamodb:Query"
Resource:
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-settings"]]
- !Join ["", ["arn:aws:dynamodb:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":table/data-lake-keys"]]
-
Effect: "Allow"
Action:
- "kms:Encrypt"
- "kms:Decrypt"
- "kms:ReEncrypt*"
- "kms:GenerateDataKey*"
- "kms:DescribeKey"
Resource:
Fn::GetAtt:
- "DataLakeKmsKey"
- "Arn"
Roles:
-
Ref: "DataLakeAuthorizerRole"
DataLakeAuthorizerService:
Type: "AWS::Lambda::Function"
Properties:
Code:
S3Bucket: !Ref SourceS3Bucket
S3Key: !Join ["/", [!Ref SourceKeyPrefix, "data-lake-authorizer.zip"]]
Description: "A data lake microservice function for custom authorizer functionality"
FunctionName: "data-lake-authorizer"
Handler: "index.handler"
MemorySize: "256"
Role:
Fn::GetAtt:
- "DataLakeAuthorizerRole"
- "Arn"
Runtime: "nodejs6.10"
Timeout: "60"
Outputs: AuthorizerArn: Description: "ARN of data lake API Gateway custom authorizer function" Value: !GetAtt DataLakeAuthorizerService.Arn AdminArn: Description: "ARN of data lake admin services function" Value: !GetAtt DataLakeAdminService.Arn CartArn: Description: "ARN of data lake cart services function" Value: !GetAtt DataLakeCartService.Arn ProfileArn: Description: "ARN of data lake profile services function" Value: !GetAtt DataLakeProfileService.Arn SearchArn: Description: "ARN of data lake search services function" Value: !GetAtt DataLakeSearchService.Arn PackageArn: Description: "ARN of data lake package services function" Value: !GetAtt DataLakePackagesService.Arn