Tohuwabohu excorcism

User Tools

Site Tools

Trace: • karma • useful_programs • elasticache • functions • bigdata • batch • elk • api • tests
cloud:aws:cloudformation

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revisionPrevious revision
Next revision		Previous revision
cloud:aws:cloudformation [2018/08/09 15:15] – skipidarcloud:aws:cloudformation [2023/12/05 15:02] (current) – skipidar
Line 1: Line 1:
 ===== CloudFormation ===== ===== CloudFormation =====
 +
 +==== Why is Terraform better? ====
 +
 +  *  "Cloudformation" relies on uploads of substacks to S3 , whereas terraform just deploys everything from the folder
 +  *  "Cloudformation" is not enough verbose on errors and changes. Terraform generates accurate changesets in human readable files.
 +  *  "Cloudformation" has problems with versioning and deploying multiple versions of same resource. Terraform uses
 +  * CloudFormation has a very NOT user friendly lifecycle. Forcing to "is in ROLLBACK_COMPLETE state and can not be updated"
 +  * Minor. Cloudformation "deploy", "create-stack" and all the historical errors are still there with broken signatures. E.g.  ''file:/ '' required for create-stack or validate-stack, but no for deploy
 +  * Cloudformation support of moving resources between stacks is very chatty
 +
 +
 +==== Deploying with cloudformation ====
 +
 +If using nested-stacks first you need a bucket, 
 +into which you will package nested stacks.
 +
 +<sxh yaml>
 +AWSTemplateFormatVersion: '2010-09-09'
 +Description: AWS template for SiteWise demo
 +Resources:
 +
 +  MyS3SubstackBucket:
 +    Type: AWS::S3::Bucket
 +    Properties:
 +      BucketName: my-alf-s3-package-bucket-2023-12-05
 +      AccessControl: Private
 +      Tags:
 +        - Key: Purpose
 +          Value: CF stacks bucket
 +
 +
 +  MyBucketPolicy:
 +    Type: AWS::S3::BucketPolicy
 +    Properties:
 +      Bucket: !Ref MyS3SubstackBucket
 +      PolicyDocument:
 +        Statement:
 +          - Sid: AllowCloudFormationAccess
 +            Effect: Allow
 +            Principal:
 +              Service: cloudformation.amazonaws.com
 +            Action: s3:*
 +            Resource: !Join
 +              - ''
 +              - - 'arn:aws:s3:::'
 +                - !Ref MyS3SubstackBucket
 +                - /*
 +</sxh>
 +
 +now deploy the bucket
 +<sxh shell>
 +
 +FILENAME="seed.cloudformation.yaml"
 +STACKNAME="MySeedStackName"
 +REGION="eu-west-1"
 +
 +# validate
 +aws cloudformation validate-template --template-body file://$FILENAME
 +
 +
 +# check the change set
 +aws cloudformation deploy --stack-name $STACKNAME --template-file $FILENAME --region $REGION --no-execute-changeset
 +
 +
 +# execute via "deploy" which automatically creates / updates stack
 +aws cloudformation deploy --stack-name $STACKNAME --template-file $FILENAME --region $REGION
 +
 +
 +# delete stack
 +# aws cloudformation delete-stack --stack-name $STACKNAME
 +</sxh>
 +
 +
 +parent1.cloudformation.yaml
 +<sxh yaml>
 +AWSTemplateFormatVersion: "2010-09-09"
 +Description: Provision a SG
 +
 +Parameters:
 +
 +  VpcIdParameter:
 +    Type: String
 +    Default: "vpc-01eb7fd6f29cea57b"
 +    
 +  packageBucket:
 +    Type: String
 +    Default: "my-alf-s3-package-bucket-2023-12-05"
 +
 +Resources:
 +
 +  SubStack1:
 +    Type: AWS::CloudFormation::Stack
 +    Properties:
 +      TemplateURL: "substack.helloworld.cloudformation.yaml"
 +      Parameters:
 +        VpcId: !Ref VpcIdParameter
 +
 +</sxh>
 +
 +substack.helloworld.cloudformation.yaml
 +<sxh yaml>
 +AWSTemplateFormatVersion: "2010-09-09"
 +Description: Provision a SG
 +
 +Parameters:
 +  VpcId:
 +    Type: String
 +
 +
 +Resources:
 +
 +  MySecurityGroup:
 +    Type: AWS::EC2::SecurityGroup
 +    Properties:
 +      GroupDescription: MySecurityGroup
 +      VpcId: !Ref VpcId
 +      SecurityGroupIngress:
 +        - IpProtocol: tcp
 +          FromPort: 80
 +          ToPort: 80
 +          CidrIp: 0.0.0.0/0 # Example: Allowing HTTP traffic from anywhere (Please adjust for your use case)
 +      Tags:
 +        - Key: Name
 +          Value: MySecurityGroup
 +
 +</sxh>
 +
 +
 +
 +now you can package the stack
 +
 +  * the sub-stacks will end up in the package-bucket.
 +  * a new file `packaged-root-template.yaml` is generated, where the `TemplateURL` field is replaced by s3 references.
 +  * you can deploy the parent stack and see nested stacks being deployed too.
 +
 +
 +
 +<sxh shell>
 +set -e
 +
 +FILENAME="parent1.cloudformation.yaml"
 +STACKNAME="MyParentStackName"
 +REGION="eu-west-1"
 +PACKAGEBUCKET="my-alf-s3-package-bucket-2023-12-05"
 +
 +
 +# validate
 +# aws cloudformation validate-template --template-body file://$FILENAME
 +
 +
 +# package uploading substacks
 +rm packaged-root-template.yaml
 +aws cloudformation package --template-file $FILENAME --s3-bucket $PACKAGEBUCKET  --output-template-file packaged-root-template.yaml --region $REGION 
 +
 +
 +
 +# check the change set, dont execute :  "no-execute-changeset"
 +# aws cloudformation deploy --stack-name $STACKNAME --template-file $FILENAME --region $REGION --no-execute-changeset
 +
 +# arn of change set is printed, here arn:aws:cloudformation:eu-west-1:913372342854:changeSet/awscli-cloudformation-package-deploy-1701783364/21bb1e0c-a0ea-41ca-9edd-5a7ab989b3a5
 +
 +# can see change-set
 +# aws cloudformation describe-change-set --change-set-name arn:aws:cloudformation:eu-west-1:913372342854:changeSet/awscli-cloudformation-package-deploy-1701783364/21bb1e0c-a0ea-41ca-9edd-5a7ab989b3a5  --region $REGION
 +
 +# can continue via 
 +# aws cloudformation execute-change-set --change-set-name arn:aws:cloudformation:eu-west-1:913372342854:changeSet/awscli-cloudformation-package-deploy-1701783364/21bb1e0c-a0ea-41ca-9edd-5a7ab989b3a5  --region $REGION
 +
 +
 +
 +
 +
 +# execute via "deploy" which automatically creates / updates stack
 +aws cloudformation deploy --stack-name $STACKNAME --template-file packaged-root-template.yaml --region $REGION  --parameter-overrides VpcIdParameter="vpc-01eb7fd6f29cea57b"
 +
 +
 +# delete stack
 +# aws cloudformation delete-stack --stack-name $STACKNAME
 +</sxh>
 +
  
 ==== Structure ==== ==== Structure ====
Line 180: Line 359:
   
 } }
 +</code>
 +
 +==== Use !Sub in commands ====
 +
 +To reference cloudformation arguments in commands - use the "env"
 +<code>
 +Resources:
 +Parameters:
 +  S3Prefix:
 +    AllowedPattern: >-
 +      ^s3://[/a-zA-Z0-9_-]*[a-zA-Z0-9_-]$
 +    Description: The s3://..... prefix to the bucket and folder, where the temp files are located.
 +    Type: String
 +
 +...
 +            04_execute:
 +              command: !Sub |
 +                ansible-playbook /ssh-jump-host/sshJumpServer.ansible.yml -v --extra-vars "host=local" --extra-vars s3prefix="${S3Prefix}" >> "/ssh-jump-host/sshJumpServer.ansible.yml.log"
 </code> </code>
  
Line 558: Line 755:
 | Human readable logs of the whole instance initiation | /var/log/cloud-init-output.log | | Human readable logs of the whole instance initiation | /var/log/cloud-init-output.log |
 | Human readable logs of the cfn-init call | /var/log/cfn-init.log | | Human readable logs of the cfn-init call | /var/log/cfn-init.log |
 +| Code from AWS::CloudFormation::Init block of a Cloud Formation template on Linux systems | /var/lib/cfn-init/data/metadata.json |
 +
 +
 +
  
 === Debug the initiation of the instance === === Debug the initiation of the instance ===
Line 645: Line 846:
 AWS specific parameters:  AWS specific parameters: 
   * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html#aws-specific-parameter-types   * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html#aws-specific-parameter-types
 +
 +
 +
 +==== Structure ====
 +
 +
 +Example of a custom definition in CLoudFormation. EFS with Fargate in this example
 +<code>
 +
 +  CustomTaskDefinition:
 +    Type: 'Custom::TaskDefinition'
 +    Version: '1.0'
 +    Properties:
 +      ServiceToken: !GetAtt 'CustomResourceFunction.Arn'
 +      TaskDefinition: {
 +        executionRoleArn: !Ref CustomTaskDefinitionRole,
 +        containerDefinitions: [
 +          {
 +            name: "navvis",
 +            image: !Ref NavvisDockerImage,
 +            memoryReservation: 2000,
 +            logConfiguration: {
 +              logDriver: "awslogs",
 +              options: {
 +                awslogs-group:  !Ref CustomLogsGroup,
 +                awslogs-datetime-format: "%Y-%m-%d %H:%M:%S.%L",
 +                awslogs-region: !Ref 'AWS::Region',
 +                awslogs-stream-prefix:  !Sub '${ProjectName}'
 +              }
 +            },
 +            portMappings: [
 +              {
 +                hostPort: 8080,
 +                protocol: "tcp",
 +                containerPort: 8080
 +              }
 +            ],
 +            command: [],
 +            "environment": [
 +              {
 +                "name": "INSTANCE_NAME",
 +                "value": !Ref 'BuildingID'
 +              },
 +              {
 +                "name": "INSTANCE_PORT",
 +                "value": "8080"
 +              },
 +              {
 +                "name": "STORAGE_PATH",
 +                "value": "/mnt"
 +              }
 +            ]
 +           ,
 +           mountPoints: [
 +             {sourceVolume: "myefs", containerPath: "/mnt"}
 +           ]
 +          }
 +        ],
 +        family: "navvis",
 +        taskRoleArn: !Ref CustomTaskDefinitionRole, #required for EFS permissions
 +        requiresCompatibilities: ["FARGATE"],
 +        cpu: "256",
 +        memory: "2048",
 +        networkMode: "awsvpc",
 +        volumes: [
 +          {
 +            name: "myefs",
 +            efsVolumeConfiguration: {
 +              fileSystemId: {'Fn::ImportValue': !Sub '${ProjectName}-arc-${TargetAwsAccount}-efsid-for-${BuildingID}----${ScanID}'}
 +            }
 +          },
 +        ]
 +      }
 +  CustomResourceFunction:
 +    Type: 'AWS::Lambda::Function'
 +    Properties:
 +      Code:
 +        ZipFile: |
 +          const aws = require('aws-sdk')
 +          const response = require('cfn-response')
 +          const ecs = new aws.ECS({apiVersion: '2014-11-13'})
 +          exports.handler = function(event, context) {
 +            console.log("REQUEST RECEIVED:\n" + JSON.stringify(event))
 +            if (event.RequestType === 'Create' || event.RequestType === 'Update') {
 +              ecs.registerTaskDefinition(event.ResourceProperties.TaskDefinition, function(err, data) {
 +                if (err) {
 +                  console.error(err);
 +                  response.send(event, context, response.FAILED)
 +                } else {
 +                  console.log(`Created/Updated task definition ${data.taskDefinition.taskDefinitionArn}`)
 +                  response.send(event, context, response.SUCCESS, {}, data.taskDefinition.taskDefinitionArn)
 +                }
 +              })
 +            } else if (event.RequestType === 'Delete') {
 +              ecs.deregisterTaskDefinition({taskDefinition: event.PhysicalResourceId}, function(err) {
 +                if (err) {
 +                  if (err.code === 'InvalidParameterException') {
 +                    console.log(`Task definition: ${event.PhysicalResourceId} does not exist. Skipping deletion.`)
 +                    response.send(event, context, response.SUCCESS)
 +                  } else {
 +                    console.error(err)
 +                    response.send(event, context, response.FAILED)
 +                  }
 +                } else {
 +                  console.log(`Removed task definition ${event.PhysicalResourceId}`)
 +                  response.send(event, context, response.SUCCESS)
 +                }
 +              })
 +            } else {
 +              console.error(`Unsupported request type: ${event.RequestType}`)
 +              response.send(event, context, response.FAILED)
 +            }
 +          }
 +      Handler: 'index.handler'
 +      MemorySize: 128
 +      Role: !GetAtt 'CustomResourceRole.Arn'
 +      Runtime: 'nodejs10.x'
 +      Timeout: 30
 +  CustomResourceRole:
 +    Type: 'AWS::IAM::Role'
 +    Properties:
 +      AssumeRolePolicyDocument:
 +        Version: '2012-10-17'
 +        Statement:
 +        - Effect: Allow
 +          Principal:
 +            Service: 'lambda.amazonaws.com'
 +          Action: 'sts:AssumeRole'
 +      Policies:
 +      - PolicyName: 'customresource'
 +        PolicyDocument:
 +          Statement:
 +          - Effect: Allow
 +            Action:
 +            - 'ecs:DeregisterTaskDefinition'
 +            - 'ecs:RegisterTaskDefinition'
 +            Resource: '*'
 +          - Effect: Allow
 +            Action:
 +            - 'logs:CreateLogGroup'
 +            - 'logs:CreateLogStream'
 +            - 'logs:PutLogEvents'
 +            Resource: '*'
 +          - Effect: Allow
 +            Action:
 +            - 'iam:PassRole'
 +            Resource: '*' # replace with value of taskRoleArn
 +  CustomLogsGroup:
 +    Type: 'AWS::Logs::LogGroup'
 +    Properties:
 +      LogGroupName: !Sub "${ProjectName}-custom"
 +      RetentionInDays: 14
 +
 +
 +  NavvisServiceFargate:
 +    Type: 'AWS::ECS::Service'
 +    Properties:
 +      Cluster: {'Fn::ImportValue': !Sub '${ProjectName}-arc-${TargetAwsAccount}-ecscluster'}
 +      LaunchType: FARGATE
 +      DesiredCount: 1
 +      PlatformVersion: "1.4.0" # by default on 30th June 2020 still pick up automatically the version 1.3.0 and not really the latest (same behaviour on web console) but to have the EFS we need the 1.4.0
 +      DeploymentConfiguration:
 +        MinimumHealthyPercent: 0 # setting to 0 as otherwise, if there are not enough resources (e.g. eni per instance) - the service will not be able to tear down the old container to replace it by the new one
 +        MaximumPercent: 250
 +      TaskDefinition: !Ref CustomTaskDefinition
 +      NetworkConfiguration:
 +        AwsvpcConfiguration:
 +          AssignPublicIp: DISABLED
 +          SecurityGroups:
 +            - {'Fn::ImportValue': !Sub '${ProjectName}-arc-${TargetAwsAccount}-codebuildfargate-sg'}
 +          Subnets:
 +            - {'Fn::ImportValue': !Sub '${PlatformPrefix}-PrivateSubnet1AID'}
 +            - {'Fn::ImportValue': !Sub '${PlatformPrefix}-PrivateSubnet1BID'}
 +            - {'Fn::ImportValue': !Sub '${PlatformPrefix}-PrivateSubnet1CID'}
 +      LoadBalancers:
 +        - ContainerName: 'navvis'
 +          ContainerPort: 8080
 +          TargetGroupArn: !Ref TargetGroupNavvisHTTP
 +      SchedulingStrategy: "REPLICA"
 +
 +
 +</code>
 +
  
cloud/aws/cloudformation.1533827715.txt.gz · Last modified: (external edit)