From c3788758ec920e8e99760807bf4d801eff6a90f1 Mon Sep 17 00:00:00 2001 From: Eamonn Faherty <273558+eamonnfaherty@users.noreply.github.com> Date: Wed, 12 Jun 2019 12:41:57 +0100 Subject: [PATCH] Feature/spoke local portfolios (#68) - Added spoke-local-portfolios to manifest and accompanying code - Exit status now corresponds to luigi results - Added timeoutInSeconds to manifest for launches Issues: #58, #59 and #72 --- CHANGELOG.md | 8 +- docs/source/conf.py | 6 +- docs/source/index.rst | 1 + docs/source/puppet/cli.rst | 5 + docs/source/puppet/designing_your_manifest.md | 422 ------------- .../source/puppet/designing_your_manifest.rst | 518 +++++++++++++++ docs/source/puppet/sharing_a_portfolio.rst | 96 +++ servicecatalog_puppet/aws.py | 127 +++- servicecatalog_puppet/cli.py | 495 +-------------- servicecatalog_puppet/cli_command_helpers.py | 594 ++++++++++++++++++ .../cli_command_helpers_unit_test.py | 94 +++ servicecatalog_puppet/cli_commands.py | 413 ++++++++++++ servicecatalog_puppet/cli_test.py | 150 ----- servicecatalog_puppet/commands/__init__.py | 0 servicecatalog_puppet/commands/bootstrap.py | 78 --- .../commands/bootstrap_org_master.py | 45 -- .../commands/bootstrap_spoke.py | 33 - .../commands/list_launches.py | 121 ---- servicecatalog_puppet/constants.py | 15 +- servicecatalog_puppet/core.py | 175 ------ .../all-tasks-for-launch-b.json | 25 + .../data/account-vending/all-tasks.json | 96 +++ .../data/account-vending/deployment-map.json | 150 +++++ .../data/account-vending/launch-a.json | 21 + .../data/account-vending/launch-b.json | 21 + .../data/account-vending/launch-c.json | 89 +++ .../launch-details-for-launch-b.json | 38 ++ .../data/account-vending/manifest.json | 287 +++++++++ .../luigi_tasks_and_targets.py | 418 ++++++++++-- .../{commands/expand.py => manifest_utils.py} | 141 ++++- servicecatalog_puppet/requirements-test.txt | 4 +- servicecatalog_puppet/targets/__init__.py | 0 servicecatalog_puppet/tasks/__init__.py | 0 .../templates/associations.template.yaml.j2 | 10 + .../launch_role_constraints.template.yaml.j2 | 14 + servicecatalog_puppet/utils/__init__.py | 0 servicecatalog_puppet/utils/manifest.py | 133 ---- setup.py | 2 +- 38 files changed, 3123 insertions(+), 1722 deletions(-) create mode 100644 docs/source/puppet/cli.rst delete mode 100644 docs/source/puppet/designing_your_manifest.md create mode 100644 docs/source/puppet/designing_your_manifest.rst create mode 100644 docs/source/puppet/sharing_a_portfolio.rst create mode 100644 servicecatalog_puppet/cli_command_helpers.py create mode 100644 servicecatalog_puppet/cli_command_helpers_unit_test.py create mode 100644 servicecatalog_puppet/cli_commands.py delete mode 100644 servicecatalog_puppet/cli_test.py delete mode 100644 servicecatalog_puppet/commands/__init__.py delete mode 100644 servicecatalog_puppet/commands/bootstrap.py delete mode 100644 servicecatalog_puppet/commands/bootstrap_org_master.py delete mode 100644 servicecatalog_puppet/commands/bootstrap_spoke.py delete mode 100644 servicecatalog_puppet/commands/list_launches.py delete mode 100644 servicecatalog_puppet/core.py create mode 100644 servicecatalog_puppet/data/account-vending/all-tasks-for-launch-b.json create mode 100644 servicecatalog_puppet/data/account-vending/all-tasks.json create mode 100644 servicecatalog_puppet/data/account-vending/deployment-map.json create mode 100644 servicecatalog_puppet/data/account-vending/launch-a.json create mode 100644 servicecatalog_puppet/data/account-vending/launch-b.json create mode 100644 servicecatalog_puppet/data/account-vending/launch-c.json create mode 100644 servicecatalog_puppet/data/account-vending/launch-details-for-launch-b.json create mode 100644 servicecatalog_puppet/data/account-vending/manifest.json rename servicecatalog_puppet/{commands/expand.py => manifest_utils.py} (58%) delete mode 100644 servicecatalog_puppet/targets/__init__.py delete mode 100644 servicecatalog_puppet/tasks/__init__.py create mode 100644 servicecatalog_puppet/templates/associations.template.yaml.j2 create mode 100644 servicecatalog_puppet/templates/launch_role_constraints.template.yaml.j2 delete mode 100644 servicecatalog_puppet/utils/__init__.py delete mode 100644 servicecatalog_puppet/utils/manifest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0524165d5..04389dfef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # This is a list of noteworthy changes made to the project +# 2019-06-12 - 0.1.14 +- Added spoke-local-portfolios to manifest and accompanying code +- Exit status now corresponds to luigi results +- Added timeoutInSeconds to manifest for launches + + # 2019-06-03 - 0.1.0 -- Replaced the deployment engine custom implementation with [Luigu](https://luigi.readthedocs.io). \ No newline at end of file +- Replaced the deployment engine custom implementation with [Luigu](https://luigi.readthedocs.io). diff --git a/docs/source/conf.py b/docs/source/conf.py index 85578ed97..d6143ddf6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -10,9 +10,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index e90cd8ede..a060686a6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,6 +8,7 @@ Welcome to aws-service-catalog-puppets's documentation! puppet/what_is_this puppet/getting_up_and_running puppet/designing_your_manifest + puppet/sharing_a_portfolio puppet/notifications puppet/utils puppet/upgrading diff --git a/docs/source/puppet/cli.rst b/docs/source/puppet/cli.rst new file mode 100644 index 000000000..8545505ee --- /dev/null +++ b/docs/source/puppet/cli.rst @@ -0,0 +1,5 @@ +cli +=== + +.. automodule:: servicecatalog_puppet.cli_commands + :members: \ No newline at end of file diff --git a/docs/source/puppet/designing_your_manifest.md b/docs/source/puppet/designing_your_manifest.md deleted file mode 100644 index b1d90ec7d..000000000 --- a/docs/source/puppet/designing_your_manifest.md +++ /dev/null @@ -1,422 +0,0 @@ -Designing your manifest -======================= - -## Purpose of the manifest file -The manifest file is there to describe what you want to provision and into which accounts you want to provision products -into. It is possible to use AWS Organizations to make your manifest file more concise and easier to work with but the -premise is the same - it is just a list of accounts and AWS Service Catalog products. - - -## Sections of the manifest file -There are three sections to a manifest file - the global parameters, the accounts list and the launches. Each of the -three are described in the following sections. - -### Parameters - -It is possible to specify global parameters that should be used when provisioning your AWS Service Catalog Products. -You can set the value to an explicit value or you can set the value to the result of a function call - using funcation -calls to set parameter values is known as using a macro. - -Here is an example of a simple global parameter: -```yaml -schema: puppet-2019-04-01 - -parameters: - CloudTrailLoggingBucketName: - default: cloudtrail-logs-for-aws -``` - -It is possible to also specify a parameter at the account level: -```yaml -accounts: - - account_id: '' - name: '' - default_region: eu-west-1 - regions_enabled: - - eu-west-1 - - eu-west-1 - tags: - - type:prod - - partition:eu - - scope:pci - parameters: - RoleName: - default: DevAdmin - Path: - default: /human-roles/ -``` - -And finally you specify parameters at the launch level: -```yaml -launches: - account-iam-for-prod: - portfolio: demo-central-it-team-portfolio - product: account-iam - version: v1 - parameters: - RoleName: - default: DevAdmin - Path: - default: /human-roles/ - deploy_to: - tags: - - tag: type:prod - regions: default_region -``` - -Whenever Puppet provisions a product it checks the parameters for the product. If it sees the name match one of the -parameter values it will use it. In order to avoid clashes with parameter names we recommend using descriptive names -like in the example - using the parameter names like ```BucketName``` will lead you into trouble pretty quickly. - -The order of precedence for parameters is account level parameters override all others and launch level parameters -override global. - -#### Retrieving AWS SSM Parameters -You can retrieve parameter values from SSM. Here is an an example: -```yaml -schema: puppet-2019-04-01 - -parameters: - CentralLoggingBucketName: - ssm: - name: central-logging-bucket-name -``` - -You can get a different value for each region: -```yaml -schema: puppet-2019-04-01 - -parameters: - CentralLoggingBucketName: - ssm: - name: central-logging-bucket-name - region: eu-west-1 -``` - -#### Setting AWS SSM Parameters -You can set the value of an SSM Parameter to the output of a CloudFormation stack output: - -```yaml - account-iam-sysops: - portfolio: demo-central-it-team-portfolio - product: account-iam - version: v1 - parameters: - Path: - default: /human-roles/ - RoleName: - default: SysOps - deploy_to: - tags: - - regions: default_region - tag: type:prod - outputs: - ssm: - - param_name: account-iam-sysops-role-arn - stack_output: RoleArn - ``` - -The example above will provision the product ```account-iam``` into an account. Once the stack has been completed it -will get the value of the output named ```RoleArn``` of the CloudFormation stack and insert it into SSM within the default -region using a parameter name of ```account-iam-sysops-role-arn``` - -You can also set override which region the output is read from and which region the SSM parameter is written to: - -```yaml - account-iam-sysops: - portfolio: demo-central-it-team-portfolio - product: account-iam - version: v1 - parameters: - Path: - default: /human-roles/ - RoleName: - default: SysOps - deploy_to: - tags: - - regions: default_region - tag: type:prod - outputs: - ssm: - - param_name: account-iam-sysops-role-arn - stack_output: RoleArn - region: us-east-1 -``` - -There is currently no capability of reading a value from a CloudFormation stack from one region and setting an SSM param -in another. - -SSM parameters can only be set using the framework when the product is deployed the first time and can only be set once -- there is no overriding. We would advise outputting SSM parameters only when a product is deployed to a single account. - - - -#### Macros -You can also use a macro to set the value of a parameter. It works in the same way as a normal parameter except it -executes a function to get the value first. Here is an an example: -```yaml -schema: puppet-2019-04-01 - -parameters: - AllAccountIds: - macro: - method: get_accounts_for_path - args: / -``` - -At the moment there are the following macros supported: - -``` -+------------------------+------------------------------+----------------------------------------------+ -| macro method name | args | description | -+========================+==============================+==============================================+ -| get_accounts_for_path | ou path to get accounts for | Returns a comma seperated list of account ids| -+------------------------+------------------------------+----------------------------------------------+ -``` - -### Accounts - -With the accounts section, you can describe your AWS accounts. You can set a default region, the enabled regions and -you can tag your accounts. This metadata describing your account is used to determine which packages get deployed into -your accounts. - -#### Setting a default region -Within your account you may have a _home_ or a default region. This may be the closest region to the team using the -account. You use ```default_region``` when describing your account and then you can use ```default_region``` again as a -target when you specify your product launches - the product will be provisioned into the region specified. - -Here is an example with a ```default_region``` set to ```us-east-1```: - -```yaml -schema: puppet-2019-04-01 - -accounts: - - account_id: '' - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci -``` - -Please note ```default_region``` can only be a string. - -#### Setting enabled regions -You may chose not to use every region within your AWS Account. When describing an AWS account you can specify which -regions are enabled for an account using ```regions_enabled```. - -Here is an example with ```regions_enabled``` set to ```us-east-1 and us-west-2```: - -```yaml -schema: puppet-2019-04-01 - -accounts: - - account_id: '' - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci -``` - -Please note ```regions_enabled``` can only be a list of strings. - - -#### Setting tags -You can describe your account using tags. Tags are specified using a list of strings. We recommend using namespaces -for your tags, adding an extra dimension to them. If you choose to do this you can use a colon to split name and values. - -Here is an example with namespaced tags: - -```yaml -schema: puppet-2019-04-01 - -accounts: - - account_id: '' - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci -``` - -In this example there the following tags: -- namespace of type and value of prod -- namespace of partition and value of us -- namespace of scope and value of pci. - -The goal of tags is to provide a classification for your accounts that can be used to a deployment time. - -#### Using an OU id or path (integration with AWS Organizations) -When specifying an account you can use short hand notation of ```ou``` instead of ```account_id``` to build out a list -of accounts with the same properties. - -For example you can use an AWS Organizations path: -```yaml -schema: puppet-2019-04-01 - -accounts: - - ou: /prod - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci -``` - -The framework will get a list of all AWS accounts within the ```/prod``` Organizational unit and expand your manifest to -look like the following (assuming accounts 0123456789010 and 0109876543210 are the only accountss within ```/prod```): - -```yaml -schema: puppet-2019-04-01 - -accounts: - - account_id: 0123456789010 - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci - - account_id: 0109876543210 - name: '' - default_region: us-east-1 - regions_enabled: - - us-east-1 - - us-west-2 - tags: - - type:prod - - partition:us - - scope:pci -``` - -### Launches -Launches allow you to decide which products get provisioned into each account. You link product launches to accounts -using tags or explicit account ids and you can set which regions the products are launched into. - -#### Tag based launches -You can specify a launch to occur using ```tags``` in the ```deploy_to``` section of a launch. - -Here is an example, it deploys a ```v1``` of a product named ```account-iam``` from the portfolio -```example-simple-central-it-team-portfolio``` into into the ```default_region``` of all accounts tagged ```type:prod```: - -```yaml -schema: puppet-2019-04-01 - -launches: - account-iam-for-prod: - portfolio: example-simple-central-it-team-portfolio - product: account-iam - version: v1 - deploy_to: - tags: - - tag: type:prod - regions: default_region -``` - - -#### Account based launches -You can also specify a launch to occur explicity in an account by using the ```accounts``` section in the -```deploy_to``` section of a launch. - -Here is an example, it deploys a ```v1``` of a product named ```account-iam``` from the portfolio -```example-simple-central-it-team-portfolio``` into into the ```default_region``` of the accounts ```0123456789010```: - -```yaml -schema: puppet-2019-04-01 - -launches: - account-iam-for-prod: - portfolio: example-simple-central-it-team-portfolio - product: account-iam - version: v1 - deploy_to: - accounts: - - account_id: '0123456789010' - regions: default_region -``` - -#### Dependencies between launches -Where possible we recommend building launches to be independent. However, there are cases where you may need to setup a -hub account before setting up a spoke or there may be times you are using AWS Lambda to back AWS CloudFormation custom -resources. In these examples it would be beneficial to be able to say deploy launch x and then launch y. To achieve this -You can use ```depends_on``` within your launch like so: -```yaml -launches: - account-vending-account-creation: - portfolio: demo-central-it-team-portfolio - product: account-vending-account-creation - version: v1 - depends_on: - - account-vending-account-bootstrap-shared - - account-vending-account-creation-shared - deploy_to: - tags: - - tag: scope:puppet-hub - regions: default_region - - account-vending-account-bootstrap-shared: - portfolio: demo-central-it-team-portfolio - product: account-vending-account-bootstrap-shared - version: v1 - deploy_to: - tags: - - tag: scope:puppet-hub - regions: default_region - - account-vending-account-creation-shared: - portfolio: demo-central-it-team-portfolio - product: account-vending-account-creation-shared - version: v1 - deploy_to: - tags: - - tag: scope:puppet-hub - regions: default_region -``` - -In this example the framework will deploy ```account-vending-account-creation``` only when -```account-vending-account-bootstrap-shared``` and ```account-vending-account-creation-shared``` have been attempted. - - -#### Termination of products -To terminate the provisioned product from a spoke account (which will delete the resources deployed) you can change -the status of the launch using the ```status``` keyword: - -```yaml -launches: - account-vending-account-creation: - portfolio: demo-central-it-team-portfolio - product: account-vending-account-creation - version: v1 - status: terminated - deploy_to: - tags: - - tag: scope:puppet-hub - regions: default_region -``` - -When you mark a launch as terminated and run your pipeline the resources will be deleted and you can then remove the -launch from your manifest. Leaving it in will not cause any errors but will result in your pipeline running time to be -longer than it needs to be. - -Please note, when mark your launch as ```terminated``` it cannot have dependencies, parameters or outputs. Leaving -these in will cause the termination action to fail. \ No newline at end of file diff --git a/docs/source/puppet/designing_your_manifest.rst b/docs/source/puppet/designing_your_manifest.rst new file mode 100644 index 000000000..55b68c767 --- /dev/null +++ b/docs/source/puppet/designing_your_manifest.rst @@ -0,0 +1,518 @@ +Designing your manifest +======================= + +Purpose of the manifest file +---------------------------- +The manifest file is there to describe what you want to provision and into which accounts you want to provision products +into. It is possible to use AWS Organizations to make your manifest file more concise and easier to work with but the +premise is the same - it is just a list of accounts and AWS Service Catalog products. + + +Sections of the manifest file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +There are three sections to a manifest file - the global parameters, the accounts list and the launches. Each of the +three are described in the following sections. + +Parameters +########## + + +It is possible to specify global parameters that should be used when provisioning your AWS Service Catalog Products. +You can set the value to an explicit value or you can set the value to the result of a function call - using funcation +calls to set parameter values is known as using a macro. + +Here is an example of a simple global parameter: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + parameters: + CloudTrailLoggingBucketName: + default: cloudtrail-logs-for-aws + +It is possible to also specify a parameter at the account level: + +.. code-block:: yaml + + accounts: + - account_id: '' + name: '' + default_region: eu-west-1 + regions_enabled: + - eu-west-1 + - eu-west-1 + tags: + - type:prod + - partition:eu + - scope:pci + parameters: + RoleName: + default: DevAdmin + Path: + default: /human-roles/ + + +And finally you specify parameters at the launch level: + +.. code-block:: yaml + + launches: + account-iam-for-prod: + portfolio: demo-central-it-team-portfolio + product: account-iam + version: v1 + parameters: + RoleName: + default: DevAdmin + Path: + default: /human-roles/ + deploy_to: + tags: + - tag: type:prod + regions: default_region + + +Whenever Puppet provisions a product it checks the parameters for the product. If it sees the name match one of the +parameter values it will use it. In order to avoid clashes with parameter names we recommend using descriptive names +like in the example - using the parameter names like ``BucketName`` will lead you into trouble pretty quickly. + +The order of precedence for parameters is account level parameters override all others and launch level parameters +override global. + +Retrieving AWS SSM Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + This was added in version 0.0.33 + +You can retrieve parameter values from SSM. Here is an an example: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + parameters: + CentralLoggingBucketName: + ssm: + name: central-logging-bucket-name + + +You can get a different value for each region: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + parameters: + CentralLoggingBucketName: + ssm: + name: central-logging-bucket-name + region: eu-west-1 + + +Setting AWS SSM Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + This was added in version 0.0.34 + +You can set the value of an SSM Parameter to the output of a CloudFormation stack output: + +.. code-block:: yaml + + account-iam-sysops: + portfolio: demo-central-it-team-portfolio + product: account-iam + version: v1 + parameters: + Path: + default: /human-roles/ + RoleName: + default: SysOps + deploy_to: + tags: + - regions: default_region + tag: type:prod + outputs: + ssm: + - param_name: account-iam-sysops-role-arn + stack_output: RoleArn + + +The example above will provision the product ``account-iam`` into an account. Once the stack has been completed it +will get the value of the output named ``RoleArn`` of the CloudFormation stack and insert it into SSM within the default +region using a parameter name of ``account-iam-sysops-role-arn`` + +You can also set override which region the output is read from and which region the SSM parameter is written to: + +.. code-block:: yaml + + account-iam-sysops: + portfolio: demo-central-it-team-portfolio + product: account-iam + version: v1 + parameters: + Path: + default: /human-roles/ + RoleName: + default: SysOps + deploy_to: + tags: + - regions: default_region + tag: type:prod + outputs: + ssm: + - param_name: account-iam-sysops-role-arn + stack_output: RoleArn + region: us-east-1 + + +.. note:: + + There is currently no capability of reading a value from a CloudFormation stack from one region and setting an SSM param in another. + + +Macros +~~~~~~ + +You can also use a macro to set the value of a parameter. It works in the same way as a normal parameter except it +executes a function to get the value first. Here is an an example: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + parameters: + AllAccountIds: + macro: + method: get_accounts_for_path + args: / + + +At the moment there are the following macros supported: + +.. code-block:: yaml + + +------------------------+------------------------------+----------------------------------------------+ + | macro method name | args | description | + +========================+==============================+==============================================+ + | get_accounts_for_path | ou path to get accounts for | Returns a comma seperated list of account ids| + +------------------------+------------------------------+----------------------------------------------+ + + +Accounts +######## + +With the accounts section, you can describe your AWS accounts. You can set a default region, the enabled regions and +you can tag your accounts. This metadata describing your account is used to determine which packages get deployed into +your accounts. + +Setting a default region +~~~~~~~~~~~~~~~~~~~~~~~~ + +Within your account you may have a _home_ or a default region. This may be the closest region to the team using the +account. You use ``default_region`` when describing your account and then you can use ``default_region`` again as a +target when you specify your product launches - the product will be provisioned into the region specified. + +Here is an example with a ``default_region`` set to ``us-east-1``: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + accounts: + - account_id: '' + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + + +.. note:: + + Please note ``default_region`` can only be a string - not a list. + +Setting enabled regions +~~~~~~~~~~~~~~~~~~~~~~~ + +You may chose not to use every region within your AWS Account. When describing an AWS account you can specify which +regions are enabled for an account using ``regions_enabled``. + +Here is an example with ``regions_enabled`` set to ``us-east-1 and us-west-2``: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + accounts: + - account_id: '' + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + + +.. note:: + + Please note ``regions_enabled`` can only be a list of strings - not a single string + + +Setting tags +~~~~~~~~~~~~ + +You can describe your account using tags. Tags are specified using a list of strings. We recommend using namespaces +for your tags, adding an extra dimension to them. If you choose to do this you can use a colon to split name and values. + +Here is an example with namespaced tags: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + accounts: + - account_id: '' + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + + +In this example there the following tags: +- namespace of type and value of prod +- namespace of partition and value of us +- namespace of scope and value of pci. + +The goal of tags is to provide a classification for your accounts that can be used to a deployment time. + +Using an OU id or path (integration with AWS Organizations) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + This was added in version 0.0.18 + +When specifying an account you can use short hand notation of ``ou`` instead of ``account_id`` to build out a list +of accounts with the same properties. + +For example you can use an AWS Organizations path: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + accounts: + - ou: /prod + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + + +The framework will get a list of all AWS accounts within the ``/prod`` Organizational unit and expand your manifest to +look like the following (assuming accounts 0123456789010 and 0109876543210 are the only accountss within ``/prod``): + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + accounts: + - account_id: 0123456789010 + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + - account_id: 0109876543210 + name: '' + default_region: us-east-1 + regions_enabled: + - us-east-1 + - us-west-2 + tags: + - type:prod + - partition:us + - scope:pci + + +Launches +######## + +Launches allow you to decide which products get provisioned into each account. You link product launches to accounts +using tags or explicit account ids and you can set which regions the products are launched into. + +Timeouts +~~~~~~~~ + +.. note:: + + This was added in version 0.1.14 + +If you are worried that a launch may fail and take a long time to fail you can set a timeout ``timeoutInSeconds``: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + launches: + account-iam-for-prod: + portfolio: example-simple-central-it-team-portfolio + product: account-iam + timeoutInSeconds: 10 + version: v1 + deploy_to: + tags: + - tag: type:prod + regions: default_region + + + +Tag based launches +~~~~~~~~~~~~~~~~~~ + +You can specify a launch to occur using ``tags`` in the ``deploy_to`` section of a launch. + +Here is an example, it deploys a ``v1`` of a product named ``account-iam`` from the portfolio +``example-simple-central-it-team-portfolio`` into into the ``default_region`` of all accounts tagged ``type:prod``: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + launches: + account-iam-for-prod: + portfolio: example-simple-central-it-team-portfolio + product: account-iam + version: v1 + deploy_to: + tags: + - tag: type:prod + regions: default_region + + +Account based launches +~~~~~~~~~~~~~~~~~~~~~~ + +You can also specify a launch to occur explicity in an account by using the ``accounts`` section in the +``deploy_to`` section of a launch. + +Here is an example, it deploys a ``v1`` of a product named ``account-iam`` from the portfolio +``example-simple-central-it-team-portfolio`` into into the ``default_region`` of the accounts ``0123456789010``: + +.. code-block:: yaml + + schema: puppet-2019-04-01 + + launches: + account-iam-for-prod: + portfolio: example-simple-central-it-team-portfolio + product: account-iam + version: v1 + deploy_to: + accounts: + - account_id: '0123456789010' + regions: default_region + + +Dependencies between launches +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Where possible we recommend building launches to be independent. However, there are cases where you may need to setup a +hub account before setting up a spoke or there may be times you are using AWS Lambda to back AWS CloudFormation custom +resources. In these examples it would be beneficial to be able to say deploy launch x and then launch y. To achieve this +You can use ``depends_on`` within your launch like so: + +.. code-block:: yaml + + launches: + account-vending-account-creation: + portfolio: demo-central-it-team-portfolio + product: account-vending-account-creation + version: v1 + depends_on: + - account-vending-account-bootstrap-shared + - account-vending-account-creation-shared + deploy_to: + tags: + - tag: scope:puppet-hub + regions: default_region + + account-vending-account-bootstrap-shared: + portfolio: demo-central-it-team-portfolio + product: account-vending-account-bootstrap-shared + version: v1 + deploy_to: + tags: + - tag: scope:puppet-hub + regions: default_region + + account-vending-account-creation-shared: + portfolio: demo-central-it-team-portfolio + product: account-vending-account-creation-shared + version: v1 + deploy_to: + tags: + - tag: scope:puppet-hub + regions: default_region + + +In this example the framework will deploy ``account-vending-account-creation`` only when +``account-vending-account-bootstrap-shared`` and ``account-vending-account-creation-shared`` have been attempted. + + +Termination of products +~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + This was added in version 0.1.11 + +To terminate the provisioned product from a spoke account (which will delete the resources deployed) you can change +the status of the launch using the ``status`` keyword: + +.. code-block:: yaml + + launches: + account-vending-account-creation: + portfolio: demo-central-it-team-portfolio + product: account-vending-account-creation + version: v1 + status: terminated + deploy_to: + tags: + - tag: scope:puppet-hub + regions: default_region + + +When you mark a launch as terminated and run your pipeline the resources will be deleted and you can then remove the +launch from your manifest. Leaving it in will not cause any errors but will result in your pipeline running time to be +longer than it needs to be. + +Please note, when mark your launch as ``terminated`` it cannot have dependencies, parameters or outputs. Leaving +these in will cause the termination action to fail. \ No newline at end of file diff --git a/docs/source/puppet/sharing_a_portfolio.rst b/docs/source/puppet/sharing_a_portfolio.rst new file mode 100644 index 000000000..29b2346d0 --- /dev/null +++ b/docs/source/puppet/sharing_a_portfolio.rst @@ -0,0 +1,96 @@ +Sharing a portfolio +=================== + +------------------------------------- +What is sharing and how does it work? +------------------------------------- + +.. note:: + + This was added in version 0.1.14 + +This framework allows you to create portfolios in other accounts that mirror the portfolio in your hub account. The +framework will create the portfolio for you and copy the products (along with their versions) from your hub account into +the newly created portfolio. + +In addition to this, you can specify associations for the created portfolio and add launch constraints for the products. + + +.. warning:: + + Once a hub product version has been copied into a spoke portfolio it will not be updated. + +-------------------- +How can I set it up? +-------------------- + +The following is an example of how to add the portfolio ``demo-central-it-team-portfolio`` to all spokes tagged +``scope:spoke``: + +.. code-block:: yaml + + spoke-local-portfolios: + account-vending-for-spokes: + portfolio: demo-central-it-team-portfolio + depends_on: + - account-iam-for-spokes + associations: + - arn:aws:iam::${AWS::AccountId}:role/MyServiceCatalogAdminRole + constraints: + launch: + - product: account-vending-account-creation-shared + roles: + - arn:aws:iam::${AWS::AccountId}:role/MyServiceCatalogAdminRole + deploy_to: + tags: + - tag: scope:spoke + regions: default_region + +The example above will create the portfolio once the ``depends_on`` launches have completed successfully. + + +How can I add an association? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The example above will add an association for the IAM principal: + +``arn:aws:iam::${AWS::AccountId}:role/MyServiceCatalogAdminRole`` + +so the portfolio will be accessible for anyone assuming that role. In addition to roles, you can also specify the ARN of +users and groups. + +.. note:: + + Using ``${AWS::AccountId}`` will evaluate in the spoke account. + + +How can I add a launch constraint? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The example above will add a launch constraint for the IAM role: + +``arn:aws:iam::${AWS::AccountId}:role/MyServiceCatalogAdminRole`` + +so they can launch the product ``account-vending-account-creation-shared`` in the spoke account. + +.. warning:: + + You can only specify an IAM role and the role must be assumable by the AWS service principal ``servicecatalog.amazonaws.com`` + +.. note:: + + Using ``${AWS::AccountId}`` will evaluate in the spoke account. + + +----------------------------------------------- +What is the recommended implementation pattern? +----------------------------------------------- + +#. Add an entry to launches that will provision a product into to your matching spokes. This product should provide the IAM roles your users will assume to interact with the portfolio you are going to add. + +#. Add an entry to spoke-local-portfolios to add a portfolio to your matching spokes. This should depend on the product you launched that contains the IAM roles you added to the launches section of your manifest. + +------------------------------------- +Is there anything else I should know? +------------------------------------- +#. It would be good to become familar with the `AWS Service Catalog pricing `_ before using this feature. diff --git a/servicecatalog_puppet/aws.py b/servicecatalog_puppet/aws.py index de41259e0..6d8b12a98 100644 --- a/servicecatalog_puppet/aws.py +++ b/servicecatalog_puppet/aws.py @@ -2,7 +2,8 @@ import time import yaml -from servicecatalog_puppet.constants import PREFIX +from servicecatalog_puppet import constants +from betterboto import client as betterboto_client logger = logging.getLogger(__file__) @@ -66,6 +67,7 @@ def get_stack_output_for(cloudformation, stack_name): def get_default_parameters_for_stack(cloudformation, stack_name): logger.info(f"Getting default parameters for for {stack_name}") existing_stack_params_dict = {} + #errored summary_response = cloudformation.get_template_summary( StackName=stack_name, ) @@ -96,7 +98,7 @@ def provision_product( params, version, ): - stack_name = "-".join([PREFIX, account_id, region, launch_name]) + stack_name = "-".join([constants.PREFIX, account_id, region, launch_name]) logger.info(f"[{launch_name}] {account_id}:{region} :: Creating a plan") regional_sns_topic = f"arn:aws:sns:{region}:{puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events" provisioning_parameters = [] @@ -176,8 +178,10 @@ def provision_product( f"waiting for change to complete: {response.get('ProvisionedProductDetail').get('Status')}" ) execute_status = response.get('ProvisionedProductDetail').get('Status') - if execute_status in ['AVAILABLE', 'TAINTED', 'ERROR']: + if execute_status in ['AVAILABLE', 'TAINTED']: break + elif execute_status == 'ERROR': + raise Exception(f"[{launch_name}] {account_id}:{region} :: Execute failed: {execute_status}") else: time.sleep(5) @@ -185,13 +189,11 @@ def provision_product( return provisioned_product_id else: - logger.error(f"[{launch_name}] {account_id}:{region} :: Execute failed: {execute_status}") - return False + raise Exception(f"[{launch_name}] {account_id}:{region} :: Execute failed: {execute_status}") else: - logger.error(f"[{launch_name}] {account_id}:{region} :: " + raise Exception(f"[{launch_name}] {account_id}:{region} :: " f"Plan failed: {response.get('ProvisionedProductPlanDetails').get('StatusMessage')}") - return False def get_path_for_product(service_catalog, product_id): @@ -241,3 +243,114 @@ def ensure_is_terminated( logger.info(f"Finished ensuring {provisioned_product_name} is terminated") return provisioned_product_id, provisioning_artifact_id + + +def get_provisioning_artifact_id_for(portfolio_name, product_name, version_name, account_id, region): + logger.info("Getting provisioning artifact id for: {} {} {} in the region: {} of account: {}".format( + portfolio_name, product_name, version_name, region, account_id + )) + role = "arn:aws:iam::{}:role/{}".format(account_id, 'servicecatalog-puppet/PuppetRole') + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, "-".join([account_id, region]), region_name=region + ) as cross_account_servicecatalog: + product_id = None + version_id = None + portfolio_id = None + args = {} + while True: + response = cross_account_servicecatalog.list_accepted_portfolio_shares() + assert response.get('NextPageToken') is None, "Pagination not supported" + for portfolio_detail in response.get('PortfolioDetails'): + if portfolio_detail.get('DisplayName') == portfolio_name: + portfolio_id = portfolio_detail.get('Id') + break + + if portfolio_id is None: + response = cross_account_servicecatalog.list_portfolios() + for portfolio_detail in response.get('PortfolioDetails', []): + if portfolio_detail.get('DisplayName') == portfolio_name: + portfolio_id = portfolio_detail.get('Id') + break + + assert portfolio_id is not None, "Could not find portfolio" + logger.info("Found portfolio: {}".format(portfolio_id)) + + args['PortfolioId'] = portfolio_id + response = cross_account_servicecatalog.search_products_as_admin( + **args + ) + for product_view_details in response.get('ProductViewDetails'): + product_view = product_view_details.get('ProductViewSummary') + if product_view.get('Name') == product_name: + logger.info('Found product: {}'.format(product_view)) + product_id = product_view.get('ProductId') + if response.get('NextPageToken', None) is not None: + args['PageToken'] = response.get('NextPageToken') + else: + break + assert product_id is not None, "Did not find product looking for" + + response = cross_account_servicecatalog.list_provisioning_artifacts( + ProductId=product_id + ) + assert response.get('NextPageToken') is None, "Pagination not support" + for provisioning_artifact_detail in response.get('ProvisioningArtifactDetails'): + if provisioning_artifact_detail.get('Name') == version_name: + version_id = provisioning_artifact_detail.get('Id') + assert version_id is not None, "Did not find version looking for" + return product_id, version_id + + +def get_portfolio_for(portfolio_name, account_id, region): + logger.info(f"Getting portfolio id for: {portfolio_name}") + role = f"arn:aws:iam::{account_id}:role/servicecatalog-puppet/PuppetRole" + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, "-".join([account_id, region]), region_name=region + ) as cross_account_servicecatalog: + portfolio = None + while True: + response = cross_account_servicecatalog.list_accepted_portfolio_shares() + assert response.get('NextPageToken') is None, "Pagination not supported" + for portfolio_detail in response.get('PortfolioDetails'): + if portfolio_detail.get('DisplayName') == portfolio_name: + portfolio = portfolio_detail + break + + if portfolio is None: + response = cross_account_servicecatalog.list_portfolios() + for portfolio_detail in response.get('PortfolioDetails', []): + if portfolio_detail.get('DisplayName') == portfolio_name: + portfolio = portfolio_detail + break + + assert portfolio is not None, "Could not find portfolio" + logger.info(f"Found portfolio: {portfolio}") + return portfolio + + +def ensure_portfolio(service_catalog, portfolio_name, provider_name, description=None): + return find_portfolio(service_catalog, portfolio_name) \ + or create_portfolio(service_catalog, portfolio_name, provider_name, description) + + +def find_portfolio(service_catalog, portfolio_searching_for): + logger.info('Searching for portfolio for: {}'.format(portfolio_searching_for)) + response = service_catalog.list_portfolios_single_page() + for detail in response.get('PortfolioDetails'): + if detail.get('DisplayName') == portfolio_searching_for: + logger.info('Found portfolio: {}'.format(portfolio_searching_for)) + return detail + return False + + +def create_portfolio(service_catalog, portfolio_name, provider_name, description=None): + logger.info(f'Creating portfolio: {portfolio_name}') + args = { + 'DisplayName': portfolio_name, + 'ProviderName': provider_name, + } + if description is not None: + args['Description'] = description + return service_catalog.create_portfolio( + **args + ).get('PortfolioDetail') diff --git a/servicecatalog_puppet/cli.py b/servicecatalog_puppet/cli.py index a66a465ae..784511880 100644 --- a/servicecatalog_puppet/cli.py +++ b/servicecatalog_puppet/cli.py @@ -1,47 +1,9 @@ # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -import copy -import shutil -import json - -import luigi import click -import pkg_resources -import yaml -import logging -import os - -from jinja2 import Template -from pykwalify.core import Core -from betterboto import client as betterboto_client - -from servicecatalog_puppet import aws -from servicecatalog_puppet.luigi_tasks_and_targets import ProvisionProductTask, SetSSMParamFromProvisionProductTask, \ - TerminateProductTask -from servicecatalog_puppet.commands.list_launches import do_list_launches -from servicecatalog_puppet.utils import manifest as manifest_utils -from servicecatalog_puppet.asset_helpers import resolve_from_site_packages, read_from_site_packages -from servicecatalog_puppet.constants import CONFIG_PARAM_NAME, CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN -from servicecatalog_puppet.core import get_org_iam_role_arn, create_share_template, \ - get_regions, get_provisioning_artifact_id_for -from servicecatalog_puppet.commands.bootstrap import do_bootstrap -from servicecatalog_puppet.commands.bootstrap_spoke import do_bootstrap_spoke -from servicecatalog_puppet.commands.expand import do_expand -from servicecatalog_puppet.utils.manifest import build_deployment_map -from servicecatalog_puppet.commands.bootstrap_org_master import do_bootstrap_org_master - -logger = logging.getLogger() -logger.setLevel(logging.INFO) - -PROVISIONED = 'provisioned' -TERMINATED = 'terminated' - -DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES = [ - 'depends_on', - 'outputs', - 'parameters', -] + +from servicecatalog_puppet import cli_commands @click.group() @@ -49,518 +11,97 @@ @click.option('--info-line-numbers/--no-info-line-numbers', default=False) def cli(info, info_line_numbers): """cli for pipeline tools""" - if info: - logging.basicConfig( - format='%(levelname)s %(threadName)s %(message)s', level=logging.INFO - ) - if info_line_numbers: - logging.basicConfig( - format='%(levelname)s %(threadName)s [%(filename)s:%(lineno)d] %(message)s', - datefmt='%Y-%m-%d:%H:%M:%S', - level=logging.INFO - ) - - -def get_puppet_account_id(): - with betterboto_client.ClientContextManager('sts') as sts: - return sts.get_caller_identity().get('Account') + cli_commands.cli(info, info_line_numbers) @cli.command() @click.argument('f', type=click.File()) def generate_shares(f): - logger.info('Starting to generate shares for: {}'.format(f.name)) - - manifest = manifest_utils.load(f) - deployment_map = build_deployment_map(manifest) - create_share_template(deployment_map, get_puppet_account_id()) - - -def set_regions_for_deployment_map(deployment_map): - logger.info('Starting to write the templates') - ALL_REGIONS = get_regions() - for account_id, account_details in deployment_map.items(): - for launch_name, launch_details in account_details.get('launches').items(): - logger.info('Looking at account: {} and launch: {}'.format(account_id, launch_name)) - if launch_details.get('match') == 'account_match': - logger.info('Setting regions for account matched') - for a in launch_details.get('deploy_to').get('accounts'): - if a.get('account_id') == account_id: - regions = a.get('regions') - if regions == "enabled": - regions = account_details.get('regions_enabled') - elif regions == "default_region" or regions is None: - regions = account_details.get('default_region') - elif regions == "all": - regions = ALL_REGIONS - elif isinstance(regions, list): - for region in regions: - if region not in ALL_REGIONS: - raise Exception("Unknown region: {}".format(region)) - elif isinstance(regions, str) and regions in ALL_REGIONS: - pass - else: - raise Exception("Unknown regions: {}".format(regions)) - if isinstance(regions, str): - regions = [regions] - launch_details['regions'] = regions - - elif launch_details.get('match') == 'tag_match': - logger.info('Setting regions for tag matched') - for t in launch_details.get('deploy_to').get('tags'): - if t.get('tag') in account_details.get('tags'): - regions = t.get('regions') - if regions == "enabled": - regions = account_details.get('regions_enabled') - elif regions == "default_region" or regions is None: - regions = account_details.get('default_region') - elif regions == "all": - regions = ALL_REGIONS - elif isinstance(regions, list): - for region in regions: - if region not in ALL_REGIONS: - raise Exception("Unknown region: {}".format(region)) - elif isinstance(regions, str) and regions in ALL_REGIONS: - pass - else: - raise Exception("Unknown regions: {}".format(regions)) - if isinstance(regions, str): - regions = [regions] - launch_details['regions'] = regions - - assert launch_details.get('regions') is not None, "Launch {} has no regions set".format(launch_name) - launch_details['regional_details'] = {} - for region in launch_details.get('regions'): - logger.info('Starting region: {}'.format(region)) - product_id, version_id = get_provisioning_artifact_id_for( - launch_details.get('portfolio'), - launch_details.get('product'), - launch_details.get('version'), - account_id, - region - ) - launch_details['regional_details'][region] = { - 'product_id': product_id, - 'version_id': version_id, - } - return deployment_map - - -def get_parameters_for_launch(required_parameters, deployment_map, manifest, launch_details, account_id, status): - regular_parameters = [] - ssm_parameters = [] - - for required_parameter_name in required_parameters.keys(): - account_ssm_param = deployment_map.get(account_id).get('parameters', {}).get(required_parameter_name, {}).get('ssm') - account_regular_param = deployment_map.get(account_id).get('parameters', {}).get(required_parameter_name, {}).get('default') - - launch_params = launch_details.get('parameters', {}) - launch_ssm_param = launch_params.get(required_parameter_name, {}).get('ssm') - launch_regular_param = launch_params.get(required_parameter_name, {}).get('default') - - manifest_params = manifest.get('parameters', {}) - manifest_ssm_param = manifest_params.get(required_parameter_name, {}).get('ssm') - manifest_regular_param = manifest_params.get(required_parameter_name, {}).get('default') - - if status == PROVISIONED and account_ssm_param: - ssm_parameters.append( - get_ssm_config_for_parameter(account_ssm_param, required_parameter_name) - ) - elif status == PROVISIONED and account_regular_param: - regular_parameters.append({ - 'name': required_parameter_name, - 'value': str(account_regular_param), - }) - - elif launch_ssm_param: - ssm_parameters.append( - get_ssm_config_for_parameter(launch_ssm_param, required_parameter_name) - ) - elif launch_regular_param: - regular_parameters.append({ - 'name': required_parameter_name, - 'value': launch_regular_param, - }) - - elif status == PROVISIONED and manifest_ssm_param: - ssm_parameters.append( - get_ssm_config_for_parameter(manifest_ssm_param, required_parameter_name) - ) - elif status == PROVISIONED and manifest_regular_param: - regular_parameters.append({ - 'name': required_parameter_name, - 'value': manifest_regular_param, - }) - - return regular_parameters, ssm_parameters - - -def get_ssm_config_for_parameter(account_ssm_param, required_parameter_name): - if account_ssm_param.get('region') is not None: - return { - 'name': account_ssm_param.get('name'), - 'region': account_ssm_param.get('region'), - 'parameter_name': required_parameter_name, - } - else: - return { - 'name': account_ssm_param.get('name'), - 'parameter_name': required_parameter_name, - } + cli_commands.generate_shares(f) @cli.command() @click.argument('f', type=click.File()) @click.option('--single-account', default=None) def deploy(f, single_account): - manifest = manifest_utils.load(f) - deployment_map = build_deployment_map(manifest) - deployment_map = set_regions_for_deployment_map(deployment_map) - - all_tasks = {} - tasks_to_run = [] - puppet_account_id = get_puppet_account_id() - - for account_id, deployments_for_account in deployment_map.items(): - for launch_name, launch_details in deployments_for_account.get('launches').items(): - for region_name, regional_details in launch_details.get('regional_details').items(): - product_id = regional_details.get('product_id') - required_parameters = {} - - role = f"arn:aws:iam::{account_id}:role/servicecatalog-puppet/PuppetRole" - with betterboto_client.CrossAccountClientContextManager( - 'servicecatalog', role, f'sc-{account_id}-{region_name}', region_name=region_name - ) as service_catalog: - response = service_catalog.describe_provisioning_parameters( - ProductId=product_id, - ProvisioningArtifactId=regional_details.get('version_id'), - PathId=aws.get_path_for_product(service_catalog, product_id), - ) - for provisioning_artifact_parameters in response.get('ProvisioningArtifactParameters', []): - parameter_key = provisioning_artifact_parameters.get('ParameterKey') - required_parameters[parameter_key] = True - - regular_parameters, ssm_parameters = get_parameters_for_launch( - required_parameters, - deployment_map, - manifest, - launch_details, - account_id, - launch_details.get('status', PROVISIONED), - ) - - logger.info(f"Found a new launch: {launch_name}") - - task = { - 'launch_name': launch_name, - 'portfolio': launch_details.get('portfolio'), - 'product': launch_details.get('product'), - 'version': launch_details.get('version'), - - 'product_id': regional_details.get('product_id'), - 'version_id': regional_details.get('version_id'), - - 'account_id': account_id, - 'region': region_name, - 'puppet_account_id': puppet_account_id, - - 'parameters': regular_parameters, - 'ssm_param_inputs': ssm_parameters, - - 'depends_on': launch_details.get('depends_on', []), - - "status": launch_details.get('status', PROVISIONED), - - 'dependencies': [], - } - - if manifest.get('configuration'): - if manifest.get('configuration').get('retry_count'): - task['retry_count'] = manifest.get('configuration').get('retry_count') - - if launch_details.get('configuration'): - if launch_details.get('configuration').get('retry_count'): - task['retry_count'] = launch_details.get('configuration').get('retry_count') - - for output in launch_details.get('outputs', {}).get('ssm', []): - t = copy.deepcopy(task) - del t['depends_on'] - tasks_to_run.append( - SetSSMParamFromProvisionProductTask(**output, dependency=t) - ) - - all_tasks[f"{task.get('account_id')}-{task.get('region')}-{task.get('launch_name')}"] = task - - logger.info(f"Deployment plan: {json.dumps(all_tasks)}") - - for task in wire_dependencies(all_tasks): - task_status = task.get('status') - del task['status'] - if task_status == PROVISIONED: - tasks_to_run.append(ProvisionProductTask(**task)) - elif task_status == TERMINATED: - for attribute in DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES: - logger.info(f"checking {launch_name} for disallowed attributes") - attribute_value = task.get(attribute) - if attribute_value is not None: - if isinstance(attribute_value, list): - if len(attribute_value) != 0: - raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") - elif isinstance(attribute_value, dict): - if len(attribute_value.keys()) != 0: - raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") - else: - raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") - - for a in ['parameters', 'ssm_param_inputs', 'outputs', 'dependencies']: - if task.get(a, None) is not None: - del task[a] - tasks_to_run.append(TerminateProductTask(**task)) - else: - raise Exception(f"Unsupported status of {task_status}") - - luigi.build( - tasks_to_run, - local_scheduler=True, - detailed_summary=True, - workers=10, - log_level='INFO', - ) - - -def wire_dependencies(all_tasks): - tasks_to_run = [] - for task_uid, task in all_tasks.items(): - for dependency in task.get('depends_on', []): - for task_uid_2, task_2 in all_tasks.items(): - if task_2.get('launch_name') == dependency: - task.get('dependencies').append(task_2) - del task['depends_on'] - logger.info(f"Scheduling ProvisionProductTask: {json.dumps(task)}") - tasks_to_run.append(task) - return tasks_to_run + cli_commands.deploy(f, single_account) @cli.command() @click.argument('puppet_account_id') @click.argument('iam_role_arns', nargs=-1) def bootstrap_spoke_as(puppet_account_id, iam_role_arns): - cross_accounts = [] - index = 0 - for role in iam_role_arns: - cross_accounts.append( - (role, 'bootstrapping-role-{}'.format(index)) - ) - index += 1 - - with betterboto_client.CrossMultipleAccountsClientContextManager( - 'cloudformation', - cross_accounts - ) as cloudformation: - do_bootstrap_spoke(puppet_account_id, cloudformation, get_puppet_version()) + cli_commands.bootstrap_spoke_as(puppet_account_id, iam_role_arns) @cli.command() @click.argument('puppet_account_id') def bootstrap_spoke(puppet_account_id): - with betterboto_client.ClientContextManager('cloudformation') as cloudformation: - do_bootstrap_spoke(puppet_account_id, cloudformation, get_puppet_version()) + cli_commands.bootstrap_spoke(puppet_account_id) @cli.command() @click.argument('branch-name') def bootstrap_branch(branch_name): - do_bootstrap("https://github.com/awslabs/aws-service-catalog-puppet/archive/{}.zip".format(branch_name)) - - -def get_puppet_version(): - return pkg_resources.require("aws-service-catalog-puppet")[0].version + cli_commands.bootstrap_branch(branch_name) @cli.command() def bootstrap(): - do_bootstrap(get_puppet_version()) + cli_commands.bootstrap() @cli.command() @click.argument('complexity', default='simple') @click.argument('p', type=click.Path(exists=True)) def seed(complexity, p): - example = "manifest-{}.yaml".format(complexity) - shutil.copy2( - resolve_from_site_packages( - os.path.sep.join(['manifests', example]) - ), - os.path.sep.join([p, "manifest.yaml"]) - ) + cli_commands.seed(complexity, p) @cli.command() @click.argument('f', type=click.File()) def list_launches(f): - manifest = manifest_utils.load(f) - do_list_launches(manifest) + cli_commands.list_launches(f) @cli.command() @click.argument('f', type=click.File()) def expand(f): - click.echo('Expanding') - manifest = manifest_utils.load(f) - org_iam_role_arn = get_org_iam_role_arn() - if org_iam_role_arn is None: - click.echo('No org role set - not expanding') - new_manifest = manifest - else: - click.echo('Expanding using role: {}'.format(org_iam_role_arn)) - with betterboto_client.CrossAccountClientContextManager( - 'organizations', org_iam_role_arn, 'org-iam-role' - ) as client: - new_manifest = do_expand(manifest, client) - click.echo('Expanded') - new_name = f.name.replace(".yaml", '-expanded.yaml') - logger.info('Writing new manifest: {}'.format(new_name)) - with open(new_name, 'w') as output: - output.write( - yaml.safe_dump(new_manifest, default_flow_style=False) - ) + cli_commands.expand(f) @cli.command() @click.argument('f', type=click.File()) def validate(f): - logger.info('Validating {}'.format(f.name)) - c = Core(source_file=f.name, schema_files=[resolve_from_site_packages('schema.yaml')]) - c.validate(raise_exception=True) - click.echo("Finished validating: {}".format(f.name)) - click.echo("Finished validating: OK") + cli_commands.validate(f) @cli.command() def version(): - click.echo("cli version: {}".format(pkg_resources.require("aws-service-catalog-puppet")[0].version)) - with betterboto_client.ClientContextManager('ssm') as ssm: - response = ssm.get_parameter( - Name="service-catalog-puppet-regional-version" - ) - click.echo( - "regional stack version: {} for region: {}".format( - response.get('Parameter').get('Value'), - response.get('Parameter').get('ARN').split(':')[3] - ) - ) - response = ssm.get_parameter( - Name="service-catalog-puppet-version" - ) - click.echo( - "stack version: {}".format( - response.get('Parameter').get('Value'), - ) - ) + cli_commands.version() @cli.command() @click.argument('p', type=click.Path(exists=True)) def upload_config(p): - content = open(p, 'r').read() - with betterboto_client.ClientContextManager('ssm') as ssm: - ssm.put_parameter( - Name=CONFIG_PARAM_NAME, - Type='String', - Value=content, - Overwrite=True, - ) - click.echo("Uploaded config") + cli_commands.upload_config(p) @cli.command() @click.argument('org-iam-role-arn') def set_org_iam_role_arn(org_iam_role_arn): - with betterboto_client.ClientContextManager('ssm') as ssm: - ssm.put_parameter( - Name=CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN, - Type='String', - Value=org_iam_role_arn, - Overwrite=True, - ) - click.echo("Uploaded config") + cli_commands.set_org_iam_role_arn(org_iam_role_arn) @cli.command() @click.argument('puppet_account_id') def bootstrap_org_master(puppet_account_id): - with betterboto_client.ClientContextManager( - 'cloudformation', - ) as cloudformation: - org_iam_role_arn = do_bootstrap_org_master( - puppet_account_id, cloudformation, get_puppet_version() - ) - click.echo("Bootstrapped org master, org-iam-role-arn: {}".format(org_iam_role_arn)) + cli_commands.bootstrap_org_master(puppet_account_id) @cli.command() def quick_start(): - click.echo("Quick Start running...") - puppet_version = get_puppet_version() - with betterboto_client.ClientContextManager('sts') as sts: - puppet_account_id = sts.get_caller_identity().get('Account') - click.echo("Going to use puppet_account_id: {}".format(puppet_account_id)) - click.echo("Bootstrapping account as a spoke") - with betterboto_client.ClientContextManager('cloudformation') as cloudformation: - do_bootstrap_spoke(puppet_account_id, cloudformation, puppet_version) - - click.echo("Setting the config") - content = yaml.safe_dump({ - "regions": [ - 'eu-west-1', - 'eu-west-2', - 'eu-west-3' - ] - }) - with betterboto_client.ClientContextManager('ssm') as ssm: - ssm.put_parameter( - Name=CONFIG_PARAM_NAME, - Type='String', - Value=content, - Overwrite=True, - ) - click.echo("Bootstrapping account as the master") - org_iam_role_arn = do_bootstrap_org_master( - puppet_account_id, cloudformation, puppet_version - ) - ssm.put_parameter( - Name=CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN, - Type='String', - Value=org_iam_role_arn, - Overwrite=True, - ) - click.echo("Bootstrapping the account now!") - do_bootstrap(puppet_version) - - if os.path.exists('ServiceCatalogPuppet'): - click.echo("Found ServiceCatalogPuppet so not cloning or seeding") - else: - click.echo("Cloning for you") - command = "git clone " \ - "--config 'credential.helper=!aws codecommit credential-helper $@' " \ - "--config 'credential.UseHttpPath=true' " \ - "https://git-codecommit.{}.amazonaws.com/v1/repos/ServiceCatalogPuppet".format( - os.environ.get("AWS_DEFAULT_REGION") - ) - os.system(command) - click.echo("Seeding") - manifest = Template( - read_from_site_packages(os.path.sep.join(["manifests", "manifest-quickstart.yaml"])) - ).render( - ACCOUNT_ID=puppet_account_id - ) - open(os.path.sep.join(["ServiceCatalogPuppet", "manifest.yaml"]), 'w').write( - manifest - ) - click.echo("Pushing manifest") - os.system("cd ServiceCatalogPuppet && git add manifest.yaml && git commit -am 'initial add' && git push") - - click.echo("All done!") + cli_commands.quick_start() if __name__ == "__main__": diff --git a/servicecatalog_puppet/cli_command_helpers.py b/servicecatalog_puppet/cli_command_helpers.py new file mode 100644 index 000000000..d7069243e --- /dev/null +++ b/servicecatalog_puppet/cli_command_helpers.py @@ -0,0 +1,594 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import click + +import pkg_resources +import json +from jinja2 import Template + +from servicecatalog_puppet import asset_helpers, manifest_utils, aws, luigi_tasks_and_targets +from servicecatalog_puppet import constants +import logging + +import os +from threading import Thread + +import yaml +from betterboto import client as betterboto_client +from jinja2 import Environment, FileSystemLoader + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def get_regions(default_region=None): + logger.info("getting regions, default_region: {}".format(default_region)) + with betterboto_client.ClientContextManager( + 'ssm', + region_name=default_region if default_region else get_home_region() + ) as ssm: + response = ssm.get_parameter(Name=constants.CONFIG_PARAM_NAME) + config = yaml.safe_load(response.get('Parameter').get('Value')) + return config.get('regions') + + +def get_home_region(): + with betterboto_client.ClientContextManager('ssm') as ssm: + response = ssm.get_parameter(Name=constants.HOME_REGION_PARAM_NAME) + return response.get('Parameter').get('Value') + + +def get_org_iam_role_arn(): + with betterboto_client.ClientContextManager('ssm', region_name=get_home_region()) as ssm: + try: + response = ssm.get_parameter(Name=constants.CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN) + return yaml.safe_load(response.get('Parameter').get('Value')) + except ssm.exceptions.ParameterNotFound as e: + logger.info("No parameter set for: {}".format(constants.CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN)) + return None + + +def generate_bucket_policies_for_shares(deployment_map, puppet_account_id): + shares = { + 'accounts': [], + 'organizations': [], + } + for account_id, deployment in deployment_map.items(): + if account_id == puppet_account_id: + continue + if deployment.get('expanded_from') is None: + if account_id not in shares['accounts']: + shares['accounts'].append(account_id) + else: + if deployment.get('organization') not in shares['organizations']: + shares['organizations'].append(deployment.get('organization')) + return shares + + +def write_share_template(portfolio_use_by_account, region, host_account_id, sharing_policies): + output = os.path.sep.join([constants.TEMPLATES, 'shares', region]) + if not os.path.exists(output): + os.makedirs(output) + + with open(os.sep.join([output, "shares.template.yaml"]), 'w') as f: + f.write( + env.get_template('shares.template.yaml.j2').render( + portfolio_use_by_account=portfolio_use_by_account, + host_account_id=host_account_id, + HOME_REGION=get_home_region(), + sharing_policies=sharing_policies, + ) + ) + + +def create_share_template(deployment_map, puppet_account_id): + logger.info("deployment_map: {}".format(deployment_map)) + ALL_REGIONS = get_regions() + for region in ALL_REGIONS: + logger.info("starting to build shares for region: {}".format(region)) + with betterboto_client.ClientContextManager('servicecatalog', region_name=region) as servicecatalog: + portfolio_ids = {} + args = {} + while True: + + response = servicecatalog.list_portfolios( + **args + ) + + for portfolio_detail in response.get('PortfolioDetails'): + portfolio_ids[portfolio_detail.get('DisplayName')] = portfolio_detail.get('Id') + + if response.get('PageToken') is not None: + args['PageToken'] = response.get('PageToken') + else: + break + + logger.info("Portfolios in use in region: {}".format(portfolio_ids)) + + portfolio_use_by_account = {} + for account_id, launch_details in deployment_map.items(): + if portfolio_use_by_account.get(account_id) is None: + portfolio_use_by_account[account_id] = [] + for launch_id, launch in launch_details.get('launches').items(): + logger.info("portfolio ids: {}".format(portfolio_ids)) + p = portfolio_ids[launch.get('portfolio')] + if p not in portfolio_use_by_account[account_id]: + portfolio_use_by_account[account_id].append(p) + host_account_id = response.get('PortfolioDetails')[0].get('ARN').split(":")[4] + sharing_policies = generate_bucket_policies_for_shares(deployment_map, puppet_account_id) + write_share_template(portfolio_use_by_account, region, host_account_id, sharing_policies) + + +template_dir = asset_helpers.resolve_from_site_packages('templates') +env = Environment( + loader=FileSystemLoader(template_dir), + extensions=['jinja2.ext.do'], +) + + +def get_puppet_account_id(): + with betterboto_client.ClientContextManager('sts') as sts: + return sts.get_caller_identity().get('Account') + + +def set_regions_for_deployment_map(deployment_map, section): + logger.info('Starting to write the templates') + ALL_REGIONS = get_regions() + for account_id, account_details in deployment_map.items(): + for launch_name, launch_details in account_details.get(section).items(): + logger.info('Looking at account: {} and launch: {}'.format(account_id, launch_name)) + if launch_details.get('match') == 'account_match': + logger.info('Setting regions for account matched') + for a in launch_details.get('deploy_to').get('accounts'): + if a.get('account_id') == account_id: + regions = a.get('regions') + if regions == "enabled": + regions = account_details.get('regions_enabled') + elif regions == "default_region" or regions is None: + regions = account_details.get('default_region') + elif regions == "all": + regions = ALL_REGIONS + elif isinstance(regions, list): + for region in regions: + if region not in ALL_REGIONS: + raise Exception("Unknown region: {}".format(region)) + elif isinstance(regions, str) and regions in ALL_REGIONS: + pass + else: + raise Exception("Unknown regions: {}".format(regions)) + if isinstance(regions, str): + regions = [regions] + launch_details['regions'] = regions + + elif launch_details.get('match') == 'tag_match': + logger.info('Setting regions for tag matched') + for t in launch_details.get('deploy_to').get('tags'): + if t.get('tag') in account_details.get('tags'): + regions = t.get('regions') + if regions == "enabled": + regions = account_details.get('regions_enabled') + elif regions == "default_region" or regions is None: + regions = account_details.get('default_region') + elif regions == "all": + regions = ALL_REGIONS + elif isinstance(regions, list): + for region in regions: + if region not in ALL_REGIONS: + raise Exception("Unknown region: {}".format(region)) + elif isinstance(regions, str) and regions in ALL_REGIONS: + pass + else: + raise Exception("Unknown regions: {}".format(regions)) + if isinstance(regions, str): + regions = [regions] + launch_details['regions'] = regions + + assert launch_details.get('regions') is not None, "Launch {} has no regions set".format(launch_name) + launch_details['regional_details'] = {} + + if section == constants.LAUNCHES: + # TODO move this to provision product task so this if statement is no longer needed + for region in launch_details.get('regions'): + logger.info('Starting region: {}'.format(region)) + product_id, version_id = aws.get_provisioning_artifact_id_for( + launch_details.get('portfolio'), + launch_details.get('product'), + launch_details.get('version'), + account_id, + region + ) + launch_details['regional_details'][region] = { + 'product_id': product_id, + 'version_id': version_id, + } + return deployment_map + + +def get_parameters_for_launch(required_parameters, deployment_map, manifest, launch_details, account_id, status): + regular_parameters = [] + ssm_parameters = [] + + for required_parameter_name in required_parameters.keys(): + account_ssm_param = deployment_map.get(account_id).get('parameters', {}).get(required_parameter_name, {}).get( + 'ssm') + account_regular_param = deployment_map.get(account_id).get('parameters', {}).get(required_parameter_name, + {}).get('default') + + launch_params = launch_details.get('parameters', {}) + launch_ssm_param = launch_params.get(required_parameter_name, {}).get('ssm') + launch_regular_param = launch_params.get(required_parameter_name, {}).get('default') + + manifest_params = manifest.get('parameters', {}) + manifest_ssm_param = manifest_params.get(required_parameter_name, {}).get('ssm') + manifest_regular_param = manifest_params.get(required_parameter_name, {}).get('default') + + if status == constants.PROVISIONED and account_ssm_param: + ssm_parameters.append( + get_ssm_config_for_parameter(account_ssm_param, required_parameter_name) + ) + elif status == constants.PROVISIONED and account_regular_param: + regular_parameters.append({ + 'name': required_parameter_name, + 'value': str(account_regular_param), + }) + + elif launch_ssm_param: + ssm_parameters.append( + get_ssm_config_for_parameter(launch_ssm_param, required_parameter_name) + ) + elif launch_regular_param: + regular_parameters.append({ + 'name': required_parameter_name, + 'value': launch_regular_param, + }) + + elif status == constants.PROVISIONED and manifest_ssm_param: + ssm_parameters.append( + get_ssm_config_for_parameter(manifest_ssm_param, required_parameter_name) + ) + elif status == constants.PROVISIONED and manifest_regular_param: + regular_parameters.append({ + 'name': required_parameter_name, + 'value': manifest_regular_param, + }) + + return regular_parameters, ssm_parameters + + +def get_ssm_config_for_parameter(account_ssm_param, required_parameter_name): + if account_ssm_param.get('region') is not None: + return { + 'name': account_ssm_param.get('name'), + 'region': account_ssm_param.get('region'), + 'parameter_name': required_parameter_name, + } + else: + return { + 'name': account_ssm_param.get('name'), + 'parameter_name': required_parameter_name, + } + + +def wire_dependencies(all_tasks): + tasks_to_run = [] + for task_uid, task in all_tasks.items(): + for dependency in task.get('depends_on', []): + for task_uid_2, task_2 in all_tasks.items(): + if task_2.get('launch_name') == dependency: + task.get('dependencies').append(task_2) + del task['depends_on'] + tasks_to_run.append(task) + return tasks_to_run + + +def get_puppet_version(): + return pkg_resources.require("aws-service-catalog-puppet")[0].version + + +def _do_bootstrap_org_master(puppet_account_id, cloudformation, puppet_version): + logger.info('Starting bootstrap of org master') + stack_name = "{}-org-master".format(constants.BOOTSTRAP_STACK_NAME) + template = asset_helpers.read_from_site_packages('{}.template.yaml'.format(stack_name)) + template = Template(template).render(VERSION=puppet_version) + args = { + 'StackName': stack_name, + 'TemplateBody': template, + 'Capabilities': ['CAPABILITY_NAMED_IAM'], + 'Parameters': [ + { + 'ParameterKey': 'PuppetAccountId', + 'ParameterValue': str(puppet_account_id), + }, { + 'ParameterKey': 'Version', + 'ParameterValue': puppet_version, + 'UsePreviousValue': False, + }, + ], + } + cloudformation.create_or_update(**args) + response = cloudformation.describe_stacks(StackName=stack_name) + if len(response.get('Stacks')) != 1: + raise Exception("Expected there to be only one {} stack".format(stack_name)) + stack = response.get('Stacks')[0] + + for output in stack.get('Outputs'): + if output.get('OutputKey') == constants.PUPPET_ORG_ROLE_FOR_EXPANDS_ARN: + logger.info('Finished bootstrap of org-master') + return output.get("OutputValue") + + raise Exception( + "Could not find output: {} in stack: {}".format(constants.PUPPET_ORG_ROLE_FOR_EXPANDS_ARN, stack_name)) + + +def _do_bootstrap_spoke(puppet_account_id, cloudformation, puppet_version): + logger.info('Starting bootstrap of spoke') + template = asset_helpers.read_from_site_packages('{}-spoke.template.yaml'.format(constants.BOOTSTRAP_STACK_NAME)) + template = Template(template).render(VERSION=puppet_version) + args = { + 'StackName': "{}-spoke".format(constants.BOOTSTRAP_STACK_NAME), + 'TemplateBody': template, + 'Capabilities': ['CAPABILITY_NAMED_IAM'], + 'Parameters': [ + { + 'ParameterKey': 'PuppetAccountId', + 'ParameterValue': str(puppet_account_id), + }, { + 'ParameterKey': 'Version', + 'ParameterValue': puppet_version, + 'UsePreviousValue': False, + }, + ], + } + cloudformation.create_or_update(**args) + logger.info('Finished bootstrap of spoke') + + +def _do_bootstrap(puppet_version): + click.echo('Starting bootstrap') + ALL_REGIONS = get_regions(os.environ.get("AWS_DEFAULT_REGION")) + with betterboto_client.MultiRegionClientContextManager('cloudformation', ALL_REGIONS) as clients: + click.echo('Creating {}-regional'.format(constants.BOOTSTRAP_STACK_NAME)) + threads = [] + template = asset_helpers.read_from_site_packages( + '{}.template.yaml'.format('{}-regional'.format(constants.BOOTSTRAP_STACK_NAME))) + template = Template(template).render(VERSION=puppet_version) + args = { + 'StackName': '{}-regional'.format(constants.BOOTSTRAP_STACK_NAME), + 'TemplateBody': template, + 'Capabilities': ['CAPABILITY_IAM'], + 'Parameters': [ + { + 'ParameterKey': 'Version', + 'ParameterValue': puppet_version, + 'UsePreviousValue': False, + }, + { + 'ParameterKey': 'DefaultRegionValue', + 'ParameterValue': os.environ.get('AWS_DEFAULT_REGION'), + 'UsePreviousValue': False, + }, + ], + } + for client_region, client in clients.items(): + process = Thread(name=client_region, target=client.create_or_update, kwargs=args) + process.start() + threads.append(process) + for process in threads: + process.join() + click.echo('Finished creating {}-regional'.format(constants.BOOTSTRAP_STACK_NAME)) + + with betterboto_client.ClientContextManager('cloudformation') as cloudformation: + click.echo('Creating {}'.format(constants.BOOTSTRAP_STACK_NAME)) + template = asset_helpers.read_from_site_packages('{}.template.yaml'.format(constants.BOOTSTRAP_STACK_NAME)) + template = Template(template).render(VERSION=puppet_version, ALL_REGIONS=ALL_REGIONS) + args = { + 'StackName': constants.BOOTSTRAP_STACK_NAME, + 'TemplateBody': template, + 'Capabilities': ['CAPABILITY_NAMED_IAM'], + 'Parameters': [ + { + 'ParameterKey': 'Version', + 'ParameterValue': puppet_version, + 'UsePreviousValue': False, + }, + { + 'ParameterKey': 'OrgIamRoleArn', + 'ParameterValue': str(get_org_iam_role_arn()), + 'UsePreviousValue': False, + }, + ], + } + cloudformation.create_or_update(**args) + + click.echo('Finished creating {}.'.format(constants.BOOTSTRAP_STACK_NAME)) + with betterboto_client.ClientContextManager('codecommit') as codecommit: + response = codecommit.get_repository(repositoryName=constants.SERVICE_CATALOG_PUPPET_REPO_NAME) + clone_url = response.get('repositoryMetadata').get('cloneUrlHttp') + clone_command = "git clone --config 'credential.helper=!aws codecommit credential-helper $@' " \ + "--config 'credential.UseHttpPath=true' {}".format(clone_url) + click.echo( + 'You need to clone your newly created repo now and will then need to seed it: \n{}'.format( + clone_command + ) + ) + + +def deploy_spoke_local_portfolios(manifest, launch_tasks): + section = constants.SPOKE_LOCAL_PORTFOLIOS + deployment_map = manifest_utils.build_deployment_map(manifest, section) + deployment_map = set_regions_for_deployment_map(deployment_map, section) + + tasks_to_run = [] + puppet_account_id = get_puppet_account_id() + + for account_id, deployments_for_account in deployment_map.items(): + for launch_name, launch_details in deployments_for_account.get(section).items(): + for region_name in launch_details.get('regions'): + + depends_on = launch_details.get('depends_on') + dependencies = [] + for dependency in depends_on: + for task_uid, task in launch_tasks.items(): + if task.get('launch_name') == dependency: + dependencies.append(task) + + hub_portfolio = aws.get_portfolio_for( + launch_details.get('portfolio'), puppet_account_id, region_name + ) + + create_spoke_local_portfolio_task_params = { + 'account_id': account_id, + 'region': region_name, + 'portfolio': launch_details.get('portfolio'), + 'provider_name': hub_portfolio.get('ProviderName'), + 'description': hub_portfolio.get('Description'), + } + create_spoke_local_portfolio_task = luigi_tasks_and_targets.CreateSpokeLocalPortfolioTask( + **create_spoke_local_portfolio_task_params + ) + tasks_to_run.append(create_spoke_local_portfolio_task) + + create_spoke_local_portfolio_task_as_dependency_params = { + 'account_id': account_id, + 'region': region_name, + 'portfolio': launch_details.get('portfolio'), + } + + create_associations_task_params = { + 'associations': launch_details.get('associations'), + 'puppet_account_id': puppet_account_id, + } + create_associations_for_portfolio_task = luigi_tasks_and_targets.CreateAssociationsForPortfolioTask( + **create_spoke_local_portfolio_task_as_dependency_params, + **create_associations_task_params, + dependencies=dependencies, + ) + tasks_to_run.append(create_associations_for_portfolio_task) + + import_into_spoke_local_portfolio_task_params = { + 'hub_portfolio_id': hub_portfolio.get('Id') + } + import_into_spoke_local_portfolio_task = luigi_tasks_and_targets.ImportIntoSpokeLocalPortfolioTask( + **create_spoke_local_portfolio_task_as_dependency_params, + **import_into_spoke_local_portfolio_task_params, + ) + tasks_to_run.append(import_into_spoke_local_portfolio_task) + + create_launch_role_constraints_for_portfolio_task_params = { + 'launch_constraints': launch_details.get('constraints', {}).get('launch', []), + 'puppet_account_id': puppet_account_id, + } + create_launch_role_constraints_for_portfolio = luigi_tasks_and_targets.CreateLaunchRoleConstraintsForPortfolio( + **create_spoke_local_portfolio_task_as_dependency_params, + **import_into_spoke_local_portfolio_task_params, + **create_launch_role_constraints_for_portfolio_task_params, + dependencies=dependencies, + ) + tasks_to_run.append(create_launch_role_constraints_for_portfolio) + + return tasks_to_run + + +def deploy_launches(manifest): + section = constants.LAUNCHES + deployment_map = manifest_utils.build_deployment_map(manifest, section) + deployment_map = set_regions_for_deployment_map(deployment_map, section) + puppet_account_id = get_puppet_account_id() + + all_tasks = deploy_launches_task_builder(deployment_map, manifest, puppet_account_id, section) + + logger.info(f"Deployment plan: {json.dumps(all_tasks)}") + return all_tasks + + +def deploy_launches_task_builder(deployment_map, manifest, puppet_account_id, section): + all_tasks = {} + for account_id, deployments_for_account in deployment_map.items(): + for launch_name, launch_details in deployments_for_account.get(section).items(): + for region_name, regional_details in launch_details.get('regional_details').items(): + these_all_tasks = deploy_launches_task_builder_for_account_launch_region( + account_id, + deployment_map, + launch_details, + launch_name, + manifest, + puppet_account_id, + region_name, + regional_details, + ) + all_tasks.update(these_all_tasks) + + return all_tasks + + +def deploy_launches_task_builder_for_account_launch_region( + account_id, deployment_map, launch_details, launch_name, manifest, + puppet_account_id, region_name, regional_details +): + # if launch_details.get('product') == 'account-vending-account-creation-shared': + # logger.info(launch_details.get('product')) + # logger.info(json.dumps(deployment_map)) + # logger.info(json.dumps(launch_details)) + # logger.info(json.dumps(manifest)) + # logger.info(json.dumps(regional_details)) + # raise Exception('doo') + all_tasks = {} + product_id = regional_details.get('product_id') + required_parameters = {} + role = f"arn:aws:iam::{account_id}:role/servicecatalog-puppet/PuppetRole" + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, f'sc-{account_id}-{region_name}', region_name=region_name + ) as service_catalog: + response = service_catalog.describe_provisioning_parameters( + ProductId=product_id, + ProvisioningArtifactId=regional_details.get('version_id'), + PathId=aws.get_path_for_product(service_catalog, product_id), + ) + for provisioning_artifact_parameters in response.get('ProvisioningArtifactParameters', []): + parameter_key = provisioning_artifact_parameters.get('ParameterKey') + required_parameters[parameter_key] = True + + regular_parameters, ssm_parameters = get_parameters_for_launch( + required_parameters, + deployment_map, + manifest, + launch_details, + account_id, + launch_details.get('status', constants.PROVISIONED), + ) + logger.info(f"Found a new launch: {launch_name}") + task = { + 'launch_name': launch_name, + 'portfolio': launch_details.get('portfolio'), + 'product': launch_details.get('product'), + 'version': launch_details.get('version'), + + 'product_id': regional_details.get('product_id'), + 'version_id': regional_details.get('version_id'), + + 'account_id': account_id, + 'region': region_name, + 'puppet_account_id': puppet_account_id, + + 'parameters': regular_parameters, + 'ssm_param_inputs': ssm_parameters, + + 'depends_on': launch_details.get('depends_on', []), + + "status": launch_details.get('status', constants.PROVISIONED), + + "worker_timeout": launch_details.get('timeoutInSeconds', constants.DEFAULT_TIMEOUT), + + "ssm_param_outputs": launch_details.get('outputs', {}).get('ssm', []), + + 'dependencies': [], + } + if manifest.get('configuration'): + if manifest.get('configuration').get('retry_count'): + task['retry_count'] = manifest.get('configuration').get('retry_count') + if launch_details.get('configuration'): + if launch_details.get('configuration').get('retry_count'): + task['retry_count'] = launch_details.get('configuration').get('retry_count') + + all_tasks[f"{task.get('account_id')}-{task.get('region')}-{task.get('launch_name')}"] = task + return all_tasks diff --git a/servicecatalog_puppet/cli_command_helpers_unit_test.py b/servicecatalog_puppet/cli_command_helpers_unit_test.py new file mode 100644 index 000000000..9e0721344 --- /dev/null +++ b/servicecatalog_puppet/cli_command_helpers_unit_test.py @@ -0,0 +1,94 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pytest import fixture +import json +from servicecatalog_puppet import constants + + +@fixture +def sut(): + from servicecatalog_puppet import cli_command_helpers + return cli_command_helpers + + +def test_wire_dependencies(sut, shared_datadir): + # setup + all_tasks = json.loads((shared_datadir / 'account-vending' / 'all-tasks.json').read_text()) + launch_a = json.loads((shared_datadir / 'account-vending' / 'launch-a.json').read_text()) + launch_b = json.loads((shared_datadir / 'account-vending' / 'launch-b.json').read_text()) + launch_c = json.loads((shared_datadir / 'account-vending' / 'launch-c.json').read_text()) + + expected_results = [ + launch_a, + launch_b, + launch_c + ] + + # exercise + actual_results = sut.wire_dependencies(all_tasks) + + # verify + assert actual_results == expected_results + + +def test_deploy_launches_task_builder_for_account_launch_region(sut, mocker, shared_datadir): + # setup + account_id = '0123456789010' + deployment_map = json.loads((shared_datadir / 'account-vending' / 'deployment-map.json').read_text()) + launch_details = json.loads((shared_datadir / 'account-vending' / 'launch-details-for-launch-b.json').read_text()) + launch_name = launch_details.get('launch_name') + manifest = json.loads((shared_datadir / 'account-vending' / 'manifest.json').read_text()) + puppet_account_id = "098765432101" + region_name = 'eu-west-1' + regional_details = {"product_id": "prod-lv3isrxiingdo", "version_id": "pa-yprmofsvvyih4"} + expected_all_tasks = json.loads((shared_datadir / 'account-vending' / 'all-tasks-for-launch-b.json').read_text()) + mocked_betterboto_client = mocker.patch.object(sut.betterboto_client, 'CrossAccountClientContextManager') + mocked_describe_provisioning_parameters_response = { + 'ProvisioningArtifactParameters': [ + {'ParameterKey': 'IamUserAccessToBilling'}, + {'ParameterKey': 'Email'}, + {'ParameterKey': 'TargetOU'}, + {'ParameterKey': 'OrganizationAccountAccessRole'}, + {'ParameterKey': 'AccountName'}, + {'ParameterKey': 'AccountVendingCreationLambdaArn'}, + {'ParameterKey': 'AccountVendingBootstrapperLambdaArn'}, + ] + } + required_parameters = { + 'IamUserAccessToBilling': True, + 'Email': True, + 'TargetOU': True, + 'OrganizationAccountAccessRole': True, + 'AccountName': True, + 'AccountVendingCreationLambdaArn': True, + 'AccountVendingBootstrapperLambdaArn': True, + } + mocked_betterboto_client().__enter__().describe_provisioning_parameters.return_value = mocked_describe_provisioning_parameters_response + mocked_get_path_for_product = mocker.patch.object(sut.aws, 'get_path_for_product') + mocked_get_path_for_product.return_value = 1 + mocked_get_parameters_for_launch = mocker.patch.object(sut, 'get_parameters_for_launch') + mocked_regular_parameters = [] + mocked_ssm_parameters = [] + mocked_get_parameters_for_launch.return_value = (mocked_regular_parameters, mocked_ssm_parameters) + + # exercise + actual_all_tasks = sut.deploy_launches_task_builder_for_account_launch_region( + account_id, deployment_map, launch_details, launch_name, manifest, + puppet_account_id, region_name, regional_details + ) + + # verify + assert len(actual_all_tasks.keys()) == 1 + assert actual_all_tasks == expected_all_tasks + mocked_get_path_for_product.assert_called_once_with( + mocked_betterboto_client().__enter__(), regional_details.get('product_id') + ) + mocked_get_parameters_for_launch.assert_called_once_with( + required_parameters, + deployment_map, + manifest, + launch_details, + account_id, + launch_details.get('status', constants.PROVISIONED), + ) diff --git a/servicecatalog_puppet/cli_commands.py b/servicecatalog_puppet/cli_commands.py new file mode 100644 index 000000000..a28a93007 --- /dev/null +++ b/servicecatalog_puppet/cli_commands.py @@ -0,0 +1,413 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import sys + +from colorclass import Color +from luigi import LuigiStatusCode +from luigi.execution_summary import LuigiRunResult +from terminaltables import AsciiTable + +import copy + +import shutil +import json + +import luigi +import pkg_resources +import yaml +import logging +import os +import click + +from jinja2 import Template +from pykwalify.core import Core +from betterboto import client as betterboto_client + + +from servicecatalog_puppet import cli_command_helpers +from servicecatalog_puppet import luigi_tasks_and_targets +from servicecatalog_puppet import manifest_utils + + +from servicecatalog_puppet import asset_helpers +from servicecatalog_puppet import constants + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def cli(info, info_line_numbers): + if info: + logging.basicConfig( + format='%(levelname)s %(threadName)s %(message)s', level=logging.INFO + ) + if info_line_numbers: + logging.basicConfig( + format='%(levelname)s %(threadName)s [%(filename)s:%(lineno)d] %(message)s', + datefmt='%Y-%m-%d:%H:%M:%S', + level=logging.INFO + ) + + +def generate_shares(f): + logger.info('Starting to generate shares for: {}'.format(f.name)) + + manifest = manifest_utils.load(f) + deployment_map = manifest_utils.build_deployment_map(manifest, constants.LAUNCHES) + cli_command_helpers.create_share_template(deployment_map, cli_command_helpers.get_puppet_account_id()) + + +def deploy(f, single_account): + manifest = manifest_utils.load(f) + + launch_tasks = {} + tasks_to_run = [] + + all_launch_tasks = cli_command_helpers.deploy_launches(manifest) + launch_tasks.update(all_launch_tasks) + + for task in cli_command_helpers.wire_dependencies(launch_tasks): + task_status = task.get('status') + del task['status'] + if task_status == constants.PROVISIONED: + tasks_to_run.append(luigi_tasks_and_targets.ProvisionProductTask(**task)) + elif task_status == constants.TERMINATED: + for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES: + logger.info(f"checking {task.get('launch_name')} for disallowed attributes") + attribute_value = task.get(attribute) + if attribute_value is not None: + if isinstance(attribute_value, list): + if len(attribute_value) != 0: + raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") + elif isinstance(attribute_value, dict): + if len(attribute_value.keys()) != 0: + raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") + else: + raise Exception(f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}") + + for a in ['parameters', 'ssm_param_inputs', 'outputs', 'dependencies']: + if task.get(a, None) is not None: + del task[a] + tasks_to_run.append(luigi_tasks_and_targets.TerminateProductTask(**task)) + else: + raise Exception(f"Unsupported status of {task_status}") + + spoke_local_portfolio_tasks_to_run = cli_command_helpers.deploy_spoke_local_portfolios(manifest, launch_tasks) + tasks_to_run += spoke_local_portfolio_tasks_to_run + + result = luigi.build( + tasks_to_run, + local_scheduler=True, + detailed_summary=True, + workers=10, + log_level='INFO', + ) + + # click.echo(f"Workflow complete {result} - {result.status.value[1]}") + exit_status_codes = { + LuigiStatusCode.SUCCESS: 0, + LuigiStatusCode.SUCCESS_WITH_RETRY: 0, + LuigiStatusCode.FAILED: 1, + LuigiStatusCode.FAILED_AND_SCHEDULING_FAILED: 2, + LuigiStatusCode.SCHEDULING_FAILED:3, + LuigiStatusCode.NOT_RUN:4, + LuigiStatusCode.MISSING_EXT:5, + } + sys.exit(exit_status_codes.get(result.status)) + + +def bootstrap_spoke_as(puppet_account_id, iam_role_arns): + cross_accounts = [] + index = 0 + for role in iam_role_arns: + cross_accounts.append( + (role, 'bootstrapping-role-{}'.format(index)) + ) + index += 1 + + with betterboto_client.CrossMultipleAccountsClientContextManager( + 'cloudformation', + cross_accounts + ) as cloudformation: + cli_command_helpers._do_bootstrap_spoke(puppet_account_id, cloudformation, cli_command_helpers.get_puppet_version()) + + +def bootstrap_spoke(puppet_account_id): + with betterboto_client.ClientContextManager('cloudformation') as cloudformation: + cli_command_helpers._do_bootstrap_spoke(puppet_account_id, cloudformation, cli_command_helpers.get_puppet_version()) + + +def bootstrap_branch(branch_name): + cli_command_helpers._do_bootstrap("https://github.com/awslabs/aws-service-catalog-puppet/archive/{}.zip".format(branch_name)) + + +def bootstrap(): + cli_command_helpers._do_bootstrap(cli_command_helpers.get_puppet_version()) + + +def seed(complexity, p): + example = "manifest-{}.yaml".format(complexity) + shutil.copy2( + asset_helpers.resolve_from_site_packages( + os.path.sep.join(['manifests', example]) + ), + os.path.sep.join([p, "manifest.yaml"]) + ) + + +def list_launches(f): + manifest = manifest_utils.load(f) + click.echo("Getting details from your account...") + ALL_REGIONS = cli_command_helpers.get_regions(os.environ.get("AWS_DEFAULT_REGION")) + deployment_map = manifest_utils.build_deployment_map(manifest, constants.LAUNCHES) + account_ids = [a.get('account_id') for a in manifest.get('accounts')] + deployments = {} + for account_id in account_ids: + for region_name in ALL_REGIONS: + role = "arn:aws:iam::{}:role/{}".format(account_id, 'servicecatalog-puppet/PuppetRole') + logger.info("Looking at region: {} in account: {}".format(region_name, account_id)) + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, 'sc-{}-{}'.format(account_id, region_name), region_name=region_name + ) as spoke_service_catalog: + + response = spoke_service_catalog.list_accepted_portfolio_shares() + portfolios = response.get('PortfolioDetails', []) + + response = spoke_service_catalog.list_portfolios() + portfolios += response.get('PortfolioDetails', []) + + for portfolio in portfolios: + portfolio_id = portfolio.get('Id') + response = spoke_service_catalog.search_products_as_admin(PortfolioId=portfolio_id) + for product_view_detail in response.get('ProductViewDetails', []): + product_view_summary = product_view_detail.get('ProductViewSummary') + product_id = product_view_summary.get('ProductId') + response = spoke_service_catalog.search_provisioned_products( + Filters={'SearchQuery': ["productId:{}".format(product_id)]}) + for provisioned_product in response.get('ProvisionedProducts', []): + launch_name = provisioned_product.get('Name') + status = provisioned_product.get('Status') + + provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact( + ProvisioningArtifactId=provisioned_product.get('ProvisioningArtifactId'), + ProductId=provisioned_product.get('ProductId'), + ).get('ProvisioningArtifactDetail') + + if deployments.get(account_id) is None: + deployments[account_id] = {'account_id': account_id, constants.LAUNCHES: {}} + + if deployments[account_id][constants.LAUNCHES].get(launch_name) is None: + deployments[account_id][constants.LAUNCHES][launch_name] = {} + + deployments[account_id][constants.LAUNCHES][launch_name][region_name] = { + 'launch_name': launch_name, + 'portfolio': portfolio.get('DisplayName'), + 'product': manifest.get(constants.LAUNCHES, {}).get(launch_name, {}).get('product'), + 'version': provisioning_artifact_response.get('Name'), + 'active': provisioning_artifact_response.get('Active'), + 'region': region_name, + 'status': status, + } + output_path = os.path.sep.join([ + constants.LAUNCHES_PATH, + account_id, + region_name, + ]) + if not os.path.exists(output_path): + os.makedirs(output_path) + + output = os.path.sep.join([output_path, "{}.json".format(provisioned_product.get('Id'))]) + with open(output, 'w') as f: + f.write(json.dumps( + provisioned_product, + indent=4, default=str + )) + + table = [ + ['account_id', 'region', 'launch', 'portfolio', 'product', 'expected_version', 'actual_version', 'active', + 'status'] + ] + for account_id, details in deployment_map.items(): + for launch_name, launch in details.get(constants.LAUNCHES, {}).items(): + if deployments.get(account_id, {}).get(constants.LAUNCHES, {}).get(launch_name) is None: + pass + else: + for region, regional_details in deployments[account_id][constants.LAUNCHES][launch_name].items(): + if regional_details.get('status') == "AVAILABLE": + status = Color("{green}" + regional_details.get('status') + "{/green}") + else: + status = Color("{red}" + regional_details.get('status') + "{/red}") + expected_version = launch.get('version') + actual_version = regional_details.get('version') + if expected_version == actual_version: + actual_version = Color("{green}" + actual_version + "{/green}") + else: + actual_version = Color("{red}" + actual_version + "{/red}") + active = regional_details.get('active') + if active: + active = Color("{green}" + str(active) + "{/green}") + else: + active = Color("{orange}" + str(active) + "{/orange}") + table.append([ + account_id, + region, + launch_name, + regional_details.get('portfolio'), + regional_details.get('product'), + expected_version, + actual_version, + active, + status, + ]) + click.echo(AsciiTable(table).table) + + +def expand(f): + click.echo('Expanding') + manifest = manifest_utils.load(f) + org_iam_role_arn = cli_command_helpers.get_org_iam_role_arn() + if org_iam_role_arn is None: + click.echo('No org role set - not expanding') + new_manifest = manifest + else: + click.echo('Expanding using role: {}'.format(org_iam_role_arn)) + with betterboto_client.CrossAccountClientContextManager( + 'organizations', org_iam_role_arn, 'org-iam-role' + ) as client: + new_manifest = manifest_utils.expand_manifest(manifest, client) + click.echo('Expanded') + new_name = f.name.replace(".yaml", '-expanded.yaml') + logger.info('Writing new manifest: {}'.format(new_name)) + with open(new_name, 'w') as output: + output.write( + yaml.safe_dump(new_manifest, default_flow_style=False) + ) + + +def validate(f): + logger.info('Validating {}'.format(f.name)) + c = Core(source_file=f.name, schema_files=[asset_helpers.resolve_from_site_packages('schema.yaml')]) + c.validate(raise_exception=True) + click.echo("Finished validating: {}".format(f.name)) + click.echo("Finished validating: OK") + + +def version(): + click.echo("cli version: {}".format(pkg_resources.require("aws-service-catalog-puppet")[0].version)) + with betterboto_client.ClientContextManager('ssm') as ssm: + response = ssm.get_parameter( + Name="service-catalog-puppet-regional-version" + ) + click.echo( + "regional stack version: {} for region: {}".format( + response.get('Parameter').get('Value'), + response.get('Parameter').get('ARN').split(':')[3] + ) + ) + response = ssm.get_parameter( + Name="service-catalog-puppet-version" + ) + click.echo( + "stack version: {}".format( + response.get('Parameter').get('Value'), + ) + ) + + +def upload_config(p): + content = open(p, 'r').read() + with betterboto_client.ClientContextManager('ssm') as ssm: + ssm.put_parameter( + Name=constants.CONFIG_PARAM_NAME, + Type='String', + Value=content, + Overwrite=True, + ) + click.echo("Uploaded config") + + +def set_org_iam_role_arn(org_iam_role_arn): + with betterboto_client.ClientContextManager('ssm') as ssm: + ssm.put_parameter( + Name=constants.CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN, + Type='String', + Value=org_iam_role_arn, + Overwrite=True, + ) + click.echo("Uploaded config") + + +def bootstrap_org_master(puppet_account_id): + with betterboto_client.ClientContextManager( + 'cloudformation', + ) as cloudformation: + org_iam_role_arn = cli_command_helpers._do_bootstrap_org_master( + puppet_account_id, cloudformation, cli_command_helpers.get_puppet_version() + ) + click.echo("Bootstrapped org master, org-iam-role-arn: {}".format(org_iam_role_arn)) + + +def quick_start(): + click.echo("Quick Start running...") + puppet_version = cli_command_helpers.get_puppet_version() + with betterboto_client.ClientContextManager('sts') as sts: + puppet_account_id = sts.get_caller_identity().get('Account') + click.echo("Going to use puppet_account_id: {}".format(puppet_account_id)) + click.echo("Bootstrapping account as a spoke") + with betterboto_client.ClientContextManager('cloudformation') as cloudformation: + cli_command_helpers._do_bootstrap_spoke(puppet_account_id, cloudformation, puppet_version) + + click.echo("Setting the config") + content = yaml.safe_dump({ + "regions": [ + 'eu-west-1', + 'eu-west-2', + 'eu-west-3' + ] + }) + with betterboto_client.ClientContextManager('ssm') as ssm: + ssm.put_parameter( + Name=constants.CONFIG_PARAM_NAME, + Type='String', + Value=content, + Overwrite=True, + ) + click.echo("Bootstrapping account as the master") + org_iam_role_arn = cli_command_helpers._do_bootstrap_org_master( + puppet_account_id, cloudformation, puppet_version + ) + ssm.put_parameter( + Name=constants.CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN, + Type='String', + Value=org_iam_role_arn, + Overwrite=True, + ) + click.echo("Bootstrapping the account now!") + cli_command_helpers._do_bootstrap(puppet_version) + + if os.path.exists('ServiceCatalogPuppet'): + click.echo("Found ServiceCatalogPuppet so not cloning or seeding") + else: + click.echo("Cloning for you") + command = "git clone " \ + "--config 'credential.helper=!aws codecommit credential-helper $@' " \ + "--config 'credential.UseHttpPath=true' " \ + "https://git-codecommit.{}.amazonaws.com/v1/repos/ServiceCatalogPuppet".format( + os.environ.get("AWS_DEFAULT_REGION") + ) + os.system(command) + click.echo("Seeding") + manifest = Template( + asset_helpers.read_from_site_packages(os.path.sep.join(["manifests", "manifest-quickstart.yaml"])) + ).render( + ACCOUNT_ID=puppet_account_id + ) + open(os.path.sep.join(["ServiceCatalogPuppet", "manifest.yaml"]), 'w').write( + manifest + ) + click.echo("Pushing manifest") + os.system("cd ServiceCatalogPuppet && git add manifest.yaml && git commit -am 'initial add' && git push") + + click.echo("All done!") + + diff --git a/servicecatalog_puppet/cli_test.py b/servicecatalog_puppet/cli_test.py deleted file mode 100644 index ecbaa6254..000000000 --- a/servicecatalog_puppet/cli_test.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -from pytest import fixture - - -@fixture -def all_tasks(): - return { - "012345678910-eu-west-1-all-enable-config": { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678910", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "depends_on": [], - "dependencies": [] - }, - "012345678910-eu-west-1-all-rules-config": { - "launch_name": "all-rules-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Rules", - "version": "v1", - "product_id": "prod-ccccccccccccc", - "version_id": "pa-ccccccccccccc", - "account_id": "012345678910", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "depends_on": [ - "all-enable-config" - ], - "dependencies": [] - }, - "012345678912-eu-west-1-all-enable-config": { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678912", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "depends_on": [], - "dependencies": [] - }, - } - - -@fixture -def sut(): - from servicecatalog_puppet import cli - return cli - - -def test_wire_dependencies(sut, all_tasks): - # setup - expected_result = [ - { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678910", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "dependencies": [] - }, - { - "launch_name": "all-rules-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Rules", - "version": "v1", - "product_id": "prod-ccccccccccccc", - "version_id": "pa-ccccccccccccc", - "account_id": "012345678910", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "dependencies": [ - { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678910", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "dependencies": [] - }, - { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678912", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "dependencies": [] - }, - ] - }, - { - "launch_name": "all-enable-config", - "portfolio": "ccoe-mandatory-product-portfolio", - "product": "Config-Enable", - "version": "v1", - "product_id": "prod-bbbbbbbbbbb", - "version_id": "pa-bbbbbbbbbbb", - "account_id": "012345678912", - "region": "eu-west-1", - "puppet_account_id": "012345678911", - "parameters": [], - "ssm_param_inputs": [], - "dependencies": [] - } - ] - - # exercise - actual_result = sut.wire_dependencies(all_tasks) - - # verify - assert expected_result[0] == actual_result[0] - e = expected_result[1] - a = actual_result[1] - assert e == a - assert expected_result[2] == actual_result[2] diff --git a/servicecatalog_puppet/commands/__init__.py b/servicecatalog_puppet/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/servicecatalog_puppet/commands/bootstrap.py b/servicecatalog_puppet/commands/bootstrap.py deleted file mode 100644 index 1b4a7a7a9..000000000 --- a/servicecatalog_puppet/commands/bootstrap.py +++ /dev/null @@ -1,78 +0,0 @@ -from threading import Thread -import os -import click -from betterboto import client as betterboto_client -from jinja2 import Template - -from servicecatalog_puppet.asset_helpers import read_from_site_packages -from servicecatalog_puppet.constants import BOOTSTRAP_STACK_NAME, SERVICE_CATALOG_PUPPET_REPO_NAME -from servicecatalog_puppet.core import get_regions, get_org_iam_role_arn - - -def do_bootstrap(puppet_version): - click.echo('Starting bootstrap') - ALL_REGIONS = get_regions(os.environ.get("AWS_DEFAULT_REGION")) - with betterboto_client.MultiRegionClientContextManager('cloudformation', ALL_REGIONS) as clients: - click.echo('Creating {}-regional'.format(BOOTSTRAP_STACK_NAME)) - threads = [] - template = read_from_site_packages('{}.template.yaml'.format('{}-regional'.format(BOOTSTRAP_STACK_NAME))) - template = Template(template).render(VERSION=puppet_version) - args = { - 'StackName': '{}-regional'.format(BOOTSTRAP_STACK_NAME), - 'TemplateBody': template, - 'Capabilities': ['CAPABILITY_IAM'], - 'Parameters': [ - { - 'ParameterKey': 'Version', - 'ParameterValue': puppet_version, - 'UsePreviousValue': False, - }, - { - 'ParameterKey': 'DefaultRegionValue', - 'ParameterValue': os.environ.get('AWS_DEFAULT_REGION'), - 'UsePreviousValue': False, - }, - ], - } - for client_region, client in clients.items(): - process = Thread(name=client_region, target=client.create_or_update, kwargs=args) - process.start() - threads.append(process) - for process in threads: - process.join() - click.echo('Finished creating {}-regional'.format(BOOTSTRAP_STACK_NAME)) - - with betterboto_client.ClientContextManager('cloudformation') as cloudformation: - click.echo('Creating {}'.format(BOOTSTRAP_STACK_NAME)) - template = read_from_site_packages('{}.template.yaml'.format(BOOTSTRAP_STACK_NAME)) - template = Template(template).render(VERSION=puppet_version, ALL_REGIONS=ALL_REGIONS) - args = { - 'StackName': BOOTSTRAP_STACK_NAME, - 'TemplateBody': template, - 'Capabilities': ['CAPABILITY_NAMED_IAM'], - 'Parameters': [ - { - 'ParameterKey': 'Version', - 'ParameterValue': puppet_version, - 'UsePreviousValue': False, - }, - { - 'ParameterKey': 'OrgIamRoleArn', - 'ParameterValue': str(get_org_iam_role_arn()), - 'UsePreviousValue': False, - }, - ], - } - cloudformation.create_or_update(**args) - - click.echo('Finished creating {}.'.format(BOOTSTRAP_STACK_NAME)) - with betterboto_client.ClientContextManager('codecommit') as codecommit: - response = codecommit.get_repository(repositoryName=SERVICE_CATALOG_PUPPET_REPO_NAME) - clone_url = response.get('repositoryMetadata').get('cloneUrlHttp') - clone_command = "git clone --config 'credential.helper=!aws codecommit credential-helper $@' " \ - "--config 'credential.UseHttpPath=true' {}".format(clone_url) - click.echo( - 'You need to clone your newly created repo now and will then need to seed it: \n{}'.format( - clone_command - ) - ) diff --git a/servicecatalog_puppet/commands/bootstrap_org_master.py b/servicecatalog_puppet/commands/bootstrap_org_master.py deleted file mode 100644 index eaf65abeb..000000000 --- a/servicecatalog_puppet/commands/bootstrap_org_master.py +++ /dev/null @@ -1,45 +0,0 @@ -from jinja2 import Template - -import logging - - -from servicecatalog_puppet.asset_helpers import read_from_site_packages -from servicecatalog_puppet.constants import BOOTSTRAP_STACK_NAME -from servicecatalog_puppet.constants import PUPPET_ORG_ROLE_FOR_EXPANDS_ARN - -logger = logging.getLogger(__file__) - - -def do_bootstrap_org_master(puppet_account_id, cloudformation, puppet_version): - logger.info('Starting bootstrap of org master') - stack_name = "{}-org-master".format(BOOTSTRAP_STACK_NAME) - template = read_from_site_packages('{}.template.yaml'.format(stack_name)) - template = Template(template).render(VERSION=puppet_version) - args = { - 'StackName': stack_name, - 'TemplateBody': template, - 'Capabilities': ['CAPABILITY_NAMED_IAM'], - 'Parameters': [ - { - 'ParameterKey': 'PuppetAccountId', - 'ParameterValue': str(puppet_account_id), - }, { - 'ParameterKey': 'Version', - 'ParameterValue': puppet_version, - 'UsePreviousValue': False, - }, - ], - } - cloudformation.create_or_update(**args) - response = cloudformation.describe_stacks(StackName=stack_name) - if len(response.get('Stacks')) != 1: - raise Exception("Expected there to be only one {} stack".format(stack_name)) - stack = response.get('Stacks')[0] - - for output in stack.get('Outputs'): - if output.get('OutputKey') == PUPPET_ORG_ROLE_FOR_EXPANDS_ARN: - logger.info('Finished bootstrap of org-master') - return output.get("OutputValue") - - raise Exception("Could not find output: {} in stack: {}".format(PUPPET_ORG_ROLE_FOR_EXPANDS_ARN, stack_name)) - diff --git a/servicecatalog_puppet/commands/bootstrap_spoke.py b/servicecatalog_puppet/commands/bootstrap_spoke.py deleted file mode 100644 index ec32199f0..000000000 --- a/servicecatalog_puppet/commands/bootstrap_spoke.py +++ /dev/null @@ -1,33 +0,0 @@ -from jinja2 import Template - -import logging - - -from servicecatalog_puppet.asset_helpers import read_from_site_packages -from servicecatalog_puppet.constants import BOOTSTRAP_STACK_NAME - - -logger = logging.getLogger(__file__) - - -def do_bootstrap_spoke(puppet_account_id, cloudformation, puppet_version): - logger.info('Starting bootstrap of spoke') - template = read_from_site_packages('{}-spoke.template.yaml'.format(BOOTSTRAP_STACK_NAME)) - template = Template(template).render(VERSION=puppet_version) - args = { - 'StackName': "{}-spoke".format(BOOTSTRAP_STACK_NAME), - 'TemplateBody': template, - 'Capabilities': ['CAPABILITY_NAMED_IAM'], - 'Parameters': [ - { - 'ParameterKey': 'PuppetAccountId', - 'ParameterValue': str(puppet_account_id), - }, { - 'ParameterKey': 'Version', - 'ParameterValue': puppet_version, - 'UsePreviousValue': False, - }, - ], - } - cloudformation.create_or_update(**args) - logger.info('Finished bootstrap of spoke') diff --git a/servicecatalog_puppet/commands/list_launches.py b/servicecatalog_puppet/commands/list_launches.py deleted file mode 100644 index 9ccc44328..000000000 --- a/servicecatalog_puppet/commands/list_launches.py +++ /dev/null @@ -1,121 +0,0 @@ -import json -import os - -import click -from betterboto import client as betterboto_client -from colorclass import Color -from terminaltables import AsciiTable - -from servicecatalog_puppet.constants import LAUNCHES -from servicecatalog_puppet.core import get_regions -from servicecatalog_puppet.utils.manifest import build_deployment_map - -import logging - -logger = logging.getLogger(__file__) - - -def do_list_launches(manifest): - click.echo("Getting details from your account...") - ALL_REGIONS = get_regions(os.environ.get("AWS_DEFAULT_REGION")) - deployment_map = build_deployment_map(manifest) - account_ids = [a.get('account_id') for a in manifest.get('accounts')] - deployments = {} - for account_id in account_ids: - for region_name in ALL_REGIONS: - role = "arn:aws:iam::{}:role/{}".format(account_id, 'servicecatalog-puppet/PuppetRole') - logger.info("Looking at region: {} in account: {}".format(region_name, account_id)) - with betterboto_client.CrossAccountClientContextManager( - 'servicecatalog', role, 'sc-{}-{}'.format(account_id, region_name), region_name=region_name - ) as spoke_service_catalog: - - response = spoke_service_catalog.list_accepted_portfolio_shares() - portfolios = response.get('PortfolioDetails', []) - - response = spoke_service_catalog.list_portfolios() - portfolios += response.get('PortfolioDetails', []) - - for portfolio in portfolios: - portfolio_id = portfolio.get('Id') - response = spoke_service_catalog.search_products_as_admin(PortfolioId=portfolio_id) - for product_view_detail in response.get('ProductViewDetails', []): - product_view_summary = product_view_detail.get('ProductViewSummary') - product_id = product_view_summary.get('ProductId') - response = spoke_service_catalog.search_provisioned_products( - Filters={'SearchQuery': ["productId:{}".format(product_id)]}) - for provisioned_product in response.get('ProvisionedProducts', []): - launch_name = provisioned_product.get('Name') - status = provisioned_product.get('Status') - - provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact( - ProvisioningArtifactId=provisioned_product.get('ProvisioningArtifactId'), - ProductId=provisioned_product.get('ProductId'), - ).get('ProvisioningArtifactDetail') - - if deployments.get(account_id) is None: - deployments[account_id] = {'account_id': account_id, 'launches': {}} - - if deployments[account_id]['launches'].get(launch_name) is None: - deployments[account_id]['launches'][launch_name] = {} - - deployments[account_id]['launches'][launch_name][region_name] = { - 'launch_name': launch_name, - 'portfolio': portfolio.get('DisplayName'), - 'product': manifest.get('launches', {}).get(launch_name, {}).get('product'), - 'version': provisioning_artifact_response.get('Name'), - 'active': provisioning_artifact_response.get('Active'), - 'region': region_name, - 'status': status, - } - output_path = os.path.sep.join([ - LAUNCHES, - account_id, - region_name, - ]) - if not os.path.exists(output_path): - os.makedirs(output_path) - - output = os.path.sep.join([output_path, "{}.json".format(provisioned_product.get('Id'))]) - with open(output, 'w') as f: - f.write(json.dumps( - provisioned_product, - indent=4, default=str - )) - - table = [ - ['account_id', 'region', 'launch', 'portfolio', 'product', 'expected_version', 'actual_version', 'active', - 'status'] - ] - for account_id, details in deployment_map.items(): - for launch_name, launch in details.get('launches', {}).items(): - if deployments.get(account_id, {}).get('launches', {}).get(launch_name) is None: - pass - else: - for region, regional_details in deployments[account_id]['launches'][launch_name].items(): - if regional_details.get('status') == "AVAILABLE": - status = Color("{green}" + regional_details.get('status') + "{/green}") - else: - status = Color("{red}" + regional_details.get('status') + "{/red}") - expected_version = launch.get('version') - actual_version = regional_details.get('version') - if expected_version == actual_version: - actual_version = Color("{green}" + actual_version + "{/green}") - else: - actual_version = Color("{red}" + actual_version + "{/red}") - active = regional_details.get('active') - if active: - active = Color("{green}" + str(active) + "{/green}") - else: - active = Color("{orange}" + str(active) + "{/orange}") - table.append([ - account_id, - region, - launch_name, - regional_details.get('portfolio'), - regional_details.get('product'), - expected_version, - actual_version, - active, - status, - ]) - click.echo(AsciiTable(table).table) \ No newline at end of file diff --git a/servicecatalog_puppet/constants.py b/servicecatalog_puppet/constants.py index 84c754b4c..806547af9 100644 --- a/servicecatalog_puppet/constants.py +++ b/servicecatalog_puppet/constants.py @@ -5,8 +5,21 @@ SERVICE_CATALOG_PUPPET_REPO_NAME = "ServiceCatalogPuppet" OUTPUT = "output" TEMPLATES = os.path.sep.join([OUTPUT, "templates"]) -LAUNCHES = os.path.sep.join([OUTPUT, "launches"]) +LAUNCHES_PATH = os.path.sep.join([OUTPUT, "launches"]) CONFIG_PARAM_NAME = "/servicecatalog-puppet/config" CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN = "/servicecatalog-puppet/org-iam-role-arn" PUPPET_ORG_ROLE_FOR_EXPANDS_ARN = "PuppetOrgRoleForExpandsArn" HOME_REGION_PARAM_NAME = "/servicecatalog-puppet/home-region" + +PROVISIONED = 'provisioned' +TERMINATED = 'terminated' + +DEFAULT_TIMEOUT = 0 +LAUNCHES = 'launches' +SPOKE_LOCAL_PORTFOLIOS = 'spoke-local-portfolios' + +DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES = [ + 'depends_on', + 'outputs', + 'parameters', +] \ No newline at end of file diff --git a/servicecatalog_puppet/core.py b/servicecatalog_puppet/core.py deleted file mode 100644 index e1c3ebaf8..000000000 --- a/servicecatalog_puppet/core.py +++ /dev/null @@ -1,175 +0,0 @@ -import logging -import os -import time -from threading import Thread -import traceback - -import yaml -from betterboto import client as betterboto_client -from jinja2 import Environment, FileSystemLoader - -from servicecatalog_puppet.asset_helpers import resolve_from_site_packages -from servicecatalog_puppet.constants import HOME_REGION_PARAM_NAME, CONFIG_PARAM_NAME, CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN, TEMPLATES, PREFIX - -logger = logging.getLogger() - - -def get_regions(default_region=None): - logger.info("getting regions, default_region: {}".format(default_region)) - with betterboto_client.ClientContextManager( - 'ssm', - region_name=default_region if default_region else get_home_region() - ) as ssm: - response = ssm.get_parameter(Name=CONFIG_PARAM_NAME) - config = yaml.safe_load(response.get('Parameter').get('Value')) - return config.get('regions') - - -def get_home_region(): - with betterboto_client.ClientContextManager('ssm') as ssm: - response = ssm.get_parameter(Name=HOME_REGION_PARAM_NAME) - return response.get('Parameter').get('Value') - - -def get_org_iam_role_arn(): - with betterboto_client.ClientContextManager('ssm', region_name=get_home_region()) as ssm: - try: - response = ssm.get_parameter(Name=CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN) - return yaml.safe_load(response.get('Parameter').get('Value')) - except ssm.exceptions.ParameterNotFound as e: - logger.info("No parameter set for: {}".format(CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN)) - return None - - -def get_provisioning_artifact_id_for(portfolio_name, product_name, version_name, account_id, region): - logger.info("Getting provisioning artifact id for: {} {} {} in the region: {} of account: {}".format( - portfolio_name, product_name, version_name, region, account_id - )) - role = "arn:aws:iam::{}:role/{}".format(account_id, 'servicecatalog-puppet/PuppetRole') - with betterboto_client.CrossAccountClientContextManager( - 'servicecatalog', role, "-".join([account_id, region]), region_name=region - ) as cross_account_servicecatalog: - product_id = None - version_id = None - portfolio_id = None - args = {} - while True: - response = cross_account_servicecatalog.list_accepted_portfolio_shares() - assert response.get('NextPageToken') is None, "Pagination not supported" - for portfolio_detail in response.get('PortfolioDetails'): - if portfolio_detail.get('DisplayName') == portfolio_name: - portfolio_id = portfolio_detail.get('Id') - break - - if portfolio_id is None: - response = cross_account_servicecatalog.list_portfolios() - for portfolio_detail in response.get('PortfolioDetails', []): - if portfolio_detail.get('DisplayName') == portfolio_name: - portfolio_id = portfolio_detail.get('Id') - break - - assert portfolio_id is not None, "Could not find portfolio" - logger.info("Found portfolio: {}".format(portfolio_id)) - - args['PortfolioId'] = portfolio_id - response = cross_account_servicecatalog.search_products_as_admin( - **args - ) - for product_view_details in response.get('ProductViewDetails'): - product_view = product_view_details.get('ProductViewSummary') - if product_view.get('Name') == product_name: - logger.info('Found product: {}'.format(product_view)) - product_id = product_view.get('ProductId') - if response.get('NextPageToken', None) is not None: - args['PageToken'] = response.get('NextPageToken') - else: - break - assert product_id is not None, "Did not find product looking for" - - response = cross_account_servicecatalog.list_provisioning_artifacts( - ProductId=product_id - ) - assert response.get('NextPageToken') is None, "Pagination not support" - for provisioning_artifact_detail in response.get('ProvisioningArtifactDetails'): - if provisioning_artifact_detail.get('Name') == version_name: - version_id = provisioning_artifact_detail.get('Id') - assert version_id is not None, "Did not find version looking for" - return product_id, version_id - - -def generate_bucket_policies_for_shares(deployment_map, puppet_account_id): - shares = { - 'accounts': [], - 'organizations': [], - } - for account_id, deployment in deployment_map.items(): - if account_id == puppet_account_id: - continue - if deployment.get('expanded_from') is None: - if account_id not in shares['accounts']: - shares['accounts'].append(account_id) - else: - if deployment.get('organization') not in shares['organizations']: - shares['organizations'].append(deployment.get('organization')) - return shares - - -def write_share_template(portfolio_use_by_account, region, host_account_id, sharing_policies): - output = os.path.sep.join([TEMPLATES, 'shares', region]) - if not os.path.exists(output): - os.makedirs(output) - - with open(os.sep.join([output, "shares.template.yaml"]), 'w') as f: - f.write( - env.get_template('shares.template.yaml.j2').render( - portfolio_use_by_account=portfolio_use_by_account, - host_account_id=host_account_id, - HOME_REGION=get_home_region(), - sharing_policies=sharing_policies, - ) - ) - - -def create_share_template(deployment_map, puppet_account_id): - logger.info("deployment_map: {}".format(deployment_map)) - ALL_REGIONS = get_regions() - for region in ALL_REGIONS: - logger.info("starting to build shares for region: {}".format(region)) - with betterboto_client.ClientContextManager('servicecatalog', region_name=region) as servicecatalog: - portfolio_ids = {} - args = {} - while True: - - response = servicecatalog.list_portfolios( - **args - ) - - for portfolio_detail in response.get('PortfolioDetails'): - portfolio_ids[portfolio_detail.get('DisplayName')] = portfolio_detail.get('Id') - - if response.get('PageToken') is not None: - args['PageToken'] = response.get('PageToken') - else: - break - - logger.info("Portfolios in use in region: {}".format(portfolio_ids)) - - portfolio_use_by_account = {} - for account_id, launch_details in deployment_map.items(): - if portfolio_use_by_account.get(account_id) is None: - portfolio_use_by_account[account_id] = [] - for launch_id, launch in launch_details.get('launches').items(): - logger.info("portfolio ids: {}".format(portfolio_ids)) - p = portfolio_ids[launch.get('portfolio')] - if p not in portfolio_use_by_account[account_id]: - portfolio_use_by_account[account_id].append(p) - host_account_id = response.get('PortfolioDetails')[0].get('ARN').split(":")[4] - sharing_policies = generate_bucket_policies_for_shares(deployment_map, puppet_account_id) - write_share_template(portfolio_use_by_account, region, host_account_id, sharing_policies) - - -template_dir = resolve_from_site_packages('templates') -env = Environment( - loader=FileSystemLoader(template_dir), - extensions=['jinja2.ext.do'], -) \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/all-tasks-for-launch-b.json b/servicecatalog_puppet/data/account-vending/all-tasks-for-launch-b.json new file mode 100644 index 000000000..1bd5e12e6 --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/all-tasks-for-launch-b.json @@ -0,0 +1,25 @@ +{ + "0123456789010-eu-west-1-account-vending-account-creation-shared": { + "launch_name": "account-vending-account-creation-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4", + "account_id": "0123456789010", + "region": "eu-west-1", + "puppet_account_id": "098765432101", + "parameters": [], + "ssm_param_inputs": [], + "depends_on": [], + "status": "provisioned", + "worker_timeout": 0, + "ssm_param_outputs": [ + { + "param_name": "/account-vending/account-custom-resource-arn", + "stack_output": "AccountCustomResourceArn" + } + ], + "dependencies": [] + } +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/all-tasks.json b/servicecatalog_puppet/data/account-vending/all-tasks.json new file mode 100644 index 000000000..e1b8bb498 --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/all-tasks.json @@ -0,0 +1,96 @@ +{ + "923822062182-eu-west-1-account-vending-account-creation-shared": { + "launch_name": "account-vending-account-creation-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "depends_on": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] + }, + "923822062182-eu-west-1-account-vending-account-bootstrap-shared": { + "launch_name": "account-vending-account-bootstrap-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "depends_on": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] + }, + "923822062182-eu-west-1-account-vending-account-002": { + "launch_name": "account-vending-account-002", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation", + "version": "v1", + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "IamUserAccessToBilling", + "value": "ALLOW" + }, + { + "name": "Email", + "value": "eamonnf+account-002@amazon.com" + }, + { + "name": "TargetOU", + "value": "/" + }, + { + "name": "OrganizationAccountAccessRole", + "value": "OrganizationAccountAccessRole" + }, + { + "name": "AccountName", + "value": "account-002" + } + ], + "ssm_param_inputs": [ + { + "name": "/account-vending/bootstrapper-project-custom-resource-arn", + "parameter_name": "AccountVendingBootstrapperLambdaArn" + }, + { + "name": "/account-vending/account-custom-resource-arn", + "parameter_name": "AccountVendingCreationLambdaArn" + } + ], + "depends_on": [ + "account-vending-account-creation-shared", + "account-vending-account-bootstrap-shared" + ], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] + } +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/deployment-map.json b/servicecatalog_puppet/data/account-vending/deployment-map.json new file mode 100644 index 000000000..edac3b37b --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/deployment-map.json @@ -0,0 +1,150 @@ +{ + "923822062182": { + "account_id": "923822062182", + "name": "923822062182", + "default_region": "eu-west-1", + "regions_enabled": [ + "eu-west-1", + "eu-west-2" + ], + "tags": [ + "type:prod", + "partition:eu", + "scope:puppet-hub" + ], + "launches": { + "account-vending-account-creation-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/account-custom-resource-arn", + "stack_output": "AccountCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-creation-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2" + } + } + }, + "account-vending-account-bootstrap-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/bootstrapper-project-custom-resource-arn", + "stack_output": "BootstrapperProjectCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-bootstrap-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o" + } + } + }, + "account-vending-account-002": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation", + "version": "v1", + "depends_on": [ + "account-vending-account-creation-shared", + "account-vending-account-bootstrap-shared" + ], + "parameters": { + "Email": { + "default": "eamonnf+account-002@amazon.com" + }, + "AccountName": { + "default": "account-002" + }, + "OrganizationAccountAccessRole": { + "default": "OrganizationAccountAccessRole" + }, + "IamUserAccessToBilling": { + "default": "ALLOW" + }, + "TargetOU": { + "default": "/" + }, + "AccountVendingCreationLambdaArn": { + "ssm": { + "name": "/account-vending/account-custom-resource-arn" + } + }, + "AccountVendingBootstrapperLambdaArn": { + "ssm": { + "name": "/account-vending/bootstrapper-project-custom-resource-arn" + } + } + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-002", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4" + } + } + } + } + } +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/launch-a.json b/servicecatalog_puppet/data/account-vending/launch-a.json new file mode 100644 index 000000000..e81f091de --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/launch-a.json @@ -0,0 +1,21 @@ +{ + "launch_name": "account-vending-account-creation-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/launch-b.json b/servicecatalog_puppet/data/account-vending/launch-b.json new file mode 100644 index 000000000..c87f37123 --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/launch-b.json @@ -0,0 +1,21 @@ +{ + "launch_name": "account-vending-account-bootstrap-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/launch-c.json b/servicecatalog_puppet/data/account-vending/launch-c.json new file mode 100644 index 000000000..6b7253842 --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/launch-c.json @@ -0,0 +1,89 @@ +{ + "launch_name": "account-vending-account-002", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation", + "version": "v1", + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "IamUserAccessToBilling", + "value": "ALLOW" + }, + { + "name": "Email", + "value": "eamonnf+account-002@amazon.com" + }, + { + "name": "TargetOU", + "value": "/" + }, + { + "name": "OrganizationAccountAccessRole", + "value": "OrganizationAccountAccessRole" + }, + { + "name": "AccountName", + "value": "account-002" + } + ], + "ssm_param_inputs": [ + { + "name": "/account-vending/bootstrapper-project-custom-resource-arn", + "parameter_name": "AccountVendingBootstrapperLambdaArn" + }, + { + "name": "/account-vending/account-custom-resource-arn", + "parameter_name": "AccountVendingCreationLambdaArn" + } + ], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [ + { + "launch_name": "account-vending-account-creation-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] + }, + { + "launch_name": "account-vending-account-bootstrap-shared", + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o", + "account_id": "923822062182", + "region": "eu-west-1", + "puppet_account_id": "923822062182", + "parameters": [ + { + "name": "AssumableRoleInRootAccountArn", + "value": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + ], + "ssm_param_inputs": [], + "status": "provisioned", + "worker_timeout": 0, + "dependencies": [] + } + ] +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/launch-details-for-launch-b.json b/servicecatalog_puppet/data/account-vending/launch-details-for-launch-b.json new file mode 100644 index 000000000..82244f9bc --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/launch-details-for-launch-b.json @@ -0,0 +1,38 @@ +{ + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/account-custom-resource-arn", + "stack_output": "AccountCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-creation-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2" + } + } +} \ No newline at end of file diff --git a/servicecatalog_puppet/data/account-vending/manifest.json b/servicecatalog_puppet/data/account-vending/manifest.json new file mode 100644 index 000000000..a76a4c60a --- /dev/null +++ b/servicecatalog_puppet/data/account-vending/manifest.json @@ -0,0 +1,287 @@ +{ + "schema": "puppet-2019-04-01", + "accounts": [ + { + "account_id": "923822062182", + "name": "923822062182", + "default_region": "eu-west-1", + "regions_enabled": [ + "eu-west-1", + "eu-west-2" + ], + "tags": [ + "type:prod", + "partition:eu", + "scope:puppet-hub" + ], + "launches": { + "account-vending-account-creation-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/account-custom-resource-arn", + "stack_output": "AccountCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-creation-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2" + } + } + }, + "account-vending-account-bootstrap-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/bootstrapper-project-custom-resource-arn", + "stack_output": "BootstrapperProjectCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-bootstrap-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o" + } + } + }, + "account-vending-account-002": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation", + "version": "v1", + "depends_on": [ + "account-vending-account-creation-shared", + "account-vending-account-bootstrap-shared" + ], + "parameters": { + "Email": { + "default": "eamonnf+account-002@amazon.com" + }, + "AccountName": { + "default": "account-002" + }, + "OrganizationAccountAccessRole": { + "default": "OrganizationAccountAccessRole" + }, + "IamUserAccessToBilling": { + "default": "ALLOW" + }, + "TargetOU": { + "default": "/" + }, + "AccountVendingCreationLambdaArn": { + "ssm": { + "name": "/account-vending/account-custom-resource-arn" + } + }, + "AccountVendingBootstrapperLambdaArn": { + "ssm": { + "name": "/account-vending/bootstrapper-project-custom-resource-arn" + } + } + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-002", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4" + } + } + } + } + } + ], + "launches": { + "account-vending-account-creation-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/account-custom-resource-arn", + "stack_output": "AccountCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-creation-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-ahhdj3q5puvhw", + "version_id": "pa-plsqbdtrqt4h2" + } + } + }, + "account-vending-account-bootstrap-shared": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-bootstrap-shared", + "version": "v1", + "parameters": { + "AssumableRoleInRootAccountArn": { + "default": "arn:aws:iam::923822062182:role/servicecatalog-puppet/AssumableRoleInRootAccount" + } + }, + "outputs": { + "ssm": [ + { + "param_name": "/account-vending/bootstrapper-project-custom-resource-arn", + "stack_output": "BootstrapperProjectCustomResourceArn" + } + ] + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-bootstrap-shared", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-itqvegxgl4wvw", + "version_id": "pa-th64llsxcn46o" + } + } + }, + "account-vending-account-002": { + "portfolio": "demo-central-it-team-portfolio", + "product": "account-vending-account-creation", + "version": "v1", + "depends_on": [ + "account-vending-account-creation-shared", + "account-vending-account-bootstrap-shared" + ], + "parameters": { + "Email": { + "default": "eamonnf+account-002@amazon.com" + }, + "AccountName": { + "default": "account-002" + }, + "OrganizationAccountAccessRole": { + "default": "OrganizationAccountAccessRole" + }, + "IamUserAccessToBilling": { + "default": "ALLOW" + }, + "TargetOU": { + "default": "/" + }, + "AccountVendingCreationLambdaArn": { + "ssm": { + "name": "/account-vending/account-custom-resource-arn" + } + }, + "AccountVendingBootstrapperLambdaArn": { + "ssm": { + "name": "/account-vending/bootstrapper-project-custom-resource-arn" + } + } + }, + "deploy_to": { + "tags": [ + { + "tag": "scope:puppet-hub", + "regions": "default_region" + } + ] + }, + "launch_name": "account-vending-account-002", + "match": "tag_match", + "matching_tag": "scope:puppet-hub", + "regions": [ + "eu-west-1" + ], + "regional_details": { + "eu-west-1": { + "product_id": "prod-lv3isrxiingdo", + "version_id": "pa-yprmofsvvyih4" + } + } + } + } +} \ No newline at end of file diff --git a/servicecatalog_puppet/luigi_tasks_and_targets.py b/servicecatalog_puppet/luigi_tasks_and_targets.py index 8b9a293af..bdddfefcd 100644 --- a/servicecatalog_puppet/luigi_tasks_and_targets.py +++ b/servicecatalog_puppet/luigi_tasks_and_targets.py @@ -1,6 +1,8 @@ +import time + from betterboto import client as betterboto_client -from servicecatalog_puppet import aws +from servicecatalog_puppet import aws, cli_command_helpers import luigi import json @@ -47,36 +49,6 @@ def run(self): pass -class SetSSMParamFromProvisionProductTask(luigi.Task): - param_name = luigi.Parameter() - param_type = luigi.Parameter(default='String') - stack_output = luigi.Parameter() - - dependency = luigi.DictParameter() - - def requires(self): - return ProvisionProductTask(**self.dependency) - - def output(self): - return SSMParamTarget(self.param_name, False) - - def run(self): - with betterboto_client.ClientContextManager('ssm') as ssm: - outputs = json.loads(self.input().open('r').read()).get('Outputs') - written = False - for output in outputs: - if output.get('OutputKey') == self.stack_output: - written = True - ssm.put_parameter( - Name=self.param_name, - Value=output.get('OutputValue'), - Type=self.param_type, - Overwrite=True, - ) - if not written: - raise Exception("Could not write SSM Param from Provisioned Product. Wrong stack_output?") - - class ProvisionProductTask(luigi.Task): launch_name = luigi.Parameter() portfolio = luigi.Parameter() @@ -98,6 +70,10 @@ class ProvisionProductTask(luigi.Task): status = luigi.Parameter(default='', significant=False) + worker_timeout = luigi.IntParameter(default=0, significant=False) + + ssm_param_outputs = luigi.ListParameter(default=[]) + try_count = 1 def add_requires(self, task): @@ -152,7 +128,9 @@ def run(self): ) as cloudformation: need_to_provision = True if provisioned_product_id: - default_cfn_params = aws.get_default_parameters_for_stack(cloudformation, f"SC-{self.account_id}-{provisioned_product_id}") + default_cfn_params = aws.get_default_parameters_for_stack( + cloudformation, f"SC-{self.account_id}-{provisioned_product_id}" + ) else: default_cfn_params = {} @@ -160,9 +138,11 @@ def run(self): if all_params.get(default_cfn_param_name) is None: all_params[default_cfn_param_name] = default_cfn_params[default_cfn_param_name] if provisioning_artifact_id == self.version_id: - logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: found previous good provision") + logger.info( + f"[{self.launch_name}] {self.account_id}:{self.region} :: found previous good provision") if provisioned_product_id: - logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: checking params for diffs") + logger.info( + f"[{self.launch_name}] {self.account_id}:{self.region} :: checking params for diffs") provisioned_parameters = aws.get_parameters_for_stack( cloudformation, f"SC-{self.account_id}-{provisioned_product_id}" @@ -214,17 +194,40 @@ def run(self): self.version, ) - f = self.output().open('w') with betterboto_client.CrossAccountClientContextManager( 'cloudformation', role, f'cfn-{self.region}-{self.account_id}', region_name=self.region - ) as cloudformation: - f.write( - json.dumps( - aws.get_stack_output_for(cloudformation, f"SC-{self.account_id}-{provisioned_product_id}"), - indent=4, - default=str, - ) + ) as spoke_cloudformation: + stack_details = aws.get_stack_output_for( + spoke_cloudformation, f"SC-{self.account_id}-{provisioned_product_id}" ) + + for ssm_param_output in self.ssm_param_outputs: + logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: " + f"writing SSM Param: {ssm_param_output.get('stack_output')}") + with betterboto_client.ClientContextManager('ssm') as ssm: + found_match = False + for output in stack_details.get('Outputs', []): + if output.get('OutputKey') == ssm_param_output.get('stack_output'): + found_match = True + logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: found value") + ssm.put_parameter( + Name=ssm_param_output.get('param_name'), + Value=output.get('OutputValue'), + Type=ssm_param_output.get('param_type', 'String'), + Overwrite=True, + ) + if not found_match: + raise Exception(f"[{self.launch_name}] {self.account_id}:{self.region} :: Could not find " + f"match for {ssm_param_output.get('stack_output')}") + + f = self.output().open('w') + f.write( + json.dumps( + stack_details, + indent=4, + default=str, + ) + ) f.close() logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: finished provisioning") @@ -278,3 +281,334 @@ def run(self): ) f.close() logger.info(f"[{self.launch_name}] {self.account_id}:{self.region} :: finished terminating") + + +class CreateSpokeLocalPortfolioTask(luigi.Task): + account_id = luigi.Parameter() + region = luigi.Parameter() + portfolio = luigi.Parameter() + + provider_name = luigi.Parameter(significant=False, default='not set') + description = luigi.Parameter(significant=False, default='not set') + + def output(self): + return luigi.LocalTarget( + f"output/CreateSpokeLocalPortfolioTask/" + f"{self.account_id}-{self.region}-{self.portfolio}.json" + ) + + def run(self): + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: starting creating portfolio") + role = f"arn:aws:iam::{self.account_id}:role/servicecatalog-puppet/PuppetRole" + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, f'sc-{self.account_id}-{self.region}', region_name=self.region + ) as spoke_service_catalog: + spoke_portfolio = aws.ensure_portfolio( + spoke_service_catalog, + self.portfolio, + self.provider_name, + self.description, + ) + f = self.output().open('w') + f.write( + json.dumps( + spoke_portfolio, + indent=4, + default=str, + ) + ) + f.close() + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: finished creating portfolio") + + +class CreateAssociationsForPortfolioTask(luigi.Task): + account_id = luigi.Parameter() + region = luigi.Parameter() + portfolio = luigi.Parameter() + puppet_account_id = luigi.Parameter() + + associations = luigi.ListParameter(default=[]) + dependencies = luigi.ListParameter(default=[]) + + def requires(self): + return { + 'create_spoke_local_portfolio_task': CreateSpokeLocalPortfolioTask( + account_id=self.account_id, + region=self.region, + portfolio=self.portfolio, + ), + 'deps': [ProvisionProductTask(**dependency) for dependency in self.dependencies] + } + + def output(self): + return luigi.LocalTarget( + f"output/CreateAssociationsForPortfolioTask/" + f"{self.account_id}-{self.region}-{self.portfolio}.json" + ) + + def run(self): + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: starting creating associations") + role = f"arn:aws:iam::{self.account_id}:role/servicecatalog-puppet/PuppetRole" + + portfolio_id = json.loads(self.input().get('create_spoke_local_portfolio_task').open('r').read()).get('Id') + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: using portfolio_id: {portfolio_id}") + + with betterboto_client.CrossAccountClientContextManager( + 'cloudformation', role, f'cfn-{self.account_id}-{self.region}', region_name=self.region + ) as cloudformation: + template = cli_command_helpers.env.get_template('associations.template.yaml.j2').render( + portfolio={ + 'DisplayName': self.portfolio, + 'Associations': self.associations + }, + portfolio_id=portfolio_id, + ) + stack_name = f"associations-for-portfolio-{portfolio_id}" + cloudformation.create_or_update( + StackName=stack_name, + TemplateBody=template, + NotificationARNs=[ + f"arn:aws:sns:{self.region}:{self.puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events" + ], + ) + result = cloudformation.describe_stacks( + StackName=stack_name, + ).get('Stacks')[0] + f = self.output().open('w') + f.write( + json.dumps( + result, + indent=4, + default=str, + ) + ) + f.close() + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: Finished importing") + + +class ImportIntoSpokeLocalPortfolioTask(luigi.Task): + account_id = luigi.Parameter() + region = luigi.Parameter() + portfolio = luigi.Parameter() + + hub_portfolio_id = luigi.Parameter() + + def requires(self): + return CreateSpokeLocalPortfolioTask( + account_id=self.account_id, + region=self.region, + portfolio=self.portfolio, + ) + + def output(self): + return luigi.LocalTarget( + f"output/ImportIntoSpokeLocalPortfolioTask/" + f"{self.account_id}-{self.region}-{self.portfolio}-{self.hub_portfolio_id}.json" + ) + + def run(self): + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: starting to import into spoke") + + product_name_to_id_dict = {} + + with betterboto_client.ClientContextManager( + 'servicecatalog', region_name=self.region + ) as service_catalog: + response = service_catalog.search_products_as_admin_single_page(PortfolioId=self.hub_portfolio_id) + for product_view_detail in response.get('ProductViewDetails', []): + product_view_summary = product_view_detail.get('ProductViewSummary') + hub_product_name = product_view_summary.get('Name') + hub_product_id = product_view_summary.get('ProductId') + + product_versions_that_should_be_copied = {} + hub_provisioning_artifact_details = service_catalog.list_provisioning_artifacts( + ProductId=hub_product_id + ).get('ProvisioningArtifactDetails', []) + for hub_provisioning_artifact_detail in hub_provisioning_artifact_details: + if hub_provisioning_artifact_detail.get('Active') and hub_provisioning_artifact_detail.get( + 'Type') == 'CLOUD_FORMATION_TEMPLATE': + product_versions_that_should_be_copied[ + f"{hub_provisioning_artifact_detail.get('Name')}"] = hub_provisioning_artifact_detail + + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: Copying {hub_product_name}") + hub_product_arn = product_view_detail.get('ProductARN') + copy_args = {'SourceProductArn': hub_product_arn} + spoke_portfolio = json.loads(self.input().open('r').read()) + portfolio_id = spoke_portfolio.get("Id") + + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} {hub_product_name} :: searching in " + f"spoke for product") + role = f"arn:aws:iam::{self.account_id}:role/servicecatalog-puppet/PuppetRole" + with betterboto_client.CrossAccountClientContextManager( + 'servicecatalog', role, f"sc-{self.account_id}-{self.region}", region_name=self.region + ) as spoke_service_catalog: + p = None + try: + p = spoke_service_catalog.search_products_as_admin_single_page( + PortfolioId=portfolio_id, + Filters={'FullTextSearch': [hub_product_name]} + ) + except spoke_service_catalog.exceptions.ResourceNotFoundException as e: + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} {hub_product_name} :: " + f"swallowing exception: {str(e)}") + + if p is not None: + for spoke_product_view_details in p.get('ProductViewDetails'): + spoke_product_view = spoke_product_view_details.get('ProductViewSummary') + if spoke_product_view.get('Name') == hub_product_name: + copy_args['TargetProductId'] = spoke_product_view.get('ProductId') + spoke_provisioning_artifact_details = spoke_service_catalog.list_provisioning_artifacts( + ProductId=spoke_product_view.get('ProductId') + ).get('ProvisioningArtifactDetails') + for provisioning_artifact_detail in spoke_provisioning_artifact_details: + id_to_delete = f"{provisioning_artifact_detail.get('Name')}" + if product_versions_that_should_be_copied.get(id_to_delete, None) is not None: + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} " + f"{hub_product_name} :: Going to skip " + f"{spoke_product_view.get('ProductId')} " + f"{provisioning_artifact_detail.get('Name')}" + ) + del product_versions_that_should_be_copied[id_to_delete] + + if len(product_versions_that_should_be_copied.keys()) == 0: + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} {hub_product_name} :: " + f"no versions to copy") + else: + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} {hub_product_name} :: " + f"about to copy product") + + copy_args['SourceProvisioningArtifactIdentifiers'] = [ + {'Id': a.get('Id')} for a in product_versions_that_should_be_copied.values() + ] + + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: about to copy product with" + f"args: {copy_args}") + copy_product_token = spoke_service_catalog.copy_product( + **copy_args + ).get('CopyProductToken') + target_product_id = None + while True: + time.sleep(5) + r = spoke_service_catalog.describe_copy_product_status( + CopyProductToken=copy_product_token + ) + target_product_id = r.get('TargetProductId') + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: " + f"{hub_product_name} status: {r.get('CopyProductStatus')}") + if r.get('CopyProductStatus') == 'FAILED': + raise Exception(f"[{self.portfolio}] {self.account_id}:{self.region} :: Copying " + f"{hub_product_name} failed: {r.get('StatusDetail')}") + elif r.get('CopyProductStatus') == 'SUCCEEDED': + break + + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: adding {target_product_id} " + f"to portfolio {portfolio_id}") + spoke_service_catalog.associate_product_with_portfolio( + ProductId=target_product_id, + PortfolioId=portfolio_id, + ) + + # associate_product_with_portfolio is not a synchronous request + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: waiting for adding of " + f"{target_product_id} to portfolio {portfolio_id}") + while True: + time.sleep(2) + response = spoke_service_catalog.search_products_as_admin_single_page( + PortfolioId=portfolio_id, + ) + products_ids = [ + product_view_detail.get('ProductViewSummary').get('ProductId') for product_view_detail + in response.get('ProductViewDetails') + ] + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: Looking for " + f"{target_product_id} in {products_ids}") + + if target_product_id in products_ids: + break + + product_name_to_id_dict[hub_product_name] = target_product_id + + f = self.output().open('w') + f.write( + json.dumps( + { + 'portfolio': spoke_portfolio, + 'product_versions_that_should_be_copied': product_versions_that_should_be_copied, + 'products': product_name_to_id_dict, + }, + indent=4, + default=str, + ) + ) + f.close() + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: Finished importing") + + +class CreateLaunchRoleConstraintsForPortfolio(luigi.Task): + account_id = luigi.Parameter() + region = luigi.Parameter() + portfolio = luigi.Parameter() + hub_portfolio_id = luigi.Parameter() + puppet_account_id = luigi.Parameter() + + launch_constraints = luigi.DictParameter() + + dependencies = luigi.ListParameter(default=[]) + + def requires(self): + return { + 'create_spoke_local_portfolio_task': ImportIntoSpokeLocalPortfolioTask( + account_id=self.account_id, + region=self.region, + portfolio=self.portfolio, + hub_portfolio_id=self.hub_portfolio_id, + ), + 'deps': [ProvisionProductTask(**dependency) for dependency in self.dependencies] + } + + def run(self): + logger.info(f"[{self.portfolio}] {self.account_id}:{self.region} :: Creating launch role constraints for " + f"{self.hub_portfolio_id}") + role = f"arn:aws:iam::{self.account_id}:role/servicecatalog-puppet/PuppetRole" + dependency_output = json.loads(self.input().get('create_spoke_local_portfolio_task').open('r').read()) + spoke_portfolio = dependency_output.get('portfolio') + portfolio_id = spoke_portfolio.get('Id') + product_name_to_id_dict = dependency_output.get('products') + with betterboto_client.CrossAccountClientContextManager( + 'cloudformation', role, f'cfn-{self.account_id}-{self.region}', region_name=self.region + ) as cloudformation: + template = cli_command_helpers.env.get_template('launch_role_constraints.template.yaml.j2').render( + portfolio={ + 'DisplayName': self.portfolio, + }, + portfolio_id=portfolio_id, + launch_constraints=self.launch_constraints, + product_name_to_id_dict=product_name_to_id_dict, + ) + time.sleep(60) + stack_name = f"launch-constraints-for-portfolio-{portfolio_id}" + cloudformation.create_or_update( + StackName=stack_name, + TemplateBody=template, + NotificationARNs=[ + f"arn:aws:sns:{self.region}:{self.puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events" + ], + ) + result = cloudformation.describe_stacks( + StackName=stack_name, + ).get('Stacks')[0] + f = self.output().open('w') + f.write( + json.dumps( + result, + indent=4, + default=str, + ) + ) + f.close() + + def output(self): + return luigi.LocalTarget( + f"output/CreateLaunchRoleConstraintsForPortfolio/" + f"{self.account_id}-{self.region}-{self.portfolio}-{self.hub_portfolio_id}.json" + ) diff --git a/servicecatalog_puppet/commands/expand.py b/servicecatalog_puppet/manifest_utils.py similarity index 58% rename from servicecatalog_puppet/commands/expand.py rename to servicecatalog_puppet/manifest_utils.py index 0f52c843e..d9497114c 100644 --- a/servicecatalog_puppet/commands/expand.py +++ b/servicecatalog_puppet/manifest_utils.py @@ -1,41 +1,91 @@ +import yaml +import logging +import json from copy import deepcopy from servicecatalog_puppet.macros import macros -import json - -import logging +from servicecatalog_puppet import constants logger = logging.getLogger(__file__) -def expand_path(account, client): - ou = client.convert_path_to_ou(account.get('ou')) - account['ou'] = ou - return expand_ou(account, client) - - -def expand_ou(original_account, client): - expanded = [] - response = client.list_children_nested(ParentId=original_account.get('ou'), ChildType='ACCOUNT') - for result in response: - new_account_id = result.get('Id') - response = client.describe_account(AccountId=new_account_id) - new_account = deepcopy(original_account) - del new_account['ou'] - if response.get('Account').get('Status') == "ACTIVE": - if response.get('Account').get('Name') is not None: - new_account['name'] = response.get('Account').get('Name') - new_account['email'] = response.get('Account').get('Email') - new_account['account_id'] = new_account_id - new_account['expanded_from'] = original_account.get('ou') - new_account['organization'] = response.get('Account').get('Arn').split(":")[5].split("/")[1] - expanded.append(new_account) - else: - logger.info(f"Skipping account as it is not ACTIVE: {json.dump(response.get('Account'), default=str)}") - return expanded - - -def do_expand(manifest, client): +def load(f): + return yaml.safe_load(f.read()) + + +def group_by_tag(launches): + logger.info('Grouping launches by tag') + launches_by_tag = {} + for launch_name, launch_details in launches.items(): + launch_details['launch_name'] = launch_name + launch_tags = launch_details.get('deploy_to').get('tags', []) + for tag_detail in launch_tags: + tag = tag_detail.get('tag') + if launches_by_tag.get(tag) is None: + launches_by_tag[tag] = [] + launches_by_tag[tag].append(launch_details) + logger.info('Finished grouping launches by tag') + return launches_by_tag + + +def group_by_account(launches): + logger.info('Grouping launches by account') + launches_by_account = {} + for launch_name, launch_details in launches.items(): + launch_details['launch_name'] = launch_name + launch_accounts = launch_details.get('deploy_to').get('accounts', []) + for account_detail in launch_accounts: + account_id = account_detail.get('account_id') + if launches_by_account.get(account_id) is None: + launches_by_account[account_id] = [] + launches_by_account[account_id].append(launch_details) + logger.info('Finished grouping launches by account') + return launches_by_account + + +def generate_launch_map(accounts, launches_by_account, launches_by_tag, section): + logger.info('Generating launch map') + deployment_map = {} + for account in accounts: + account_id = account.get('account_id') + deployment_map[account_id] = account + launches = account[section] = {} + for launch in launches_by_account.get(account_id, []): + launch['match'] = "account_match" + launches[launch.get('launch_name')] = launch + for tag in account.get('tags'): + for launch in launches_by_tag.get(tag, []): + launch['match'] = "tag_match" + launch['matching_tag'] = tag + launches[launch.get('launch_name')] = launch + logger.info('Finished generating launch map') + return deployment_map + + +def build_deployment_map(manifest, section): + accounts = manifest.get('accounts') + launches = manifest.get(section, {}) + + verify_no_ous_in_manifest(accounts) + + launches_by_tag = group_by_tag(launches) + launches_by_account = group_by_account(launches) + + return generate_launch_map( + accounts, + launches_by_account, + launches_by_tag, + section + ) + + +def verify_no_ous_in_manifest(accounts): + for account in accounts: + if account.get('account_id') is None: + raise Exception("{} account object does not have an account_id".format(account.get('name'))) + + +def expand_manifest(manifest, client): new_manifest = deepcopy(manifest) new_accounts = new_manifest['accounts'] = [] @@ -86,7 +136,7 @@ def do_expand(manifest, client): ) raise Exception(message) - for launch_name, launch_details in new_manifest.get('launches').items(): + for launch_name, launch_details in new_manifest.get(constants.LAUNCHES, {}).items(): for parameter_name, parameter_details in launch_details.get('parameters', {}).items(): if parameter_details.get('macro'): macro_to_run = macros.get(parameter_details.get('macro').get('method')) @@ -95,3 +145,30 @@ def do_expand(manifest, client): del parameter_details['macro'] return new_manifest + + +def expand_path(account, client): + ou = client.convert_path_to_ou(account.get('ou')) + account['ou'] = ou + return expand_ou(account, client) + + +def expand_ou(original_account, client): + expanded = [] + response = client.list_children_nested(ParentId=original_account.get('ou'), ChildType='ACCOUNT') + for result in response: + new_account_id = result.get('Id') + response = client.describe_account(AccountId=new_account_id) + new_account = deepcopy(original_account) + del new_account['ou'] + if response.get('Account').get('Status') == "ACTIVE": + if response.get('Account').get('Name') is not None: + new_account['name'] = response.get('Account').get('Name') + new_account['email'] = response.get('Account').get('Email') + new_account['account_id'] = new_account_id + new_account['expanded_from'] = original_account.get('ou') + new_account['organization'] = response.get('Account').get('Arn').split(":")[5].split("/")[1] + expanded.append(new_account) + else: + logger.info(f"Skipping account as it is not ACTIVE: {json.dump(response.get('Account'), default=str)}") + return expanded diff --git a/servicecatalog_puppet/requirements-test.txt b/servicecatalog_puppet/requirements-test.txt index 5c97cd917..23968f25c 100644 --- a/servicecatalog_puppet/requirements-test.txt +++ b/servicecatalog_puppet/requirements-test.txt @@ -1,4 +1,6 @@ pytest==4.4.0 pytest-cov==2.6.1 codecov==2.0.15 -pylint==2.3.1 \ No newline at end of file +pylint==2.3.1 +pytest-mock==1.10.4 +pytest-datadir==1.3.0 \ No newline at end of file diff --git a/servicecatalog_puppet/targets/__init__.py b/servicecatalog_puppet/targets/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/servicecatalog_puppet/tasks/__init__.py b/servicecatalog_puppet/tasks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/servicecatalog_puppet/templates/associations.template.yaml.j2 b/servicecatalog_puppet/templates/associations.template.yaml.j2 new file mode 100644 index 000000000..573d70260 --- /dev/null +++ b/servicecatalog_puppet/templates/associations.template.yaml.j2 @@ -0,0 +1,10 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Associations for {{portfolio.DisplayName}} +Resources: +{% for association in portfolio.Associations %} + Association{{ loop.index }}: + Type: AWS::ServiceCatalog::PortfolioPrincipalAssociation + Properties: + PrincipalARN: !Sub "{{ association }}" + PortfolioId: {{ portfolio_id }} + PrincipalType: IAM{% endfor %} diff --git a/servicecatalog_puppet/templates/launch_role_constraints.template.yaml.j2 b/servicecatalog_puppet/templates/launch_role_constraints.template.yaml.j2 new file mode 100644 index 000000000..ab5fbba70 --- /dev/null +++ b/servicecatalog_puppet/templates/launch_role_constraints.template.yaml.j2 @@ -0,0 +1,14 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Launch role contraints for {{portfolio.DisplayName}} + +Resources: +{% for launch_constraint in launch_constraints %} + {% set parentloop = loop %} + {% for roleArn in launch_constraint.roles %} + LaunchRoleConstraintA{{parentloop.index}}B{{loop.index}}: + Type: AWS::ServiceCatalog::LaunchRoleConstraint + Properties: + PortfolioId: {{ portfolio_id }} + ProductId: {{ product_name_to_id_dict.get(launch_constraint.product) }} + RoleArn: !Sub "{{ roleArn }}"{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/servicecatalog_puppet/utils/__init__.py b/servicecatalog_puppet/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/servicecatalog_puppet/utils/manifest.py b/servicecatalog_puppet/utils/manifest.py deleted file mode 100644 index 2a25f2ad7..000000000 --- a/servicecatalog_puppet/utils/manifest.py +++ /dev/null @@ -1,133 +0,0 @@ -import yaml -import logging - -logger = logging.getLogger(__file__) - - -def load(f): - return yaml.safe_load(f.read()) - - -def group_by_tag(launches): - logger.info('Grouping launches by tag') - launches_by_tag = {} - for launch_name, launch_details in launches.items(): - launch_details['launch_name'] = launch_name - launch_tags = launch_details.get('deploy_to').get('tags', []) - for tag_detail in launch_tags: - tag = tag_detail.get('tag') - if launches_by_tag.get(tag) is None: - launches_by_tag[tag] = [] - launches_by_tag[tag].append(launch_details) - logger.info('Finished grouping launches by tag') - return launches_by_tag - - -def group_by_account(launches): - logger.info('Grouping launches by account') - launches_by_account = {} - for launch_name, launch_details in launches.items(): - launch_details['launch_name'] = launch_name - launch_accounts = launch_details.get('deploy_to').get('accounts', []) - for account_detail in launch_accounts: - account_id = account_detail.get('account_id') - if launches_by_account.get(account_id) is None: - launches_by_account[account_id] = [] - launches_by_account[account_id].append(launch_details) - logger.info('Finished grouping launches by account') - return launches_by_account - - -def group_by_product(launches): - logger.info('Grouping launches by product') - launches_by_product = {} - for launch_name, launch_details in launches.items(): - product = launch_details.get('product') - if launches_by_product.get(product) is None: - launches_by_product[product] = [] - launch_details['launch_name'] = launch_name - launches_by_product[product].append(launch_details) - logger.info('Finished grouping launches by product') - return launches_by_product - - -def generate_launch_map(accounts, launches_by_account, launches_by_tag): - logger.info('Generating launch map') - deployment_map = {} - for account in accounts: - account_id = account.get('account_id') - deployment_map[account_id] = account - launches = account['launches'] = {} - for launch in launches_by_account.get(account_id, []): - launch['match'] = "account_match" - launches[launch.get('launch_name')] = launch - for tag in account.get('tags'): - for launch in launches_by_tag.get(tag, []): - launch['match'] = "tag_match" - launch['matching_tag'] = tag - launches[launch.get('launch_name')] = launch - logger.info('Finished generating launch map') - return deployment_map - - -def build_deployment_map(manifest): - accounts = manifest.get('accounts') - launches = manifest.get('launches') - - verify_no_ous_in_manifest(accounts) - - launches_by_product = group_by_product(launches) - # check_for_duplicate_products_in_launches(launches_by_product) - launches_by_tag = group_by_tag(launches) - launches_by_account = group_by_account(launches) - - return generate_launch_map( - accounts, - launches_by_account, - launches_by_tag, - ) - - -def verify_no_ous_in_manifest(accounts): - for account in accounts: - if account.get('account_id') is None: - raise Exception("{} account object does not have an account_id".format(account.get('name'))) - - -def check_for_duplicate_products_in_launches(launches_by_product): - logger.info('Checking for duplicate products by tag') - for product_name, product_launches in launches_by_product.items(): - tags_seen = {} - for product_launch in product_launches: - for tag in product_launch.get('deploy_to').get('tags', []): - tag_name = tag.get('tag') - if tags_seen.get(tag_name) is None: - tags_seen[tag_name] = product_launch - else: - raise Exception( - "Cannot process {}. Already added {} because of tag: {}".format( - product_launch.get('launch_name'), - tags_seen[tag_name].get('launch_name'), - tag_name - ) - ) - logger.info('Finished checking for duplicate products by tag') - - logger.info('Checking for duplicate products by account listed twice') - for product_name, product_launches in launches_by_product.items(): - accounts_seen = {} - for product_launch in product_launches: - for account in product_launch.get('deploy_to').get('accounts', []): - account_id = account.get('account_id') - if accounts_seen.get(account_id) is None: - accounts_seen[account_id] = product_launch - else: - raise Exception( - "Cannot process {}. {} is already receiving product: {} as it was listed in launch: {}".format( - product_launch.get('launch_name'), - account_id, - accounts_seen[account_id].get('product'), - accounts_seen[account_id].get('launch_name'), - ) - ) - logger.info('Finished checking for duplicate products by account listed twice') diff --git a/setup.py b/setup.py index 352d276c2..035980b71 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setuptools.setup( name="aws-service-catalog-puppet", - version="0.1.13", + version="0.1.14", author="Eamonn Faherty", author_email="aws-service-catalog-tools@amazon.com", description="Making it easier to deploy ServiceCatalog products",