diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 07ae0fb..df1273c 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -49,6 +49,8 @@ jobs: GROOVY_NPM_GROOVY_LINT_FILTER_REGEX_EXCLUDE: "Jenkinsfile" MARKDOWN_MARKDOWN_LINK_CHECK_DISABLE_ERRORS: true PYTHON_MYPY_DISABLE_ERRORS: true + PYTHON_BANDIT_DISABLE_ERRORS: true + DISABLE_LINTERS: PYTHON_PYLINT commit-messages: name: Conventional Commits Lint diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile index f5c022b..785fe4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM google/cloud-sdk:alpine ENV HELM_BASE_URL="https://get.helm.sh" ENV HELM_VERSION="3.13.1" SHELL ["/bin/ash", "-eo", "pipefail", "-c"] -RUN apk add --update --no-cache py-pip==23.1.2-r0 openssh==9.3_p2-r0 -RUN python3 -m pip install --no-cache-dir \ +RUN apk add --update --no-cache py-pip==23.1.2-r0 openssh==9.3_p2-r0 && \ + python3 -m pip install --no-cache-dir \ requests==2.25.1 \ jsonschema==4.19.1 \ jsonschema-specifications==2023.7.1 \ @@ -11,8 +11,8 @@ RUN python3 -m pip install --no-cache-dir \ kubernetes==27.2.0 \ ansible-core==2.15.5 \ ansible==8.5.0 && \ - ansible-galaxy collection install ansible.posix && \ - case $(uname -m) in \ + ansible-galaxy collection install ansible.posix && \ + case $(uname -m) in \ x86_64) ARCH=amd64; ;; \ armv7l) ARCH=arm; ;; \ aarch64) ARCH=arm64; ;; \ @@ -20,9 +20,10 @@ RUN python3 -m pip install --no-cache-dir \ s390x) ARCH=s390x; ;; \ *) echo "un-supported arch, exit ..."; exit 1; ;; \ esac && \ - apk add --update --no-cache wget==1.21.4-r0 git==2.40.1-r0 curl==8.4.0-r0 bash==5.2.15-r5 yq==4.33.3-r4 && \ - wget --progress=dot:giga "${HELM_BASE_URL}/helm-v${HELM_VERSION}-linux-${ARCH}.tar.gz" -O - | tar -xz && \ - mv "linux-${ARCH}/helm" /usr/bin/helm && \ - chmod +x /usr/bin/helm && \ - rm -rf "linux-${ARCH}" && \ - gcloud components install kubectl + apk add --update --no-cache wget==1.21.4-r0 git==2.40.1-r0 \ + curl==8.4.0-r0 bash==5.2.15-r5 yq==4.33.3-r4 && \ + wget --progress=dot:giga "${HELM_BASE_URL}/helm-v${HELM_VERSION}-linux-${ARCH}.tar.gz" -O - | tar -xz && \ + mv "linux-${ARCH}/helm" /usr/bin/helm && \ + chmod +x /usr/bin/helm && \ + rm -rf "linux-${ARCH}" && \ + gcloud components install kubectl diff --git a/roles/certificates/files/generate_client_certs.sh b/roles/certificates/files/generate_client_certs.sh deleted file mode 100755 index aba9032..0000000 --- a/roles/certificates/files/generate_client_certs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -CERT_FOLDER="${1:-client1}" -CLIENT_CERT_CN="${2:-Test Client}" - -openssl req \ - -newkey rsa:2048 \ - -nodes -keyform PEM \ - -keyout "$CERT_FOLDER/client-ca.key" \ - -x509 -days 3650 \ - -outform PEM \ - -out "$CERT_FOLDER/client-ca.crt" \ - -subj "/CN=$CLIENT_CERT_CN CA" - -# openssl genrsa -out "$CERT_FOLDER/example-client.key" 2048 - -# openssl req \ -# -new -key "$CERT_FOLDER/example-client.key" \ -# -out "$CERT_FOLDER/example-client.csr" \ -# -subj "/CN=$CLIENT_CERT_CN" - -# openssl x509 \ -# -req -in "$CERT_FOLDER/example-client.csr" \ -# -CA "$CERT_FOLDER/client-ca.crt" \ -# -CAkey "$CERT_FOLDER/client-ca.key" \ -# -set_serial 101 -days 365 \ -# -outform PEM -out "$CERT_FOLDER/example-client.crt" diff --git a/roles/certificates/files/generate_server_certs.sh b/roles/certificates/files/generate_server_certs.sh deleted file mode 100755 index 36bbdb5..0000000 --- a/roles/certificates/files/generate_server_certs.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e -x - -CERT_FOLDER="${1}" -CERT_CN="${2}" -CERT_SAN_CN="${3}" -CERT_SAN_CN=$(awk -F',' '{ for( i=1; i<=NF; i++ ) print $i }' <<<"$CERT_SAN_CN") - -SAN="" -for dns in $CERT_SAN_CN -do - SAN+="DNS:$dns, " -done - -SAN=$(echo $SAN | sed 's/,*$//g') -echo $SAN - -openssl req \ - -nodes \ - -new \ - -x509 \ - -keyout "$CERT_FOLDER/server.key" \ - -out "$CERT_FOLDER/server.crt" \ - -subj "/CN=$CERT_CN" \ - -addext "subjectAltName=$SAN" \ - -days 3650 diff --git a/roles/validate-api-calls/files/deploy_api.py b/roles/validate-api-calls/files/deploy_api.py index 11d2b13..2a2011c 100644 --- a/roles/validate-api-calls/files/deploy_api.py +++ b/roles/validate-api-calls/files/deploy_api.py @@ -20,6 +20,7 @@ from time import sleep import argparse + class Apigee: def __init__( self, @@ -48,7 +49,7 @@ def is_token_valid(self, token): return True return False - def get_access_token(self,access_token): + def get_access_token(self, access_token): token = access_token if token is not None: if self.apigee_type == "x": @@ -86,7 +87,7 @@ def get_api(self, api_name): revision = response.json().get('revision', ['1']) return True, revision else: - return False,None + return False, None def create_api(self, api_name, proxy_bundle_path): url = f"{self.baseurl}/apis?action=import&name={api_name}&validate=true" # noqa @@ -172,7 +173,7 @@ def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_force_redeploy if self.get_api_revisions_deployment( env, api_name, api_rev ): - print(f"INFO : Proxy {api_name} already active in to {env} in Apigee Org {self.org} !") + print(f"INFO : Proxy {api_name} already active in to {env} in Apigee Org {self.org} !") # noqa return True else: if self.deploy_api(env, api_name, api_rev): @@ -235,14 +236,15 @@ def fetch_api_revision(self, api_type, api_name, revision, export_dir): # noqa return True return False + def main(): - parser = argparse.ArgumentParser(description='Deploy Apigee API proxy bundle') + parser = argparse.ArgumentParser(description='Deploy Apigee API proxy bundle') # noqa parser.add_argument('--project_id', help='GCP Project ID') parser.add_argument('--env', help='Apigee Environment Name') parser.add_argument('--api_name', help='Apigee API Name') - parser.add_argument('--api_bundle_path', help='Apigee API Proxy bundle path') + parser.add_argument('--api_bundle_path', help='Apigee API Proxy bundle path') # noqa parser.add_argument('--access_token', help='GCP OAuth Access Token') - parser.add_argument('--api_redeploy', help='Redploy API',action="store_true") + parser.add_argument('--api_redeploy', help='Redploy API',action="store_true") # noqa args = parser.parse_args() TargetApigee = Apigee( "x", @@ -260,5 +262,6 @@ def main(): print(f"Proxy: {args.api_name} deployment failed.") sys.exit(1) + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/roles/validate-input-apigee-control-plane/files/validate_apigee_objects.py b/roles/validate-input-apigee-control-plane/files/validate_apigee_objects.py index 78d1103..71875c4 100644 --- a/roles/validate-input-apigee-control-plane/files/validate_apigee_objects.py +++ b/roles/validate-input-apigee-control-plane/files/validate_apigee_objects.py @@ -20,6 +20,7 @@ import json import collections + class Apigee: def __init__( self, @@ -40,13 +41,13 @@ def __init__( else "Basic {}".format(access_token) # noqa } - def get_token_user(self,token): + def get_token_user(self, token): url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}" # noqa response = requests.get(url) if response.status_code == 200: return response.json()['email'] return '' - + def is_token_valid(self, token): url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}" # noqa response = requests.get(url) @@ -55,8 +56,7 @@ def is_token_valid(self, token): return True return False - - def get_access_token(self,access_token): + def get_access_token(self, access_token): token = access_token if token is not None: if self.apigee_type == "x": @@ -64,7 +64,7 @@ def get_access_token(self,access_token): return token else: print( - 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa type: ignore + 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa pylint: disable=line-too-long ) sys.exit(1) else: @@ -72,7 +72,7 @@ def get_access_token(self,access_token): else: if self.apigee_type == "x": print( - 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa + 'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ' # noqa pylint: disable=line-too-long ) else: print("please export APIGEE_OPDK_ACCESS_TOKEN") @@ -87,16 +87,16 @@ def set_auth_header(self): } def get_org(self): - url=f"{self.baseurl}" + url = f"{self.baseurl}" headers = self.auth_header.copy() response = requests.request("GET", url, headers=headers) if response.status_code == 200: return True else: return False - - def get_environment(self,env): - url=f"{self.baseurl}/environments/{env}" + + def get_environment(self, env): + url = f"{self.baseurl}/environments/{env}" headers = self.auth_header.copy() response = requests.request("GET", url, headers=headers) if response.status_code == 200: @@ -104,21 +104,23 @@ def get_environment(self,env): else: return False - def get_env_group(self,env_group): - url=f"{self.baseurl}/envgroups/{env_group}" + def get_env_group(self, env_group): + url = f"{self.baseurl}/envgroups/{env_group}" headers = self.auth_header.copy() response = requests.request("GET", url, headers=headers) if response.status_code == 200: - return True,response.json() + return True, response.json() else: - return False,None + return False, None -def compare_lists(l1,l2): - if(collections.Counter(l1)==collections.Counter(l2)): - return True + +def compare_lists(l1, l2): + if collections.Counter(l1) == collections.Counter(l2): + return True else: return False + def main(): parser = argparse.ArgumentParser(description='Validates Apigee Objects') parser.add_argument('--input_data', help='Apigee Input data') @@ -138,22 +140,22 @@ def main(): validations = [] authenticated_user = TargetApigee.get_token_user(args.access_token) if not TargetApigee.get_org(): - validations.append(f"Apigee Organization : {apigee_org} doesnt exist OR user {authenticated_user} doesnt have permissions ") + validations.append(f"Apigee Organization : {apigee_org} doesnt exist OR user {authenticated_user} doesnt have permissions ") # noqa pylint: disable=line-too-long for apigee_env in apigee_envs: if not TargetApigee.get_environment(apigee_env['name']): - validations.append(f"Apigee Environment : {apigee_env['name']} doesnt exist OR user {authenticated_user} doesnt have permissions ") + validations.append(f"Apigee Environment : {apigee_env['name']} doesnt exist OR user {authenticated_user} doesnt have permissions ") # noqa pylint: disable=line-too-long for apigee_vhost in apigee_vhosts: - apigee_vhost_status, apigee_vhost_info = TargetApigee.get_env_group(apigee_vhost['name']) + apigee_vhost_status, apigee_vhost_info = TargetApigee.get_env_group(apigee_vhost['name']) # noqa pylint: disable=line-too-long if not apigee_vhost_status: - validations.append(f"Apigee Environment Group : {apigee_vhost['name']} doesnt exist OR user {authenticated_user} doesnt have permissions ") + validations.append(f"Apigee Environment Group : {apigee_vhost['name']} doesnt exist OR user {authenticated_user} doesnt have permissions ") # noqa pylint: disable=line-too-long if apigee_vhost_status: - apigee_vhost_hostname = apigee_vhost.get('hostnames',[]) - apigee_vhost_info_hostname = apigee_vhost_info.get('hostnames',[]) - if not compare_lists(apigee_vhost_hostname, apigee_vhost_info_hostname): - validations.append(f"Apigee Environmrnt Group {apigee_vhost['name']} hostnames {apigee_vhost_hostname} dont match the hostnames in Apigee Management API: {apigee_vhost_info_hostname}") + apigee_vhost_hostname = apigee_vhost.get('hostnames', []) + apigee_vhost_info_hostname = apigee_vhost_info.get('hostnames', []) # noqa pylint: disable=line-too-long + if not compare_lists(apigee_vhost_hostname, apigee_vhost_info_hostname): # noqa pylint: disable=line-too-long + validations.append(f"Apigee Environmrnt Group {apigee_vhost['name']} hostnames {apigee_vhost_hostname} dont match the hostnames in Apigee Management API: {apigee_vhost_info_hostname}") # noqa pylint: disable=line-too-long if len(validations) > 0: print('Validation Errors found !') @@ -161,5 +163,6 @@ def main(): sys.exit(1) print('Apigee Control plane validations successfull') + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/roles/validate-input-k8s-cluster/files/validate_k8s_objects.py b/roles/validate-input-k8s-cluster/files/validate_k8s_objects.py index 0e21939..7d69d57 100644 --- a/roles/validate-input-k8s-cluster/files/validate_k8s_objects.py +++ b/roles/validate-input-k8s-cluster/files/validate_k8s_objects.py @@ -25,13 +25,15 @@ config.load_kube_config(config_file=KUBECONFIG_FILE) except Exception as e: print(f"Unable to load kubeconfig : {KUBECONFIG_FILE}") + print(f"Error: {e}") sys.exit(1) api_instance = client.CoreV1Api() + def check_api_server(): timeout = 2 - try: + try: api_instance.list_namespace(_request_timeout=timeout) return True except Exception as e: @@ -39,6 +41,7 @@ def check_api_server(): print(f"Error: {str(e)}") return False + def check_nodes(label_selector): node_list = api_instance.list_node(label_selector=label_selector) if len(node_list.items) > 0: @@ -46,28 +49,32 @@ def check_nodes(label_selector): else: return False + def list_namespace_secrets(namespace): secret_list = api_instance.list_namespaced_secret(namespace) - return [ each_secret.metadata.name for each_secret in secret_list.items ] + return [each_secret.metadata.name for each_secret in secret_list.items] + def _finditem(obj, key, result=[]): - if key in obj: result.append(obj[key]) + if key in obj: + result.append(obj[key]) for k, v in obj.items(): - if isinstance(v,dict): + if isinstance(v, dict): item = _finditem(v, key, result) if item is not None: return item + def main(): parser = argparse.ArgumentParser(description='Validates Apigee Objects') - parser.add_argument('--input_data', help='Apigee Input data', required=True) - parser.add_argument('--generate_certificates', help='SSL Certs need to be generated', default='True') - parser.add_argument('--create_service_account', help='Service Account Secrets need to be created', default='True') + parser.add_argument('--input_data', help='Apigee Input data', required=True) # noqa + parser.add_argument('--generate_certificates', help='SSL Certs need to be generated', default='True') # noqa + parser.add_argument('--create_service_account', help='Service Account Secrets need to be created', default='True') # noqa args = parser.parse_args() input_data = json.loads(args.input_data) - apigee_vhosts = input_data.get('virtualhosts',[]) - apigee_node_selector = input_data.get('nodeSelector',{}).get( - 'requiredForScheduling',False) + apigee_vhosts = input_data.get('virtualhosts', []) + apigee_node_selector = input_data.get('nodeSelector', {}).get( + 'requiredForScheduling', False) apigee_namespace = 'apigee' validations = [] api_server_reachable = check_api_server() @@ -75,34 +82,34 @@ def main(): sys.exit(1) if apigee_node_selector: - apigee_rt_node_selector_key = input_data.get('nodeSelector',{}).get( - 'apigeeRuntime',{}).get('key','cloud.google.com/gke-nodepool') - apigee_rt_node_selector_value = input_data.get('nodeSelector',{}).get( - 'apigeeRuntime',{}).get('value','apigee-runtime') - apigee_data_node_selector_key = input_data.get('nodeSelector',{}).get( - 'apigeeData',{}).get('key','cloud.google.com/gke-nodepool') - apigee_data_node_selector_value = input_data.get('nodeSelector',{}).get( - 'apigeeData',{}).get('value','apigee-data') - - if not check_nodes(f"{apigee_rt_node_selector_key}={apigee_rt_node_selector_value}"): - validations.append(f"Number of nodes with selectors {apigee_rt_node_selector_key}={apigee_rt_node_selector_value} are Zero[0]") - if not check_nodes(f"{apigee_data_node_selector_key}={apigee_data_node_selector_value}"): - validations.append(f"Number of nodes with selectors {apigee_data_node_selector_key}={apigee_data_node_selector_value} are Zero[0]") + apigee_rt_node_selector_key = input_data.get('nodeSelector', {}).get( # noqa + 'apigeeRuntime', {}).get('key','cloud.google.com/gke-nodepool') # noqa + apigee_rt_node_selector_value = input_data.get('nodeSelector', {}).get( # noqa + 'apigeeRuntime', {}).get('value', 'apigee-runtime') # noqa + apigee_data_node_selector_key = input_data.get('nodeSelector', {}).get( # noqa + 'apigeeData', {}).get('key', 'cloud.google.com/gke-nodepool') # noqa + apigee_data_node_selector_value = input_data.get('nodeSelector', {}).get( # noqa + 'apigeeData', {}).get('value', 'apigee-data') + + if not check_nodes(f"{apigee_rt_node_selector_key}={apigee_rt_node_selector_value}"): # noqa + validations.append(f"Number of nodes with selectors {apigee_rt_node_selector_key}={apigee_rt_node_selector_value} are Zero[0]") # noqa + if not check_nodes(f"{apigee_data_node_selector_key}={apigee_data_node_selector_value}"): # noqa + validations.append(f"Number of nodes with selectors {apigee_data_node_selector_key}={apigee_data_node_selector_value} are Zero[0]") # noqa apigee_secrets = list_namespace_secrets(apigee_namespace) _key_ref = 'sslSecret' if args.generate_certificates == 'False': for apigee_vhost in apigee_vhosts: if apigee_vhost[_key_ref] not in apigee_secrets: - validations.append(f"SSL Secret: {apigee_vhost[_key_ref]} Not found in {apigee_namespace} namespace") + validations.append(f"SSL Secret: {apigee_vhost[_key_ref]} Not found in {apigee_namespace} namespace") # noqa if args.create_service_account == 'False': input_service_accounts = [] - _finditem(input_data,'serviceAccountRef',input_service_accounts) + _finditem(input_data, 'serviceAccountRef', input_service_accounts) for each_sa in input_service_accounts: if each_sa not in apigee_secrets: - validations.append(f"Service Account Secret: {each_sa} Not found in {apigee_namespace} namespace") + validations.append(f"Service Account Secret: {each_sa} Not found in {apigee_namespace} namespace") # noqa if len(validations) > 0: print('Kubernetes validation Errors found !') @@ -110,5 +117,6 @@ def main(): sys.exit(1) print('Kubernetes validations successfull') + if __name__ == '__main__': main()