From 09f362e1361e9a7b20223f56a7b0fd79437ebdda Mon Sep 17 00:00:00 2001 From: Torsten Wendland Date: Thu, 8 Jul 2021 15:18:26 +0200 Subject: [PATCH] Fixes #220 Fixes #220 Wildcard move Fixes #220 Wildcard move Fixes #220 Updated doc Fixes #220 Updated doc Fixes #220 Updated Docu Fixes #220 Updated Docu Update var.tfvars-doc.md Fixes #220 Update var.tfvars-doc.md Updated documentation, that selecting supported branches are also supported Enable external DNS/LB support Signed-off-by: CS Zhang Update the doc Signed-off-by: CS Zhang Update the helpernode_tag to latest level Signed-off-by: CS Zhang Add RHCOS kernel options before installation Signed-off-by: Aishwarya Kamat Allow OCP network customization before installation. (#224) add clusterNetwork_CIDR, serviceNetwork, hostprefix vars Added cs-zhang as approver To set mtu on private network Signed-off-by: Aishwarya Kamat remove mkumatag from reviewer list Not actively involved, hence removing my entry from the reviewers to avoid getting assigned automatically for the review force centos stream to use ansible 2.9 like rhel8 Accessing cluster using non-root user Signed-off-by: Aishwarya Kamat bastion fqdn with clusterID as subdmain To remove the scp error with Terraform v1.1.x Signed-off-by: Aishwarya Kamat To Update the Terraform Version Signed-off-by: Aishwarya Kamat FIPS enablement Signed-off-by: Aishwarya Kamat Merging the code #220 Merging the code #220 --- OWNERS | 2 +- docs/automation_host_prereqs.md | 2 +- docs/var.tfvars-doc.md | 69 ++++++++- modules/1_bastion/bastion.tf | 20 +-- modules/1_bastion/versions.tf | 2 +- modules/2_network/versions.tf | 2 +- modules/3_helpernode/helpernode.tf | 71 ++++++++- .../templates/helpernode_inventory | 2 +- .../templates/helpernode_vars.yaml | 8 +- modules/3_helpernode/variables.tf | 3 + modules/3_helpernode/versions.tf | 2 +- modules/4_nodes/versions.tf | 2 +- modules/5_install/install.tf | 142 ++++++++++++++---- modules/5_install/templates/install_inventory | 2 +- modules/5_install/templates/install_vars.yaml | 8 + modules/5_install/variables.tf | 8 + modules/5_install/versions.tf | 2 +- ocp.tf | 10 ++ var.tfvars | 10 +- variables.tf | 54 ++++++- versions.tf | 2 +- 21 files changed, 356 insertions(+), 67 deletions(-) diff --git a/OWNERS b/OWNERS index eaf34e802..cd719e6a7 100644 --- a/OWNERS +++ b/OWNERS @@ -1,5 +1,4 @@ reviewers: - - mkumatag - Prajyot-Parab - sudeeshjohn - yussufsh @@ -7,4 +6,5 @@ reviewers: - cs-zhang approvers: - bpradipt + - cs-zhang - yussufsh diff --git a/docs/automation_host_prereqs.md b/docs/automation_host_prereqs.md index 00fa56e9b..901c09710 100644 --- a/docs/automation_host_prereqs.md +++ b/docs/automation_host_prereqs.md @@ -22,7 +22,7 @@ Install the following packages on the automation host. Select the appropriate in **Terraform >= 0.13.0**: Please refer to the [link](https://learn.hashicorp.com/terraform/getting-started/install.html) for instructions on installing Terraform. For validating the version run `terraform version` command after install. Install Terraform and providers for Power environment: -1. Download the Terraform binary version 0.13.5 from https://www.power-devops.com/terraform and install it to /usr/local/bin. +1. Download and install the Terraform binary (>= 0.13.0) for Linux/ppc64le from https://www.power-devops.com/terraform. 2. Download the required Terraform providers for Power into your TF project directory: ``` $ cd diff --git a/docs/var.tfvars-doc.md b/docs/var.tfvars-doc.md index b680b97c0..c3818e517 100644 --- a/docs/var.tfvars-doc.md +++ b/docs/var.tfvars-doc.md @@ -83,10 +83,11 @@ worker = {instance_type = "", i ``` These set of variables specify the username and the SSH key to be used for accessing the bastion node. ``` -rhel_username = "root" +rhel_username = "root" #Set it to an appropriate username for non-root user access public_key_file = "data/id_rsa.pub" private_key_file = "data/id_rsa" ``` +rhel_username is set to root. rhel_username can be set to an appropriate username having superuser privileges with no password prompt. Please note that only OpenSSH formatted keys are supported. Refer to the following links for instructions on creating SSH key based on your platform. - Windows 10 - https://phoenixnap.com/kb/generate-ssh-key-windows-10 - Mac OSX - https://www.techrepublic.com/article/how-to-generate-ssh-keys-on-macos-mojave/ @@ -139,10 +140,24 @@ If `cluster_if_prefix` is not set, the `cluster_id` will be used only without pr A random value will be used for `cluster_id` if not set. The total length of `cluster_id_prefix`.`cluster_id` should not exceed 14 characters. +### FIPS Variable for OpenShift deployment + +These variables will be used for deploying OCP in FIPS mode. +Change the values as per your requirement. +``` +fips_compliant = false +``` + ### Misc Customizations These variables provides miscellaneous customizations. For common usage scenarios these are not required and should be left unchanged. +The following variables are used to define the IP address for the preconfigured external DNS and the Load-balancer +``` +lb_ipaddr = "" +ext_dns = "" +``` + The following variable is used to set the network adapter type for the VMs. By default the VMs will use SEA. If SRIOV is required then uncomment the variable ``` network_type = "SRIOV" @@ -179,14 +194,49 @@ This variable can be used for trying out custom OpenShift install image for deve release_image_override = "" ``` -These variables specify the ansible playbooks that are used for OpenShift install and post-install customizations. +These variables specify the ansible playbooks that are used for OpenShift install and post-install customizations. If the URL ends with a file name extension .zip, then it is assumed that it points to a HTTP/HTTPS server and curl/unzip will be used to extract the package. URLs without ending with .zip are recognized as GitHub repositories and git clone && git checkout are used. +`Only .zip is supported file format on web servers. The all files must be placed in folders starting with ocp4-playbooks, or ocp4-helpernode! It is allowed to extend the directory name with additional informations: e.g. ocp4-helpernode- - 50 for OpenshiftSDN and - 100 for OVNKubernetes network provider. +``` +private_network_mtu = 1450 +``` + These variables can be used when debugging ansible playbooks ``` installer_log_level = "info" @@ -198,6 +248,16 @@ This variable specifies the external DNS servers to forward DNS queries that can dns_forwarders = "1.1.1.1; 9.9.9.9" ``` +List of [day-1 kernel arguments](https://docs.openshift.com/container-platform/4.8/installing/install_config/installing-customizing.html#installation-special-config-kargs_installing-customizing) for the cluster nodes. +To add kernel arguments to master or worker nodes, using MachineConfig object and inject that object into the set of manifest files used by Ignition during cluster setup. +``` +rhcos_pre_kernel_options = [] +``` +- Example 1 + ``` + rhcos_pre_kernel_options = ["rd.multipath=default","root=/dev/disk/by-label/dm-mpath-root"] + ``` + List of [kernel arguments](https://docs.openshift.com/container-platform/4.4/nodes/nodes/nodes-nodes-working.html#nodes-nodes-kernel-arguments_nodes-nodes-working) for the cluster nodes. Note that this will be applied after the cluster is installed and all the nodes are in `Ready` status. ``` @@ -255,4 +315,7 @@ This variable is used to set the default Container Network Interface (CNI) netwo ``` cni_network_provider = "OpenshiftSDN" -``` +cluster_network_cidr = "10.128.0.0/14" +cluster_network_hostprefix = "23" +service_network = "172.30.0.0/16" +``` \ No newline at end of file diff --git a/modules/1_bastion/bastion.tf b/modules/1_bastion/bastion.tf index 9d0a3a28d..65d27563b 100644 --- a/modules/1_bastion/bastion.tf +++ b/modules/1_bastion/bastion.tf @@ -93,18 +93,18 @@ resource "null_resource" "bastion_init" { } provisioner "file" { content = var.private_key - destination = "$HOME/.ssh/id_rsa" + destination = ".ssh/id_rsa" } provisioner "file" { content = var.public_key - destination = "$HOME/.ssh/id_rsa.pub" + destination = ".ssh/id_rsa.pub" } provisioner "remote-exec" { inline = [ - "sudo chmod 600 $HOME/.ssh/id_rsa*", + "sudo chmod 600 .ssh/id_rsa*", "sudo sed -i.bak -e 's/^ - set_hostname/# - set_hostname/' -e 's/^ - update_hostname/# - update_hostname/' /etc/cloud/cloud.cfg", - "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}", - "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null", + "sudo hostnamectl set-hostname --static ${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}", + "echo 'HOSTNAME=${lower(var.cluster_id)}-bastion-${count.index}.${lower(var.cluster_id)}.${var.cluster_domain}' | sudo tee -a /etc/sysconfig/network > /dev/null", "sudo hostname -F /etc/hostname", "echo 'vm.max_map_count = 262144' | sudo tee --append /etc/sysctl.conf > /dev/null", ] @@ -265,12 +265,12 @@ resource "null_resource" "bastion_packages" { provisioner "remote-exec" { inline = [ "#sudo yum update -y --skip-broken", - "sudo yum install -y wget jq git net-tools vim python3 tar" + "sudo yum install -y wget jq git net-tools vim python3 tar curl unzip" ] } provisioner "remote-exec" { inline = [ - "sudo yum install -y ansible" + "sudo yum install -y ansible-2.9.*" ] } provisioner "remote-exec" { @@ -327,11 +327,11 @@ resource "null_resource" "setup_nfs_disk" { } provisioner "remote-exec" { inline = [ - "rm -rf mkdir ${local.storage_path}; mkdir -p ${local.storage_path}; chmod -R 755 ${local.storage_path}", + "sudo rm -rf mkdir ${local.storage_path}; sudo mkdir -p ${local.storage_path}; sudo chmod -R 755 ${local.storage_path}", "sudo chmod +x /tmp/create_disk_link.sh", # Fix for copying file from Windows OS having CR - "sed -i 's/\r//g' /tmp/create_disk_link.sh", - "/tmp/create_disk_link.sh", + "sudo sed -i 's/\r//g' /tmp/create_disk_link.sh", + "sudo /tmp/create_disk_link.sh", "sudo mkfs.ext4 -F /dev/${local.disk_config.disk_name}", "echo '/dev/${local.disk_config.disk_name} ${local.storage_path} ext4 defaults 0 0' | sudo tee -a /etc/fstab > /dev/null", "sudo mount ${local.storage_path}", diff --git a/modules/1_bastion/versions.tf b/modules/1_bastion/versions.tf index 052785e22..afb32cda1 100644 --- a/modules/1_bastion/versions.tf +++ b/modules/1_bastion/versions.tf @@ -33,5 +33,5 @@ terraform { version = "~> 2.3" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" } diff --git a/modules/2_network/versions.tf b/modules/2_network/versions.tf index efa46b12e..fcb9ad20f 100644 --- a/modules/2_network/versions.tf +++ b/modules/2_network/versions.tf @@ -25,5 +25,5 @@ terraform { version = "~> 1.32" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" } diff --git a/modules/3_helpernode/helpernode.tf b/modules/3_helpernode/helpernode.tf index ebe46693d..be3002dad 100644 --- a/modules/3_helpernode/helpernode.tf +++ b/modules/3_helpernode/helpernode.tf @@ -38,6 +38,8 @@ locals { bastion_master_ip = var.bastion_ip[0] bastion_backup_ip = length(var.bastion_ip) > 1 ? slice(var.bastion_ip, 1, length(var.bastion_ip)) : [] forwarders = var.dns_forwarders + lb_ipaddr = var.lb_ipaddr + ext_dns = var.ext_dns gateway_ip = var.gateway_ip netmask = cidrnetmask(var.cidr) broadcast = cidrhost(var.cidr,-1) @@ -67,20 +69,21 @@ locals { ] local_registry = local.local_registry + helm_repo = var.helm_repo client_tarball = var.openshift_client_tarball install_tarball = var.openshift_install_tarball } helpernode_inventory = { - bastion_ip = var.bastion_ip + rhel_username = var.rhel_username + bastion_ip = var.bastion_ip } } -resource "null_resource" "config" { - +resource "null_resource" "prep_helpernode_tools_git" { triggers = { bootstrap_count = var.bootstrap_port_ip == "" ? 0 : 1 - worker_count = length(var.worker_port_ips) } + count = length(regexall("\\.zip$", var.helpernode_repo)) == 0 ? 1 : 0 connection { type = "ssh" @@ -101,23 +104,75 @@ resource "null_resource" "config" { "cd ocp4-helpernode && git checkout ${var.helpernode_tag}" ] } +} + +resource "null_resource" "prep_helpernode_tools_curl" { + triggers = { + bootstrap_count = var.bootstrap_port_ip == "" ? 0 : 1 + } + count = length(regexall("\\.zip$", var.helpernode_repo)) > 0 ? 1 : 0 + + connection { + type = "ssh" + user = var.rhel_username + host = var.bastion_ip[0] + private_key = var.private_key + agent = var.ssh_agent + timeout = "${var.connection_timeout}m" + bastion_host = var.jump_host + } + + provisioner "remote-exec" { + inline = [ + "mkdir -p .openshift", + "rm -rf ocp4-helpernode", + "rm -rf ocp4-extract-helper", + "mkdir -p ocp4-extract-helper", + "echo 'Downloading ocp4-helpernode...'", + "curl -o ocp4-extract-helper/ocp4-helpernode.zip ${var.helpernode_repo}", + "echo 'Extracting ocp4-helpernode...'", + "cd ocp4-extract-helper && unzip ocp4-helpernode.zip", + "cd .. && rm -rf ocp4-extract-helper/ocp4-helpernode.zip", + "mv ocp4-extract-helper/ocp4-helpernode* ocp4-helpernode", + "rm -rf ocp4-extract-helper" + ] + } +} + +resource "null_resource" "config" { + depends_on = [null_resource.prep_helpernode_tools_git, null_resource.prep_helpernode_tools_curl] + triggers = { + bootstrap_count = var.bootstrap_port_ip == "" ? 0 : 1 + worker_count = length(var.worker_port_ips) + } + + connection { + type = "ssh" + user = var.rhel_username + host = var.bastion_ip[0] + private_key = var.private_key + agent = var.ssh_agent + timeout = "${var.connection_timeout}m" + bastion_host = var.jump_host + } + provisioner "file" { content = templatefile("${path.module}/templates/helpernode_inventory", local.helpernode_inventory) - destination = "$HOME/ocp4-helpernode/inventory" + destination = "ocp4-helpernode/inventory" } provisioner "file" { content = var.pull_secret - destination = "$HOME/.openshift/pull-secret" + destination = ".openshift/pull-secret" } provisioner "file" { content = templatefile("${path.module}/templates/helpernode_vars.yaml", local.helpernode_vars) - destination = "$HOME/ocp4-helpernode/helpernode_vars.yaml" + destination = "ocp4-helpernode/helpernode_vars.yaml" } provisioner "remote-exec" { inline = [ "sed -i \"/^helper:.*/a \\ \\ networkifacename: $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}')\" ocp4-helpernode/helpernode_vars.yaml", "echo 'Running ocp4-helpernode playbook...'", - "cd ocp4-helpernode && ansible-playbook -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options}" + "cd ocp4-helpernode && ansible-playbook -i inventory -e @helpernode_vars.yaml tasks/main.yml ${var.ansible_extra_options} --become" ] } } diff --git a/modules/3_helpernode/templates/helpernode_inventory b/modules/3_helpernode/templates/helpernode_inventory index ac1eaddde..d6b2ad059 100644 --- a/modules/3_helpernode/templates/helpernode_inventory +++ b/modules/3_helpernode/templates/helpernode_inventory @@ -1,4 +1,4 @@ [vmhost] %{ for ip in bastion_ip ~} -${ip} ansible_connection=ssh ansible_user=root +${ip} ansible_connection=ssh ansible_user=${rhel_username} %{ endfor ~} diff --git a/modules/3_helpernode/templates/helpernode_vars.yaml b/modules/3_helpernode/templates/helpernode_vars.yaml index 5f3edc414..c04c40603 100644 --- a/modules/3_helpernode/templates/helpernode_vars.yaml +++ b/modules/3_helpernode/templates/helpernode_vars.yaml @@ -21,10 +21,16 @@ dns: domain: "${cluster_domain}" clusterid: "${cluster_id}" forwarder1: "${forwarders}" +%{ if lb_ipaddr != "" } + lb_ipaddr: "${lb_ipaddr}" +%{ endif } dhcp: router: "${gateway_ip}" bcast: "${broadcast}" netmask: "${netmask}" +%{ if ext_dns != "" } + dns: "${ext_dns}" +%{ endif } ipid: "${ipid}" netmaskid: "${netmask}" poolstart: "${pool.start}" @@ -84,4 +90,4 @@ ocp_initramfs: "file:///dev/null" ocp_install_kernel: "file:///dev/null" # This is required for latest helpernode. TODO: Remove when https://github.com/RedHatOfficial/ocp4-helpernode/pull/140 is merged -helm_source: "https://get.helm.sh/helm-v3.4.0-linux-ppc64le.tar.gz" +helm_source: "${helm_repo}" diff --git a/modules/3_helpernode/variables.tf b/modules/3_helpernode/variables.tf index e17d6f869..9b9661edb 100644 --- a/modules/3_helpernode/variables.tf +++ b/modules/3_helpernode/variables.tf @@ -29,6 +29,8 @@ variable "dns_forwarders" { default = "8.8.8.8; 9.9.9.9" } +variable "lb_ipaddr" {} +variable "ext_dns" {} variable "gateway_ip" {} variable "cidr" {} variable "allocation_pools" {} @@ -58,6 +60,7 @@ variable "ocp_release_tag" {} variable "helpernode_repo" {} variable "helpernode_tag" {} +variable "helm_repo" {} variable "ansible_extra_options" {} diff --git a/modules/3_helpernode/versions.tf b/modules/3_helpernode/versions.tf index 103cb6a14..f280c68c9 100644 --- a/modules/3_helpernode/versions.tf +++ b/modules/3_helpernode/versions.tf @@ -25,5 +25,5 @@ terraform { version = "~> 2.1" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" } diff --git a/modules/4_nodes/versions.tf b/modules/4_nodes/versions.tf index 2b54931e9..dd2344582 100644 --- a/modules/4_nodes/versions.tf +++ b/modules/4_nodes/versions.tf @@ -33,5 +33,5 @@ terraform { version = "~> 2.3" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" } diff --git a/modules/5_install/install.tf b/modules/5_install/install.tf index fce534370..91edd05b3 100644 --- a/modules/5_install/install.tf +++ b/modules/5_install/install.tf @@ -24,7 +24,10 @@ locals { ocp_release_repo = "ocp4/openshift4" + bastion_count = lookup(var.bastion, "count", 1) + install_inventory = { + rhel_username = var.rhel_username bastion_hosts = [for ix in range(length(var.bastion_ip)) : "${var.cluster_id}-bastion-${ix}"] bootstrap_host = var.bootstrap_ip == "" ? "" : "bootstrap" master_hosts = [for ix in range(length(var.master_ips)) : "master-${ix}"] @@ -40,28 +43,36 @@ locals { local_registry_ocp_image = "registry.${var.cluster_id}.${local.cluster_domain}:5000/${local.ocp_release_repo}:${var.ocp_release_tag}" install_vars = { - bastion_vip = var.bastion_vip - cluster_id = var.cluster_id - cluster_domain = local.cluster_domain - pull_secret = var.pull_secret - public_ssh_key = var.public_key - storage_type = var.storage_type - log_level = var.log_level - release_image_override = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override - enable_local_registry = var.enable_local_registry - node_connection_timeout = 60 * var.connection_timeout - rhcos_kernel_options = var.rhcos_kernel_options - sysctl_tuned_options = var.sysctl_tuned_options - sysctl_options = var.sysctl_options - match_array = indent(2,var.match_array) - setup_squid_proxy = var.setup_squid_proxy - squid_source_range = var.cidr - proxy_url = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}" - no_proxy = var.cidr - chrony_config = var.chrony_config - chrony_config_servers = var.chrony_config_servers - chrony_allow_range = var.cidr - cni_network_provider = var.cni_network_provider + bastion_vip = var.bastion_vip + cluster_id = var.cluster_id + cluster_domain = local.cluster_domain + pull_secret = var.pull_secret + public_ssh_key = var.public_key + storage_type = var.storage_type + log_level = var.log_level + release_image_override = var.enable_local_registry ? local.local_registry_ocp_image : var.release_image_override + enable_local_registry = var.enable_local_registry + fips_compliant = var.fips_compliant + node_connection_timeout = 60 * var.connection_timeout + rhcos_pre_kernel_options = var.rhcos_pre_kernel_options + rhcos_kernel_options = var.rhcos_kernel_options + sysctl_tuned_options = var.sysctl_tuned_options + sysctl_options = var.sysctl_options + match_array = indent(2,var.match_array) + setup_squid_proxy = var.setup_squid_proxy + squid_source_range = var.cidr + proxy_url = local.proxy.server == "" ? "" : "http://${local.proxy.user_pass}${local.proxy.server}:${local.proxy.port}" + no_proxy = var.cidr + chrony_config = var.chrony_config + chrony_config_servers = var.chrony_config_servers + chrony_allow_range = var.cidr + cni_network_provider = var.cni_network_provider + cluster_network_cidr = var.cluster_network_cidr + cluster_network_hostprefix = var.cluster_network_hostprefix + service_network = var.service_network + # Set CNI network MTU to MTU - 100 for OVNKubernetes and MTU - 50 for OpenShiftSDN(default). + # Add new conditions here when we have more network providers + cni_network_mtu = var.cni_network_provider == "OVNKubernetes" ? var.private_network_mtu - 100 : var.private_network_mtu - 50 } upgrade_vars = { @@ -73,10 +84,8 @@ locals { } } -resource "null_resource" "install" { - triggers = { - worker_count = length(var.worker_ips) - } +resource "null_resource" "prep_playbooks_tools_git" { + count = length(regexall("\\.zip$", var.install_playbook_repo)) == 0 ? 1 : 0 connection { type = "ssh" @@ -96,13 +105,87 @@ resource "null_resource" "install" { "cd ocp4-playbooks && git checkout ${var.install_playbook_tag}" ] } +} + +resource "null_resource" "prep_playbooks_tools_curl" { + count = length(regexall("\\.zip$", var.install_playbook_repo)) > 0 ? 1 : 0 + + connection { + type = "ssh" + user = var.rhel_username + host = var.bastion_ip[0] + private_key = var.private_key + agent = var.ssh_agent + timeout = "${var.connection_timeout}m" + bastion_host = var.jump_host + } + + provisioner "remote-exec" { + inline = [ + "rm -rf ocp4-playbooks", + "rm -rf ocp4-extract-helper", + "mkdir -p ocp4-extract-helper", + "echo 'Downloading ocp4-playbooks...'", + "curl -o ocp4-extract-helper/ocp4-playbooks.zip ${var.install_playbook_repo}", + "echo 'Extracting ocp4-playbooks...'", + "cd ocp4-extract-helper && unzip ocp4-playbooks.zip", + "cd .. && rm -rf ocp4-extract-helper/ocp4-playbooks.zip", + "mv ocp4-extract-helper/ocp4-playbooks* ocp4-playbooks", + "rm -rf ocp4-extract-helper" + ] + } +} + +resource "null_resource" "pre_install" { + count = local.bastion_count + + connection { + type = "ssh" + user = var.rhel_username + host = var.bastion_ip[count.index] + private_key = var.private_key + agent = var.ssh_agent + timeout = "${var.connection_timeout}m" + bastion_host = var.jump_host + } + + # DHCP config for setting MTU; Since helpernode DHCP template does not support MTU setting + provisioner "remote-exec" { + inline = [ + # Set specified mtu for private interface. + "sudo ip link set dev $(ip r | grep \"${var.cidr} dev\" | awk '{print $3}') mtu ${var.private_network_mtu}", + "echo MTU=${var.private_network_mtu} | sudo tee -a /etc/sysconfig/network-scripts/ifcfg-$(ip r | grep ${var.cidr} | awk '{print $3}')", + # DHCP config for setting MTU; + "sed -i.mtubak '/option routers/i option interface-mtu ${var.private_network_mtu};' /etc/dhcp/dhcpd.conf", + "sudo systemctl restart dhcpd.service" + ] + } +} + +resource "null_resource" "install" { + depends_on = [null_resource.prep_playbooks_tools_git, null_resource.prep_playbooks_tools_curl, null_resource.pre_install] + + triggers = { + worker_count = length(var.worker_ips) + } + + connection { + type = "ssh" + user = var.rhel_username + host = var.bastion_ip[0] + private_key = var.private_key + agent = var.ssh_agent + timeout = "${var.connection_timeout}m" + bastion_host = var.jump_host + } + provisioner "file" { content = templatefile("${path.module}/templates/install_inventory", local.install_inventory) - destination = "$HOME/ocp4-playbooks/inventory" + destination = "ocp4-playbooks/inventory" } provisioner "file" { content = templatefile("${path.module}/templates/install_vars.yaml", local.install_vars) - destination = "$HOME/ocp4-playbooks/install_vars.yaml" + destination = "ocp4-playbooks/install_vars.yaml" } provisioner "remote-exec" { inline = [ @@ -128,7 +211,7 @@ resource "null_resource" "upgrade" { provisioner "file" { content = templatefile("${path.module}/templates/upgrade_vars.yaml", local.upgrade_vars) - destination = "$HOME/ocp4-playbooks/upgrade_vars.yaml" + destination = "ocp4-playbooks/upgrade_vars.yaml" } provisioner "remote-exec" { inline = [ @@ -137,4 +220,3 @@ resource "null_resource" "upgrade" { ] } } - diff --git a/modules/5_install/templates/install_inventory b/modules/5_install/templates/install_inventory index aec3cd88c..5afd4aff3 100644 --- a/modules/5_install/templates/install_inventory +++ b/modules/5_install/templates/install_inventory @@ -1,6 +1,6 @@ [bastion] %{ for bastion in bastion_hosts ~} -${bastion} ansible_connection=ssh ansible_user=root +${bastion} ansible_connection=ssh ansible_user=${rhel_username} %{ endfor ~} %{ if bootstrap_host != "" ~} diff --git a/modules/5_install/templates/install_vars.yaml b/modules/5_install/templates/install_vars.yaml index f83262ab9..244ec6b9a 100644 --- a/modules/5_install/templates/install_vars.yaml +++ b/modules/5_install/templates/install_vars.yaml @@ -10,9 +10,12 @@ storage_type: ${storage_type} log_level: ${log_level} release_image_override: '${release_image_override}' enable_local_registry: ${enable_local_registry} +fips_compliant: "${fips_compliant}" node_connection_timeout: ${node_connection_timeout} +rhcos_pre_kernel_options: [%{ for opt in rhcos_pre_kernel_options ~}"${opt}",%{ endfor ~}] + rhcos_kernel_options: [%{ for opt in rhcos_kernel_options ~}"${opt}",%{ endfor ~}] sysctl_tuned_options: ${sysctl_tuned_options} @@ -54,3 +57,8 @@ bastion_vip: "${bastion_vip}" %{ endif ~} cni_network_provider: "${cni_network_provider}" + +cluster_network_cidr: "${cluster_network_cidr}" +cluster_network_hostprefix: "${cluster_network_hostprefix}" +service_network: "${service_network}" +cni_network_mtu: "${cni_network_mtu}" diff --git a/modules/5_install/variables.tf b/modules/5_install/variables.tf index 076dd66a8..97408ff45 100644 --- a/modules/5_install/variables.tf +++ b/modules/5_install/variables.tf @@ -35,6 +35,7 @@ variable "ssh_agent" {} variable "connection_timeout" {} variable "jump_host" {} +variable "bastion" {} variable "bootstrap_ip" {} variable "master_ips" {} variable "worker_ips" {} @@ -42,6 +43,9 @@ variable "worker_ips" {} variable "public_key" {} variable "pull_secret" {} variable "release_image_override" {} +variable "fips_compliant" {} + +variable "private_network_mtu" {} variable "enable_local_registry" {} variable "local_registry_image" {} @@ -54,6 +58,7 @@ variable "storage_type" {} variable "log_level" {} variable "ansible_extra_options" {} +variable "rhcos_pre_kernel_options" {} variable "rhcos_kernel_options" {} variable "sysctl_tuned_options" {} @@ -72,3 +77,6 @@ variable "upgrade_pause_time" {} variable "upgrade_delay_time" {} variable "cni_network_provider" {} +variable "cluster_network_cidr" {} +variable "cluster_network_hostprefix" {} +variable "service_network" {} diff --git a/modules/5_install/versions.tf b/modules/5_install/versions.tf index 103cb6a14..f280c68c9 100644 --- a/modules/5_install/versions.tf +++ b/modules/5_install/versions.tf @@ -25,5 +25,5 @@ terraform { version = "~> 2.1" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" } diff --git a/ocp.tf b/ocp.tf index 54d447c7d..a12ae4152 100644 --- a/ocp.tf +++ b/ocp.tf @@ -88,6 +88,8 @@ module "helpernode" { cluster_domain = var.cluster_domain cluster_id = local.cluster_id dns_forwarders = var.dns_forwarders + lb_ipaddr = var.lb_ipaddr + ext_dns = var.ext_dns gateway_ip = module.network.gateway_ip cidr = module.network.cidr allocation_pools = module.network.allocation_pools @@ -111,6 +113,7 @@ module "helpernode" { ocp_release_tag = var.ocp_release_tag helpernode_repo = var.helpernode_repo helpernode_tag = var.helpernode_tag + helm_repo = var.helm_repo ansible_extra_options = var.ansible_extra_options chrony_config = var.chrony_config chrony_config_servers = var.chrony_config_servers @@ -145,6 +148,7 @@ module "install" { cluster_domain = var.cluster_domain cluster_id = local.cluster_id cidr = module.network.cidr + bastion = var.bastion bastion_vip = module.network.bastion_vip bastion_ip = module.bastion.bastion_ip rhel_username = var.rhel_username @@ -159,13 +163,16 @@ module "install" { pull_secret = file(coalesce(var.pull_secret_file, "/dev/null")) storage_type = local.storage_type release_image_override = var.release_image_override + private_network_mtu = var.private_network_mtu enable_local_registry = var.enable_local_registry + fips_compliant = var.fips_compliant local_registry_image = var.local_registry_image ocp_release_tag = var.ocp_release_tag install_playbook_repo = var.install_playbook_repo install_playbook_tag = var.install_playbook_tag log_level = var.installer_log_level ansible_extra_options = var.ansible_extra_options + rhcos_pre_kernel_options = var.rhcos_pre_kernel_options rhcos_kernel_options = var.rhcos_kernel_options sysctl_tuned_options = var.sysctl_tuned_options sysctl_options = var.sysctl_options @@ -180,4 +187,7 @@ module "install" { chrony_config = var.chrony_config chrony_config_servers = var.chrony_config_servers cni_network_provider = var.cni_network_provider + cluster_network_cidr = var.cluster_network_cidr + cluster_network_hostprefix = var.cluster_network_hostprefix + service_network = var.service_network } diff --git a/var.tfvars b/var.tfvars index ccdc5bddf..fe4a92090 100644 --- a/var.tfvars +++ b/var.tfvars @@ -21,7 +21,7 @@ worker = {instance_type = "", # worker = {instance_type = "", image_id = "", availability_zone = "", "count" = 2, data_volume_count = 0, data_volume_size = 100} -rhel_username = "root" +rhel_username = "root" #Set it to an appropriate username for non-root user access public_key_file = "data/id_rsa.pub" private_key_file = "data/id_rsa" rhel_subscription_username = "" #Leave this as-is if using CentOS as bastion image @@ -41,7 +41,7 @@ pull_secret_file = "data/pull-secret.txt" cluster_domain = "ibm.com" # Set domain to nip.io or xip.io if you prefer using online wildcard domain and avoid modifying /etc/hosts cluster_id_prefix = "test-ocp" # Set it to empty if just want to use cluster_id without prefix cluster_id = "" # It will use random generated id with cluster_id_prefix if this is not set - +#fips_compliant = false # Set it true if you prefer to use FIPS enable in ocp deployment ### Misc Customizations @@ -61,11 +61,13 @@ cluster_id = "" # It will use random generated id with #helpernode_tag = "" #install_playbook_repo = "https://github.com/ocp-power-automation/ocp4-playbooks" #install_playbook_tag = "" +#helm_repo = "https://get.helm.sh/helm-v3.6.3-linux-ppc64le.tar.gz" #installer_log_level = "info" #ansible_extra_options = "-v" #ansible_repo_name = "ansible-2.9-for-rhel-8-ppc64le-rpms" #dns_forwarders = "1.1.1.1; 9.9.9.9" +#rhcos_pre_kernel_options = [] #rhcos_kernel_options = [] #chrony_config = true #chrony_config_servers = [ {server = "0.centos.pool.ntp.org", options = "iburst"}, {server = "1.centos.pool.ntp.org", options = "iburst"} ] @@ -90,3 +92,7 @@ cluster_id = "" # It will use random generated id with #upgrade_delay_time = "600" #cni_network_provider = "OpenshiftSDN" +#cluster_network_cidr = "10.128.0.0/14" +#cluster_network_hostprefix = "23" +#service_network = "172.30.0.0/16" +#private_network_mtu = "1450" diff --git a/variables.tf b/variables.tf index b878c92a1..23984752c 100644 --- a/variables.tf +++ b/variables.tf @@ -199,6 +199,12 @@ variable "rhel_subscription_org" { variable "rhel_subscription_activationkey" { default = "" } + +variable "rhcos_pre_kernel_options" { + description = "List of kernel arguments for the cluster nodes for pre-installation" + default = [] +} + variable "rhcos_kernel_options" { description = "List of kernel arguments for the cluster nodes" default = [] @@ -249,6 +255,12 @@ variable "jump_host" { default = "" } +variable "private_network_mtu" { + type = number + description = "MTU value for the private network interface on RHEL and RHCOS nodes" + default = 1450 +} + variable "installer_log_level" { description = "Set the log level required for openshift-install commands" default = "info" @@ -263,7 +275,7 @@ variable "helpernode_repo" { variable "helpernode_tag" { description = "Set the branch/tag name or commit# for using ocp4-helpernode repo" # Checkout level for https://github.com/RedHatOfficial/ocp4-helpernode which is used for setting up services required on bastion node - default = "1ac7f276b537cd734240eda9ed554a254ba80629" + default = "324e09e3d303101874f540730c993cd986ddbc04" } variable "install_playbook_repo" { @@ -275,7 +287,12 @@ variable "install_playbook_repo" { variable "install_playbook_tag" { description = "Set the branch/tag name or commit# for using ocp4-playbooks repo" # Checkout level for https://github.com/ocp-power-automation/ocp4-playbooks which is used for running ocp4 installations steps - default = "10fec74c9e987b39f7af1127abe304a9e41f8e65" + default = "284b597b3e88c635e3069b82926aa16812238492" +} + +variable "helm_repo" { + description = "Set the URL after http_server_repo_main_dir pointing to the Python helm modules" + default = "https://get.helm.sh/helm-v3.6.3-linux-ppc64le.tar.gz" } variable "ansible_extra_options" { @@ -329,10 +346,26 @@ variable "cluster_id" { default = "" } +variable "fips_compliant" { + type = bool + description = "Set to true to enable usage of FIPS for OCP deployment." + default = false +} + variable "dns_forwarders" { default = "8.8.8.8; 8.8.4.4" } +variable "lb_ipaddr" { + description = "Define the preconfigured external Load Balancer" + default = "" +} + +variable "ext_dns" { + description = "Define the preconfigured external DNS and Load Balancer" + default = "" +} + variable "mount_etcd_ramdisk" { description = "Whether mount etcd directory in the ramdisk (Only for dev/test) on low performance disk" default = false @@ -401,6 +434,21 @@ variable "cni_network_provider" { default = "OpenshiftSDN" } +variable "cluster_network_cidr" { + description = "blocks of IP addresses from which pod IP addresses are allocated." + default = "10.128.0.0/14" +} + +variable "cluster_network_hostprefix" { + description = "The subnet prefix length to assign to each individual node." + default = "23" +} + +variable "service_network" { + description = "blocks of IP addresses from which service addresses are allocated." + default = "172.30.0.0/16" +} + ################################################################ # Local registry variables ( used only in disconnected install ) ################################################################ @@ -418,4 +466,4 @@ variable "local_registry_image" { variable "ocp_release_tag" { description = "The version of OpenShift you want to sync." default = "4.4.9-ppc64le" -} +} \ No newline at end of file diff --git a/versions.tf b/versions.tf index 7016f11ab..47460246b 100644 --- a/versions.tf +++ b/versions.tf @@ -29,5 +29,5 @@ terraform { version = "~> 2.3" } } - required_version = "~> 0.13.0" + required_version = ">= 0.13.0" }