WIP Initial support for Bottlerocket OS

cmd/kops/integration_test.go

@@ -71,18 +71,20 @@ type integrationTest struct {
 	sshKey                           bool
 	bastionUserData                  bool
 	ciliumEtcd                       bool
+	nodeupConfigNodes                bool
 	// nth is true if we should check for files created by nth queue processor add on
 	nth bool
 }
 
 func newIntegrationTest(clusterName, srcDir string) *integrationTest {
 	return &integrationTest{
-		clusterName:    clusterName,
-		srcDir:         srcDir,
-		version:        "v1alpha2",
-		zones:          1,
-		expectPolicies: true,
-		sshKey:         true,
+		clusterName:       clusterName,
+		srcDir:            srcDir,
+		version:           "v1alpha2",
+		zones:             1,
+		expectPolicies:    true,
+		sshKey:            true,
+		nodeupConfigNodes: true,
 	}
 }
 
@@ -106,6 +108,11 @@ func (i *integrationTest) withoutPolicies() *integrationTest {
 	return i
 }
 
+func (i *integrationTest) withoutNodeupConfigNodes() *integrationTest {
+	i.nodeupConfigNodes = false
+	return i
+}
+
 func (i *integrationTest) withLifecycleOverrides(lco []string) *integrationTest {
 	i.lifecycleOverrides = lco
 	return i
@@ -253,6 +260,18 @@ func TestCompress(t *testing.T) {
 		runTestTerraformAWS(t)
 }
 
+// TestBottlerocket runs the test with nodes using a bottlerocket OS
+func TestBottlerocket(t *testing.T) {
+	featureflag.ParseFlags("+Bottlerocket")
+	unsetFeaureFlag := func() {
+		featureflag.ParseFlags("-Bottlerocket")
+	}
+	defer unsetFeaureFlag()
+	newIntegrationTest("bottlerocket.example.com", "bottlerocket").
+		withoutNodeupConfigNodes().
+		withAddons("authentication.aws-k8s-1.12", "csr-approver.addons.k8s.io-k8s-1.23", awsEBSCSIAddon, dnsControllerAddon, ciliumAddon).runTestTerraformAWS(t)
+}
+
 // TestExternalPolicies tests external policies output
 func TestExternalPolicies(t *testing.T) {
 	newIntegrationTest("externalpolicies.example.com", "externalpolicies").
@@ -1022,7 +1041,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 		"aws_s3_bucket_object_manifests-etcdmanager-events_content",
 		"aws_s3_bucket_object_manifests-etcdmanager-main_content",
 		"aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content",
-		"aws_s3_bucket_object_nodeupconfig-nodes_content",
 		"aws_s3_bucket_object_"+i.clusterName+"-addons-bootstrap_content",
 		"aws_s3_bucket_object_"+i.clusterName+"-addons-core.addons.k8s.io_content",
 		"aws_s3_bucket_object_"+i.clusterName+"-addons-kops-controller.addons.k8s.io-k8s-1.16_content",
@@ -1036,6 +1054,10 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 		expectedFilenames = append(expectedFilenames, "aws_s3_bucket_object_"+i.clusterName+"-addons-coredns.addons.k8s.io-k8s-1.12_content")
 	}
 
+	if i.nodeupConfigNodes {
+		expectedFilenames = append(expectedFilenames, "aws_s3_bucket_object_nodeupconfig-nodes_content")
+	}
+
 	if i.discovery {
 		expectedFilenames = append(expectedFilenames,
 			"aws_s3_bucket_object_discovery.json_content",

docs/operations/images.md

@@ -36,6 +36,7 @@ The following table provides the support status for various distros with regards
 | Distro | Experimental | Stable | Deprecated | Removed | 
 | ------------ | -----------: | -----: | ---------: | ------: |
 | [Amazon Linux 2](#amazon-linux-2) | 1.10 | 1.18 | - | - |
+| [Bottlerocket](#bottlerocket) | 1.23 | - | - | - |
 | [CentOS 7](#centos-7) | - | 1.5 | 1.21 | - |
 | [CentOS 8](#centos-8) | 1.15 | - | 1.21 | - |
 | CoreOS | 1.6 | 1.9 | 1.17 | 1.18 |
@@ -68,6 +69,37 @@ aws ec2 describe-images --region us-east-1 --output table \
   --filters "Name=name,Values=amzn2-ami-kernel-5.10-hvm-2*-x86_64-gp2"
 ```
 
+### Bottlerocket
+
+[Bottlerocket](https://github.com/bottlerocket-os/bottlerocket) is a container-native linux distribution created by AWS.
+
+Support for Bottlerocket is still experimental and requires a featureflag: `export KOPS_FEATURE_FLAGS=Bottlerocket`.
+
+Be aware of the following limitations:
+* Only AWS is supported
+* Only certain regions have Bottlerocket AMIs (see the containers-roadmap issue [here](https://github.com/aws/containers-roadmap/issues/827#issuecomment-675176968))
+* Bottlerocket AMIs are specific to K8s versions and may not have versions that Kops otherwise supports
+* Only the node InstanceGroupRole is supported; Master, APIServer, and Bastion are not
+* Many node customizations are not supported such as `additionalUserdata` and will fail API validation
+* Gossip clusters are not supported
+* Certain networking configurations are not supported such as Kubenet, Kuberouter, and Cilium in EtcdManaged mode
+* Clusters must have the [AWS IAM Authenticator](../authentication.md#aws-iam-authenticator) enabled with version >= 0.5.1 and the CRD Backend mode enabled
+
+To use Bottlerocket, find the available image (note that the Owner IDs vary per region)
+```bash
+aws ec2 describe-images --output table \
+  --owners amazon \
+  --query "reverse(sort_by(Images, &CreationDate))[*].[CreationDate,OwnerId,Name,ImageId]" \
+  --filters Name=name,Values="bottlerocket-aws-k8s*"
+```
+
+Then use the AMI ID in the image field and add the following block to the InstanceGroupSpec:
+```yaml
+spec:
+  imageFamily:
+    bottlerocket: {}
+```
+
 ### Debian 10 (Buster)
 
 Debian 10 is based on Kernel version **4.19** which fixes some of the bugs present in Debian 9 and effects are less visible.

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -229,6 +229,15 @@ spec:
               image:
                 description: Image is the instance (ami etc) we should use
                 type: string
+              imageFamily:
+                description: ImageFamily defines the Image's family. Only required
+                  for certain images - see allowed values.
+                properties:
+                  bottlerocket:
+                    description: Bottlerocket indicates the instances use AWS Bottlerocket
+                      OS
+                    type: object
+                type: object
               instanceInterruptionBehavior:
                 description: InstanceInterruptionBehavior defines if a spot instance
                   should be terminated, hibernated, or stopped after interruption

pkg/apis/kops/instancegroup.go

@@ -182,6 +182,8 @@ type InstanceGroupSpec struct {
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 	// WarmPool specifies a pool of pre-warmed instances for later use (AWS only).
 	WarmPool *WarmPoolSpec `json:"warmPool,omitempty"`
+	// ImageFamily defines the Image's family. Only required for certain images - see allowed values.
+	ImageFamily *ImageFamily `json:"imageFamily,omitempty"`
 }
 
 const (
@@ -289,6 +291,15 @@ type IAMProfileSpec struct {
 	Profile *string `json:"profile,omitempty"`
 }
 
+// ImageFamily specifies OS settings that need to be known before launchtime
+type ImageFamily struct {
+	Bottlerocket *Bottlerocket `json:"bottlerocket,omitempty"`
+}
+
+// Bottlerocket indicates the instances use AWS Bottlerocket OS
+type Bottlerocket struct {
+}
+
 // IsMaster checks if instanceGroup is a master
 func (g *InstanceGroup) IsMaster() bool {
 	switch g.Spec.Role {

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -151,6 +151,8 @@ type InstanceGroupSpec struct {
 	UpdatePolicy *string `json:"updatePolicy,omitempty"`
 	// WarmPool configures an ASG warm pool for the instance group
 	WarmPool *WarmPoolSpec `json:"warmPool,omitempty"`
+	// ImageFamily defines the Image's family. Only required for certain images - see allowed values.
+	ImageFamily *ImageFamily `json:"imageFamily,omitempty"`
 }
 
 // InstanceMetadataOptions defines the EC2 instance metadata service options (AWS Only)
@@ -246,3 +248,12 @@ type LoadBalancer struct {
 	// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
 	TargetGroupARN *string `json:"targetGroupArn,omitempty"`
 }
+
+// ImageFamily specifies OS settings that need to be known before launchtime
+type ImageFamily struct {
+	Bottlerocket *Bottlerocket `json:"bottlerocket,omitempty"`
+}
+
+// Bottlerocket indicates the instances use AWS Bottlerocket OS
+type Bottlerocket struct {
+}