Fix kube-router config generation (kubernetes-sigs#5531)

Signed-off-by: Arthur Outhenin-Chalandre <[email protected]>
SndR85 · Mar 10, 2020 · 6b89d63 · 6b89d63
1 parent 47991f7
commit 6b89d63
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 76 deletions.
diff --git a/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml b/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml
@@ -8,8 +8,8 @@
     resource: "ds"
     namespace: "kube-system"
     state: "latest"
-  when:
-    - inventory_hostname == groups['kube-master'][0]
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  run_once: true
 
 - name: kube-router | Wait for kube-router pods to be ready
   command: "{{ bin_dir }}/kubectl -n kube-system get pods -l k8s-app=kube-router -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"   # noqa 601
@@ -18,5 +18,6 @@
   retries: 30
   delay: 10
   ignore_errors: yes
-  when:
-    - inventory_hostname == groups['kube-master'][0]
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  run_once: true
+  changed_when: false
diff --git a/roles/network_plugin/kube-router/handlers/main.yml b/roles/network_plugin/kube-router/handlers/main.yml
@@ -0,0 +1,20 @@
+---
+- name: reset_kube_router
+  command: /bin/true
+  notify:
+    - docker | delete kube-router containers
+    - containerd | delete kube-router containers
+
+- name: docker | delete kube-router containers
+  shell: "docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f"
+  register: docker_kube_router_remove
+  until: docker_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["docker"]
+
+- name: containerd | delete kube-router containers
+  shell: 'crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "crictl stopp % && crictl rmp %"'
+  register: crictl_kube_router_remove
+  until: crictl_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["crio", "containerd"]
diff --git a/roles/network_plugin/kube-router/tasks/main.yml b/roles/network_plugin/kube-router/tasks/main.yml
@@ -19,7 +19,63 @@
     owner: kube
     remote_src: yes
 
+- name: kube-router | Create config directory
+  file:
+    path: /var/lib/kube-router
+    state: directory
+    owner: kube
+    recurse: true
+    mode: 0755
+
+- name: kube-router | Create kubeconfig
+  template:
+    src: kubeconfig.yml.j2
+    dest: /var/lib/kube-router/kubeconfig
+    owner: kube
+  notify:
+    - reset_kube_router
+
+- name: kube-router | Slurp cni config
+  slurp:
+    src: /etc/cni/net.d/10-kuberouter.conf
+  register: cni_config_slurp
+  ignore_errors: true
+
+- name: kube-router | Set cni_config variable
+  set_fact:
+    cni_config: "{{ cni_config_slurp.content | b64decode | from_json }}"
+  when:
+    - not cni_config_slurp.failed
+
+- name: kube-router | Set host_subnet variable
+  set_fact:
+    host_subnet: "{{ cni_config.ipam.subnet }}"
+  when:
+    - cni_config is defined
+    - cni_config.ipam is defined
+    - cni_config.ipam.subnet is defined
+
+- name: kube-router | Set wanted cni config variable
+  set_fact:
+    wanted_cni_config: "{{ lookup('template', 'cni-conf.json.j2') }}"
+
+- name: kube-router | Set wanted_cni_config variable
+  set_fact:
+    wanted_cni_config: "{{ wanted_cni_config | combine({ 'ipam': { 'subnet': host_subnet }}, recursive=True) }}"
+  when: host_subnet is defined
+
+- name: kube-router | Create cni config
+  copy:
+    content: "{{ wanted_cni_config | to_nice_json }}"
+    dest: /etc/cni/net.d/10-kuberouter.conf
+    owner: kube
+  changed_when: wanted_cni_config != cni_config
+  notify:
+    - reset_kube_router
+
 - name: kube-router | Create manifest
   template:
     src: kube-router.yml.j2
     dest: "{{ kube_config_dir }}/kube-router.yml"
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  run_once: true
diff --git a/roles/network_plugin/kube-router/templates/cni-conf.json.j2 b/roles/network_plugin/kube-router/templates/cni-conf.json.j2
@@ -0,0 +1,13 @@
+{
+  "name":"kubernetes",
+  "cniVersion": "0.2.0",
+  "type":"bridge",
+  "bridge":"kube-bridge",
+  "isDefaultGateway":true,
+{% if kube_router_support_hairpin_mode %}
+  "hairpinMode":true,
+{% endif %}
+  "ipam": {
+    "type":"host-local"
+  }
+}
diff --git a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/roles/network_plugin/kube-router/templates/kube-router.yml.j2
@@ -1,47 +1,3 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: kube-router
-data:
-  cni-conf.json: |
-    {
-      "name":"kubernetes",
-      "cniVersion": "0.2.0",
-      "type":"bridge",
-      "bridge":"kube-bridge",
-      "isDefaultGateway":true,
-{% if kube_router_support_hairpin_mode %}
-      "hairpinMode":true,
-{% endif %}
-      "ipam": {
-        "type":"host-local"
-      }
-    }
-  kubeconfig: |
-    apiVersion: v1
-    kind: Config
-    clusterCIDR: {{ kube_pods_subnet }}
-    clusters:
-    - name: cluster
-      cluster:
-        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        server: {{ kube_apiserver_endpoint }}
-    users:
-    - name: kube-router
-      user:
-        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-    contexts:
-    - context:
-        cluster: cluster
-        user: kube-router
-      name: kube-router-context
-    current-context: kube-router-context
-
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -146,31 +102,6 @@ spec:
           name: metrics
           protocol: TCP
 {% endif %}
-      initContainers:
-      - name: install-cni
-        image: {{ busybox_image_repo }}:{{ busybox_image_tag }}
-        imagePullPolicy: IfNotPresent
-        command:
-        - /bin/sh
-        - -c
-        - set -e -x;
-          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
-            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
-            cp /etc/kube-router/cni-conf.json ${TMP};
-            mv ${TMP} /etc/cni/net.d/10-kuberouter.conf;
-          fi;
-          if [ ! -f /var/lib/kube-router/kubeconfig ]; then
-            TMP=/var/lib/kube-router/.tmp-kubeconfig;
-            cp /etc/kube-router/kubeconfig ${TMP};
-            mv ${TMP} /var/lib/kube-router/kubeconfig;
-          fi
-        volumeMounts:
-        - mountPath: /etc/cni/net.d
-          name: cni-conf-dir
-        - mountPath: /etc/kube-router
-          name: kube-router-cfg
-        - name: kubeconfig
-          mountPath: /var/lib/kube-router
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
 {% if kube_router_enable_dsr %}
@@ -195,9 +126,6 @@ spec:
       - name: cni-conf-dir
         hostPath:
           path: /etc/cni/net.d
-      - name: kube-router-cfg
-        configMap:
-          name: kube-router-cfg
       - name: kubeconfig
         hostPath:
           path: /var/lib/kube-router

diff --git a/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 b/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusterCIDR: {{ kube_pods_subnet }}
+clusters:
+- name: cluster
+  cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-router
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+contexts:
+- context:
+    cluster: cluster
+    user: kube-router
+  name: kube-router-context
+current-context: kube-router-context