diff --git a/charts/opencost-config/templates/opencost/dashboards.yaml b/charts/opencost-config/templates/opencost/dashboards.yaml index c91e911..653e4df 100644 --- a/charts/opencost-config/templates/opencost/dashboards.yaml +++ b/charts/opencost-config/templates/opencost/dashboards.yaml @@ -1,3 +1,4 @@ +{{ if and .Values.dashboards }} {{- range $path, $_ := .Files.Glob "dashboards/**.json" }} --- apiVersion: v1 @@ -11,3 +12,4 @@ data: {{ base $path }}: |- {{ ($.Files.Get $path) | nindent 6 }} {{ end }} +{{ end }} diff --git a/installation/Helmfile_eks.yaml b/installation/Helmfile_eks.yaml new file mode 100644 index 0000000..eae5dd1 --- /dev/null +++ b/installation/Helmfile_eks.yaml @@ -0,0 +1,139 @@ +repositories: +- name: kyverno + url: https://kyverno.github.io/kyverno/ +- name: prometheus-opencost-exporter + url: https://prometheus-community.github.io/helm-charts +- name: prometheus + url: https://prometheus-community.github.io/helm-charts +- name: grafana + url: https://grafana.github.io/helm-charts +- name: vpa + url: https://charts.fairwinds.com/stable +- name: cert-manager + url: https://charts.jetstack.io +- name: fairwinds-stable + url: https://charts.fairwinds.com/stable + +--- +helmDefaults: + wait: true + timeout: 1200 +--- +environments: + default: + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/enabled.yaml" +--- +releases: +- name: kyverno + version: "3.2.6" + chart: kyverno/kyverno + condition: kyverno.enabled + namespace: finops-stack + values: + - "./config/common/kyverno-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/kyverno-values.yaml" + +- name: finops-policies + version: "0.1.0" + chart: "../charts/finops-policies" + condition: finops-policies.enabled + namespace: finops-stack + disableValidationOnInstall: true + needs: + - kyverno + +- name: cert-manager + version: v1.15.3 + chart: cert-manager/cert-manager + condition: cert-manager.enabled + namespace: cert-manager + values: + - "./config/common/cert-manager-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/cert-manager-values.yaml" + - global: + leaderElection: + namespace: cert-manager + commonLabels: + cost-center-label: "xyz" + - serviceAccount: + annotations: + {{ env "CERT_MANAGER_SA_ANNOTATION" }} + +- name: cert-manager-config + version: "0.1.0" + chart: "../charts/cert-manager-config" + condition: cert-manager.enabled + namespace: finops-stack + disableValidationOnInstall: true + needs: + - cert-manager/cert-manager + values: + - email: {{ env "CERT_MANAGER_EMAIL" }} + - grafanaTLSCert: + hostname: {{ env "GRAFANA_FQDN" }} + +- name: grafana + version: "8.4.7" + chart: grafana/grafana + condition: grafana.enabled + namespace: finops-stack + values: + - "./config/common/grafana-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/grafana-values.yaml" + - adminUser: {{ env "GRAFANA_ADMIN_USER" }} + - adminPassword: {{ env "GRAFANA_ADMIN_PW" }} + - serviceAccount: + annotations: + {{ env "GRAFANA_SA_ANNOTATION" }} + + # If you don't require ingress via an FQDN remove this ingress section + - ingress: + enabled: {{ env "GRAFANA_INGRESS" }} + annotations: + kubernetes.io/ingress.global-static-ip-name: {{ env "GRAFANA_PUBLIC_IP_NAME" }} + hosts: + - {{ env "GRAFANA_FQDN" }} + # If you don't require https access to the Grafana dashboard remove this tls section + tls: + - secretName: {{ env "GRAFANA_FQDN" }}-tls + hosts: + - {{ env "GRAFANA_FQDN" }} + needs: + - cert-manager/cert-manager + +- name: vpa + version: "4.5.0" + chart: "vpa/vpa" + condition: vpa.enabled + namespace: finops-stack + values: + - "./config/common/vpa-values.yaml" + +- name: prometheus-opencost-exporter + version: "0.1.1" + chart: "prometheus-opencost-exporter/prometheus-opencost-exporter" + condition: opencost-exporter.enabled + namespace: finops-stack + values: + - "./config/common/prometheus-opencost-exporter-values.yaml" + - "./config/{{ requiredEnv "HOST_ENV" }}/prometheus-opencost-exporter-values.yaml" + +- name: finops-stack-opencost-templates + version: "0.1.0" + chart: "../charts/opencost-config" + condition: opencost-templates.enabled + namespace: finops-stack + disableValidationOnInstall: true + values: + - "./config/{{ requiredEnv "HOST_ENV" }}/opencost-templates-values.yaml" + +- name: goldilocks + version: "9.0.0" + chart: fairwinds-stable/goldilocks + condition: goldilocks.enabled + namespace: finops-stack + values: + - "./config/common/goldilocks-values.yaml" + + diff --git a/installation/Makefile b/installation/Makefile index d1ac14a..f20be81 100644 --- a/installation/Makefile +++ b/installation/Makefile @@ -1,10 +1,5 @@ .DEFAULT_GOAL := help -CLUSTER_NAME := ambient -PROJECT_ID := "$(shell gcloud config get-value project)" -M_TYPE := n1-standard-2 -ZONE := europe-west2-a - cluster: ## Setup cluster kind create cluster diff --git a/installation/config/eks/enabled.yaml b/installation/config/eks/enabled.yaml new file mode 100644 index 0000000..5a707b4 --- /dev/null +++ b/installation/config/eks/enabled.yaml @@ -0,0 +1,29 @@ +# -- Enable / Disable the installation of Kyverno +kyverno: + enabled: true + +# -- Enable / Disable the installation of Kyverno FinOps Policies +finops-policies: + enabled: true +# -- Enable / Disable the installation of the VPA Controller +vpa: + enabled: true +# -- Enable / Disable the installation of the Grafana +grafana: + enabled: true +# -- Enable / Disable the installation of cert-manager +cert-manager: + enabled: false +# -- Enable / Disable the installation of OpenCost +opencost: + enabled: false +# -- Enable / Disable the installation of OpenCost Exporter +opencost-exporter: + enabled: true + +opencost-templates: + enabled: true + +# -- Enable / Disable the installation of Fairwinds' Goldilocks +goldilocks: + enabled: true diff --git a/installation/config/eks/grafana-values.yaml b/installation/config/eks/grafana-values.yaml new file mode 100644 index 0000000..781dc41 --- /dev/null +++ b/installation/config/eks/grafana-values.yaml @@ -0,0 +1,16 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + uid: "PFB5ABA51A8A585D7" + url: http://prometheus-server + isDefault: true + editable: true +ingress: + enabled: false + annotations: + kubernetes.io/ingress.global-static-ip-name: "ip-name" + hosts: + - grafana.example.com diff --git a/installation/config/eks/kyverno-values.yaml b/installation/config/eks/kyverno-values.yaml new file mode 100644 index 0000000..2d4980c --- /dev/null +++ b/installation/config/eks/kyverno-values.yaml @@ -0,0 +1,13 @@ +features: + backgroundScan: + skipResourceFilters: false # Important so that background Scan's Exclude Namespaces, etc +config: + resourceFiltersIncludeNamespaces: + - kube-system + webhooks: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system diff --git a/installation/config/eks/opencost-templates-values.yaml b/installation/config/eks/opencost-templates-values.yaml new file mode 100644 index 0000000..3a6f68b --- /dev/null +++ b/installation/config/eks/opencost-templates-values.yaml @@ -0,0 +1,5 @@ +dashboards: true +opencost: + customPricing: + # -- This is only to enable custom pricing on on-premises or kind + enabled: false diff --git a/installation/config/eks/opencost-values.yaml b/installation/config/eks/opencost-values.yaml new file mode 100644 index 0000000..bc6dbdf --- /dev/null +++ b/installation/config/eks/opencost-values.yaml @@ -0,0 +1,18 @@ +cloudCost: + enabled: false +opencost: + cloudCost: + enabled: false + prometheus: + external: + enabled: false + # url: http://prometheus-server + internal: + # -- Use in-cluster Prometheus + enabled: true + # -- Service name of in-cluster Prometheus + serviceName: prometheus-server + # -- Namespace of in-cluster Prometheus + namespaceName: finops-stack + # -- Service port of in-cluster Prometheus + port: 80 diff --git a/installation/config/eks/prometheus-opencost-exporter-values.yaml b/installation/config/eks/prometheus-opencost-exporter-values.yaml new file mode 100644 index 0000000..de70cd1 --- /dev/null +++ b/installation/config/eks/prometheus-opencost-exporter-values.yaml @@ -0,0 +1,27 @@ +service: + enabled: true +opencost: + prometheus: + external: + enabled: false + # url: http://prometheus-server + internal: + # -- Use in-cluster Prometheus + enabled: true + # -- Service name of in-cluster Prometheus + serviceName: prometheus-server + # -- Namespace of in-cluster Prometheus + namespaceName: finops-stack + # -- Service port of in-cluster Prometheus + port: 80 + exporter: + persistence: + enabled: false + # -- Annotations for persistent volume + annotations: {} + # -- Access mode for persistent volume + accessMode: "" + # -- Storage class for persistent volume + storageClass: "standard-rwo" + # -- Size for persistent volume + size: "10g" diff --git a/installation/config/eks/prometheus-values.yaml b/installation/config/eks/prometheus-values.yaml new file mode 100644 index 0000000..8c0a0b1 --- /dev/null +++ b/installation/config/eks/prometheus-values.yaml @@ -0,0 +1,17 @@ +extraScrapeConfigs: | + - job_name: opencost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - opencost.opencost + type: 'A' + port: 9003 + +alertmanager: + enabled: false +prometheus-pushgateway: + enabled: false diff --git a/installation/config/gke/enabled.yaml b/installation/config/gke/enabled.yaml new file mode 100644 index 0000000..81f356a --- /dev/null +++ b/installation/config/gke/enabled.yaml @@ -0,0 +1,33 @@ +# -- Enable / Disable the installation of Kyverno +kyverno: + enabled: true + +# -- Enable / Disable the installation of Kyverno FinOps Policies +finops-policies: + enabled: true +# -- Enable / Disable the installation of the VPA Controller +vpa: + enabled: true +# -- Enable / Disable the installation of the Grafana +grafana: + enabled: true +# -- Enable / Disable the installation of cert-manager +cert-manager: + enabled: false +# -- Enable / Disable the installation of OpenCost +opencost: + enabled: false +# -- Enable / Disable the installation of OpenCost Exporter +opencost-exporter: + enabled: true + +opencost-templates: + enabled: true + +# -- Enable / Disable the installation of GMP Proxy (required when using Google Managed Prometheus) +gmp-proxy: + enabled: true + +# -- Enable / Disable the installation of Fairwinds' Goldilocks +goldilocks: + enabled: true diff --git a/installation/config/gke/opencost-templates-values.yaml b/installation/config/gke/opencost-templates-values.yaml index 92397d3..417b1e2 100644 --- a/installation/config/gke/opencost-templates-values.yaml +++ b/installation/config/gke/opencost-templates-values.yaml @@ -1,2 +1,7 @@ +dashboards: true +opencost: + customPricing: + # -- This is only to enable custom pricing on on-premises or kind + enabled: false gke: - podmonitoring: true \ No newline at end of file + podmonitoring: true diff --git a/installation/config/kind/opencost-templates-values.yaml b/installation/config/kind/opencost-templates-values.yaml index 93d7d52..c9efd85 100644 --- a/installation/config/kind/opencost-templates-values.yaml +++ b/installation/config/kind/opencost-templates-values.yaml @@ -1,3 +1,4 @@ +dashboards: true opencost: configFileName: default provider: kind diff --git a/installation/eks.md b/installation/eks.md new file mode 100644 index 0000000..d8f3e23 --- /dev/null +++ b/installation/eks.md @@ -0,0 +1,79 @@ +# Installation using Helmfile + +Installing Helm charts with lots of dependencies and CRDs is challenging; these instructions use Helmfile to mitigate issues with Helm. + +This documentation focuses on installing the FinOps Stack in EKS clusters. + +## Pre-requisites + +- A EKS cluster with: + - kubectl access + - (Optional) If your cluster has Spot Instances, EKS Pod Identities need to be configured. See [documentation](https://www.opencost.io/docs/configuration/aws#eks-pod-identities). +- [Helmfile](https://helmfile.readthedocs.io/en/latest/#installation) installed on your local machine +- Unless you want to access the Grafana dashboard via `kubectl port-forward` you'll need a domain name or external public IP. + +## Installation + +### Configuration changes for your cluster environment + +1. To control which Finops Stack components to install, edit the [enabled.yaml](./installation/config/common/enabled.yaml) file +1. Copy `./env_eks.tmpl` to `./.env` and replace the env var values accordingly. + +### Install everything using Helmfile + +For the first run: + +```bash +set -a; source .env; set +a; helmfile apply --file Helmfile_eks.yaml --interactive +``` + +NOTE: it will take several minutes for all workloads to install and start running. Helmfile does display its progress in the terminal. All workloads get installed into the `finops-stack` namespace so you can also view progress using `kubectl`. + +To speed up subsequent runs: + +```bash +set -a; source .env; set +a; helmfile apply --file Helmfile_eks.yaml --interactive --skip-deps +``` + +## Optional: Configure ingress for Grafana + +### Pre-requisites + +Already have an FQDN setup and registered with a public IP, e.g. grafana.example.com + +### Grafana Helm values + +These are specified in `config/common/grafana-values.yaml`, `config/gke/grafana-values.yaml` and under the Grafana release in `helmfile.yaml`. Probably all the changes you will want to make can be done by changing the values in `helmfile.yaml`, e.g. the admin user and what type of ingress you require. + +General guidance when configuring ingress: +- Update the `.env` file with the FQDN and public IP for you domain. +- If you wish to enable tls, then ensure that cert-manager.enabled is set to true and update the values in `.env` accordingly. + +## Enable Goldilocks for namespaces + +For Goldilocks to analyse namespaces and add then to its dashboard you need to add this label to the namespace resource: `goldilocks.fairwinds.com/enabled=true`, e.g. +`kubectl label ns finops-stack goldilocks.fairwinds.com/enabled=true` + +## Useful commands + +To port forward to Grafana: + +```bash +kubectl --namespace finops-stack port-forward service/grafana 3000:80 +``` + +Access via http://localhost:3000 + +To port forward to the metrics endpoint of the Opencost Prometheus exporter (to examine what metrics are being scraped): + +```bash +kubectl --namespace finops-stack port-forward service/prometheus-opencost-exporter 9003:9003 +``` + +To access the Goldilocks dashboard (assuming you've enabled it): + +```bash +kubectl -n finops-stack port-forward svc/goldilocks-dashboard 8080:80 +``` + +Then goto http://localhost:8080 \ No newline at end of file diff --git a/installation/env_eks.tmpl b/installation/env_eks.tmpl new file mode 100644 index 0000000..5a8b849 --- /dev/null +++ b/installation/env_eks.tmpl @@ -0,0 +1,19 @@ +# Sub-dir under ./config that holds your hosting env specific Helm values.files, e.g. gke +HOST_ENV=eks + +# -- AWS secret access key and access key id +AWS_SECRET_ACCESS_KEY="" +AWS_ACCESS_KEY_ID="" + +# Grafana admin user credentials +GRAFANA_ADMIN_USER=finops +GRAFANA_ADMIN_PW=s7@ck + +# Grafana ingress settings +GRAFANA_INGRESS="false" +# GRAFANA_PUBLIC_IP_NAME="name-of-public-ip" +# GRAFANA_FQDN="grafana.host.name" + +## GCP SA for workload identity for cert-manager (need to be defined but only used if cert-manager is being installed) +CERT_MANAGER_SA_ANNOTATION="" +CERT_MANAGER_EMAIL="issuer@example.com" diff --git a/installation/gke.md b/installation/gke.md index 710cf35..97f22f4 100644 --- a/installation/gke.md +++ b/installation/gke.md @@ -27,14 +27,14 @@ This documentation focuses on installing the FinOps Stack in GKE standard/autopi ### Configuration changes for your cluster environment 1. To control which Finops Stack components to install, edit the [enabled.yaml](./installation/config/common/enabled.yaml) file -1. Copy `./env.tmpl` to `./.env` and replace the env var values accordingly. As a minimum, you will need to change the `GCP_PROJECT`, `CSP_API_KEY`, `GRAFANA_SA_ANNOTATION` values. +1. Copy `./env_gke.tmpl` to `./.env` and replace the env var values accordingly. As a minimum, you will need to change the `GCP_PROJECT`, `CSP_API_KEY`, `GRAFANA_SA_ANNOTATION` values. ### Install everything using Helmfile For the first run: ```bash -set -a; source .env; set +a; helmfile apply --interactive +set -a; source .env; set +a; helmfile apply --file Helmfile_gke.yaml --interactive ``` NOTE: it will take several minutes for all workloads to install and start running. Helmfile does display its progress in the terminal. All workloads get installed into the `finops-stack` namespace so you can also view progress using `kubectl`.