From 3b9b97d33a641e9a97ba34542dc2c141b608ae93 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:18:59 -0700 Subject: [PATCH 01/11] feat: comment out standalone grafana in start/stop scripts feat: update test-forwards utility script for prometheus operator use --- pulumi/aws/destroy.sh | 2 +- pulumi/aws/sirius/src | 2 +- pulumi/aws/start_all.sh | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pulumi/aws/destroy.sh b/pulumi/aws/destroy.sh index 7f2c8629..82587b83 100755 --- a/pulumi/aws/destroy.sh +++ b/pulumi/aws/destroy.sh @@ -91,7 +91,7 @@ if command -v aws > /dev/null; then validate_aws_credentials fi -k8s_projects=(sirius observability grafana prometheus certmgr logagent logstore kic-helm-chart) +k8s_projects=(sirius observability prometheus certmgr logagent logstore kic-helm-chart) # Test to see if EKS has been destroy AND there are still Kubernetes resources # that are being managed by Pulumi. If so, we have to destroy the stack for diff --git a/pulumi/aws/sirius/src b/pulumi/aws/sirius/src index c6966910..96e7df33 160000 --- a/pulumi/aws/sirius/src +++ b/pulumi/aws/sirius/src @@ -1 +1 @@ -Subproject commit c696691064f093d53b30e6b5e9b7940fce5517f7 +Subproject commit 96e7df33a7808079bd8ccd61e0c8ad99eeef8f0f diff --git a/pulumi/aws/start_all.sh b/pulumi/aws/start_all.sh index 5905afef..eaaa80cb 100755 --- a/pulumi/aws/start_all.sh +++ b/pulumi/aws/start_all.sh @@ -182,6 +182,10 @@ fi # Admin password for grafana (see note in __main__.py in grafana project as to why not encrypted) # We run in the vpc project directory because we need the pulumi yaml to point us to the correct # configuration. +# +# This same password will be used for the Grafana deployment that is stood up as part of +# the prometheus operator driven prometheus-kube-stack. +# if pulumi config get grafana:adminpass -C ${script_dir}/vpc >/dev/null 2>&1; then echo "Password found for grafana admin account" else @@ -313,9 +317,15 @@ header "Prometheus" cd "${script_dir}/prometheus" pulumi $pulumi_args up -header "Grafana" -cd "${script_dir}/grafana" -pulumi $pulumi_args up +# This is commented out, as the new prometheus deployment provides this +# functionality. +# +# If you wish to use this intead, you will need to adjust the helm manifest +# values in the __main__.py file in the prometheus directory. +# +#header "Grafana" +#cd "${script_dir}/grafana" +#pulumi $pulumi_args up header "Observability" cd "${script_dir}/observability" From 269efa1ac9ffe953a0869420b408593bc02ee0ca Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:20:47 -0700 Subject: [PATCH 02/11] feat: convert prometheus to kube-prometheus-stack --- pulumi/aws/prometheus/__main__.py | 100 +++++++++++++++++- .../manifests/nginx-service-mon.yaml | 21 ++++ 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 pulumi/aws/prometheus/manifests/nginx-service-mon.yaml diff --git a/pulumi/aws/prometheus/__main__.py b/pulumi/aws/prometheus/__main__.py index 0459ff60..4b372e63 100644 --- a/pulumi/aws/prometheus/__main__.py +++ b/pulumi/aws/prometheus/__main__.py @@ -4,6 +4,8 @@ import pulumi_kubernetes as k8s from pulumi_kubernetes.helm.v3 import Release, ReleaseArgs, RepositoryOptsArgs from pulumi import Output +from pulumi_kubernetes.yaml import ConfigFile +from pulumi_kubernetes.yaml import ConfigGroup from kic_util import pulumi_config @@ -14,6 +16,12 @@ def project_name_from_project_dir(dirname: str): return pulumi_config.get_pulumi_project_name(project_path) +def servicemon_manifests_location(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + servicemon_manifests_path = os.path.join(script_dir, 'manifests', '*.yaml') + return servicemon_manifests_path + + stack_name = pulumi.get_stack() project_name = pulumi.get_project() pulumi_user = pulumi_config.get_pulumi_user() @@ -30,10 +38,12 @@ def project_name_from_project_dir(dirname: str): metadata={'name': 'prometheus'}, opts=pulumi.ResourceOptions(provider=k8s_provider)) + + config = pulumi.Config('prometheus') chart_name = config.get('chart_name') if not chart_name: - chart_name = 'prometheus' + chart_name = 'kube-prometheus-stack' chart_version = config.get('chart_version') if not chart_version: chart_version = '14.6.0' @@ -43,6 +53,12 @@ def project_name_from_project_dir(dirname: str): helm_repo_url = config.get('prometheus_helm_repo_url') if not helm_repo_url: helm_repo_url = 'https://prometheus-community.github.io/helm-charts' + +grafana_config = pulumi.Config('grafana') +# Require an admin password, but do not encrypt it due to the +# issues we experienced with Anthos; this can be adjusted at the +# same time that we fix the Anthos issues. +adminpass = grafana_config.require('adminpass') prometheus_release_args = ReleaseArgs( chart=chart_name, @@ -54,6 +70,75 @@ def project_name_from_project_dir(dirname: str): # Values from Chart's parameters specified hierarchically, values={ + "prometheus": { + "serviceAccount": { + "create": True, + "name": "prometheus", + "annotations": {} + }, + "prometheusSpec": { + "podMonitorSelectorNilUsesHelmValues": False, + "serviceMonitorSelectorNilUsesHelmValues": False, + "serviceMonitorSelector": {}, + "serviceMonitorNamespaceSelector ": { + "matchLabels": { + "prometheus": True + } + }, + "storageSpec": { + "volumeClaimTemplate": { + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "5Gi" + } + } + } + } + } + } + }, + "grafana": { + "serviceAccount": { + "create": False, + "name": "prometheus", + "annotations": {} + }, + "adminPassword": adminpass, + "persistence": { + "enabled": True, + "accessModes": [ + "ReadWriteOnce" + ], + "size": "5Gi" + } + }, + "alertmanager": { + "serviceAccount": { + "create": False, + "name": "prometheus", + "annotations": {} + }, + "alertmanagerSpec": { + "storage": { + "volumeClaimTemplate": { + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "5Gi" + } + } + } + } + } + } + } }, # By default Release resource will wait till all created resources # are available. Set this to true to skip waiting on resources being @@ -71,11 +156,18 @@ def project_name_from_project_dir(dirname: str): prom_status = prometheus_release.status +servicemon_manifests = servicemon_manifests_location() + +servicemon = ConfigGroup( + 'servicemon', + files=[servicemon_manifests], + opts=pulumi.ResourceOptions(depends_on=[prometheus_release]) +) + # # Deploy the statsd collector # - config = pulumi.Config('prometheus') statsd_chart_name = config.get('statsd_chart_name') if not statsd_chart_name: @@ -100,6 +192,10 @@ def project_name_from_project_dir(dirname: str): # Values from Chart's parameters specified hierarchically, values={ + "serviceMonitor": { + "enabled": True, + "namespace": "prometheus" + }, "serviceAccount": { "create": True, "annotations": {}, diff --git a/pulumi/aws/prometheus/manifests/nginx-service-mon.yaml b/pulumi/aws/prometheus/manifests/nginx-service-mon.yaml new file mode 100644 index 00000000..80027ef8 --- /dev/null +++ b/pulumi/aws/prometheus/manifests/nginx-service-mon.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + # needs to match above matchLabels + prometheus: nginx-monitor + name: nginx-monitor + namespace: nginx-ingress +spec: + endpoints: + - interval: 15s + port: prometheus + scheme: http + jobLabel: app.kubernetes.io/name + selector: + matchLabels: + app: kic-nginx-ingress + namespaceSelector: + any: true + From 39ce4f5a9780872948a094ec5d5dc5733b77b34f Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:21:23 -0700 Subject: [PATCH 03/11] feat: Update utility script to use new services from prometheus operator --- pulumi/aws/extras/scripts/test-forward.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulumi/aws/extras/scripts/test-forward.sh b/pulumi/aws/extras/scripts/test-forward.sh index cae2552f..52985f8a 100755 --- a/pulumi/aws/extras/scripts/test-forward.sh +++ b/pulumi/aws/extras/scripts/test-forward.sh @@ -51,7 +51,7 @@ kubectl port-forward service/elastic-kibana --namespace logstore 5601:5601 & echo $! > $PID01 ## Grafana Tunnel -kubectl port-forward service/grafana --namespace grafana 3000:80 & +kubectl port-forward service/prometheus-grafana --namespace prometheus 3000:80 & echo $! > $PID02 ## Loadgenerator Tunnel @@ -59,7 +59,7 @@ kubectl port-forward service/loadgenerator --namespace bos 8089:8089 & echo $! > $PID03 ## Prometheus Tunnel -kubectl port-forward service/prometheus-server --namespace prometheus 9090:80 & +kubectl port-forward service/prometheus-kube-prometheus-prometheus --namespace prometheus 9090:9090 & echo $! > $PID04 ## Elasticsearch Tunnel From 178f8cfdd7b72fa3a08a19e4258b443b07badaa3 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:21:47 -0700 Subject: [PATCH 04/11] feat: add extras script to fix permissions on kube-proxy metrics --- pulumi/aws/prometheus/extras/README.md | 13 +++++ pulumi/aws/prometheus/extras/kube-proxy.yaml | 55 ++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 pulumi/aws/prometheus/extras/README.md create mode 100644 pulumi/aws/prometheus/extras/kube-proxy.yaml diff --git a/pulumi/aws/prometheus/extras/README.md b/pulumi/aws/prometheus/extras/README.md new file mode 100644 index 00000000..a485fc80 --- /dev/null +++ b/pulumi/aws/prometheus/extras/README.md @@ -0,0 +1,13 @@ +## Purpose +This directory contains a manifest that can be used to change the metrics bind port +for the kube-proxy from 127.0.0.1 to 0.0.0.0 in order to allow the metrics to be scraped +by the prometheus service. + +This is not being automatically applied, since it is changing the bind address that is +being used for the metrics port. That said, this should be secure since it's internal +to the installation and the connection is done via HTTPS. + +However, please see this +[github issue](https://github.com/prometheus-community/helm-charts/issues/977) +for the full discussion of why this is required. + diff --git a/pulumi/aws/prometheus/extras/kube-proxy.yaml b/pulumi/aws/prometheus/extras/kube-proxy.yaml new file mode 100644 index 00000000..100b8276 --- /dev/null +++ b/pulumi/aws/prometheus/extras/kube-proxy.yaml @@ -0,0 +1,55 @@ +--- +# This version of the kube proxy configuration is required to change the +# bind address for metrics from 127.0.0.1 to 0.0.0.0. See: +# https://github.com/prometheus-community/helm-charts/issues/977 +# for details as why this is required. +# +# Note this is a hack, and as such should be tested with any version +# changes to Kubernetes. You have been warned. +# +apiVersion: v1 +data: + config: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + bindAddress: 0.0.0.0 + clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig + qps: 5 + clusterCIDR: "" + configSyncPeriod: 15m0s + conntrack: + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s + enableProfiling: false + healthzBindAddress: 0.0.0.0:10256 + hostnameOverride: "" + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0:10249 + mode: "iptables" + nodePortAddresses: null + oomScoreAdj: -998 + portRange: "" + udpIdleTimeout: 250ms +kind: ConfigMap +metadata: + annotations: + labels: + eks.amazonaws.com/component: kube-proxy + k8s-app: kube-proxy + name: kube-proxy-config + namespace: kube-system From 63c4837c18aa2e5441f37e72e5feb244244b6e82 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:22:23 -0700 Subject: [PATCH 05/11] feat: modifications to NGINX IC to allow prometheus service monitor to pull metrics --- pulumi/aws/kic-helm-chart/__main__.py | 30 ++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/pulumi/aws/kic-helm-chart/__main__.py b/pulumi/aws/kic-helm-chart/__main__.py index 7eae40b6..2be3e46d 100644 --- a/pulumi/aws/kic-helm-chart/__main__.py +++ b/pulumi/aws/kic-helm-chart/__main__.py @@ -51,7 +51,25 @@ def build_chart_values(repository: dict) -> helm.ChartOpts: 'service': { 'annotations': { 'co.elastic.logs/module': 'nginx' - } + }, + "extraLabels": { + "app": "kic-nginx-ingress" + }, + "customPorts": [ + { + "name": "dashboard", + "targetPort": 8080, + "protocol": "TCP", + "port": 8080 + }, + { + "name": "prometheus", + "targetPort": 9113, + "protocol": "TCP", + "port": 9113 + } + ] + }, 'pod': { 'annotations': { @@ -62,7 +80,10 @@ def build_chart_values(repository: dict) -> helm.ChartOpts: 'prometheus': { 'create': True, 'port': 9113 - } + }, + "opentracing-tracer": "/usr/local/lib/libjaegertracing_plugin.so", + "opentracing-tracer-config": "{\n \"service_name\": \"nginx-ingress\",\n \"propagation_format\": \"w3c\",\n \"sampler\": {\n \"type\": \"const\",\n \"param\": 1\n },\n \"reporter\": {\n \"localAgentHostPort\": \"simplest-collector.observability.svc.cluster.local:9978\"\n }\n} \n", + "opentracing": True } has_image_tag = 'image_tag' in repository or 'image_tag_alias' in repository @@ -109,7 +130,10 @@ def build_chart_values(repository: dict) -> helm.ChartOpts: kubeconfig=kubeconfig) ns = k8s.core.v1.Namespace(resource_name='nginx-ingress', - metadata={'name': 'nginx-ingress'}, + metadata={'name': 'nginx-ingress', + 'labels': { + 'prometheus': 'scrape' } + }, opts=pulumi.ResourceOptions(provider=k8s_provider)) chart_values = ecr_repository.apply(build_chart_values) From 263aed65f3aaf20a64e301ba145a349a319c35a3 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:23:11 -0700 Subject: [PATCH 06/11] feat: added service monitor for ledgerdb and accountdb postgres --- pulumi/aws/sirius/__main__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pulumi/aws/sirius/__main__.py b/pulumi/aws/sirius/__main__.py index 9be68fca..75e6aa4f 100644 --- a/pulumi/aws/sirius/__main__.py +++ b/pulumi/aws/sirius/__main__.py @@ -383,7 +383,11 @@ def add_namespace(obj): namespace=ns, # Values from Chart's parameters specified hierarchically, - values={ + values = { + "serviceMonitor": { + "enabled": True, + "namespace": "prometheus" + }, "config": { "datasource": { "host": "accounts-db", @@ -426,7 +430,11 @@ def add_namespace(obj): namespace=ns, # Values from Chart's parameters specified hierarchically, - values={ + values = { + "serviceMonitor": { + "enabled": True, + "namespace": "prometheus" + }, "config": { "datasource": { "host": "ledger-db", From ca5d32a536d4c0086f08502a1ff7a3985171061d Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 14:53:38 -0700 Subject: [PATCH 07/11] feat: update README to reflect current configuration --- pulumi/aws/README.md | 46 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/pulumi/aws/README.md b/pulumi/aws/README.md index d5e0f01f..27494c8d 100644 --- a/pulumi/aws/README.md +++ b/pulumi/aws/README.md @@ -28,9 +28,8 @@ vpc - defines and installs the VPC and subnets to use with EKS └─logagent - deploys a logging agent (filebeat) to the EKS cluster └─certmgr - deploys the open source cert-manager.io helm chart to the EKS cluster └─prometheus - deploys prometheus server, node exporter, and statsd collector for metrics - └─grafana - deploys the grafana visualization platform - └─observability - deploys the OTEL operator and instantiates a simple collector - └─sirius - deploys the Bank of Sirus application to the EKS cluster + └─observability - deploys the OTEL operator and instantiates a simple collector + └─sirius - deploys the Bank of Sirus application to the EKS cluster ``` @@ -146,15 +145,37 @@ deployment. ### Prometheus Prometheus is deployed and configured to enable the collection of metrics for all components that have -properties `prometheus.io:scrape: true` set in the annotations -(along with any other connection information). This includes the prometheus `node-exporter` -daemonset which is deployed in this step as well. +a defined service monitor. At installation time, the deployment will instantiate: +- Node Exporters +- Kubernetes Service Monitors +- Grafana preloaded with dashboards and datasources for Kubernetes management +- The NGINX Ingress Controller +- Statsd receiver + +The former behavior of using the `prometheus.io:scrape: true` property set in the annotations +indicating pods where metrics should be scraped has been deprecated, and these annotations will +be removed in the near future. + +Also, the standalone Grafana deployment has been removed from the standard deployment scripts, but has been left as +a project in the event someone wishes to run this standalone. + +Finally, this namespace will hold service monitors created by other projects, for example the Bank of Sirius +deployment currently deploys a service monitor for each of the postgres monitors that are deployed. + +Notes: +1. The NGINX IC needs to be configured to expose prometheus metrics; this is currently done by default. +2. The default address binding of the `kube-proxy` component is set to `127.0.0.1` and as such will cause errors when the +canned prometheus scrape configurations are run. The fix is to set this address to `0.0.0.0`. An example manifest +has been provided in [prometheus/extras](./prometheus/extras) that can be applied against your installation with +`kubectl apply -f ./filename`. Please only apply this change once you have verified that it will work with your +version of Kubernetes. -This also pulls data from the NGINX KIC, provided the KIC is configured to allow prometheus access (which is enabled by -default). ### Grafana +**NOTE:** This deployment has been deprecated but the project has been left as an example on how to deploy Grafana in this +architecture. + Grafana is deployed and configured with a connection to the prometheus datasource installed above. At the time of this writing, the NGINX Plus KIC dashboard is installed as part of the initial setup. Additional datasources and dashboards can be added by the user either in the code, or via the standard Grafana tooling. @@ -188,7 +209,10 @@ As part of the Bank of Sirius deployment, we deploy a cluster-wide [self-signed](https://cert-manager.io/docs/configuration/selfsigned/) issuer using the cert-manager deployed above. This is then used by the Ingress object created to enable TLS access to the application. Note that this Issuer can be changed out by the user, for example to use the -[ACME](https://cert-manager.io/docs/configuration/acme/) issuer. +[ACME](https://cert-manager.io/docs/configuration/acme/) issuer. The use of the ACME issuer has been tested and works +without issues, provided the FQDN meets the length requirements. As of this writing the AWS ELB hostname is too long +to work with the ACME server. Additional work in this area will be undertaken to provide dynamic DNS record creation +as part of this process so legitimate certificates can be issued. In order to provide visibility into the Postgres databases that are running as part of the application, the Prometheus Postgres data exporter will be deployed into the same namespace as the application and will be configured to be scraped @@ -204,4 +228,6 @@ provides better tools for hierarchical configuration files. In order to help enable simple load testing, a script has been provided that uses the `kubectl` command to port-forward monitoring and management connections to the local workstation. This command -is [`test-foward.sh`](./extras/test-forward.sh) and is located in the [`extras`](./extras) directory. \ No newline at end of file +is [`test-foward.sh`](./extras/test-forward.sh) and is located in the [`extras`](./extras) directory. + +**NOTE:** This script has been modified to use the new Prometheus Operator based deployment. From dcc9617054f1f6b442af0349da0c9fce8d6543c4 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 9 Dec 2021 15:09:30 -0700 Subject: [PATCH 08/11] feat: update documentation to clarify Grafana Password --- pulumi/aws/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulumi/aws/README.md b/pulumi/aws/README.md index 27494c8d..11f2e66d 100644 --- a/pulumi/aws/README.md +++ b/pulumi/aws/README.md @@ -169,6 +169,9 @@ canned prometheus scrape configurations are run. The fix is to set this address has been provided in [prometheus/extras](./prometheus/extras) that can be applied against your installation with `kubectl apply -f ./filename`. Please only apply this change once you have verified that it will work with your version of Kubernetes. +3. The _grafana_ namespace has been maintained in the conifugration file to be used by the prometheus operator deployed +version of Grafana. This version only accepts a password; you can still specify a username for the admin account but it +will be silently ignored. ### Grafana From 2cf217d2d3a668501a770bac757fb8b0a75f9214 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Thu, 16 Dec 2021 16:58:20 -0700 Subject: [PATCH 09/11] fix: adjust depends_on for prometheus deployment --- pulumi/aws/prometheus/__main__.py | 25 ++++++++----------------- pulumi/aws/sirius/src | 2 +- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/pulumi/aws/prometheus/__main__.py b/pulumi/aws/prometheus/__main__.py index 4b372e63..f064cc31 100644 --- a/pulumi/aws/prometheus/__main__.py +++ b/pulumi/aws/prometheus/__main__.py @@ -38,8 +38,6 @@ def servicemon_manifests_location(): metadata={'name': 'prometheus'}, opts=pulumi.ResourceOptions(provider=k8s_provider)) - - config = pulumi.Config('prometheus') chart_name = config.get('chart_name') if not chart_name: @@ -53,7 +51,7 @@ def servicemon_manifests_location(): helm_repo_url = config.get('prometheus_helm_repo_url') if not helm_repo_url: helm_repo_url = 'https://prometheus-community.github.io/helm-charts' - + grafana_config = pulumi.Config('grafana') # Require an admin password, but do not encrypt it due to the # issues we experienced with Anthos; this can be adjusted at the @@ -81,9 +79,9 @@ def servicemon_manifests_location(): "serviceMonitorSelectorNilUsesHelmValues": False, "serviceMonitorSelector": {}, "serviceMonitorNamespaceSelector ": { - "matchLabels": { - "prometheus": True - } + "matchLabels": { + "prometheus": True + } }, "storageSpec": { "volumeClaimTemplate": { @@ -152,7 +150,7 @@ def servicemon_manifests_location(): # Force update if required force_update=True) -prometheus_release = Release("prometheus", args=prometheus_release_args) +prometheus_release = Release("prometheus", args=prometheus_release_args, opts=pulumi.ResourceOptions(depends_on=[ns])) prom_status = prometheus_release.status @@ -161,7 +159,7 @@ def servicemon_manifests_location(): servicemon = ConfigGroup( 'servicemon', files=[servicemon_manifests], - opts=pulumi.ResourceOptions(depends_on=[prometheus_release]) + opts=pulumi.ResourceOptions(depends_on=[ns, prometheus_release]) ) # @@ -200,14 +198,6 @@ def servicemon_manifests_location(): "create": True, "annotations": {}, "name": "" - }, - "podAnnotations": { - "prometheus.io/scrape": "true", - "prometheus.io/port": "9102" - }, - "annotations": { - "prometheus.io/scrape": "true", - "prometheus.io/port": "9102" } }, # By default Release resource will wait till all created resources @@ -223,7 +213,8 @@ def servicemon_manifests_location(): # Force update if required force_update=True) -statsd_release = Release("statsd", args=statsd_release_args) +statsd_release = Release("statsd", args=statsd_release_args, + opts=pulumi.ResourceOptions(depends_on=[ns, prometheus_release])) statsd_status = statsd_release.status diff --git a/pulumi/aws/sirius/src b/pulumi/aws/sirius/src index 96e7df33..27bfff2f 160000 --- a/pulumi/aws/sirius/src +++ b/pulumi/aws/sirius/src @@ -1 +1 @@ -Subproject commit 96e7df33a7808079bd8ccd61e0c8ad99eeef8f0f +Subproject commit 27bfff2f6f93b80fc230878c450126b889c2551f From 646977fe3aa0a9cdd24a54fd1091bcedd21337e7 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Fri, 17 Dec 2021 10:50:52 -0700 Subject: [PATCH 10/11] feat: remove grafana standalone in favor of prometheus kube stack --- .../aws/config/Pulumi.stackname.yaml.example | 12 +- pulumi/aws/grafana/Pulumi.yaml | 7 - pulumi/aws/grafana/__main__.py | 139 ------------------ pulumi/aws/start_all.sh | 10 -- 4 files changed, 1 insertion(+), 167 deletions(-) delete mode 100644 pulumi/aws/grafana/Pulumi.yaml delete mode 100644 pulumi/aws/grafana/__main__.py diff --git a/pulumi/aws/config/Pulumi.stackname.yaml.example b/pulumi/aws/config/Pulumi.stackname.yaml.example index 867e8e60..7c44c015 100644 --- a/pulumi/aws/config/Pulumi.stackname.yaml.example +++ b/pulumi/aws/config/Pulumi.stackname.yaml.example @@ -178,16 +178,6 @@ config: ############################################################################ # Grafana Configuration - grafana:chart_name: grafana - # Chart name for the helm chart for grafana - grafana:chart_version: 6.13.7 - # Chart version for the helm chart for grafana - grafana:helm_repo_name: grafana - # Name of the repo to pull the grafana chart from - grafana:helm_repo_url: https://grafana.github.io/helm-charts - # URL of the chart repo to pull grafana from - grafana:adminuser: admin - # The username for the grafana installation grafana:adminpass: strongpass # The password for the grafana installation; note that this is not exposed to the internet # and requires kubeproxy to access. However, this should be encrypted which is dependent on @@ -197,7 +187,7 @@ config: ############################################################################ # Prometheus Configuration - prometheus:chart_name: prometheus + prometheus:chart_name: kube-prometheus-stack # Chart name for the helm chart for prometheus prometheus:chart_version: 14.6.0 # Chart version for the helm chart for prometheus diff --git a/pulumi/aws/grafana/Pulumi.yaml b/pulumi/aws/grafana/Pulumi.yaml deleted file mode 100644 index 12d471e6..00000000 --- a/pulumi/aws/grafana/Pulumi.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: grafana -runtime: - name: python - options: - virtualenv: ../venv -config: ../config -description: Deploys Grafana diff --git a/pulumi/aws/grafana/__main__.py b/pulumi/aws/grafana/__main__.py deleted file mode 100644 index 5e542530..00000000 --- a/pulumi/aws/grafana/__main__.py +++ /dev/null @@ -1,139 +0,0 @@ -import os - -import pulumi -import pulumi_kubernetes as k8s -from pulumi_kubernetes.helm.v3 import Release, ReleaseArgs, RepositoryOptsArgs - -from kic_util import pulumi_config - - -def project_name_from_project_dir(dirname: str): - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_path = os.path.join(script_dir, '..', dirname) - return pulumi_config.get_pulumi_project_name(project_path) - - -def pulumi_prometheus_project_name(): - script_dir = os.path.dirname(os.path.abspath(__file__)) - prometheus_project_path = os.path.join(script_dir, '..', 'prometheus') - return pulumi_config.get_pulumi_project_name(prometheus_project_path) - - -stack_name = pulumi.get_stack() -project_name = pulumi.get_project() -pulumi_user = pulumi_config.get_pulumi_user() - -eks_project_name = project_name_from_project_dir('eks') -eks_stack_ref_id = f"{pulumi_user}/{eks_project_name}/{stack_name}" -eks_stack_ref = pulumi.StackReference(eks_stack_ref_id) -kubeconfig = eks_stack_ref.require_output('kubeconfig').apply(lambda c: str(c)) - -k8s_provider = k8s.Provider(resource_name=f'ingress-setup-sample', - kubeconfig=kubeconfig) - -ns = k8s.core.v1.Namespace(resource_name='grafana', - metadata={'name': 'grafana'}, - opts=pulumi.ResourceOptions(provider=k8s_provider)) - -config = pulumi.Config('grafana') -adminuser = config.get('admin_user') -if not adminuser: - adminuser = 'admin' - -# Require an admin password, but do not encrypt it due to the -# issues we experienced with Anthos; this can be adjusted at the -# same time that we fix the Anthos issues. -adminpass = config.require('adminpass') - -chart_name = config.get('chart_name') -if not chart_name: - chart_name = 'grafana' -chart_version = config.get('chart_version') -if not chart_version: - chart_version = '6.13.7' -helm_repo_name = config.get('grafana_helm_repo_name') -if not helm_repo_name: - helm_repo_name = 'grafana' -helm_repo_url = config.get('grafana_helm_repo_url') -if not helm_repo_url: - helm_repo_url = 'https://grafana.github.io/helm-charts' - -# Logic to extract the FQDN of prometheus -prometheus_project_name = pulumi_prometheus_project_name() -prometheus_stack_ref_id = f"{pulumi_user}/{prometheus_project_name}/{stack_name}" -prometheus_stack_ref = pulumi.StackReference(prometheus_stack_ref_id) -prometheus_hostname = prometheus_stack_ref.get_output('prometheus_hostname') - -grafana_release_args = ReleaseArgs( - chart=chart_name, - repository_opts=RepositoryOptsArgs( - repo=helm_repo_url - ), - version=chart_version, - namespace=ns.metadata.name, - - # Values from Chart's parameters specified hierarchically, - values={ - 'persistence': { - 'enabled': True - }, - 'adminUser': adminuser, - 'adminPassword': adminpass, - "datasources": { - "datasources.yaml": { - "apiVersion": 1, - "datasources": [ - { - "name": "Prometheus", - "type": "prometheus", - "url": "http://prometheus-server.prometheus.svc.cluster.local:80", - "access": "proxy", - "isDefault": True - } - ] - } - }, - "dashboardProviders": { - "dashboardproviders.yaml": { - "apiVersion": 1, - "providers": [ - { - "name": "default", - "orgId": 1, - "folder": "", - "type": "file", - "disableDeletion": False, - "editable": True, - "options": { - "path": "/var/lib/grafana/dashboards/default" - } - } - ] - } - }, - "dashboards": { - "default": { - "local-dashboard": { - "url": "https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/master/grafana/NGINXPlusICDashboard.json", - "token": "", - "datasoure": "Prometheus" - } - } - } - }, - # By default Release resource will wait till all created resources - # are available. Set this to true to skip waiting on resources being - # available. - skip_await=False, - # If we fail, clean up - cleanup_on_fail=True, - # Provide a name for our release - name="grafana", - # Lint the chart before installing - lint=True, - # Force update if required - force_update=True) -grafana_release = Release("grafana", args=grafana_release_args) - -status = grafana_release.status -pulumi.export("grafana_status", status) diff --git a/pulumi/aws/start_all.sh b/pulumi/aws/start_all.sh index eaaa80cb..6bcd92a0 100755 --- a/pulumi/aws/start_all.sh +++ b/pulumi/aws/start_all.sh @@ -317,16 +317,6 @@ header "Prometheus" cd "${script_dir}/prometheus" pulumi $pulumi_args up -# This is commented out, as the new prometheus deployment provides this -# functionality. -# -# If you wish to use this intead, you will need to adjust the helm manifest -# values in the __main__.py file in the prometheus directory. -# -#header "Grafana" -#cd "${script_dir}/grafana" -#pulumi $pulumi_args up - header "Observability" cd "${script_dir}/observability" pulumi $pulumi_args up From 6686f48012b52b6caa651272c31eddba9e462a32 Mon Sep 17 00:00:00 2001 From: Jason Schmidt Date: Fri, 17 Dec 2021 12:58:22 -0700 Subject: [PATCH 11/11] chore: upgrade pulumi version --- pulumi/aws/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulumi/aws/requirements.txt b/pulumi/aws/requirements.txt index 86f10509..4cda9402 100644 --- a/pulumi/aws/requirements.txt +++ b/pulumi/aws/requirements.txt @@ -3,7 +3,7 @@ fart~=0.1.5 lolcat~=1.4 nodeenv~=1.6.0 passlib~=1.7.4 -pulumi~=3.16.0 +pulumi~=3.19.0 pulumi-aws>=4.25.0 pulumi-docker~=3.1.0 pulumi-eks>=0.34.0 @@ -13,4 +13,4 @@ PyYAML~=5.4.1 requests~=2.26.0 setuptools==58.3.0 wheel==0.37.0 -yamlreader==3.0.4 \ No newline at end of file +yamlreader==3.0.4