From e7dd69f38b91d14bde8abe8c30bcfcb305184b65 Mon Sep 17 00:00:00 2001 From: decentral1se Date: Fri, 18 Mar 2022 14:50:28 +0100 Subject: [PATCH] hackity hack hack loki now working --- README.md | 7 + compose.yml | 39 +- env | 11 +- grafana-dashboards.yml | 14 - grafana-datasources.yml | 11 - grafana-stacks-dashboard.json | 745 -------------- grafana-swarm-dashboard.json | 1729 -------------------------------- grafana-traefik-dashboard.json | 1536 ---------------------------- loki.htpasswd.tmpl | 2 +- loki.yml.tmpl | 2 +- nginx.conf.tmpl | 31 +- prometheus.yml.tmpl | 3 + prometheus_web.yml.tmpl | 2 +- 13 files changed, 55 insertions(+), 4077 deletions(-) delete mode 100644 grafana-dashboards.yml delete mode 100644 grafana-datasources.yml delete mode 100644 grafana-stacks-dashboard.json delete mode 100644 grafana-swarm-dashboard.json delete mode 100644 grafana-traefik-dashboard.json diff --git a/README.md b/README.md index 170d55a..f5d6898 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ - [p.monitor.autonomic.zone](https://p.monitor.autonomic.zone) - [l.monitor.autonomic.zone](https://l.monitor.autonomic.zone) +## setup + ``` printf $(pass show hosts/swarm.autonomic.zone/minio/secret_key) | docker secret create gp_monitoring_loki_aws_secret_access_key_v1 - printf password | docker secret create gp_monitoring_grafana_admin_password_v1 - @@ -24,3 +26,8 @@ docker context use monitor.autonomic.zone docker stack deploy -c compose.yml gp_monitoring ``` + +## questions / TODO + +- [ ] how to load in secrets from multiple hosts & keep a recipe generic? +- [ ] basic auth on gathering stack diff --git a/compose.yml b/compose.yml index f33295c..4b3dc06 100644 --- a/compose.yml +++ b/compose.yml @@ -10,16 +10,6 @@ services: - grafana_admin_password - grafana_oauth_client_secret configs: - - source: grafana_datasources_yml - target: /etc/grafana/provisioning/datasources/datasources.yml - - source: grafana_dashboards_yml - target: /etc/grafana/provisioning/dashboards/dashboards.yml - - source: grafana_swarm_dashboard_json - target: /var/lib/grafana/dashboards/docker-swarm-nodes.json - - source: grafana_stacks_dashboard_json - target: /var/lib/grafana/dashboards/docker-swarm-stacks.json - - source: grafana_traefik_dashboard_json - target: /var/lib/grafana/dashboards/traefik.json - source: grafana_custom_ini target: /etc/grafana/grafana.ini networks: @@ -56,6 +46,7 @@ services: image: prom/prometheus:v2.34.0 secrets: - prometheus_admin_password + - prometheus_admin_password_hashed volumes: - prometheus-data:/prometheus:rw configs: @@ -102,7 +93,6 @@ services: - ALERTMANAGER_SMTP_HOST - ALERTMANAGER_SMTP_TO - # Note(d1): https://grafana.com/docs/loki/latest/operations/authentication/ web: image: nginx:1.20.0 networks: @@ -117,7 +107,7 @@ services: - source: htpasswd_conf target: /etc/nginx/conf.d/loki.htpasswd secrets: - - loki_admin_password + - loki_admin_password_hashed deploy: restart_policy: condition: on-failure @@ -141,10 +131,11 @@ services: secrets: - loki_aws_secret_access_key environment: + - LOKI_ACCESS_KEY_ID - LOKI_AWS_ENDPOINT - LOKI_AWS_REGION - - LOKI_ACCESS_KEY_ID - LOKI_BUCKET_NAMES + - STACK_NAME configs: grafana_custom_ini: @@ -175,21 +166,6 @@ configs: template_driver: golang name: ${STACK_NAME}_htpasswd_${HTPASSWD_CONFIG_VERSION} file: loki.htpasswd.tmpl - grafana_datasources_yml: - name: ${STACK_NAME}_grafana_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION} - file: grafana-datasources.yml - grafana_dashboards_yml: - name: ${STACK_NAME}_grafana_dashboards_yml_${GRAFANA_DASHBOARDS_YML_VERSION} - file: grafana-dashboards.yml - grafana_swarm_dashboard_json: - name: ${STACK_NAME}_grafana_swarm_dashboard_json_${GRAFANA_SWARM_DASHBOARD_JSON_VERSION} - file: grafana-swarm-dashboard.json - grafana_stacks_dashboard_json: - name: ${STACK_NAME}_grafana_stacks_dashboard_json_${GRAFANA_STACKS_DASHBOARD_JSON_VERSION} - file: grafana-stacks-dashboard.json - grafana_traefik_dashboard_json: - name: ${STACK_NAME}_grafana_traefik_dashboard_json_${GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION} - file: grafana-traefik-dashboard.json volumes: prometheus-data: @@ -212,12 +188,15 @@ secrets: grafana_oauth_client_secret: external: true name: ${STACK_NAME}_grafana_oauth_client_secret_${SECRET_GRAFANA_OAUTH_CLIENT_SECRET_VERSION} + prometheus_admin_password_hashed: + external: true + name: ${STACK_NAME}_prometheus_admin_password_hashed_${SECRET_PROMETHEUS_ADMIN_PASSWORD_HASHED_VERSION} prometheus_admin_password: external: true name: ${STACK_NAME}_prometheus_admin_password_${SECRET_PROMETHEUS_ADMIN_PASSWORD_VERSION} alertmanager_smtp_password: external: true name: ${STACK_NAME}_alertmanager_smtp_password_${SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION} - loki_admin_password: + loki_admin_password_hashed: external: true - name: ${STACK_NAME}_loki_admin_password_${SECRET_LOKI_ADMIN_PASSWORD_VERSION} + name: ${STACK_NAME}_loki_admin_password_hashed_${SECRET_LOKI_ADMIN_PASSWORD_HASHED_VERSION} diff --git a/env b/env index f44da55..61ba9a7 100644 --- a/env +++ b/env @@ -31,9 +31,9 @@ LOKI_ACCESS_KEY_ID=bush-debrief-approval-robust-scraggly-molecule LOKI_BUCKET_NAMES=loki # NOTE(d1): abra.sh env vars, while we deploy things manually -PROMETHEUS_YML_VERSION=v2 +PROMETHEUS_YML_VERSION=v6 PROMTAIL_YML_VERSION=v1 -LOKI_YML_VERSION=v1 +LOKI_YML_VERSION=v2 NODE_EXPORTER_ENTRYPOINT_VERSION=v1 GRAFANA_DATASOURCES_YML_VERSION=v1 GRAFANA_DASHBOARDS_YML_VERSION=v1 @@ -41,9 +41,9 @@ GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v1 GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v1 GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v1 GRAFANA_CUSTOM_INI_VERSION=v1 -PROMETHEUS_WEB_YML_VERSION=v1 +PROMETHEUS_WEB_YML_VERSION=v2 ALERTMANAGER_CONFIG_VERSION=v1 -NGINX_CONFIG_VERSION=v1 +NGINX_CONFIG_VERSION=v5 HTPASSWD_CONFIG_VERSION=v1 KEYCLOAK_AUTH_URL="https://id.autonomic.zone/auth/realms/autonomic/protocol/openid-connect/auth" @@ -54,8 +54,9 @@ SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION=v1 SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1 SECRET_GRAFANA_OAUTH_CLIENT_SECRET_VERSION=v1 SECRET_PROMETHEUS_ADMIN_PASSWORD_VERSION=v1 +SECRET_PROMETHEUS_ADMIN_PASSWORD_HASHED_VERSION=v1 SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION=v1 -SECRET_LOKI_ADMIN_PASSWORD_VERSION=v1 +SECRET_LOKI_ADMIN_PASSWORD_HASHED_VERSION=v1 ALERTMANAGER_SMTP_FROM=noreply@autonomic.zone ALERTMANAGER_SMTP_HOST=mail.gandi.net:465 diff --git a/grafana-dashboards.yml b/grafana-dashboards.yml deleted file mode 100644 index 73888c4..0000000 --- a/grafana-dashboards.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: 1 - -providers: - - name: 'default-dashboard-provider' - orgId: 1 - folder: 'default-dashboards' - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: true diff --git a/grafana-datasources.yml b/grafana-datasources.yml deleted file mode 100644 index 21e1a7b..0000000 --- a/grafana-datasources.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - orgId: 1 - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/grafana-stacks-dashboard.json b/grafana-stacks-dashboard.json deleted file mode 100644 index a3322a2..0000000 --- a/grafana-stacks-dashboard.json +++ /dev/null @@ -1,745 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Simple docker swarm monitoring with relation to individual stacks.", - "editable": true, - "gnetId": 7007, - "graphTooltip": 1, - "id": 2, - "iteration": 1626744694222, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 10, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "(time() - min(container_start_time_seconds{container_label_com_docker_stack_namespace=~\"$stack\"}))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Stack Uptime", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": null, - "fill": 5, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 20, - "x": 4, - "y": 0 - }, - "height": "270px", - "hiddenSeries": false, - "id": 1, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\"}[$interval])) by (container_label_com_docker_swarm_service_name, container_label_com_docker_swarm_task_id)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_label_com_docker_swarm_service_name }} - {{ container_label_com_docker_swarm_task_id }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage per Container", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 3 - }, - "id": 3, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "count(rate(container_last_seen{container_label_com_docker_stack_namespace=~\"$stack\"}[$interval]))", - "format": "time_series", - "intervalFactor": 2, - "refId": "B", - "step": 4 - } - ], - "title": "Containers", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 3, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 7 - }, - "height": "270px", - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{container_label_com_docker_stack_namespace=~\"$stack\"}) by (container_label_com_docker_swarm_service_name, container_label_com_docker_swarm_task_id)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_label_com_docker_swarm_service_name }} - {{ container_label_com_docker_swarm_task_id }}", - "metric": "container_memory_rss", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage per Container", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 14 - }, - "height": "270px", - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_receive_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\"}[$interval])) by (container_label_com_docker_swarm_service_name, container_label_com_docker_swarm_task_id)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_label_com_docker_swarm_service_name }} - {{ container_label_com_docker_swarm_task_id }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Incoming Network Traffic per Container", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 21 - }, - "height": "270px", - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_transmit_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\"}[$interval])) by (container_label_com_docker_swarm_service_name, container_label_com_docker_swarm_task_id)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_label_com_docker_swarm_service_name }} - {{ container_label_com_docker_swarm_task_id }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Outgoing Network Traffic per Container", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 30, - "style": "dark", - "tags": [ - "prometheus", - "cAdvisor", - "node-exporter", - "alertmanager" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "pretix", - "value": "pretix" - }, - "datasource": "Prometheus", - "definition": "", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Stack", - "multi": false, - "name": "stack", - "options": [], - "query": { - "query": "query_result(count(container_last_seen{container_label_com_docker_stack_namespace =~\".+\"}) by (container_label_com_docker_stack_namespace))", - "refId": "Prometheus-stack-Variable-Query" - }, - "refresh": 1, - "regex": "/container_label_com_docker_stack_namespace=\"(.*)\"/", - "skipUrlSync": false, - "sort": 2, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "auto": true, - "auto_count": 50, - "auto_min": "50s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "description": null, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": false, - "text": "3m", - "value": "3m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "7m", - "value": "7m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Swarm Stack Monitoring", - "uid": "KdVoGQm7z", - "version": 1 -} diff --git a/grafana-swarm-dashboard.json b/grafana-swarm-dashboard.json deleted file mode 100644 index 6c9468a..0000000 --- a/grafana-swarm-dashboard.json +++ /dev/null @@ -1,1729 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Docker Swarm nodes metrics", - "editable": true, - "gnetId": 7461, - "graphTooltip": 0, - "id": 1, - "iteration": 1626741910840, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 0 - }, - "hideTimeOverride": true, - "id": 2, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "topk(1, sum((node_time_seconds - node_boot_time_seconds) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 2 - } - ], - "timeFrom": "1m", - "timeShift": null, - "title": "Uptime", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 1, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "Nodes", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 8 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "hideTimeOverride": true, - "id": 4, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "count(node_cpu_seconds_total{mode=\"idle\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 2 - } - ], - "timeFrom": "1m", - "timeShift": null, - "title": "CPUs", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 25 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "hideTimeOverride": true, - "id": 11, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) * 100 / count(node_cpu_seconds_total{mode=\"user\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) ", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 2 - } - ], - "timeFrom": "1m", - "timeShift": null, - "title": "CPU Idle", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "rightSide": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node_load5 * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "load5 {{node_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "System Load by Node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 4 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) by (node_name))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{node_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage by Node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": "100", - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 11 - }, - "hideTimeOverride": true, - "id": 3, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "sum(node_memory_MemTotal_bytes * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Memory", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 25 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 11 - }, - "id": 8, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "sum((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "Available Memory", - "type": "gauge" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed" - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 11 - }, - "hideTimeOverride": true, - "id": 9, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "exemplar": true, - "expr": "sum(node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Disk Space", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 10 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 25 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 11 - }, - "id": 10, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "exemplar": true, - "expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 20 - } - ], - "title": "Available Disk Space", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum((node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Cached_bytes - node_memory_Buffers_bytes - node_memory_Slab_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Used {{node_name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(node_memory_Cached * on(instance) group_left(node_name) node_meta{node_id=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Cached {{node_name}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory usage by Node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 22 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(node_disk_read_bytes_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Read {{node_name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(node_disk_written_bytes_total[$interval]) * on(instance) group_left(node_name) node_meta{node_id=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Written {{node_name}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk I/O by Node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(node_disk_reads_completed_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Reads {{node_name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(node_disk_writes_completed_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Writes {{node_name}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "IOPS by Node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": true, - "rightSide": true, - "show": false, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(avg(irate(node_cpu_seconds_total{mode=\"iowait\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) by (node_name))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{node_name}}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU IO Wait by Node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "fill": 3, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 18, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_last_seen[5m]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (container_label_com_docker_swarm_service_name)", - "format": "time_series", - "hide": false, - "intervalFactor": 10, - "legendFormat": "{{ container_label_com_docker_swarm_service_name }}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Running Containers by Service", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 36 - }, - "id": 7, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "expr": "count(rate(container_last_seen[5m]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"$node_id\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 20 - } - ], - "title": "Total Containers", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 43 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_receive_bytes_total[$interval]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (node_name)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "IN {{node_name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "- sum(rate(container_network_transmit_bytes_total[$interval]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (node_name)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "OUT {{node_name}}", - "metric": "", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Containers Network Traffic by Node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "columns": [], - "datasource": "Prometheus", - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 50 - }, - "hideTimeOverride": true, - "id": 20, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(node_meta) by (node_id, node_name, instance)", - "format": "table", - "intervalFactor": 2, - "refId": "A", - "step": 2 - } - ], - "timeFrom": "1s", - "title": "Cluster members", - "transform": "table", - "type": "table-old" - } - ], - "refresh": "5s", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "swarmprom", - "prometheus", - "node-exporter", - "cadvisor" - ], - "templating": { - "list": [ - { - "allValue": ".+", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "definition": "", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Swarm Node", - "multi": false, - "name": "node_id", - "options": [], - "query": { - "query": "node_meta", - "refId": "Prometheus-node_id-Variable-Query" - }, - "refresh": 1, - "regex": "/node_name=\"([^\"]+)\"/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "30s", - "current": { - "selected": false, - "text": "1m", - "value": "1m" - }, - "description": null, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Docker Swarm Nodes", - "uid": "BPlb-Sgik", - "version": 4 -} diff --git a/grafana-traefik-dashboard.json b/grafana-traefik-dashboard.json deleted file mode 100644 index 78b6c14..0000000 --- a/grafana-traefik-dashboard.json +++ /dev/null @@ -1,1536 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Simple dashboard for Traefik 2", - "editable": true, - "gnetId": 11462, - "graphTooltip": 0, - "id": 4, - "iteration": 1628548257934, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 22, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "time() - process_start_time_seconds{job=\"$job\"}", - "format": "time_series", - "instant": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "0", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#37872D", - "value": null - }, - { - "color": "#C4162A", - "value": 1 - }, - { - "color": "#C4162A", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 3, - "y": 0 - }, - "id": 32, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "max" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "sum(increase(traefik_service_requests_total{code=\"499\"}[$interval]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Max Service Error count ($interval)", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#37872D", - "value": null - }, - { - "color": "#37872D", - "value": 0 - }, - { - "color": "rgb(156, 156, 156)", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 7, - "y": 0 - }, - "id": 26, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "count" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "sum(increase(traefik_service_requests_total{code=\"404\",protocol=~\"$protocol\"}[$interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "metric": "traefik_requests_total", - "refId": "A", - "step": 60 - } - ], - "title": "Max 404 Error Count ($interval)", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 0, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#C4162A", - "value": null - }, - { - "color": "#FF9830", - "value": 10 - }, - { - "color": "green", - "value": 50 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 10, - "y": 0 - }, - "id": 33, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "sum(increase(traefik_service_requests_total{code=\"200\",protocol=~\"$protocol\"}[$interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "traefik_requests_total", - "refId": "A", - "step": 60 - } - ], - "title": "200 Status Count (last $interval)", - "type": "stat" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 6, - "w": 10, - "x": 14, - "y": 0 - }, - "id": 18, - "interval": null, - "legend": { - "percentage": true, - "show": true, - "sort": null, - "sortDesc": null, - "values": true - }, - "legendType": "Right side", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "pieType": "pie", - "strokeWidth": 1, - "targets": [ - { - "expr": "topk(5, sum(traefik_service_requests_total{protocol=~\"$protocol\"}) by (code))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{code}}", - "refId": "A" - } - ], - "title": "Top 5 $protocol return codes", - "type": "grafana-piechart-panel", - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 6 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(traefik_service_requests_total{code=\"499\",method=\"GET\"}[$interval])) by (service)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{service}} ", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bad Status Code Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [], - "unit": "ms" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 10, - "x": 10, - "y": 6 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum(traefik_entrypoint_request_duration_seconds_sum{protocol=~\"$protocol\"}) / sum(traefik_entrypoint_requests_total{protocol=~\"$protocol\"}) * 1000) - (sum(traefik_service_request_duration_seconds_sum{protocol=~\"$protocol\"}) / sum(traefik_service_requests_total{protocol=~\"$protocol\"}) * 1000)", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Average processing time (ms)", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average processing time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 10, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 10 - }, - { - "color": "#EAB839", - "value": 200 - }, - { - "color": "red", - "value": 500 - }, - { - "color": "purple", - "value": 1000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 6 - }, - "id": 20, - "interval": null, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "sum(traefik_entrypoint_request_duration_seconds_sum) / sum(traefik_entrypoint_requests_total) * 1000", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "title": "Average response time", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 17, - "x": 0, - "y": 12 - }, - "hiddenSeries": false, - "hideTimeOverride": false, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(traefik_service_requests_total[$interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Total requests", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 17, - "y": 12 - }, - "hideTimeOverride": false, - "id": 34, - "links": [], - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.5", - "targets": [ - { - "expr": "sum(rate(traefik_service_requests_total[$interval]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total requests", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests per second", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{job=~\"$job\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ instance }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Used sockets", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 14, - "x": 10, - "y": 18 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(traefik_service_requests_total{protocol=~\"http|https\"}[$interval])) by (service)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{service}} ", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Access to services", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 7, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(traefik_entrypoint_open_connections) by (method)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ method }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "ENTRYPOINT - Open Connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 7, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "avg", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(traefik_service_open_connections) by (method)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ method }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "SERVICE - Open Connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.5", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/^[^234].*/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(traefik_service_requests_total{protocol=~\"$protocol\"}[$interval])) by (code)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{code}}", - "refId": "A", - "step": 120 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Status Code Count ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "", - "schemaVersion": 26, - "style": "dark", - "tags": [ - "traefik", - "load-balancer", - "docker", - "prometheus" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "traefik", - "value": "traefik" - }, - "datasource": "Prometheus", - "definition": "", - "error": null, - "hide": 0, - "includeAll": false, - "label": "Job:", - "multi": false, - "name": "job", - "options": [], - "query": "label_values(job)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 2, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "definition": "label_values(traefik_service_requests_total, protocol)", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Service:", - "multi": true, - "name": "protocol", - "options": [], - "query": "label_values(traefik_service_requests_total, protocol)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Traefik 2", - "uid": "3ipsWfViz", - "version": 4 -} diff --git a/loki.htpasswd.tmpl b/loki.htpasswd.tmpl index 7ea69e6..74f33cc 100644 --- a/loki.htpasswd.tmpl +++ b/loki.htpasswd.tmpl @@ -1 +1 @@ -loki:{{ secret "loki_admin_password" }} +loki:{{ secret "loki_admin_password_hashed" }} diff --git a/loki.yml.tmpl b/loki.yml.tmpl index a356dff..979f298 100644 --- a/loki.yml.tmpl +++ b/loki.yml.tmpl @@ -22,7 +22,7 @@ memberlist: abort_if_cluster_join_fails: false bind_port: 7946 join_members: - - loki:7946 + - {{ env "STACK_NAME" }}_loki:7946 max_join_backoff: 1m max_join_retries: 10 min_join_backoff: 1s diff --git a/nginx.conf.tmpl b/nginx.conf.tmpl index 497d39b..b8af0ba 100644 --- a/nginx.conf.tmpl +++ b/nginx.conf.tmpl @@ -7,14 +7,37 @@ events { http { include /etc/nginx/mime.types; + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + server { listen 80; server_name {{ env "LOKI_DOMAIN" }}; - location /loki/api/v1 { - auth_basic "loki"; - auth_basic_user_file /etc/nginx/conf.d/loki.htpasswd; - proxy_pass http://{{ env "STACK_NAME" }}_loki:3100/loki/api/v1; + auth_basic "loki"; + auth_basic_user_file /etc/nginx/conf.d/loki.htpasswd; + + location / { + proxy_read_timeout 1800s; + proxy_connect_timeout 1600s; + proxy_pass http://{{ env "STACK_NAME" }}_loki:3100; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Connection "Keep-Alive"; + proxy_set_header Proxy-Connection "Keep-Alive"; + proxy_redirect off; + } + + location /ready { + proxy_pass http://{{ env "STACK_NAME" }}_loki:3100; + proxy_http_version 1.1; + proxy_set_header Connection "Keep-Alive"; + proxy_set_header Proxy-Connection "Keep-Alive"; + proxy_redirect off; + auth_basic "off"; } } } diff --git a/prometheus.yml.tmpl b/prometheus.yml.tmpl index a701f55..2cf984e 100644 --- a/prometheus.yml.tmpl +++ b/prometheus.yml.tmpl @@ -13,3 +13,6 @@ scrape_configs: static_configs: - targets: - localhost:9090 + basic_auth: + username: 'admin' + password: '{{ secret "prometheus_admin_password" }}' diff --git a/prometheus_web.yml.tmpl b/prometheus_web.yml.tmpl index 193205f..38c005a 100644 --- a/prometheus_web.yml.tmpl +++ b/prometheus_web.yml.tmpl @@ -1,2 +1,2 @@ basic_auth_users: - admin: {{ secret "prometheus_admin_password" }} + admin: {{ secret "prometheus_admin_password_hashed" }}