loki auth

monitoring/README.md

@@ -4,6 +4,7 @@
 
 - [g.monitor.autonomic.zone](https://g.monitor.autonomic.zone)
 - [p.monitor.autonomic.zone](https://p.monitor.autonomic.zone)
+- [l.monitor.autonomic.zone](https://l.monitor.autonomic.zone)
 
 ```
 printf $(pass show hosts/swarm.autonomic.zone/minio/secret_key) | docker secret create gp_monitoring_loki_aws_secret_access_key_v1 -
@@ -13,6 +14,9 @@ printf <...> | docker secret create gp_monitoring_grafana_oauth_client_secret_v1
 pwgen -s 64 1; ./scripts/genpw.py # input password & get hashed output for secret
 printf <...> | docker secret create gp_monitoring_prometheus_admin_password_v1 -
 
+pwgen -s 64 1; ./scripts/genpw.py # input password & get hashed output for secret
+printf <...> | docker secret create gp_monitoring_loki_admin_password_v1 -
+
 printf <...> | docker secret create gp_monitoring_alertmanager_smtp_password_v1 -
 
 set -a && source env && set +a

monitoring/alertmanager.yml.tmpl

@@ -1,10 +1,10 @@
 global:
-  smtp_from: {{ env "SMTP_FROM" }}
+  smtp_from: {{ env "ALERTMANAGER_SMTP_FROM" }}
-  smtp_smarthost: {{ env "SMTP_HOST" }}
+  smtp_smarthost: {{ env "ALERTMANAGER_SMTP_HOST" }}
-  smtp_auth_username: {{ env "SMTP_FROM" }}
+  smtp_auth_username: {{ env "ALERTMANAGER_SMTP_FROM" }}
   smtp_auth_password: {{ secret "alertmanager_smtp_password" }}
 
 receivers:
   - name: "kaboom mailer"
     email_configs:
-      - to: {{ env "SMTP_TO" }}
+      - to: {{ env "ALERTMANAGER_SMTP_TO" }}

monitoring/compose.yml

@@ -98,22 +98,41 @@ services:
       - source: alertmanager_config
         target: /etc/alertmanager/config.yml
     environment:
-      - SMTP_FROM
+      - ALERTMANAGER_SMTP_FROM
-      - SMTP_HOST
+      - ALERTMANAGER_SMTP_HOST
-      - SMTP_TO
+      - ALERTMANAGER_SMTP_TO
+
+  # Note(d1): https://grafana.com/docs/loki/latest/operations/authentication/
+  web:
+    image: nginx:1.20.0
+    networks:
+      - proxy
+      - internal
+    environment:
+      - LOKI_DOMAIN
+      - STACK_NAME
+    configs:
+      - source: nginx_config
+        target: /etc/nginx/nginx.conf
+      - source: htpasswd_conf
+        target: /etc/nginx/conf.d/loki.htpasswd
+    secrets:
+      - loki_admin_password
+    deploy:
+      restart_policy:
+        condition: on-failure
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.services.${STACK_NAME}-web.loadbalancer.server.port=80"
+        - "traefik.http.routers.${STACK_NAME}-web.rule=Host(`${LOKI_DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-web.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-web.tls.certresolver=${LETS_ENCRYPT_ENV}"
 
   loki:
     image: grafana/loki:2.0.0
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - internal
-    deploy:
-      endpoint_mode: dnsrr
-    ports:
-      - target: 3100
-        published: 3100
-        protocol: tcp
-        mode: host
     configs:
       - source: loki_yml
         target: /etc/loki/local-config.yaml
@@ -148,6 +167,14 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_alertmanager_config_${ALERTMANAGER_CONFIG_VERSION}
     file: ./alertmanager.yml.tmpl
+  nginx_config:
+    template_driver: golang
+    name: ${STACK_NAME}_nginx_config_${NGINX_CONFIG_VERSION}
+    file: nginx.conf.tmpl
+  htpasswd_conf:
+    template_driver: golang
+    name: ${STACK_NAME}_htpasswd_${HTPASSWD_CONFIG_VERSION}
+    file: loki.htpasswd.tmpl
   grafana_datasources_yml:
     name: ${STACK_NAME}_grafana_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION}
     file: grafana-datasources.yml
@@ -191,3 +218,6 @@ secrets:
   alertmanager_smtp_password:
     external: true
     name: ${STACK_NAME}_alertmanager_smtp_password_${SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION}
+  loki_admin_password:
+    external: true
+    name: ${STACK_NAME}_loki_admin_password_${SECRET_LOKI_ADMIN_PASSWORD_VERSION}

monitoring/env

@@ -4,6 +4,7 @@ STACK_NAME=gp_monitoring
 
 GRAFANA_DOMAIN=g.monitor.autonomic.zone
 PROMETHEUS_DOMAIN=p.monitor.autonomic.zone
+LOKI_DOMAIN=l.monitor.autonomic.zone
 
 LETS_ENCRYPT_ENV=production
 
@@ -42,6 +43,8 @@ GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v1
 GRAFANA_CUSTOM_INI_VERSION=v1
 PROMETHEUS_WEB_YML_VERSION=v1
 ALERTMANAGER_CONFIG_VERSION=v1
+NGINX_CONFIG_VERSION=v1
+HTPASSWD_CONFIG_VERSION=v1
 
 KEYCLOAK_AUTH_URL="https://id.autonomic.zone/auth/realms/autonomic/protocol/openid-connect/auth"
 KEYCLOAK_API_URL="https://id.autonomic.zone/auth/realms/autonomic/protocol/openid-connect/userinfo"
@@ -52,7 +55,8 @@ SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
 SECRET_GRAFANA_OAUTH_CLIENT_SECRET_VERSION=v1
 SECRET_PROMETHEUS_ADMIN_PASSWORD_VERSION=v1
 SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION=v1
+SECRET_LOKI_ADMIN_PASSWORD_VERSION=v1
 
-SMTP_FROM=noreply@autonomic.zone
+ALERTMANAGER_SMTP_FROM=noreply@autonomic.zone
-SMTP_HOST=mail.gandi.net:465
+ALERTMANAGER_SMTP_HOST=mail.gandi.net:465
-SMTP_TO=kaboom@autonomic.zone
+ALERTMANAGER_SMTP_TO=kaboom@autonomic.zone

monitoring/loki.htpasswd.tmpl

@@ -0,0 +1 @@
+loki:{{ secret "loki_admin_password" }}

monitoring/loki.yml.tmpl

@@ -20,16 +20,9 @@ ingester:
 
 memberlist:
   abort_if_cluster_join_fails: false
-
-  # Expose this port on all distributor, ingester
-  # and querier replicas.
   bind_port: 7946
-
-  # You can use a headless k8s service for all distributor,
-  # ingester and querier components.
   join_members:
   - loki:7946
-
   max_join_backoff: 1m
   max_join_retries: 10
   min_join_backoff: 1s

monitoring/nginx.conf.tmpl

@@ -0,0 +1,20 @@
+user www-data;
+
+events {
+  worker_connections 768;
+}
+
+http {
+  include /etc/nginx/mime.types;
+
+  server {
+    listen 80;
+    server_name {{ env "LOKI_DOMAIN" }};
+
+    location /loki/api/v1 {
+        auth_basic "loki";
+        auth_basic_user_file /etc/nginx/conf.d/loki.htpasswd;
+        proxy_pass http://{{ env "STACK_NAME" }}_loki:3100/loki/api/v1;
+    }
+  }
+}