Untested port of peertube to coop-cloud

missing:

 * live streaming and RTMP proxying with traefik
 * webseed cross-instance redundancy
 * option to use S3-compatible storage
 * testing, there might be silly bugs

Did I mention this is untested?

.env.sample

@@ -4,3 +4,15 @@ DOMAIN=peertube.example.com
 ## Domain aliases
 #EXTRA_DOMAINS=', `www.peertube.example.com`'
 LETS_ENCRYPT_ENV=production
+
+PEERTUBE_WEBSERVER_PORT: 443
+PEERTUBE_WEBSERVER_HTTPS: "true"
+PEERTUBE_ADMIN_EMAIL: admin@example.com
+PEERTUBE_SIGNUP_ENABLED: "false"
+PEERTUBE_TRANSCODING_ENABLED: "true"
+PEERTUBE_SMTP_HOSTNAME: smtpd
+PEERTUBE_SMTP_PORT: 25
+PEERTUBE_SMTP_FROM: peertube@example.com
+PEERTUBE_SMTP_TLS: "false"
+PEERTUBE_SMTP_DISABLE_STARTTLS: "true"
+PEERTUBE_TRUST_PROXY: '["127.0.0.1", "loopback", "172.16.0.0/12", "10.0.0.0/8"]'

compose.yml

@@ -2,10 +2,21 @@
 version: "3.8"
 
 services:
-  app:
+  nginx:
     image: nginx:1.19.2
     networks:
       - proxy
+      - app
+    volumes:
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+      - app-data:/var/www/peertube/storage:ro
+      - app-assets:/var/www/peertube/peertube-latest/client/dist:ro
+      - nginx-tmp:/var/lib/nginx/client-tmp:rw
+      - nginx-cache:/var/cache/nginx/peertube-media-cache:rw
+    configs:
+      - source: nginx_config
+        target: /etc/nginx/conf.d/default.conf
     deploy:
       restart_policy:
         condition: on-failure
@@ -19,13 +30,101 @@ services:
         #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
+    depends_on:
+      - app
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost"]
       interval: 30s
       timeout: 10s
       retries: 10
+      start_period: 10s
+
+  app:
+    image: chocobozzz/peertube:v3.1.0-rc.1-buster
+    environment:
+      PEERTUBE_WEBSERVER_HOSTNAME: ${DOMAIN}
+      PEERTUBE_DB_HOSTNAME: db
+      PEERTUBE_DB_USERNAME: peertube
+      PEERTUBE_DB_PASSWORD_FILE: /run/secrets/db_password
+      PEERTUBE_REDIS_HOSTNAME: redis
+    volumes:
+      - app-data:/data
+      - app-config:/config
+      - app-assets:/app/client/dist
+    secrets:
+      - db_password
+    configs:
+      - source: entrypoint_conf
+        target: /docker-entrypoint.sh
+        mode: 0555
+    entrypoint: /docker-entrypoint.sh
+    networks:
+      app:
+      postfix_smtp:
+      redis:
+      db:
+    depends_on:
+      - db
+      - redis
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
       start_period: 1m
 
+  db:
+    image: postgres:10-alpine
+    environment:
+      POSTGRES_USER: peertube
+      POSTGRES_PASSWORD_FILE: /run/secrets/db_password
+      POSTGRES_DB: peertube
+    secrets:
+      - db_password
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    networks:
+      db:
+
+  redis:
+    image: redis:4-alpine
+    volumes:
+      - redis-data:/data
+    networks:
+      redis:
+
 networks:
   proxy:
     external: true
+  app:
+    driver: overlay
+    attachable: true
+  redis:
+    driver: overlay
+    attachable: true
+  db:
+    driver: overlay
+    attachable: true
+
+volumes:
+  app-data:
+  app-config:
+  app-assets:
+  nginx-tmp:
+  nginx-cache:
+  postgres-data:
+  redis-data:
+
+configs:
+  nginx_config:
+    name: ${STACK_NAME}_nginx_config_${NGINX_CONFIG_VERSION}
+    file: nginx.conf.tmpl
+    template_driver: golang
+  app_entrypoint:
+    name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION}
+    file: entrypoint.sh
+
+secrets:
+  db_password:
+    external: true
+    name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+file_env() {
+   local var="$1"
+   local fileVar="${var}_FILE"
+   local def="${2:-}"
+
+   if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+      echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+      exit 1
+   fi
+   local val="$def"
+   if [ "${!var:-}" ]; then
+      val="${!var}"
+   elif [ "${!fileVar:-}" ]; then
+      val="$(< "${!fileVar}")"
+   fi
+   export "$var"="$val"
+   unset "$fileVar"
+}
+
+file_env "PEERTUBE_DB_PASSWORD"
+
+exec /entrypoint.sh "$@"

nginx.conf.tmpl

@@ -0,0 +1,234 @@
+upstream backend {
+  server {{ env "STACK_NAME" }}_app:9000;
+}
+
+proxy_cache_path /var/cache/nginx/peertube-media-cache levels=1:2 keys_zone=peertube_media_cache:5m max_size=40g inactive=72h use_temp_path=off;
+
+server {
+  listen 80;
+  server_name _;
+
+  ##
+  # Application
+  ##
+
+  location @api {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host            $host;
+
+    client_max_body_size  100k; # default is 1M
+
+    proxy_connect_timeout 10m;
+    proxy_send_timeout    10m;
+    proxy_read_timeout    10m;
+    send_timeout          10m;
+
+    proxy_pass http://backend;
+  }
+
+  location / {
+    try_files /dev/null @api;
+  }
+
+  location = /api/v1/videos/upload {
+    limit_except POST HEAD { deny all; }
+
+    # This is the maximum upload size, which roughly matches the maximum size of a video file.
+    # Note that temporary space is needed equal to the total size of all concurrent uploads.
+    # This data gets stored in /var/lib/nginx by default, so you may want to put this directory
+    # on a dedicated filesystem.
+    client_max_body_size                      12G; # default is 1M
+    add_header            X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
+
+    try_files /dev/null @api;
+  }
+
+  location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) {
+    client_max_body_size                      3M; # default is 1M
+    add_header            X-File-Maximum-Size 2M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size)
+
+    try_files /dev/null @api;
+  }
+
+  ##
+  # Websocket
+  ##
+
+  location @api_websocket {
+    proxy_http_version 1.1;
+    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header   Host            $host;
+    proxy_set_header   X-Real-IP       $remote_addr;
+    proxy_set_header   Upgrade         $http_upgrade;
+    proxy_set_header   Connection      "upgrade";
+
+    proxy_pass http://backend;
+  }
+
+  location /socket.io {
+    try_files /dev/null @api_websocket;
+  }
+
+  location /tracker/socket {
+    # Peers send a message to the tracker every 15 minutes
+    # Don't close the websocket before then
+    proxy_read_timeout 15m; # default is 60s
+
+    try_files /dev/null @api_websocket;
+  }
+
+  ##
+  # Performance optimizations
+  # For extra performance please refer to https://github.com/denji/nginx-tuning
+  ##
+
+  root /var/www/peertube/storage;
+
+  # Enable compression for JS/CSS/HTML, for improved client load times.
+  # It might be nice to compress JSON/XML as returned by the API, but
+  # leaving that out to protect against potential BREACH attack.
+  gzip              on;
+  gzip_vary         on;
+  gzip_types        # text/html is always compressed by HttpGzipModule
+                    text/css
+                    application/javascript
+                    font/truetype
+                    font/opentype
+                    application/vnd.ms-fontobject
+                    image/svg+xml;
+  gzip_min_length   1000; # default is 20 bytes
+  gzip_buffers      16 8k;
+  gzip_comp_level   2; # default is 1
+
+  client_body_timeout       30s; # default is 60
+  client_header_timeout     10s; # default is 60
+  send_timeout              10s; # default is 60
+  keepalive_timeout         10s; # default is 75
+  resolver_timeout          10s; # default is 30
+  reset_timedout_connection on;
+  proxy_ignore_client_abort on;
+
+  tcp_nopush                on; # send headers in one piece
+  tcp_nodelay               on; # don't buffer data sent, good for small data bursts in real time
+
+  # If you have a small /var/lib partition, it could be interesting to store temp nginx uploads in a different place
+  # See https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_temp_path
+  client_body_temp_path /var/lib/nginx/client-tmp;
+
+  # Bypass PeerTube for performance reasons. Optional.
+  # Should be consistent with client-overrides assets list in /server/controllers/client.ts
+  location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png))$ {
+    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year
+
+    root /var/www/peertube;
+
+    try_files /storage/client-overrides/$1 /peertube-latest/client/dist/$1 @api;
+  }
+
+  # Bypass PeerTube for performance reasons. Optional.
+  location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ {
+    add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year
+
+    alias /var/www/peertube/peertube-latest/client/dist/$1;
+  }
+
+  # Bypass PeerTube for performance reasons. Optional.
+  location ~ ^/static/(thumbnails|avatars)/ {
+    if ($request_method = 'OPTIONS') {
+      add_header Access-Control-Allow-Origin  '*';
+      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
+      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
+      add_header Content-Type                 'text/plain charset=UTF-8';
+      add_header Content-Length               0;
+      return 204;
+    }
+
+    add_header Access-Control-Allow-Origin    '*';
+    add_header Access-Control-Allow-Methods   'GET, OPTIONS';
+    add_header Access-Control-Allow-Headers   'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+    add_header Cache-Control                  "public, max-age=7200"; # Cache response 2 hours
+
+    rewrite ^/static/(.*)$ /$1 break;
+
+    try_files $uri @api;
+  }
+
+  # Bypass PeerTube for performance reasons. Optional.
+  location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {
+    limit_rate_after            5M;
+
+    # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client
+    set $peertube_limit_rate    800k;
+
+    # Increase rate limit in HLS mode, because we don't have multiple simultaneous connections
+    if ($request_uri ~ -fragmented.mp4$) {
+      set $peertube_limit_rate  5M;
+    }
+
+    # Use this line with nginx >= 1.17.0
+    limit_rate $peertube_limit_rate;
+    # Or this line if your nginx < 1.17.0
+    #set $limit_rate $peertube_limit_rate;
+
+    if ($request_method = 'OPTIONS') {
+      add_header Access-Control-Allow-Origin  '*';
+      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
+      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+      add_header Access-Control-Max-Age       1728000; # Preflight request can be cached 20 days
+      add_header Content-Type                 'text/plain charset=UTF-8';
+      add_header Content-Length               0;
+      return 204;
+    }
+
+    if ($request_method = 'GET') {
+      add_header Access-Control-Allow-Origin  '*';
+      add_header Access-Control-Allow-Methods 'GET, OPTIONS';
+      add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+      add_header X-Cache-Status $upstream_cache_status;
+      add_header X-Cache-Range $slice_range;
+      add_header X-Fuuu ok;
+
+      add_header X-Accel-Expires 604800;
+
+      # Don't spam access log file with byte range requests
+      access_log off;
+    }
+
+    # Enabling the sendfile directive eliminates the step of copying the data into the buffer
+    # and enables direct copying data from one file descriptor to another.
+    sendfile on;
+    sendfile_max_chunk 1M; # prevent one fast connection from entirely occupying the worker process. should be > 800k.
+    aio threads;
+
+    proxy_connect_timeout       70;
+    proxy_read_timeout          1200;
+    proxy_send_timeout          1200;
+
+    proxy_buffering on;
+    proxy_buffers 8 128k;
+    proxy_cache peertube_media_cache;
+
+    slice              64k;
+    proxy_cache_key    $host$uri$is_args$args$slice_range;
+    proxy_set_header   Range $slice_range;
+    proxy_cache_valid  200 206 1h;
+    proxy_cache_revalidate on;
+
+    proxy_http_version 1.1;
+    proxy_redirect off;
+
+
+    # Use this in tandem with fuse-mounting i.e. https://docs.joinpeertube.org/admin-remote-storage
+    # to serve files directly from a public bucket without proxying.
+    # Assumes you have buckets named after the storage subdirectories, i.e. 'videos', 'redundancy', etc.
+    #set $cdn https://bucket-name.cdn.cloud;
+    #rewrite ^/static/webseed/(.*)$ $cdn/videos/$1 redirect;
+    #rewrite ^/static/(.*)$         $cdn/$1        redirect;
+    rewrite ^/static/webseed/(.*)$ /videos/$1 break;
+    rewrite ^/static/(.*)$         /$1        break;
+
+    try_files $uri @api;
+    #proxy_pass https://bucket-name.s3.api-endpoint.cloud;
+  }
+}