Convert to a django-list structure

spikes/README.md

@@ -0,0 +1,27 @@
+# Run It
+
+```bash
+$ python3 -m venv .venv
+$ source .venv/bin/activate
+$ pip install -r requirements.txt
+$ python first.py / second.py / third.py
+```
+
+## first.py
+
+- List apps from an `app.json` (points to https://git.autonomic.zone/compose-stacks)
+- Clone selected app template and parse configuration for inputs (env vars and secrets)
+- Generate a form so those values can be filled out and allow it to be saved
+- Save the form inputs to a `db.json` (as a start)
+- Deploy an applicaiton to a local swarm (assumes access to local docker socket)
+- Create an "edit app" page where the `db.json` is re-called and can be updated
+- Make sure re-deploy works (taking care of updating secret and app versions)
+
+## second.py
+
+- Don't try to be smart with the auto-generation, hard-code everything. We
+  maintain the app template (`compose.yml`) and this code anyway, so we just
+  need to be aware of each other and keep in sync. This would optimise for
+  trust and collaboration and not "smart" code.
+- Hard-code the secrets/configs required to make the code for generating
+  versions simpler as well.

spikes/first.py

@@ -0,0 +1,151 @@
+"""The all singing all dancing auto-generation from compose.yml attempt.
+
+Largely unfinished, just a stab in the dark.
+
+"""
+
+from json import loads
+from os.path import exists
+from pathlib import Path
+from re import findall
+from shlex import split
+from subprocess import run
+
+from flask import Flask, render_template, send_from_directory
+from flask_wtf import FlaskForm
+from ruamel.yaml import YAML
+from wtforms import StringField
+from wtforms.validators import DataRequired
+
+app = Flask(__name__)
+
+# Note(decentral1se): load from env vars at some point
+app.secret_key = b'_5#y2L"F4Q8z\n\xec]/'
+
+yaml = YAML()
+
+# Note(decentral1se): paths needs to change dependening
+# on deployment mode ./data in dev, /data in production
+DATA_DIR = Path("./data")
+
+STATIC_DIR = Path(".").absolute() / "static"
+
+
+class InstallAppInputForm(FlaskForm):
+    """Dynamically generated input form for app deploy."""
+
+    pass
+
+
+def get_apps_json():
+    """Retrieve the apps listing."""
+    global STATIC_DIR
+
+    try:
+        apps_path = STATIC_DIR / "apps.json"
+        with open(apps_path, "r") as handle:
+            return loads(handle.read())
+    except Exception as exception:
+        app.logger.error(f"Failed to load apps.json, saw {exception}")
+
+
+def clone_app_template(app_name):
+    """Git clone an app template repository."""
+    global DATA_DIR
+
+    try:
+        apps_json = get_apps_json()
+        app_repo_url = apps_json[app_name]
+    except KeyError:
+        app.logger.error(f"No repository for {app_name} found")
+        # Note(decentral1se): should catch this or make more specific error
+        raise RuntimeError()
+
+    try:
+        clone_path = DATA_DIR / app_name
+        command = f"git clone {app_repo_url} {clone_path}"
+        run(split(command))
+    except Exception as exception:
+        app.logger.error(f"Failed to run {command}, saw {exception}")
+        # Note(decentral1se): should catch this or make more specific error
+        raise RuntimeError()
+
+
+def get_env_vars(path):
+    """Get required env vars from a compose.yml."""
+    try:
+        compose_path = Path(path) / "compose.yml"
+        with open(compose_path, "r") as handle:
+            compose_yml = handle.read()
+    except Exception as exception:
+        app.logger.error(f"Failed to load {compose_path}, saw {exception}")
+        # Note(decentral1se): should catch this or make more specific error
+        raise RuntimeError()
+
+    # Note(decentral1se): matches all env vars (e.g ${FOO} and returns FOO)
+    return findall(r"\${(.*?)}", compose_yml)
+
+
+def get_secrets(path):
+    """Get required secrets from a compose.yml."""
+    try:
+        compose_path = Path(path) / "compose.yml"
+        with open(compose_path, "r") as handle:
+            compose_yml = yaml.load(handle.read())
+    except Exception as exception:
+        app.logger.error(f"Failed to load {compose_path}, saw {exception}")
+        # Note(decentral1se): should catch this or make more specific error
+        raise RuntimeError()
+
+    # Note(decentral1se): retrieves top level names like "db_passwd" not
+    # "gitea_db_passwd_v1" and I am unsure at this point which is the best to
+    # bring in but at least the plumbing is there
+    return [secret for secret in compose_yml["secrets"].keys()]
+
+
+def get_compose_inputs(path):
+    """Retrieve a list of all required inputs for stack deploy."""
+    return {
+        "env_vars": get_env_vars(path),
+        "secrets": get_secrets(path),
+    }
+
+
+@app.route("/")
+def home():
+    """Home page."""
+    apps = [app_name for app_name in get_apps_json()]
+    return render_template("first/index.html", apps=apps)
+
+
+@app.route("/apps")
+def apps_list():
+    """The JSON list of all apps."""
+    return send_from_directory("static", "apps.json")
+
+
+@app.route("/<app_name>")
+def install_app(app_name):
+    """Install an app"""
+    clone_app_template(app_name)
+
+    inputs = get_compose_inputs(DATA_DIR / app_name)
+
+    for env_var in inputs["env_vars"]:
+        setattr(InstallAppInputForm, env_var, StringField())
+
+    for secret in inputs["secrets"]:
+        setattr(InstallAppInputForm, secret, StringField())
+
+    form = InstallAppInputForm()
+
+    return render_template("first/install.html", app_name=app_name, form=form)
+
+
+@app.route("/deploy/<app_name>")
+def deploy_app(app_name):
+    return "TODO"
+
+
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0")

spikes/requirements.txt

@@ -0,0 +1,3 @@
+flask-wtf>=0.14.3,<0.15
+flask>=1.1.2,<2
+ruamel.yaml>=0.16.10,<0.17

spikes/second.py

@@ -0,0 +1,205 @@
+"""Less flashy version. More hard-coding."""
+
+from hashlib import md5
+from json import dumps, loads
+from os import environ, mkdir
+from os.path import exists
+from pathlib import Path
+from shlex import split
+from subprocess import run
+from typing import Dict
+
+from flask import Flask, render_template, request
+from flask_wtf import FlaskForm
+from ruamel.yaml import YAML
+from wtforms import PasswordField, StringField
+from wtforms.validators import URL, DataRequired, Length
+
+app = Flask(__name__)
+app.secret_key = b'_5#y2L"F4Q8z\n\xec]/'
+
+yaml = YAML()
+
+APPS_SPEC = {
+    "gitea": {
+        "url": "https://git.autonomic.zone/compose-stacks/gitea",
+        "secrets": [
+            "db_passwd",
+            "db_root_passwd",
+            "internal_token",
+            "jwt_secret",
+            "secret_key",
+        ],
+        "configs": ["app_ini"],
+    }
+}
+DATA_DIR = Path("./data")
+
+
+def clone_app_template(app_name: str) -> None:
+    """Git clone an app template repository."""
+    clone_path = DATA_DIR / app_name
+    clone_url = APPS_SPEC[app_name]["url"]
+    run(split(f"git clone {clone_url} {clone_path}"))
+
+
+def create_docker_secret(key: str, value: str) -> None:
+    """Load a docker secret into swarm."""
+    command = f"echo {value} | docker secret create {key} -"
+    run(command, shell=True)
+
+
+def dump_db(db: Dict) -> None:
+    """Dump the database."""
+    with open(DATA_DIR / "db.json", "w") as handle:
+        handle.write(dumps(db))
+
+
+def load_db() -> Dict:
+    """Load the database."""
+    db_path = DATA_DIR / "db.json"
+    if exists(db_path):
+        with open(db_path, "r") as handle:
+            return loads(handle.read())
+    return {}
+
+
+def get_hash(value: str) -> str:
+    """Hash a value for swarm versioning with good 'ol md5."""
+    hasher = md5()
+    hasher.update(value.encode())
+    return hasher.hexdigest()
+
+
+def arrange_configs_and_secrets(app_name, form_data, env):
+    """Version secrets and configs for swarm."""
+
+    def _create_versions(values, secrets=False):
+        """Step through listing and create an env var key/val.
+
+        Takes "app_ini" and produces {"APP_INI_VERSION": "app_ini_laksjdklajsdkla"}.
+        Hash is based on the value of the input passed in from the form.
+        """
+        for value in values:
+            if value in form_data:
+                hashed = get_hash(form_data[value])
+                env_key = f"{value.upper()}_VERSION"
+                env_val = f"{value}_{hashed}"
+                env[env_key] = env_val
+                if secrets:
+                    create_docker_secret(env_val, form_data[value])
+
+    _create_versions(APPS_SPEC[app_name]["configs"])
+    _create_versions(APPS_SPEC[app_name]["secrets"], secrets=True)
+
+    return env
+
+
+def stack_deploy(app_name, stack_name, env):
+    """Depoy an application to the swarm."""
+    compose_yml = DATA_DIR / app_name / "compose.yml"
+    command = f"docker stack deploy -c {compose_yml} {stack_name}"
+
+    # Note(decentral1se): we're not sure if this ever succeeds actually stuff
+    # like https://github.com/issuu/sure-deploy could be used and once we
+    # graduate this to a dockerise app, we could install that all in the
+    # container and run it from here. Of course, if we're using JS or not might
+    # affect this, say, we have a page that is real-time reporting or we're
+    # using a static page? Unsure what the UX could be like, thinking of
+    # cloudron loading bar but never like those...
+    run(split(command), env=env)
+
+
+def get_loaded_env(app_name, request_form):
+    """Load environment from install form for compose.yml injection."""
+    env = environ.copy()
+
+    for key in request_form.keys():
+        # Note(decentral1se): {"app_ini": "foo"} -> {"APP_INI": "foo"}
+        env[key.upper()] = request.form[key]
+
+    # Note(decentral1se): maybe we don't need this in the end since we will be
+    # quite sure that our configs are working and we are not getting rate
+    # limited
+    env["LETS_ENCRYPT_ENV"] = "staging"
+
+    return env
+
+
+class GiteaInstallForm(FlaskForm):
+    """Gitea installation form."""
+
+    app_name = StringField("Application name", default="Git with a cup of tea")
+    domain = StringField("Domain name", validators=[DataRequired()],)
+    stack_name = StringField("Stack name", default="magic-app-gitea")
+
+    db_host = StringField("Database host", default="mariadb:3306")
+    db_name = StringField("Database name", default="gitea")
+    db_passwd = PasswordField(
+        "Database password", validators=[DataRequired(), Length(min=32)],
+    )
+    db_root_passwd = PasswordField(
+        "Database root password", validators=[DataRequired(), Length(min=32)],
+    )
+    db_type = StringField("Database type", default="mysql")
+    db_user = StringField("Database user", default="mysql")
+    internal_token = PasswordField(
+        "Internal secret token", validators=[DataRequired(), Length(min=105)],
+    )
+    jwt_secret = PasswordField(
+        "JWT secret", validators=[DataRequired(), Length(min=43)],
+    )
+    secret_key = PasswordField(
+        "Secret key", validators=[DataRequired(), Length(min=64)],
+    )
+    ssh_host_port = StringField("SSH host port", default="2225")
+
+
+@app.route("/")
+def index():
+    """Home page for app installation possibilities."""
+    return render_template("second/index.html", apps=[app for app in APPS_SPEC])
+
+
+@app.route("/install/<app_name>")
+def install(app_name):
+    """Installation page for an app."""
+    if app_name == "gitea":
+        form = GiteaInstallForm()
+    return render_template("second/install.html", app_name=app_name, form=form)
+
+
+@app.route("/deploy/<app_name>", methods=["POST"])
+def deploy(app_name):
+    """Deployment end-point for an app."""
+    if app_name == "gitea":
+        form = GiteaInstallForm(request.form)
+
+    if not form.validate():
+        return render_template("second/install.html", app_name=app_name, form=form)
+
+    try:
+        mkdir(DATA_DIR)
+    except FileExistsError:
+        # Note(decentral1se): we already cloned it, so keep going
+        pass
+
+    clone_app_template(app_name)
+
+    # Note(decentral1se): there are badly named and the functions have too many
+    # concerns but they basically load every value required into the
+    # environment so that it can be injected when docker stack deploy is run,
+    # oh and secrets are also created as part of this step ;)
+    env = get_loaded_env(app_name, form.data)
+    env = arrange_configs_and_secrets(app_name, form.data, env)
+
+    stack_name = form.data["stack_name"]
+    dump_db({stack_name: form.data})
+    stack_deploy(app_name, stack_name, env)
+    return render_template(
+        "second/deploy.html", app_name=app_name, stack_name=stack_name
+    )
+
+
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0")

spikes/static/apps.json

@@ -0,0 +1,3 @@
+{
+  "gitea": "https://git.autonomic.zone/compose-stacks/gitea"
+}

spikes/templates/first/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Magic App</title>
+  </head>
+  <body>
+    <p>List of apps that can be installed:</p>
+    <ul>
+      {% for app in apps %}
+      <li><a href="/{{ app }}">{{ app }}</a></li>
+    </ul>
+    {% endfor %}
+  </body>
+</html>

spikes/templates/first/install.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Install {{ app_name }}</title>
+  </head>
+  <body>
+    <p>Install {{ app_name }}</p>
+    <form method="POST" action="/deploy">
+      {% for field in form %}
+        <div>
+        {{ field.label() }}
+        {{ field() }}
+        </div>
+      {% endfor %}
+      <input type="submit" value="Deploy" />
+    </form>
+  </body>
+</html>

spikes/templates/second/deploy.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Deployed {{ app_name }}</title>
+  </head>
+  <body>
+    <p>{{ app_name }} deployed as the {{ stack_name }} stack!</p>
+    <p>Not sure if it actually succeeded but hey, we got here at least!</p>
+  </body>
+</html>

spikes/templates/second/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Magic App</title>
+  </head>
+  <body>
+    <p>List of apps that can be installed:</p>
+    <ul>
+      {% for app in apps %}
+      <li><a href="/install/{{ app }}">{{ app }}</a></li>
+    </ul>
+    {% endfor %}
+  </body>
+</html>

spikes/templates/second/install.html

@@ -0,0 +1,20 @@
+{% from "second/macros.html" import with_errors %}
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Install {{ app_name }}</title>
+  </head>
+  <body>
+    <p>Install {{ app_name }}</p>
+    <form method="POST" action="/deploy/{{ app_name }}">
+      {% for field in form %}
+        {{ field.label() }}
+        {{ with_errors(field, style='font-weight: bold') }}
+      {% endfor %}
+      <input type="submit" value="Deploy" />
+    </form>
+  </body>
+</html>

spikes/templates/second/macros.html

@@ -0,0 +1,16 @@
+{#
+  From:  https://wtforms.readthedocs.io/en/2.3.x/specific_problems/#rendering-errors
+  Usage: with_errors(form.field, style='font-weight: bold')
+#}
+
+{% macro with_errors(field) %}
+    <div class="form_field">
+    {% if field.errors %}
+        {% set css_class = 'has_error ' + kwargs.pop('class', '') %}
+        {{ field(class=css_class, **kwargs) }}
+        <ul class="errors">{% for error in field.errors %}<li>{{ error|e }}</li>{% endfor %}</ul>
+    {% else %}
+        {{ field(**kwargs) }}
+    {% endif %}
+    </div>
+{% endmacro %}