Re-work hacking from last night

autonomic/__main__.py

@@ -1,21 +1,13 @@
 """Command-line entrypoint."""
 
-import os
-
 import click
 
 from autonomic.command import actions, init
 
 
 @click.group()
-@click.option(
-    "--debug/--no-debug",
-    default=False,
-    help="Enable or disable debug mode",
-    show_default=True,
-)
 @click.pass_context
-def autonomic(ctx, debug):
+def autonomic(ctx):
     """
     \b
       ___        _                              _
@@ -25,9 +17,7 @@ def autonomic(ctx, debug):
     | | | | |_| | || (_) | | | | (_) | | | | | | | (__
     \_| |_/\__,_|\__\___/|_| |_|\___/|_| |_| |_|_|\___|
     """  # noqa
-    ctx.obj = {}
+    pass
-    ctx.obj["DEBUG"] = debug
-    ctx.obj["HOME"] = "/home/{user}".format(user=os.getlogin())
 
 
 autonomic.add_command(init.init)

autonomic/ansible.py

@@ -1,12 +0,0 @@
-"""Ansible management module."""
-
-from autonomic.settings import INFRASTRUCTURE_PATH
-from autonomic.system import run_command
-
-
-def run_ansible_playbook(play, extra_env=None):
-    """Run ansible-playbook against a play."""
-    command = [".venv/bin/ansible-playbook", play]
-    run_command(
-        command, cwd=INFRASTRUCTURE_PATH, output=True, extra_env=extra_env
-    )

autonomic/command/actions.py

@@ -1,63 +1,35 @@
-"""Ansible Actions module."""
+"""Actions module."""
+
+from os import environ
 
 import click
 from PyInquirer import prompt
 
-from autonomic import logger
+from autonomic.config import ACTIONS_DIR
-from autonomic.ansible import run_ansible_playbook
+from autonomic.infra import get_passwd, run_play
-from autonomic.passstore import get_from_pass
+from autonomic.utils import qlist
-from autonomic.settings import ACTIONS_PATH
-
-log = logger.get_logger(__name__)
 
 
 @click.command()
 @click.pass_context
 def actions(ctx):
     """Run an Ansible action."""
-    question = [
+    choices = ["addusers", "newhetzner", "rmhetzner", "newdokku"]
-        {
+    question = qlist("action", "Which Ansible action?", choices,)
-            "type": "list",
+    action = prompt(question)["action"]
-            "name": "action",
-            "message": "Which Ansible action do you want to run?",
-            "choices": [
-                {"name": "addusers"},
-                {"name": "newhetzner"},
-                {"name": "rmhetzner"},
-                {"name": "newdokku"},
-            ],
-            "filter": lambda val: val.lower(),
-        }
-    ]
-    action_answer = prompt(question)
 
-    env = {}
+    if action in any(("newhetzner", "rmhetzner")):
-    if (
+        choices = ["prod", "test", "cicd"]
-        action_answer["action"] == "newhetzner"
+        question = qlist("key", "Which Hetzner API key?", choices)
-        or action_answer["action"] == "rmhetzner"
+        key = prompt(question)["key"]
-    ):
-        question = [
-            {
-                "type": "list",
-                "name": "key",
-                "message": "Which Hetzner API key do you need?",
-                "choices": [
-                    {"name": "prod"},
-                    {"name": "test"},
-                    {"name": "cicd"},
-                ],
-                "filter": lambda val: val.lower(),
-            }
-        ]
 
-        key_answer = prompt(question)
+        path = "logins/hetzner/{}/api_key".format(key)
-        path = "logins/hetzner/{}/api_key".format(key_answer["key"])
+        secret = get_passwd(path)
-        secret = get_from_pass(path)
+        env = environ.copy()
         env["HCLOUD_TOKEN"] = secret
 
-    play = "{}/{}.yml".format(ACTIONS_PATH, action_answer["action"])
+    play = "{}/{}.yml".format(ACTIONS_DIR, action)
-
+    run_play(play, env=env)
-    run_ansible_playbook(play, extra_env=env)
 
     # TODO(decentral1se):
     # git commit and push on infrastructure if we:

autonomic/command/init.py

@@ -1,23 +1,18 @@
 """Initialise the toolbelt."""
 
-import os
+
+from os import mkdir
+from os.path import exists
 
 import click
-import emoji
+from emoji import emojize
 from PyInquirer import prompt
 
-from autonomic import logger
+from autonomic.config import CONFIG_DIR, CONFIG_YAML, INFRA_DIR, INFRA_REPO
-from autonomic.config import add_to_config
+from autonomic.infra import members
-from autonomic.infrastructure import get_members
+from autonomic.logger import log
-from autonomic.settings import (
+from autonomic.settings import add, get
-    AUTONOMIC_YAML,
+from autonomic.utils import qlist, run
-    CONFIG_PATH,
-    INFRASTRUCTURE_PATH,
-    INFRASTRUCTURE_REPOSITORY,
-)
-from autonomic.system import run_command
-
-log = logger.get_logger(__name__)
 
 
 @click.command()
@@ -25,71 +20,67 @@ log = logger.get_logger(__name__)
 def init(ctx):
     """Initialise the toolbelt."""
     create_configuration_directory()
-    create_configuration_file()
+    create_settings_file()
     clone_infrastructure_repo()
-    ask_to_login()
+    store_username()
     install_dependencies()
 
 
 def create_configuration_directory():
     """Create toolbelt config directory."""
-    if not os.path.exists(CONFIG_PATH):
+    if not exists(CONFIG_DIR):
-        os.mkdir(CONFIG_PATH)
+        mkdir(CONFIG_DIR)
 
 
 def clone_infrastructure_repo():
-    """Clone the infrastructure repository."""
+    """Clone or update the infrastructure repository."""
-    if not os.path.exists(INFRASTRUCTURE_PATH):
+    if not exists(INFRA_DIR):
-        run_command(
+        cmd = ["git", "clone", INFRA_REPO, INFRA_DIR]
-            ["git", "clone", INFRASTRUCTURE_REPOSITORY, INFRASTRUCTURE_PATH]
+        run(cmd)
-        )
     else:
-        run_command(
+        cmd = ["git", "pull", "origin", "master"]
-            ["git", "pull", "origin", "master"], cwd=INFRASTRUCTURE_PATH
+        run(cmd, cwd=INFRA_DIR)
-        )
 
 
-def ask_to_login():
+def store_username():
-    """Log in as your autonomic member username."""
+    """Store Autonomic username in the settings."""
-    members = get_members()
+    if exists(CONFIG_YAML):
-    choices = [info["username"] for info in members["autonomic_members"]]
+        username = get("username")
-    question = [
+        if username is not None:
-        {
+            msg = "Username already configured as {}".format(username)
-            "type": "list",
+            log.info(msg)
-            "name": "username",
+            return
-            "message": "What is your Autonomic username?",
-            "choices": choices,
-            "filter": lambda val: val.lower(),
-        }
-    ]
 
+    usernames = members(flatten="username")
+    question = qlist("username", "What is you Autonomic username?", usernames)
     answer = prompt(question)
-    add_to_config(answer)
+    add(answer)
 
     msg = "Welcome comrade {} :kissing:".format(answer["username"])
-    emojized = emoji.emojize(msg, use_aliases=True)
+    emojized = emojize(msg, use_aliases=True)
     log.info(emojized)
 
 
-def create_configuration_file():
+def create_settings_file():
-    """Create toolbelt config file."""
+    """Create settings file."""
-    if not os.path.exists(AUTONOMIC_YAML):
+    if not exists(CONFIG_YAML):
-        with open(AUTONOMIC_YAML, "w") as handle:
+        with open(CONFIG_YAML, "w") as handle:
             handle.write("---")
 
 
 def install_dependencies():
     """Install infrastructure dependencies."""
-    run_command(
+    cmd = ["/usr/bin/python3", "-m", "venv", ".venv"]
-        ["/usr/bin/python3", "-m", "venv", ".venv"], cwd=INFRASTRUCTURE_PATH,
+    run(cmd, cwd=INFRA_DIR)
-    )
 
-    run_command(
+    cmd = [".venv/bin/pip", "install", "-r", "requirements.txt"]
-        [".venv/bin/pip", "install", "-r", "requirements.txt"],
+    run(cmd, cwd=INFRA_DIR)
-        cwd=INFRASTRUCTURE_PATH,
-    )
 
-    run_command(
+    cmd = [
-        [".venv/bin/ansible-galaxy", "install", "-r", "requirements.yml"],
+        ".venv/bin/ansible-galaxy",
-        cwd=INFRASTRUCTURE_PATH,
+        "install",
-    )
+        "-r",
+        "requirements.yml",
+        "--force",
+    ]
+    run(cmd, cwd=INFRA_DIR)

autonomic/config.py

@@ -1,48 +1,32 @@
-"""Configuration handling module."""
+"""Tool configuration."""
 
 import os
 
-from autonomic import logger
+# current user
-from autonomic.settings import AUTONOMIC_YAML
+WHOAMI = os.getlogin()
-from autonomic.system import exit_with_msg
-from autonomic.yaml import yaml
 
-log = logger.get_logger(__name__)
+# home directory of the current user
+HOME_DIR = "/home/{}".format(WHOAMI)
 
+# configuration directory for the toolbelt
+CONFIG_DIR = "{}/.autonomic".format(HOME_DIR)
 
-def ensure_config():
+# SSH url for our infrastructure repository
-    """Ensure the configuration exists."""
+INFRA_REPO = (
-    if not os.path.exists(AUTONOMIC_YAML):
+    "ssh://git@git.autonomic.zone:222/autonomic-cooperative/infrastructure.git"
-        msg = "{} is missing, run: autonomic init".format(AUTONOMIC_YAML)
+)
-        exit_with_msg(msg)
 
+# infrastructure directory
+INFRA_DIR = "{}/infrastructure".format(CONFIG_DIR)
 
-def add_to_config(data):
+# list of autonomic members yaml file
-    """Add values to the autonomic.yml file."""
+MEMBERS_YAML = "{}/resources/members.yml".format(INFRA_DIR)
-    ensure_config()
 
-    with open(AUTONOMIC_YAML, "r") as handle:
+# directory where we store our ansible action plays
-        config = yaml.load(handle.read())
+ACTIONS_DIR = "{}/actions".format(INFRA_DIR)
 
-    if config is None:
+# toolbelt configuration file
-        config = {}
+CONFIG_YAML = "{}/autonomic.yml".format(CONFIG_DIR)
 
-    for key in data:
+# password store directory
-        config[key] = data[key]
+PASS_STORE_DIR = "{}/credentials/password-store".format(INFRA_DIR)
-
-    with open(AUTONOMIC_YAML, "w") as handle:
-        yaml.dump(config, handle)
-
-
-def get_from_config(key):
-    """Get values from the autonomic.yml file."""
-    ensure_config()
-
-    with open(AUTONOMIC_YAML, "r") as handle:
-        config = yaml.load(handle.read())
-
-    try:
-        return config[key]
-    except KeyError:
-        msg = "Unable to retrieve {} from {}".format(key, AUTONOMIC_YAML)
-        exit_with_msg(msg)

autonomic/infra.py

@@ -0,0 +1,45 @@
+"""Infrastructure module.
+
+All functionality that requires interacting with the infrastructure repository
+should live in here. This module relies on the file structure and conventions
+of the infrastructure repository directory layout. That can change without the
+code knowing, so the code must act more defensively than normal. This is an
+experiment.
+"""
+
+from os import environ
+
+from autonomic.config import INFRA_DIR, MEMBERS_YAML, PASS_STORE_DIR
+from autonomic.utils import run, yaml_load
+
+
+def members(flatten=None):
+    """The list of Autonomic members."""
+    members = yaml_load(MEMBERS_YAML)
+
+    if members is None:
+        return
+
+    if flatten:
+        return [info[flatten] for info in members["autonomic_members"]]
+
+    return members
+
+
+def run_play(play, env=None):
+    """Run an Ansible playbook."""
+    if env is None:
+        env = environ.copy()
+    cmd = [".venv/bin/ansible-playbook", play]
+    run(cmd, cwd=INFRA_DIR, env=env)
+
+
+def get_passwd(path):
+    """Lookup a password from the password store."""
+    env = environ.copy()
+    env.update({"PASSWORD_STORE_DIR": PASS_STORE_DIR})
+
+    cmd = ["pass", "show", path]
+    output = run(cmd, cwd=INFRA_DIR, env=env)
+
+    return output.decode("utf-8").strip()

autonomic/infrastructure.py

@@ -1,20 +0,0 @@
-"""Infrastructure handling module."""
-
-import os
-
-from autonomic.settings import MEMBERS_YAML
-from autonomic.system import exit_with_msg
-from autonomic.yaml import yaml
-
-
-def get_members():
-    """Get list of Autonomic members."""
-    if not os.path.exists(MEMBERS_YAML):
-        msg = "{} is missing, run: autonomic init".format(MEMBERS_YAML)
-        exit_with_msg(msg)
-
-    with open(MEMBERS_YAML, "r") as handle:
-        try:
-            return yaml.load(handle.read())
-        except Exception as exception:
-            exit_with_msg(str(exception))

autonomic/logger.py

@@ -1,4 +1,9 @@
-"""Logging Module."""
+"""Logging Module.
+
+This module is largely borrowed from Ansible Molecule and that is why it is
+largely uncommented since we didn't write it and luckily, didn't have to! This
+code just works TM. The same upstream license applies.
+"""
 
 import logging
 import os
@@ -140,3 +145,6 @@ def color_text(color, msg):
     if should_do_markup():
         return "{}{}{}".format(color, msg, colorama.Style.RESET_ALL)
     return msg
+
+
+log = get_logger(__name__)

autonomic/passstore.py

@@ -1,22 +0,0 @@
-"""Password store management module."""
-
-import os
-import subprocess
-
-from autonomic.settings import INFRASTRUCTURE_PATH, PASSWORD_STORE_PATH
-from autonomic.system import exit_with_msg
-
-
-def get_from_pass(path):
-    """Retrieve a password from the password store."""
-    env = os.environ.copy()
-    env.update({"PASSWORD_STORE_DIR": PASSWORD_STORE_PATH})
-
-    try:
-        os.chdir(INFRASTRUCTURE_PATH)
-        command = ["pass", "show", path]
-        output = subprocess.check_output(command, env=env)
-        return output.decode("utf-8").strip()
-    except subprocess.CalledProcessError as exception:
-        msg = "{} failed! Saw {}".format(" ".join(command), str(exception))
-        exit_with_msg(msg)

autonomic/settings.py

@@ -1,17 +1,32 @@
-"""Static settings."""
+"""Toolbelt settings module.
 
-import os
+All code related to working with the ~/.autonomic/settings.yml file. This file
+is useful for storing information for future lookup when dealing with
+repetitive cooperative tasks.
+"""
 
-USER = os.getlogin()
+from autonomic.config import CONFIG_YAML
-HOME = "/home/{}".format(USER)
+from autonomic.utils import yaml_dump, yaml_load
-CONFIG_PATH = "{}/.autonomic".format(HOME)
+
-INFRASTRUCTURE_REPOSITORY = (
+
-    "ssh://git@git.autonomic.zone:222/autonomic-cooperative/infrastructure.git"
+def add(data):
-)
+    """Add values to the settings file."""
-INFRASTRUCTURE_PATH = "{}/infrastructure".format(CONFIG_PATH)
+    settings = yaml_load(CONFIG_YAML)
-MEMBERS_YAML = "{}/resources/members.yml".format(INFRASTRUCTURE_PATH)
+
-ACTIONS_PATH = "{}/actions".format(INFRASTRUCTURE_PATH)
+    if settings is None:
-AUTONOMIC_YAML = "{}/autonomic.yml".format(CONFIG_PATH)
+        settings = {}
-PASSWORD_STORE_PATH = "{}/credentials/password-store".format(
+
-    INFRASTRUCTURE_PATH
+    for key in data:
-)
+        settings[key] = data[key]
+
+    yaml_dump(CONFIG_YAML, settings)
+
+
+def get(key):
+    """Retrieve setting values."""
+    settings = yaml_load(CONFIG_YAML)
+
+    try:
+        return settings[key]
+    except Exception:
+        return None

autonomic/system.py

@@ -1,53 +0,0 @@
-"""System related functions."""
-
-import os
-import subprocess
-import sys
-
-from autonomic import logger
-from autonomic.settings import PASSWORD_STORE_PATH
-
-log = logger.get_logger(__name__)
-
-
-def run_command(command, cwd=None, output=False, extra_env=None):
-    """Run a command."""
-    from autonomic.config import get_from_config
-
-    username = get_from_config("username")
-
-    env = os.environ.copy()
-    env.update({"PASSWORD_STORE_DIR": PASSWORD_STORE_PATH})
-    env.update({"REMOTE_USER": username})
-    env.update({"ANSIBLE_USER": username})
-
-    if extra_env:
-        for key, val in extra_env.items():
-            env.update({key: val})
-
-    try:
-        if cwd:
-            os.chdir(cwd)
-
-        log.info(
-            "Running {}{}".format(
-                " ".join(command), " in {}".format(cwd) if cwd else ""
-            )
-        )
-
-        if output:
-            subprocess.call(command, env=env)
-        else:
-            subprocess.check_output(command, env=env, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as exception:
-        msg = "{} failed! Saw {}".format(" ".join(command), str(exception))
-        exit_with_msg(msg)
-
-
-def exit(code=1):
-    sys.exit(code)
-
-
-def exit_with_msg(msg, code=1):
-    log.critical(msg)
-    exit(code)

autonomic/utils.py

@@ -0,0 +1,73 @@
+"""Utility functions.
+
+While mostly a vague concept and sometimes a bad idea, the utilities module is
+somewhere where reusable system related, data processing and other general
+functions can live. This codebase seems to have a few of those, so it seems
+like a necessary module. Let us be critical of our requirements when adding
+things here.
+"""
+
+from os import chdir
+from subprocess import CalledProcessError, check_output
+
+from autonomic.logger import log
+from autonomic.yaml import yaml
+
+
+def run(cmd, cwd=None, **kwargs):
+    """Run a system command.
+
+    Please note, all **kwargs will be passed into the check_output command so
+    that system call customisation can happen. Please name your keyword
+    arguments (like `cwd`) if you intend that they are used for other logic.
+    """
+    try:
+        if cwd:
+            chdir(cwd)
+            log.info("Changed directory to {}".format(cwd))
+        log.info("Running {}".format(" ".join(cmd)))
+        return check_output(cmd, **kwargs)
+    except CalledProcessError as exception:
+        msg = "{} failed! Saw {}".format(" ".join(cmd), str(exception))
+        exit(msg)
+
+
+def exit(msg, code=1):
+    """Exit and log appropriate level."""
+    if code != 0:
+        log.critical(msg)
+    else:
+        log.info(msg)
+
+    exit(code)
+
+
+def qlist(name, message, choices):
+    """A question in list format."""
+    return [
+        {
+            "type": "list",
+            "name": name,
+            "message": message,
+            "choices": choices,
+            "filter": lambda answer: answer.lower(),
+        }
+    ]
+
+
+def yaml_load(fpath):
+    """Load a YAML file."""
+    try:
+        with open(fpath, "r") as handle:
+            return yaml.load(handle.read())
+    except Exception as exception:
+        log.error(str(exception))
+
+
+def yaml_dump(fpath, data):
+    """Dump a YAML file."""
+    try:
+        with open(fpath, "w") as handle:
+            yaml.dump(data, handle)
+    except Exception as exception:
+        log.error(str(exception))

autonomic/yaml.py

@@ -1,6 +1,13 @@
-"""YAML initialisation module."""
+"""YAML initialisation module.
+
+Since we would like a consistent approach to how we load, manipulate and dump
+YAML, we centralise the approach into a single module to avoid duplicating all
+the customisation code across the code base.
+"""
 
 from ruamel.yaml import YAML
 
 yaml = YAML()
+
+# ensure that "---" is always at the top of the YAML file
 yaml.explicit_start = True

autonomic/yaml.yml

@@ -1,7 +0,0 @@
-"""YAML initialisation module."""
-
-from ruamel.yaml import YAML
-
-
-yaml = YAML()
-yaml.explicit_start = True