autonomic-cooperative
/
autonomic
Archived
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Re-work hacking from last night

This commit is contained in:
Luke Murphy 2020-04-11 21:10:29 +02:00
parent c6608c5568
commit b58b9c3296
No known key found for this signature in database
GPG Key ID: 5E2EF5A63E3718CC
14 changed files with 253 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
autonomic/__main__.py
View File

@ -1,21 +1,13 @@
"""Command-line entrypoint."""
import os
import click
from autonomic.command import actions, init
@click.group()
@click.option(
"--debug/--no-debug",
default=False,
help="Enable or disable debug mode",
show_default=True,
)
@click.pass_context
def autonomic(ctx, debug):
def autonomic(ctx):
"""
\b
___ _ _
@ -25,9 +17,7 @@ def autonomic(ctx, debug):
| | | | |_| | || (_) | | | | (_) | | | | | | | (__
\_| |_/\__,_|\__\___/|_| |_|\___/|_| |_| |_|_|\___|
""" # noqa
ctx.obj = {}
ctx.obj["DEBUG"] = debug
ctx.obj["HOME"] = "/home/{user}".format(user=os.getlogin())
pass
autonomic.add_command(init.init)

12
autonomic/ansible.py
View File

@ -1,12 +0,0 @@
"""Ansible management module."""
from autonomic.settings import INFRASTRUCTURE_PATH
from autonomic.system import run_command
def run_ansible_playbook(play, extra_env=None):
"""Run ansible-playbook against a play."""
command = [".venv/bin/ansible-playbook", play]
run_command(
command, cwd=INFRASTRUCTURE_PATH, output=True, extra_env=extra_env
)

64
autonomic/command/actions.py
View File

@ -1,63 +1,35 @@
"""Ansible Actions module."""
"""Actions module."""
from os import environ
import click
from PyInquirer import prompt
from autonomic import logger
from autonomic.ansible import run_ansible_playbook
from autonomic.passstore import get_from_pass
from autonomic.settings import ACTIONS_PATH
log = logger.get_logger(__name__)
from autonomic.config import ACTIONS_DIR
from autonomic.infra import get_passwd, run_play
from autonomic.utils import qlist
@click.command()
@click.pass_context
def actions(ctx):
"""Run an Ansible action."""
question = [
{
"type": "list",
"name": "action",
"message": "Which Ansible action do you want to run?",
"choices": [
{"name": "addusers"},
{"name": "newhetzner"},
{"name": "rmhetzner"},
{"name": "newdokku"},
],
"filter": lambda val: val.lower(),
}
]
action_answer = prompt(question)
choices = ["addusers", "newhetzner", "rmhetzner", "newdokku"]
question = qlist("action", "Which Ansible action?", choices,)
action = prompt(question)["action"]
env = {}
if (
action_answer["action"] == "newhetzner"
or action_answer["action"] == "rmhetzner"
):
question = [
{
"type": "list",
"name": "key",
"message": "Which Hetzner API key do you need?",
"choices": [
{"name": "prod"},
{"name": "test"},
{"name": "cicd"},
],
"filter": lambda val: val.lower(),
}
]
if action in any(("newhetzner", "rmhetzner")):
choices = ["prod", "test", "cicd"]
question = qlist("key", "Which Hetzner API key?", choices)
key = prompt(question)["key"]
key_answer = prompt(question)
path = "logins/hetzner/{}/api_key".format(key_answer["key"])
secret = get_from_pass(path)
path = "logins/hetzner/{}/api_key".format(key)
secret = get_passwd(path)
env = environ.copy()
env["HCLOUD_TOKEN"] = secret
play = "{}/{}.yml".format(ACTIONS_PATH, action_answer["action"])
run_ansible_playbook(play, extra_env=env)
play = "{}/{}.yml".format(ACTIONS_DIR, action)
run_play(play, env=env)
# TODO(decentral1se):
# git commit and push on infrastructure if we:

103
autonomic/command/init.py
View File

@ -1,23 +1,18 @@
"""Initialise the toolbelt."""
import os
from os import mkdir
from os.path import exists
import click
import emoji
from emoji import emojize
from PyInquirer import prompt
from autonomic import logger
from autonomic.config import add_to_config
from autonomic.infrastructure import get_members
from autonomic.settings import (
AUTONOMIC_YAML,
CONFIG_PATH,
INFRASTRUCTURE_PATH,
INFRASTRUCTURE_REPOSITORY,
)
from autonomic.system import run_command
log = logger.get_logger(__name__)
from autonomic.config import CONFIG_DIR, CONFIG_YAML, INFRA_DIR, INFRA_REPO
from autonomic.infra import members
from autonomic.logger import log
from autonomic.settings import add, get
from autonomic.utils import qlist, run
@click.command()
@ -25,71 +20,67 @@ log = logger.get_logger(__name__)
def init(ctx):
"""Initialise the toolbelt."""
create_configuration_directory()
create_configuration_file()
create_settings_file()
clone_infrastructure_repo()
ask_to_login()
store_username()
install_dependencies()
def create_configuration_directory():
"""Create toolbelt config directory."""
if not os.path.exists(CONFIG_PATH):
os.mkdir(CONFIG_PATH)
if not exists(CONFIG_DIR):
mkdir(CONFIG_DIR)
def clone_infrastructure_repo():
"""Clone the infrastructure repository."""
if not os.path.exists(INFRASTRUCTURE_PATH):
run_command(
["git", "clone", INFRASTRUCTURE_REPOSITORY, INFRASTRUCTURE_PATH]
)
"""Clone or update the infrastructure repository."""
if not exists(INFRA_DIR):
cmd = ["git", "clone", INFRA_REPO, INFRA_DIR]
run(cmd)
else:
run_command(
["git", "pull", "origin", "master"], cwd=INFRASTRUCTURE_PATH
)
cmd = ["git", "pull", "origin", "master"]
run(cmd, cwd=INFRA_DIR)
def ask_to_login():
"""Log in as your autonomic member username."""
members = get_members()
choices = [info["username"] for info in members["autonomic_members"]]
question = [
{
"type": "list",
"name": "username",
"message": "What is your Autonomic username?",
"choices": choices,
"filter": lambda val: val.lower(),
}
]
def store_username():
"""Store Autonomic username in the settings."""
if exists(CONFIG_YAML):
username = get("username")
if username is not None:
msg = "Username already configured as {}".format(username)
log.info(msg)
return
usernames = members(flatten="username")
question = qlist("username", "What is you Autonomic username?", usernames)
answer = prompt(question)
add_to_config(answer)
add(answer)
msg = "Welcome comrade {} :kissing:".format(answer["username"])
emojized = emoji.emojize(msg, use_aliases=True)
emojized = emojize(msg, use_aliases=True)
log.info(emojized)
def create_configuration_file():
"""Create toolbelt config file."""
if not os.path.exists(AUTONOMIC_YAML):
with open(AUTONOMIC_YAML, "w") as handle:
def create_settings_file():
"""Create settings file."""
if not exists(CONFIG_YAML):
with open(CONFIG_YAML, "w") as handle:
handle.write("---")
def install_dependencies():
"""Install infrastructure dependencies."""
run_command(
["/usr/bin/python3", "-m", "venv", ".venv"], cwd=INFRASTRUCTURE_PATH,
)
cmd = ["/usr/bin/python3", "-m", "venv", ".venv"]
run(cmd, cwd=INFRA_DIR)
run_command(
[".venv/bin/pip", "install", "-r", "requirements.txt"],
cwd=INFRASTRUCTURE_PATH,
)
cmd = [".venv/bin/pip", "install", "-r", "requirements.txt"]
run(cmd, cwd=INFRA_DIR)
run_command(
[".venv/bin/ansible-galaxy", "install", "-r", "requirements.yml"],
cwd=INFRASTRUCTURE_PATH,
)
cmd = [
".venv/bin/ansible-galaxy",
"install",
"-r",
"requirements.yml",
"--force",
]
run(cmd, cwd=INFRA_DIR)

58
autonomic/config.py
View File

@ -1,48 +1,32 @@
"""Configuration handling module."""
"""Tool configuration."""
import os
from autonomic import logger
from autonomic.settings import AUTONOMIC_YAML
from autonomic.system import exit_with_msg
from autonomic.yaml import yaml
# current user
WHOAMI = os.getlogin()
log = logger.get_logger(__name__)
# home directory of the current user
HOME_DIR = "/home/{}".format(WHOAMI)
# configuration directory for the toolbelt
CONFIG_DIR = "{}/.autonomic".format(HOME_DIR)
def ensure_config():
"""Ensure the configuration exists."""
if not os.path.exists(AUTONOMIC_YAML):
msg = "{} is missing, run: autonomic init".format(AUTONOMIC_YAML)
exit_with_msg(msg)
# SSH url for our infrastructure repository
INFRA_REPO = (
"ssh://git@git.autonomic.zone:222/autonomic-cooperative/infrastructure.git"
)
# infrastructure directory
INFRA_DIR = "{}/infrastructure".format(CONFIG_DIR)
def add_to_config(data):
"""Add values to the autonomic.yml file."""
ensure_config()
# list of autonomic members yaml file
MEMBERS_YAML = "{}/resources/members.yml".format(INFRA_DIR)
with open(AUTONOMIC_YAML, "r") as handle:
config = yaml.load(handle.read())
# directory where we store our ansible action plays
ACTIONS_DIR = "{}/actions".format(INFRA_DIR)
if config is None:
config = {}
# toolbelt configuration file
CONFIG_YAML = "{}/autonomic.yml".format(CONFIG_DIR)
for key in data:
config[key] = data[key]
with open(AUTONOMIC_YAML, "w") as handle:
yaml.dump(config, handle)
def get_from_config(key):
"""Get values from the autonomic.yml file."""
ensure_config()
with open(AUTONOMIC_YAML, "r") as handle:
config = yaml.load(handle.read())
try:
return config[key]
except KeyError:
msg = "Unable to retrieve {} from {}".format(key, AUTONOMIC_YAML)
exit_with_msg(msg)
# password store directory
PASS_STORE_DIR = "{}/credentials/password-store".format(INFRA_DIR)

45
autonomic/infra.py Normal file
View File

@ -0,0 +1,45 @@
"""Infrastructure module.
All functionality that requires interacting with the infrastructure repository
should live in here. This module relies on the file structure and conventions
of the infrastructure repository directory layout. That can change without the
code knowing, so the code must act more defensively than normal. This is an
experiment.
"""
from os import environ
from autonomic.config import INFRA_DIR, MEMBERS_YAML, PASS_STORE_DIR
from autonomic.utils import run, yaml_load
def members(flatten=None):
"""The list of Autonomic members."""
members = yaml_load(MEMBERS_YAML)
if members is None:
return
if flatten:
return [info[flatten] for info in members["autonomic_members"]]
return members
def run_play(play, env=None):
"""Run an Ansible playbook."""
if env is None:
env = environ.copy()
cmd = [".venv/bin/ansible-playbook", play]
run(cmd, cwd=INFRA_DIR, env=env)
def get_passwd(path):
"""Lookup a password from the password store."""
env = environ.copy()
env.update({"PASSWORD_STORE_DIR": PASS_STORE_DIR})
cmd = ["pass", "show", path]
output = run(cmd, cwd=INFRA_DIR, env=env)
return output.decode("utf-8").strip()

20
autonomic/infrastructure.py
View File

@ -1,20 +0,0 @@
"""Infrastructure handling module."""
import os
from autonomic.settings import MEMBERS_YAML
from autonomic.system import exit_with_msg
from autonomic.yaml import yaml
def get_members():
"""Get list of Autonomic members."""
if not os.path.exists(MEMBERS_YAML):
msg = "{} is missing, run: autonomic init".format(MEMBERS_YAML)
exit_with_msg(msg)
with open(MEMBERS_YAML, "r") as handle:
try:
return yaml.load(handle.read())
except Exception as exception:
exit_with_msg(str(exception))

10
autonomic/logger.py
View File

@ -1,4 +1,9 @@
"""Logging Module."""
"""Logging Module.
This module is largely borrowed from Ansible Molecule and that is why it is
largely uncommented since we didn't write it and luckily, didn't have to! This
code just works TM. The same upstream license applies.
"""
import logging
import os
@ -140,3 +145,6 @@ def color_text(color, msg):
if should_do_markup():
return "{}{}{}".format(color, msg, colorama.Style.RESET_ALL)
return msg
log = get_logger(__name__)

22
autonomic/passstore.py
View File

@ -1,22 +0,0 @@
"""Password store management module."""
import os
import subprocess
from autonomic.settings import INFRASTRUCTURE_PATH, PASSWORD_STORE_PATH
from autonomic.system import exit_with_msg
def get_from_pass(path):
"""Retrieve a password from the password store."""
env = os.environ.copy()
env.update({"PASSWORD_STORE_DIR": PASSWORD_STORE_PATH})
try:
os.chdir(INFRASTRUCTURE_PATH)
command = ["pass", "show", path]
output = subprocess.check_output(command, env=env)
return output.decode("utf-8").strip()
except subprocess.CalledProcessError as exception:
msg = "{} failed! Saw {}".format(" ".join(command), str(exception))
exit_with_msg(msg)

45
autonomic/settings.py
View File

@ -1,17 +1,32 @@
"""Static settings."""
"""Toolbelt settings module.
import os
All code related to working with the ~/.autonomic/settings.yml file. This file
is useful for storing information for future lookup when dealing with
repetitive cooperative tasks.
"""
USER = os.getlogin()
HOME = "/home/{}".format(USER)
CONFIG_PATH = "{}/.autonomic".format(HOME)
INFRASTRUCTURE_REPOSITORY = (
"ssh://git@git.autonomic.zone:222/autonomic-cooperative/infrastructure.git"
)
INFRASTRUCTURE_PATH = "{}/infrastructure".format(CONFIG_PATH)
MEMBERS_YAML = "{}/resources/members.yml".format(INFRASTRUCTURE_PATH)
ACTIONS_PATH = "{}/actions".format(INFRASTRUCTURE_PATH)
AUTONOMIC_YAML = "{}/autonomic.yml".format(CONFIG_PATH)
PASSWORD_STORE_PATH = "{}/credentials/password-store".format(
INFRASTRUCTURE_PATH
)
from autonomic.config import CONFIG_YAML
from autonomic.utils import yaml_dump, yaml_load
def add(data):
"""Add values to the settings file."""
settings = yaml_load(CONFIG_YAML)
if settings is None:
settings = {}
for key in data:
settings[key] = data[key]
yaml_dump(CONFIG_YAML, settings)
def get(key):
"""Retrieve setting values."""
settings = yaml_load(CONFIG_YAML)
try:
return settings[key]
except Exception:
return None

53
autonomic/system.py
View File

@ -1,53 +0,0 @@
"""System related functions."""
import os
import subprocess
import sys
from autonomic import logger
from autonomic.settings import PASSWORD_STORE_PATH
log = logger.get_logger(__name__)
def run_command(command, cwd=None, output=False, extra_env=None):
"""Run a command."""
from autonomic.config import get_from_config
username = get_from_config("username")
env = os.environ.copy()
env.update({"PASSWORD_STORE_DIR": PASSWORD_STORE_PATH})
env.update({"REMOTE_USER": username})
env.update({"ANSIBLE_USER": username})
if extra_env:
for key, val in extra_env.items():
env.update({key: val})
try:
if cwd:
os.chdir(cwd)
log.info(
"Running {}{}".format(
" ".join(command), " in {}".format(cwd) if cwd else ""
)
)
if output:
subprocess.call(command, env=env)
else:
subprocess.check_output(command, env=env, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exception:
msg = "{} failed! Saw {}".format(" ".join(command), str(exception))
exit_with_msg(msg)
def exit(code=1):
sys.exit(code)
def exit_with_msg(msg, code=1):
log.critical(msg)
exit(code)

73
autonomic/utils.py Normal file
View File

@ -0,0 +1,73 @@
"""Utility functions.
While mostly a vague concept and sometimes a bad idea, the utilities module is
somewhere where reusable system related, data processing and other general
functions can live. This codebase seems to have a few of those, so it seems
like a necessary module. Let us be critical of our requirements when adding
things here.
"""
from os import chdir
from subprocess import CalledProcessError, check_output
from autonomic.logger import log
from autonomic.yaml import yaml
def run(cmd, cwd=None, **kwargs):
"""Run a system command.
Please note, all **kwargs will be passed into the check_output command so
that system call customisation can happen. Please name your keyword
arguments (like `cwd`) if you intend that they are used for other logic.
"""
try:
if cwd:
chdir(cwd)
log.info("Changed directory to {}".format(cwd))
log.info("Running {}".format(" ".join(cmd)))
return check_output(cmd, **kwargs)
except CalledProcessError as exception:
msg = "{} failed! Saw {}".format(" ".join(cmd), str(exception))
exit(msg)
def exit(msg, code=1):
"""Exit and log appropriate level."""
if code != 0:
log.critical(msg)
else:
log.info(msg)
exit(code)
def qlist(name, message, choices):
"""A question in list format."""
return [
{
"type": "list",
"name": name,
"message": message,
"choices": choices,
"filter": lambda answer: answer.lower(),
}
]
def yaml_load(fpath):
"""Load a YAML file."""
try:
with open(fpath, "r") as handle:
return yaml.load(handle.read())
except Exception as exception:
log.error(str(exception))
def yaml_dump(fpath, data):
"""Dump a YAML file."""
try:
with open(fpath, "w") as handle:
yaml.dump(data, handle)
except Exception as exception:
log.error(str(exception))

9
autonomic/yaml.py
View File

@ -1,6 +1,13 @@
"""YAML initialisation module."""
"""YAML initialisation module.
Since we would like a consistent approach to how we load, manipulate and dump
YAML, we centralise the approach into a single module to avoid duplicating all
the customisation code across the code base.
"""
from ruamel.yaml import YAML
yaml = YAML()
# ensure that "---" is always at the top of the YAML file
yaml.explicit_start = True

7
autonomic/yaml.yml
View File

@ -1,7 +0,0 @@
"""YAML initialisation module."""
from ruamel.yaml import YAML
yaml = YAML()
yaml.explicit_start = True