Take a cleanup pass on generation script

This commit is contained in:
decentral1se 2021-04-02 20:40:31 +02:00
parent 55dc3a1d2a
commit 6f776a8c51
Signed by untrusted user who does not match committer: decentral1se
GPG Key ID: 92DAD76BD9567B8A

View File

@ -7,12 +7,14 @@
# https://abra-apps.cloud.autonomic.zone
from json import dump
from logging import DEBUG, basicConfig, getLogger
from os import chdir, listdir, mkdir
from os.path import exists, expanduser
from pathlib import Path
from re import findall, search
from shlex import split
from subprocess import check_output, run
from sys import exit
from requests import get
@ -20,44 +22,74 @@ HOME_PATH = expanduser("~/")
CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
SCRIPT_PATH = Path(__file__).absolute().parent
log = getLogger(__name__)
basicConfig()
log.setLevel(DEBUG)
def clone_apps():
def _run_cmd(cmd, shell=False):
"""Run a shell command."""
args = [split(cmd)]
kwargs = {}
if shell:
args = [cmd]
kwargs = {"shell": shell}
try:
return check_output(*args, **kwargs).decode("utf-8").strip()
except Exception as exception:
log.error(f"Failed to run {cmd}, saw {str(exception)}")
exit(1)
def clone_all_apps():
"""Clone all Co-op Cloud apps to ~/.abra/apps."""
if not exists(CLONES_PATH):
mkdir(CLONES_PATH)
response = get("https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos")
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
log.info(f"Retrieving {url}")
try:
response = get(url, timeout=30)
except Exception as exception:
log.error(f"Failed to retrieve {url}, saw {str(exception)}")
exit(1)
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
for name, url in repos:
if name in skips:
continue
try:
if not exists(f"{CLONES_PATH}/{name}"):
run(split(f"git clone {url} {CLONES_PATH}/{name}"))
chdir(f"{CLONES_PATH}/{name}")
if not int(check_output("git branch --list | wc -l", shell=True)):
run(split("git checkout main"))
except Exception:
pass
if not exists(f"{CLONES_PATH}/{name}"):
run(split(f"git clone {url} {CLONES_PATH}/{name}"))
chdir(f"{CLONES_PATH}/{name}")
if not int(_run_cmd("git branch --list | wc -l", shell=True)):
log.info(f"Guessing main branch is HEAD for {name}")
run(split("git checkout main"))
def gen_apps_json():
def generate_apps_json():
"""Generate the abra-apps.json application versions file."""
apps_json = {}
for app in listdir(CLONES_PATH):
app_path = f"{CLONES_PATH}/{app}"
chdir(app_path)
output = check_output("git tag --list", shell=True)
tags = output.decode("utf-8").strip().split()
tags = _run_cmd("git tag --list").split()
if not tags:
log.info(f"No tags discovered for {app}, moving on")
continue
for tag in tags:
log.info(f"Processing {tag} for {app}")
apps_json[app] = {
"category": "apps",
"repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
@ -69,18 +101,19 @@ def gen_apps_json():
def get_app_features(app_path, tag):
print(f"Gathering features for {app_path}, tag: {tag}")
"""Parse features from app repo README files."""
features = {}
chdir(app_path)
run(f"git checkout {tag}", shell=True)
_run_cmd(f"git checkout {tag}")
with open(f"{app_path}/README.md", "r") as handle:
log.info(f"{app_path}/README.md")
contents = handle.read()
for match in findall(r"\*\*.*\s\*", contents):
title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
if title == "image":
value = {
"image": search(r"(?<=`).*(?=`)", match).group(),
@ -90,49 +123,59 @@ def get_app_features(app_path, tag):
}
else:
value = match.split(":")[-1].replace("*", "").strip()
features[title] = value
run(f"git checkout HEAD", shell=True)
log.info(f"Parsed {features}")
_run_cmd("git checkout HEAD")
return features
def get_app_versions(app_path, tag):
print(f"Gathering versions for {app_path}, tag: {tag}")
versions = []
chdir(app_path)
run(f"git checkout {tag}", shell=True)
services_command = "yq e '.services | keys | .[]' compose*.yml"
services = check_output(services_command, shell=True).decode("utf-8").split()
_run_cmd(f"git checkout {tag}")
services_cmd = "yq e '.services | keys | .[]' compose*.yml"
services = _run_cmd(services_cmd, shell=True).split()
for service in services:
images_command = f"yq e '.services.{service}.image' compose*.yml"
images = check_output(images_command, shell=True).decode("utf-8").split()
services_cmd = f"yq e '.services.{service}.image' compose*.yml"
images = _run_cmd(services_cmd, shell=True).split()
for image in images:
if image in ("null", "---"):
continue
digest_command = f"skopeo inspect docker://{image} | jq '.Digest'"
output = check_output(digest_command, shell=True).decode("utf-8")
digest = output.strip().split(":")[-1][:8]
images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
output = _run_cmd(images_cmd, shell=True)
image_name = image.split(":")[0]
versions.append(
{service: {"image": image_name, "tag": tag, "digest": digest}}
)
new_version_info = {
service: {
"image": image.split(":")[0],
"tag": tag,
"digest": output.split(":")[-1][:8],
}
}
versions.append(new_version_info)
log.info(f"Parsed {new_version_info}")
_run_cmd("git checkout HEAD")
run(f"git checkout HEAD", shell=True)
return versions
clone_apps()
def main():
"""Run the script."""
clone_all_apps()
target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
with open(target, "w", encoding="utf-8") as handle:
dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
with open(target, "w", encoding="utf-8") as handle:
dump(gen_apps_json(), handle, ensure_ascii=False, indent=4)
log.info(f"Successfully generated {target}")
print(f"Output saved to {target}")
main()