Take a cleanup pass on generation script

This commit is contained in:
decentral1se 2021-04-02 20:40:31 +02:00
parent 55dc3a1d2a
commit 6f776a8c51
Signed by untrusted user who does not match committer: decentral1se
GPG Key ID: 92DAD76BD9567B8A

View File

@ -7,12 +7,14 @@
# https://abra-apps.cloud.autonomic.zone # https://abra-apps.cloud.autonomic.zone
from json import dump from json import dump
from logging import DEBUG, basicConfig, getLogger
from os import chdir, listdir, mkdir from os import chdir, listdir, mkdir
from os.path import exists, expanduser from os.path import exists, expanduser
from pathlib import Path from pathlib import Path
from re import findall, search from re import findall, search
from shlex import split from shlex import split
from subprocess import check_output, run from subprocess import check_output, run
from sys import exit
from requests import get from requests import get
@ -20,44 +22,74 @@ HOME_PATH = expanduser("~/")
CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute() CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
SCRIPT_PATH = Path(__file__).absolute().parent SCRIPT_PATH = Path(__file__).absolute().parent
log = getLogger(__name__)
basicConfig()
log.setLevel(DEBUG)
def clone_apps():
def _run_cmd(cmd, shell=False):
"""Run a shell command."""
args = [split(cmd)]
kwargs = {}
if shell:
args = [cmd]
kwargs = {"shell": shell}
try:
return check_output(*args, **kwargs).decode("utf-8").strip()
except Exception as exception:
log.error(f"Failed to run {cmd}, saw {str(exception)}")
exit(1)
def clone_all_apps():
"""Clone all Co-op Cloud apps to ~/.abra/apps."""
if not exists(CLONES_PATH): if not exists(CLONES_PATH):
mkdir(CLONES_PATH) mkdir(CLONES_PATH)
response = get("https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos")
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra") skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
log.info(f"Retrieving {url}")
try:
response = get(url, timeout=30)
except Exception as exception:
log.error(f"Failed to retrieve {url}, saw {str(exception)}")
exit(1)
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
for name, url in repos: for name, url in repos:
if name in skips: if name in skips:
continue continue
try:
if not exists(f"{CLONES_PATH}/{name}"): if not exists(f"{CLONES_PATH}/{name}"):
run(split(f"git clone {url} {CLONES_PATH}/{name}")) run(split(f"git clone {url} {CLONES_PATH}/{name}"))
chdir(f"{CLONES_PATH}/{name}") chdir(f"{CLONES_PATH}/{name}")
if not int(check_output("git branch --list | wc -l", shell=True)): if not int(_run_cmd("git branch --list | wc -l", shell=True)):
log.info(f"Guessing main branch is HEAD for {name}")
run(split("git checkout main")) run(split("git checkout main"))
except Exception:
pass
def gen_apps_json(): def generate_apps_json():
"""Generate the abra-apps.json application versions file."""
apps_json = {} apps_json = {}
for app in listdir(CLONES_PATH): for app in listdir(CLONES_PATH):
app_path = f"{CLONES_PATH}/{app}" app_path = f"{CLONES_PATH}/{app}"
chdir(app_path) chdir(app_path)
output = check_output("git tag --list", shell=True) tags = _run_cmd("git tag --list").split()
tags = output.decode("utf-8").strip().split()
if not tags: if not tags:
log.info(f"No tags discovered for {app}, moving on")
continue continue
for tag in tags: for tag in tags:
log.info(f"Processing {tag} for {app}")
apps_json[app] = { apps_json[app] = {
"category": "apps", "category": "apps",
"repository": f"https://git.autonomic.zone/coop-cloud/{app}.git", "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
@ -69,18 +101,19 @@ def gen_apps_json():
def get_app_features(app_path, tag): def get_app_features(app_path, tag):
print(f"Gathering features for {app_path}, tag: {tag}") """Parse features from app repo README files."""
features = {} features = {}
chdir(app_path) chdir(app_path)
run(f"git checkout {tag}", shell=True) _run_cmd(f"git checkout {tag}")
with open(f"{app_path}/README.md", "r") as handle: with open(f"{app_path}/README.md", "r") as handle:
log.info(f"{app_path}/README.md")
contents = handle.read() contents = handle.read()
for match in findall(r"\*\*.*\s\*", contents): for match in findall(r"\*\*.*\s\*", contents):
title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower() title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
if title == "image": if title == "image":
value = { value = {
"image": search(r"(?<=`).*(?=`)", match).group(), "image": search(r"(?<=`).*(?=`)", match).group(),
@ -90,49 +123,59 @@ def get_app_features(app_path, tag):
} }
else: else:
value = match.split(":")[-1].replace("*", "").strip() value = match.split(":")[-1].replace("*", "").strip()
features[title] = value features[title] = value
run(f"git checkout HEAD", shell=True) log.info(f"Parsed {features}")
_run_cmd("git checkout HEAD")
return features return features
def get_app_versions(app_path, tag): def get_app_versions(app_path, tag):
print(f"Gathering versions for {app_path}, tag: {tag}")
versions = [] versions = []
chdir(app_path) chdir(app_path)
run(f"git checkout {tag}", shell=True) _run_cmd(f"git checkout {tag}")
services_command = "yq e '.services | keys | .[]' compose*.yml"
services = check_output(services_command, shell=True).decode("utf-8").split()
services_cmd = "yq e '.services | keys | .[]' compose*.yml"
services = _run_cmd(services_cmd, shell=True).split()
for service in services: for service in services:
images_command = f"yq e '.services.{service}.image' compose*.yml" services_cmd = f"yq e '.services.{service}.image' compose*.yml"
images = check_output(images_command, shell=True).decode("utf-8").split() images = _run_cmd(services_cmd, shell=True).split()
for image in images: for image in images:
if image in ("null", "---"): if image in ("null", "---"):
continue continue
digest_command = f"skopeo inspect docker://{image} | jq '.Digest'" images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
output = check_output(digest_command, shell=True).decode("utf-8") output = _run_cmd(images_cmd, shell=True)
digest = output.strip().split(":")[-1][:8]
image_name = image.split(":")[0] new_version_info = {
versions.append( service: {
{service: {"image": image_name, "tag": tag, "digest": digest}} "image": image.split(":")[0],
) "tag": tag,
"digest": output.split(":")[-1][:8],
}
}
versions.append(new_version_info)
log.info(f"Parsed {new_version_info}")
_run_cmd("git checkout HEAD")
run(f"git checkout HEAD", shell=True)
return versions return versions
clone_apps() def main():
"""Run the script."""
clone_all_apps()
target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json" target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
with open(target, "w", encoding="utf-8") as handle:
dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
with open(target, "w", encoding="utf-8") as handle: log.info(f"Successfully generated {target}")
dump(gen_apps_json(), handle, ensure_ascii=False, indent=4)
print(f"Output saved to {target}")
main()