Take a cleanup pass on generation script
This commit is contained in:
decentral1se
parent
55dc3a1d2a
commit
6f776a8c51
GPG Key ID:
92DAD76BD9567B8A
125
bin/app-json.py
125
bin/app-json.py
@ -7,12 +7,14 @@
|
||||
# https://abra-apps.cloud.autonomic.zone
|
||||
|
||||
from json import dump
|
||||
from logging import DEBUG, basicConfig, getLogger
|
||||
from os import chdir, listdir, mkdir
|
||||
from os.path import exists, expanduser
|
||||
from pathlib import Path
|
||||
from re import findall, search
|
||||
from shlex import split
|
||||
from subprocess import check_output, run
|
||||
from sys import exit
|
||||
|
||||
from requests import get
|
||||
|
||||
@ -20,44 +22,74 @@ HOME_PATH = expanduser("~/")
|
||||
CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
|
||||
SCRIPT_PATH = Path(__file__).absolute().parent
|
||||
|
||||
log = getLogger(__name__)
|
||||
basicConfig()
|
||||
log.setLevel(DEBUG)
|
||||
|
||||
def clone_apps():
|
||||
|
||||
def _run_cmd(cmd, shell=False):
|
||||
"""Run a shell command."""
|
||||
args = [split(cmd)]
|
||||
kwargs = {}
|
||||
|
||||
if shell:
|
||||
args = [cmd]
|
||||
kwargs = {"shell": shell}
|
||||
|
||||
try:
|
||||
return check_output(*args, **kwargs).decode("utf-8").strip()
|
||||
except Exception as exception:
|
||||
log.error(f"Failed to run {cmd}, saw {str(exception)}")
|
||||
exit(1)
|
||||
|
||||
|
||||
def clone_all_apps():
|
||||
"""Clone all Co-op Cloud apps to ~/.abra/apps."""
|
||||
if not exists(CLONES_PATH):
|
||||
mkdir(CLONES_PATH)
|
||||
|
||||
response = get("https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos")
|
||||
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
|
||||
|
||||
skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
|
||||
url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
|
||||
|
||||
log.info(f"Retrieving {url}")
|
||||
|
||||
try:
|
||||
response = get(url, timeout=30)
|
||||
except Exception as exception:
|
||||
log.error(f"Failed to retrieve {url}, saw {str(exception)}")
|
||||
exit(1)
|
||||
|
||||
repos = [[p["name"], p["ssh_url"]] for p in response.json()]
|
||||
|
||||
for name, url in repos:
|
||||
if name in skips:
|
||||
continue
|
||||
|
||||
try:
|
||||
if not exists(f"{CLONES_PATH}/{name}"):
|
||||
run(split(f"git clone {url} {CLONES_PATH}/{name}"))
|
||||
chdir(f"{CLONES_PATH}/{name}")
|
||||
if not int(check_output("git branch --list | wc -l", shell=True)):
|
||||
run(split("git checkout main"))
|
||||
except Exception:
|
||||
pass
|
||||
if not exists(f"{CLONES_PATH}/{name}"):
|
||||
run(split(f"git clone {url} {CLONES_PATH}/{name}"))
|
||||
|
||||
chdir(f"{CLONES_PATH}/{name}")
|
||||
if not int(_run_cmd("git branch --list | wc -l", shell=True)):
|
||||
log.info(f"Guessing main branch is HEAD for {name}")
|
||||
run(split("git checkout main"))
|
||||
|
||||
|
||||
def gen_apps_json():
|
||||
def generate_apps_json():
|
||||
"""Generate the abra-apps.json application versions file."""
|
||||
apps_json = {}
|
||||
|
||||
for app in listdir(CLONES_PATH):
|
||||
app_path = f"{CLONES_PATH}/{app}"
|
||||
chdir(app_path)
|
||||
|
||||
output = check_output("git tag --list", shell=True)
|
||||
tags = output.decode("utf-8").strip().split()
|
||||
tags = _run_cmd("git tag --list").split()
|
||||
|
||||
if not tags:
|
||||
log.info(f"No tags discovered for {app}, moving on")
|
||||
continue
|
||||
|
||||
for tag in tags:
|
||||
log.info(f"Processing {tag} for {app}")
|
||||
apps_json[app] = {
|
||||
"category": "apps",
|
||||
"repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
|
||||
@ -69,18 +101,19 @@ def gen_apps_json():
|
||||
|
||||
|
||||
def get_app_features(app_path, tag):
|
||||
print(f"Gathering features for {app_path}, tag: {tag}")
|
||||
|
||||
"""Parse features from app repo README files."""
|
||||
features = {}
|
||||
|
||||
chdir(app_path)
|
||||
run(f"git checkout {tag}", shell=True)
|
||||
_run_cmd(f"git checkout {tag}")
|
||||
|
||||
with open(f"{app_path}/README.md", "r") as handle:
|
||||
log.info(f"{app_path}/README.md")
|
||||
contents = handle.read()
|
||||
|
||||
for match in findall(r"\*\*.*\s\*", contents):
|
||||
title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
|
||||
|
||||
if title == "image":
|
||||
value = {
|
||||
"image": search(r"(?<=`).*(?=`)", match).group(),
|
||||
@ -90,49 +123,59 @@ def get_app_features(app_path, tag):
|
||||
}
|
||||
else:
|
||||
value = match.split(":")[-1].replace("*", "").strip()
|
||||
|
||||
features[title] = value
|
||||
|
||||
run(f"git checkout HEAD", shell=True)
|
||||
log.info(f"Parsed {features}")
|
||||
|
||||
_run_cmd("git checkout HEAD")
|
||||
|
||||
return features
|
||||
|
||||
|
||||
def get_app_versions(app_path, tag):
|
||||
print(f"Gathering versions for {app_path}, tag: {tag}")
|
||||
|
||||
versions = []
|
||||
|
||||
chdir(app_path)
|
||||
run(f"git checkout {tag}", shell=True)
|
||||
|
||||
services_command = "yq e '.services | keys | .[]' compose*.yml"
|
||||
services = check_output(services_command, shell=True).decode("utf-8").split()
|
||||
_run_cmd(f"git checkout {tag}")
|
||||
|
||||
services_cmd = "yq e '.services | keys | .[]' compose*.yml"
|
||||
services = _run_cmd(services_cmd, shell=True).split()
|
||||
for service in services:
|
||||
images_command = f"yq e '.services.{service}.image' compose*.yml"
|
||||
images = check_output(images_command, shell=True).decode("utf-8").split()
|
||||
|
||||
services_cmd = f"yq e '.services.{service}.image' compose*.yml"
|
||||
images = _run_cmd(services_cmd, shell=True).split()
|
||||
for image in images:
|
||||
if image in ("null", "---"):
|
||||
continue
|
||||
|
||||
digest_command = f"skopeo inspect docker://{image} | jq '.Digest'"
|
||||
output = check_output(digest_command, shell=True).decode("utf-8")
|
||||
digest = output.strip().split(":")[-1][:8]
|
||||
images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
|
||||
output = _run_cmd(images_cmd, shell=True)
|
||||
|
||||
image_name = image.split(":")[0]
|
||||
versions.append(
|
||||
{service: {"image": image_name, "tag": tag, "digest": digest}}
|
||||
)
|
||||
new_version_info = {
|
||||
service: {
|
||||
"image": image.split(":")[0],
|
||||
"tag": tag,
|
||||
"digest": output.split(":")[-1][:8],
|
||||
}
|
||||
}
|
||||
|
||||
versions.append(new_version_info)
|
||||
log.info(f"Parsed {new_version_info}")
|
||||
|
||||
_run_cmd("git checkout HEAD")
|
||||
|
||||
run(f"git checkout HEAD", shell=True)
|
||||
return versions
|
||||
|
||||
|
||||
clone_apps()
|
||||
def main():
|
||||
"""Run the script."""
|
||||
clone_all_apps()
|
||||
|
||||
target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
|
||||
target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
|
||||
with open(target, "w", encoding="utf-8") as handle:
|
||||
dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
|
||||
|
||||
with open(target, "w", encoding="utf-8") as handle:
|
||||
dump(gen_apps_json(), handle, ensure_ascii=False, indent=4)
|
||||
log.info(f"Successfully generated {target}")
|
||||
|
||||
print(f"Output saved to {target}")
|
||||
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user