Take a cleanup pass on generation script

bin/app-json.py

@@ -7,12 +7,14 @@
 # https://abra-apps.cloud.autonomic.zone
 
 from json import dump
+from logging import DEBUG, basicConfig, getLogger
 from os import chdir, listdir, mkdir
 from os.path import exists, expanduser
 from pathlib import Path
 from re import findall, search
 from shlex import split
 from subprocess import check_output, run
+from sys import exit
 
 from requests import get
 
@@ -20,44 +22,74 @@ HOME_PATH = expanduser("~/")
 CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
 SCRIPT_PATH = Path(__file__).absolute().parent
 
+log = getLogger(__name__)
+basicConfig()
+log.setLevel(DEBUG)
 
-def clone_apps():
+
+def _run_cmd(cmd, shell=False):
+    """Run a shell command."""
+    args = [split(cmd)]
+    kwargs = {}
+
+    if shell:
+        args = [cmd]
+        kwargs = {"shell": shell}
+
+    try:
+        return check_output(*args, **kwargs).decode("utf-8").strip()
+    except Exception as exception:
+        log.error(f"Failed to run {cmd}, saw {str(exception)}")
+        exit(1)
+
+
+def clone_all_apps():
+    """Clone all Co-op Cloud apps to ~/.abra/apps."""
     if not exists(CLONES_PATH):
         mkdir(CLONES_PATH)
 
-    response = get("https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos")
-    repos = [[p["name"], p["ssh_url"]] for p in response.json()]
-
     skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
+    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
+
+    log.info(f"Retrieving {url}")
+
+    try:
+        response = get(url, timeout=30)
+    except Exception as exception:
+        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
+        exit(1)
+
+    repos = [[p["name"], p["ssh_url"]] for p in response.json()]
 
     for name, url in repos:
         if name in skips:
             continue
 
-        try:
         if not exists(f"{CLONES_PATH}/{name}"):
             run(split(f"git clone {url} {CLONES_PATH}/{name}"))
+
             chdir(f"{CLONES_PATH}/{name}")
-                if not int(check_output("git branch --list | wc -l", shell=True)):
+            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
+                log.info(f"Guessing main branch is HEAD for {name}")
                 run(split("git checkout main"))
-        except Exception:
-            pass
 
 
-def gen_apps_json():
+def generate_apps_json():
+    """Generate the abra-apps.json application versions file."""
     apps_json = {}
 
     for app in listdir(CLONES_PATH):
         app_path = f"{CLONES_PATH}/{app}"
         chdir(app_path)
 
-        output = check_output("git tag --list", shell=True)
+        tags = _run_cmd("git tag --list").split()
-        tags = output.decode("utf-8").strip().split()
 
         if not tags:
+            log.info(f"No tags discovered for {app}, moving on")
             continue
 
         for tag in tags:
+            log.info(f"Processing {tag} for {app}")
             apps_json[app] = {
                 "category": "apps",
                 "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
@@ -69,18 +101,19 @@ def gen_apps_json():
 
 
 def get_app_features(app_path, tag):
-    print(f"Gathering features for {app_path}, tag: {tag}")
+    """Parse features from app repo README files."""
-
     features = {}
 
     chdir(app_path)
-    run(f"git checkout {tag}", shell=True)
+    _run_cmd(f"git checkout {tag}")
 
     with open(f"{app_path}/README.md", "r") as handle:
+        log.info(f"{app_path}/README.md")
         contents = handle.read()
 
     for match in findall(r"\*\*.*\s\*", contents):
         title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
+
         if title == "image":
             value = {
                 "image": search(r"(?<=`).*(?=`)", match).group(),
@@ -90,49 +123,59 @@ def get_app_features(app_path, tag):
             }
         else:
             value = match.split(":")[-1].replace("*", "").strip()
+
         features[title] = value
 
-    run(f"git checkout HEAD", shell=True)
+    log.info(f"Parsed {features}")
+
+    _run_cmd("git checkout HEAD")
+
     return features
 
 
 def get_app_versions(app_path, tag):
-    print(f"Gathering versions for {app_path}, tag: {tag}")
-
     versions = []
 
     chdir(app_path)
-    run(f"git checkout {tag}", shell=True)
+    _run_cmd(f"git checkout {tag}")
-
-    services_command = "yq e '.services | keys | .[]' compose*.yml"
-    services = check_output(services_command, shell=True).decode("utf-8").split()
 
+    services_cmd = "yq e '.services | keys | .[]' compose*.yml"
+    services = _run_cmd(services_cmd, shell=True).split()
     for service in services:
-        images_command = f"yq e '.services.{service}.image' compose*.yml"
+        services_cmd = f"yq e '.services.{service}.image' compose*.yml"
-        images = check_output(images_command, shell=True).decode("utf-8").split()
+        images = _run_cmd(services_cmd, shell=True).split()
-
         for image in images:
             if image in ("null", "---"):
                 continue
 
-            digest_command = f"skopeo inspect docker://{image} | jq '.Digest'"
+            images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
-            output = check_output(digest_command, shell=True).decode("utf-8")
+            output = _run_cmd(images_cmd, shell=True)
-            digest = output.strip().split(":")[-1][:8]
 
-            image_name = image.split(":")[0]
+            new_version_info = {
-            versions.append(
+                service: {
-                {service: {"image": image_name, "tag": tag, "digest": digest}}
+                    "image": image.split(":")[0],
-            )
+                    "tag": tag,
+                    "digest": output.split(":")[-1][:8],
+                }
+            }
+
+            versions.append(new_version_info)
+            log.info(f"Parsed {new_version_info}")
+
+    _run_cmd("git checkout HEAD")
 
-    run(f"git checkout HEAD", shell=True)
     return versions
 
 
-clone_apps()
+def main():
+    """Run the script."""
+    clone_all_apps()
 
-target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
+    target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
+    with open(target, "w", encoding="utf-8") as handle:
+        dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
 
-with open(target, "w", encoding="utf-8") as handle:
+    log.info(f"Successfully generated {target}")
-    dump(gen_apps_json(), handle, ensure_ascii=False, indent=4)
 
-print(f"Output saved to {target}")
+
+main()