abra/bin/app-json.py

#!/usr/bin/env python3

# Usage: ./app-json.py
#
# Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
# ~/.abra/apps), and format it as JSON so that it can be hosted here:
# https://abra-apps.cloud.autonomic.zone

from json import dump
from logging import DEBUG, basicConfig, getLogger
from os import chdir, listdir, mkdir
from os.path import exists, expanduser
from pathlib import Path
from re import findall, search
from shlex import split
from subprocess import check_output, run
from sys import exit

from requests import get

HOME_PATH = expanduser("~/")
CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
SCRIPT_PATH = Path(__file__).absolute().parent

log = getLogger(__name__)
basicConfig()
log.setLevel(DEBUG)


def _run_cmd(cmd, shell=False):
    """Run a shell command."""
    args = [split(cmd)]
    kwargs = {}

    if shell:
        args = [cmd]
        kwargs = {"shell": shell}

    try:
        return check_output(*args, **kwargs).decode("utf-8").strip()
    except Exception as exception:
        log.error(f"Failed to run {cmd}, saw {str(exception)}")
        exit(1)


def clone_all_apps():
    """Clone all Co-op Cloud apps to ~/.abra/apps."""
    if not exists(CLONES_PATH):
        mkdir(CLONES_PATH)

    skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"

    log.info(f"Retrieving {url}")

    try:
        response = get(url, timeout=30)
    except Exception as exception:
        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
        exit(1)

    repos = [[p["name"], p["ssh_url"]] for p in response.json()]

    for name, url in repos:
        if name in skips:
            continue

        if not exists(f"{CLONES_PATH}/{name}"):
            run(split(f"git clone {url} {CLONES_PATH}/{name}"))

            chdir(f"{CLONES_PATH}/{name}")
            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
                log.info(f"Guessing main branch is HEAD for {name}")
                run(split("git checkout main"))


def generate_apps_json():
    """Generate the abra-apps.json application versions file."""
    apps_json = {}

    for app in listdir(CLONES_PATH):
        app_path = f"{CLONES_PATH}/{app}"
        chdir(app_path)

        tags = _run_cmd("git tag --list").split()

        if not tags:
            log.info(f"No tags discovered for {app}, moving on")
            continue

        for tag in tags:
            log.info(f"Processing {tag} for {app}")
            apps_json[app] = {
                "category": "apps",
                "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
                "features": get_app_features(app_path, tag),
                "versions": get_app_versions(app_path, tag),
            }

    return apps_json


def get_app_features(app_path, tag):
    """Parse features from app repo README files."""
    features = {}

    chdir(app_path)
    _run_cmd(f"git checkout {tag}")

    with open(f"{app_path}/README.md", "r") as handle:
        log.info(f"{app_path}/README.md")
        contents = handle.read()

    for match in findall(r"\*\*.*\s\*", contents):
        title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()

        if title == "image":
            value = {
                "image": search(r"(?<=`).*(?=`)", match).group(),
                "url": search(r"(?<=\().*(?=\))", match).group(),
                "rating": match.split(",")[1].strip(),
                "source": match.split(",")[-1].replace("*", "").strip(),
            }
        else:
            value = match.split(":")[-1].replace("*", "").strip()

        features[title] = value

    log.info(f"Parsed {features}")

    _run_cmd("git checkout HEAD")

    return features


def get_app_versions(app_path, tag):
    versions = []

    chdir(app_path)
    _run_cmd(f"git checkout {tag}")

    services_cmd = "yq e '.services | keys | .[]' compose*.yml"
    services = _run_cmd(services_cmd, shell=True).split()
    for service in services:
        services_cmd = f"yq e '.services.{service}.image' compose*.yml"
        images = _run_cmd(services_cmd, shell=True).split()
        for image in images:
            if image in ("null", "---"):
                continue

            images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
            output = _run_cmd(images_cmd, shell=True)

            new_version_info = {
                service: {
                    "image": image.split(":")[0],
                    "tag": tag,
                    "digest": output.split(":")[-1][:8],
                }
            }

            versions.append(new_version_info)
            log.info(f"Parsed {new_version_info}")

    _run_cmd("git checkout HEAD")

    return versions


def main():
    """Run the script."""
    clone_all_apps()

    target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
    with open(target, "w", encoding="utf-8") as handle:
        dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)

    log.info(f"Successfully generated {target}")


main()