From 6b0f8a3d45ad3a5ec35d030c9a1f9ba9baa8d1b8 Mon Sep 17 00:00:00 2001 From: decentral1se Date: Fri, 2 Apr 2021 21:00:55 +0200 Subject: [PATCH] Fix loads of bugs and generate apps JSON again --- bin/app-json.py | 104 ++-- .../abra-apps.json | 454 +++++++++++++++++- 2 files changed, 503 insertions(+), 55 deletions(-) diff --git a/bin/app-json.py b/bin/app-json.py index 4c67d4c..02cb56b 100755 --- a/bin/app-json.py +++ b/bin/app-json.py @@ -21,6 +21,16 @@ from requests import get HOME_PATH = expanduser("~/") CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute() SCRIPT_PATH = Path(__file__).absolute().parent +REPOS_TO_SKIP = ( + "abra", + "backup-bot", + "cloud.autonomic.zone", + "docs.cloud.autonomic.zone", + "example", + "organising", + "pyabra", + "stack-ssh-deploy", +) log = getLogger(__name__) basicConfig() @@ -47,8 +57,6 @@ def clone_all_apps(): """Clone all Co-op Cloud apps to ~/.abra/apps.""" if not exists(CLONES_PATH): mkdir(CLONES_PATH) - - skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra") url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos" log.info(f"Retrieving {url}") @@ -62,7 +70,7 @@ def clone_all_apps(): repos = [[p["name"], p["ssh_url"]] for p in response.json()] for name, url in repos: - if name in skips: + if name in REPOS_TO_SKIP: continue if not exists(f"{CLONES_PATH}/{name}"): @@ -71,7 +79,7 @@ def clone_all_apps(): chdir(f"{CLONES_PATH}/{name}") if not int(_run_cmd("git branch --list | wc -l", shell=True)): log.info(f"Guessing main branch is HEAD for {name}") - run(split("git checkout main")) + _run_cmd("git checkout main") def generate_apps_json(): @@ -79,33 +87,34 @@ def generate_apps_json(): apps_json = {} for app in listdir(CLONES_PATH): + if app in REPOS_TO_SKIP: + log.info(f"Skipping {app}") + continue + app_path = f"{CLONES_PATH}/{app}" chdir(app_path) - tags = _run_cmd("git tag --list").split() - - if not tags: - log.info(f"No tags discovered for {app}, moving on") - continue - - for tag in tags: - log.info(f"Processing {tag} for {app}") - apps_json[app] = { - "category": "apps", - "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git", - "features": get_app_features(app_path, tag), - "versions": get_app_versions(app_path, tag), - } + log.info(f"Processing {app}") + apps_json[app] = { + "category": "apps", + "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git", + # Note(decentral1se): please note that the app features do not + # correspond to version tags. We simply parse the latest features + # list from HEAD. This may lead to unexpected situations where + # users believe X feature is available under Y version but it is + # not. + "features": get_app_features(app_path), + "versions": get_app_versions(app_path), + } return apps_json -def get_app_features(app_path, tag): +def get_app_features(app_path): """Parse features from app repo README files.""" features = {} chdir(app_path) - _run_cmd(f"git checkout {tag}") with open(f"{app_path}/README.md", "r") as handle: log.info(f"{app_path}/README.md") @@ -133,34 +142,47 @@ def get_app_features(app_path, tag): return features -def get_app_versions(app_path, tag): - versions = [] +def get_app_versions(app_path): + versions = {} chdir(app_path) - _run_cmd(f"git checkout {tag}") - services_cmd = "yq e '.services | keys | .[]' compose*.yml" - services = _run_cmd(services_cmd, shell=True).split() - for service in services: - services_cmd = f"yq e '.services.{service}.image' compose*.yml" - images = _run_cmd(services_cmd, shell=True).split() - for image in images: - if image in ("null", "---"): - continue + tags = _run_cmd("git tag --list").split() - images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'" - output = _run_cmd(images_cmd, shell=True) + if not tags: + log.info(f"No tags discovered, moving on") + return {} - new_version_info = { - service: { - "image": image.split(":")[0], - "tag": tag, - "digest": output.split(":")[-1][:8], + for tag in tags: + _run_cmd(f"git checkout {tag}") + + services_cmd = "yq e '.services | keys | .[]' compose*.yml" + services = _run_cmd(services_cmd, shell=True).split() + + service_versions = [] + for service in services: + services_cmd = f"yq e '.services.{service}.image' compose*.yml" + images = _run_cmd(services_cmd, shell=True).split() + + for image in images: + if image in ("null", "---"): + continue + + images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'" + output = _run_cmd(images_cmd, shell=True) + + service_version_info = { + service: { + "image": image.split(":")[0], + "tag": tag, + "digest": output.split(":")[-1][:8], + } } - } - versions.append(new_version_info) - log.info(f"Parsed {new_version_info}") + log.info(f"Parsed {service_version_info}") + service_versions.append(service_version_info) + + versions[tag] = service_versions _run_cmd("git checkout HEAD") diff --git a/deploy/abra-apps.cloud.autonomic.zone/abra-apps.json b/deploy/abra-apps.cloud.autonomic.zone/abra-apps.json index a91994b..ce28980 100644 --- a/deploy/abra-apps.cloud.autonomic.zone/abra-apps.json +++ b/deploy/abra-apps.cloud.autonomic.zone/abra-apps.json @@ -1,4 +1,124 @@ { + "drone": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/drone.git", + "features": { + "category": "Development", + "status": "?", + "image": { + "image": "drone/drone", + "url": "https://hub.docker.com/r/drone/drone", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "?", + "email": "?", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "traefik-forward-auth": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/traefik-forward-auth.git", + "features": { + "category": "Utilities", + "status": "?", + "image": { + "image": "crazymax/swarm-cronjob", + "url": "https://hub.docker.com/r/crazymax/swarm-cronjob/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "?", + "backups": "?", + "email": "?", + "tests": "?" + }, + "versions": {} + }, + "postfix-relay": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/postfix-relay.git", + "features": { + "category": "Utilities", + "status": "β·πŸ’›", + "image": { + "image": "boky/postfix", + "url": "https://hub.docker.com/r/boky/postfix/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "N/A", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "matomo": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/matomo.git", + "features": { + "category": "Apps", + "status": "❸🍎", + "image": { + "image": "matomo", + "url": "https://hub.docker.com/_/matomo", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "rocketchat": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/rocketchat.git", + "features": { + "category": "Apps", + "status": "β·πŸ’›", + "image": { + "image": "rocketchat/rocket.chat", + "url": "https://hub.docker.com/r/rocketchat/rocket.chat/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "renovate": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/renovate.git", + "features": {}, + "versions": {} + }, + "keycloak": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/keycloak.git", + "features": { + "category": "Apps", + "status": "β·πŸ’›", + "image": { + "image": "jboss/keycloak", + "url": "https://hub.docker.com/r/jboss/keycloak", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "?", + "email": "❸🍎", + "tests": "β·πŸ’›" + }, + "versions": {} + }, "gitea": { "category": "apps", "repository": "https://git.autonomic.zone/coop-cloud/gitea.git", @@ -16,21 +136,327 @@ "email": "?", "tests": "β·πŸ’›" }, - "versions": [ - { - "app": { - "image": "gitea/gitea", - "tag": "1.13.6", - "digest": "1d90f984" + "versions": { + "1.13.6": [ + { + "app": { + "image": "gitea/gitea", + "tag": "1.13.6", + "digest": "1d90f984" + } + }, + { + "db": { + "image": "mariadb", + "tag": "1.13.6", + "digest": "ab7c906b" + } } + ] + } + }, + "wordpress": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/wordpress.git", + "features": { + "category": "Apps", + "status": "βΆπŸ’š", + "image": { + "image": "wordpress", + "url": "https://hub.docker.com/_/wordpress", + "rating": "βΆπŸ’š", + "source": "upstream" }, - { - "db": { - "image": "mariadb", - "tag": "1.13.6", - "digest": "ab7c906b" - } - } - ] + "healthcheck": "Yes", + "backups": "Yes", + "email": "βΆπŸ’š", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "mediawiki": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/mediawiki.git", + "features": { + "category": "Apps", + "status": "❸🍎", + "image": { + "image": "mediawiki", + "url": "https://hub.docker.com/_/mediawiki", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "No", + "backups": "Yes", + "email": "βΆπŸ’š", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "swarm-cronjob": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/swarm-cronjob.git", + "features": { + "category": "Utilities", + "status": "?", + "image": { + "image": "crazymax/swarm-cronjob", + "url": "https://hub.docker.com/r/crazymax/swarm-cronjob/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "?", + "backups": "?", + "email": "?", + "tests": "?" + }, + "versions": {} + }, + "swarmpit": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/swarmpit.git", + "features": { + "category": "Utilities", + "status": "β·πŸ’›", + "image": { + "image": "swarmpit", + "url": "https://hub.docker.com/_/swarmpit", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "codimd": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/codimd.git", + "features": { + "category": "Apps", + "status": "β·πŸ’›", + "image": { + "image": "hackmdio/hackmd", + "url": "https://hub.docker.com/r/hackmdio/hackmd/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "traefik": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/traefik.git", + "features": { + "category": "Utilities", + "status": "?", + "image": { + "image": "traefik", + "url": "https://hub.docker.com/_/traefik", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "N/A", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "nextcloud": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/nextcloud.git", + "features": { + "category": "Apps", + "status": "β·πŸ’›", + "image": { + "image": "nextcloud", + "url": "https://hub.docker.com/_/nextcloud", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "βΆπŸ’š", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "strapi": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/strapi.git", + "features": { + "category": "Development", + "status": "❸🍎", + "image": { + "image": "strapi/strapi", + "url": "https://hub.docker.com/r/strapi/strapi", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "No", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "selfoss": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/selfoss.git", + "features": { + "category": "Apps", + "status": "❸🍎", + "image": { + "image": "akito13/selfoss", + "url": "https://hub.docker.com/r/akito13/selfoss", + "rating": "❸🍎", + "source": "3rd-party" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "❸🍎" + }, + "versions": {} + }, + "matrix-synapse": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/matrix-synapse.git", + "features": { + "category": "Apps", + "status": "βΉπŸ’£", + "image": { + "image": "matrixdotorg/synapse", + "url": "https://hub.docker.com/r/matrixdotorg/synapse", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "No" + }, + "versions": {} + }, + "kimai": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/kimai.git", + "features": { + "category": "Apps", + "status": "?", + "image": { + "image": "kimai/kimai2", + "url": "https://hub.docker.com/kimai/kimai2", + "rating": "β·πŸ’›", + "source": "official" + }, + "healthcheck": "No", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "distribution": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/distribution.git", + "features": { + "category": "Development", + "status": "βΉπŸ’£", + "image": { + "image": "registry", + "url": "https://hub.docker.com/_/registry/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "?", + "backups": "?", + "email": "?", + "tests": "?" + }, + "versions": {} + }, + "invoiceninja": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/invoiceninja.git", + "features": { + "category": "Apps", + "status": "βΉπŸ’£", + "image": { + "image": "invoiceninja/invoiceninja", + "url": "https://hub.docker.com/r/invoiceninja/invoiceninja", + "rating": "❸🍎", + "source": "3rd party" + }, + "healthcheck": "No", + "backups": "No", + "email": "?", + "tests": "No" + }, + "versions": {} + }, + "portainer": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/portainer.git", + "features": { + "category": "Utilities", + "status": "?", + "image": { + "image": "portainer/portainer", + "url": "https://hub.docker.com/r/portainer/portainer", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "No", + "backups": "?", + "email": "?", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "adapt_authoring": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/adapt_authoring.git", + "features": { + "category": "Apps", + "status": "❸🍎", + "image": { + "image": "3wordchant/adaptauthoring", + "url": "https://hub.docker.com/r/3wordchant/adaptauthoring", + "rating": "βΉπŸ’£", + "source": "own" + }, + "healthcheck": "Yes", + "backups": "No", + "email": "No", + "tests": "β·πŸ’›" + }, + "versions": {} + }, + "drone-docker-runner": { + "category": "apps", + "repository": "https://git.autonomic.zone/coop-cloud/drone-docker-runner.git", + "features": { + "category": "Development", + "status": "βΉπŸ’£", + "image": { + "image": "drone/drone-docker-runner", + "url": "https://hub.docker.com/r/drone/drone-docker-runner/", + "rating": "βΆπŸ’š", + "source": "upstream" + }, + "healthcheck": "?", + "backups": "?", + "email": "?", + "tests": "?" + }, + "versions": {} } } \ No newline at end of file