Fix loads of bugs and generate apps JSON again

bin/app-json.py

@@ -21,6 +21,16 @@ from requests import get
 HOME_PATH = expanduser("~/")
 CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
 SCRIPT_PATH = Path(__file__).absolute().parent
+REPOS_TO_SKIP = (
+    "abra",
+    "backup-bot",
+    "cloud.autonomic.zone",
+    "docs.cloud.autonomic.zone",
+    "example",
+    "organising",
+    "pyabra",
+    "stack-ssh-deploy",
+)
 
 log = getLogger(__name__)
 basicConfig()
@@ -47,8 +57,6 @@ def clone_all_apps():
     """Clone all Co-op Cloud apps to ~/.abra/apps."""
     if not exists(CLONES_PATH):
         mkdir(CLONES_PATH)
-
-    skips = ("organising", "cloud.autonomic.zone", "docs.cloud.autonomic.zone", "abra")
     url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
 
     log.info(f"Retrieving {url}")
@@ -62,7 +70,7 @@ def clone_all_apps():
     repos = [[p["name"], p["ssh_url"]] for p in response.json()]
 
     for name, url in repos:
-        if name in skips:
+        if name in REPOS_TO_SKIP:
             continue
 
         if not exists(f"{CLONES_PATH}/{name}"):
@@ -71,7 +79,7 @@ def clone_all_apps():
             chdir(f"{CLONES_PATH}/{name}")
             if not int(_run_cmd("git branch --list | wc -l", shell=True)):
                 log.info(f"Guessing main branch is HEAD for {name}")
-                run(split("git checkout main"))
+                _run_cmd("git checkout main")
 
 
 def generate_apps_json():
@@ -79,33 +87,34 @@ def generate_apps_json():
     apps_json = {}
 
     for app in listdir(CLONES_PATH):
+        if app in REPOS_TO_SKIP:
+            log.info(f"Skipping {app}")
+            continue
+
         app_path = f"{CLONES_PATH}/{app}"
         chdir(app_path)
 
-        tags = _run_cmd("git tag --list").split()
+        log.info(f"Processing {app}")
-
+        apps_json[app] = {
-        if not tags:
+            "category": "apps",
-            log.info(f"No tags discovered for {app}, moving on")
+            "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
-            continue
+            # Note(decentral1se): please note that the app features do not
-
+            # correspond to version tags. We simply parse the latest features
-        for tag in tags:
+            # list from HEAD. This may lead to unexpected situations where
-            log.info(f"Processing {tag} for {app}")
+            # users believe X feature is available under Y version but it is
-            apps_json[app] = {
+            # not.
-                "category": "apps",
+            "features": get_app_features(app_path),
-                "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
+            "versions": get_app_versions(app_path),
-                "features": get_app_features(app_path, tag),
+        }
-                "versions": get_app_versions(app_path, tag),
-            }
 
     return apps_json
 
 
-def get_app_features(app_path, tag):
+def get_app_features(app_path):
     """Parse features from app repo README files."""
     features = {}
 
     chdir(app_path)
-    _run_cmd(f"git checkout {tag}")
 
     with open(f"{app_path}/README.md", "r") as handle:
         log.info(f"{app_path}/README.md")
@@ -133,34 +142,47 @@ def get_app_features(app_path, tag):
     return features
 
 
-def get_app_versions(app_path, tag):
+def get_app_versions(app_path):
-    versions = []
+    versions = {}
 
     chdir(app_path)
-    _run_cmd(f"git checkout {tag}")
 
-    services_cmd = "yq e '.services | keys | .[]' compose*.yml"
+    tags = _run_cmd("git tag --list").split()
-    services = _run_cmd(services_cmd, shell=True).split()
-    for service in services:
-        services_cmd = f"yq e '.services.{service}.image' compose*.yml"
-        images = _run_cmd(services_cmd, shell=True).split()
-        for image in images:
-            if image in ("null", "---"):
-                continue
 
-            images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
+    if not tags:
-            output = _run_cmd(images_cmd, shell=True)
+        log.info(f"No tags discovered, moving on")
+        return {}
 
-            new_version_info = {
+    for tag in tags:
-                service: {
+        _run_cmd(f"git checkout {tag}")
-                    "image": image.split(":")[0],
+
-                    "tag": tag,
+        services_cmd = "yq e '.services | keys | .[]' compose*.yml"
-                    "digest": output.split(":")[-1][:8],
+        services = _run_cmd(services_cmd, shell=True).split()
+
+        service_versions = []
+        for service in services:
+            services_cmd = f"yq e '.services.{service}.image' compose*.yml"
+            images = _run_cmd(services_cmd, shell=True).split()
+
+            for image in images:
+                if image in ("null", "---"):
+                    continue
+
+                images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
+                output = _run_cmd(images_cmd, shell=True)
+
+                service_version_info = {
+                    service: {
+                        "image": image.split(":")[0],
+                        "tag": tag,
+                        "digest": output.split(":")[-1][:8],
+                    }
                 }
-            }
 
-            versions.append(new_version_info)
+                log.info(f"Parsed {service_version_info}")
-            log.info(f"Parsed {new_version_info}")
+                service_versions.append(service_version_info)
+
+        versions[tag] = service_versions
 
     _run_cmd("git checkout HEAD")
 

deploy/abra-apps.cloud.autonomic.zone/abra-apps.json

@@ -1,4 +1,124 @@
 {
+    "drone": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/drone.git",
+        "features": {
+            "category": "Development",
+            "status": "?",
+            "image": {
+                "image": "drone/drone",
+                "url": "https://hub.docker.com/r/drone/drone",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "?",
+            "email": "?",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "traefik-forward-auth": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/traefik-forward-auth.git",
+        "features": {
+            "category": "Utilities",
+            "status": "?",
+            "image": {
+                "image": "crazymax/swarm-cronjob",
+                "url": "https://hub.docker.com/r/crazymax/swarm-cronjob/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "?",
+            "backups": "?",
+            "email": "?",
+            "tests": "?"
+        },
+        "versions": {}
+    },
+    "postfix-relay": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/postfix-relay.git",
+        "features": {
+            "category": "Utilities",
+            "status": "❷💛",
+            "image": {
+                "image": "boky/postfix",
+                "url": "https://hub.docker.com/r/boky/postfix/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "N/A",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "matomo": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/matomo.git",
+        "features": {
+            "category": "Apps",
+            "status": "❸🍎",
+            "image": {
+                "image": "matomo",
+                "url": "https://hub.docker.com/_/matomo",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "rocketchat": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/rocketchat.git",
+        "features": {
+            "category": "Apps",
+            "status": "❷💛",
+            "image": {
+                "image": "rocketchat/rocket.chat",
+                "url": "https://hub.docker.com/r/rocketchat/rocket.chat/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "renovate": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/renovate.git",
+        "features": {},
+        "versions": {}
+    },
+    "keycloak": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/keycloak.git",
+        "features": {
+            "category": "Apps",
+            "status": "❷💛",
+            "image": {
+                "image": "jboss/keycloak",
+                "url": "https://hub.docker.com/r/jboss/keycloak",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "?",
+            "email": "❸🍎",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
     "gitea": {
         "category": "apps",
         "repository": "https://git.autonomic.zone/coop-cloud/gitea.git",
@@ -16,21 +136,327 @@
             "email": "?",
             "tests": "❷💛"
         },
-        "versions": [
+        "versions": {
-            {
+            "1.13.6": [
-                "app": {
+                {
-                    "image": "gitea/gitea",
+                    "app": {
-                    "tag": "1.13.6",
+                        "image": "gitea/gitea",
-                    "digest": "1d90f984"
+                        "tag": "1.13.6",
+                        "digest": "1d90f984"
+                    }
+                },
+                {
+                    "db": {
+                        "image": "mariadb",
+                        "tag": "1.13.6",
+                        "digest": "ab7c906b"
+                    }
                 }
+            ]
+        }
+    },
+    "wordpress": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/wordpress.git",
+        "features": {
+            "category": "Apps",
+            "status": "❶💚",
+            "image": {
+                "image": "wordpress",
+                "url": "https://hub.docker.com/_/wordpress",
+                "rating": "❶💚",
+                "source": "upstream"
             },
-            {
+            "healthcheck": "Yes",
-                "db": {
+            "backups": "Yes",
-                    "image": "mariadb",
+            "email": "❶💚",
-                    "tag": "1.13.6",
+            "tests": "❷💛"
-                    "digest": "ab7c906b"
+        },
-                }
+        "versions": {}
-            }
+    },
-        ]
+    "mediawiki": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/mediawiki.git",
+        "features": {
+            "category": "Apps",
+            "status": "❸🍎",
+            "image": {
+                "image": "mediawiki",
+                "url": "https://hub.docker.com/_/mediawiki",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "No",
+            "backups": "Yes",
+            "email": "❶💚",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "swarm-cronjob": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/swarm-cronjob.git",
+        "features": {
+            "category": "Utilities",
+            "status": "?",
+            "image": {
+                "image": "crazymax/swarm-cronjob",
+                "url": "https://hub.docker.com/r/crazymax/swarm-cronjob/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "?",
+            "backups": "?",
+            "email": "?",
+            "tests": "?"
+        },
+        "versions": {}
+    },
+    "swarmpit": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/swarmpit.git",
+        "features": {
+            "category": "Utilities",
+            "status": "❷💛",
+            "image": {
+                "image": "swarmpit",
+                "url": "https://hub.docker.com/_/swarmpit",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "codimd": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/codimd.git",
+        "features": {
+            "category": "Apps",
+            "status": "❷💛",
+            "image": {
+                "image": "hackmdio/hackmd",
+                "url": "https://hub.docker.com/r/hackmdio/hackmd/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "traefik": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/traefik.git",
+        "features": {
+            "category": "Utilities",
+            "status": "?",
+            "image": {
+                "image": "traefik",
+                "url": "https://hub.docker.com/_/traefik",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "N/A",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "nextcloud": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/nextcloud.git",
+        "features": {
+            "category": "Apps",
+            "status": "❷💛",
+            "image": {
+                "image": "nextcloud",
+                "url": "https://hub.docker.com/_/nextcloud",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "❶💚",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "strapi": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/strapi.git",
+        "features": {
+            "category": "Development",
+            "status": "❸🍎",
+            "image": {
+                "image": "strapi/strapi",
+                "url": "https://hub.docker.com/r/strapi/strapi",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "No",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "selfoss": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/selfoss.git",
+        "features": {
+            "category": "Apps",
+            "status": "❸🍎",
+            "image": {
+                "image": "akito13/selfoss",
+                "url": "https://hub.docker.com/r/akito13/selfoss",
+                "rating": "❸🍎",
+                "source": "3rd-party"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❸🍎"
+        },
+        "versions": {}
+    },
+    "matrix-synapse": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/matrix-synapse.git",
+        "features": {
+            "category": "Apps",
+            "status": "❹💣",
+            "image": {
+                "image": "matrixdotorg/synapse",
+                "url": "https://hub.docker.com/r/matrixdotorg/synapse",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "No"
+        },
+        "versions": {}
+    },
+    "kimai": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/kimai.git",
+        "features": {
+            "category": "Apps",
+            "status": "?",
+            "image": {
+                "image": "kimai/kimai2",
+                "url": "https://hub.docker.com/kimai/kimai2",
+                "rating": "❷💛",
+                "source": "official"
+            },
+            "healthcheck": "No",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "distribution": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/distribution.git",
+        "features": {
+            "category": "Development",
+            "status": "❹💣",
+            "image": {
+                "image": "registry",
+                "url": "https://hub.docker.com/_/registry/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "?",
+            "backups": "?",
+            "email": "?",
+            "tests": "?"
+        },
+        "versions": {}
+    },
+    "invoiceninja": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/invoiceninja.git",
+        "features": {
+            "category": "Apps",
+            "status": "❹💣",
+            "image": {
+                "image": "invoiceninja/invoiceninja",
+                "url": "https://hub.docker.com/r/invoiceninja/invoiceninja",
+                "rating": "❸🍎",
+                "source": "3rd party"
+            },
+            "healthcheck": "No",
+            "backups": "No",
+            "email": "?",
+            "tests": "No"
+        },
+        "versions": {}
+    },
+    "portainer": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/portainer.git",
+        "features": {
+            "category": "Utilities",
+            "status": "?",
+            "image": {
+                "image": "portainer/portainer",
+                "url": "https://hub.docker.com/r/portainer/portainer",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "No",
+            "backups": "?",
+            "email": "?",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "adapt_authoring": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/adapt_authoring.git",
+        "features": {
+            "category": "Apps",
+            "status": "❸🍎",
+            "image": {
+                "image": "3wordchant/adaptauthoring",
+                "url": "https://hub.docker.com/r/3wordchant/adaptauthoring",
+                "rating": "❹💣",
+                "source": "own"
+            },
+            "healthcheck": "Yes",
+            "backups": "No",
+            "email": "No",
+            "tests": "❷💛"
+        },
+        "versions": {}
+    },
+    "drone-docker-runner": {
+        "category": "apps",
+        "repository": "https://git.autonomic.zone/coop-cloud/drone-docker-runner.git",
+        "features": {
+            "category": "Development",
+            "status": "❹💣",
+            "image": {
+                "image": "drone/drone-docker-runner",
+                "url": "https://hub.docker.com/r/drone/drone-docker-runner/",
+                "rating": "❶💚",
+                "source": "upstream"
+            },
+            "healthcheck": "?",
+            "backups": "?",
+            "email": "?",
+            "tests": "?"
+        },
+        "versions": {}
     }
 }