apps-json.py: more metadata, skip abra-apps, pagination

bin/app-json.py

@@ -24,6 +24,7 @@ YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
 SCRIPT_PATH = Path(__file__).absolute().parent
 REPOS_TO_SKIP = (
     "abra",
+    "abra-apps",
     "backup-bot",
     "cloud.autonomic.zone",
     "docs.cloud.autonomic.zone",
@@ -62,13 +63,24 @@ def get_repos_json():
 
     log.info(f"Retrieving {url}")
 
+    repos = []
+    response = True
+    page = 1
+
     try:
-        return get(url, timeout=10).json()
+        while response:
+            log.info(f"Trying to fetch page {page}")
+            response = get(url + f"?page={page}",timeout=10).json()
+            repos.extend(response)
+            page += 1
+
+        return repos
     except Exception as exception:
         log.error(f"Failed to retrieve {url}, saw {str(exception)}")
         exit(1)
 
 
+
 def get_published_apps_json():
     """Retrieve already published apps json."""
     url = "https://apps.coopcloud.tech"
@@ -123,18 +135,23 @@ def generate_apps_json(repos_json):
         app_path = f"{CLONES_PATH}/{app}"
         chdir(app_path)
 
-        features = get_app_features(app_path)
+        metadata = get_app_metadata(app_path)
+        name = metadata.pop("name", "")
 
         log.info(f"Processing {app}")
         apps_json[app] = {
-            "category": features.get("category", ""),
+            "name": name,
-            "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
+            "category": metadata.get("category", ""),
-            # Note(decentral1se): please note that the app features do not
+            "repository": repo_details.get("clone_url", ""),
-            # correspond to version tags. We simply parse the latest features
+            "default_branch": repo_details.get("default_branch", ""),
+            "description": repo_details.get("description", ""),
+            "website": repo_details.get("website", ""),
+            # Note(decentral1se): please note that the app metadata do not
+            # correspond to version tags. We simply parse the latest metadata
             # list from HEAD. This may lead to unexpected situations where
             # users believe X feature is available under Y version but it is
             # not.
-            "features": features,
+            "features": metadata,
             "versions": get_app_versions(app_path, cached_apps_json),
             "icon": repo_details.get("avatar_url", "")
         }
@@ -142,9 +159,9 @@ def generate_apps_json(repos_json):
     return apps_json
 
 
-def get_app_features(app_path):
+def get_app_metadata(app_path):
-    """Parse features from app repo README files."""
+    """Parse metadata from app repo README files."""
-    features = {}
+    metadata = {}
 
     chdir(app_path)
 
@@ -166,16 +183,17 @@ def get_app_features(app_path):
             else:
                 value = match.split(":")[-1].replace("*", "").strip()
 
-            features[title] = value
+            metadata[title] = value
+        metadata["name"] = findall(r"^# (.*)", contents)[0]
     except (IndexError, AttributeError):
         log.info(f"Can't parse {app_path}/README.md")
         return {}
     finally:
         _run_cmd("git checkout HEAD")
 
-    log.info(f"Parsed {features}")
+    log.info(f"Parsed {metadata}")
 
-    return features
+    return metadata
 
 
 def get_app_versions(app_path, cached_apps_json):
@@ -255,7 +273,7 @@ def main():
     repos_json = get_repos_json()
     clone_all_apps(repos_json)
 
-    target = f"{SCRIPT_PATH}/../deploy/apps.coopcloud.tech/abra-apps.json"
+    target = f"{SCRIPT_PATH}/../deploy/apps.coopcloud.tech/apps.json"
     with open(target, "w", encoding="utf-8") as handle:
         dump(generate_apps_json(repos_json), handle, ensure_ascii=False, indent=4)