From 7b0fb50e7f08158190b8d8e1d6ea39b325e7cfce Mon Sep 17 00:00:00 2001 From: decentral1se Date: Sat, 5 Jun 2021 07:55:05 +0200 Subject: [PATCH] Abstract common functions into a library --- bin/__pycache__/abralib.cpython-39.pyc | Bin 0 -> 2782 bytes bin/abralib.py | 103 +++++++++++++++++++++++ bin/app-json.py | 109 +++---------------------- bin/clone-all-apps.py | 16 ++++ bin/renovate-ls-apps.py | 72 +--------------- 5 files changed, 132 insertions(+), 168 deletions(-) create mode 100644 bin/__pycache__/abralib.cpython-39.pyc create mode 100644 bin/abralib.py create mode 100755 bin/clone-all-apps.py diff --git a/bin/__pycache__/abralib.cpython-39.pyc b/bin/__pycache__/abralib.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23cc42cb0d8e8654cca93db002370aaebc0a99dd GIT binary patch literal 2782 zcmZWr&2JmW6`z@1F3BZDQD1iCIL=0C1273m2PlxXaNVY|YzIkg#ZnSvkzl*r8If!4 z?yP2pva}$5sC-Ro`Z2|m5_4EdL7(|={IUC>&89i%I>*@v(CSDDRT1MM|tognaY?euj=z>(8IwDI0@O#?dCv_Tph#A2RhZ9}b!5kg^huXe`P~x(@@$4m*ma zlBLUz@-U-g30AnYw?j38u_ANW6_{o;V$puU$0{Fd*@k6$w7_>k9`}nzzZVWf*bnnu zc9wiw3q@Viu2vPUVTZ6qTr>?of|VH)g*ynN{c+wMa8er#v% zY`l)Hg9boVEmj|lGcWYyh$V>^@ic|B^dQ+fC45*`m=Ii)7R-t!*mo+W6{S?P z%-Vg3=1`Wlp>=k?^j}E43iD_0^hZ2peb|9zN`%RKlFb7yMby`2pfi`m11N^F7QjgZ z@clL{mCHa7afyu`Tq6Wu!7X^jYoLwv)cJFsqD@f9?|@J|g*2T)Z_xpMYU*@|rqEZW zgxXW9xH}<&8jH%rI=4_@O`Vy18PO`OeNA-oreuOo?TPU+v8eq%npy`N6Fg69<@nz( zl8PXyb(2*07fFQ_&y(7iC)JE5sSQYKvv}Qmz#swaka>dT+)JRYy*8n^0Mm;@E`pIp(khIl9cR$k7J&8b zhZ3*@jY&aGi4{iQP?>boz=lt8@`WgQ~%n}ZF#1}Yac03Higf2-`%3bikQ$)N^X)VW~k zX#c8uK?7@b#Vbd{jn==0i%0&~_;QVJ13H*(XgkX@gclVXfY^)$aMD1Zu6HBh zQJBb^z?>VO;l9vGFI=bq&)9os*elkx2yG+-9Cy2>ZYJKN$m=G%9c=nk@y4Aollrpo z(lE}>jbrKE-TL5`xsr<|<5XXhfOun8)IZH>sPw9$sb^=U(hW80t8kFAvH9`+ds{of z_6NIng`+hK54N`N?*zN|gPo5)+%~l#JeV+jOX!G}R-CDhUAq5=vv^Zm{92O@O)_j+H_K`ez||TAeieH95I#xh z3=2yWT@)qZ(j3oxhF|5fm&1Pu^O-_dOj*@3bZSut$%o4(VMpc`DP1`pK;n|I1 z$zy3MUDx&jL|(~RYpCD7|CcQjEx#5HBu~bQiFfrGy4q>=i%-lX+yucM++IOYIywA( z%9P*O*?jQf_U?;m4YwdL@%3vG{vbi$OlCpWMQ=X+LL5y*O(&_{NI4xR?51uD@+}}X dS%uKGadY+~O$h5UwpUuX1)PIj%U!I!@)KpE(pdli literal 0 HcmV?d00001 diff --git a/bin/abralib.py b/bin/abralib.py new file mode 100644 index 0000000..e7722b7 --- /dev/null +++ b/bin/abralib.py @@ -0,0 +1,103 @@ +"""Shared utilities for bin/*.py scripts.""" + +from logging import DEBUG, basicConfig, getLogger +from os import chdir, mkdir +from os.path import exists, expanduser +from pathlib import Path +from shlex import split +from subprocess import check_output +from sys import exit + +from requests import get + +HOME_PATH = expanduser("~/") +CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute() +REPOS_TO_SKIP = ( + "abra", + "abra-apps", + "abra-gandi", + "abra-hetzner", + "backup-bot", + "coopcloud.tech", + "coturn", + "docker-cp-deploy", + "docker-dind-bats-kcov", + "docs.coopcloud.tech", + "example", + "gardening", + "organising", + "pyabra", + "radicle-seed-node", + "stack-ssh-deploy", + "swarm-cronjob", +) +YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq") +SCRIPT_PATH = Path(__file__).absolute().parent + +log = getLogger(__name__) +basicConfig() +log.setLevel(DEBUG) + + +def _run_cmd(cmd, shell=False, **kwargs): + """Run a shell command.""" + args = [split(cmd)] + + if shell: + args = [cmd] + kwargs = {"shell": shell} + + try: + return check_output(*args, **kwargs).decode("utf-8").strip() + except Exception as exception: + log.error(f"Failed to run {cmd}, saw {str(exception)}") + exit(1) + + +def get_repos_json(): + """ Retrieve repo list from Gitea """ + + url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos" + + log.info(f"Retrieving {url}") + + repos = [] + response = True + page = 1 + + try: + while response: + log.info(f"Trying to fetch page {page}") + response = get(url + f"?page={page}", timeout=10).json() + repos.extend(response) + page += 1 + + return repos + except Exception as exception: + log.error(f"Failed to retrieve {url}, saw {str(exception)}") + exit(1) + + +def clone_all_apps(repos_json): + """Clone all Co-op Cloud apps to ~/.abra/apps.""" + if not exists(CLONES_PATH): + mkdir(CLONES_PATH) + + repos = [[p["name"], p["ssh_url"]] for p in repos_json] + + for name, url in repos: + if name in REPOS_TO_SKIP: + continue + + if not exists(f"{CLONES_PATH}/{name}"): + log.info(f"Retrieving {url}") + _run_cmd(f"git clone {url} {CLONES_PATH}/{name}") + + chdir(f"{CLONES_PATH}/{name}") + if not int(_run_cmd("git branch --list | wc -l", shell=True)): + log.info(f"Guessing main branch is HEAD for {name}") + _run_cmd("git checkout main") + else: + log.info(f"Updating {name}") + chdir(f"{CLONES_PATH}/{name}") + _run_cmd("git fetch -a") diff --git a/bin/app-json.py b/bin/app-json.py index eb75dab..ba056ac 100755 --- a/bin/app-json.py +++ b/bin/app-json.py @@ -7,84 +7,24 @@ # https://apps.coopcloud.tech from json import dump -from logging import DEBUG, basicConfig, getLogger -from os import chdir, listdir, mkdir -from os.path import basename, exists, expanduser -from pathlib import Path +from os import chdir, listdir +from os.path import basename from re import findall, search -from shlex import split -from subprocess import DEVNULL, check_output -from sys import exit +from subprocess import DEVNULL from requests import get -HOME_PATH = expanduser("~/") -CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute() -YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq") -SCRIPT_PATH = Path(__file__).absolute().parent -REPOS_TO_SKIP = ( - "abra", - "abra-apps", - "abra-gandi", - "abra-hetzner", - "backup-bot", - "coopcloud.tech", - "coturn", - "docker-cp-deploy", - "docker-dind-bats-kcov", - "docs.coopcloud.tech", - "example", - "gardening", - "organising", - "pyabra", - "radicle-seed-node", - "stack-ssh-deploy", - "swarm-cronjob", +from abralib import ( + CLONES_PATH, + REPOS_TO_SKIP, + SCRIPT_PATH, + YQ_PATH, + _run_cmd, + clone_all_apps, + get_repos_json, + log, ) -log = getLogger(__name__) -basicConfig() -log.setLevel(DEBUG) - - -def _run_cmd(cmd, shell=False, **kwargs): - """Run a shell command.""" - args = [split(cmd)] - - if shell: - args = [cmd] - kwargs = {"shell": shell} - - try: - return check_output(*args, **kwargs).decode("utf-8").strip() - except Exception as exception: - log.error(f"Failed to run {cmd}, saw {str(exception)}") - exit(1) - - -def get_repos_json(): - """ Retrieve repo list from Gitea """ - - url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos" - - log.info(f"Retrieving {url}") - - repos = [] - response = True - page = 1 - - try: - while response: - log.info(f"Trying to fetch page {page}") - response = get(url + f"?page={page}", timeout=10).json() - repos.extend(response) - page += 1 - - return repos - except Exception as exception: - log.error(f"Failed to retrieve {url}, saw {str(exception)}") - exit(1) - def get_published_apps_json(): """Retrieve already published apps json.""" @@ -99,31 +39,6 @@ def get_published_apps_json(): return {} -def clone_all_apps(repos_json): - """Clone all Co-op Cloud apps to ~/.abra/apps.""" - if not exists(CLONES_PATH): - mkdir(CLONES_PATH) - - repos = [[p["name"], p["ssh_url"]] for p in repos_json] - - for name, url in repos: - if name in REPOS_TO_SKIP: - continue - - if not exists(f"{CLONES_PATH}/{name}"): - log.info(f"Retrieving {url}") - _run_cmd(f"git clone {url} {CLONES_PATH}/{name}") - - chdir(f"{CLONES_PATH}/{name}") - if not int(_run_cmd("git branch --list | wc -l", shell=True)): - log.info(f"Guessing main branch is HEAD for {name}") - _run_cmd("git checkout main") - else: - log.info(f"Updating {name}") - chdir(f"{CLONES_PATH}/{name}") - _run_cmd("git fetch -a") - - def generate_apps_json(repos_json): """Generate the abra-apps.json application versions file.""" apps_json = {} diff --git a/bin/clone-all-apps.py b/bin/clone-all-apps.py new file mode 100755 index 0000000..b9ec1f8 --- /dev/null +++ b/bin/clone-all-apps.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +# Usage: ./clone-all-apps.py +# +# Clone all available apps into ~/.abra/apps using ssh:// URLs + +from abralib import clone_all_apps, get_repos_json + + +def main(): + """Run the script.""" + repos_json = get_repos_json() + clone_all_apps(repos_json) + + +main() diff --git a/bin/renovate-ls-apps.py b/bin/renovate-ls-apps.py index 19839cb..1b602ac 100755 --- a/bin/renovate-ls-apps.py +++ b/bin/renovate-ls-apps.py @@ -4,77 +4,7 @@ # # Output list of apps for Renovate bot configuration -from logging import DEBUG, basicConfig, getLogger -from os.path import expanduser -from pathlib import Path -from shlex import split -from subprocess import check_output -from sys import exit - -from requests import get - -HOME_PATH = expanduser("~/") -CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute() -YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq") -SCRIPT_PATH = Path(__file__).absolute().parent -REPOS_TO_SKIP = ( - "abra", - "abra-apps", - "abra-gandi", - "abra-hetzner", - "backup-bot", - "coopcloud.tech", - "coturn", - "docker-cp-deploy", - "docker-dind-bats-kcov", - "docs.coopcloud.tech", - "example", - "gardening", - "organising", - "pyabra", - "radicle-seed-node", - "stack-ssh-deploy", - "swarm-cronjob", -) - -log = getLogger(__name__) -basicConfig() -log.setLevel(DEBUG) - - -def _run_cmd(cmd, shell=False, **kwargs): - """Run a shell command.""" - args = [split(cmd)] - - if shell: - args = [cmd] - kwargs = {"shell": shell} - - try: - return check_output(*args, **kwargs).decode("utf-8").strip() - except Exception as exception: - log.error(f"Failed to run {cmd}, saw {str(exception)}") - exit(1) - - -def get_repos_json(): - """ Retrieve repo list from Gitea """ - url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos" - - repos = [] - response = True - page = 1 - - try: - while response: - response = get(url + f"?page={page}", timeout=10).json() - repos.extend(response) - page += 1 - - return repos - except Exception as exception: - log.error(f"Failed to retrieve {url}, saw {str(exception)}") - exit(1) +from abralib import REPOS_TO_SKIP, get_repos_json def main():