postgres automatic schema management roughly working

.gitignore

@@ -0,0 +1,17 @@
+notes.txt
+.vscode
+
+*.pyc
+__pycache__/
+
+instance/
+
+.pytest_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+*.egg-info/
+
+.venv

README.md

@@ -0,0 +1,41 @@
+# capsulflask
+
+Python Flask web application for capsul.org
+
+## postgres database schema management
+
+capsulflask has a concept of a schema version. When the application starts, it will query the database for a table named
+`schemaversion` that has one row and one column (`version`). If the `version` it finds is not equal to the `desiredSchemaVersion` variable set in `db.py`, it will run  migration scripts from the `schema_migrations` folder one by one until the `schemaversion` table shows the correct version.
+
+For example, the script named `02_up_xyz.sql` should contain code that migrates the database in a reverse-able fashion from schema version 1 to schema version 2. Likewise, the script `02_down_xyz.sql` should contain code that migrates from schema version 2 back to schema version 1.
+
+**IMPORTANT: if you need to make changes to the schema, make a NEW schema version. DO NOT EDIT the existing schema versions.**
+
+## how to run locally
+
+Ensure you have the pre-requisites for the psycopg2 Postgres database adapter package
+
+```
+sudo apt-get install python3-dev libpq-dev
+pg_config --version
+```
+
+Create python virtual environment and install packages
+
+```
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+Run an instance of Postgres (I used docker for this, point is its listening on localhost:5432)
+
+```
+docker run -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres
+```
+
+Run the app 
+
+```
+FLASK_APP=capsulflask flask run
+```

capsulflask/__init__.py

@@ -0,0 +1,24 @@
+
+from flask import Flask
+import os
+
+def create_app():
+  app = Flask(__name__)
+  app.config.from_mapping(
+    SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"),
+    DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
+    DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
+  )
+
+  from capsulflask import db
+
+  db.init_app(app)
+
+  # from capsulflask import auth, blog
+
+  # app.register_blueprint(auth.bp)
+  # app.register_blueprint(blog.bp)
+
+  app.add_url_rule("/", endpoint="index")
+
+  return app

capsulflask/db.py

@@ -0,0 +1,117 @@
+import psycopg2
+import re
+import sys
+from urllib.parse import urlparse
+from os import listdir
+from os.path import isfile, join
+from psycopg2 import pool
+from flask import current_app
+from flask import g
+
+
+def init_app(app):
+  databaseUrl = urlparse(app.config['DATABASE_URL'])
+
+  app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool(
+    1, 
+    20,
+    user = databaseUrl.username,
+    password = databaseUrl.password,
+    host = databaseUrl.hostname,
+    port = databaseUrl.port,
+    database = databaseUrl.path[1:]
+  )
+
+  schemaMigrations = {}
+  schemaMigrationsPath = join(app.root_path, 'schema_migrations')
+  print("loading schema migration scripts from {}".format(schemaMigrationsPath))
+  for filename in listdir(schemaMigrationsPath):
+    key = re.search(r"^\d+_(up|down)", filename).group()
+    with open(join(schemaMigrationsPath, filename), 'rb') as file:
+      schemaMigrations[key] = file.read().decode("utf8")
+  
+  db = app.config['PSYCOPG2_CONNECTION_POOL'].getconn()
+
+  hasSchemaVersionTable = False
+  actionWasTaken = False
+  schemaVersion = 0
+  desiredSchemaVersion = 2
+
+  cursor = db.cursor()
+
+  cursor.execute("""
+    SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema = '{}'
+  """.format(app.config['DATABASE_SCHEMA']))
+
+  rows = cursor.fetchall()
+  for row in rows:
+    if row[0] == "schemaversion":
+      hasSchemaVersionTable = True
+
+  if hasSchemaVersionTable == False:
+    print("no table named schemaversion found in the {} schema. running migration 01_up".format(app.config['DATABASE_SCHEMA']))
+    try:
+      cursor.execute(schemaMigrations["01_up"])
+      db.commit()
+    except:
+      print("unable to create the schemaversion table because: {}".format(my_exec_info_message(sys.exc_info())))
+      exit(1)
+    actionWasTaken = True
+
+  cursor.execute("SELECT Version FROM schemaversion")
+  schemaVersion = cursor.fetchall()[0][0]
+  
+  # print(schemaVersion)
+  while schemaVersion < desiredSchemaVersion:
+    migrationKey = "%02d_up" % (schemaVersion+1)
+    print("schemaVersion ({}) < desiredSchemaVersion ({}). running migration {}".format(
+      schemaVersion, desiredSchemaVersion, migrationKey
+    ))
+    try:
+      cursor.execute(schemaMigrations[migrationKey])
+      db.commit()
+    except KeyError:
+      print("missing schema migration script: {}_xyz.sql".format(migrationKey))
+      exit(1)
+    except:
+      print("unable to execute the schema migration {} because: {}".format(migrationKey, my_exec_info_message(sys.exc_info())))
+      exit(1)
+    actionWasTaken = True
+
+    schemaVersion += 1
+    cursor.execute("SELECT Version FROM schemaversion")
+    versionFromDatabase = cursor.fetchall()[0][0]
+
+    if schemaVersion != versionFromDatabase:
+      print("incorrect schema version value \"{}\" after running migration {}, expected \"{}\". exiting.".format(
+        versionFromDatabase,
+        migrationKey, 
+        schemaVersion
+      ))
+      exit(1)
+  
+  cursor.close()
+
+  app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db)
+
+  print("schema migration completed. {}current schemaVersion: \"{}\"".format(
+    ("" if actionWasTaken else "(no action was taken). "), schemaVersion
+  ))
+
+  app.teardown_appcontext(close_db)
+
+
+def get_db():
+    if 'db' not in g:
+        g.db = current_app.config['PSYCOPG2_CONNECTION_POOL'].getconn()
+    return g.db
+
+
+def close_db(e=None):
+  db = g.pop("db", None)
+
+  if db is not None:
+    current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db)
+
+def my_exec_info_message(exec_info):
+  return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])

capsulflask/schema_migrations/01_up_create_SchemaVersion.sql

@@ -0,0 +1,5 @@
+CREATE TABLE schemaversion (
+   version INT PRIMARY KEY NOT NULL
+);
+
+INSERT INTO schemaversion(version) VALUES (1);

capsulflask/schema_migrations/02_up_test.sql

@@ -0,0 +1 @@
+UPDATE schemaversion SET version = 2;

requirements.txt

@@ -0,0 +1,17 @@
+astroid==2.4.1
+click==7.1.2
+Flask==1.1.2
+isort==4.3.21
+itsdangerous==1.1.0
+Jinja2==2.11.2
+lazy-object-proxy==1.4.3
+MarkupSafe==1.1.1
+mccabe==0.6.1
+pkg-resources==0.0.0
+psycopg2==2.8.5
+pylint==2.5.2
+six==1.14.0
+toml==0.10.0
+typed-ast==1.4.1
+Werkzeug==1.0.1
+wrapt==1.12.1

setup.cfg

@@ -0,0 +1,17 @@
+[metadata]
+name = capsulflask
+version = 0.0.0
+url = https://giit.cyberia.club/~forest/capsul-flask
+license = BSD-3-Clause
+maintainer = cyberia
+maintainer_email = forest.n.johnson@gmail.com
+description = Python Flask web application for capsul.org
+long_description = file: README.md
+long_description_content_type = text/markdown
+
+[options]
+packages = find:
+include_package_data = true
+install_requires =
+    Flask
+    psycopg2

setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()