postgres automatic schema management roughly working

This commit is contained in:
forest 2020-05-09 19:13:20 -05:00
commit 4a1924587c
9 changed files with 242 additions and 0 deletions

17
.gitignore vendored Normal file
View File

@ -0,0 +1,17 @@
notes.txt
.vscode
*.pyc
__pycache__/
instance/
.pytest_cache/
.coverage
htmlcov/
dist/
build/
*.egg-info/
.venv

41
README.md Normal file
View File

@ -0,0 +1,41 @@
# capsulflask
Python Flask web application for capsul.org
## postgres database schema management
capsulflask has a concept of a schema version. When the application starts, it will query the database for a table named
`schemaversion` that has one row and one column (`version`). If the `version` it finds is not equal to the `desiredSchemaVersion` variable set in `db.py`, it will run migration scripts from the `schema_migrations` folder one by one until the `schemaversion` table shows the correct version.
For example, the script named `02_up_xyz.sql` should contain code that migrates the database in a reverse-able fashion from schema version 1 to schema version 2. Likewise, the script `02_down_xyz.sql` should contain code that migrates from schema version 2 back to schema version 1.
**IMPORTANT: if you need to make changes to the schema, make a NEW schema version. DO NOT EDIT the existing schema versions.**
## how to run locally
Ensure you have the pre-requisites for the psycopg2 Postgres database adapter package
```
sudo apt-get install python3-dev libpq-dev
pg_config --version
```
Create python virtual environment and install packages
```
python3 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
```
Run an instance of Postgres (I used docker for this, point is its listening on localhost:5432)
```
docker run -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres
```
Run the app
```
FLASK_APP=capsulflask flask run
```

24
capsulflask/__init__.py Normal file
View File

@ -0,0 +1,24 @@
from flask import Flask
import os
def create_app():
app = Flask(__name__)
app.config.from_mapping(
SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"),
DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
)
from capsulflask import db
db.init_app(app)
# from capsulflask import auth, blog
# app.register_blueprint(auth.bp)
# app.register_blueprint(blog.bp)
app.add_url_rule("/", endpoint="index")
return app

117
capsulflask/db.py Normal file
View File

@ -0,0 +1,117 @@
import psycopg2
import re
import sys
from urllib.parse import urlparse
from os import listdir
from os.path import isfile, join
from psycopg2 import pool
from flask import current_app
from flask import g
def init_app(app):
databaseUrl = urlparse(app.config['DATABASE_URL'])
app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool(
1,
20,
user = databaseUrl.username,
password = databaseUrl.password,
host = databaseUrl.hostname,
port = databaseUrl.port,
database = databaseUrl.path[1:]
)
schemaMigrations = {}
schemaMigrationsPath = join(app.root_path, 'schema_migrations')
print("loading schema migration scripts from {}".format(schemaMigrationsPath))
for filename in listdir(schemaMigrationsPath):
key = re.search(r"^\d+_(up|down)", filename).group()
with open(join(schemaMigrationsPath, filename), 'rb') as file:
schemaMigrations[key] = file.read().decode("utf8")
db = app.config['PSYCOPG2_CONNECTION_POOL'].getconn()
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 2
cursor = db.cursor()
cursor.execute("""
SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema = '{}'
""".format(app.config['DATABASE_SCHEMA']))
rows = cursor.fetchall()
for row in rows:
if row[0] == "schemaversion":
hasSchemaVersionTable = True
if hasSchemaVersionTable == False:
print("no table named schemaversion found in the {} schema. running migration 01_up".format(app.config['DATABASE_SCHEMA']))
try:
cursor.execute(schemaMigrations["01_up"])
db.commit()
except:
print("unable to create the schemaversion table because: {}".format(my_exec_info_message(sys.exc_info())))
exit(1)
actionWasTaken = True
cursor.execute("SELECT Version FROM schemaversion")
schemaVersion = cursor.fetchall()[0][0]
# print(schemaVersion)
while schemaVersion < desiredSchemaVersion:
migrationKey = "%02d_up" % (schemaVersion+1)
print("schemaVersion ({}) < desiredSchemaVersion ({}). running migration {}".format(
schemaVersion, desiredSchemaVersion, migrationKey
))
try:
cursor.execute(schemaMigrations[migrationKey])
db.commit()
except KeyError:
print("missing schema migration script: {}_xyz.sql".format(migrationKey))
exit(1)
except:
print("unable to execute the schema migration {} because: {}".format(migrationKey, my_exec_info_message(sys.exc_info())))
exit(1)
actionWasTaken = True
schemaVersion += 1
cursor.execute("SELECT Version FROM schemaversion")
versionFromDatabase = cursor.fetchall()[0][0]
if schemaVersion != versionFromDatabase:
print("incorrect schema version value \"{}\" after running migration {}, expected \"{}\". exiting.".format(
versionFromDatabase,
migrationKey,
schemaVersion
))
exit(1)
cursor.close()
app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db)
print("schema migration completed. {}current schemaVersion: \"{}\"".format(
("" if actionWasTaken else "(no action was taken). "), schemaVersion
))
app.teardown_appcontext(close_db)
def get_db():
if 'db' not in g:
g.db = current_app.config['PSYCOPG2_CONNECTION_POOL'].getconn()
return g.db
def close_db(e=None):
db = g.pop("db", None)
if db is not None:
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db)
def my_exec_info_message(exec_info):
return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])

View File

@ -0,0 +1,5 @@
CREATE TABLE schemaversion (
version INT PRIMARY KEY NOT NULL
);
INSERT INTO schemaversion(version) VALUES (1);

View File

@ -0,0 +1 @@
UPDATE schemaversion SET version = 2;

17
requirements.txt Normal file
View File

@ -0,0 +1,17 @@
astroid==2.4.1
click==7.1.2
Flask==1.1.2
isort==4.3.21
itsdangerous==1.1.0
Jinja2==2.11.2
lazy-object-proxy==1.4.3
MarkupSafe==1.1.1
mccabe==0.6.1
pkg-resources==0.0.0
psycopg2==2.8.5
pylint==2.5.2
six==1.14.0
toml==0.10.0
typed-ast==1.4.1
Werkzeug==1.0.1
wrapt==1.12.1

17
setup.cfg Normal file
View File

@ -0,0 +1,17 @@
[metadata]
name = capsulflask
version = 0.0.0
url = https://giit.cyberia.club/~forest/capsul-flask
license = BSD-3-Clause
maintainer = cyberia
maintainer_email = forest.n.johnson@gmail.com
description = Python Flask web application for capsul.org
long_description = file: README.md
long_description_content_type = text/markdown
[options]
packages = find:
include_package_data = true
install_requires =
Flask
psycopg2

3
setup.py Normal file
View File

@ -0,0 +1,3 @@
from setuptools import setup
setup()