CI, celery, compose setup, hackity hack hack

This commit is contained in:
Luke Murphy 2020-07-05 01:21:17 +02:00
parent a05cfe3337
commit b914d1ae73
No known key found for this signature in database
GPG Key ID: 5E2EF5A63E3718CC
13 changed files with 89 additions and 44 deletions

View File

@ -1,10 +1,6 @@
--- ---
matrix: matrix:
include: include:
- IMAGE: 3.6-stretch
TOXENV: py36
- IMAGE: 3.7-stretch
TOXENV: py37
- IMAGE: 3.8-buster - IMAGE: 3.8-buster
TOXENV: py38 TOXENV: py38
- IMAGE: 3.8-buster - IMAGE: 3.8-buster
@ -15,14 +11,10 @@ matrix:
TOXENV: format TOXENV: format
- IMAGE: 3.8-buster - IMAGE: 3.8-buster
TOXENV: type TOXENV: type
- IMAGE: 3.8-buster
TOXENV: docs
- IMAGE: 3.8-buster
TOXENV: metadata-release
pipeline: pipeline:
build: build:
image: python:${IMAGE} image: python:${IMAGE}
commands: commands:
- pip install tox==3.14.6 - pip install tox
- tox -e ${TOXENV} - tox -e ${TOXENV}

View File

@ -1,6 +1,7 @@
export CELERY_BROKER_URL=redis://localhost:6379/1 export CELERY_BROKER_URL=redis://localhost:6379
export CELERY_RESULT_BACKEND=redis://localhost:6379/1 export CELERY_RESULT_BACKEND=redis://localhost:6379
export ENV=development export FLASK_ENV=development
export FLASK_APP=wsgi:app
export REDIS_HOST=localhost export REDIS_HOST=localhost
export REDIS_PORT=6379 export REDIS_PORT=6379
export REDIS_SESSION_DB=0 export REDIS_SESSION_DB=0

View File

@ -12,15 +12,18 @@ RUN apk add --update \
git \ git \
libffi-dev \ libffi-dev \
libsasl \ libsasl \
openssl-dev \
python3-dev python3-dev
RUN addgroup -S ${CELERY_USER} RUN addgroup -S ${CELERY_USER}
RUN adduser -D \ RUN adduser -D \
-h ${APP_ROOT} \ -h ${APP_ROOT} \
-s /usr/sbin/nologin \ -s /usr/sbin/nologin \
-G ${CELERY_USER} ${CELERY_USER} -G ${CELERY_USER} \
${CELERY_USER}
RUN pip install "poetry>=1.0.9,<2.0" RUN pip install "poetry>=1.0.9,<2.0"
RUN poetry install RUN poetry install \
--no-dev \
--no-interaction

View File

@ -4,4 +4,8 @@ A swarm of dreams.
# Development # Development
TODO. ```
$ docker-compose up
```
Then see http://localhost:5000.

BIN
celerybeat-schedule Normal file

Binary file not shown.

View File

@ -4,6 +4,6 @@ from os import environ
from magic_app.app import celery, create_app # noqa from magic_app.app import celery, create_app # noqa
from magic_app.config import CONFIG from magic_app.config import CONFIG
config = CONFIG[environ["ENV"]] config = CONFIG[environ["FLASK_ENV"]]
app = create_app(config=config) app = create_app(config=config)
app.app_context().push() app.app_context().push()

36
docker-compose.yml Normal file
View File

@ -0,0 +1,36 @@
---
version: "3.8"
x-env: &env
environment:
CELERY_BROKER_URL: "redis://redis:6379"
CELERY_RESULT_BACKEND: "redis://redis:6379"
FLASK_APP: "wsgi:app"
FLASK_ENV: "development"
REDIS_HOST: "redis"
REDIS_PORT: "6379"
REDIS_SESSION_DB: "0"
services:
flask:
build: .
<<: *env
ports:
- "5000:5000"
command: |
poetry run
flask run --host 0.0.0.0
volumes:
- "./:/magic_app/"
celery:
build: .
<<: *env
command: |
poetry run
celery worker -A celworker.celery -B -l debug
volumes:
- "./:/magic_app/"
redis:
image: redis:alpine

View File

@ -7,7 +7,9 @@ from redis import Redis
from magic_app.config import Base, Production from magic_app.config import Base, Production
celery = Celery(__name__, broker=Base.CELERY_BROKER_URL) celery = Celery(
__name__, backend=Base.CELERY_RESULT_BACKEND, broker=Base.CELERY_BROKER_URL
)
def create_app(config=Production): def create_app(config=Production):
@ -15,8 +17,7 @@ def create_app(config=Production):
app = Flask(__name__.split(".")[0]) app = Flask(__name__.split(".")[0])
app.config.from_object(config) app.config.from_object(config)
celery.conf.update(app.config) configure_celery(app)
configure_redis(app) configure_redis(app)
configure_views(app) configure_views(app)
configure_logging(app) configure_logging(app)
@ -28,11 +29,11 @@ def configure_redis(app):
"""Configure Redis connection.""" """Configure Redis connection."""
from magic_app.session import RedisSessionDB from magic_app.session import RedisSessionDB
host = (Base.REDIS_HOST,) host = Base.REDIS_HOST
port = (Base.REDIS_PORT,) port = Base.REDIS_PORT
db_num = Base.REDIS_SESSION_DB db_num = Base.REDIS_SESSION_DB
if app.debug: if app.testing:
from fakeredis import FakeRedis from fakeredis import FakeRedis
connection = FakeRedis() connection = FakeRedis()
@ -42,6 +43,18 @@ def configure_redis(app):
app.config["SESSION"] = RedisSessionDB(connection) app.config["SESSION"] = RedisSessionDB(connection)
def configure_celery(app):
"""Configure celery."""
celery.conf.update(app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
def configure_views(app): def configure_views(app):
"""Configure API resource views.""" """Configure API resource views."""
from magic_app.views import home from magic_app.views import home

View File

@ -25,14 +25,16 @@ class Development(Base):
"""The Development configuration.""" """The Development configuration."""
ENV = "development" ENV = "development"
DEBUG = True
CELERY_ALWAYS_EAGER = True CELERY_ALWAYS_EAGER = True
DEBUG = True
class Testing(Base): class Testing(Base):
"""The testing configuration.""" """The testing configuration."""
ENV = "testing" ENV = "testing"
TESTING = True
class Production(Base): class Production(Base):

View File

@ -6,4 +6,5 @@ home = Blueprint("home", __name__)
@home.route("/") @home.route("/")
def hello_world(): def hello_world():
return "Hello, World" return "Hello, World"

View File

@ -16,7 +16,8 @@ keywords = ["docker", "swarm", "packaging"]
celery = "^4.4.6" celery = "^4.4.6"
flask = "^1.1.2" flask = "^1.1.2"
flask-wtf = "^0.14.3" flask-wtf = "^0.14.3"
python = "^3.6" gunicorn = "^20.0.4"
python = "^3.8"
redis = "^3.5.3" redis = "^3.5.3"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]

14
tox.ini
View File

@ -1,6 +1,6 @@
[tox] [tox]
envlist = envlist =
{py36,py37,py38} py38
lint lint
sort sort
format format
@ -9,12 +9,6 @@ skip_missing_interpreters = True
isolated_build = True isolated_build = True
[testenv] [testenv]
description = run the unit tests
deps =
pytest
pytest-cov
pytest-mock
commands = pytest test/ --cov={toxinidir}/magic_app/ --no-cov-on-fail {posargs}
[testenv:lint] [testenv:lint]
description = lint the source description = lint the source
@ -26,25 +20,23 @@ commands = flake8 {posargs} magic_app/ test/
description = sort the source description = sort the source
skipdist = True skipdist = True
deps = isort deps = isort
commands = isort {posargs:-rc -c} -sp setup.cfg magic_app/ test/ commands = isort {posargs:-c} -sp setup.cfg magic_app/ test/
[testenv:format] [testenv:format]
description = format the source description = format the source
skipdist = True skipdist = True
basepython = python3.8
deps = black deps = black
commands = black {posargs:--check} magic_app/ test/ commands = black {posargs:--check} magic_app/ test/
[testenv:type] [testenv:type]
description = type check the source description = type check the source
basepython = python3.8
skipdist = True skipdist = True
deps = mypy deps = mypy
commands = mypy magic_app/ test/ commands = mypy magic_app/ test/
[testenv:release] [testenv:release]
description = make a release description = make a release
deps = {[testenv:metadata-release]deps} deps = twine
commands = commands =
rm -rf {toxworkdir}/dist rm -rf {toxworkdir}/dist
python -m setup sdist --dist-dir {toxworkdir}/dist bdist_wheel python -m setup sdist --dist-dir {toxworkdir}/dist bdist_wheel

View File

@ -4,5 +4,5 @@ import os
from magic_app.app import create_app from magic_app.app import create_app
from magic_app.config import CONFIG from magic_app.config import CONFIG
config = CONFIG[os.environ["ENV"]] config = CONFIG[os.environ["FLASK_ENV"]]
app = create_app(config=config) app = create_app(config=config)