Merge branch 'webapi' into docker-api

.. using server-side API tokens

.drone.yml

@@ -0,0 +1,14 @@
+---
+kind: pipeline
+name: publish docker image
+steps:
+  - name: build and publish
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_reg_username_3wc
+      password:
+        from_secret: docker_reg_passwd_3wc
+      repo: 3wordchant/capsul-flask
+      tags: latest
+

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.8-alpine as build
+
+RUN apk add gettext git gcc python3-dev musl-dev \
+    libffi-dev zlib-dev jpeg-dev libjpeg postgresql-dev build-base \
+    --virtual .build-dependencies
+
+RUN mkdir -p /app/{code,venv}
+WORKDIR /app/code
+COPY Pipfile Pipfile.lock /app/code/
+
+RUN python3 -m venv /app/venv
+RUN pip install pipenv setuptools
+ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
+RUN pip install wheel cppy
+# Install dependencies into the virtual environment with Pipenv
+RUN pipenv install --deploy --verbose
+
+FROM python:3.8-alpine
+
+RUN apk add --no-cache libpq libstdc++ libjpeg
+
+COPY . /app/code/
+WORKDIR /app/code
+
+COPY --from=build /app/venv /app/venv
+ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
+
+CMD ["gunicorn", "--bind", "0.0.0.0:5000", "-k", "gevent", "--worker-connections", "1000", "app:app"]
+
+VOLUME /app/code
+
+EXPOSE 5000

Pipfile

@@ -9,7 +9,6 @@ blinker = "==1.4"
 click = "==7.1.2"
 Flask = "==1.1.2"
 Flask-Mail = "==0.9.1"
-Flask-Testing = "==0.8.1"
 gunicorn = "==20.0.4"
 isort = "==4.3.21"
 itsdangerous = "==1.1.0"

app.py

@@ -1,4 +1,2 @@
 
-from capsulflask import create_app
-
-create_app()
+from capsulflask import app

capsulflask/__init__.py

@@ -8,7 +8,7 @@ import requests
 import sys
 
 import stripe
-from dotenv import find_dotenv, dotenv_values
+from dotenv import load_dotenv, find_dotenv
 from flask import Flask
 from flask_mail import Mail, Message
 from flask import render_template
@@ -22,112 +22,106 @@ from capsulflask import hub_model, spoke_model, cli
 from capsulflask.btcpay import client as btcpay
 from capsulflask.http_client import MyHTTPClient
 
-
 class StdoutMockFlaskMail:
     def send(self, message: Message):
       current_app.logger.info(f"Email would have been sent if configured:\n\nto: {','.join(message.recipients)}\nsubject: {message.subject}\nbody:\n\n{message.body}\n\n")
 
-def create_app():
+load_dotenv(find_dotenv())
 
-  config = {
-      **dotenv_values(find_dotenv()),
-      **os.environ,  # override loaded values with environment variables
-  }
+app = Flask(__name__)
 
-  app = Flask(__name__)
+app.config.from_mapping(
+  
+  BASE_URL=os.environ.get("BASE_URL", default="http://localhost:5000"),
+  SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"),
+  HUB_MODE_ENABLED=os.environ.get("HUB_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
+  SPOKE_MODE_ENABLED=os.environ.get("SPOKE_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
+  INTERNAL_HTTP_TIMEOUT_SECONDS=os.environ.get("INTERNAL_HTTP_TIMEOUT_SECONDS", default="300"),
+  HUB_MODEL=os.environ.get("HUB_MODEL", default="capsul-flask"),
+  SPOKE_MODEL=os.environ.get("SPOKE_MODEL", default="mock"),
+  LOG_LEVEL=os.environ.get("LOG_LEVEL", default="INFO"),
+  SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="baikal"),
+  SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="changeme"),
+  HUB_TOKEN=os.environ.get("HUB_TOKEN", default="changeme"),
 
-  app.config.from_mapping(
-    TESTING=config.get("TESTING", False),
-    BASE_URL=config.get("BASE_URL", "http://localhost:5000"),
-    SECRET_KEY=config.get("SECRET_KEY", "dev"),
-    HUB_MODE_ENABLED=config.get("HUB_MODE_ENABLED", "True").lower() in ['true', '1', 't', 'y', 'yes'],
-    SPOKE_MODE_ENABLED=config.get("SPOKE_MODE_ENABLED", "True").lower() in ['true', '1', 't', 'y', 'yes'],
-    INTERNAL_HTTP_TIMEOUT_SECONDS=config.get("INTERNAL_HTTP_TIMEOUT_SECONDS", "300"),
-    HUB_MODEL=config.get("HUB_MODEL", "capsul-flask"),
-    SPOKE_MODEL=config.get("SPOKE_MODEL", "mock"),
-    LOG_LEVEL=config.get("LOG_LEVEL", "INFO"),
-    SPOKE_HOST_ID=config.get("SPOKE_HOST_ID", "baikal"),
-    SPOKE_HOST_TOKEN=config.get("SPOKE_HOST_TOKEN", "changeme"),
-    HUB_TOKEN=config.get("HUB_TOKEN", "changeme"),
+  # https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
+  # https://stackoverflow.com/questions/56332906/where-to-put-ssl-certificates-when-trying-to-connect-to-a-remote-database-using
+  # TLS example: sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=db.example.com port=5432 user=postgres password=dev dbname=postgres
+  POSTGRES_CONNECTION_PARAMETERS=os.environ.get(
+    "POSTGRES_CONNECTION_PARAMETERS", 
+    default="host=localhost port=5432 user=postgres password=dev dbname=postgres"
+  ),
 
-    # https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
-    # https://stackoverflow.com/questions/56332906/where-to-put-ssl-certificates-when-trying-to-connect-to-a-remote-database-using
-    # TLS example: sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=db.example.com port=5432 user=postgres password=dev dbname=postgres
-    POSTGRES_CONNECTION_PARAMETERS=config.get(
-      "POSTGRES_CONNECTION_PARAMETERS", 
-      "host=localhost port=5432 user=postgres password=dev dbname=postgres"
-    ),
+  DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
 
-    DATABASE_SCHEMA=config.get("DATABASE_SCHEMA", "public"),
+  MAIL_SERVER=os.environ.get("MAIL_SERVER", default=""),
+  MAIL_PORT=os.environ.get("MAIL_PORT", default="465"),
+  MAIL_USE_TLS=os.environ.get("MAIL_USE_TLS", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
+  MAIL_USE_SSL=os.environ.get("MAIL_USE_SSL", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
+  MAIL_USERNAME=os.environ.get("MAIL_USERNAME", default=""),
+  MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
+  MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
+  ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"),
+  ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"),
 
-    MAIL_SERVER=config.get("MAIL_SERVER", ""),
-    MAIL_PORT=config.get("MAIL_PORT", "465"),
-    MAIL_USE_TLS=config.get("MAIL_USE_TLS", "False").lower() in ['true', '1', 't', 'y', 'yes'],
-    MAIL_USE_SSL=config.get("MAIL_USE_SSL", "True").lower() in ['true', '1', 't', 'y', 'yes'],
-    MAIL_USERNAME=config.get("MAIL_USERNAME", ""),
-    MAIL_PASSWORD=config.get("MAIL_PASSWORD", ""),
-    MAIL_DEFAULT_SENDER=config.get("MAIL_DEFAULT_SENDER", "no-reply@capsul.org"),
-    ADMIN_EMAIL_ADDRESSES=config.get("ADMIN_EMAIL_ADDRESSES", "ops@cyberia.club"),
-    ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=config.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", "forest.n.johnson@gmail.com,capsul@cyberia.club"),
+  PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
 
-    PROMETHEUS_URL=config.get("PROMETHEUS_URL", "https://prometheus.cyberia.club"),
+  STRIPE_API_VERSION=os.environ.get("STRIPE_API_VERSION", default="2020-03-02"),
+  STRIPE_SECRET_KEY=os.environ.get("STRIPE_SECRET_KEY", default=""),
+  STRIPE_PUBLISHABLE_KEY=os.environ.get("STRIPE_PUBLISHABLE_KEY", default=""),
+  #STRIPE_WEBHOOK_SECRET=os.environ.get("STRIPE_WEBHOOK_SECRET", default="")
 
-    STRIPE_API_VERSION=config.get("STRIPE_API_VERSION", "2020-03-02"),
-    STRIPE_SECRET_KEY=config.get("STRIPE_SECRET_KEY", ""),
-    STRIPE_PUBLISHABLE_KEY=config.get("STRIPE_PUBLISHABLE_KEY", ""),
-    #STRIPE_WEBHOOK_SECRET=config.get("STRIPE_WEBHOOK_SECRET", "")
+  BTCPAY_PRIVATE_KEY=os.environ.get("BTCPAY_PRIVATE_KEY", default="").replace("\\n", "\n"),
+  BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club")
+)
 
-    BTCPAY_PRIVATE_KEY=config.get("BTCPAY_PRIVATE_KEY", "").replace("\\n", "\n"),
-    BTCPAY_URL=config.get("BTCPAY_URL", "https://btcpay.cyberia.club")
-  )
+app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL'])
 
-  app.config['HUB_URL'] = config.get("HUB_URL", app.config['BASE_URL'])
-
-  class SetLogLevelToDebugForHeartbeatRelatedMessagesFilter(logging.Filter):
-    def isHeartbeatRelatedString(self, thing):
-      # thing_string = "<error>"
-      is_in_string = False
-      try:
-        thing_string = "%s" % thing
-        is_in_string = 'heartbeat-task' in thing_string or 'hub/heartbeat' in thing_string or 'spoke/heartbeat' in thing_string
-      except:
-        pass
-      # self.warning("isHeartbeatRelatedString(%s): %s", thing_string, is_in_string )
-      return is_in_string
-
-    def filter(self, record):
-      if app.config['LOG_LEVEL'] == "DEBUG":
-        return True
-
-      if self.isHeartbeatRelatedString(record.msg):
-        return False
-      for arg in record.args:
-        if self.isHeartbeatRelatedString(arg):
-          return False
+class SetLogLevelToDebugForHeartbeatRelatedMessagesFilter(logging.Filter):
+  def isHeartbeatRelatedString(self, thing):
+    # thing_string = "<error>"
+    is_in_string = False
+    try:
+      thing_string = "%s" % thing
+      is_in_string = 'heartbeat-task' in thing_string or 'hub/heartbeat' in thing_string or 'spoke/heartbeat' in thing_string
+    except:
+      pass
+    # self.warning("isHeartbeatRelatedString(%s): %s", thing_string, is_in_string )
+    return is_in_string
 
+  def filter(self, record):
+    if app.config['LOG_LEVEL'] == "DEBUG":
       return True
 
-  logging_dict_config({
-    'version': 1,
-    'formatters': {'default': {
-      'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
-    }},
-    'filters': {
-      'setLogLevelToDebugForHeartbeatRelatedMessages': {
-          '()': SetLogLevelToDebugForHeartbeatRelatedMessagesFilter,
-      }
-    },
-    'handlers': {'wsgi': {
-      'class': 'logging.StreamHandler',
-      'stream': 'ext://flask.logging.wsgi_errors_stream',
-      'formatter': 'default',
-      'filters': ['setLogLevelToDebugForHeartbeatRelatedMessages']
-    }},
-    'root': {
-      'level': app.config['LOG_LEVEL'],
-      'handlers': ['wsgi']
+    if self.isHeartbeatRelatedString(record.msg):
+      return False
+    for arg in record.args:
+      if self.isHeartbeatRelatedString(arg):
+        return False
+
+    return True
+
+logging_dict_config({
+  'version': 1,
+  'formatters': {'default': {
+    'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+  }},
+  'filters': {
+    'setLogLevelToDebugForHeartbeatRelatedMessages': {
+        '()': SetLogLevelToDebugForHeartbeatRelatedMessagesFilter,
     }
-  })
+  },
+  'handlers': {'wsgi': {
+    'class': 'logging.StreamHandler',
+    'stream': 'ext://flask.logging.wsgi_errors_stream',
+    'formatter': 'default',
+    'filters': ['setLogLevelToDebugForHeartbeatRelatedMessages']
+  }},
+  'root': {
+    'level': app.config['LOG_LEVEL'],
+    'handlers': ['wsgi']
+  }
+})
 
 # app.logger.critical("critical")
 # app.logger.error("error")
@@ -135,61 +129,57 @@ def create_app():
 # app.logger.info("info")
 # app.logger.debug("debug")
 
-  stripe.api_key = app.config['STRIPE_SECRET_KEY']
-  stripe.api_version = app.config['STRIPE_API_VERSION']
+stripe.api_key = app.config['STRIPE_SECRET_KEY']
+stripe.api_version = app.config['STRIPE_API_VERSION']
 
-  if app.config['MAIL_SERVER'] != "":
-    app.config['FLASK_MAIL_INSTANCE'] = Mail(app)
-  else:
-    app.logger.warning("No MAIL_SERVER configured. capsul will simply print emails to stdout.")
-    app.config['FLASK_MAIL_INSTANCE'] = StdoutMockFlaskMail()
+if app.config['MAIL_SERVER'] != "":
+  app.config['FLASK_MAIL_INSTANCE'] = Mail(app)
+else:
+  app.logger.warning("No MAIL_SERVER configured. capsul will simply print emails to stdout.")
+  app.config['FLASK_MAIL_INSTANCE'] = StdoutMockFlaskMail()
 
-  app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
+app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
 
-  try:
-    app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
-  except:
-    app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
+try:
+  app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
+except:
+  app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
 
 # only start the scheduler and attempt to migrate the database if we are running the app.
 # otherwise we are running a CLI command.
-  command_line = ' '.join(sys.argv)
-  is_running_server = (
-    ('flask run' in command_line) or
-    ('gunicorn' in command_line) or
-    ('test' in command_line)
-  )
+command_line = ' '.join(sys.argv)
+is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
 
-  app.logger.info(f"is_running_server: {is_running_server}")
+app.logger.info(f"is_running_server: {is_running_server}")
 
-  if app.config['HUB_MODE_ENABLED']:
-    if app.config['HUB_MODEL'] == "capsul-flask":
-      app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
+if app.config['HUB_MODE_ENABLED']:
+  if app.config['HUB_MODEL'] == "capsul-flask":
+    app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
 
-      # debug mode (flask reloader) runs two copies of the app. When running in debug mode,
-      # we only want to start the scheduler one time.
-      if is_running_server and (not app.debug or config.get('WERKZEUG_RUN_MAIN') == 'true'):
-        scheduler = BackgroundScheduler()
-        heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
-        heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
-        heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
-        scheduler.add_job(name="heartbeat-task", func=heartbeat_task, trigger="interval", seconds=5)
-        scheduler.start()
+    # debug mode (flask reloader) runs two copies of the app. When running in debug mode,
+    # we only want to start the scheduler one time.
+    if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
+      scheduler = BackgroundScheduler()
+      heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
+      heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
+      heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
+      scheduler.add_job(name="heartbeat-task", func=heartbeat_task, trigger="interval", seconds=5)
+      scheduler.start()
 
-        atexit.register(lambda: scheduler.shutdown())
+      atexit.register(lambda: scheduler.shutdown())
 
-    else:
-      app.config['HUB_MODEL'] = hub_model.MockHub()
+  else:
+    app.config['HUB_MODEL'] = hub_model.MockHub()
 
-    from capsulflask import db
-    db.init_app(app, is_running_server)
+  from capsulflask import db
+  db.init_app(app, is_running_server)
 
   from capsulflask import (
     auth, landing, console, payment, metrics, cli, hub_api, publicapi, admin
   )
 
-  app.register_blueprint(auth.bp)
   app.register_blueprint(landing.bp)
+  app.register_blueprint(auth.bp)
   app.register_blueprint(console.bp)
   app.register_blueprint(payment.bp)
   app.register_blueprint(metrics.bp)
@@ -200,57 +190,57 @@ def create_app():
 
   app.add_url_rule("/", endpoint="index")
 
-  if app.config['SPOKE_MODE_ENABLED']:
-    if app.config['SPOKE_MODEL'] == "shell-scripts":
-      app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
-    else:
-      app.config['SPOKE_MODEL'] = spoke_model.MockSpoke()
-
-    from capsulflask import spoke_api
-
-    app.register_blueprint(spoke_api.bp)
-
-  @app.after_request
-  def security_headers(response):
-    response.headers['X-Frame-Options'] = 'SAMEORIGIN'
-    if 'Content-Security-Policy' not in response.headers:
-      response.headers['Content-Security-Policy'] = "default-src 'self'"
-    response.headers['X-Content-Type-Options'] = 'nosniff'
-    return response
 
 
-  @app.context_processor
-  def override_url_for():
-    """
-    override the url_for function built into flask 
-    with our own custom implementation that busts the cache correctly when files change 
-    """
-    return dict(url_for=url_for_with_cache_bust)
+if app.config['SPOKE_MODE_ENABLED']:
+  if app.config['SPOKE_MODEL'] == "shell-scripts":
+    app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
+  else:
+    app.config['SPOKE_MODEL'] = spoke_model.MockSpoke()
+
+  from capsulflask import spoke_api
+
+  app.register_blueprint(spoke_api.bp)
+
+@app.after_request
+def security_headers(response):
+  response.headers['X-Frame-Options'] = 'SAMEORIGIN'
+  if 'Content-Security-Policy' not in response.headers:
+    response.headers['Content-Security-Policy'] = "default-src 'self'"
+  response.headers['X-Content-Type-Options'] = 'nosniff'
+  return response
 
 
-  def url_for_with_cache_bust(endpoint, **values):
-    """
-    Add a query parameter based on the hash of the file, this acts as a cache bust
-    """
-    
-    if endpoint == 'static':
-      filename = values.get('filename', None)
-      if filename:
-        if 'STATIC_FILE_HASH_CACHE' not in current_app.config:
-          current_app.config['STATIC_FILE_HASH_CACHE'] = dict()
-        
-        if filename not in current_app.config['STATIC_FILE_HASH_CACHE']:
-          filepath = os.path.join(current_app.root_path, endpoint, filename)
-          #print(filepath)
-          if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
+@app.context_processor
+def override_url_for():
+  """
+  override the url_for function built into flask 
+  with our own custom implementation that busts the cache correctly when files change 
+  """
+  return dict(url_for=url_for_with_cache_bust)
+
+
+def url_for_with_cache_bust(endpoint, **values):
+  """
+  Add a query parameter based on the hash of the file, this acts as a cache bust
+  """
+  
+  if endpoint == 'static':
+    filename = values.get('filename', None)
+    if filename:
+      if 'STATIC_FILE_HASH_CACHE' not in current_app.config:
+        current_app.config['STATIC_FILE_HASH_CACHE'] = dict()
+      
+      if filename not in current_app.config['STATIC_FILE_HASH_CACHE']:
+        filepath = os.path.join(current_app.root_path, endpoint, filename)
+        #print(filepath)
+        if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
+          
+          with open(filepath, 'rb') as file:
+            hasher = hashlib.md5()
+            hasher.update(file.read())
+            current_app.config['STATIC_FILE_HASH_CACHE'][filename] = hasher.hexdigest()[-6:]
             
-            with open(filepath, 'rb') as file:
-              hasher = hashlib.md5()
-              hasher.update(file.read())
-              current_app.config['STATIC_FILE_HASH_CACHE'][filename] = hasher.hexdigest()[-6:]
-              
-        values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
+      values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
 
-    return url_for(endpoint, **values)
-
-  return app
+  return url_for(endpoint, **values)

capsulflask/admin.py

@@ -17,9 +17,9 @@ bp = Blueprint("admin", __name__, url_prefix="/admin")
 @bp.route("/")
 @admin_account_required
 def index():
-  hosts = get_model().list_hosts_with_networks(None)
-  vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
-  network_display_width_px = float(270)
+  hosts = get_model().list_hosts_with_networks()
+  vms_by_host_and_network = get_model().all_non_deleted_vms_by_host_and_network()
+  network_display_width_px = float(500);
   #operations = get_model().list_all_operations()
 
   display_hosts = []
@@ -35,18 +35,26 @@ def index():
     display_host = dict(name=host_id, networks=value['networks'])
 
     for network in display_host['networks']:
-
-      network_start_int = int(ipaddress.ip_address(network["public_ipv4_first_usable_ip"]))
-      network_end_int = int(ipaddress.ip_address(network["public_ipv4_last_usable_ip"]))
-
+      ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False)
+      network_start_int = -1
+      network_end_int = -1
+      i = 0
+      for ipv4_address in ipv4_network: 
+        i += 1
+        if i > 2:
+          if network_start_int == -1:
+            network_start_int = int(ipv4_address)
+          
+          network_end_int = int(ipv4_address)
+      
       network['allocations'] = []
-      network_addresses_width = float((network_end_int-network_start_int)+1)
+      network_addresses_width = float((network_end_int-network_start_int))
 
       if host_id in vms_by_host_and_network:
         if network['network_name'] in vms_by_host_and_network[host_id]:
           for vm in vms_by_host_and_network[host_id][network['network_name']]:
             ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
-            if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
+            if network_start_int < ip_address_int and ip_address_int < network_end_int:
               allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
               inline_styles.append(
                 f"""

capsulflask/auth.py

@@ -41,6 +41,7 @@ def account_required(view):
 
     return wrapped_view
 
+
 def admin_account_required(view):
     """View decorator that redirects non-admin users to the login page."""
 

capsulflask/console.py

@@ -237,11 +237,16 @@ def _create(vm_sizes, operating_systems, public_keys_for_account, server_data):
 
   if len(errors) == 0:
     id = make_capsul_id()
+    get_model().create_vm(
+      email=session["account"], 
+      id=id, 
+      size=size, 
+      os=os,
+      ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys))
+    )
     current_app.config["HUB_MODEL"].create(
       email = session["account"],
       id=id,
-      os=os,
-      size=size,
       template_image_file_name=operating_systems[os]['template_image_file_name'],
       vcpus=vm_sizes[size]['vcpus'],
       memory_mb=vm_sizes[size]['memory_mb'],
@@ -259,7 +264,6 @@ def create():
   public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
   account_balance = get_account_balance(get_vms(), get_payments(), datetime.utcnow())
   capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
-  errors = list()
 
   if request.method == "POST":
     if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:

capsulflask/db.py

@@ -43,7 +43,7 @@ def init_app(app, is_running_server):
   hasSchemaVersionTable = False
   actionWasTaken = False
   schemaVersion = 0
-  desiredSchemaVersion = 19
+  desiredSchemaVersion = 17
 
   cursor = connection.cursor()
 

capsulflask/db_model.py

@@ -1,5 +1,3 @@
-import re
-
 # I was never able to get this type hinting to work correctly 
 # from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
 import hashlib
@@ -67,18 +65,8 @@ class DBModel:
 
   #     ------    VM & ACCOUNT MANAGEMENT     ---------  
 
-  def non_deleted_vms_by_host_and_network(self, host_id):
-    query = "SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL"
-    if host_id is None: 
-      self.cursor.execute(query)
-    else:
-      if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
-        raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")
-
-      # I kept getting "TypeError: not all arguments converted during string formatting"
-      # when I was trying to mix python string templating with psycopg2 safe parameter passing. 
-      # so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
-      self.cursor.execute(f"{query} AND host = '{host_id}'")
+  def all_non_deleted_vms_by_host_and_network(self):
+    self.cursor.execute("SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL")
 
     hosts = dict()
     for row in self.cursor.fetchall():
@@ -193,12 +181,12 @@ class DBModel:
       )
     self.connection.commit()
 
-  def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys):
+  def create_vm(self, email, id, size, os, ssh_authorized_keys):
     self.cursor.execute(""" 
-      INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4)
-      VALUES  (%s, %s, %s, %s, %s, %s, %s)
+      INSERT INTO vms (email, id, size, os, network_name, host)
+      VALUES  (%s, %s, %s, %s, 'virbr1', 'baikal')
       """, 
-      (email, id, size, os, host, network_name, public_ipv4)
+      (email, id, size, os)
     )
     
     for ssh_authorized_key in ssh_authorized_keys:
@@ -365,35 +353,18 @@ class DBModel:
 
   #     ------    HOSTS     ---------  
 
-  def list_hosts_with_networks(self, host_id: str):
-    query = """
-    SELECT hosts.id, hosts.last_health_check, host_network.network_name, 
-    host_network.public_ipv4_cidr_block, host_network.public_ipv4_first_usable_ip, host_network.public_ipv4_last_usable_ip
-    FROM hosts
+  def list_hosts_with_networks(self):
+    self.cursor.execute("""
+    SELECT hosts.id, hosts.last_health_check, host_network.network_name, host_network.public_ipv4_cidr_block FROM hosts
     JOIN host_network ON host_network.host = hosts.id
-    """
-    if host_id is None: 
-      self.cursor.execute(query)
-    else:
-      if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
-        raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")
-
-      # I kept getting "TypeError: not all arguments converted during string formatting"
-      # when I was trying to mix python query string templating with psycopg2 safe parameter passing. 
-      # so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
-      self.cursor.execute(f"{query} WHERE hosts.id = '{host_id}'")
+    """)
 
     hosts = dict()
     for row in self.cursor.fetchall():
       if row[0] not in hosts:
         hosts[row[0]] = dict(last_health_check=row[1], networks=[])
 
-      hosts[row[0]]["networks"].append(dict(
-        network_name=row[2], 
-        public_ipv4_cidr_block=row[3], 
-        public_ipv4_first_usable_ip=row[4], 
-        public_ipv4_last_usable_ip=row[5]
-      ))
+      hosts[row[0]]["networks"].append(dict(network_name=row[2], public_ipv4_cidr_block=row[3]))
 
     return hosts
 
@@ -443,13 +414,6 @@ class DBModel:
     self.connection.commit()
     return operation_id
 
-  def update_operation(self, operation_id: int, payload: str):
-    self.cursor.execute(
-      "UPDATE operations SET payload = %s WHERE id = %s", 
-      (payload, operation_id)
-    )
-    self.connection.commit()
-
   def update_host_operation(self, host_id: str, operation_id: int, assignment_status: str, result: str):
     if assignment_status and not result:
       self.cursor.execute(
@@ -476,20 +440,9 @@ class DBModel:
     else:
       return None
 
-  def get_payload_json_from_host_operation(self, operation_id: int, host_id: str) -> str:
-    self.cursor.execute(
-      """
-        SELECT operations.payload FROM operations 
-          JOIN host_operation ON host_operation.operation = operations.id
-          WHERE host_operation.host = %s AND host_operation.operation = %s
-      """,
-      (host_id, operation_id)
-    )
-    row = self.cursor.fetchone()
-    if row:
-      return row[0]
-    else:
-      return None
+  def host_operation_exists(self, operation_id: int, host_id: str) -> bool:
+    self.cursor.execute("SELECT operation FROM host_operation WHERE host = %s AND operation = %s",(host_id, operation_id))
+    return len(self.cursor.fetchall()) != 0
 
   def claim_operation(self, operation_id: int, host_id: str) -> bool:
     # have to make a new cursor to set isolation level

capsulflask/hub_api.py

@@ -1,9 +1,7 @@
-import json
-import ipaddress
 
 from flask import Blueprint
 from flask import current_app
-from flask import request, make_response
+from flask import request
 from werkzeug.exceptions import abort
 
 from capsulflask.db import get_model
@@ -48,134 +46,15 @@ def heartbeat(host_id):
 @bp.route("/claim-operation/<int:operation_id>/<string:host_id>", methods=("POST",))
 def claim_operation(operation_id: int, host_id: str):
   if authorized_for_host(host_id):
-    payload_json = get_model().get_payload_json_from_host_operation(operation_id, host_id)
-    if payload_json is None:
-      error_message = f"{host_id} can't claim operation {operation_id} because host_operation row not found"
-      current_app.logger.error(error_message)
-      return abort(404, error_message)
-
-    # right now if there is a can_claim_handler there needs to be a corresponding on_claimed_handler.
-    # this is sole-ly due to laziness in error handling.
-    can_claim_handlers = {
-      "create": can_claim_create,
-    }
-    on_claimed_handlers = {
-      "create": on_create_claimed,
-    }
-    error_message = ""
-    payload = None
-    payload_is_dict = False
-    payload_has_type = False
-    payload_has_valid_type = False
-    try:
-      payload = json.loads(payload_json)
-      payload_is_dict = isinstance(payload, dict)
-      payload_has_type = payload_is_dict and 'type' in payload
-      payload_has_valid_type = payload_has_type and payload['type'] in can_claim_handlers
-      
-      if not payload_is_dict:
-        error_message = "invalid json: expected an object"
-      elif not payload_has_type:
-        error_message = "invalid json: 'type' field is required"
-      elif not payload_has_valid_type:
-        error_message = f"invalid json: expected type \"{payload['type']}\" to be one of [{', '.join(can_claim_handlers.keys())}]"
-    except:
-      error_message = "could not parse payload as json"
-
-    if error_message != "":
-      error_message = f"{host_id} can't claim operation {operation_id} because {error_message}"
-      current_app.logger.error(error_message)
-      return abort(400, error_message)
-
-    # we will only return this payload as json if claiming succeeds, so might as well do this now...
-    payload['assignment_status'] = 'assigned'
-
-    # invoke the appropriate can_claim_handler for this operation type
-    result_tuple = can_claim_handlers[payload['type']](payload, host_id)
-    modified_payload = result_tuple[0]
-    error_message = result_tuple[1]
-
-    if error_message != "":
-      error_message = f"{host_id} can't claim operation {operation_id} because {error_message}"
-      current_app.logger.error(error_message)
-      return abort(400, error_message)
-
+    exists = get_model().host_operation_exists(operation_id, host_id)
+    if not exists:
+      return abort(404, "host operation not found")
     claimed = get_model().claim_operation(operation_id, host_id)
     if claimed:
-      modified_payload_json = json.dumps(modified_payload)
-      get_model().update_operation(operation_id, modified_payload_json)
-      on_claimed_handlers[payload['type']](modified_payload, host_id)
-      
-      response = make_response(modified_payload_json)
-      response.headers.set("Content-Type", "application/json")
-
-      return response
+      return "ok"
     else:
-      return abort(409, f"operation was already assigned to another host")
+      return abort(409, "operation was already assigned to another host")
   else:
     current_app.logger.warning(f"/hub/claim-operation/{operation_id}/{host_id} returned 401: invalid token")
     return abort(401, "invalid host id or token")
 
-def can_claim_create(payload, host_id) -> (str, str):
-
-  hosts = get_model().list_hosts_with_networks(host_id)
-
-  if host_id not in hosts:
-    return "", f"the host \"{host_id}\" does not appear to have any networks."
-
-  networks = hosts[host_id]['networks']
-
-  vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(host_id)
-
-  vms_by_network = dict()
-  if host_id in vms_by_host_and_network:
-    vms_by_network = vms_by_host_and_network[host_id]
-
-  allocated_ipv4_address = None
-  allocated_network_name = None
-  for network in networks:
-    vms = []
-    if network["network_name"] in vms_by_network:
-      vms = vms_by_network[network["network_name"]]
-
-    claimed_ipv4s = dict()
-    for vm in vms:
-      claimed_ipv4s[vm['public_ipv4']] = True
-
-
-    ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False)
-    ipv4_first_usable_ip = network["public_ipv4_first_usable_ip"]
-    ipv4_last_usable_ip = network["public_ipv4_last_usable_ip"]
-        
-    for ipv4_address in ipv4_network: 
-      within_usable_range = ipv4_first_usable_ip <= str(ipv4_address) and str(ipv4_address) <= ipv4_last_usable_ip
-      if within_usable_range and str(ipv4_address) not in claimed_ipv4s:
-        allocated_ipv4_address = str(ipv4_address)
-        break 
-
-    if allocated_ipv4_address is not None:
-      allocated_network_name = network["network_name"]
-      break
-        
-  if allocated_network_name is None or allocated_ipv4_address is None:
-    return "", f"host \"{host_id}\" does not have any avaliable IP addresses on any of its networks."
-
-  # payload["network_name"] = allocated_network_name
-  # hard-code the network name for now until we can fix the phantom dhcp lease issues.
-
-  payload["network_name"] = 'public3'
-  payload["public_ipv4"] = allocated_ipv4_address
-
-  return payload, ""
-
-def on_create_claimed(payload, host_id):
-  get_model().create_vm(
-    email=payload['email'], 
-    id=payload['id'], 
-    size=payload['size'], 
-    os=payload['os'],
-    host=host_id,
-    network_name=payload['network_name'],
-    public_ipv4=payload['public_ipv4'],
-    ssh_authorized_keys=list(map(lambda x: x["name"], payload['ssh_authorized_keys'])),
-  )

capsulflask/hub_model.py

@@ -36,7 +36,7 @@ class MockHub(VirtualizationInterface):
   def list_ids(self) -> list:
     return get_model().all_non_deleted_vm_ids()
 
-  def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
+  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
     current_app.logger.info(f"mock create: {id} for {email}")
     sleep(1)
@@ -180,7 +180,7 @@ class CapsulFlaskHub(VirtualizationInterface):
 
     return to_return
 
-  def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
+  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
     online_hosts = get_model().get_online_hosts()
     #current_app.logger.debug(f"hub_model.create(): ${len(online_hosts)} hosts")
@@ -188,8 +188,6 @@ class CapsulFlaskHub(VirtualizationInterface):
       type="create", 
       email=email, 
       id=id, 
-      os=os,
-      size=size,
       template_image_file_name=template_image_file_name, 
       vcpus=vcpus,
       memory_mb=memory_mb,

capsulflask/schema_migrations/16_up_managed_ips.sql

@@ -3,12 +3,12 @@ CREATE TABLE host_network (
   public_ipv4_cidr_block  TEXT NOT NULL,
   network_name       TEXT NOT NULL,
   host               TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT,
-  CONSTRAINT host_network_pkey PRIMARY KEY (host, network_name)
+  PRIMARY KEY (host, network_name)
 );
 
+
 INSERT INTO host_network (host, network_name, public_ipv4_cidr_block) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
-                                                                             ('baikal', 'virbr2', '69.61.2.194/26'),
-                                                                             ('baikal', 'virbr3', '69.61.38.193/26');
+                                                                             ('baikal', 'virbr2', '69.61.2.194/26');
 ALTER TABLE vms RENAME COLUMN last_seen_ipv4 TO public_ipv4;
 ALTER TABLE vms RENAME COLUMN last_seen_ipv6 TO public_ipv6;
 ALTER TABLE vms ADD COLUMN network_name TEXT;
@@ -18,8 +18,7 @@ UPDATE vms SET network_name = 'virbr2' WHERE public_ipv4 >= '69.61.2.192';
 
 ALTER TABLE vms ALTER COLUMN network_name SET NOT NULL;
 
-ALTER TABLE vms ADD CONSTRAINT vms_host_network_name_fkey FOREIGN KEY (host, network_name) 
-                                               REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
+ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
 
 
 UPDATE schemaversion SET version = 16;

capsulflask/schema_migrations/17_down_api_tokens.py

@@ -0,0 +1,2 @@
+DROP TABLE api_keys;
+UPDATE schemaversion SET version = 16;

capsulflask/schema_migrations/17_down_bridge_and_network_name.sql

@@ -1,18 +0,0 @@
-
-
-ALTER TABLE vms DROP CONSTRAINT vms_host_network_name_fkey;
-
-UPDATE host_network SET network_name = 'virbr1' WHERE virtual_bridge_name = "virbr1";
-UPDATE host_network SET network_name = 'virbr2' WHERE virtual_bridge_name = "virbr1";
-UPDATE host_network SET network_name = 'virbr3' WHERE virtual_bridge_name = "virbr2";
-
-UPDATE vms SET network_name = 'virbr1' WHERE network_name = "public1";
-UPDATE vms SET network_name = 'virbr2' WHERE network_name = "public2";
-UPDATE vms SET network_name = 'virbr3' WHERE network_name = "public3";
-
-ALTER TABLE host_network DROP COLUMN virtual_bridge_name;
-
-ALTER TABLE vms ADD CONSTRAINT vms_host_network_name_fkey FOREIGN KEY (host, network_name) 
-                                               REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
-
-UPDATE schemaversion SET version = 16;

capsulflask/schema_migrations/17_up_api_tokens.py

@@ -6,4 +6,4 @@ CREATE TABLE api_tokens (
   token              TEXT NOT NULL
 );
 
-UPDATE schemaversion SET version = 19;
+UPDATE schemaversion SET version = 17;

capsulflask/schema_migrations/17_up_bridge_and_network_name.sql

@@ -1,26 +0,0 @@
-
-
-ALTER TABLE vms DROP CONSTRAINT vms_host_network_name_fkey;
-
-ALTER TABLE host_network RENAME COLUMN network_name TO virtual_bridge_name;
-
-ALTER TABLE host_network ADD COLUMN network_name TEXT;
-
-UPDATE host_network SET network_name = 'public1' WHERE virtual_bridge_name = 'virbr1';
-UPDATE host_network SET network_name = 'public2' WHERE virtual_bridge_name = 'virbr2';
-UPDATE host_network SET network_name = 'public3' WHERE virtual_bridge_name = 'virbr3';
-
-UPDATE vms SET network_name = 'public1' WHERE network_name = 'virbr1';
-UPDATE vms SET network_name = 'public2' WHERE network_name = 'virbr2';
-UPDATE vms SET network_name = 'public3' WHERE network_name = 'virbr3';
-
-ALTER TABLE host_network ALTER COLUMN network_name SET NOT NULL;
-
-ALTER TABLE host_network DROP CONSTRAINT host_network_pkey;
-ALTER TABLE host_network ADD CONSTRAINT host_network_pkey PRIMARY KEY (host, network_name);
-
-ALTER TABLE vms ADD CONSTRAINT vms_host_network_name_fkey FOREIGN KEY (host, network_name) 
-                                               REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
-
-UPDATE schemaversion SET version = 17;
-

capsulflask/schema_migrations/18_down_start_ip_end_ip.sql

@@ -1,6 +0,0 @@
-
-
-ALTER TABLE host_network DROP COLUMN public_ipv4_first_usable_ip;
-ALTER TABLE host_network DROP COLUMN public_ipv4_last_usable_ip;
-
-UPDATE schemaversion SET version = 17;

capsulflask/schema_migrations/18_up_start_ip_end_ip.sql

@@ -1,25 +0,0 @@
-
-
-ALTER TABLE host_network ADD COLUMN public_ipv4_first_usable_ip TEXT;
-ALTER TABLE host_network ADD COLUMN public_ipv4_last_usable_ip TEXT;
-
--- public1, 69.61.2.162/27
-UPDATE host_network 
-SET public_ipv4_first_usable_ip = '69.61.2.163', public_ipv4_last_usable_ip = '69.61.2.190' 
-WHERE network_name = 'public1';
-
--- public2, 69.61.2.194/26
-UPDATE host_network 
-SET public_ipv4_first_usable_ip = '69.61.2.195', public_ipv4_last_usable_ip = '69.61.2.254' 
-WHERE network_name = 'public2';
-
--- public3, 69.61.38.193/26
-UPDATE host_network 
-SET public_ipv4_first_usable_ip = '69.61.38.194', public_ipv4_last_usable_ip = '69.61.38.254' 
-WHERE network_name = 'public3';
-
-ALTER TABLE host_network ALTER COLUMN public_ipv4_first_usable_ip SET NOT NULL;
-ALTER TABLE host_network ALTER COLUMN public_ipv4_last_usable_ip SET NOT NULL;
-
-UPDATE schemaversion SET version = 18;
-

capsulflask/schema_migrations/19_down_api_tokens.py

@@ -1,2 +0,0 @@
-DROP TABLE api_keys;
-UPDATE schemaversion SET version = 18;

capsulflask/shell_scripts/capacity-avaliable.sh

@@ -19,9 +19,9 @@ if [ "$ram_bytes_remainder" -le $((20 * 1024 * 1024 * 1024)) ]; then
   exit 1
 fi
 
-ipv4_limit=61
-used_ips=$(grep ip-add "/var/lib/libvirt/dnsmasq/virbr3.status" | cut -d '"' -f 4)
-reserved_ips=$(cat "/var/lib/libvirt/dnsmasq/public3.hostsfile" | cut -d ',' -f 2)
+ipv4_limit=28
+used_ips=$(grep ip-add "/var/lib/libvirt/dnsmasq/virbr1.status" | cut -d '"' -f 4)
+reserved_ips=$(cat "/var/lib/libvirt/dnsmasq/public1.hostsfile" | cut -d ',' -f 2)
 total_addresses_used=$(printf "$used_ips\n$reserved_ips" | sort | uniq | wc -l)
 ipv4_count=$(printf "$total_addresses_used")
 

capsulflask/shell_scripts/create.sh

@@ -9,8 +9,6 @@ template_file="/tank/img/$2"
 vcpus="$3"
 memory="$4"
 pubkeys="$5"
-network_name="$6"
-public_ipv4="$7"
 root_volume_size="25G"
 
 if echo "$vmname" | grep -vqE '^capsul-[a-z0-9]{10}$'; then
@@ -40,16 +38,6 @@ echo "$pubkeys" | while IFS= read -r line; do
   fi
 done
 
-if echo "$network_name" | grep -vqE "^[a-zA-Z0-9_-]+"; then
-  echo "network_name \"$network_name\" must match ^[a-zA-Z0-9_-]+"
-  exit 1
-fi
-
-if echo "$public_ipv4" | grep -vqE "^[0-9.]+$"; then
-  echo "public_ipv4 \"$public_ipv4\" must match ^[0-9.]+$"
-  exit 1
-fi
-
 disk="/tank/vm/$vmname.qcow2"
 cdrom="/tank/vm/$vmname.iso"
 xml="/tank/vm/$vmname.xml"
@@ -78,7 +66,7 @@ virt-install \
     --os-variant generic \
     --virt-type kvm \
     --graphics vnc,listen=127.0.0.1 \
-    --network network=$network_name,filterref=clean-traffic,model=virtio \
+    --network network=public1,filterref=clean-traffic,model=virtio \
     --import \
     --print-xml > "$xml"
 

capsulflask/spoke_api.py

@@ -104,7 +104,7 @@ def handle_create(operation_id, request_body):
     current_app.logger.error(f"/hosts/operation returned 400: operation_id is required for create ")
     return abort(400, f"bad request; operation_id is required. try POST /spoke/operation/<id>")
 
-  parameters = ["email", "id", "os", "size", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"]
+  parameters = ["email", "id", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"]
   error_message = ""
   for parameter in parameters:
     if parameter not in request_body:
@@ -115,28 +115,14 @@ def handle_create(operation_id, request_body):
     return abort(400, f"bad request; {error_message}")
 
   # only one host should create the vm, so we first race to assign this create operation to ourselves.
-  # only one host will win this race.
+  # only one host will win this race
   authorization_header = f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}"
   url = f"{current_app.config['HUB_URL']}/hub/claim-operation/{operation_id}/{current_app.config['SPOKE_HOST_ID']}"
   result = current_app.config['HTTP_CLIENT'].do_http_sync(url, body=None, authorization_header=authorization_header)
 
   assignment_status = ""
   if result.status_code == 200:
-    try:
-      assignment_info = json.loads(result.body)
-      if not isinstance(assignment_info, dict):
-        return abort(503, f"hub at '{url}' returned 200, but did not return assignment_info json object")
-      if 'network_name' not in assignment_info:
-        return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include network_name")
-      if 'public_ipv4' not in assignment_info:
-        return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include public_ipv4")
-
-      assignment_status = "assigned"
-      request_body['network_name'] = assignment_info['network_name']
-      request_body['public_ipv4'] = assignment_info['public_ipv4']
-    except:
-      return abort(503, f"hub at '{url}' returned 200, but did not return valid json")
-    
+    assignment_status = "assigned"
   elif result.status_code == 409:
     assignment_status = "assigned_to_other_host"
   else:
@@ -151,9 +137,7 @@ def handle_create(operation_id, request_body):
         template_image_file_name=request_body['template_image_file_name'],
         vcpus=request_body['vcpus'],
         memory_mb=request_body['memory_mb'],
-        ssh_authorized_keys=list(map(lambda x: x['content'], request_body['ssh_authorized_keys'])),
-        network_name=request_body['network_name'],
-        public_ipv4=request_body['public_ipv4'],
+        ssh_authorized_keys=request_body['ssh_authorized_keys'],
       )
     except:
       error_message = my_exec_info_message(sys.exc_info())
@@ -163,11 +147,7 @@ def handle_create(operation_id, request_body):
       params=  f"{params} vcpus='{request_body['vcpus'] if 'vcpus' in request_body else 'KeyError'}', "
       params=  f"{params} memory_mb='{request_body['memory_mb'] if 'memory_mb' in request_body else 'KeyError'}', "
       params=  f"{params} ssh_authorized_keys='{request_body['ssh_authorized_keys'] if 'ssh_authorized_keys' in request_body else 'KeyError'}', "
-      params=  f"{params} network_name='{request_body['network_name'] if 'network_name' in request_body else 'KeyError'}', "
-      params=  f"{params} public_ipv4='{request_body['public_ipv4'] if 'public_ipv4' in request_body else 'KeyError'}', "
-      
       current_app.logger.error(f"spoke_model.create({params}) failed: {error_message}")
-
       return jsonify(dict(assignment_status=assignment_status, error_message=error_message))
 
   return jsonify(dict(assignment_status=assignment_status))

capsulflask/spoke_model.py

@@ -14,37 +14,28 @@ from capsulflask.shared import VirtualizationInterface, VirtualMachine, validate
 
 
 class MockSpoke(VirtualizationInterface):
-
-  def __init__(self):
-    self.capsuls = dict()
-
   def capacity_avaliable(self, additional_ram_bytes):
     return True
 
   def get(self, id, get_ssh_host_keys):
     validate_capsul_id(id)
 
-    ipv4 = "1.1.1.1"
-    if id in self.capsuls:
-      ipv4 = self.capsuls[id]['public_ipv4']
-
     if get_ssh_host_keys:
       ssh_host_keys = json.loads("""[
         {"key_type":"ED25519", "content":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN8cna0zeKSKl/r8whdn/KmDWhdzuWRVV0GaKIM+eshh", "sha256":"V4X2apAF6btGAfS45gmpldknoDX0ipJ5c6DLfZR2ttQ"},
         {"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"},
         {"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"}
       ]""")
-      return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running", ssh_host_keys=ssh_host_keys)
+      return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running", ssh_host_keys=ssh_host_keys)
 
-    return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
+    return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running")
 
   def list_ids(self) -> list:
     return get_model().all_non_deleted_vm_ids()
 
-  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
+  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
     current_app.logger.info(f"mock create: {id} for {email}")
-    self.capsuls[id] = dict(email=email, id=id, network_name=network_name, public_ipv4=public_ipv4)
     sleep(1)
 
   def destroy(self, email: str, id: str):
@@ -138,7 +129,7 @@ class ShellScriptSpoke(VirtualizationInterface):
     self.validate_completed_process(completedProcess)
     return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() ))
 
-  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
+  def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
 
     if not re.match(r"^[a-zA-Z0-9/_.-]+$", template_image_file_name):
@@ -148,18 +139,12 @@ class ShellScriptSpoke(VirtualizationInterface):
       if not re.match(r"^(ssh|ecdsa)-[0-9A-Za-z+/_=@:. -]+$", ssh_authorized_key):
         raise ValueError(f"ssh_authorized_key \"{ssh_authorized_key}\" must match \"^(ssh|ecdsa)-[0-9A-Za-z+/_=@:. -]+$\"")
 
-    if isinstance(vcpus, int) and (vcpus < 1 or vcpus > 8):
+    if vcpus < 1 or vcpus > 8:
       raise ValueError(f"vcpus \"{vcpus}\" must match 1 <= vcpus <= 8")
 
-    if isinstance(memory_mb, int) and (memory_mb < 512 or memory_mb > 16384):
+    if memory_mb < 512 or memory_mb > 16384:
       raise ValueError(f"memory_mb \"{memory_mb}\" must match 512 <= memory_mb <= 16384")
 
-    if not re.match(r"^[a-zA-Z0-9_-]+$", network_name):
-      raise ValueError(f"network_name \"{network_name}\" must match \"^[a-zA-Z0-9_-]+\"")
-
-    if not re.match(r"^[0-9.]+$", public_ipv4):
-      raise ValueError(f"public_ipv4 \"{public_ipv4}\" must match \"^[0-9.]+$\"")
-
     ssh_keys_string = "\n".join(ssh_authorized_keys)
 
     completedProcess = run([
@@ -168,9 +153,7 @@ class ShellScriptSpoke(VirtualizationInterface):
       template_image_file_name,
       str(vcpus),
       str(memory_mb),
-      ssh_keys_string,
-      network_name,
-      public_ipv4
+      ssh_keys_string
     ], capture_output=True)
 
     self.validate_completed_process(completedProcess, email)
@@ -183,8 +166,6 @@ class ShellScriptSpoke(VirtualizationInterface):
       vcpus={str(vcpus)}
       memory={str(memory_mb)}
       ssh_authorized_keys={ssh_keys_string}
-      network_name={network_name}
-      public_ipv4={public_ipv4}
     """
 
     if not status == "success":

capsulflask/tests/test_auth.py

@@ -1,23 +0,0 @@
-from flask import url_for, session
-
-from capsulflask.db import get_model
-from capsulflask.tests_base import BaseTestCase
-
-
-class LoginTests(BaseTestCase):
-    render_templates = False
-
-    def test_login_request(self):
-        with self.client as client:
-            response = client.get(url_for("auth.login"))
-            self.assert_200(response)
-
-            # FIXME test generated login link
-
-    def test_login_magiclink(self):
-        token, ignoreCaseMatches = get_model().login('test@example.com')
-
-        with self.client as client:
-            response = client.get(url_for("auth.magiclink", token=token))
-            self.assertRedirects(response, url_for("console.index"))
-            self.assertEqual(session['account'], 'test@example.com')

capsulflask/tests/test_console.py

@@ -1,170 +0,0 @@
-from unittest.mock import patch
-
-from flask import url_for
-
-from capsulflask.hub_model import MockHub
-from capsulflask.db import get_model
-from capsulflask.tests_base import BaseTestCase
-
-
-class ConsoleTests(BaseTestCase):
-    capsul_data = {
-        "size": "f1-xs",
-        "os": "debian10",
-        "ssh_authorized_key_count": 1,
-        "ssh_key_0": "key"
-    }
-
-    ssh_key_data = {
-        "name": "key2",
-        "action": "upload_ssh_key",
-        "content": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDntq1t8Ddsa2q4p+PM7W4CLYYmxakokRRVLlf7AQlsTJFPsgBe9u0zuoOaKDMkBr0dlnuLm4Eub1Mj+BrdqAokto0YDiAnxUKRuYQKuHySKK8bLkisi2k47jGBDikx/jihgiuFTawo1mYsJJepC7PPwZGsoCImJEgq1L+ug0p3Zrj3QkUx4h25MpCSs2yvfgWjDyN8hEC76O42P+4ETezYrzrd1Kj26hdzHRnrxygvIUOtfau+5ydlaz8xQBEPrEY6/+pKDuwtXg1pBL7GmoUxBXVfHQSgq5s9jIJH+G0CR0ZoHMB25Ln4X/bsCQbLOu21+IGYKSDVM5TIMLtkKUkERQMVWvnpOp1LZKir4dC0m7SW74wpA8+2b1IsURIr9ARYGJpCEv1Q1Wz/X3yTf6Mfey7992MjUc9HcgjgU01/+kYomoXHprzolk+22Gjfgo3a4dRIoTY82GO8kkUKiaWHvDkkVURCY5dpteLA05sk3Z9aRMYsNXPLeOOPfzTlDA0="
-    }
-
-    def test_index(self):
-        self._login('test@example.com')
-        with self.client as client:
-            response = client.get(url_for("console.index"))
-            self.assert_200(response)
-
-    def test_create_loads(self):
-        self._login('test@example.com')
-        with self.client as client:
-            response = client.get(url_for("console.create"))
-            self.assert_200(response)
-
-    def test_create_fails_capacity(self):
-        with self.client as client:
-            client.get(url_for("console.create"))
-            csrf_token = self.get_context_variable('csrf_token')
-
-            data = self.capsul_data
-            data['csrf-token'] = csrf_token
-
-            # Override MockHub.capacity_avaliable to always return False
-            with patch.object(MockHub, 'capacity_avaliable', return_value=False) as mock_method:
-                self.app.config["HUB_MODEL"] = MockHub()
-
-                client.post(url_for("console.create"), data=data)
-                capacity_message = \
-                    '\n      host(s) at capacity. no capsuls can be created at this time. sorry. \n    '
-                self.assert_message_flashed(capacity_message, category='message')
-
-                self.assertEqual(
-                    len(get_model().list_vms_for_account('test@example.com')),
-                    0
-                )
-            mock_method.assert_called_with(512 * 1024 * 1024)
-
-    def test_create_fails_invalid(self):
-        with self.client as client:
-            client.get(url_for("console.create"))
-            csrf_token = self.get_context_variable('csrf_token')
-
-            data = self.capsul_data
-            data['csrf-token'] = csrf_token
-            data['os'] = ''
-            client.post(url_for("console.create"), data=data)
-
-            self.assert_message_flashed(
-                'OS is required', 
-                category='message'
-            )
-
-            self.assertEqual(
-                len(get_model().list_vms_for_account('test@example.com')),
-                0
-            )
-
-    def test_create_succeeds(self):
-        with self.client as client:
-            client.get(url_for("console.create"))
-            csrf_token = self.get_context_variable('csrf_token')
-
-            data = self.capsul_data
-            data['csrf-token'] = csrf_token
-            response = client.post(url_for("console.create"), data=data)
-
-            # FIXME: mock create doesn't create, see #83
-            # vms = get_model().list_vms_for_account('test@example.com')
-            # self.assertEqual(
-            #     len(vms),
-            #     1 
-            # )
-            #
-            # vm_id = vms[0].id
-            #
-            # self.assertRedirects(
-            #     response, 
-            #     url_for("console.index") + f'?{vm_id}'
-            # )
-
-    def test_keys_loads(self):
-        self._login('test@example.com')
-        with self.client as client:
-            response = client.get(url_for("console.ssh_api_keys"))
-            self.assert_200(response)
-            keys = self.get_context_variable('ssh_public_keys')
-            self.assertEqual(keys[0]['name'], 'key')
-
-    def test_keys_add_ssh_fails_invalid(self):
-        self._login('test@example.com')
-        with self.client as client:
-            client.get(url_for("console.ssh_api_keys"))
-            csrf_token = self.get_context_variable('csrf_token')
-
-            data = self.ssh_key_data
-            data['csrf-token'] = csrf_token
-
-            data_invalid_content = data
-            data_invalid_content['content'] = 'foo'
-            client.post(
-                url_for("console.ssh_api_keys"),
-                data=data_invalid_content
-            )
-
-            self.assert_message_flashed(
-                'Content must match "^(ssh|ecdsa)-[0-9A-Za-z+/_=@:. -]+$"',
-                category='message'
-            )
-
-            data_missing_content = data
-            data_missing_content['content'] = ''
-            client.post(url_for("console.ssh_api_keys"), data=data_missing_content)
-
-            self.assert_message_flashed(
-                'Content is required', category='message'
-            )
-
-    def test_keys_add_ssh_fails_duplicate(self):
-        self._login('test@example.com')
-        with self.client as client:
-            client.get(url_for("console.ssh_api_keys"))
-            csrf_token = self.get_context_variable('csrf_token')
-
-            data = self.ssh_key_data
-            data['csrf-token'] = csrf_token
-            data['name'] = 'key'
-            client.post(url_for("console.ssh_api_keys"), data=data)
-
-            self.assert_message_flashed(
-                'A key with that name already exists',
-                category='message'
-            )
-
-            data = self.ssh_key_data
-            data['csrf-token'] = csrf_token
-            data['name'] = 'key'
-            client.post(url_for("console.ssh_api_keys"), data=data)
-
-            self.assert_message_flashed(
-                'A key with that name already exists',
-                category='message'
-            )
-
-    def setUp(self):
-        self._login('test@example.com')
-        get_model().create_ssh_public_key('test@example.com', 'key', 'foo') 
-
-    def tearDown(self):
-        get_model().delete_ssh_public_key('test@example.com', 'key') 

capsulflask/tests/test_landing.py

@@ -1,14 +0,0 @@
-from capsulflask.tests_base import BaseTestCase
-
-
-class LandingTests(BaseTestCase):
-    #: Do not render templates, we're only testing logic here.
-    render_templates = False
-
-    def test_landing(self):
-        pages = ['/', 'pricing', 'faq', 'about-ssh', 'changelog', 'support']
-
-        with self.client as client:
-            for page in pages:
-                response = client.get(page)
-                self.assert_200(response)

capsulflask/tests/test_publicapi.py

@@ -1,45 +0,0 @@
-from base64 import b64encode
-import requests
-
-from flask import url_for
-
-from capsulflask.db import get_model
-from capsulflask.tests_base import BaseLiveServerTestCase
-
-
-class PublicAPITests(BaseLiveServerTestCase):
-    def test_server_is_up_and_running(self):
-        response = requests.get(self.get_server_url())
-        self.assertEqual(response.status_code, 200)
-
-    def test_capsul_create_succeeds(self):
-        response = requests.post(
-            self.get_server_url() +
-            url_for('publicapi.capsul_create'),
-            headers={'Authorization': self.token},
-            json={
-                'size': 'f1-xs',
-                'os': 'openbsd68',
-                'ssh_key_0': 'key'
-            }
-        )
-
-        self.assertEqual(response.status_code, 200)
-
-        # FIXME: mock create doesn't create, see #83
-        # vms = get_model().list_vms_for_account('test@example.com')
-        # 
-        # self.assertEqual(
-        #     len(vms),
-        #     1
-        # )
-
-    def setUp(self):
-        get_model().create_ssh_public_key('test@example.com', 'key', 'foo') 
-        self.token = b64encode(
-            get_model().generate_api_token('test@example.com', 'apikey').encode('utf-8')
-        ).decode('utf-8')
-
-    def tearDown(self):
-        get_model().delete_ssh_public_key('test@example.com', 'key') 
-        get_model().delete_api_token('test@example.com', 1)

capsulflask/tests_base.py

@@ -1,35 +0,0 @@
-import os
-from nanoid import generate
-
-from flask_testing import TestCase, LiveServerTestCase
-
-from capsulflask import create_app
-from capsulflask.db import get_model
-
-
-class BaseSharedTestCase(object):
-    def create_app(self):
-        # Use default connection paramaters
-        os.environ['POSTGRES_CONNECTION_PARAMETERS'] = "host=localhost port=5432 user=postgres password=dev dbname=capsulflask_test"
-        os.environ['TESTING'] = '1'
-        os.environ['SPOKE_MODEL'] = 'mock'
-        os.environ['HUB_MODEL'] = 'mock'
-        return create_app()
-
-    def setUp(self):
-        pass
-
-    def tearDown(self):
-        pass
-
-
-class BaseTestCase(BaseSharedTestCase, TestCase):
-    def _login(self, user_email):
-        get_model().login(user_email)
-        with self.client.session_transaction() as session:
-            session['account'] = user_email
-            session['csrf-token'] = generate()
-
-
-class BaseLiveServerTestCase(BaseSharedTestCase, LiveServerTestCase):
-    pass

docker-compose.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+
+services:
+  app:
+    image: 3wordchant/capsul-flask:latest
+    build: .
+    volumes:
+      - "./:/app/code"
+    depends_on:
+      - db
+    ports:
+      - "5000:5000"
+    environment:
+      - "POSTGRES_CONNECTION_PARAMETERS=host=db port=5432 user=capsul password=capsul dbname=capsul"
+    # The image uses gunicorn by default, let's override it with Flask's
+    # built-in development server
+    command: ["flask", "run", "-h", "0.0.0.0", "-p", "5000"]
+  db:
+    image: "postgres:9.6.5"
+    volumes:
+      - "postgres:/var/lib/postgresql/data"
+    environment:
+      POSTGRES_USER: capsul
+      POSTGRES_PASSWORD: capsul
+      POSTGRES_DB: capsul
+
+volumes:
+  postgres: