capsul-flask/capsulflask/db_model.py

import re

# I was never able to get this type hinting to work correctly 
# from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
import hashlib
from nanoid import generate
from flask import current_app
from typing import List

from capsulflask.shared import OnlineHost


class DBModel:
  #def __init__(self, connection: Psycopg2Connection, cursor: Psycopg2Cursor):
  def __init__(self, connection, cursor):
    self.connection = connection
    self.cursor = cursor


  #     ------    LOGIN     ---------     


  def login(self, email):
    self.cursor.execute("SELECT * FROM accounts WHERE email = %s", (email, ))
    hasExactMatch = len(self.cursor.fetchall())
    self.cursor.execute("SELECT * FROM accounts WHERE email = %s AND ever_logged_in = TRUE", (email, ))
    everLoggedIn = len(self.cursor.fetchall())
    ignoreCaseMatches = []
    if everLoggedIn == 0:
      self.cursor.execute("SELECT email FROM accounts WHERE lower_case_email = %s AND email != %s", (email.lower(), email))
      ignoreCaseMatches = list(map(lambda x: x[0], self.cursor.fetchall()))
  
    if hasExactMatch == 0:
      self.cursor.execute("INSERT INTO accounts (email, lower_case_email) VALUES (%s, %s)", (email, email.lower()))

    self.cursor.execute("SELECT token FROM login_tokens WHERE email = %s and created > (NOW() - INTERVAL '20 min')", (email, ))
    if len(self.cursor.fetchall()) > 2:
      return (None, ignoreCaseMatches)

    token = generate()
    self.cursor.execute("INSERT INTO login_tokens (email, token) VALUES (%s, %s)", (email, token))
    self.connection.commit()

    return (token, ignoreCaseMatches)

  def authenticate_token(self, token):
    m = hashlib.md5()
    m.update(token.encode('utf-8'))
    hash_token = m.hexdigest()
    self.cursor.execute("SELECT email FROM api_tokens WHERE token = %s", (hash_token, ))
    result = self.cursor.fetchall()
    if len(result) == 1:
      return result[0]
    return None
    
  def consume_token(self, token):
    self.cursor.execute("SELECT email FROM login_tokens WHERE token = %s and created > (NOW() - INTERVAL '20 min')", (token, ))
    row = self.cursor.fetchone()
    if row:
      email = row[0]
      self.cursor.execute("DELETE FROM login_tokens WHERE email = %s", (email, ))
      self.cursor.execute("UPDATE accounts SET ever_logged_in = TRUE WHERE email = %s", (email, ))
      self.connection.commit()
      return email
    return None


  #     ------    VM & ACCOUNT MANAGEMENT     ---------  

  def non_deleted_vms_by_host_and_network(self, host_id):
    query = "SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL"
    if host_id is None: 
      self.cursor.execute(query)
    else:
      if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
        raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")

      # I kept getting "TypeError: not all arguments converted during string formatting"
      # when I was trying to mix python string templating with psycopg2 safe parameter passing. 
      # so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
      self.cursor.execute(f"{query} AND host = '{host_id}'")

    hosts = dict()
    for row in self.cursor.fetchall():
      host_id = row[1]
      network_name = row[2]
      if host_id not in hosts:
        hosts[host_id] = dict()
      if network_name not in hosts[host_id]:
        hosts[host_id][network_name] = []
      
      hosts[host_id][network_name].append(
        dict(id=row[0], public_ipv4=row[3], public_ipv6=row[4])
      )

    return hosts

  def all_non_deleted_vm_ids(self):
    self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
    return list(map(lambda x: x[0], self.cursor.fetchall()))

  def operating_systems_dict(self):
    self.cursor.execute("SELECT id, template_image_file_name, description FROM os_images WHERE deprecated = FALSE")

    operatingSystems = dict()
    for row in self.cursor.fetchall():
      operatingSystems[row[0]] = dict(template_image_file_name=row[1], description=row[2])

    return operatingSystems

  def vm_sizes_dict(self):
    self.cursor.execute("SELECT id, dollars_per_month, vcpus, memory_mb, bandwidth_gb_per_month FROM vm_sizes")

    vmSizes = dict()
    for row in self.cursor.fetchall():
      vmSizes[row[0]] = dict(dollars_per_month=row[1], vcpus=row[2], memory_mb=row[3], bandwidth_gb_per_month=row[4])

    return vmSizes

  def list_ssh_public_keys_for_account(self, email):
    self.cursor.execute("SELECT name, content, created FROM ssh_public_keys WHERE email = %s", (email, ))
    return list(map(
      lambda x: dict(name=x[0], content=x[1], created=x[2]), 
      self.cursor.fetchall()
    ))

  def ssh_public_key_name_exists(self, email, name):
    self.cursor.execute( "SELECT name FROM ssh_public_keys where email = %s AND name = %s", (email, name) )
    return len(self.cursor.fetchall()) > 0

  def create_ssh_public_key(self, email, name, content):
    self.cursor.execute(""" 
      INSERT INTO ssh_public_keys (email, name, content)
      VALUES  (%s, %s, %s)
      """, 
      (email, name, content)
    )
    self.connection.commit()

  def delete_ssh_public_key(self, email, name):
    self.cursor.execute( "DELETE FROM ssh_public_keys where email = %s AND name = %s", (email, name) )
    self.connection.commit()

  def list_api_tokens(self, email):
    self.cursor.execute(
      "SELECT id, token, name, created FROM api_tokens WHERE email = %s", 
      (email, )
    )
    return list(map(
      lambda x: dict(id=x[0], token=x[1], name=x[2], created=x[3]), 
      self.cursor.fetchall()
    ))

  def generate_api_token(self, email, name):
    token = generate()
    m = hashlib.md5()
    m.update(token.encode('utf-8'))
    hash_token = m.hexdigest()
    self.cursor.execute(
      "INSERT INTO api_tokens (email, name, token) VALUES (%s, %s, %s)", 
      (email, name, hash_token)
    )
    self.connection.commit()
    return token

  def delete_api_token(self, email, id_):
    self.cursor.execute( "DELETE FROM api_tokens where email = %s AND id = %s", (email, id_))
    self.connection.commit()

  def list_vms_for_account(self, email):
    self.cursor.execute(""" 
      SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
        FROM vms JOIN vm_sizes on vms.size = vm_sizes.id
      WHERE vms.email = %s""", 
      (email, )
    )
    return list(map(
      lambda x: dict(id=x[0], ipv4=x[1], ipv6=x[2], size=x[3], os=x[4], created=x[5], deleted=x[6], dollars_per_month=x[7]), 
      self.cursor.fetchall()
    ))

  def update_vm_ip(self, email, id, ipv4):
    self.cursor.execute("UPDATE vms SET public_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
    self.connection.commit()

  def update_vm_ssh_host_keys(self, email, id, ssh_host_keys):
    for key in ssh_host_keys:
      self.cursor.execute(""" 
        INSERT INTO vm_ssh_host_key (email, vm_id, key_type, content, sha256)
        VALUES  (%s, %s, %s, %s, %s)
        """, 
        (email, id, key['key_type'], key['content'], key['sha256'])
      )
    self.connection.commit()

  def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys):
    self.cursor.execute(""" 
      INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4)
      VALUES  (%s, %s, %s, %s, %s, %s, %s)
      """, 
      (email, id, size, os, host, network_name, public_ipv4)
    )
    
    for ssh_authorized_key in ssh_authorized_keys:
      self.cursor.execute(""" 
        INSERT INTO vm_ssh_authorized_key (email, vm_id, ssh_public_key_name)
        VALUES  (%s, %s, %s)
        """, 
        (email, id, ssh_authorized_key)
      )
    self.connection.commit()

  def delete_vm(self, email, id):
    self.cursor.execute("UPDATE vms SET deleted = now() WHERE email = %s AND id = %s", ( email, id))
    self.connection.commit()

  def get_vm_detail(self, email, id):
    self.cursor.execute(""" 
      SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted,
            vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
      FROM vms 
        JOIN os_images on vms.os = os_images.id
        JOIN vm_sizes on vms.size = vm_sizes.id
      WHERE vms.email = %s AND vms.id = %s""", 
      (email, id)
    )
    row = self.cursor.fetchone()
    if not row:
      return None

    vm = dict(
      id=row[0], ipv4=row[1], ipv6=row[2], os_description=row[3], created=row[4], deleted=row[5],
      size=row[6], dollars_per_month=row[7], vcpus=row[8], memory_mb=row[9], bandwidth_gb_per_month=row[10]
    )

    self.cursor.execute(""" 
      SELECT ssh_public_key_name FROM vm_ssh_authorized_key 
      WHERE vm_ssh_authorized_key.email = %s AND vm_ssh_authorized_key.vm_id = %s""", 
      (email, id)
    )
    vm["ssh_authorized_keys"] = list(map( lambda x: x[0], self.cursor.fetchall() ))


    self.cursor.execute(""" 
      SELECT key_type, content, sha256 FROM vm_ssh_host_key 
      WHERE vm_ssh_host_key.email = %s AND vm_ssh_host_key.vm_id = %s""", 
      (email, id)
    )
    vm["ssh_host_keys"] = list(map( lambda x: dict(key_type=x[0], content=x[1], sha256=x[2]), self.cursor.fetchall() ))

    return vm


  #     ------    PAYMENTS & ACCOUNT BALANCE     ---------  


  def list_payments_for_account(self, email):
    self.cursor.execute(""" 
      SELECT id, dollars, invalidated, created
      FROM payments WHERE payments.email = %s""", 
      (email, )
    )
    return list(map(
      lambda x: dict(id=x[0], dollars=x[1], invalidated=x[2], created=x[3]), 
      self.cursor.fetchall()
    ))
  
  def create_payment_session(self, payment_type, id, email, dollars):
    self.cursor.execute(""" 
      INSERT INTO payment_sessions (id, type, email, dollars)
      VALUES  (%s, %s, %s, %s)
      """, 
      (id, payment_type, email, dollars)
    )
    self.connection.commit()

  def list_payment_sessions_for_account(self, email):
    self.cursor.execute(""" 
      SELECT id, type, dollars, created
      FROM payment_sessions WHERE email = %s""", 
      (email, )
    )
    return list(map(
      lambda x: dict(id=x[0], type=x[1], dollars=x[2], created=x[3]), 
      self.cursor.fetchall()
    ))

  def payment_session_redirect(self, email, id):
    self.cursor.execute("SELECT redirected FROM payment_sessions WHERE email = %s AND id = %s", 
      (email, id)
    )
    row = self.cursor.fetchone()
    if row:
      self.cursor.execute("UPDATE payment_sessions SET redirected = TRUE WHERE email = %s AND id = %s", 
        (email, id)
      )
      self.connection.commit()
      return row[0]
    
    return None


  def consume_payment_session(self, payment_type, id, dollars):
    self.cursor.execute("SELECT email, dollars FROM payment_sessions WHERE id = %s AND type = %s", (id, payment_type))
    row = self.cursor.fetchone()
    if row:
      if int(row[1]) != int(dollars):
        current_app.logger.warning(f"""
          {payment_type} gave us a completed payment session with a different dollar amount than what we had recorded!!
          id: {id}
          account: {row[0]}
          Recorded amount: {int(row[1])}
          {payment_type} sent: {int(dollars)}
        """)
        # not sure what to do here. For now just log and do nothing.
      self.cursor.execute( "DELETE FROM payment_sessions WHERE id = %s AND type = %s", (id, payment_type) )
      self.cursor.execute( "INSERT INTO payments (email, dollars) VALUES (%s, %s) RETURNING id", (row[0], row[1]) )

      if payment_type == "btcpay":
        payment_id = self.cursor.fetchone()[0]
        self.cursor.execute(
          "INSERT INTO unresolved_btcpay_invoices (id, email, payment_id) VALUES (%s, %s, %s)",
           (id, row[0], payment_id)
        )

      self.connection.commit()
      return row[0]
    else:
      return None

  def delete_payment_session(self, payment_type, id):
    self.cursor.execute( "DELETE FROM payment_sessions WHERE id = %s AND type = %s", (id, payment_type) )
    self.connection.commit()

  def btcpay_invoice_resolved(self, id, completed):
    self.cursor.execute("SELECT email, payment_id FROM unresolved_btcpay_invoices WHERE id = %s ", (id,))
    row = self.cursor.fetchone()
    if row:
      self.cursor.execute( "DELETE FROM unresolved_btcpay_invoices WHERE id = %s", (id,) )
      if not completed:
        self.cursor.execute("UPDATE payments SET invalidated = TRUE WHERE email = %s id = %s", (row[0], row[1]))
      
      self.connection.commit()
        

  def get_unresolved_btcpay_invoices(self):
    self.cursor.execute("""
      SELECT unresolved_btcpay_invoices.id, payments.created, payments.dollars, unresolved_btcpay_invoices.email 
      FROM unresolved_btcpay_invoices JOIN payments on payment_id = payments.id
    """)
    return list(map(lambda row: dict(id=row[0], created=row[1], dollars=row[2], email=row[3]), self.cursor.fetchall()))

  def get_account_balance_warning(self, email):
    self.cursor.execute("SELECT account_balance_warning FROM accounts WHERE email = %s", (email,))
    return self.cursor.fetchone()[0]

  def set_account_balance_warning(self, email, account_balance_warning):
    self.cursor.execute("UPDATE accounts SET account_balance_warning = %s WHERE email = %s", (account_balance_warning, email))
    self.connection.commit()

  def all_accounts(self):
    self.cursor.execute("SELECT email, account_balance_warning FROM accounts WHERE ever_logged_in = TRUE ")
    return list(map(lambda row: dict(email=row[0], account_balance_warning=row[1]), self.cursor.fetchall()))


  #     ------    HOSTS     ---------  

  def list_hosts_with_networks(self, host_id: str):
    query = """
    SELECT hosts.id, hosts.last_health_check, host_network.network_name, 
    host_network.public_ipv4_cidr_block, host_network.public_ipv4_first_usable_ip, host_network.public_ipv4_last_usable_ip
    FROM hosts
    JOIN host_network ON host_network.host = hosts.id
    """
    if host_id is None: 
      self.cursor.execute(query)
    else:
      if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
        raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")

      # I kept getting "TypeError: not all arguments converted during string formatting"
      # when I was trying to mix python query string templating with psycopg2 safe parameter passing. 
      # so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
      self.cursor.execute(f"{query} WHERE hosts.id = '{host_id}'")

    hosts = dict()
    for row in self.cursor.fetchall():
      if row[0] not in hosts:
        hosts[row[0]] = dict(last_health_check=row[1], networks=[])

      hosts[row[0]]["networks"].append(dict(
        network_name=row[2], 
        public_ipv4_cidr_block=row[3], 
        public_ipv4_first_usable_ip=row[4], 
        public_ipv4_last_usable_ip=row[5]
      ))

    return hosts

  def authorized_for_host(self, id, token) -> bool:
    self.cursor.execute("SELECT id FROM hosts WHERE id = %s AND token = %s", (id, token))
    return self.cursor.fetchone() != None

  def host_heartbeat(self, id) -> None:
    self.cursor.execute("UPDATE hosts SET last_health_check = NOW() WHERE id = %s", (id,))
    self.connection.commit()

  def get_all_hosts(self) -> List[OnlineHost]:
    self.cursor.execute("SELECT id, https_url FROM hosts")
    return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))

  def get_online_hosts(self) -> List[OnlineHost]:
    self.cursor.execute("SELECT id, https_url FROM hosts WHERE last_health_check > NOW() - INTERVAL '20 seconds'")
    return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))

  def list_all_operations(self): 
    self.cursor.execute("""
    SELECT operations.id, operations.email, operations.created, operations.payload,
           host_operation.host host_operation.assignment_status, host_operation.assigned, 
           host_operation.completed, host_operation.results FROM operations
    JOIN host_operation ON host_operation.operation = operations.id
    """)

    operations = dict()
    for row in self.cursor.fetchall():
      if row[0] not in operations:
        operations[row[0]] = dict(email=row[1], created=row[2], payload=row[3], hosts=[])

      operations[row[0]]["hosts"].append(dict(
        host=row[4], assignment_status=row[5], assigned=row[6], 
        completed=row[7], results=row[8],
      ))

    return operations

  def create_operation(self, online_hosts: List[OnlineHost], email: str, payload: str) -> int:
    self.cursor.execute( "INSERT INTO operations (email, payload) VALUES (%s, %s) RETURNING id", (email, payload) )
    operation_id = self.cursor.fetchone()[0]
  
    for host in online_hosts:
      self.cursor.execute( "INSERT INTO host_operation (host, operation) VALUES (%s, %s)", (host.id, operation_id) )
      
    self.connection.commit()
    return operation_id

  def update_operation(self, operation_id: int, payload: str):
    self.cursor.execute(
      "UPDATE operations SET payload = %s WHERE id = %s", 
      (payload, operation_id)
    )
    self.connection.commit()

  def update_host_operation(self, host_id: str, operation_id: int, assignment_status: str, result: str):
    if assignment_status and not result:
      self.cursor.execute(
        "UPDATE host_operation SET assignment_status = %s, assigned = NOW() WHERE host = %s AND operation = %s", 
        (assignment_status, host_id, operation_id)
      )
    elif not assignment_status and result:
      self.cursor.execute(
        "UPDATE host_operation SET results = %s, completed = NOW() WHERE host = %s AND operation = %s", 
        (result, host_id, operation_id)
      )
    elif assignment_status and result:
      self.cursor.execute(
        "UPDATE host_operation SET assignment_status = %s, assigned = NOW(), results = %s, completed = NOW() WHERE host = %s AND operation = %s", 
        (assignment_status, result, host_id, operation_id)
      )
    self.connection.commit()

  def host_of_capsul(self, capsul_id: str) -> OnlineHost:
    self.cursor.execute("SELECT hosts.id, hosts.https_url from vms JOIN hosts on hosts.id = vms.host where vms.id = %s",  (capsul_id,))
    row = self.cursor.fetchone()
    if row:
      return OnlineHost(row[0], row[1])
    else:
      return None

  def get_payload_json_from_host_operation(self, operation_id: int, host_id: str) -> str:
    self.cursor.execute(
      """
        SELECT operations.payload FROM operations 
          JOIN host_operation ON host_operation.operation = operations.id
          WHERE host_operation.host = %s AND host_operation.operation = %s
      """,
      (host_id, operation_id)
    )
    row = self.cursor.fetchone()
    if row:
      return row[0]
    else:
      return None

  def claim_operation(self, operation_id: int, host_id: str) -> bool:
    # have to make a new cursor to set isolation level
    # cursor = self.connection.cursor()
    # self.cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE;")
    # psycopg2.errors.ActiveSqlTransaction: SET TRANSACTION ISOLATION LEVEL must be called before any query
    self.cursor.execute("""
      BEGIN TRANSACTION;
      UPDATE host_operation SET assignment_status = 'assigned' 
        WHERE host = %s AND operation = %s AND operation != (
          SELECT COALESCE(
            (SELECT operation FROM host_operation WHERE operation = %s AND assignment_status = 'assigned'),
            -1
          ) as already_assigned_operation_id
        );
      COMMIT TRANSACTION;
    """,  (host_id, operation_id, operation_id))

    to_return = self.cursor.rowcount != 0

    self.connection.commit()
    #cursor.close()

    return to_return