390 lines
16 KiB
Python
390 lines
16 KiB
Python
|
|
import os
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timedelta
|
|
|
|
import click
|
|
from flask.cli import with_appcontext
|
|
from flask import Blueprint
|
|
from flask import current_app
|
|
from psycopg2 import ProgrammingError
|
|
from flask_mail import Message
|
|
|
|
from capsulflask.db import get_model
|
|
from capsulflask.shared import my_exec_info_message, get_account_balance
|
|
from capsulflask.payment import check_if_shortterm_flag_can_be_unset
|
|
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
|
|
|
|
bp = Blueprint('cli', __name__)
|
|
|
|
@bp.cli.command('sql')
|
|
@click.option('-f', help='script filename')
|
|
@click.option('-c', help='sql command')
|
|
@with_appcontext
|
|
def sql_script(f, c):
|
|
"""Run a sql script against the database. script is run 1 command at a time inside a single transaction."""
|
|
|
|
model = get_model()
|
|
script = ""
|
|
if f:
|
|
filepath = os.path.join(os.getcwd(), f)
|
|
if not os.path.isfile(filepath):
|
|
raise f"{filepath} is not a file"
|
|
|
|
with open(filepath, 'rb') as file:
|
|
script = file.read().decode("utf8")
|
|
elif c:
|
|
script = c
|
|
else:
|
|
click.echo(f"you must provide sql to run either inline with the -c argument or in a file with the -f argument")
|
|
return
|
|
|
|
commands = re.split(";\\s+", script)
|
|
|
|
for command in commands:
|
|
if command.strip() != "":
|
|
|
|
model.cursor.execute(command)
|
|
if re.match("^\\s*select", command, re.IGNORECASE) is not None:
|
|
for row in model.cursor.fetchall():
|
|
def format_value(x):
|
|
if isinstance(x, bool):
|
|
return "true" if x else "false"
|
|
if not x :
|
|
return "null"
|
|
if isinstance(x, datetime):
|
|
return x.isoformat()
|
|
return f"{x}"
|
|
|
|
click.echo(", ".join(list(map(format_value, row))))
|
|
else:
|
|
click.echo(f"{model.cursor.rowcount} rows affected.")
|
|
|
|
model.connection.commit()
|
|
|
|
@bp.cli.command('account-balance')
|
|
@click.option('-u', help='users email address')
|
|
@with_appcontext
|
|
def account_balance(u):
|
|
vms = get_model().list_vms_for_account(u)
|
|
payments = get_model().list_payments_for_account(u)
|
|
click.echo(".")
|
|
click.echo(".")
|
|
click.echo(get_account_balance(vms, payments, datetime.utcnow()))
|
|
click.echo(".")
|
|
|
|
|
|
|
|
@bp.cli.command('cron-task')
|
|
@with_appcontext
|
|
def cron_task():
|
|
|
|
# make sure btcpay payments get completed (in case we miss a webhook), otherwise invalidate the payment
|
|
current_app.logger.info("cron_task: starting clean_up_unresolved_btcpay_invoices")
|
|
clean_up_unresolved_btcpay_invoices()
|
|
current_app.logger.info("cron_task: finished clean_up_unresolved_btcpay_invoices")
|
|
|
|
# notify when funds are about to run out and delete long-term vms once account reaches -$10
|
|
current_app.logger.info("cron_task: starting notify_users_about_account_balance")
|
|
notify_users_about_account_balance()
|
|
current_app.logger.info("cron_task: finished notify_users_about_account_balance")
|
|
|
|
# delete short-term vms and notify user once account reaches $0
|
|
current_app.logger.info("cron_task: starting delete_shortterm_vms_if_account_is_empty")
|
|
delete_shortterm_vms_if_account_is_empty()
|
|
current_app.logger.info("cron_task: finished delete_shortterm_vms_if_account_is_empty")
|
|
|
|
# make sure vm system and DB are synced
|
|
current_app.logger.info("cron_task: starting ensure_vms_and_db_are_synced")
|
|
ensure_vms_and_db_are_synced()
|
|
current_app.logger.info("cron_task: finished ensure_vms_and_db_are_synced")
|
|
|
|
|
|
|
|
def clean_up_unresolved_btcpay_invoices():
|
|
unresolved_btcpay_invoices = get_model().get_unresolved_btcpay_invoices()
|
|
for unresolved_invoice in unresolved_btcpay_invoices:
|
|
invoice_id = unresolved_invoice['id']
|
|
btcpay_invoice = None
|
|
try:
|
|
btcpay_invoice = current_app.config['BTCPAY_CLIENT'].get_invoice(invoice_id)
|
|
except:
|
|
current_app.logger.error(f"""
|
|
error was thrown when contacting btcpay server for invoice {invoice_id}:
|
|
{my_exec_info_message(sys.exc_info())}"""
|
|
)
|
|
continue
|
|
|
|
days = float((datetime.now() - unresolved_invoice['created']).total_seconds())/float(60*60*24)
|
|
|
|
if btcpay_invoice['status'] == "complete":
|
|
current_app.logger.info(
|
|
f"resolving btcpay invoice {invoice_id} "
|
|
f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as completed "
|
|
)
|
|
resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True)
|
|
|
|
if resolved_invoice_email is not None:
|
|
check_if_shortterm_flag_can_be_unset(resolved_invoice_email)
|
|
|
|
elif days >= 1:
|
|
current_app.logger.info(
|
|
f"resolving btcpay invoice {invoice_id} "
|
|
f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as invalidated, "
|
|
f"btcpay server invoice status: {btcpay_invoice['status']}"
|
|
)
|
|
get_model().btcpay_invoice_resolved(invoice_id, False)
|
|
get_model().delete_payment_session("btcpay", invoice_id)
|
|
|
|
delete_at_account_balance_dollars = -10
|
|
|
|
def get_warning_headline(warning_id, pluralize_capsul):
|
|
return dict(
|
|
zero_1w= (
|
|
"According to our calculations, your Capsul account will run out of funds before this time next week.\n\n"
|
|
),
|
|
zero_1d= (
|
|
"According to our calculations, your Capsul account will run out of funds by this time tomorrow.\n\n"
|
|
),
|
|
zero_now= (
|
|
f"You have run out of funds! You will no longer be able to create Capsuls.\n\n"
|
|
f"As a courtesy, we'll let your existing Capsul{pluralize_capsul} keep running until your account "
|
|
"reaches a -$10 balance, at which point they will be deleted.\n\n"
|
|
),
|
|
delete_1w= (
|
|
"You have run out of funds and have not refilled your account.\n\n"
|
|
f"As a courtesy, we've let your existing Capsul{pluralize_capsul} keep running. "
|
|
f"However, your account will reach a -$10 balance some time next week and your Capsul{pluralize_capsul} "
|
|
"will be deleted.\n\n"
|
|
),
|
|
delete_1d= (
|
|
"You have run out of funds and have not refilled your account.\n\n"
|
|
f"As a courtesy, we have let your existing Capsul{pluralize_capsul} keep running. "
|
|
f"However, your account will reach a -$10 balance by this time tomorrow and "
|
|
f"your Capsul{pluralize_capsul} will be deleted.\n\n"
|
|
f"Last chance to deposit funds now and keep your Capsul{pluralize_capsul} running! "
|
|
),
|
|
delete_now= (
|
|
f"Your account reached a -$10 balance and your Capsul{pluralize_capsul} were deleted."
|
|
)
|
|
)[warning_id]
|
|
|
|
def get_warnings_list():
|
|
return [
|
|
dict(
|
|
id='zero_1w',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < 0,
|
|
get_subject=lambda _: "Capsul One Week Payment Reminder",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('zero_1w', pluralize_capsul)}"
|
|
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
|
|
"If you believe you have recieved this message in error, please let us know: support@cyberia.club"
|
|
)
|
|
),
|
|
dict(
|
|
id='zero_1d',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < 0,
|
|
get_subject=lambda _: "Capsul One Day Payment Reminder",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('zero_1d', pluralize_capsul)}"
|
|
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
|
|
"If you believe you have recieved this message in error, please let us know: support@cyberia.club"
|
|
)
|
|
),
|
|
dict(
|
|
id='zero_now',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_now < 0,
|
|
get_subject=lambda _: "Your Capsul Account is No Longer Funded",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('zero_now', pluralize_capsul)}"
|
|
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
|
|
f"If you need help decomissioning your Capsul{pluralize_capsul}, "
|
|
"would like to request backups, or de-activate your account, please contact: support@cyberia.club"
|
|
)
|
|
),
|
|
dict(
|
|
id='delete_1w',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < delete_at_account_balance_dollars,
|
|
get_subject=lambda pluralize_capsul: f"Your Capsul{pluralize_capsul} Will be Deleted In Less Than a Week",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('delete_1w', pluralize_capsul)}"
|
|
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
|
|
f"If you need help decomissioning your Capsul{pluralize_capsul}, "
|
|
"would like to request backups, or de-activate your account, please contact: support@cyberia.club"
|
|
)
|
|
),
|
|
dict(
|
|
id='delete_1d',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < delete_at_account_balance_dollars,
|
|
get_subject=lambda pluralize_capsul: f"Last Chance to Save your Capsul{pluralize_capsul}: Gone Tomorrow",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('delete_1d', pluralize_capsul)}"
|
|
f"{base_url}/console/account-balance"
|
|
)
|
|
),
|
|
dict(
|
|
id='delete_now',
|
|
get_active=lambda balance_1w, balance_1d, balance_now: balance_now < delete_at_account_balance_dollars,
|
|
get_subject=lambda pluralize_capsul: f"Capsul{pluralize_capsul} Deleted",
|
|
get_body=lambda base_url, pluralize_capsul: (
|
|
f"{get_warning_headline('delete_now', pluralize_capsul)}"
|
|
)
|
|
)
|
|
]
|
|
|
|
def notify_users_about_account_balance():
|
|
accounts = get_model().all_accounts()
|
|
|
|
out_of_bounds_accounts = dict()
|
|
|
|
for account in accounts:
|
|
vms = get_model().list_vms_for_account(account['email'])
|
|
payments = get_model().list_payments_for_account(account['email'])
|
|
balance_1w = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=7))
|
|
balance_1d = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=1))
|
|
balance_now = get_account_balance(vms, payments, datetime.utcnow())
|
|
current_warning = account['account_balance_warning']
|
|
|
|
if balance_now < -11 and len(list(filter(lambda vm: not vm['deleted'], vms))) > 0:
|
|
out_of_bounds_accounts[account['email']] = balance_now
|
|
|
|
longterm_vms = list(filter(lambda vm: vm['shortterm'] == False, vms))
|
|
|
|
if len(longterm_vms) == 0:
|
|
continue
|
|
|
|
pluralize_capsul = "s" if len(longterm_vms) > 1 else ""
|
|
|
|
warnings = get_warnings_list()
|
|
current_warning_index = -1
|
|
if current_warning:
|
|
for i in range(0, len(warnings)):
|
|
if warnings[i]['id'] == current_warning:
|
|
current_warning_index = i
|
|
|
|
index_to_send = -1
|
|
for i in range(0, len(warnings)):
|
|
if i > current_warning_index and warnings[i]['get_active'](balance_1w, balance_1d, balance_now):
|
|
index_to_send = i
|
|
|
|
if index_to_send > -1:
|
|
current_app.logger.info(f"cron_task: sending {warnings[index_to_send]['id']} warning email to {account['email']}.")
|
|
get_body = warnings[index_to_send]['get_body']
|
|
get_subject = warnings[index_to_send]['get_subject']
|
|
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
|
Message(
|
|
get_subject(pluralize_capsul),
|
|
body=get_body(current_app.config['BASE_URL'], pluralize_capsul),
|
|
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
|
recipients=[account['email']]
|
|
)
|
|
)
|
|
get_model().set_account_balance_warning(account['email'], warnings[index_to_send]['id'])
|
|
if index_to_send == len(warnings)-1:
|
|
for vm in longterm_vms:
|
|
current_app.logger.warning(f"cron_task: deleting {vm['id']} ( {account['email']} ) due to negative account balance.")
|
|
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
|
|
get_model().delete_vm(email=account["email"], id=vm['id'])
|
|
|
|
if len(out_of_bounds_accounts) > 0:
|
|
lines_redacted = ["The following accounts have out-of-bounds account balances: (Un-redacted email addresses avaliable in the logs)", ""]
|
|
lines = ["The following accounts have out-of-bounds account balances:", ""]
|
|
|
|
for email, balance in out_of_bounds_accounts.items():
|
|
lines.append(f"{email}: ${format(balance, '.2f')}")
|
|
lines_redacted.append(f"*******: ${format(balance, '.2f')}")
|
|
|
|
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
|
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
|
|
|
current_app.logger.info(f"notify_users_about_account_balance: sending out of bounds account balances email to {','.join(email_addresses)}:")
|
|
for line in lines:
|
|
current_app.logger.info(f"notify_users_about_account_balance: {line}.")
|
|
|
|
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
|
Message(
|
|
"Capsul Out Of Bounds Account Balance Notification",
|
|
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
|
body="\n".join(lines_redacted),
|
|
recipients=email_addresses
|
|
)
|
|
)
|
|
|
|
|
|
def delete_shortterm_vms_if_account_is_empty():
|
|
accounts = get_model().all_accounts()
|
|
for account in accounts:
|
|
vms = get_model().list_vms_for_account(account['email'])
|
|
payments = get_model().list_payments_for_account(account['email'])
|
|
balance = get_account_balance(vms, payments, datetime.utcnow())
|
|
shortterm_vms = list(filter(lambda vm: vm['shortterm'] == True, vms))
|
|
|
|
if len(shortterm_vms) > 0 and balance <= 0:
|
|
|
|
pluralize_capsul = "s" if len(shortterm_vms) > 1 else ""
|
|
pluralize_past_tense = "have" if len(shortterm_vms) > 1 else "has"
|
|
|
|
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
|
Message(
|
|
f"Short-term Capsul{pluralize_capsul} Deleted",
|
|
body=(
|
|
f"You have run out of funds! Your Short-term Capsul{pluralize_capsul} {pluralize_past_tense} been deleted.\n\n"
|
|
),
|
|
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
|
recipients=[account['email']]
|
|
)
|
|
)
|
|
|
|
for vm in shortterm_vms:
|
|
current_app.logger.warning(f"cron_task: deleting shortterm vm {vm['id']} ( {account['email']} ) due to negative account balance.")
|
|
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
|
|
get_model().delete_vm(email=account["email"], id=vm['id'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ensure_vms_and_db_are_synced():
|
|
db_vms_by_id = get_all_vms_from_db()
|
|
virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)
|
|
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
|
|
|
|
errors = list()
|
|
|
|
for vm in inconsistency_info['in_db_but_not_in_virt']:
|
|
errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model")
|
|
|
|
for vm in inconsistency_info['state_not_equal_to_desired_state']:
|
|
errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}")
|
|
|
|
for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']:
|
|
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
|
|
|
for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']:
|
|
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
|
|
|
for vm in inconsistency_info['has_wrong_ip']:
|
|
errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
|
|
|
if len(errors) > 0:
|
|
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
|
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
|
|
|
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
|
for error in errors:
|
|
current_app.logger.info(f"cron_task: {error}.")
|
|
|
|
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
|
Message(
|
|
"Capsul Consistency Check Failed",
|
|
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
|
body="\n".join(errors),
|
|
recipients=email_addresses
|
|
)
|
|
)
|
|
|