import os import re import sys from datetime import datetime, timedelta import click from flask.cli import with_appcontext from flask import Blueprint from flask import current_app from psycopg2 import ProgrammingError from flask_mail import Message from capsulflask.db import get_model from capsulflask.shared import my_exec_info_message, get_account_balance from capsulflask.payment import check_if_shortterm_flag_can_be_unset from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information bp = Blueprint('cli', __name__) @bp.cli.command('sql') @click.option('-f', help='script filename') @click.option('-c', help='sql command') @with_appcontext def sql_script(f, c): """Run a sql script against the database. script is run 1 command at a time inside a single transaction.""" model = get_model() script = "" if f: filepath = os.path.join(os.getcwd(), f) if not os.path.isfile(filepath): raise f"{filepath} is not a file" with open(filepath, 'rb') as file: script = file.read().decode("utf8") elif c: script = c else: click.echo(f"you must provide sql to run either inline with the -c argument or in a file with the -f argument") return commands = re.split(";\\s+", script) for command in commands: if command.strip() != "": model.cursor.execute(command) if re.match("^\\s*select", command, re.IGNORECASE) is not None: for row in model.cursor.fetchall(): def format_value(x): if isinstance(x, bool): return "true" if x else "false" if not x : return "null" if isinstance(x, datetime): return x.isoformat() return f"{x}" click.echo(", ".join(list(map(format_value, row)))) else: click.echo(f"{model.cursor.rowcount} rows affected.") model.connection.commit() @bp.cli.command('account-balance') @click.option('-u', help='users email address') @with_appcontext def account_balance(u): vms = get_model().list_vms_for_account(u) payments = get_model().list_payments_for_account(u) click.echo(".") click.echo(".") click.echo(get_account_balance(vms, payments, datetime.utcnow())) click.echo(".") @bp.cli.command('cron-task') @with_appcontext def cron_task(): # make sure btcpay payments get completed (in case we miss a webhook), otherwise invalidate the payment current_app.logger.info("cron_task: starting clean_up_unresolved_btcpay_invoices") clean_up_unresolved_btcpay_invoices() current_app.logger.info("cron_task: finished clean_up_unresolved_btcpay_invoices") # notify when funds are about to run out and delete long-term vms once account reaches -$10 current_app.logger.info("cron_task: starting notify_users_about_account_balance") notify_users_about_account_balance() current_app.logger.info("cron_task: finished notify_users_about_account_balance") # delete short-term vms and notify user once account reaches $0 current_app.logger.info("cron_task: starting delete_shortterm_vms_if_account_is_empty") delete_shortterm_vms_if_account_is_empty() current_app.logger.info("cron_task: finished delete_shortterm_vms_if_account_is_empty") # make sure vm system and DB are synced current_app.logger.info("cron_task: starting ensure_vms_and_db_are_synced") ensure_vms_and_db_are_synced() current_app.logger.info("cron_task: finished ensure_vms_and_db_are_synced") def clean_up_unresolved_btcpay_invoices(): unresolved_btcpay_invoices = get_model().get_unresolved_btcpay_invoices() for unresolved_invoice in unresolved_btcpay_invoices: invoice_id = unresolved_invoice['id'] btcpay_invoice = None try: btcpay_invoice = current_app.config['BTCPAY_CLIENT'].get_invoice(invoice_id) except: current_app.logger.error(f""" error was thrown when contacting btcpay server for invoice {invoice_id}: {my_exec_info_message(sys.exc_info())}""" ) continue days = float((datetime.now() - unresolved_invoice['created']).total_seconds())/float(60*60*24) if btcpay_invoice['status'] == "complete": current_app.logger.info( f"resolving btcpay invoice {invoice_id} " f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as completed " ) resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True) if resolved_invoice_email is not None: check_if_shortterm_flag_can_be_unset(resolved_invoice_email) elif days >= 1: current_app.logger.info( f"resolving btcpay invoice {invoice_id} " f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as invalidated, " f"btcpay server invoice status: {btcpay_invoice['status']}" ) get_model().btcpay_invoice_resolved(invoice_id, False) get_model().delete_payment_session("btcpay", invoice_id) delete_at_account_balance_dollars = -10 def get_warning_headline(warning_id, pluralize_capsul): return dict( zero_1w= ( "According to our calculations, your Capsul account will run out of funds before this time next week.\n\n" ), zero_1d= ( "According to our calculations, your Capsul account will run out of funds by this time tomorrow.\n\n" ), zero_now= ( f"You have run out of funds! You will no longer be able to create Capsuls.\n\n" f"As a courtesy, we'll let your existing Capsul{pluralize_capsul} keep running until your account " "reaches a -$10 balance, at which point they will be deleted.\n\n" ), delete_1w= ( "You have run out of funds and have not refilled your account.\n\n" f"As a courtesy, we've let your existing Capsul{pluralize_capsul} keep running. " f"However, your account will reach a -$10 balance some time next week and your Capsul{pluralize_capsul} " "will be deleted.\n\n" ), delete_1d= ( "You have run out of funds and have not refilled your account.\n\n" f"As a courtesy, we have let your existing Capsul{pluralize_capsul} keep running. " f"However, your account will reach a -$10 balance by this time tomorrow and " f"your Capsul{pluralize_capsul} will be deleted.\n\n" f"Last chance to deposit funds now and keep your Capsul{pluralize_capsul} running! " ), delete_now= ( f"Your account reached a -$10 balance and your Capsul{pluralize_capsul} were deleted." ) )[warning_id] def get_warnings_list(): return [ dict( id='zero_1w', get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < 0, get_subject=lambda _: "Capsul One Week Payment Reminder", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('zero_1w', pluralize_capsul)}" f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n" "If you believe you have recieved this message in error, please let us know: support@cyberia.club" ) ), dict( id='zero_1d', get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < 0, get_subject=lambda _: "Capsul One Day Payment Reminder", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('zero_1d', pluralize_capsul)}" f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n" "If you believe you have recieved this message in error, please let us know: support@cyberia.club" ) ), dict( id='zero_now', get_active=lambda balance_1w, balance_1d, balance_now: balance_now < 0, get_subject=lambda _: "Your Capsul Account is No Longer Funded", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('zero_now', pluralize_capsul)}" f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n" f"If you need help decomissioning your Capsul{pluralize_capsul}, " "would like to request backups, or de-activate your account, please contact: support@cyberia.club" ) ), dict( id='delete_1w', get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < delete_at_account_balance_dollars, get_subject=lambda pluralize_capsul: f"Your Capsul{pluralize_capsul} Will be Deleted In Less Than a Week", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('delete_1w', pluralize_capsul)}" f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n" f"If you need help decomissioning your Capsul{pluralize_capsul}, " "would like to request backups, or de-activate your account, please contact: support@cyberia.club" ) ), dict( id='delete_1d', get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < delete_at_account_balance_dollars, get_subject=lambda pluralize_capsul: f"Last Chance to Save your Capsul{pluralize_capsul}: Gone Tomorrow", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('delete_1d', pluralize_capsul)}" f"{base_url}/console/account-balance" ) ), dict( id='delete_now', get_active=lambda balance_1w, balance_1d, balance_now: balance_now < delete_at_account_balance_dollars, get_subject=lambda pluralize_capsul: f"Capsul{pluralize_capsul} Deleted", get_body=lambda base_url, pluralize_capsul: ( f"{get_warning_headline('delete_now', pluralize_capsul)}" ) ) ] def notify_users_about_account_balance(): accounts = get_model().all_accounts() out_of_bounds_accounts = dict() for account in accounts: vms = get_model().list_vms_for_account(account['email']) payments = get_model().list_payments_for_account(account['email']) balance_1w = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=7)) balance_1d = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=1)) balance_now = get_account_balance(vms, payments, datetime.utcnow()) current_warning = account['account_balance_warning'] if balance_now < -11 and len(list(filter(lambda vm: not vm['deleted'], vms))) > 0: out_of_bounds_accounts[account['email']] = balance_now longterm_vms = list(filter(lambda vm: vm['shortterm'] == False, vms)) if len(longterm_vms) == 0: continue pluralize_capsul = "s" if len(longterm_vms) > 1 else "" warnings = get_warnings_list() current_warning_index = -1 if current_warning: for i in range(0, len(warnings)): if warnings[i]['id'] == current_warning: current_warning_index = i index_to_send = -1 for i in range(0, len(warnings)): if i > current_warning_index and warnings[i]['get_active'](balance_1w, balance_1d, balance_now): index_to_send = i if index_to_send > -1: current_app.logger.info(f"cron_task: sending {warnings[index_to_send]['id']} warning email to {account['email']}.") get_body = warnings[index_to_send]['get_body'] get_subject = warnings[index_to_send]['get_subject'] current_app.config["FLASK_MAIL_INSTANCE"].send( Message( get_subject(pluralize_capsul), body=get_body(current_app.config['BASE_URL'], pluralize_capsul), sender=current_app.config["MAIL_DEFAULT_SENDER"], recipients=[account['email']] ) ) get_model().set_account_balance_warning(account['email'], warnings[index_to_send]['id']) if index_to_send == len(warnings)-1: for vm in longterm_vms: current_app.logger.warning(f"cron_task: deleting {vm['id']} ( {account['email']} ) due to negative account balance.") current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id']) get_model().delete_vm(email=account["email"], id=vm['id']) if len(out_of_bounds_accounts) > 0: lines_redacted = ["The following accounts have out-of-bounds account balances: (Un-redacted email addresses avaliable in the logs)", ""] lines = ["The following accounts have out-of-bounds account balances:", ""] for email, balance in out_of_bounds_accounts.items(): lines.append(f"{email}: ${format(balance, '.2f')}") lines_redacted.append(f"*******: ${format(balance, '.2f')}") email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) current_app.logger.info(f"notify_users_about_account_balance: sending out of bounds account balances email to {','.join(email_addresses)}:") for line in lines: current_app.logger.info(f"notify_users_about_account_balance: {line}.") current_app.config["FLASK_MAIL_INSTANCE"].send( Message( "Capsul Out Of Bounds Account Balance Notification", sender=current_app.config["MAIL_DEFAULT_SENDER"], body="\n".join(lines_redacted), recipients=email_addresses ) ) def delete_shortterm_vms_if_account_is_empty(): accounts = get_model().all_accounts() for account in accounts: vms = get_model().list_vms_for_account(account['email']) payments = get_model().list_payments_for_account(account['email']) balance = get_account_balance(vms, payments, datetime.utcnow()) shortterm_vms = list(filter(lambda vm: vm['shortterm'] == True, vms)) if len(shortterm_vms) > 0 and balance <= 0: pluralize_capsul = "s" if len(shortterm_vms) > 1 else "" pluralize_past_tense = "have" if len(shortterm_vms) > 1 else "has" current_app.config["FLASK_MAIL_INSTANCE"].send( Message( f"Short-term Capsul{pluralize_capsul} Deleted", body=( f"You have run out of funds! Your Short-term Capsul{pluralize_capsul} {pluralize_past_tense} been deleted.\n\n" ), sender=current_app.config["MAIL_DEFAULT_SENDER"], recipients=[account['email']] ) ) for vm in shortterm_vms: current_app.logger.warning(f"cron_task: deleting shortterm vm {vm['id']} ( {account['email']} ) due to negative account balance.") current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id']) get_model().delete_vm(email=account["email"], id=vm['id']) def ensure_vms_and_db_are_synced(): db_vms_by_id = get_all_vms_from_db() virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id) inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id) errors = list() for vm in inconsistency_info['in_db_but_not_in_virt']: errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model") for vm in inconsistency_info['state_not_equal_to_desired_state']: errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}") for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']: errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']: errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") for vm in inconsistency_info['has_wrong_ip']: errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") if len(errors) > 0: email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") for error in errors: current_app.logger.info(f"cron_task: {error}.") current_app.config["FLASK_MAIL_INSTANCE"].send( Message( "Capsul Consistency Check Failed", sender=current_app.config["MAIL_DEFAULT_SENDER"], body="\n".join(errors), recipients=email_addresses ) )