Merge branch 'master' into multiple-hosts

Conflicts:
	capsulflask/console.py
	capsulflask/db.py
	capsulflask/shared.py
	capsulflask/virt_model.py
This commit is contained in:
forest 2021-02-15 17:52:17 -06:00
commit dad6547825
23 changed files with 819 additions and 43 deletions

View File

@ -38,6 +38,10 @@ app.config.from_mapping(
HUB_TOKEN=os.environ.get("HUB_TOKEN", default="default"),
DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
# https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
DATABASE_SSLMODE=os.environ.get("DATABASE_SSLMODE", default="prefer"),
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
MAIL_SERVER=os.environ.get("MAIL_SERVER", default="m1.nullhex.com"),

View File

@ -43,17 +43,28 @@ def login():
errors.append("enter a valid email address")
if len(errors) == 0:
token = get_model().login(email)
result = get_model().login(email)
token = result[0]
ignoreCaseMatches = result[1]
if token is None:
errors.append("too many logins. please use one of the existing login links that have been emailed to you")
else:
link = f"{current_app.config['BASE_URL']}/auth/magic/{token}"
message = (f"Navigate to {link} to log into Capsul.\n"
"\nIf you didn't request this, ignore this message.")
if len(ignoreCaseMatches) > 0:
joinedMatches = " or ".join(map(lambda x: f"'{x}'", ignoreCaseMatches))
message = (f"You tried to log in as '{email}', but that account doesn't exist yet. \n"
f"If you would like to create a new account for '{email}', click here {link} \n\n"
f"If you meant to log in as {joinedMatches}, please return to https://capsul.org \n"
"and log in again with the correct (case-sensitive) email address.")
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Click This Link to Login to Capsul",
body=(f"Navigate to {link} to log into Capsul.\n"
"\nIf you didn't request this, ignore this message."),
body=message,
recipients=[email]
)
)

View File

@ -1,5 +1,6 @@
import re
import sys
import json
from datetime import datetime, timedelta
from flask import Blueprint
from flask import flash
@ -26,19 +27,22 @@ def makeCapsulId():
lettersAndNumbers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
return f"capsul-{lettersAndNumbers}"
def double_check_capsul_address(id, ipv4):
def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
try:
result = current_app.config["HUB_MODEL"].get(id)
result = current_app.config["HUB_MODEL"].get(id, get_ssh_host_keys)
if result.ipv4 != ipv4:
ipv4 = result.ipv4
get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4)
if get_ssh_host_keys:
get_model().update_vm_ssh_host_keys(email=session["account"], id=id, ssh_host_keys=result.ssh_host_keys)
except:
current_app.logger.error(f"""
the virtualization model threw an error in double_check_capsul_address of {id}:
{my_exec_info_message(sys.exc_info())}"""
)
return None
return ipv4
return result
@bp.route("/")
@account_required
@ -54,7 +58,9 @@ def index():
# for now we are going to check the IP according to the virt model
# on every request. this could be done by a background job and cached later on...
for vm in vms:
vm["ipv4"] = double_check_capsul_address(vm["id"], vm["ipv4"])
result = double_check_capsul_address(vm["id"], vm["ipv4"], False)
if result is not None:
vm["ipv4"] = result.ipv4
vms = list(map(
lambda x: dict(
@ -105,9 +111,17 @@ def detail(id):
return render_template("capsul-detail.html", vm=vm, delete=True, deleted=True)
else:
vm["ipv4"] = double_check_capsul_address(vm["id"], vm["ipv4"])
needs_ssh_host_keys = "ssh_host_keys" not in vm or len(vm["ssh_host_keys"]) == 0
vm_from_virt_model = double_check_capsul_address(vm["id"], vm["ipv4"], needs_ssh_host_keys)
if vm_from_virt_model is not None:
vm["ipv4"] = vm_from_virt_model.ipv4
if needs_ssh_host_keys:
vm["ssh_host_keys"] = vm_from_virt_model.ssh_host_keys
vm["created"] = vm['created'].strftime("%b %d %Y %H:%M")
vm["ssh_public_keys"] = ", ".join(vm["ssh_public_keys"]) if len(vm["ssh_public_keys"]) > 0 else "<missing>"
vm["ssh_authorized_keys"] = ", ".join(vm["ssh_authorized_keys"]) if len(vm["ssh_authorized_keys"]) > 0 else "<missing>"
return render_template(
"capsul-detail.html",
@ -124,7 +138,7 @@ def detail(id):
def create():
vm_sizes = get_model().vm_sizes_dict()
operating_systems = get_model().operating_systems_dict()
ssh_public_keys = get_model().list_ssh_public_keys_for_account(session["account"])
public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance(get_vms(), get_payments(), datetime.utcnow())
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
errors = list()
@ -155,7 +169,7 @@ def create():
if f"ssh_key_{i}" in request.form:
posted_name = request.form[f"ssh_key_{i}"]
key = None
for x in ssh_public_keys:
for x in public_keys_for_account:
if x['name'] == posted_name:
key = x
if key:
@ -180,7 +194,7 @@ def create():
id=id,
size=size,
os=os,
ssh_public_keys=list(map(lambda x: x["name"], posted_keys))
ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys))
)
current_app.config["HUB_MODEL"].create(
email = session["account"],
@ -188,14 +202,17 @@ def create():
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_public_keys=list(map(lambda x: x["content"], posted_keys))
ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys))
)
return redirect(f"{url_for('console.index')}?created={id}")
affordable_vm_sizes = dict()
for key, vm_size in vm_sizes.items():
if vm_size["dollars_per_month"] <= account_balance:
# if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month,
# then they have to delete the vm and re-create it, they will not be able to, they will have to pay again.
# so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake
if vm_size["dollars_per_month"] <= account_balance+0.25:
affordable_vm_sizes[key] = vm_size
for error in errors:
@ -209,9 +226,9 @@ def create():
csrf_token = session["csrf-token"],
capacity_avaliable=capacity_avaliable,
account_balance=format(account_balance, '.2f'),
ssh_public_keys=ssh_public_keys,
ssh_public_key_count=len(ssh_public_keys),
no_ssh_public_keys=len(ssh_public_keys) == 0,
ssh_public_keys=public_keys_for_account,
ssh_public_key_count=len(public_keys_for_account),
no_ssh_public_keys=len(public_keys_for_account) == 0,
operating_systems=operating_systems,
cant_afford=len(affordable_vm_sizes) == 0,
vm_sizes=affordable_vm_sizes

View File

@ -9,6 +9,7 @@ from flask import current_app
from flask import g
from capsulflask.db_model import DBModel
from capsulflask.shared import my_exec_info_message
def init_app(app):
databaseUrl = urlparse(app.config['DATABASE_URL'])
@ -20,7 +21,8 @@ def init_app(app):
password = databaseUrl.password,
host = databaseUrl.hostname,
port = databaseUrl.port,
database = databaseUrl.path[1:]
database = databaseUrl.path[1:],
sslmode = app.config['DATABASE_SSLMODE']
)
schemaMigrations = {}
@ -40,7 +42,7 @@ def init_app(app):
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 9
desiredSchemaVersion = 13
cursor = connection.cursor()
@ -126,4 +128,3 @@ def close_db(e=None):
db_model.cursor.close()
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)

View File

@ -21,18 +21,26 @@ class DBModel:
def login(self, email):
self.cursor.execute("SELECT * FROM accounts WHERE email = %s", (email, ))
if len(self.cursor.fetchall()) == 0:
self.cursor.execute("INSERT INTO accounts (email) VALUES (%s)", (email, ))
hasExactMatch = len(self.cursor.fetchall())
self.cursor.execute("SELECT * FROM accounts WHERE email = %s AND ever_logged_in = TRUE", (email, ))
everLoggedIn = len(self.cursor.fetchall())
ignoreCaseMatches = []
if everLoggedIn == 0:
self.cursor.execute("SELECT email FROM accounts WHERE lower_case_email = %s AND email != %s", (email.lower(), email))
ignoreCaseMatches = list(map(lambda x: x[0], self.cursor.fetchall()))
self.cursor.execute("SELECT token FROM login_tokens WHERE email = %s", (email, ))
if hasExactMatch == 0:
self.cursor.execute("INSERT INTO accounts (email, lower_case_email) VALUES (%s, %s)", (email, email.lower()))
self.cursor.execute("SELECT token FROM login_tokens WHERE email = %s and created > (NOW() - INTERVAL '20 min')", (email, ))
if len(self.cursor.fetchall()) > 2:
return None
return (None, ignoreCaseMatches)
token = generate()
self.cursor.execute("INSERT INTO login_tokens (email, token) VALUES (%s, %s)", (email, token))
self.connection.commit()
return token
return (token, ignoreCaseMatches)
def consume_token(self, token):
self.cursor.execute("SELECT email FROM login_tokens WHERE token = %s and created > (NOW() - INTERVAL '20 min')", (token, ))
@ -40,6 +48,7 @@ class DBModel:
if row:
email = row[0]
self.cursor.execute("DELETE FROM login_tokens WHERE email = %s", (email, ))
self.cursor.execute("UPDATE accounts SET ever_logged_in = TRUE WHERE email = %s", (email, ))
self.connection.commit()
return email
return None
@ -110,7 +119,17 @@ class DBModel:
self.cursor.execute("UPDATE vms SET last_seen_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
self.connection.commit()
def create_vm(self, email, id, size, os, ssh_public_keys):
def update_vm_ssh_host_keys(self, email, id, ssh_host_keys):
for key in ssh_host_keys:
self.cursor.execute("""
INSERT INTO vm_ssh_host_key (email, vm_id, key_type, content, sha256)
VALUES (%s, %s, %s, %s, %s)
""",
(email, id, key['key_type'], key['content'], key['sha256'])
)
self.connection.commit()
def create_vm(self, email, id, size, os, ssh_authorized_keys):
self.cursor.execute("""
INSERT INTO vms (email, id, size, os)
VALUES (%s, %s, %s, %s)
@ -118,12 +137,12 @@ class DBModel:
(email, id, size, os)
)
for ssh_public_key in ssh_public_keys:
for ssh_authorized_key in ssh_authorized_keys:
self.cursor.execute("""
INSERT INTO vm_ssh_public_key (email, vm_id, ssh_public_key_name)
INSERT INTO vm_ssh_authorized_key (email, vm_id, ssh_public_key_name)
VALUES (%s, %s, %s)
""",
(email, id, ssh_public_key)
(email, id, ssh_authorized_key)
)
self.connection.commit()
@ -151,11 +170,19 @@ class DBModel:
)
self.cursor.execute("""
SELECT ssh_public_key_name FROM vm_ssh_public_key
WHERE vm_ssh_public_key.email = %s AND vm_ssh_public_key.vm_id = %s""",
SELECT ssh_public_key_name FROM vm_ssh_authorized_key
WHERE vm_ssh_authorized_key.email = %s AND vm_ssh_authorized_key.vm_id = %s""",
(email, id)
)
vm["ssh_public_keys"] = list(map( lambda x: x[0], self.cursor.fetchall() ))
vm["ssh_authorized_keys"] = list(map( lambda x: x[0], self.cursor.fetchall() ))
self.cursor.execute("""
SELECT key_type, content, sha256 FROM vm_ssh_host_key
WHERE vm_ssh_host_key.email = %s AND vm_ssh_host_key.vm_id = %s""",
(email, id)
)
vm["ssh_host_keys"] = list(map( lambda x: dict(key_type=x[0], content=x[1], sha256=x[2]), self.cursor.fetchall() ))
return vm
@ -247,7 +274,7 @@ class DBModel:
if row:
self.cursor.execute( "DELETE FROM unresolved_btcpay_invoices WHERE id = %s", (id,) )
if not completed:
self.cursor.execute("UPDATE payments SET invalidated = True WHERE email = %s id = %s", (row[0], row[1]))
self.cursor.execute("UPDATE payments SET invalidated = TRUE WHERE email = %s id = %s", (row[0], row[1]))
self.connection.commit()
@ -268,7 +295,7 @@ class DBModel:
self.connection.commit()
def all_accounts(self):
self.cursor.execute("SELECT email, account_balance_warning FROM accounts")
self.cursor.execute("SELECT email, account_balance_warning FROM accounts WHERE ever_logged_in = TRUE ")
return list(map(lambda row: dict(email=row[0], account_balance_warning=row[1]), self.cursor.fetchall()))

View File

@ -22,6 +22,10 @@ def pricing():
def faq():
return render_template("faq.html")
@bp.route("/about-ssh")
def about_ssh():
return render_template("about-ssh.html")
@bp.route("/changelog")
def changelog():
return render_template("changelog.html")

View File

@ -0,0 +1,5 @@
ALTER TABLE accounts DROP COLUMN lower_case_email;
ALTER TABLE accounts DROP COLUMN ever_logged_in;
UPDATE schemaversion SET version = 8;

View File

@ -0,0 +1,10 @@
ALTER TABLE accounts
ADD COLUMN lower_case_email TEXT NULL;
ALTER TABLE accounts
ADD COLUMN ever_logged_in BOOLEAN NOT NULL DEFAULT FALSE;
UPDATE accounts set lower_case_email = LOWER(accounts.email);
UPDATE accounts set ever_logged_in = TRUE;
UPDATE schemaversion SET version = 9;

View File

@ -0,0 +1,3 @@
DELETE FROM os_images WHERE id = 'guixsystem120';
UPDATE schemaversion SET version = 9;

View File

@ -0,0 +1,4 @@
INSERT INTO os_images (id, template_image_file_name, description, deprecated)
VALUES ('guixsystem120', 'guixsystem/1.2.0/root.img.qcow2', 'Guix System 1.2.0', FALSE);
UPDATE schemaversion SET version = 10;

View File

@ -0,0 +1,5 @@
DELETE FROM os_images WHERE id = 'alpine313';
UPDATE os_images SET deprecated = FALSE WHERE id = 'alpine312';
UPDATE schemaversion SET version = 10;

View File

@ -0,0 +1,6 @@
INSERT INTO os_images (id, template_image_file_name, description, deprecated)
VALUES ('alpine313', 'alpine/3.13/root.img.qcow2', 'Alpine Linux 3.13', FALSE);
UPDATE os_images SET deprecated = TRUE WHERE id = 'alpine312';
UPDATE schemaversion SET version = 11;

View File

@ -0,0 +1,7 @@
DROP TABLE vm_ssh_host_key;
ALTER TABLE vm_ssh_authorized_key RENAME TO vm_ssh_public_key;
UPDATE schemaversion SET version = 11;

View File

@ -0,0 +1,14 @@
CREATE TABLE vm_ssh_host_key (
email TEXT NOT NULL,
vm_id TEXT NOT NULL,
key_type TEXT NOT NULL,
content TEXT NOT NULL,
sha256 TEXT NOT NULL,
FOREIGN KEY (email, vm_id) REFERENCES vms(email, id) ON DELETE CASCADE,
PRIMARY KEY (email, vm_id, key_type)
);
ALTER TABLE vm_ssh_public_key RENAME TO vm_ssh_authorized_key;
UPDATE schemaversion SET version = 12;

View File

@ -0,0 +1,11 @@
DROP TABLE host_operation;
DROP TABLE operations;
ALTER TABLE vms DROP COLUMN host;
DROP TABLE hosts;
UPDATE schemaversion SET version = 12;

View File

@ -0,0 +1,32 @@
CREATE TABLE hosts (
id TEXT PRIMARY KEY NOT NULL,
last_health_check TIMESTAMP NOT NULL DEFAULT NOW(),
https_url TEXT NOT NULL,
token TEXT NOT NULL
);
INSERT INTO hosts (id, https_url, token) VALUES ('baikal', 'http://localhost:5000', 'changeme');
ALTER TABLE vms
ADD COLUMN host TEXT REFERENCES hosts(id) ON DELETE RESTRICT DEFAULT 'baikal';
CREATE TABLE operations (
id SERIAL PRIMARY KEY ,
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
created TIMESTAMP NOT NULL DEFAULT NOW(),
payload TEXT NOT NULL
);
CREATE TABLE host_operation (
host TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT,
operation INTEGER NOT NULL REFERENCES operations(id) ON DELETE RESTRICT,
assignment_status TEXT NULL,
assigned TIMESTAMP NULL,
completed TIMESTAMP NULL,
results TEXT NULL,
PRIMARY KEY (host, operation)
);
UPDATE schemaversion SET version = 13;

View File

@ -1,18 +1,27 @@
import re
from flask import current_app
from typing import List
class OnlineHost:
def __init__(self, id: str, url: str):
self.id = id
self.url = url
# I decided to just use dict everywhere instead because I have to use dict to read it from json
# class SSHHostKey:
# def __init__(self, key_type=None, content=None, sha256=None):
# self.key_type = key_type
# self.content = content
# self.sha256 = sha256
class VirtualMachine:
def __init__(self, id, host, ipv4=None, ipv6=None):
def __init__(self, id, host, ipv4=None, ipv6=None, ssh_host_keys: List[dict] = list()):
self.id = id
self.host = host
self.ipv4 = ipv4
self.ipv6 = ipv6
self.ssh_host_keys = ssh_host_keys
class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:

View File

@ -0,0 +1,23 @@
#!/bin/sh
ip_address="$1"
if echo "$ip_address" | grep -vqE '^([0-9]{1,3}\.){3}[0-9]{1,3}$'; then
echo "ip_address $ip_address must match "'"^([0-9]{1,3}\.){3}[0-9]{1,3}$"'
exit 1
fi
printf '['
DELIMITER=""
ssh-keyscan "$ip_address" 2>/dev/null | while read -r line; do
if echo "$line" | grep -qE "^$ip_address"' +(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$'; then
KEY_CONTENT="$(echo "$line" | awk '{ print $2 " " $3 }')"
FINGERPRINT_OUTPUT="$(echo "$KEY_CONTENT" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')"
SHA256_HASH="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $1 }')"
KEY_TYPE="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $2 }')"
printf '%s\n {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$DELIMITER" "$KEY_TYPE" "$KEY_CONTENT" "$SHA256_HASH"
DELIMITER=","
fi
done
printf '\n]\n'

View File

@ -211,10 +211,22 @@ input[type=image].submit {
}
ul li {
ul li, ol li {
margin: 0.5em 0;
margin-left: 1.2rem;
}
.long-form p, .long-form li {
line-height: 2em;
}
.long-form p .code, .long-form li .code{
line-height: 1em;
padding: 5px;
padding-top: 3px;
margin-top: 2px;
padding-bottom: 4px;
border-radius: 4px;
}
hr {
@ -277,8 +289,12 @@ div.metric {
border: 1px solid #777e73;
background: #bdc7b810;
}
.break-word {
word-break: break-word;
pre.code.wrap {
white-space: pre-wrap;
}
.break-all {
word-break: break-all;
}
.dim {

View File

@ -0,0 +1,347 @@
{% extends 'base.html' %}
{% block title %}About SSH{% endblock %}
{% block content %}
<div class="row full-margin"><h1>Understanding the Secure Shell Protocol (SSH)</h1></div>
{% endblock %}
{% block subcontent %}
<div class="long-form">
<p>
In order to use our service, you will have to use the Secure Shell protocol (SSH) to connect to your capsul.
</p>
<p>
<a href="https://en.wikipedia.org/wiki/SSH_(Secure_Shell)">SSH</a> is a very old tool, created back when the internet was a different place, with different use cases and concerns.
In many ways, the protocol has failed to evolve to meet the needs of our 21st century global internet.
Instead, the users of SSH (tech heads, sysadmins, etc) have had to evolve our processes to work around SSH's limitations.
</p>
<p>
These days, we use SSH + public-key cryptography to establish secure connections to our servers.
If you are not familiar with the concept of public key cryptography, cryptographic signatures,
or diffie-hellman key exchange, you may wish to see
<a href="https://en.wikipedia.org/wiki/Public-key_cryptography">the wikipedia article</a> for a refresher.
</p>
<div class="row half-margin"><h1>Public Key Crypto and Key Exchange: The TL;DR</h1></div>
<p>
Computers can generate <b>"key pairs"</b> which consist of a public key and a private key. Given a <b>public key pair A</b>:
</p>
<ol>
<li>
A computer which has access to <b>public key A</b> can encrypt data,
and then <b>ONLY</b> a computer which has access <b>private key A</b> can decrypt & read it
</li>
<li>
Likewise, a computer which has access to <b>private key A</b> can encrypt data,
and any a computer which has access <b>public key A</b> can decrypt it,
thus <b>PROVING</b> the message must have come from someone who posesses <b>private key A</b>
</li>
</ol>
<p>
Key exchange is a process in which two computers, Computer A and Computer B (often referred to as Alice and Bob)
both create key pairs, so you have <b>key pair A</b> and <b>key pair B</b>, for a total of 4 keys:
</p>
<ol>
<li><b>public key A</b></li>
<li><b>private key A</b></li>
<li><b>public key B</b></li>
<li><b>private key B</b></li>
</ol>
<p>
In simplified terms, during a key exchange,
</p>
<ol>
<li><b>computer A</b> sends <b>computer B</b> its public key</li>
<li><b>computer B</b> sends <b>computer A</b> its public key</li>
<li><b>computer A</b> sends <b>computer B</b>
a message which is encrypted with <b>computer B</b>'s public key</li>
<li><b>computer B</b> sends <b>computer A</b>
a message which is encrypted with <b>computer A</b>'s public key</li>
</ol>
<p>
The way this process is carried out allows A and B to communicate with each-other securely, which is great, <br/><br/>
<b><u>HOWEVER, there is a catch!!</u></b>
</p>
<p>
When computers A and B are trying to establish a secure connection for the first time,
we assume that the way they communicate right now is NOT secure. That means that someone on the network
between A and B can read & modify
all messages they send to each-other! You might be able to see where this is heading...
</p>
<p>
When <b>computer A</b> sends its public key to <b>computer B</b>,
someone in the middle (lets call it <b>computer E, or Eve</b>) could record that message, save it,
and then replace it with a forged message to <b>computer B</b> containing <b>public key E</b>
(from a key pair that <b>computer E</b> generated).
If this happens, when <b>computer B</b> sends an encrypted message to <b>computer A</b>,
B thinks that A's public key is actually <b>public key E</b>, so it will use <b>public key E</b> to encrypt.
And again, <b>computer E</b> in the middle can intercept the message, and they can decrypt it as well
because they have <b>private key E</b>.
Finally, they can relay the same message to <b>computer A</b>, this time encrypted with <b>computer A</b>'s public key.
This is called a <a href="https://en.wikipedia.org/wiki/Man-in-the-middle_attack">Man In The Middle (MITM)</a> attack.
</p>
<p>
Without some additional verification method,
<b><u>Computer A AND Computer B can both be duped and the connection is NOT really secure</u></b>.
</p>
<div class="row half-margin"><h1>Authenticating Public Keys: A Tale of Two Protocols</h1></div>
<p>
Now that we have seen how key exhange works,
and we understand that in order to prevent MITM attacks, all participants have to have a way of knowing
whether a given public key is authentic or not, I can explain what I meant when I said
</p>
<p>
> [SSH] has failed to evolve to meet the needs of our 21st century global internet
</p>
<p>
In order to explain this, let's first look at how a different, more modern protocol,
<a href="https://en.wikipedia.org/wiki/Transport_Layer_Security">Transport Layer Security (or TLS)</a> solved this problem.
TLS, (still sometimes called by its olde name "Secure Sockets Layer", or SSL) was created to enable HTTPS, to allow
internet users to log into web sites securely and purchase things online by entering their credit card number.
Of course, this required security that actually works; if someone could MITM attack the connection, they could easily
steal tons of credit card numbers and passwords.
</p>
<p>
In order to enable this, a new standard called <a href="https://en.wikipedia.org/wiki/X.509">X.509</a> was created.
X.509 dictates the data format of certificates and keys (public keys and private keys), and it also defines
a simple and easy way to determine whether a given certificate (public key) is authentic.
X.509 introduced the concept of a Certificate Authority, or CA.
These CAs were supposed to be bank-like public institutions of power which everyone could trust.
The CA would create a key pair on an extremely secure computer, and then a CA Certificate (the public side of that key pair)
would be distributed along with every copy of Windows, Mac OS, and Linux. Then folks who wanted to run a secure web server
could generate thier OWN key pair for thier web server,
and pay the CA to sign thier web server's X.509 certificate (public key) with the highly protected CA private key.
Critically, issue date, expiration date, and the domain name of the web server, like foo.example.com, would have to be included
in the x.509 certiciate along with the public key.
This way, when the user types https://foo.example.com into thier web browser:
</p>
<ol>
<li>The web browser sends a TLS ClientHello request to the server</li>
<li>
The server responds with a ServerHello & ServerCertificate message
<ul>
<li>The ServerCertificate message contains the X.509 certificate for the web server at foo.example.com</li>
</ul>
</li>
<li>The web browser inspects the X.509 certificate
<ul>
<li>
Is the current date in between the issued date and expiry date of the certificate?
If not, display an <a href="https://expired.badssl.com/">EXPIRED_CERTIFICATE error</a>.
</li>
<li>
Does the domain name the user typed in, foo.example.com, match the domain name in the certificate?
If not, display a <a href="https://wrong.host.badssl.com/">BAD_CERT_DOMAIN error</a>.
</li>
<li>
Does the certificate contain a valid CA signature?
(can the signature on the certificate be decrypted by one of the CA Certificates included with the operating system?)
If not, display an <a href="https://untrusted-root.badssl.com/">UNKNOWN_ISSUER error</a>.
</li>
</ul>
</li>
<li>Assuming all the checks pass, the web browser trusts the certificate and connects</li>
</ol>
<p>
This system enabled the internet to grow and flourish:
purchasing from a CA was the only way to get a valid X.509 certificate for a website,
and guaranteeing authenticity was in the CA's business interest.
The CAs kept their private keys behind razor wire and armed guards, and followed strict rules to ensure that only the right
people got thier certificates signed.
Only the CAs themselves or anyone who had enough power to force them to create a fraudulent certificate
would be able to execute MITM attacks.
</p>
<p>
The TLS+X.509 Certificate Authority works well for HTTP and other application protocols, because
</p>
<ul>
<li>Most internet users don't have the patience to manually verify the authenticity of digital certificates.</li>
<li>Most internet users don't understand or care how it works; they just want to connect right now.</li>
<li>Businesses and organizations that run websites are generally willing to jump through hoops and
subjugate themselves to authorities in order to offer a more secure application experience to thier users.</li>
<li>The centralization & problematic power dynamic which CAs represent
is easily swept under the rug, if it doesn't directly or noticably impact the average person, who cares?</li>
</ul>
<p>
However, this would never fly with SSH. You have to understand, SSH does not come from Microsoft, it does not come from Apple,
in fact, it does not even come from Linux or GNU. <a href="https://www.openssh.com/">SSH comes from BSD</a>.
<a href="https://en.wikipedia.org/wiki/BSD">Berkeley Software Distribution</a>. Most people don't even know
what BSD is. It's <i>Deep Nerdcore</i> material. The people who maintain SSH are not playing around, they would never
allow themselves to be subjugated by so-called "Certificate Authorities".
So, what are they doing instead? Where is SSH at? Well, back when it was created, computer security was easy —
a very minimal defense was enough to deter attackers.
In order to help prevent these MITM attacks, instead of something like X.509, SSH employs a policy called
<a href="https://en.wikipedia.org/wiki/Trust_on_first_use">Trust On First Use (TOFU)</a>.
</p>
<p>
The SSH client application keeps a record of every server it has ever connected to
in a file <span class="code">~/.ssh/known_hosts</span>.
</p>
<p>
(the tilde <span class="code">~</span> here represents the user's home directory,
<span class="code">/home/username</span> on linux,
<span class="code">C:\Users\username</span> on Windows, and
<span class="code">/Users/username</span> on MacOS).
</p>
<p>
If the user asks the SSH client to connect to a server it has never seen before,
it will print a prompt like this to the terminal:
</p>
<pre class="code">The authenticity of host 'fooserver.com (69.4.20.69)' can't be established.
ECDSA key fingerprint is SHA256:EXAMPLE1xY4JUVhYirOVlfuDFtgTbaiw3x29xYizEeU.
Are you sure you want to continue connecting (yes/no/[fingerprint])?</pre>
<p>
Here, the SSH client is displaying the fingerprint (<a href="https://en.wikipedia.org/wiki/SHA-2">SHA256 hash</a>)
of the public key provided by the server at <span class="code">fooserver.com</span>.
Back in the day, when SSH was created, servers lived for months to years, not minutes, and they were installed by hand.
So it would have been perfectly reasonable to call the person installing the server on thier
<a href="https://nokiamuseum.info/nokia-909/">Nokia 909</a>
and ask them to log into it & read off the host key fingerprint over the phone.
After verifing that the fingerprints match in the phone call, the user would type <span class="code">yes</span>
to continue.
</p>
<p>
After the SSH client connects to a server for the first time, it will record the server's IP address and public key in the
<span class="code">~/.ssh/known_hosts</span> file. All subsequent connections will simply check the public key
the server presents against the public key it has recorded in the <span class="code">~/.ssh/known_hosts</span> file.
If the two public keys match, the connection will continue without prompting the user, however, if they don't match,
the SSH client will display a scary warning message:
</p>
<pre class="code">
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ WARNING: POSSIBLE DNS SPOOFING DETECTED! @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
The ECDSA host key for fooserver.com has changed,
and the key for the corresponding IP address 69.4.20.42
is unknown. This could either mean that
DNS SPOOFING is happening or the IP address for the host
and its host key have changed at the same time.
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!
Someone could be eavesdropping on you right now (man-in-the-middle attack)!
It is also possible that a host key has just been changed.
The fingerprint for the ECDSA key sent by the remote host is
SHA256:EXAMPLEpDDefcNcIROtFpuTiHC1j3iNU74aaKFO03+0.
Please contact your system administrator.
Add correct host key in /root/.ssh/known_hosts to get rid of this message.
Offending ECDSA key in /root/.ssh/known_hosts:1
remove with:
ssh-keygen -f "/root/.ssh/known_hosts" -R "fooserver.com"
ECDSA host key for fooserver.com has changed and you have requested strict checking.
Host key verification failed.
</pre>
<p>
This is why it's called <b>Trust On First Use</b>:
SSH protocol assumes that when you type <span class="code">yes</span> in response to the prompt during your first connection,
you <b>really did</b> verify that the server's public key fingerprint matches.
If you type <span class="code">yes</span> here without checking the server's host key somehow, you could add an attackers public key to the trusted
list in your <span class="code">~/.ssh/known_hosts</span> file; if you type <span class="code">yes</span> blindly, you are
<b>completely disabling all security of the SSH connection</b>.
It can be fully man-in-the-middle attacked & you are
vulnerable to surveillance, command injection, even emulation/falsification of the entire stream.
Will anyone actually attack you like that? Who knows. Personally, I'd rather not find out.
</p>
<p>
So what are technologists to do? Most cloud providers don't "provide" an easy way to get the SSH host public keys
for instances that users create on thier platform. For example, see this
<a href="https://serverfault.com/questions/941915/verify-authenticity-of-ssh-host-on-digital-ocean-droplet-freebsd">
question posted by a frustrated user trying to secure thier connection to a digitalocean droplet</a>.
Besides using the provider's HTTPS-based console to log into the machine & directly read the public key,
providers also recommend using a "userdata script".
This script would run on boot & upload the machine's SSH public keys to a
trusted location like <a href="https://www.backblaze.com/b2/cloud-storage.html">Backblaze B2</a> or
<del>Amazon S3</del><sup><a href="#ref_1">[1]</a></sup>, for an application to retrieve later.
As an example, I wrote a
<a href="https://git.sequentialread.com/forest/rootsystem/src/1cdbe53974d20da97d9f522d4bd62c34487817c0/terraform-modules/gateway-instance-digitalocean/upload_known_hosts.tpl#L5">
userdata script which does this</a>
for my own cloud compute management tool called
<a href="https://git.sequentialread.com/forest/rootsystem">rootsystem</a>.
Later in the process, rootsystem will
<a href="https://git.sequentialread.com/forest/rootsystem/src/1cdbe53974d20da97d9f522d4bd62c34487817c0/host-key-poller/main.go#L33">
download the public keys from the Object Storage provider
and add them to the ~/.ssh/known_hosts file</a>
before finally
<a href="https://git.sequentialread.com/forest/rootsystem/src/1cdbe53974d20da97d9f522d4bd62c34487817c0/terraform-modules/ansible-threshold-server/main.tf#L32">
invoking the ssh client against the cloud host</a>.
</p>
<p>
Personally, I think it's disgusting and irresponsible to require users to go through that much work
just to be able to connect to their instance securely. However, this practice appears to be an industry standard.
It's gross, but it's where we're at right now.
</p>
<p>
So for <a href="https://capsul.org">capsul</a>, we obviously wanted to do better.
We wanted to make this kind of thing as easy as possible for the user,
so I'm proud to announce as of today, capsul SSH host key fingerprints will be displayed on the capsul detail page,
as well as the host's SSH public keys themselves in <span class="code">~/.ssh/known_hosts</span> format.
Users can simply copy and paste these keys into thier <span class="code">~/.ssh/known_hosts</span> file and connect
with confidence that they are not being MITM attacked.
</p>
<div class="row half-margin"><h1>Why ssh more ssh</h1></div>
<p>
SSH is a relatively low-level protocol, it should be kept simple and it should not depend on anything external.
It has to be this way, because often times SSH is the first service that runs on a server, before any other
services or processes launch. SSH server has to run no matter what, because it's what we're gonna depend on to
log in there and fix everything else which is broken! Also, SSH has to work for all computers, not just the ones which
have internet access or are reachable publically.
So, arguing that SSH should be wrapped in TLS or that SSH should use x.509 doesn't make much sense.
</p>
<hr/>
<p>
> ssh didnt needed an upgrade. SSH is perfect
</p>
<hr/>
<p>
Because of the case for absolute simplicity, I think that in a cloud based use-case
it might even make sense to remove the TOFU and make the ssh client even less user friendly; requiring the
expected host key to be passed in on every command by default
would dramatically increase the security of real-world SSH usage.
In order to make it more human-friendly again while keeping the security benefits,
we can create a new layer of abstraction on top of SSH, create regime-specific automation & wrapper scripts.
</p>
<p>
For example, when we build a JSON API for capsul, we could also provide a <span class="code">capsul-cli</span>
application which contains an SSH wrapper that knows how to automatically grab & inject the authentic host keys and invoke ssh
in a single command.
</p>
<p>
Cheers and best wishes,<br/>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Forest
</p>
<hr/>
<p>
<sup id="ref_1">[1]</sup> <a href="https://www.doitwithoutdues.com/">fuck amazon</a>
</p>
</div>
{% endblock %}
{% block pagesource %}/templates/about-ssh.html{% endblock %}

View File

@ -73,8 +73,8 @@
<span id="ssh_username">cyberian</span>
</div>
<div class="row justify-start">
<label class="align" for="ssh_public_keys">SSH Public Keys</label>
<a id="ssh_public_keys" href="/console/ssh">{{ vm['ssh_public_keys'] }}</a>
<label class="align" for="ssh_authorized_keys">SSH Authorized Keys</label>
<a id="ssh_authorized_keys" href="/console/ssh">{{ vm['ssh_authorized_keys'] }}</a>
</div>
<div class="row center justify-start">
<label class="align" for="delete_action">Actions</label>
@ -85,6 +85,17 @@
</form>
</div>
</div>
<div class="row third-margin">
<h1>ssh host key fingerprints</h1>
</div>
<div class="row">
<pre class="code">{% for key in vm['ssh_host_keys'] %}
SHA256:{{ key.sha256 }} ({{ key.key_type }}){% endfor %}</pre>
</div>
<div class="row">
<span>(What's this? see <a href="/about-ssh">Understanding the Secure Shell Protocol (SSH)</a>)</span>
</div>
<div class="row ">
<hr/>
</div>
@ -136,6 +147,20 @@
</a>
</div>
</div>
<div class="row ">
<hr/>
</div>
<div class="row half-margin">
add the following to your ~/.ssh/known_hosts file (optional)
</div>
<div class="row">
<pre class="code wrap break-all smalltext">{% for key in vm['ssh_host_keys'] %}
{{ vm['ipv4'] }} {{ key.content }}{% endfor %}
</pre>
</div>
<div class="row">
<span>(What's this? see <a href="/about-ssh">Understanding the Secure Shell Protocol (SSH)</a>)</span>
</div>
{% endif %}
{% endblock %}

View File

@ -8,7 +8,9 @@
{% block subcontent %}
<p>
<ul>
<li>2020-10-29: Add OpenBSD 6.8 and Alpine 3.12 support, remove previous versions.</li>
<li>2021-01-19: Add Alpine Linux 3.13 support</li>
<li>2021-01-14: Add Guix System 1.2.0 support, thanks to jgart, ryanprior, and raghavgururajan</li>
<li>2020-10-29: Add OpenBSD 6.8 and Alpine 3.12 support, remove previous versions</li>
<li>2020-10-23: Automate VM build system (backend)</li>
<li>2020-10-22: Re-worked FAQ, added more supporting docs</li>
<li>2020-05-16: Beta version of new Capsul web application</li>

193
capsulflask/virt_model.py Normal file
View File

@ -0,0 +1,193 @@
import subprocess
import re
import sys
import json
from flask import current_app
from time import sleep
from os.path import join
from subprocess import run
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message, VirtualMachine
def validate_capsul_id(id):
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
pass
def get(self, id: str, get_ssh_host_keys: bool) -> VirtualMachine:
pass
def list_ids(self) -> list:
pass
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list):
pass
def destroy(self, email: str, id: str):
pass
class MockVirtualization(VirtualizationInterface):
def capacity_avaliable(self, additional_ram_bytes):
return True
def get(self, id, get_ssh_host_keys):
validate_capsul_id(id)
if get_ssh_host_keys:
ssh_host_keys = json.loads("""[
{"key_type":"ED25519", "content":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN8cna0zeKSKl/r8whdn/KmDWhdzuWRVV0GaKIM+eshh", "sha256":"V4X2apAF6btGAfS45gmpldknoDX0ipJ5c6DLfZR2ttQ"},
{"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"},
{"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"}
]""")
return VirtualMachine(id, ipv4="1.1.1.1", ssh_host_keys=ssh_host_keys)
return VirtualMachine(id, ipv4="1.1.1.1")
def list_ids(self) -> list:
return get_model().all_non_deleted_vm_ids()
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
current_app.logger.info(f"mock create: {id} for {email}")
sleep(1)
def destroy(self, email: str, id: str):
current_app.logger.info(f"mock destroy: {id} for {email}")
class ShellScriptVirtualization(VirtualizationInterface):
def validate_completed_process(self, completedProcess, email=None):
emailPart = ""
if email != None:
emailPart = f"for {email}"
if completedProcess.returncode != 0:
raise RuntimeError(f"""{" ".join(completedProcess.args)} failed {emailPart} with exit code {completedProcess.returncode}
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")
def capacity_avaliable(self, additional_ram_bytes):
my_args=[join(current_app.root_path, 'shell_scripts/capacity-avaliable.sh'), str(additional_ram_bytes)]
completedProcess = run(my_args, capture_output=True)
if completedProcess.returncode != 0:
current_app.logger.error(f"""
capacity-avaliable.sh exited {completedProcess.returncode} with
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")
return False
lines = completedProcess.stdout.splitlines()
output = lines[len(lines)-1]
if not output == b"yes":
current_app.logger.error(f"capacity-avaliable.sh exited 0 and returned {output} but did not return \"yes\" ")
return False
return True
def get(self, id, get_ssh_host_keys):
validate_capsul_id(id)
completedProcess = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True)
self.validate_completed_process(completedProcess)
ipaddr_lines = completedProcess.stdout.splitlines()
if len(ipaddr_lines) == 0:
return None
ipaddr = ipaddr_lines[0].decode("utf-8")
if not re.match(r"^([0-9]{1,3}\.){3}[0-9]{1,3}$", ipaddr):
return None
if get_ssh_host_keys:
try:
completedProcess2 = run([join(current_app.root_path, 'shell_scripts/ssh-keyscan.sh'), ipaddr], capture_output=True)
self.validate_completed_process(completedProcess2)
ssh_host_keys = json.loads(completedProcess2.stdout.decode("utf-8"))
return VirtualMachine(id, ipv4=ipaddr, ssh_host_keys=ssh_host_keys)
except:
current_app.logger.warning(f"""
failed to ssh-keyscan {id} at {ipaddr}:
{my_exec_info_message(sys.exc_info())}"""
)
return VirtualMachine(id, ipv4=ipaddr)
def list_ids(self) -> list:
completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True)
self.validate_completed_process(completedProcess)
return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() ))
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
if not re.match(r"^[a-zA-Z0-9/_.-]+$", template_image_file_name):
raise ValueError(f"template_image_file_name \"{template_image_file_name}\" must match \"^[a-zA-Z0-9/_.-]+$\"")
for ssh_public_key in ssh_authorized_keys:
if not re.match(r"^(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$", ssh_public_key):
raise ValueError(f"ssh_public_key \"{ssh_public_key}\" must match \"^(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$\"")
if vcpus < 1 or vcpus > 8:
raise ValueError(f"vcpus \"{vcpus}\" must match 1 <= vcpus <= 8")
if memory_mb < 512 or memory_mb > 16384:
raise ValueError(f"memory_mb \"{memory_mb}\" must match 512 <= memory_mb <= 16384")
ssh_keys_string = "\n".join(ssh_authorized_keys)
completedProcess = run([
join(current_app.root_path, 'shell_scripts/create.sh'),
id,
template_image_file_name,
str(vcpus),
str(memory_mb),
ssh_keys_string
], capture_output=True)
self.validate_completed_process(completedProcess, email)
lines = completedProcess.stdout.splitlines()
status = lines[len(lines)-1].decode("utf-8")
vmSettings = f"""
id={id}
template_image_file_name={template_image_file_name}
vcpus={str(vcpus)}
memory={str(memory_mb)}
ssh_authorized_keys={ssh_keys_string}
"""
if not status == "success":
raise ValueError(f"""failed to create vm for {email} with:
{vmSettings}
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")
def destroy(self, email: str, id: str):
validate_capsul_id(id)
completedProcess = run([join(current_app.root_path, 'shell_scripts/destroy.sh'), id], capture_output=True)
self.validate_completed_process(completedProcess, email)
lines = completedProcess.stdout.splitlines()
status = lines[len(lines)-1].decode("utf-8")
if not status == "success":
raise ValueError(f"""failed to destroy vm "{id}" for {email}:
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")