capsul-flask/capsulflask/metrics.py

261 lines
7.6 KiB
Python
Raw Permalink Normal View History

2020-05-13 05:28:53 +00:00
import matplotlib.ticker as ticker
import matplotlib.pyplot as pyplot
import matplotlib.dates as mdates
2020-05-13 05:34:38 +00:00
from functools import reduce
2020-05-13 05:28:53 +00:00
import requests
2021-01-04 21:18:23 +00:00
import json
2020-05-13 05:28:53 +00:00
from datetime import datetime
2021-01-04 21:18:23 +00:00
from threading import Lock
2020-05-13 05:28:53 +00:00
from io import BytesIO
from flask import Blueprint
from flask import current_app
from flask import session
from flask import render_template, make_response
from werkzeug.exceptions import abort
from capsulflask.db import get_model
from capsulflask.auth import account_required
from capsulflask.shared import *
2020-05-13 05:28:53 +00:00
2021-01-04 21:18:23 +00:00
mutex = Lock()
2020-05-13 05:28:53 +00:00
bp = Blueprint("metrics", __name__, url_prefix="/metrics")
durations = dict(
_5m=[60*5, 15],
_1h=[60*60, 60],
_1d=[60*60*24, 60*20],
_30d=[60*60*24*30, 60*300]
)
sizes = dict(
s=[0.77, 0.23, 4],
m=[1, 1, 2],
l=[6, 4, 1],
)
green = (121/255, 240/255, 50/255)
blue = (70/255, 150/255, 255/255)
red = (255/255, 50/255, 8/255)
gray = (128/255, 128/255, 128/255)
2020-05-13 05:28:53 +00:00
@bp.route("/html/<string:metric>/<string:capsulid>/<string:duration>")
@account_required
def display_metric(metric, capsulid, duration):
vm = get_model().get_vm_detail(session["account"], capsulid)
return render_template(
"display-metric.html",
vm=vm,
duration=duration,
durations=list(map(lambda x: x.strip("_"), durations.keys())),
metric=metric
)
@bp.route("/<string:metric>/<string:capsulid>/<string:duration>/<string:size>")
@account_required
def metric_png(metric, capsulid, duration, size):
result = get_plot_bytes(metric, capsulid, duration, size)
if result[0] != 200:
abort(result[0])
response = make_response(result[1])
response.headers.set('Content-Type', 'image/png')
return response
def get_plot_bytes(metric, capsulid, duration, size):
duration = f"_{duration}"
if duration not in durations:
return (404, None)
if size not in sizes:
return (404, None)
vm = get_model().get_vm_detail(session["account"], capsulid)
if not vm:
return (404, None)
now_unix = int(datetime.strftime(datetime.now(), "%s"))
duration_seconds = durations[duration][0]
interval_seconds = durations[duration][1] * sizes[size][2]
if interval_seconds < 30:
interval_seconds = 30
2020-05-13 05:49:08 +00:00
# Prometheus queries to pull metrics for VMs
2020-05-13 05:28:53 +00:00
metric_queries = dict(
cpu=f"irate(libvirtd_domain_info_cpu_time_seconds_total{{domain='{capsulid}'}}[30s])",
memory=f"libvirtd_domain_info_maximum_memory_bytes{{domain='{capsulid}'}}-libvirtd_domain_info_memory_unused_bytes{{domain='{capsulid}'}}",
2020-05-13 05:28:53 +00:00
network_in=f"rate(libvirtd_domain_interface_stats_receive_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
network_out=f"rate(libvirtd_domain_interface_stats_transmit_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
disk=f"rate(libvirtd_domain_block_stats_read_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])%2Brate(libvirtd_domain_block_stats_write_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
)
2020-05-13 05:49:08 +00:00
# These represent the top of the graph for graphs that are designed to be viewed at a glance.
# they are also used to colorize the graph at any size.
2020-05-13 05:28:53 +00:00
scales = dict(
cpu=vm["vcpus"],
memory=vm["memory_mb"]*1024*1024,
network_in=1024*1024*2,
network_out=1024*200,
2020-05-13 19:19:10 +00:00
disk=1024*1024*8,
2020-05-13 05:28:53 +00:00
)
if metric not in metric_queries:
return (404, None)
range_and_interval = f"start={now_unix-duration_seconds}&end={now_unix}&step={interval_seconds}"
prometheus_range_url = f"{current_app.config['PROMETHEUS_URL']}/api/v1/query_range"
#print(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")
prometheus_response = requests.get(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")
if prometheus_response.status_code >= 300:
return (502, None)
series = prometheus_response.json()["data"]["result"]
2021-01-04 21:18:23 +00:00
if len(series) == 0:
now_timestamp = datetime.timestamp(datetime.now())
series = [
dict(
values=[[now_timestamp - interval_seconds, float(0)],[now_timestamp, float(0)]]
)
]
2020-05-13 05:28:53 +00:00
time_series_data = list(map(
lambda x: (datetime.fromtimestamp(x[0]), float(x[1])),
series[0]["values"]
2020-05-13 05:28:53 +00:00
))
2021-01-04 21:18:23 +00:00
mutex.acquire()
try:
plot_bytes = draw_plot_png_bytes(time_series_data, scale=scales[metric], size_x=sizes[size][0], size_y=sizes[size][1])
finally:
mutex.release()
2020-05-13 05:28:53 +00:00
return (200, plot_bytes)
2020-05-13 05:49:08 +00:00
def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
2020-05-13 05:28:53 +00:00
#mylog_info(current_app, json.dumps(data, indent=4, default=str))
2021-01-04 21:18:23 +00:00
2020-05-13 05:28:53 +00:00
pyplot.style.use("seaborn-dark")
fig, my_plot = pyplot.subplots(figsize=(size_x, size_y))
# x=range(1, 15)
# y=[1,4,6,8,4,5,3,2,4,1,5,6,8,7]
divide_by = 1
unit = ""
if scale > 1024 and scale < 1024*1024*1024:
divide_by = 1024*1024
unit = "MB"
if scale > 1024*1024*1024:
divide_by = 1024*1024*1024
unit = "GB"
scale /= divide_by
if scale > 10:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{}{}".format(int(x), unit)) )
elif scale > 1:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.1f}{}".format(x, unit)) )
else:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.2f}{}".format(x, unit)) )
x=list(map(lambda x: x[0], data))
y=list(map(lambda x: x[1]/divide_by, data))
minutes = float((x[len(x)-1] - x[0]).total_seconds())/float(60)
hours = minutes/float(60)
days = hours/float(24)
2020-05-13 05:49:08 +00:00
week_locator = mdates.WeekdayLocator()
2020-05-13 05:28:53 +00:00
minute_locator = mdates.MinuteLocator()
ten_minute_locator = mdates.MinuteLocator(interval=10)
hour_locator = mdates.HourLocator(interval=6)
hour_minute_formatter = mdates.DateFormatter('%H:%M')
day_formatter = mdates.DateFormatter('%b %d')
if minutes < 10:
my_plot.xaxis.set_major_locator(minute_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
elif hours < 2:
my_plot.xaxis.set_major_locator(ten_minute_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
elif days < 2:
my_plot.xaxis.set_major_locator(hour_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
else:
2020-05-13 05:49:08 +00:00
my_plot.xaxis.set_major_locator(week_locator)
2020-05-13 05:28:53 +00:00
my_plot.xaxis.set_major_formatter(day_formatter)
2020-05-13 05:34:38 +00:00
max_value = reduce(lambda a, b: a if a > b else b, y, scale)
2020-05-13 18:56:43 +00:00
if len(data) > 2:
average=(sum(y)/len(y))/scale
average=average*1.25+0.1
2020-05-13 05:28:53 +00:00
bg_color=color_gradient(average)
2020-05-13 05:28:53 +00:00
average -= 0.1
2020-05-13 05:28:53 +00:00
fill_color=color_gradient(average)
highlight_color=lerp_rgb_tuples(fill_color, (1,1,1), 0.5)
else:
bg_color = fill_color = highlight_color = gray
2020-05-13 05:28:53 +00:00
2020-05-13 05:34:38 +00:00
my_plot.fill_between( x, max_value, color=bg_color, alpha=0.13)
2020-05-13 05:28:53 +00:00
my_plot.fill_between( x, y, color=highlight_color, alpha=0.3)
my_plot.plot(x, y, 'r-', color=highlight_color)
if size_y < 4:
2020-05-13 18:56:43 +00:00
my_plot.set_yticks([0, scale])
2020-05-13 05:28:53 +00:00
my_plot.set_ylim(0, scale)
my_plot.xaxis.label.set_color(highlight_color)
my_plot.tick_params(axis='x', colors=highlight_color)
my_plot.yaxis.label.set_color(highlight_color)
my_plot.tick_params(axis='y', colors=highlight_color)
if size_x < 4:
my_plot.set_xticklabels([])
if size_y < 1:
my_plot.set_yticklabels([])
image_binary = BytesIO()
fig.savefig(image_binary, transparent=True, bbox_inches="tight", pad_inches=0.05)
2020-05-13 19:19:10 +00:00
pyplot.close('all')
2020-05-13 05:28:53 +00:00
return image_binary.getvalue()
def lerp_rgb_tuples(a, b, lerp):
if lerp < 0:
lerp = 0
if lerp > 1:
lerp = 1
return (
a[0]*(1.0-lerp)+b[0]*lerp,
a[1]*(1.0-lerp)+b[1]*lerp,
a[2]*(1.0-lerp)+b[2]*lerp
)
def color_gradient(value):
if value < 0:
value = 0
if value > 1:
value = 1
if value < 0.5:
return lerp_rgb_tuples(green, blue, value*2)
else:
return lerp_rgb_tuples(blue, red, (value-0.5)*2)