capsul-flask/capsulflask/metrics.py

261 lines
7.6 KiB
Python

import matplotlib.ticker as ticker
import matplotlib.pyplot as pyplot
import matplotlib.dates as mdates
from functools import reduce
import requests
import json
from datetime import datetime
from threading import Lock
from io import BytesIO
from flask import Blueprint
from flask import current_app
from flask import session
from flask import render_template, make_response
from werkzeug.exceptions import abort
from capsulflask.db import get_model
from capsulflask.auth import account_required
from capsulflask.shared import *
mutex = Lock()
bp = Blueprint("metrics", __name__, url_prefix="/metrics")
durations = dict(
_5m=[60*5, 15],
_1h=[60*60, 60],
_1d=[60*60*24, 60*20],
_30d=[60*60*24*30, 60*300]
)
sizes = dict(
s=[0.77, 0.23, 4],
m=[1, 1, 2],
l=[6, 4, 1],
)
green = (121/255, 240/255, 50/255)
blue = (70/255, 150/255, 255/255)
red = (255/255, 50/255, 8/255)
gray = (128/255, 128/255, 128/255)
@bp.route("/html/<string:metric>/<string:capsulid>/<string:duration>")
@account_required
def display_metric(metric, capsulid, duration):
vm = get_model().get_vm_detail(session["account"], capsulid)
return render_template(
"display-metric.html",
vm=vm,
duration=duration,
durations=list(map(lambda x: x.strip("_"), durations.keys())),
metric=metric
)
@bp.route("/<string:metric>/<string:capsulid>/<string:duration>/<string:size>")
@account_required
def metric_png(metric, capsulid, duration, size):
result = get_plot_bytes(metric, capsulid, duration, size)
if result[0] != 200:
abort(result[0])
response = make_response(result[1])
response.headers.set('Content-Type', 'image/png')
return response
def get_plot_bytes(metric, capsulid, duration, size):
duration = f"_{duration}"
if duration not in durations:
return (404, None)
if size not in sizes:
return (404, None)
vm = get_model().get_vm_detail(session["account"], capsulid)
if not vm:
return (404, None)
now_unix = int(datetime.strftime(datetime.now(), "%s"))
duration_seconds = durations[duration][0]
interval_seconds = durations[duration][1] * sizes[size][2]
if interval_seconds < 30:
interval_seconds = 30
# Prometheus queries to pull metrics for VMs
metric_queries = dict(
cpu=f"irate(libvirtd_domain_info_cpu_time_seconds_total{{domain='{capsulid}'}}[30s])",
memory=f"libvirtd_domain_info_maximum_memory_bytes{{domain='{capsulid}'}}-libvirtd_domain_info_memory_unused_bytes{{domain='{capsulid}'}}",
network_in=f"rate(libvirtd_domain_interface_stats_receive_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
network_out=f"rate(libvirtd_domain_interface_stats_transmit_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
disk=f"rate(libvirtd_domain_block_stats_read_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])%2Brate(libvirtd_domain_block_stats_write_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
)
# These represent the top of the graph for graphs that are designed to be viewed at a glance.
# they are also used to colorize the graph at any size.
scales = dict(
cpu=vm["vcpus"],
memory=vm["memory_mb"]*1024*1024,
network_in=1024*1024*2,
network_out=1024*200,
disk=1024*1024*8,
)
if metric not in metric_queries:
return (404, None)
range_and_interval = f"start={now_unix-duration_seconds}&end={now_unix}&step={interval_seconds}"
prometheus_range_url = f"{current_app.config['PROMETHEUS_URL']}/api/v1/query_range"
#print(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")
prometheus_response = requests.get(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")
if prometheus_response.status_code >= 300:
return (502, None)
series = prometheus_response.json()["data"]["result"]
if len(series) == 0:
now_timestamp = datetime.timestamp(datetime.now())
series = [
dict(
values=[[now_timestamp - interval_seconds, float(0)],[now_timestamp, float(0)]]
)
]
time_series_data = list(map(
lambda x: (datetime.fromtimestamp(x[0]), float(x[1])),
series[0]["values"]
))
mutex.acquire()
try:
plot_bytes = draw_plot_png_bytes(time_series_data, scale=scales[metric], size_x=sizes[size][0], size_y=sizes[size][1])
finally:
mutex.release()
return (200, plot_bytes)
def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
#mylog_info(current_app, json.dumps(data, indent=4, default=str))
pyplot.style.use("seaborn-dark")
fig, my_plot = pyplot.subplots(figsize=(size_x, size_y))
# x=range(1, 15)
# y=[1,4,6,8,4,5,3,2,4,1,5,6,8,7]
divide_by = 1
unit = ""
if scale > 1024 and scale < 1024*1024*1024:
divide_by = 1024*1024
unit = "MB"
if scale > 1024*1024*1024:
divide_by = 1024*1024*1024
unit = "GB"
scale /= divide_by
if scale > 10:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{}{}".format(int(x), unit)) )
elif scale > 1:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.1f}{}".format(x, unit)) )
else:
my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.2f}{}".format(x, unit)) )
x=list(map(lambda x: x[0], data))
y=list(map(lambda x: x[1]/divide_by, data))
minutes = float((x[len(x)-1] - x[0]).total_seconds())/float(60)
hours = minutes/float(60)
days = hours/float(24)
week_locator = mdates.WeekdayLocator()
minute_locator = mdates.MinuteLocator()
ten_minute_locator = mdates.MinuteLocator(interval=10)
hour_locator = mdates.HourLocator(interval=6)
hour_minute_formatter = mdates.DateFormatter('%H:%M')
day_formatter = mdates.DateFormatter('%b %d')
if minutes < 10:
my_plot.xaxis.set_major_locator(minute_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
elif hours < 2:
my_plot.xaxis.set_major_locator(ten_minute_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
elif days < 2:
my_plot.xaxis.set_major_locator(hour_locator)
my_plot.xaxis.set_major_formatter(hour_minute_formatter)
else:
my_plot.xaxis.set_major_locator(week_locator)
my_plot.xaxis.set_major_formatter(day_formatter)
max_value = reduce(lambda a, b: a if a > b else b, y, scale)
if len(data) > 2:
average=(sum(y)/len(y))/scale
average=average*1.25+0.1
bg_color=color_gradient(average)
average -= 0.1
fill_color=color_gradient(average)
highlight_color=lerp_rgb_tuples(fill_color, (1,1,1), 0.5)
else:
bg_color = fill_color = highlight_color = gray
my_plot.fill_between( x, max_value, color=bg_color, alpha=0.13)
my_plot.fill_between( x, y, color=highlight_color, alpha=0.3)
my_plot.plot(x, y, 'r-', color=highlight_color)
if size_y < 4:
my_plot.set_yticks([0, scale])
my_plot.set_ylim(0, scale)
my_plot.xaxis.label.set_color(highlight_color)
my_plot.tick_params(axis='x', colors=highlight_color)
my_plot.yaxis.label.set_color(highlight_color)
my_plot.tick_params(axis='y', colors=highlight_color)
if size_x < 4:
my_plot.set_xticklabels([])
if size_y < 1:
my_plot.set_yticklabels([])
image_binary = BytesIO()
fig.savefig(image_binary, transparent=True, bbox_inches="tight", pad_inches=0.05)
pyplot.close('all')
return image_binary.getvalue()
def lerp_rgb_tuples(a, b, lerp):
if lerp < 0:
lerp = 0
if lerp > 1:
lerp = 1
return (
a[0]*(1.0-lerp)+b[0]*lerp,
a[1]*(1.0-lerp)+b[1]*lerp,
a[2]*(1.0-lerp)+b[2]*lerp
)
def color_gradient(value):
if value < 0:
value = 0
if value > 1:
value = 1
if value < 0.5:
return lerp_rgb_tuples(green, blue, value*2)
else:
return lerp_rgb_tuples(blue, red, (value-0.5)*2)