capsul-flask/capsulflask/metrics.py

import matplotlib.ticker as ticker
import matplotlib.pyplot as pyplot
import matplotlib.dates as mdates
from functools import reduce 
import requests
import json
from datetime import datetime
from threading import Lock
from io import BytesIO
from flask import Blueprint
from flask import current_app
from flask import session
from flask import render_template, make_response
from werkzeug.exceptions import abort

from capsulflask.db import get_model
from capsulflask.auth import account_required

mutex = Lock()
bp = Blueprint("metrics", __name__, url_prefix="/metrics")

durations = dict(
  _5m=[60*5, 15],
  _1h=[60*60, 60],
  _1d=[60*60*24, 60*20],
  _30d=[60*60*24*30, 60*300]
)
sizes = dict(
  s=[0.77, 0.23, 4],
  m=[1, 1, 2],
  l=[6, 4, 1],
)

green = (121/255, 240/255, 50/255)
blue = (70/255, 150/255, 255/255)
red = (255/255, 50/255, 8/255)

gray = (128/255, 128/255, 128/255)

@bp.route("/html/<string:metric>/<string:capsulid>/<string:duration>")
@account_required
def display_metric(metric, capsulid, duration):
  vm = get_model().get_vm_detail(session["account"], capsulid)

  return render_template(
    "display-metric.html", 
    vm=vm, 
    duration=duration, 
    durations=list(map(lambda x: x.strip("_"), durations.keys())),
    metric=metric
  )


@bp.route("/<string:metric>/<string:capsulid>/<string:duration>/<string:size>")
@account_required
def metric_png(metric, capsulid, duration, size):
  result = get_plot_bytes(metric, capsulid, duration, size)

  if result[0] != 200:
    abort(result[0])

  response = make_response(result[1])
  response.headers.set('Content-Type', 'image/png')
  return response


def get_plot_bytes(metric, capsulid, duration, size):

  duration = f"_{duration}"

  if duration not in durations:
    return (404, None)

  if size not in sizes:
    return (404, None)

  vm = get_model().get_vm_detail(session["account"], capsulid)

  if not vm:
    return (404, None)

  now_unix = int(datetime.strftime(datetime.now(), "%s"))
  duration_seconds = durations[duration][0]
  interval_seconds = durations[duration][1] * sizes[size][2]
  if interval_seconds < 30:
    interval_seconds = 30

  # Prometheus queries to pull metrics for VMs
  metric_queries = dict(
    cpu=f"irate(libvirtd_domain_info_cpu_time_seconds_total{{domain='{capsulid}'}}[30s])",
    memory=f"libvirtd_domain_info_maximum_memory_bytes{{domain='{capsulid}'}}-libvirtd_domain_info_memory_unused_bytes{{domain='{capsulid}'}}",
    network_in=f"rate(libvirtd_domain_interface_stats_receive_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
    network_out=f"rate(libvirtd_domain_interface_stats_transmit_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
    disk=f"rate(libvirtd_domain_block_stats_read_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])%2Brate(libvirtd_domain_block_stats_write_bytes_total{{domain='{capsulid}'}}[{interval_seconds}s])",
  )

  # These represent the top of the graph for graphs that are designed to be viewed at a glance.
  # they are also used to colorize the graph at any size.
  scales = dict(
    cpu=vm["vcpus"],
    memory=vm["memory_mb"]*1024*1024,
    network_in=1024*1024*2,
    network_out=1024*200,
    disk=1024*1024*8,
  )

  if metric not in metric_queries:
    return (404, None)

  range_and_interval = f"start={now_unix-duration_seconds}&end={now_unix}&step={interval_seconds}"

  prometheus_range_url = f"{current_app.config['PROMETHEUS_URL']}/api/v1/query_range"

  #print(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")

  prometheus_response = requests.get(f"{prometheus_range_url}?query={metric_queries[metric]}&{range_and_interval}")
  if prometheus_response.status_code >= 300:
    return (502, None)

  series = prometheus_response.json()["data"]["result"]
  
  if len(series) == 0:
    now_timestamp = datetime.timestamp(datetime.now())
    series = [
      dict(
        values=[[now_timestamp - interval_seconds, float(0)],[now_timestamp, float(0)]]
      )
    ]

  time_series_data = list(map(
    lambda x: (datetime.fromtimestamp(x[0]), float(x[1])), 
    series[0]["values"]
  ))

  mutex.acquire()
  try:
    plot_bytes = draw_plot_png_bytes(time_series_data, scale=scales[metric], size_x=sizes[size][0], size_y=sizes[size][1])
  finally:
    mutex.release()

  return (200, plot_bytes)


def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):

  #current_app.logger.info(json.dumps(data, indent=4, default=str))

  pyplot.style.use("seaborn-dark")
  fig, my_plot = pyplot.subplots(figsize=(size_x, size_y)) 

  # x=range(1, 15)
  # y=[1,4,6,8,4,5,3,2,4,1,5,6,8,7]

  divide_by = 1
  unit = ""

  if scale > 1024 and scale < 1024*1024*1024:
    divide_by = 1024*1024
    unit = "MB"
  if scale > 1024*1024*1024:
    divide_by = 1024*1024*1024
    unit = "GB"
  
  scale /= divide_by

  if scale > 10:
    my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{}{}".format(int(x), unit)) )
  elif scale > 1:
    my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.1f}{}".format(x, unit)) )
  else:
    my_plot.get_yaxis().set_major_formatter( ticker.FuncFormatter(lambda x, p: "{:.2f}{}".format(x, unit)) )

  x=list(map(lambda x: x[0], data))
  y=list(map(lambda x: x[1]/divide_by, data))

  minutes = float((x[len(x)-1] - x[0]).total_seconds())/float(60)
  hours = minutes/float(60)
  days = hours/float(24)

  week_locator = mdates.WeekdayLocator()
  minute_locator = mdates.MinuteLocator()
  ten_minute_locator = mdates.MinuteLocator(interval=10)
  hour_locator = mdates.HourLocator(interval=6)
  hour_minute_formatter = mdates.DateFormatter('%H:%M')
  day_formatter = mdates.DateFormatter('%b %d')

  if minutes < 10:
    my_plot.xaxis.set_major_locator(minute_locator)
    my_plot.xaxis.set_major_formatter(hour_minute_formatter)
  elif hours < 2:
    my_plot.xaxis.set_major_locator(ten_minute_locator)
    my_plot.xaxis.set_major_formatter(hour_minute_formatter)
  elif days < 2:
    my_plot.xaxis.set_major_locator(hour_locator)
    my_plot.xaxis.set_major_formatter(hour_minute_formatter)
  else:
    my_plot.xaxis.set_major_locator(week_locator)
    my_plot.xaxis.set_major_formatter(day_formatter)

  max_value = reduce(lambda a, b: a if a > b else b, y, scale)

  if len(data) > 2:
    average=(sum(y)/len(y))/scale
    average=average*1.25+0.1

    bg_color=color_gradient(average)

    average -= 0.1

    fill_color=color_gradient(average)
    highlight_color=lerp_rgb_tuples(fill_color, (1,1,1), 0.5)
  else:
    bg_color = fill_color = highlight_color = gray

  my_plot.fill_between( x, max_value, color=bg_color, alpha=0.13)
  my_plot.fill_between( x, y, color=highlight_color, alpha=0.3)
  my_plot.plot(x, y, 'r-', color=highlight_color)

  if size_y < 4:
    my_plot.set_yticks([0, scale])
    my_plot.set_ylim(0, scale)

  my_plot.xaxis.label.set_color(highlight_color)
  my_plot.tick_params(axis='x', colors=highlight_color)
  my_plot.yaxis.label.set_color(highlight_color)
  my_plot.tick_params(axis='y', colors=highlight_color)

  if size_x < 4:
    my_plot.set_xticklabels([])
  if size_y < 1:
    my_plot.set_yticklabels([])

  image_binary = BytesIO()
  fig.savefig(image_binary, transparent=True, bbox_inches="tight", pad_inches=0.05)

  pyplot.close('all')

  return image_binary.getvalue()


def lerp_rgb_tuples(a, b, lerp):
  if lerp < 0:
    lerp = 0
  if lerp > 1:
    lerp = 1
  return (
    a[0]*(1.0-lerp)+b[0]*lerp,
    a[1]*(1.0-lerp)+b[1]*lerp,
    a[2]*(1.0-lerp)+b[2]*lerp
  )

def color_gradient(value):
  if value < 0:
    value = 0
  if value > 1:
    value = 1
  if value < 0.5:
    return lerp_rgb_tuples(green, blue, value*2)
  else:
    return lerp_rgb_tuples(blue, red, (value-0.5)*2)