From 81db5818df876886e2a3ba0fd755f0988fd31507 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 15 Oct 2024 22:25:45 +0300 Subject: [PATCH] Improved metrics handling - remove timestamps --- app.py | 77 ++++++++++++++++++++++++++++---------------------- pyproject.toml | 1 - uv.lock | 11 -------- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/app.py b/app.py index 43e02bc..f4af4de 100644 --- a/app.py +++ b/app.py @@ -1,71 +1,82 @@ import contextlib import os -from datetime import datetime +from collections.abc import Awaitable, Callable +from typing import Any import aiofiles import orjson from environs import Env -env = Env() +env: Env = Env() env.read_env() unit_log_file: str = env.str("UNIT_LOG_FILE") -def escape_label_value(value): +def escape_label_value(value: str) -> str: return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") -async def read_and_process_logs(): - metric_lines = [] +async def read_and_process_logs() -> dict[tuple[str, str], dict[str, float]]: + metrics_data: dict[tuple[str, str], dict[str, float]] = {} if os.path.exists(unit_log_file): async with aiofiles.open(unit_log_file, mode="r+") as f: - lines = await f.readlines() + lines: list[str] = await f.readlines() await f.truncate(0) for line in lines: with contextlib.suppress(Exception): - log_data = orjson.loads(line) - request_time = float(log_data["request_time"]) - url = log_data["url"] - http_method = log_data["http_method"] - datetime_str = log_data["datetime"] + log_data: dict[str, str] = orjson.loads(line) + request_time: float = float(log_data["request_time"]) + url: str = log_data["url"] + http_method: str = log_data["http_method"] + key: tuple[str, str] = (url, http_method) - # Parse datetime_str to Unix timestamp in milliseconds - dt = datetime.strptime(datetime_str, "%d/%b/%Y:%H:%M:%S %z") - timestamp_ms = int(dt.timestamp() * 1000) + if key in metrics_data: + metrics_data[key]["count"] += 1 + metrics_data[key]["sum"] += request_time + else: + metrics_data[key] = {"count": 1, "sum": request_time} - labels = ( - f'url="{escape_label_value(url)}",' - f'http_method="{escape_label_value(http_method)}"' - ) - - # Generate the metric line - metric_line = f"request_duration_seconds{{{labels}}} {request_time} {timestamp_ms}" - metric_lines.append(metric_line) - return metric_lines + return metrics_data -async def generate_metrics_text(): - lines = [ - "# HELP request_duration_seconds Request duration in seconds", - "# TYPE request_duration_seconds gauge", +async def generate_metrics_text() -> str: + lines: list[str] = [ + "# HELP request_duration_seconds_total Total request duration in seconds", + "# TYPE request_duration_seconds_total counter", + "# HELP request_duration_seconds_count Total number of requests", + "# TYPE request_duration_seconds_count counter", ] - metric_lines = await read_and_process_logs() + metrics_data: dict[tuple[str, str], dict[str, float]] = await read_and_process_logs() - lines.extend(metric_lines) + for key, data in metrics_data.items(): + url, http_method = key + count: int = int(data["count"]) + sum_values: float = data["sum"] + + labels: str = ( + f'url="{escape_label_value(url)}",' f'http_method="{escape_label_value(http_method)}"' + ) + + lines.append(f"request_duration_seconds_count{{{labels}}} {count}") + lines.append(f"request_duration_seconds_total{{{labels}}} {sum_values}") return "\n".join(lines) + "\n" -async def app(scope, receive, send): +async def app( + scope: dict[str, Any], + _: Callable[[], Awaitable[dict[str, Any]]], + send: Callable[[dict[str, Any]], Awaitable[None]], +) -> None: if scope["type"] == "http": if scope["path"] == "/metrics": - headers = [(b"content-type", b"text/plain; charset=utf-8")] - metrics_text = await generate_metrics_text() - body = metrics_text.encode("utf-8") + headers: list[tuple[bytes, bytes]] = [(b"content-type", b"text/plain; charset=utf-8")] + metrics_text: str = await generate_metrics_text() + body: bytes = metrics_text.encode("utf-8") await send({"type": "http.response.start", "status": 200, "headers": headers}) await send({"type": "http.response.body", "body": body}) diff --git a/pyproject.toml b/pyproject.toml index f9c8586..07b0e19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ dependencies = [ "environs>=11.0.0", "granian>=1.6.0", "orjson>=3.10.7", - "prometheus-client>=0.21.0", "uvloop>=0.20.0", ] diff --git a/uv.lock b/uv.lock index 7b56d08..6f92a38 100644 --- a/uv.lock +++ b/uv.lock @@ -91,7 +91,6 @@ dependencies = [ { name = "environs" }, { name = "granian" }, { name = "orjson" }, - { name = "prometheus-client" }, { name = "uvloop" }, ] @@ -101,7 +100,6 @@ requires-dist = [ { name = "environs", specifier = ">=11.0.0" }, { name = "granian", specifier = ">=1.6.0" }, { name = "orjson", specifier = ">=3.10.7" }, - { name = "prometheus-client", specifier = ">=0.21.0" }, { name = "uvloop", specifier = ">=0.20.0" }, ] @@ -138,15 +136,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124", size = 53985 }, ] -[[package]] -name = "prometheus-client" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/54/a369868ed7a7f1ea5163030f4fc07d85d22d7a1d270560dab675188fb612/prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e", size = 78634 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166", size = 54686 }, -] - [[package]] name = "python-dotenv" version = "1.0.1"