From d6c048fb5b54e14728f3168a64023c8f365f301e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Wed, 18 Oct 2023 16:31:47 +0200 Subject: [PATCH] Speed-up SQL queriying by using DISTINCT statement --- .gitignore | 3 ++- README.md | 1 + argos/server/main.py | 4 +--- argos/server/routes/api.py | 4 ++-- argos/server/routes/views.py | 27 +++++++++++++++++++++------ argos/server/static/styles.css | 1 + argos/server/templates/domain.html | 6 +++--- argos/server/templates/index.html | 5 +++-- argos/server/templates/results.html | 2 +- 9 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 9ec7806..72a0c53 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__ *.egg-info .vscode -venv \ No newline at end of file +venv +.env \ No newline at end of file diff --git a/README.md b/README.md index fd67250..ca02ac3 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Todo: - [x] Use Postgresql as a database - [x] Expose a simple read-only website. +- [ ] Agents should wait and retry on timeout - [ ] Last seen agents - [ ] Use background tasks for alerting - [ ] Add a command to generate new authentication tokens diff --git a/argos/server/main.py b/argos/server/main.py index 1ed634d..b9e78b2 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -71,9 +71,7 @@ def setup_database(app, settings): logger.debug(f"Using database URL {settings.database_url}") if settings.database_url.startswith("sqlite:////tmp"): logger.warning("Using sqlite in /tmp is not recommended for production") - engine = create_engine( - settings.database_url, - ) + engine = create_engine(settings.database_url) app.state.SessionLocal = sessionmaker( autocommit=False, autoflush=False, bind=engine ) diff --git a/argos/server/routes/api.py b/argos/server/routes/api.py index 8964c4f..c779e9a 100644 --- a/argos/server/routes/api.py +++ b/argos/server/routes/api.py @@ -55,7 +55,7 @@ async def create_results( return {"result_ids": [r.id for r in db_results]} -@route.get("/stats", dependencies=[Depends(verify_token)]) +@route.get("/stats") async def get_stats(db: Session = Depends(get_db)): return { "upcoming_tasks_count": await queries.count_tasks(db, selected=False), @@ -64,7 +64,7 @@ async def get_stats(db: Session = Depends(get_db)): } -@route.get("/severities", dependencies=[Depends(verify_token)]) +@route.get("/severities") async def get_severity_counts(db: Session = Depends(get_db)): """Returns the number of results per severity""" counts = await queries.get_severity_counts(db) diff --git a/argos/server/routes/views.py b/argos/server/routes/views.py index 0dbcc91..d6cbd4b 100644 --- a/argos/server/routes/views.py +++ b/argos/server/routes/views.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse from fastapi import APIRouter, Depends, Request from fastapi.templating import Jinja2Templates +from sqlalchemy import desc from sqlalchemy.orm import Session from argos.schemas import Config @@ -18,24 +19,38 @@ templates = Jinja2Templates(directory="argos/server/templates") async def read_tasks(request: Request, db: Session = Depends(get_db)): tasks = db.query(Task).order_by(Task.domain).all() + results = ( + db.query(Task, Result) + .join(Result) + .distinct(Task.id) + .order_by(Task.id, desc(Result.submitted_at)) + .all() + ) + domains_severities = defaultdict(list) domains_last_checks = defaultdict(list) - for task in tasks: - severity = task.severity or "to-process" + for task, result in results: + severity = result.severity or "to-process" domain = urlparse(task.url).netloc domains_severities[domain].append(severity) - domains_last_checks[domain].append(task.last_result.submitted_at) + domains_last_checks[domain].append(result.submitted_at) def _max_severity(severities): - severity_level = {"ok": 1, "warning": 2, "critical": 3, "to-process": "4"} + severity_level = {"ok": 1, "warning": 2, "critical": 3, "to-process": 4} return max(severities, key=severity_level.get) - domains = {key: _max_severity(value) for key, value in domains_severities.items()} + domains = [(key, _max_severity(value)) for key, value in domains_severities.items()] last_checks = {key: max(value) for key, value in domains_last_checks.items()} + domains.sort(key=lambda x: x[1]) return templates.TemplateResponse( "index.html", - {"request": request, "domains": domains, "last_checks": last_checks}, + { + "request": request, + "domains": domains, + "last_checks": last_checks, + "total_task_count": len(tasks), + }, ) diff --git a/argos/server/static/styles.css b/argos/server/static/styles.css index 2fe50e0..e1333c1 100644 --- a/argos/server/static/styles.css +++ b/argos/server/static/styles.css @@ -2,5 +2,6 @@ code { white-space: pre-wrap; + } \ No newline at end of file diff --git a/argos/server/templates/domain.html b/argos/server/templates/domain.html index 2143e3e..17d36e8 100644 --- a/argos/server/templates/domain.html +++ b/argos/server/templates/domain.html @@ -2,7 +2,7 @@ {% block title %}

{{domain}}

{% endblock %} {% block content %}
- +
@@ -15,11 +15,11 @@ {% for task in tasks %} - + - + {% endfor %} diff --git a/argos/server/templates/index.html b/argos/server/templates/index.html index 23824b3..d6b8ac3 100644 --- a/argos/server/templates/index.html +++ b/argos/server/templates/index.html @@ -1,8 +1,9 @@ {% extends "base.html" %} {% block content %}
+

{{domains | length}} domains, {{ total_task_count }} tasks

-
URL
{{ task.url }} {{ task.check }} {{ task.expected }}{{ task.status }}{{ task.status }} view all
+
@@ -11,7 +12,7 @@ - {% for domain, status in domains.items() %} + {% for (domain, status) in domains %}
Domain
diff --git a/argos/server/templates/results.html b/argos/server/templates/results.html index c9f8774..79c638b 100644 --- a/argos/server/templates/results.html +++ b/argos/server/templates/results.html @@ -2,7 +2,7 @@ {% block title %}

{{ task }}

{% endblock %} {% block content %} {{ description }} - +
Submitted at