diff --git a/Pipfile b/Pipfile index f729f7b..7d25139 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,8 @@ sqlalchemy = "*" sqlalchemy-utils = "*" uvicorn = "*" pyyaml = "*" +httpx = "*" +click = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index a0fda17..ee24ae4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b95f0093b5a31fb370823a1219b4cde22ee7b0e07d9c7bc331aad379f05ee7da" + "sha256": "65abfc821a32d62f8da703a8df902b964e88c2acf91eaeb8b7cf9c2dd2e6b4dd" }, "pipfile-spec": 6, "requires": { @@ -32,22 +32,31 @@ "markers": "python_version >= '3.7'", "version": "==3.7.1" }, + "certifi": { + "hashes": [ + "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + ], + "markers": "python_version >= '3.6'", + "version": "==2023.7.22" + }, "click": { "hashes": [ "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" ], + "index": "pypi", "markers": "python_version >= '3.7'", "version": "==8.1.7" }, "fastapi": { "hashes": [ - "sha256:345844e6a82062f06a096684196aaf96c1198b25c06b72c1311b882aa2d8a35d", - "sha256:5e5f17e826dbd9e9b5a5145976c5cd90bcaa61f2bf9a69aca423f2bcebe44d83" + "sha256:3270de872f0fe9ec809d4bd3d4d890c6d5cc7b9611d721d6438f9dacc8c4ef2e", + "sha256:75a11f6bfb8fc4d2bec0bd710c2d5f2829659c0e8c0afd5560fdda6ce25ec653" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==0.103.1" + "version": "==0.103.2" }, "h11": { "hashes": [ @@ -57,6 +66,23 @@ "markers": "python_version >= '3.7'", "version": "==0.14.0" }, + "httpcore": { + "hashes": [ + "sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9", + "sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced" + ], + "markers": "python_version >= '3.8'", + "version": "==0.18.0" + }, + "httpx": { + "hashes": [ + "sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100", + "sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==0.25.0" + }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -67,11 +93,11 @@ }, "pydantic": { "hashes": [ - "sha256:2b2240c8d54bb8f84b88e061fac1bdfa1761c2859c367f9d3afe0ec2966deddc", - "sha256:b172505886028e4356868d617d2d1a776d7af1625d1313450fd51bdd19d9d61f" + "sha256:94f336138093a5d7f426aac732dcfe7ab4eb4da243c88f891d65deb4a2556ee7", + "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1" ], "markers": "python_version >= '3.7'", - "version": "==2.4.1" + "version": "==2.4.2" }, "pydantic-core": { "hashes": [ diff --git a/README.md b/README.md index d2f732c..602ce32 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ Features : - [ ] Checks can be distributed on the network thanks to a job queue ; - [x] Multiple paths per websites can be tested ; - [ ] Handles multiple alerting backends (email, sms, gotify) ; -- [ ] Exposes an HTTP API that can be consumed by other systems ; +- [x] Exposes an HTTP API that can be consumed by other systems ; - [ ] Exposes a simple read-only website. +- [ ] Packaging (and argos-client / argos-server commands) Implemented checks : @@ -21,6 +22,76 @@ Implemented checks : - [ ] Returned body matches what you expect ; - [ ] SSL certificate expires in more than X days ; +## How to run ? + +We're using [pipenv](https://pipenv.pypa.io/) to manage the virtual environment and the dependencies. +You can install it with [pipx](https://pypa.github.io/pipx/): + +```bash +pipx install pipenv +``` + +And then, checkout this repository and sync its pipenv + +```bash +pipenv sync +``` + +Once all the dependencies are in place, here is how to run the server: + +```bash +pipenv run uvicorn argos.server:app --reload +``` + +The server will read a `config.yaml` file at startup, and will populate the tasks specified in it. See the configuration section below for more information on how to configure the checks you want to run. + +And here is how to run the client: + +```bash +pipenv run python -m argos.client.cli --server http://localhost:8000 +``` + +NB: `argos-server` and `argos-client` commands will be provided in the future. + +## Configuration + +Here is a simple configuration file: + +```yaml +general: + frequency: 4h # Run checks every 4 hours. + alerts: + error: + - local + warning: + - local + alert: + - local +service: + port: 8888 + # Can be generated using `openssl rand -base64 32`. + secrets: + - "O4kt8Max9/k0EmHaEJ0CGGYbBNFmK8kOZNIoUk3Kjwc" + - "x1T1VZR51pxrv5pQUyzooMG4pMUvHNMhA5y/3cUsYVs=" + +ssl: + thresholds: + critical: "1d" + warning: "10d" + +websites: + - domain: "https://blog.notmyidea.org" + paths: + - path: "/" + checks: + - status-is: 200 + - body-contains: "Alexis" + - ssl-certificate-expiration: "on-check" + - path: "/foo" + checks: + - status-is: 400 +``` + ## Development notes ### On service start. diff --git a/argos/checks/__init__.py b/argos/checks/__init__.py index 620034d..581dee2 100644 --- a/argos/checks/__init__.py +++ b/argos/checks/__init__.py @@ -1,26 +1,82 @@ -class HTTPStatusCheck: +import httpx +from argos.logging import logger + +from argos.schemas import Task +from pydantic import BaseModel, Field +from typing import Type + + +class BaseExpectedValue(BaseModel): + expected: str + + def get_converted(self): + return self.expected + + +class ExpectedIntValue(BaseExpectedValue): + def get_converted(self): + return int(self.expected) + + +class ExpectedStringValue(BaseExpectedValue): + pass + + +class BaseCheck: + config: str + expected_cls : Type[BaseExpectedValue] = None + + def __init__(self, client: httpx.AsyncClient, task: Task): + self.client = client + self.task = task + + @property + def expected(self): + return self.expected_cls(expected=self.task.expected).get_converted() + + +class HTTPStatusCheck(BaseCheck): config = "status-is" + expected_cls = ExpectedIntValue - def check(resp): - return True + async def run(self): + # XXX Get the method from the task + task = self.task + response = await self.client.request(method="get", url=task.url) + logger.error(f"{response.status_code=}, {self.expected=}") + return response.status_code == self.expected -class HTTPBodyContains: +class HTTPBodyContains(BaseCheck): config = "body-contains" + expected_cls = ExpectedStringValue - def check(resp): + async def run(self): return True -class SSLCertificateExpiration: +class SSLCertificateExpiration(BaseCheck): config = "ssl-certificate-expiration" + expected_cls = ExpectedStringValue - def check(resp): + async def run(self): return True AVAILABLE_CHECKS = (HTTPStatusCheck, HTTPBodyContains, SSLCertificateExpiration) +class CheckNotFound(Exception): + pass + + def get_names(checks=AVAILABLE_CHECKS): return [c.config for c in checks] + + +def get_check_by_name(name, checks=AVAILABLE_CHECKS): + checks_dict = {c.config: c for c in checks} + check = checks_dict.get(name) + if not check: + raise CheckNotFound(name) + return check diff --git a/argos/client/__init__.py b/argos/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/argos/client/cli.py b/argos/client/cli.py new file mode 100644 index 0000000..5bb988e --- /dev/null +++ b/argos/client/cli.py @@ -0,0 +1,69 @@ +import httpx +import asyncio +import click +from typing import List + +from argos import logging +from argos.logging import logger +from argos.checks import CheckNotFound, get_check_by_name + +from argos.schemas import Task + + +async def complete_task(client: httpx.AsyncClient, task: dict) -> dict: + task = Task(**task) + check_class = get_check_by_name(task.check) + check = check_class(client, task) + result = await check.run() + logger.error(f"{result=}") + return {"id": task.id, "result": "completed"} + + +async def post_results(client: httpx.AsyncClient, server: str, results: List[dict]): + response = await client.post(f"{server}/results", json={"results": results}) + + if response.status_code == httpx.codes.OK: + logger.info("Successfully posted results") + else: + logger.error(f"Failed to post results: {response.read()}") + + +async def run(server: str, max_tasks: int): + tasks = [] + + async with httpx.AsyncClient() as client: + # Fetch the list of tasks + response = await client.get(f"{server}/tasks") + + if response.status_code == httpx.codes.OK: + # XXX Maybe we want to group the tests by URL ? (to issue one request per URL) + data = response.json() + logger.info(f"Received {len(data)} tasks from the server") + + for task in data: + tasks.append(complete_task(client, task)) + + # Run up to max_tasks concurrent tasks + results = await asyncio.gather(*tasks) + + # Post the results + await post_results(client, server, results) + else: + logger.error(f"Failed to fetch tasks: {response.read()}") + + +@click.command() +@click.option("--server", required=True, help="Server URL") +@click.option("--max-tasks", default=10, help="Maximum number of concurrent tasks") +@click.option( + "--log-level", + default="INFO", + type=click.Choice(logging.LOG_LEVELS, case_sensitive=False), +) +def main(server, max_tasks, log_level): + logging.set_log_level(log_level) + asyncio.run(run(server, max_tasks)) + + +if __name__ == "__main__": + main() diff --git a/argos/logging.py b/argos/logging.py index eea436a..fa9ec2d 100644 --- a/argos/logging.py +++ b/argos/logging.py @@ -1,3 +1,14 @@ import logging +LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + +# XXX We probably want different loggers for client and server. logger = logging.getLogger(__name__) + +# XXX Does not work ? +def set_log_level(log_level): + level = getattr(logging, log_level.upper(), None) + if not isinstance(level, int): + raise ValueError(f"Invalid log level: {log_level}") + logger.setLevel(level=level) + logger.info("Log level set to {}".format(log_level)) diff --git a/argos/schemas/config.py b/argos/schemas/config.py index 9257a42..c38aa36 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -7,7 +7,8 @@ import yaml from pydantic import BaseModel, Field, HttpUrl, validator from datetime import datetime -from argos.checks import get_names as get_check_names +# from argos.checks import get_names as get_check_names +# XXX Find a way to check without having cirular imports # This file contains the pydantic schemas. For the database models, check in argos.model. @@ -21,7 +22,8 @@ class SSL(BaseModel): thresholds: Thresholds -WebsiteCheck = dict[StrEnum("Check", get_check_names()), str | int] +WebsiteCheck = dict[str, str | int] +# StrEnum("Check", get_check_names()) ? class WebsitePath(BaseModel): diff --git a/argos/schemas/models.py b/argos/schemas/models.py index 7e411c4..913b9d2 100644 --- a/argos/schemas/models.py +++ b/argos/schemas/models.py @@ -1,15 +1,16 @@ from pydantic import BaseModel from datetime import datetime +# XXX Refactor using SQLModel to avoid duplication of model data class Task(BaseModel): id : int - max_delta_days: int - status: str - response: dict - last_check: datetime | None - selected_by: str | None + url: str + domain: str + check: str + expected: str selected_at: datetime | None + selected_by : str | None class Config: from_attributes = True diff --git a/argos/server/__init__.py b/argos/server/__init__.py new file mode 100644 index 0000000..6213c67 --- /dev/null +++ b/argos/server/__init__.py @@ -0,0 +1 @@ +from argos.server.api import app \ No newline at end of file diff --git a/argos/main.py b/argos/server/api.py similarity index 76% rename from argos/main.py rename to argos/server/api.py index 148b3ac..e6110a0 100644 --- a/argos/main.py +++ b/argos/server/api.py @@ -1,14 +1,16 @@ from fastapi import Depends, FastAPI, HTTPException from sqlalchemy.orm import Session -from . import queries, models, schemas -from .database import SessionLocal, engine -from .logging import logger +from argos.server import queries, models +from argos import schemas +from argos.server.database import SessionLocal, engine +from argos.logging import logger models.Base.metadata.create_all(bind=engine) app = FastAPI() + # Dependency def get_db(): db = SessionLocal() @@ -17,6 +19,7 @@ def get_db(): finally: db.close() + @app.on_event("startup") async def read_config_and_populate_db(): # XXX Get filename from environment. @@ -29,7 +32,7 @@ async def read_config_and_populate_db(): db.close() -@app.get("/", response_model=list[schemas.Task]) +@app.get("/tasks", response_model=list[schemas.Task]) async def read_tasks(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)): users = queries.list_tasks(db, limit) return users diff --git a/argos/database.py b/argos/server/database.py similarity index 100% rename from argos/database.py rename to argos/server/database.py diff --git a/argos/models.py b/argos/server/models.py similarity index 97% rename from argos/models.py rename to argos/server/models.py index 3640d61..ac1de6b 100644 --- a/argos/models.py +++ b/argos/server/models.py @@ -6,7 +6,7 @@ from sqlalchemy.orm import mapped_column from datetime import datetime -from .schemas import WebsiteCheck +from argos.schemas import WebsiteCheck class Base(DeclarativeBase): diff --git a/argos/queries.py b/argos/server/queries.py similarity index 53% rename from argos/queries.py rename to argos/server/queries.py index 0903cc5..d177e9f 100644 --- a/argos/queries.py +++ b/argos/server/queries.py @@ -1,14 +1,15 @@ from sqlalchemy.orm import Session from sqlalchemy import exists -from . import schemas -from .models import Task -from .logging import logger +from argos import schemas +from argos.logging import logger +from argos.server.models import Task + from urllib.parse import urljoin def list_tasks(db: Session, limit: int = 100): - return db.query(models.Task).limit(limit).all() + return db.query(Task).limit(limit).all() def update_from_config(db: Session, config: schemas.Config): @@ -18,23 +19,24 @@ def update_from_config(db: Session, config: schemas.Config): url = urljoin(domain, str(p.path)) for check in p.checks: for check_key, expected in check.items(): - # Check the db for already existing tasks. - existing_task = db.query(exists().where( - Task.url == url - and Task.check == check_key - and Task.expected == expected - )).scalar() - + # Check the db for already existing tasks. + existing_task = db.query( + exists().where( + Task.url == url + and Task.check == check_key + and Task.expected == expected + ) + ).scalar() + if not existing_task: task = Task( - domain = domain, - url = url, - check = check_key, - expected = expected + domain=domain, url=url, check=check_key, expected=expected ) logger.debug(f"Adding a new task in the db: {task=}") db.add(task) else: - logger.debug(f"Skipping db task creation for {url=}, {check_key=}, {expected=}.") + logger.debug( + f"Skipping db task creation for {url=}, {check_key=}, {expected=}." + ) db.commit()