From d35be89f4bd0f85208d58468a139cf3a7061dcee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Tue, 10 Oct 2023 11:45:33 +0200 Subject: [PATCH] Support `!include filename` in the yaml files. - run isort on the codebase --- Pipfile | 1 + Pipfile.lock | 11 +++++- argos/agent.py | 11 +++--- argos/checks/__init__.py | 6 ++-- argos/checks/base.py | 4 +-- argos/checks/checks.py | 14 +++----- argos/commands.py | 10 ++++-- argos/schemas/config.py | 29 +++++++++------- argos/schemas/models.py | 5 +-- argos/server/api.py | 8 ++--- argos/server/database.py | 6 ++-- argos/server/models.py | 23 ++++--------- argos/server/queries.py | 10 +++--- config.yaml | 67 +----------------------------------- pyproject.toml | 1 + tests/test_checks_base.py | 1 + tests/test_schemas_config.py | 1 + 17 files changed, 76 insertions(+), 132 deletions(-) diff --git a/Pipfile b/Pipfile index 63e6491..5f37610 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ sqlalchemy = {extras = ["asyncio"] } pyopenssl = "*" ipdb = "*" argos = {extras = ["dev"], file = ".", editable = true} +pyyaml-include = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 4b6305c..a8c98e5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "545ec239a057ec56cb3b4e5d7d6b8922a837d9ce2340e4b2def368c8064acf73" + "sha256": "8cc9237ff86d00019539f36e3df5b20edcbbc60f52d1b0fce2b03e51c089ad39" }, "pipfile-spec": 6, "requires": { @@ -669,6 +669,15 @@ "markers": "python_version >= '3.6'", "version": "==6.0.1" }, + "pyyaml-include": { + "hashes": [ + "sha256:4cb3b4e1baae2ec251808fe1e8aed5d3d20699c541864c8e47ed866ab2f15039", + "sha256:e58525721a2938d29c4046350f8aad86f848660a770c29605e6f2700925fa753" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.3.1" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", diff --git a/argos/agent.py b/argos/agent.py index 2094b82..fba8768 100644 --- a/argos/agent.py +++ b/argos/agent.py @@ -1,10 +1,11 @@ -import httpx import asyncio from typing import List -from argos.logging import logger -from argos.checks import CheckNotFound, get_registered_check -from argos.schemas import Task, AgentResult, SerializableException +import httpx + +from argos.checks import CheckNotFound, get_registered_check +from argos.logging import logger +from argos.schemas import AgentResult, SerializableException, Task async def complete_task(http_client: httpx.AsyncClient, task: dict) -> dict: @@ -58,4 +59,4 @@ async def run_agent(server: str, auth: str, max_tasks: int): # Post the results await post_results(http_client, server, results) else: - logger.error(f"Failed to fetch tasks: {response.read()}") \ No newline at end of file + logger.error(f"Failed to fetch tasks: {response.read()}") diff --git a/argos/checks/__init__.py b/argos/checks/__init__.py index 21c44ff..2f70f43 100644 --- a/argos/checks/__init__.py +++ b/argos/checks/__init__.py @@ -1,2 +1,4 @@ -from argos.checks.checks import HTTPStatus, HTTPBodyContains, SSLCertificateExpiration -from argos.checks.base import get_registered_checks, get_registered_check, CheckNotFound +from argos.checks.base import (CheckNotFound, get_registered_check, + get_registered_checks) +from argos.checks.checks import (HTTPBodyContains, HTTPStatus, + SSLCertificateExpiration) diff --git a/argos/checks/base.py b/argos/checks/base.py index ff9908d..427d6b4 100644 --- a/argos/checks/base.py +++ b/argos/checks/base.py @@ -1,8 +1,8 @@ -from pydantic import BaseModel, Field from dataclasses import dataclass - from typing import Type + import httpx +from pydantic import BaseModel, Field from argos.schemas import Task diff --git a/argos/checks/checks.py b/argos/checks/checks.py index 6ca1da2..d1cc6ae 100644 --- a/argos/checks/checks.py +++ b/argos/checks/checks.py @@ -1,17 +1,13 @@ -from argos.logging import logger -from argos.checks.base import ( - BaseCheck, - Response, - Status, - ExpectedIntValue, - ExpectedStringValue, -) import ssl import time - from datetime import datetime + from OpenSSL import crypto +from argos.checks.base import (BaseCheck, ExpectedIntValue, + ExpectedStringValue, Response, Status) +from argos.logging import logger + class HTTPStatus(BaseCheck): config = "status-is" diff --git a/argos/commands.py b/argos/commands.py index 75c5539..f471425 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -1,15 +1,18 @@ -import click -import subprocess import asyncio +import subprocess + +import click -from argos.agent import run_agent from argos import logging +from argos.agent import run_agent from argos.logging import logger + @click.group() def cli(): pass + @cli.command() @click.option("--server", required=True, help="Server URL") @click.option("--auth", required=True, help="The authentication token") @@ -36,5 +39,6 @@ def server(host, port, reload): command.append("--reload") subprocess.run(command) + if __name__ == "__main__": cli() diff --git a/argos/schemas/config.py b/argos/schemas/config.py index 712cc02..f3ba29c 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -1,14 +1,11 @@ -from pydantic import BaseModel - +import os +from datetime import datetime from enum import StrEnum -from typing import List, Optional, Tuple, Literal - -from typing import Dict, Union, List +from typing import Dict, List, Literal, Optional, Tuple, Union import yaml from pydantic import BaseModel, Field, HttpUrl, validator - -from datetime import datetime +from yamlinclude import YamlIncludeConstructor # XXX Find a way to check without having cirular imports @@ -66,7 +63,8 @@ class WebsitePath(BaseModel): @validator("checks", each_item=True, pre=True) def parse_checks(cls, value): - from argos.checks import get_registered_checks # To avoid circular imports + from argos.checks import \ + get_registered_checks # To avoid circular imports available_names = get_registered_checks().keys() @@ -109,11 +107,16 @@ def validate_config(config: dict): return Config(**config) -def from_yaml(file_name): - parsed = load_yaml(file_name) +def from_yaml(filename): + parsed = load_yaml(filename) return validate_config(parsed) -def load_yaml(file_name): - with open(file_name, "r") as stream: - return yaml.safe_load(stream) +def load_yaml(filename): + base_dir = os.path.dirname(filename) + YamlIncludeConstructor.add_to_loader_class( + loader_class=yaml.FullLoader, base_dir=base_dir + ) + + with open(filename, "r") as stream: + return yaml.load(stream, Loader=yaml.FullLoader) diff --git a/argos/schemas/models.py b/argos/schemas/models.py index a98a1a0..d79b83f 100644 --- a/argos/schemas/models.py +++ b/argos/schemas/models.py @@ -1,7 +1,8 @@ -from pydantic import BaseModel +import traceback from datetime import datetime from typing import Literal -import traceback + +from pydantic import BaseModel # XXX Refactor using SQLModel to avoid duplication of model data diff --git a/argos/server/api.py b/argos/server/api.py index 9171fe0..c3a2507 100644 --- a/argos/server/api.py +++ b/argos/server/api.py @@ -1,8 +1,8 @@ import sys from typing import Annotated, List, Optional -from fastapi import Depends, FastAPI, HTTPException, Request, Header -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from fastapi import Depends, FastAPI, Header, HTTPException, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from pydantic import BaseModel, ValidationError from sqlalchemy.orm import Session @@ -60,7 +60,7 @@ async def read_tasks(request: Request, db: Session = Depends(get_db), limit: int return tasks -@app.post("/results", status_code=201) +@app.post("/results", status_code=201, dependencies=[Depends(verify_token)]) async def create_result(results: List[AgentResult], db: Session = Depends(get_db)): """Get the results from the agents and store them locally. @@ -89,7 +89,7 @@ async def create_result(results: List[AgentResult], db: Session = Depends(get_db return {"result_ids": [r.id for r in db_results]} -@app.get("/stats") +@app.get("/stats", dependencies=[Depends(verify_token)]) async def get_stats(db: Session = Depends(get_db)): return { "tasks_count": await queries.count_tasks(db), diff --git a/argos/server/database.py b/argos/server/database.py index e01c07e..715c93e 100644 --- a/argos/server/database.py +++ b/argos/server/database.py @@ -1,8 +1,6 @@ from sqlalchemy import create_engine -from sqlalchemy.orm import DeclarativeBase -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession - +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import DeclarativeBase, sessionmaker SQLALCHEMY_DATABASE_URL = "sqlite:////tmp/argos.db" # SQLALCHEMY_DATABASE_URL = "postgresql://user:password@postgresserver/db" diff --git a/argos/server/models.py b/argos/server/models.py index bed3ef2..4cbfa6e 100644 --- a/argos/server/models.py +++ b/argos/server/models.py @@ -1,22 +1,13 @@ -from typing import List, Literal -from sqlalchemy import ( - Boolean, - Column, - ForeignKey, - Integer, - String, - JSON, - DateTime, - Enum, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase -from sqlalchemy_utils import ChoiceType -from sqlalchemy.orm import mapped_column, relationship - from datetime import datetime +from typing import List, Literal + +from sqlalchemy import (JSON, Boolean, Column, DateTime, Enum, ForeignKey, + Integer, String) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from sqlalchemy_utils import ChoiceType -from argos.schemas import WebsiteCheck from argos.checks import get_registered_check +from argos.schemas import WebsiteCheck class Base(DeclarativeBase): diff --git a/argos/server/queries.py b/argos/server/queries.py index 4600439..3ad4a8c 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -1,12 +1,12 @@ -from sqlalchemy.orm import Session +from datetime import datetime +from urllib.parse import urljoin + from sqlalchemy import exists +from sqlalchemy.orm import Session from argos import schemas from argos.logging import logger -from argos.server.models import Task, Result - -from urllib.parse import urljoin -from datetime import datetime +from argos.server.models import Result, Task async def list_tasks(db: Session, agent_id: str, limit: int = 100): diff --git a/config.yaml b/config.yaml index c9eb083..d39a530 100644 --- a/config.yaml +++ b/config.yaml @@ -19,69 +19,4 @@ ssl: - "1d": critical "5d": warning -websites: - - domain: "https://mypads.framapad.org" - paths: - - path: "/mypads/" - checks: - - status-is: 200 - - body-contains: '
' - - ssl-certificate-expiration: "on-check" - - path: "/admin/" - checks: - - status-is: 401 - - domain: "https://munin.framasoft.org" - paths: - - path: "/" - checks: - - status-is: 301 - - path: "/munin/" - checks: - - status-is: 401 - - domain: "https://framagenda.org" - paths: - - path: "/status.php" - checks: - - status-is: 200 - # Là, idéalement, il faudrait un json-contains, - # qui serait une table de hachage - - body-contains: '"maintenance":false' - - ssl-certificate-expiration: "on-check" - - path: "/" - checks: - - status-is: 302 - - path: "/login" - checks: - - status-is: 200 - - domain: "https://framadrive.org" - paths: - - path: "/status.php" - checks: - - status-is: 200 - - body-contains: '"maintenance":false' - - ssl-certificate-expiration: "on-check" - - path: "/" - checks: - - status-is: 302 - - path: "/login" - checks: - - status-is: 200 - - domain: "https://cloud.framabook.org" - paths: - - path: "/status.php" - checks: - - status-is: 200 - - body-contains: '"maintenance":false' - - ssl-certificate-expiration: "on-check" - - path: "/" - checks: - - status-is: 302 - - path: "/login" - checks: - - status-is: 200 - - domain: "https://framasoft.org" - paths: - - path: "/" - checks: - - status-is: 200 - - ssl-certificate-expiration: "on-check" +websites: !include websites.yaml diff --git a/pyproject.toml b/pyproject.toml index 96a8155..036c2f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "httpx>=0.25,<1", "pydantic>=2.4,<3", "pyyaml>=6.0,<7", + "pyyaml-include>=1.3,<2", "sqlalchemy[asyncio]>=2.0,<3", "sqlalchemy-utils>=0.41,<1", "uvicorn>=0.23,<1", diff --git a/tests/test_checks_base.py b/tests/test_checks_base.py index fea85ed..def4ae3 100644 --- a/tests/test_checks_base.py +++ b/tests/test_checks_base.py @@ -1,4 +1,5 @@ import pytest + from argos.checks.base import Response, Status diff --git a/tests/test_schemas_config.py b/tests/test_schemas_config.py index 0e9676e..4191af4 100644 --- a/tests/test_schemas_config.py +++ b/tests/test_schemas_config.py @@ -1,4 +1,5 @@ import pytest + from argos.schemas.config import SSL, WebsitePath