Support !include filename in the yaml files.

- run isort on the codebase

Pipfile

@@ -15,6 +15,7 @@ sqlalchemy = {extras = ["asyncio"] }
 pyopenssl = "*"
 ipdb = "*"
 argos = {extras = ["dev"], file = ".", editable = true}
+pyyaml-include = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "545ec239a057ec56cb3b4e5d7d6b8922a837d9ce2340e4b2def368c8064acf73"
+            "sha256": "8cc9237ff86d00019539f36e3df5b20edcbbc60f52d1b0fce2b03e51c089ad39"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -669,6 +669,15 @@
             "markers": "python_version >= '3.6'",
             "version": "==6.0.1"
         },
+        "pyyaml-include": {
+            "hashes": [
+                "sha256:4cb3b4e1baae2ec251808fe1e8aed5d3d20699c541864c8e47ed866ab2f15039",
+                "sha256:e58525721a2938d29c4046350f8aad86f848660a770c29605e6f2700925fa753"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",

argos/agent.py

@@ -1,10 +1,11 @@
-import httpx
 import asyncio
 from typing import List
-from argos.logging import logger
-from argos.checks import CheckNotFound, get_registered_check
 
-from argos.schemas import Task, AgentResult, SerializableException
+import httpx
+
+from argos.checks import CheckNotFound, get_registered_check
+from argos.logging import logger
+from argos.schemas import AgentResult, SerializableException, Task
 
 
 async def complete_task(http_client: httpx.AsyncClient, task: dict) -> dict:
@@ -58,4 +59,4 @@ async def run_agent(server: str, auth: str, max_tasks: int):
             # Post the results
             await post_results(http_client, server, results)
         else:
-            logger.error(f"Failed to fetch tasks: {response.read()}")
+            logger.error(f"Failed to fetch tasks: {response.read()}")

argos/checks/__init__.py

@@ -1,2 +1,4 @@
-from argos.checks.checks import HTTPStatus, HTTPBodyContains, SSLCertificateExpiration
-from argos.checks.base import get_registered_checks, get_registered_check, CheckNotFound
+from argos.checks.base import (CheckNotFound, get_registered_check,
+                               get_registered_checks)
+from argos.checks.checks import (HTTPBodyContains, HTTPStatus,
+                                 SSLCertificateExpiration)

argos/checks/base.py

@@ -1,8 +1,8 @@
-from pydantic import BaseModel, Field
 from dataclasses import dataclass
-
 from typing import Type
+
 import httpx
+from pydantic import BaseModel, Field
 
 from argos.schemas import Task
 

argos/checks/checks.py

@@ -1,17 +1,13 @@
-from argos.logging import logger
-from argos.checks.base import (
-    BaseCheck,
-    Response,
-    Status,
-    ExpectedIntValue,
-    ExpectedStringValue,
-)
 import ssl
 import time
-
 from datetime import datetime
+
 from OpenSSL import crypto
 
+from argos.checks.base import (BaseCheck, ExpectedIntValue,
+                               ExpectedStringValue, Response, Status)
+from argos.logging import logger
+
 
 class HTTPStatus(BaseCheck):
     config = "status-is"

argos/commands.py

@@ -1,15 +1,18 @@
-import click
-import subprocess
 import asyncio
+import subprocess
+
+import click
 
-from argos.agent import run_agent
 from argos import logging
+from argos.agent import run_agent
 from argos.logging import logger
 
+
 @click.group()
 def cli():
     pass
 
+
 @cli.command()
 @click.option("--server", required=True, help="Server URL")
 @click.option("--auth", required=True, help="The authentication token")
@@ -36,5 +39,6 @@ def server(host, port, reload):
         command.append("--reload")
     subprocess.run(command)
 
+
 if __name__ == "__main__":
     cli()

argos/schemas/config.py

@@ -1,14 +1,11 @@
-from pydantic import BaseModel
-
+import os
+from datetime import datetime
 from enum import StrEnum
-from typing import List, Optional, Tuple, Literal
-
-from typing import Dict, Union, List
+from typing import Dict, List, Literal, Optional, Tuple, Union
 
 import yaml
 from pydantic import BaseModel, Field, HttpUrl, validator
-
-from datetime import datetime
+from yamlinclude import YamlIncludeConstructor
 
 # XXX Find a way to check without having cirular imports
 
@@ -66,7 +63,8 @@ class WebsitePath(BaseModel):
 
     @validator("checks", each_item=True, pre=True)
     def parse_checks(cls, value):
-        from argos.checks import get_registered_checks  # To avoid circular imports
+        from argos.checks import \
+            get_registered_checks  # To avoid circular imports
 
         available_names = get_registered_checks().keys()
 
@@ -109,11 +107,16 @@ def validate_config(config: dict):
     return Config(**config)
 
 
-def from_yaml(file_name):
-    parsed = load_yaml(file_name)
+def from_yaml(filename):
+    parsed = load_yaml(filename)
     return validate_config(parsed)
 
 
-def load_yaml(file_name):
-    with open(file_name, "r") as stream:
-        return yaml.safe_load(stream)
+def load_yaml(filename):
+    base_dir = os.path.dirname(filename)
+    YamlIncludeConstructor.add_to_loader_class(
+        loader_class=yaml.FullLoader, base_dir=base_dir
+    )
+
+    with open(filename, "r") as stream:
+        return yaml.load(stream, Loader=yaml.FullLoader)

argos/schemas/models.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel
+import traceback
 from datetime import datetime
 from typing import Literal
-import traceback
+
+from pydantic import BaseModel
 
 # XXX Refactor using SQLModel to avoid duplication of model data
 

argos/server/api.py

@@ -1,8 +1,8 @@
 import sys
 from typing import Annotated, List, Optional
 
-from fastapi import Depends, FastAPI, HTTPException, Request, Header
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from fastapi import Depends, FastAPI, Header, HTTPException, Request
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from pydantic import BaseModel, ValidationError
 from sqlalchemy.orm import Session
 
@@ -60,7 +60,7 @@ async def read_tasks(request: Request, db: Session = Depends(get_db), limit: int
     return tasks
 
 
-@app.post("/results", status_code=201)
+@app.post("/results", status_code=201, dependencies=[Depends(verify_token)])
 async def create_result(results: List[AgentResult], db: Session = Depends(get_db)):
     """Get the results from the agents and store them locally.
 
@@ -89,7 +89,7 @@ async def create_result(results: List[AgentResult], db: Session = Depends(get_db
     return {"result_ids": [r.id for r in db_results]}
 
 
-@app.get("/stats")
+@app.get("/stats", dependencies=[Depends(verify_token)])
 async def get_stats(db: Session = Depends(get_db)):
     return {
         "tasks_count": await queries.count_tasks(db),

argos/server/database.py

@@ -1,8 +1,6 @@
 from sqlalchemy import create_engine
-from sqlalchemy.orm import DeclarativeBase
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
-
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
+from sqlalchemy.orm import DeclarativeBase, sessionmaker
 
 SQLALCHEMY_DATABASE_URL = "sqlite:////tmp/argos.db"
 # SQLALCHEMY_DATABASE_URL = "postgresql://user:password@postgresserver/db"

argos/server/models.py

@@ -1,22 +1,13 @@
-from typing import List, Literal
-from sqlalchemy import (
-    Boolean,
-    Column,
-    ForeignKey,
-    Integer,
-    String,
-    JSON,
-    DateTime,
-    Enum,
-)
-from sqlalchemy.orm import relationship, Mapped, mapped_column, DeclarativeBase
-from sqlalchemy_utils import ChoiceType
-from sqlalchemy.orm import mapped_column, relationship
-
 from datetime import datetime
+from typing import List, Literal
+
+from sqlalchemy import (JSON, Boolean, Column, DateTime, Enum, ForeignKey,
+                        Integer, String)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+from sqlalchemy_utils import ChoiceType
 
-from argos.schemas import WebsiteCheck
 from argos.checks import get_registered_check
+from argos.schemas import WebsiteCheck
 
 
 class Base(DeclarativeBase):

argos/server/queries.py

@@ -1,12 +1,12 @@
-from sqlalchemy.orm import Session
+from datetime import datetime
+from urllib.parse import urljoin
+
 from sqlalchemy import exists
+from sqlalchemy.orm import Session
 
 from argos import schemas
 from argos.logging import logger
-from argos.server.models import Task, Result
-
-from urllib.parse import urljoin
-from datetime import datetime
+from argos.server.models import Result, Task
 
 
 async def list_tasks(db: Session, agent_id: str, limit: int = 100):

config.yaml

@@ -19,69 +19,4 @@ ssl:
     - "1d": critical
       "5d": warning
 
-websites:
-  - domain: "https://mypads.framapad.org"
-    paths:
-      - path: "/mypads/"
-        checks:
-          - status-is: 200
-          - body-contains: '<div id= "mypads"></div>'
-          - ssl-certificate-expiration: "on-check"
-      - path: "/admin/"
-        checks:
-          - status-is: 401
-  - domain: "https://munin.framasoft.org"
-    paths:
-      - path: "/"
-        checks:
-          - status-is: 301
-      - path: "/munin/"
-        checks:
-          - status-is: 401
-  - domain: "https://framagenda.org"
-    paths:
-      - path: "/status.php"
-        checks:
-          - status-is: 200
-          # Là, idéalement, il faudrait un json-contains,
-          # qui serait une table de hachage
-          - body-contains: '"maintenance":false'
-          - ssl-certificate-expiration: "on-check"
-      - path: "/"
-        checks:
-          - status-is: 302
-      - path: "/login"
-        checks:
-          - status-is: 200
-  - domain: "https://framadrive.org"
-    paths:
-      - path: "/status.php"
-        checks:
-          - status-is: 200
-          - body-contains: '"maintenance":false'
-          - ssl-certificate-expiration: "on-check"
-      - path: "/"
-        checks:
-          - status-is: 302
-      - path: "/login"
-        checks:
-          - status-is: 200
-  - domain: "https://cloud.framabook.org"
-    paths:
-      - path: "/status.php"
-        checks:
-          - status-is: 200
-          - body-contains: '"maintenance":false'
-          - ssl-certificate-expiration: "on-check"
-      - path: "/"
-        checks:
-          - status-is: 302
-      - path: "/login"
-        checks:
-          - status-is: 200
-  - domain: "https://framasoft.org"
-    paths:
-      - path: "/"
-        checks:
-          - status-is: 200
-          - ssl-certificate-expiration: "on-check"
+websites: !include websites.yaml

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
   "httpx>=0.25,<1",
   "pydantic>=2.4,<3",
   "pyyaml>=6.0,<7",
+  "pyyaml-include>=1.3,<2",
   "sqlalchemy[asyncio]>=2.0,<3",
   "sqlalchemy-utils>=0.41,<1",
   "uvicorn>=0.23,<1",

tests/test_checks_base.py

@@ -1,4 +1,5 @@
 import pytest
+
 from argos.checks.base import Response, Status
 
 

tests/test_schemas_config.py

@@ -1,4 +1,5 @@
 import pytest
+
 from argos.schemas.config import SSL, WebsitePath