diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31cf6d9..6c6aca3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,12 @@ djlint: script: - make djlint +mypy: + <<: *pull_cache + stage: test + script: + - make mypy + pylint: <<: *pull_cache stage: test @@ -61,6 +67,10 @@ release_job: release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties tag_name: '$CI_COMMIT_TAG' description: './release.md' + assets: + links: + - name: 'PyPI page' + url: 'https://pypi.org/project/argos-monitoring/$CI_COMMIT_TAG/' pages: <<: *pull_cache diff --git a/CHANGELOG.md b/CHANGELOG.md index ef59901..8308fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +- 🩹 — Fix release documentation +- ✅ — Add mypy test +- ✨ — Add new check type: status-in +- 🩹 — Close menu after rescheduling non-ok checks (#55) +- ✨ — Add new check types: headers-contain and headers-have (#56) +- ✨ — Add command to test email configuration (!66) +- 💄 — Enhance the mobile view (!67) +- ✨ — Allow to run Argos in a subfolder (i.e. not on /) (#59) +- ✨ — Add new check types: json-contains, json-has and json-is (#57) + ## 0.2.2 Date: 2024-07-04 diff --git a/Makefile b/Makefile index 722f7ba..33d445a 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,12 @@ ruff: venv ruff-format: venv venv/bin/ruff format . djlint: venv ## Format the templates - venv/bin/djlint --ignore=H030,H031,H006 --profile jinja --lint argos/server/templates/*html + venv/bin/djlint --ignore=H006 --profile jinja --lint argos/server/templates/*html pylint: venv ## Runs pylint on the code venv/bin/pylint argos -lint: djlint pylint ruff +mypy: venv + venv/bin/mypy argos tests +lint: djlint pylint mypy ruff help: @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) diff --git a/argos/agent.py b/argos/agent.py index 7cc3209..e0a4608 100644 --- a/argos/agent.py +++ b/argos/agent.py @@ -54,12 +54,12 @@ class ArgosAgent: while "forever": retry_now = await self._get_and_complete_tasks() if not retry_now: - logger.error("Waiting %i seconds before next retry", self.wait_time) + logger.info("Waiting %i seconds before next retry", self.wait_time) await asyncio.sleep(self.wait_time) - async def _complete_task(self, task: dict) -> dict: + async def _complete_task(self, _task: dict) -> AgentResult: try: - task = Task(**task) + task = Task(**_task) check_class = get_registered_check(task.check) check = check_class(self._http_client, task) result = await check.run() @@ -69,7 +69,7 @@ class ArgosAgent: except Exception as err: # pylint: disable=broad-except status = "error" context = SerializableException.from_exception(err) - msg = f"An exception occured when running {task}. {err.__class__.__name__} : {err}" + msg = f"An exception occured when running {_task}. {err.__class__.__name__} : {err}" logger.error(msg) return AgentResult(task_id=task.id, status=status, context=context) @@ -94,7 +94,7 @@ class ArgosAgent: await self._post_results(results) return True - logger.error("Got no tasks from the server.") + logger.info("Got no tasks from the server.") return False logger.error("Failed to fetch tasks: %s", response.read()) @@ -102,12 +102,19 @@ class ArgosAgent: async def _post_results(self, results: List[AgentResult]): data = [r.model_dump() for r in results] - response = await self._http_client.post( - f"{self.server}/api/results", params={"agent_id": self.agent_id}, json=data - ) + if self._http_client is not None: + response = await self._http_client.post( + f"{self.server}/api/results", + params={"agent_id": self.agent_id}, + json=data, + ) - if response.status_code == httpx.codes.CREATED: - logger.error("Successfully posted results %s", json.dumps(response.json())) - else: - logger.error("Failed to post results: %s", response.read()) - return response + if response.status_code == httpx.codes.CREATED: + logger.info( + "Successfully posted results %s", json.dumps(response.json()) + ) + else: + logger.error("Failed to post results: %s", response.read()) + return response + + logger.error("self._http_client is None") diff --git a/argos/checks/base.py b/argos/checks/base.py index 20521b9..3221809 100644 --- a/argos/checks/base.py +++ b/argos/checks/base.py @@ -1,7 +1,7 @@ """Various base classes for checks""" from dataclasses import dataclass -from typing import Type, Union +from typing import Type import httpx from pydantic import BaseModel @@ -71,7 +71,7 @@ class InvalidResponse(Exception): class BaseCheck: config: str - expected_cls: Union[None, Type[BaseExpectedValue]] = None + expected_cls: None | Type[BaseExpectedValue] = None _registry = [] # type: ignore[var-annotated] diff --git a/argos/checks/checks.py b/argos/checks/checks.py index 42a6848..15c038f 100644 --- a/argos/checks/checks.py +++ b/argos/checks/checks.py @@ -1,7 +1,10 @@ """Define the available checks""" +import json from datetime import datetime +from jsonpointer import resolve_pointer, JsonPointerException + from argos.checks.base import ( BaseCheck, ExpectedIntValue, @@ -31,6 +34,82 @@ class HTTPStatus(BaseCheck): ) +class HTTPStatusIn(BaseCheck): + """Checks that the HTTP status code is in the list of expected values.""" + + config = "status-in" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + return self.response( + status=response.status_code in json.loads(self.expected), + expected=self.expected, + retrieved=response.status_code, + ) + + +class HTTPHeadersContain(BaseCheck): + """Checks that response headers contains the expected headers + (without checking their values)""" + + config = "headers-contain" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + status = True + for header in json.loads(self.expected): + if header not in response.headers: + status = False + break + + return self.response( + status=status, + expected=self.expected, + retrieved=json.dumps(list(dict(response.headers).keys())), + ) + + +class HTTPHeadersHave(BaseCheck): + """Checks that response headers contains the expected headers and values""" + + config = "headers-have" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + status = True + for header, value in json.loads(self.expected).items(): + if header not in response.headers: + status = False + break + if response.headers[header] != value: + status = False + break + + return self.response( + status=status, + expected=self.expected, + retrieved=json.dumps(dict(response.headers)), + ) + + class HTTPBodyContains(BaseCheck): """Checks that the HTTP body contains the expected string.""" @@ -44,6 +123,94 @@ class HTTPBodyContains(BaseCheck): return self.response(status=self.expected in response.text) +class HTTPJsonContains(BaseCheck): + """Checks that JSON response contains the expected structure + (without checking the value)""" + + config = "json-contains" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + obj = response.json() + + status = True + for pointer in json.loads(self.expected): + try: + resolve_pointer(obj, pointer) + except JsonPointerException: + status = False + break + + return self.response( + status=status, + expected=self.expected, + retrieved=json.dumps(obj), + ) + + +class HTTPJsonHas(BaseCheck): + """Checks that JSON response contains the expected structure and values""" + + config = "json-has" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + obj = response.json() + + status = True + for pointer, exp_value in json.loads(self.expected).items(): + try: + value = resolve_pointer(obj, pointer) + if value != exp_value: + status = False + break + except JsonPointerException: + status = False + break + + return self.response( + status=status, + expected=self.expected, + retrieved=json.dumps(obj), + ) + + +class HTTPJsonIs(BaseCheck): + """Checks that JSON response is the exact expected JSON object""" + + config = "json-is" + expected_cls = ExpectedStringValue + + async def run(self) -> dict: + # XXX Get the method from the task + task = self.task + response = await self.http_client.request( + method="get", url=task.url, timeout=60 + ) + + obj = response.json() + + status = response.json() == json.loads(self.expected) + + return self.response( + status=status, + expected=self.expected, + retrieved=json.dumps(obj), + ) + + class SSLCertificateExpiration(BaseCheck): """Checks that the SSL certificate will not expire soon.""" diff --git a/argos/commands.py b/argos/commands.py index 0fc1491..5b13764 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -10,8 +10,7 @@ import uvicorn from alembic import command from alembic.config import Config -from argos import logging -from argos import VERSION +from argos import VERSION, logging from argos.agent import ArgosAgent @@ -305,9 +304,10 @@ async def add(config, name, password): os.environ["ARGOS_YAML_FILE"] = config # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries from passlib.context import CryptContext + from argos.server import queries + db = await get_db() _user = await queries.get_user(db, name) if _user is not None: @@ -339,9 +339,10 @@ async def change_password(config, name, password): os.environ["ARGOS_YAML_FILE"] = config # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries from passlib.context import CryptContext + from argos.server import queries + db = await get_db() _user = await queries.get_user(db, name) if _user is None: @@ -374,9 +375,10 @@ async def verify_password(config, name, password): os.environ["ARGOS_YAML_FILE"] = config # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries from passlib.context import CryptContext + from argos.server import queries + db = await get_db() _user = await queries.get_user(db, name) if _user is None: @@ -548,5 +550,69 @@ async def generate_config(): print(f.read()) +@server.command() +@click.option( + "--config", + default="argos-config.yaml", + help="Path of the configuration file. " + "If ARGOS_YAML_FILE environment variable is set, its value will be used instead.", + envvar="ARGOS_YAML_FILE", + callback=validate_config_access, +) +@click.option("--domain", help="Domain for the notification", default="example.org") +@click.option("--severity", help="Severity", default="CRITICAL") +@coroutine +async def test_mail(config, domain, severity): + """Send a test email""" + os.environ["ARGOS_YAML_FILE"] = config + + from datetime import datetime + + from argos.logging import set_log_level + from argos.server.alerting import notify_by_mail + from argos.server.main import read_config + from argos.server.models import Result, Task + + conf = read_config(config) + + if not conf.general.mail: + click.echo("Mail is not configured, cannot test", err=True) + sysexit(1) + else: + now = datetime.now() + task = Task( + url=f"https://{domain}", + domain=domain, + check="body-contains", + expected="foo", + frequency=1, + selected_by="test", + selected_at=now, + ) + + result = Result( + submitted_at=now, + status="success", + context={"foo": "bar"}, + task=task, + agent_id="test", + severity="ok", + ) + + class _FalseRequest: + def url_for(*args, **kwargs): + return "/url" + + set_log_level("debug") + notify_by_mail( + result, + task, + severity="SEVERITY", + old_severity="OLD SEVERITY", + config=conf.general.mail, + request=_FalseRequest(), + ) + + if __name__ == "__main__": cli() diff --git a/argos/config-example.yaml b/argos/config-example.yaml index 41f00d6..b5558ee 100644 --- a/argos/config-example.yaml +++ b/argos/config-example.yaml @@ -1,10 +1,14 @@ +--- general: db: - # The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls + # The database URL, as defined in SQLAlchemy docs : + # https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls # Example for SQLite: "sqlite:////tmp/argos.db" url: "postgresql://argos:argos@localhost/argos" - # You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?) - # See https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size for details + # You configure the size of the database pool of connection, and + # the max overflow (until when new connections are accepted ?) + # For details, see + # https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size pool_size: 10 max_overflow: 20 # Can be "production", "dev", "test". @@ -29,6 +33,11 @@ general: - local unknown: - local + # Argos root path + # If not present, default value is "" + # Set it to /foo if you want to use argos at /foo/ instead of / + # on your web server + # root_path: "/foo" # Mail configuration is quite straight-forward # mail: # mailfrom: no-reply@example.org @@ -71,12 +80,45 @@ websites: paths: - path: "/mypads/" checks: + # Check that the returned HTTP status is 200 - status-is: 200 + # Check that the response contains this string - body-contains: '
' + # Check that the SSL certificate is no older than ssl.thresholds - ssl-certificate-expiration: "on-check" + # Check that the response contains this headers + # The comparison is case insensitive + - headers-contain: + - "content-encoding" + - "content-type" - path: "/admin/" checks: - - status-is: 401 + # Check that the return HTTP status is one of those + # Similar to status-is, verify that you don’t mistyped it! + - status-in: + - 401 + - 301 + # Check that the response contains this headers and values + # It’s VERY important to respect the 4 spaces indentation here! + # The name of the headers is case insensitive + - headers-have: + content-encoding: "gzip" + content-type: "text/html" + - path: "/my-stats.json" + checks: + # Check that JSON response contains the expected structure + - json-contains: + - /foo/bar/0 + - /foo/bar/1 + - /timestamp + # Check that JSON response contains the expected structure and values + # It’s VERY important to respect the 4 spaces indentation here! + - json-has: + /maintenance: false + /productname: "Nextcloud" + # Check that JSON response is the exact expected JSON object + # The order of the items in the object does not matter. + - json-is: '{"foo": "bar", "baz": 42}' - domain: "https://munin.example.org" frequency: "20m" paths: diff --git a/argos/schemas/config.py b/argos/schemas/config.py index be90f7b..f4887ca 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -2,6 +2,9 @@ For database models, see argos.server.models. """ + +import json + from typing import Dict, List, Literal, Optional, Tuple from pydantic import ( @@ -78,6 +81,10 @@ def parse_checks(value): raise ValueError(msg) if isinstance(expected, int): expected = str(expected) + if isinstance(expected, list): + expected = json.dumps(expected) + if isinstance(expected, dict): + expected = json.dumps(expected) return (name, expected) @@ -157,6 +164,7 @@ class General(BaseModel): frequency: int db: DbSettings env: Environment = "production" + root_path: str = "" alerts: Alert mail: Optional[Mail] = None gotify: Optional[List[GotifyUrl]] = None diff --git a/argos/schemas/utils.py b/argos/schemas/utils.py index d3a2fc1..f225241 100644 --- a/argos/schemas/utils.py +++ b/argos/schemas/utils.py @@ -1,9 +1,9 @@ -from typing import Literal, Union +from typing import Literal def string_to_duration( value: str, target: Literal["days", "hours", "minutes"] -) -> Union[int, float]: +) -> int | float: """Convert a string to a number of hours, days or minutes""" num = int("".join(filter(str.isdigit, value))) diff --git a/argos/server/main.py b/argos/server/main.py index 898b202..b6ee412 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -18,11 +18,19 @@ from argos.server.settings import read_yaml_config def get_application() -> FastAPI: """Spawn Argos FastAPI server""" - appli = FastAPI(lifespan=lifespan) config_file = os.environ["ARGOS_YAML_FILE"] - config = read_config(config_file) + root_path = config.general.root_path + + if root_path != "": + logger.info("Root path for Argos: %s", root_path) + if root_path.endswith("/"): + root_path = root_path[:-1] + logger.info("Fixed root path for Argos: %s", root_path) + + appli = FastAPI(lifespan=lifespan, root_path=root_path) + # Config is the argos config object (built from yaml) appli.state.config = config appli.add_exception_handler(NotAuthenticatedException, auth_exception_handler) diff --git a/argos/server/queries.py b/argos/server/queries.py index b0bf979..e887ebe 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -51,7 +51,7 @@ async def list_users(db: Session): return db.query(User).order_by(asc(User.username)) -async def get_task(db: Session, task_id: int) -> Task: +async def get_task(db: Session, task_id: int) -> None | Task: return db.get(Task, task_id) @@ -71,9 +71,9 @@ async def count_tasks(db: Session, selected: None | bool = None): query = db.query(Task) if selected is not None: if selected: - query = query.filter(Task.selected_by is not None) + query = query.filter(Task.selected_by is not None) # type: ignore[arg-type] else: - query = query.filter(Task.selected_by is None) + query = query.filter(Task.selected_by is None) # type: ignore[arg-type] return query.count() @@ -98,7 +98,7 @@ async def has_config_changed(db: Session, config: schemas.Config) -> bool: case "general_frequency": if conf.val != str(config.general.frequency): same_config = False - conf.val = config.general.frequency + conf.val = str(config.general.frequency) conf.updated_at = datetime.now() db.commit() @@ -208,7 +208,7 @@ async def get_severity_counts(db: Session) -> dict: # Execute the query and fetch the results task_counts_by_severity = query.all() - counts_dict = dict(task_counts_by_severity) + counts_dict = dict(task_counts_by_severity) # type: ignore[var-annotated,arg-type] for key in ("ok", "warning", "critical", "unknown"): counts_dict.setdefault(key, 0) return counts_dict diff --git a/argos/server/routes/api.py b/argos/server/routes/api.py index 8e0e177..ec132ca 100644 --- a/argos/server/routes/api.py +++ b/argos/server/routes/api.py @@ -1,5 +1,5 @@ """Web interface for machines""" -from typing import List, Union +from typing import List from fastapi import APIRouter, BackgroundTasks, Depends, Request from sqlalchemy.orm import Session @@ -18,10 +18,13 @@ async def read_tasks( request: Request, db: Session = Depends(get_db), limit: int = 10, - agent_id: Union[None, str] = None, + agent_id: None | str = None, ): """Return a list of tasks to execute""" - agent_id = agent_id or request.client.host + host = "" + if request.client is not None: + host = request.client.host + agent_id = agent_id or host tasks = await queries.list_tasks(db, agent_id=agent_id, limit=limit) return tasks @@ -33,7 +36,7 @@ async def create_results( background_tasks: BackgroundTasks, db: Session = Depends(get_db), config: Config = Depends(get_config), - agent_id: Union[None, str] = None, + agent_id: None | str = None, ): """Get the results from the agents and store them locally. @@ -42,7 +45,10 @@ async def create_results( - If it's an error, determine its severity ; - Trigger the reporting calls """ - agent_id = agent_id or request.client.host + host = "" + if request.client is not None: + host = request.client.host + agent_id = agent_id or host db_results = [] for agent_result in results: # XXX Maybe offload this to a queue. diff --git a/argos/server/routes/dependencies.py b/argos/server/routes/dependencies.py index 938d7a6..f26d5ee 100644 --- a/argos/server/routes/dependencies.py +++ b/argos/server/routes/dependencies.py @@ -1,5 +1,6 @@ from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from fastapi_login import LoginManager auth_scheme = HTTPBearer() @@ -16,7 +17,7 @@ def get_config(request: Request): return request.app.state.config -async def get_manager(request: Request): +async def get_manager(request: Request) -> LoginManager: return await request.app.state.manager(request) diff --git a/argos/server/routes/views.py b/argos/server/routes/views.py index ef7d4de..fe48566 100644 --- a/argos/server/routes/views.py +++ b/argos/server/routes/views.py @@ -125,7 +125,7 @@ async def get_domains_view( tasks = db.query(Task).all() domains_severities = defaultdict(list) - domains_last_checks = defaultdict(list) + domains_last_checks = defaultdict(list) # type: ignore[var-annotated] for task in tasks: domain = urlparse(task.url).netloc @@ -210,7 +210,9 @@ async def get_task_results_view( .all() ) task = db.query(Task).get(task_id) - description = task.get_check().get_description(config) + description = "" + if task is not None: + description = task.get_check().get_description(config) return templates.TemplateResponse( "results.html", { @@ -251,8 +253,8 @@ async def set_refresh_cookies_view( request.url_for("get_severity_counts_view"), status_code=status.HTTP_303_SEE_OTHER, ) - response.set_cookie(key="auto_refresh_enabled", value=auto_refresh_enabled) + response.set_cookie(key="auto_refresh_enabled", value=str(auto_refresh_enabled)) response.set_cookie( - key="auto_refresh_seconds", value=max(5, int(auto_refresh_seconds)) + key="auto_refresh_seconds", value=str(max(5, int(auto_refresh_seconds))) ) return response diff --git a/argos/server/settings.py b/argos/server/settings.py index 160e609..7d26a49 100644 --- a/argos/server/settings.py +++ b/argos/server/settings.py @@ -7,12 +7,12 @@ from yamlinclude import YamlIncludeConstructor from argos.schemas.config import Config -def read_yaml_config(filename): +def read_yaml_config(filename: str) -> Config: parsed = _load_yaml(filename) return Config(**parsed) -def _load_yaml(filename): +def _load_yaml(filename: str): base_dir = Path(filename).resolve().parent YamlIncludeConstructor.add_to_loader_class( loader_class=yaml.FullLoader, base_dir=str(base_dir) diff --git a/argos/server/static/styles.css b/argos/server/static/styles.css index 769c0a9..6b03c9e 100644 --- a/argos/server/static/styles.css +++ b/argos/server/static/styles.css @@ -4,10 +4,6 @@ code { white-space: pre-wrap; } -body > header, -body > main { - padding: 0 !important; -} #title { margin-bottom: 0; } @@ -53,3 +49,7 @@ label[for="select-status"] { #refresh-delay { max-width: 120px; } +/* Remove chevron on menu */ +#nav-menu summary::after { + background-image: none !important; +} diff --git a/argos/server/templates/base.html b/argos/server/templates/base.html index 513e7a6..4964031 100644 --- a/argos/server/templates/base.html +++ b/argos/server/templates/base.html @@ -3,6 +3,10 @@