diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ba1a1c..ef25432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## [Unreleased] +- ✨ — Allow to customize agent User-Agent header (#78) +- 📝 — Document how to add data to requests (#77) +- ✨ — No need cron tasks for DB cleaning anymore (#74 and #75) +- ✨ — No need cron tasks for agents watching (#76) +- ✨ — Reload configuration asynchronously (#79) +- 🐛 — Automatically reconnect to LDAP if unreachable (#81) +- 🐛 — Better httpx.RequestError handling (#83) + +💥 Warning: there is now new settings to add to your configuration file. +Use `argos server generate-config > /etc/argos/config.yaml-dist` to generate +a new example configuration file. + +💥 You don’t need cron tasks anymore! +Remove your old cron tasks as they will now do nothing but generating errors. + ## 0.7.4 Date: 2025-02-12 diff --git a/argos/agent.py b/argos/agent.py index 195ffdc..dec1e50 100644 --- a/argos/agent.py +++ b/argos/agent.py @@ -37,11 +37,17 @@ def log_failure(retry_state): class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes """The Argos agent is responsible for running the checks and reporting the results.""" - def __init__(self, server: str, auth: str, max_tasks: int, wait_time: int): + def __init__( # pylint: disable-msg=too-many-positional-arguments + self, server: str, auth: str, max_tasks: int, wait_time: int, user_agent: str + ): self.server = server self.max_tasks = max_tasks self.wait_time = wait_time self.auth = auth + if user_agent == "": + self.ua = user_agent + else: + self.ua = f" - {user_agent}" self._http_client: httpx.AsyncClient | None = None self._http_client_v4: httpx.AsyncClient | None = None self._http_client_v6: httpx.AsyncClient | None = None @@ -53,13 +59,13 @@ class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes async def run(self): auth_header = { "Authorization": f"Bearer {self.auth}", - "User-Agent": f"Argos Panoptes agent {VERSION}", + "User-Agent": f"Argos Panoptes agent {VERSION}{self.ua}", } self._http_client = httpx.AsyncClient(headers=auth_header) ua_header = { "User-Agent": f"Argos Panoptes {VERSION} " - "(about: https://argos-monitoring.framasoft.org/)", + f"(about: https://argos-monitoring.framasoft.org/){self.ua}", } self._http_client_v4 = httpx.AsyncClient( headers=ua_header, @@ -78,6 +84,7 @@ class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes await asyncio.sleep(self.wait_time) async def _do_request(self, group: str, details: dict): + logger.debug("_do_request for group %s", group) headers = {} if details["request_data"] is not None: request_data = json.loads(details["request_data"]) @@ -114,6 +121,7 @@ class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes ) except httpx.ReadError: sleep(1) + logger.warning("httpx.ReadError for group %s, re-emit request", group) if details["request_data"] is None or request_data["data"] is None: response = await http_client.request( # type: ignore[union-attr] method=details["method"], url=details["url"], timeout=60 @@ -132,6 +140,9 @@ class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes data=request_data["data"], timeout=60, ) + except httpx.RequestError as err: + logger.warning("httpx.RequestError for group %s", group) + response = err self._res_cache[group] = response @@ -141,15 +152,21 @@ class ArgosAgent: # pylint: disable-msg=too-many-instance-attributes check_class = get_registered_check(task.check) check = check_class(task) - result = await check.run(self._res_cache[task.task_group]) - status = result.status - context = result.context + response = self._res_cache[task.task_group] + if isinstance(response, httpx.Response): + result = await check.run(response) + status = result.status + context = result.context + else: + status = "failure" + context = SerializableException.from_exception(response) except Exception as err: # pylint: disable=broad-except status = "error" context = SerializableException.from_exception(err) msg = f"An exception occured when running {_task}. {err.__class__.__name__} : {err}" logger.error(msg) + return AgentResult(task_id=task.id, status=status, context=context) async def _get_and_complete_tasks(self): diff --git a/argos/commands.py b/argos/commands.py index ac1309f..7bd5690 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -92,7 +92,12 @@ def version(): default="INFO", type=click.Choice(logging.LOG_LEVELS, case_sensitive=False), ) -def agent(server_url, auth, max_tasks, wait_time, log_level): +@click.option( + "--user-agent", + default="", + help="A custom string to append to the User-Agent header", +) +def agent(server_url, auth, max_tasks, wait_time, log_level, user_agent): # pylint: disable-msg=too-many-positional-arguments """Get and run tasks for the provided server. Will wait for new tasks. Usage: argos agent https://argos.example.org "auth-token-here" @@ -108,7 +113,7 @@ def agent(server_url, auth, max_tasks, wait_time, log_level): from argos.logging import logger logger.setLevel(log_level) - agent_ = ArgosAgent(server_url, auth, max_tasks, wait_time) + agent_ = ArgosAgent(server_url, auth, max_tasks, wait_time, user_agent) asyncio.run(agent_.run()) @@ -135,101 +140,6 @@ def start(host, port, config, reload): uvicorn.run("argos.server:app", host=host, port=port, reload=reload) -def validate_max_lock_seconds(ctx, param, value): - if value <= 60: - raise click.BadParameter("Should be strictly higher than 60") - return value - - -def validate_max_results(ctx, param, value): - if value <= 0: - raise click.BadParameter("Should be a positive integer") - return value - - -@server.command() -@click.option( - "--max-results", - default=100, - help="Number of results per task to keep", - callback=validate_max_results, -) -@click.option( - "--max-lock-seconds", - default=100, - help=( - "The number of seconds after which a lock is " - "considered stale, must be higher than 60 " - "(the checks have a timeout value of 60 seconds)" - ), - callback=validate_max_lock_seconds, -) -@click.option( - "--config", - default="argos-config.yaml", - help="Path of the configuration file. " - "If ARGOS_YAML_FILE environment variable is set, its value will be used instead. " - "Default value: argos-config.yaml and /etc/argos/config.yaml as fallback.", - envvar="ARGOS_YAML_FILE", - callback=validate_config_access, -) -@coroutine -async def cleandb(max_results, max_lock_seconds, config): - """Clean the database (to run routinely) - - \b - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. - """ - # It’s mandatory to do it before the imports - os.environ["ARGOS_YAML_FILE"] = config - - # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries - - db = await get_db() - removed = await queries.remove_old_results(db, max_results) - updated = await queries.release_old_locks(db, max_lock_seconds) - - click.echo(f"{removed} results removed") - click.echo(f"{updated} locks released") - - -@server.command() -@click.option( - "--time-without-agent", - default=5, - help="Time without seeing an agent after which a warning will be issued, in minutes. " - "Default is 5 minutes.", - callback=validate_max_results, -) -@click.option( - "--config", - default="argos-config.yaml", - help="Path of the configuration file. " - "If ARGOS_YAML_FILE environment variable is set, its value will be used instead.", - envvar="ARGOS_YAML_FILE", - callback=validate_config_access, -) -@coroutine -async def watch_agents(time_without_agent, config): - """Watch agents (to run routinely) - - Issues a warning if no agent has been seen by the server for a given time. - """ - # It’s mandatory to do it before the imports - os.environ["ARGOS_YAML_FILE"] = config - - # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries - - db = await get_db() - agents = await queries.get_recent_agents_count(db, time_without_agent) - if agents == 0: - click.echo(f"No agent has been seen in the last {time_without_agent} minutes.") - sysexit(1) - - @server.command(short_help="Load or reload tasks’ configuration") @click.option( "--config", @@ -240,23 +150,40 @@ async def watch_agents(time_without_agent, config): envvar="ARGOS_YAML_FILE", callback=validate_config_access, ) +@click.option( + "--enqueue/--no-enqueue", + default=False, + help="Let Argos main recurring tasks handle configuration’s loading. " + "It may delay the application of the new configuration up to 2 minutes. " + "Default is --no-enqueue", +) @coroutine -async def reload_config(config): +async def reload_config(config, enqueue): """Read tasks’ configuration and add/delete tasks in database if needed""" # It’s mandatory to do it before the imports os.environ["ARGOS_YAML_FILE"] = config # The imports are made here otherwise the agent will need server configuration files. from argos.server import queries - from argos.server.main import read_config + from argos.server.settings import read_config _config = read_config(config) db = await get_db() - changed = await queries.update_from_config(db, _config) - click.echo(f"{changed['added']} tasks added") - click.echo(f"{changed['vanished']} tasks deleted") + config_changed = await queries.has_config_changed(db, _config) + if not config_changed: + click.echo("Config has not change") + else: + if enqueue: + msg = await queries.update_from_config_later(db, config_file=config) + + click.echo(msg) + else: + changed = await queries.update_from_config(db, _config) + + click.echo(f"{changed['added']} task(s) added") + click.echo(f"{changed['vanished']} task(s) deleted") @server.command() @@ -570,8 +497,8 @@ async def test_mail(config, domain, severity): from argos.logging import set_log_level from argos.server.alerting import notify_by_mail - from argos.server.main import read_config from argos.server.models import Result, Task + from argos.server.settings import read_config conf = read_config(config) @@ -586,6 +513,7 @@ async def test_mail(config, domain, severity): check="body-contains", expected="foo", frequency=1, + ip_version=4, selected_by="test", selected_at=now, ) @@ -634,8 +562,8 @@ async def test_gotify(config, domain, severity): from argos.logging import set_log_level from argos.server.alerting import notify_with_gotify - from argos.server.main import read_config from argos.server.models import Result, Task + from argos.server.settings import read_config conf = read_config(config) @@ -650,6 +578,7 @@ async def test_gotify(config, domain, severity): check="body-contains", expected="foo", frequency=1, + ip_version=4, selected_by="test", selected_at=now, ) @@ -701,8 +630,8 @@ async def test_apprise(config, domain, severity, apprise_group): from argos.logging import set_log_level from argos.server.alerting import notify_with_apprise - from argos.server.main import read_config from argos.server.models import Result, Task + from argos.server.settings import read_config conf = read_config(config) @@ -717,6 +646,7 @@ async def test_apprise(config, domain, severity, apprise_group): check="body-contains", expected="foo", frequency=1, + ip_version=4, selected_by="test", selected_at=now, ) diff --git a/argos/config-example.yaml b/argos/config-example.yaml index 5508bff..3b9141f 100644 --- a/argos/config-example.yaml +++ b/argos/config-example.yaml @@ -81,6 +81,12 @@ general: # To disable the IPv6 check of domains: # ipv6: false + # Argos root path + # If not present, default value is "" + # Set it to /foo if you want to use argos at /foo/ instead of / + # on your web server + # root_path: "/foo" + # Which way do you want to be warned when a check goes to that severity? # "local" emits a message in the server log # You’ll need to configure mail, gotify or apprise below to be able to use @@ -96,11 +102,10 @@ general: - local unknown: - local - # Argos root path - # If not present, default value is "" - # Set it to /foo if you want to use argos at /foo/ instead of / - # on your web server - # root_path: "/foo" + # This alert is triggered when no Argos agent has been seen in a while + # See recurring_tasks.time_without_agent below + no_agent: + - local # Mail configuration is quite straight-forward # mail: # mailfrom: no-reply@example.org @@ -144,6 +149,20 @@ ssl: - "1d": critical - "5d": warning +# Argos will execute some tasks in the background for you +# every 2 minutes and needs some configuration for that +recurring_tasks: + # Max number of results per tasks you want to keep + # Minimum value is 1, default is 100 + max_results: 100 + # Max number of seconds a task can be locked + # Minimum value is 61, default is 100 + max_lock_seconds: 100 + # Max number of minutes without seing an agent + # before sending an alert + # Minimum value is 1, default is 5 + time_without_agent: 5 + # It's also possible to define the checks in another file # with the include syntax: # diff --git a/argos/logging.py b/argos/logging.py index 5aba551..e67bfb9 100644 --- a/argos/logging.py +++ b/argos/logging.py @@ -14,9 +14,10 @@ logger = logging.getLogger(__name__) # XXX Does not work ? -def set_log_level(log_level): +def set_log_level(log_level: str, quiet: bool = False): level = getattr(logging, log_level.upper(), None) if not isinstance(level, int): raise ValueError(f"Invalid log level: {log_level}") logger.setLevel(level=level) - logger.info("Log level set to %s", log_level) + if not quiet: + logger.info("Log level set to %s", log_level) diff --git a/argos/schemas/config.py b/argos/schemas/config.py index 1baa7ed..26d0198 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -48,6 +48,36 @@ class SSL(BaseModel): thresholds: List[Annotated[Tuple[int, Severity], BeforeValidator(parse_threshold)]] +class RecurringTasks(BaseModel): + max_results: int + max_lock_seconds: int + time_without_agent: int + + @field_validator("max_results", mode="before") + def parse_max_results(cls, value): + """Ensure that max_results is higher than 0""" + if value >= 1: + return value + + return 100 + + @field_validator("max_lock_seconds", mode="before") + def parse_max_lock_seconds(cls, value): + """Ensure that max_lock_seconds is higher or equal to agent’s requests timeout (60)""" + if value > 60: + return value + + return 100 + + @field_validator("time_without_agent", mode="before") + def parse_time_without_agent(cls, value): + """Ensure that time_without_agent is at least one minute""" + if value >= 1: + return value + + return 5 + + class WebsiteCheck(BaseModel): key: str value: str | List[str] | Dict[str, str] @@ -190,6 +220,7 @@ class Alert(BaseModel): warning: List[str] critical: List[str] unknown: List[str] + no_agent: List[str] class GotifyUrl(BaseModel): @@ -264,4 +295,5 @@ class Config(BaseModel): general: General service: Service ssl: SSL + recurring_tasks: RecurringTasks websites: List[Website] diff --git a/argos/schemas/models.py b/argos/schemas/models.py index ed1bc20..cd7f4cc 100644 --- a/argos/schemas/models.py +++ b/argos/schemas/models.py @@ -8,11 +8,25 @@ from typing import Literal from pydantic import BaseModel, ConfigDict -from argos.schemas.utils import IPVersion, Method +from argos.schemas.utils import IPVersion, Method, Todo # XXX Refactor using SQLModel to avoid duplication of model data +class Job(BaseModel): + """Tasks needing to be executed in recurring tasks processing. + It’s quite like a job queue.""" + + id: int + todo: Todo + args: str + current: bool + added_at: datetime + + def __str__(self): + return f"Job ({self.id}): {self.todo}" + + class Task(BaseModel): """A task corresponds to a check to execute""" diff --git a/argos/schemas/utils.py b/argos/schemas/utils.py index a160ee1..ed81b59 100644 --- a/argos/schemas/utils.py +++ b/argos/schemas/utils.py @@ -6,3 +6,5 @@ IPVersion = Literal["4", "6"] Method = Literal[ "GET", "HEAD", "POST", "OPTIONS", "CONNECT", "TRACE", "PUT", "PATCH", "DELETE" ] + +Todo = Literal["RELOAD_CONFIG"] diff --git a/argos/server/alerting.py b/argos/server/alerting.py index d533ed4..8cec99a 100644 --- a/argos/server/alerting.py +++ b/argos/server/alerting.py @@ -74,6 +74,91 @@ def get_icon_from_severity(severity: str) -> str: return icon +def send_mail(mail: EmailMessage, config: Mail): + """Send message by mail""" + + if config.ssl: + logger.debug("Mail notification: SSL") + context = ssl.create_default_context() + smtp = smtplib.SMTP_SSL(host=config.host, port=config.port, context=context) + else: + smtp = smtplib.SMTP( + host=config.host, # type: ignore + port=config.port, + ) + if config.starttls: + logger.debug("Mail notification: STARTTLS") + context = ssl.create_default_context() + smtp.starttls(context=context) + + if config.auth is not None: + logger.debug("Mail notification: authentification") + smtp.login(config.auth.login, config.auth.password) + + for address in config.addresses: + logger.debug("Sending mail to %s", address) + logger.debug(mail.get_body()) + smtp.send_message(mail, to_addrs=address) + + +def send_gotify_msg(config, payload): + """Send message with gotify""" + headers = {"accept": "application/json", "content-type": "application/json"} + + for url in config: + logger.debug("Sending gotify message(s) to %s", url.url) + for token in url.tokens: + try: + res = httpx.post( + f"{url.url}message", + params={"token": token}, + headers=headers, + json=payload, + ) + res.raise_for_status() + except httpx.RequestError as err: + logger.error( + "An error occurred while sending a message to %s with token %s", + err.request.url, + token, + ) + + +def no_agent_alert(config: Config): + """Alert""" + msg = "You should check what’s going on with your Argos agents." + twa = config.recurring_tasks.time_without_agent + if twa > 1: + subject = f"No agent has been seen within the last {twa} minutes" + else: + subject = "No agent has been seen within the last minute" + + if "local" in config.general.alerts.no_agent: + logger.error(subject) + + if config.general.mail is not None and "mail" in config.general.alerts.no_agent: + mail = EmailMessage() + mail["Subject"] = f"[Argos] {subject}" + mail["From"] = config.general.mail.mailfrom + mail.set_content(msg) + send_mail(mail, config.general.mail) + + if config.general.gotify is not None and "gotify" in config.general.alerts.no_agent: + priority = 9 + payload = {"title": subject, "message": msg, "priority": priority} + send_gotify_msg(config.general.gotify, payload) + + if config.general.apprise is not None: + for notif_way in config.general.alerts.no_agent: + if notif_way.startswith("apprise:"): + group = notif_way[8:] + apobj = apprise.Apprise() + for channel in config.general.apprise[group]: + apobj.add(channel) + + apobj.notify(title=subject, body=msg) + + def handle_alert(config: Config, result, task, severity, old_severity, request): # pylint: disable-msg=too-many-positional-arguments """Dispatch alert through configured alert channels""" @@ -163,36 +248,13 @@ See results of task on {request.url_for('get_task_results_view', task_id=task.id ] = f"[Argos] {icon} {urlparse(task.url).netloc} (IPv{task.ip_version}): status {severity}" mail["From"] = config.mailfrom mail.set_content(msg) - - if config.ssl: - logger.debug("Mail notification: SSL") - context = ssl.create_default_context() - smtp = smtplib.SMTP_SSL(host=config.host, port=config.port, context=context) - else: - smtp = smtplib.SMTP( - host=config.host, # type: ignore - port=config.port, - ) - if config.starttls: - logger.debug("Mail notification: STARTTLS") - context = ssl.create_default_context() - smtp.starttls(context=context) - - if config.auth is not None: - logger.debug("Mail notification: authentification") - smtp.login(config.auth.login, config.auth.password) - - for address in config.addresses: - logger.debug("Sending mail to %s", address) - logger.debug(msg) - smtp.send_message(mail, to_addrs=address) + send_mail(mail, config) def notify_with_gotify( # pylint: disable-msg=too-many-positional-arguments result, task, severity: str, old_severity: str, config: List[GotifyUrl], request ) -> None: logger.debug("Will send gotify notification") - headers = {"accept": "application/json", "content-type": "application/json"} icon = get_icon_from_severity(severity) priority = 9 @@ -228,20 +290,4 @@ See results of task on <{request.url_for('get_task_results_view', task_id=task.i payload = {"title": subject, "message": msg, "priority": priority, "extras": extras} - for url in config: - logger.debug("Sending gotify message(s) to %s", url.url) - for token in url.tokens: - try: - res = httpx.post( - f"{url.url}message", - params={"token": token}, - headers=headers, - json=payload, - ) - res.raise_for_status() - except httpx.RequestError as err: - logger.error( - "An error occurred while sending a message to %s with token %s", - err.request.url, - token, - ) + send_gotify_msg(config, payload) diff --git a/argos/server/main.py b/argos/server/main.py index 0543182..c4623d2 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -1,19 +1,19 @@ import os -import sys from contextlib import asynccontextmanager from pathlib import Path from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from fastapi_login import LoginManager -from pydantic import ValidationError +from fastapi_utils.tasks import repeat_every from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker -from argos.logging import logger +from argos.logging import logger, set_log_level from argos.server import models, routes, queries +from argos.server.alerting import no_agent_alert from argos.server.exceptions import NotAuthenticatedException, auth_exception_handler -from argos.server.settings import read_yaml_config +from argos.server.settings import read_config def get_application() -> FastAPI: @@ -39,9 +39,7 @@ def get_application() -> FastAPI: if config.general.ldap is not None: import ldap - l = ldap.initialize(config.general.ldap.uri) - l.simple_bind_s(config.general.ldap.bind_dn, config.general.ldap.bind_pwd) - appli.state.ldap = l + appli.state.ldap = ldap.initialize(config.general.ldap.uri) @appli.state.manager.user_loader() async def query_user(user: str) -> None | str | models.User: @@ -71,17 +69,6 @@ async def connect_to_db(appli): return appli.state.db -def read_config(yaml_file): - try: - config = read_yaml_config(yaml_file) - return config - except ValidationError as err: - logger.error("Errors where found while reading configuration:") - for error in err.errors(): - logger.error("%s is %s", error["loc"], error["type"]) - sys.exit(1) - - def setup_database(appli): config = appli.state.config db_url = str(config.general.db.url) @@ -126,8 +113,31 @@ def create_manager(cookie_secret: str) -> LoginManager: ) +@repeat_every(seconds=120, logger=logger) +async def recurring_tasks() -> None: + """Recurring DB cleanup and watch-agents tasks""" + set_log_level("info", quiet=True) + logger.info("Start background recurring tasks") + with app.state.SessionLocal() as db: + config = app.state.config.recurring_tasks + removed = await queries.remove_old_results(db, config.max_results) + logger.info("%i result(s) removed", removed) + + updated = await queries.release_old_locks(db, config.max_lock_seconds) + logger.info("%i lock(s) released", updated) + + agents = await queries.get_recent_agents_count(db, config.time_without_agent) + if agents == 0: + no_agent_alert(app.state.config) + + processed_jobs = await queries.process_jobs(db) + logger.info("%i job(s) processed", processed_jobs) + + logger.info("Background recurring tasks ended") + + @asynccontextmanager -async def lifespan(appli): +async def lifespan(appli: FastAPI): """Server start and stop actions Setup database connection then close it at shutdown. @@ -142,6 +152,7 @@ async def lifespan(appli): "There is no tasks in the database. " 'Please launch the command "argos server reload-config"' ) + await recurring_tasks() yield diff --git a/argos/server/migrations/versions/5f6cb30db996_add_job_queue.py b/argos/server/migrations/versions/5f6cb30db996_add_job_queue.py new file mode 100644 index 0000000..cf5d9e2 --- /dev/null +++ b/argos/server/migrations/versions/5f6cb30db996_add_job_queue.py @@ -0,0 +1,43 @@ +"""Add job queue + +Revision ID: 5f6cb30db996 +Revises: bd4b4962696a +Create Date: 2025-02-17 16:56:36.673511 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "5f6cb30db996" +down_revision: Union[str, None] = "bd4b4962696a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + enum = sa.Enum( + "RELOAD_CONFIG", + name="todo_enum", + create_type=False, + ) + enum.create(op.get_bind(), checkfirst=True) + op.create_table( + "jobs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("todo", enum, nullable=False), + sa.Column("args", sa.String(), nullable=False), + sa.Column( + "current", sa.Boolean(), server_default=sa.sql.false(), nullable=False + ), + sa.Column("added_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("jobs") + sa.Enum(name="todo_enum").drop(op.get_bind(), checkfirst=True) diff --git a/argos/server/models.py b/argos/server/models.py index c503e20..eab35ac 100644 --- a/argos/server/models.py +++ b/argos/server/models.py @@ -14,7 +14,7 @@ from sqlalchemy.schema import Index from argos.checks import BaseCheck, get_registered_check from argos.schemas import WebsiteCheck -from argos.schemas.utils import IPVersion, Method +from argos.schemas.utils import IPVersion, Method, Todo def compute_task_group(context) -> str: @@ -33,6 +33,19 @@ class Base(DeclarativeBase): type_annotation_map = {List[WebsiteCheck]: JSON, dict: JSON} +class Job(Base): + """ + Job queue emulation + """ + + __tablename__ = "jobs" + id: Mapped[int] = mapped_column(primary_key=True) + todo: Mapped[Todo] = mapped_column(Enum("RELOAD_CONFIG", name="todo_enum")) + args: Mapped[str] = mapped_column() + current: Mapped[bool] = mapped_column(insert_default=False) + added_at: Mapped[datetime] = mapped_column() + + class Task(Base): """ There is one task per check. diff --git a/argos/server/queries.py b/argos/server/queries.py index be9afd7..0329eb9 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -9,7 +9,8 @@ from sqlalchemy.orm import Session from argos import schemas from argos.logging import logger -from argos.server.models import Result, Task, ConfigCache, User +from argos.server.models import ConfigCache, Job, Result, Task, User +from argos.server.settings import read_config async def list_tasks(db: Session, agent_id: str, limit: int = 100): @@ -219,12 +220,50 @@ async def has_config_changed(db: Session, config: schemas.Config) -> bool: # py return True +async def update_from_config_later(db: Session, config_file): + """Ask Argos to reload configuration in a recurring task""" + jobs = ( + db.query(Job) + .filter( + Job.todo == "RELOAD_CONFIG", + Job.args == config_file, + Job.current == False, + ) + .all() + ) + if jobs: + return "There is already a config reloading job in the job queue, for the same file" + + job = Job(todo="RELOAD_CONFIG", args=config_file, added_at=datetime.now()) + db.add(job) + db.commit() + + return "Config reloading has been added in the job queue" + + +async def process_jobs(db: Session) -> int: + """Process job queue""" + jobs = db.query(Job).filter(Job.current == False).all() + if jobs: + for job in jobs: + job.current = True + db.commit() + if job.todo == "RELOAD_CONFIG": + logger.info("Processing job %i: %s %s", job.id, job.todo, job.args) + _config = read_config(job.args) + changed = await update_from_config(db, _config) + logger.info("%i task(s) added", changed["added"]) + logger.info("%i task(s) deleted", changed["vanished"]) + db.delete(job) + + db.commit() + return len(jobs) + + return 0 + + async def update_from_config(db: Session, config: schemas.Config): # pylint: disable-msg=too-many-branches """Update tasks from config file""" - config_changed = await has_config_changed(db, config) - if not config_changed: - return {"added": 0, "vanished": 0} - max_task_id = ( db.query(func.max(Task.id).label("max_id")).all() # pylint: disable-msg=not-callable )[0].max_id @@ -339,7 +378,8 @@ async def update_from_config(db: Session, config: schemas.Config): # pylint: di ) db.commit() logger.info( - "%i tasks has been removed since not in config file anymore", vanished_tasks + "%i task(s) has been removed since not in config file anymore", + vanished_tasks, ) return {"added": len(tasks), "vanished": vanished_tasks} diff --git a/argos/server/routes/dependencies.py b/argos/server/routes/dependencies.py index e61b77a..ed0399a 100644 --- a/argos/server/routes/dependencies.py +++ b/argos/server/routes/dependencies.py @@ -2,6 +2,8 @@ from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from fastapi_login import LoginManager +from argos.logging import logger + auth_scheme = HTTPBearer() @@ -33,12 +35,19 @@ async def verify_token( return token -async def find_ldap_user(config, ldap, user: str) -> str | None: +async def find_ldap_user(config, ldapobj, user: str) -> str | None: """Do a LDAP search for user and return its dn""" + import ldap import ldap.filter as ldap_filter from ldapurl import LDAP_SCOPE_SUBTREE - result = ldap.search_s( + try: + ldapobj.simple_bind_s(config.general.ldap.bind_dn, config.general.ldap.bind_pwd) + except ldap.LDAPError as err: # pylint: disable-msg=no-member + logger.error("LDAP error: %s", err) + return None + + result = ldapobj.search_s( config.general.ldap.user_tree, LDAP_SCOPE_SUBTREE, filterstr=ldap_filter.filter_format( diff --git a/argos/server/routes/views.py b/argos/server/routes/views.py index 3242972..3da2a6b 100644 --- a/argos/server/routes/views.py +++ b/argos/server/routes/views.py @@ -90,6 +90,15 @@ async def post_login( from ldap import INVALID_CREDENTIALS # pylint: disable-msg=no-name-in-module from argos.server.routes.dependencies import find_ldap_user + invalid_credentials = templates.TemplateResponse( + "login.html", + { + "request": request, + "msg": "Sorry, invalid username or bad password. " + "Or the LDAP server is unreachable (see logs to verify).", + }, + ) + ldap_dn = await find_ldap_user(config, request.app.state.ldap, username) if ldap_dn is None: return invalid_credentials diff --git a/argos/server/settings.py b/argos/server/settings.py index 7d26a49..25f3903 100644 --- a/argos/server/settings.py +++ b/argos/server/settings.py @@ -1,12 +1,26 @@ """Pydantic schemas for server""" +import sys from pathlib import Path import yaml from yamlinclude import YamlIncludeConstructor +from pydantic import ValidationError +from argos.logging import logger from argos.schemas.config import Config +def read_config(yaml_file): + try: + config = read_yaml_config(yaml_file) + return config + except ValidationError as err: + logger.error("Errors where found while reading configuration:") + for error in err.errors(): + logger.error("%s is %s", error["loc"], error["type"]) + sys.exit(1) + + def read_yaml_config(filename: str) -> Config: parsed = _load_yaml(filename) return Config(**parsed) diff --git a/docs/checks.md b/docs/checks.md index 8e961e2..82ee77e 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -82,6 +82,48 @@ caption: argos-config.yaml - json-is: '{"foo": "bar", "baz": 42}' ``` +## Add data to requests + +If you want to specify query parameters, just put them in the path: + +```{code-block} yaml +websites: + - domain: "https://contact.example.org" + paths: + - path: "/index.php?action=show_messages" + method: "GET" +``` + +If you want, for example, to test a form and send some data to it: + +```{code-block} yaml +websites: + - domain: "https://contact.example.org" + paths: + - path: "/" + method: "POST" + request_data: + # These are the data sent to the server: title and msg + data: + title: "Hello my friend" + msg: "How are you today?" + # To send data as JSON (optional, default is false): + is_json: true +``` + +If you need to send some headers in the request: + +```{code-block} yaml +websites: + - domain: "https://contact.example.org" + paths: + - path: "/api/mail" + method: "PUT" + request_data: + headers: + Authorization: "Bearer foo-bar-baz" +``` + ## SSL certificate expiration Checks that the SSL certificate will not expire soon. You need to define the thresholds in the configuration, and set the `on-check` option to enable the check. diff --git a/docs/cli.md b/docs/cli.md index a9aac5e..f04ddb4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -61,6 +61,8 @@ Options: --wait-time INTEGER Waiting time between two polls on the server (seconds) --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] + --user-agent TEXT A custom string to append to the User-Agent + header --help Show this message and exit. ``` @@ -82,7 +84,6 @@ Options: --help Show this message and exit. Commands: - cleandb Clean the database (to run routinely) generate-config Output a self-documented example config file. generate-token Generate a token for agents migrate Run database migrations @@ -93,7 +94,6 @@ Commands: test-gotify Send a test gotify notification test-mail Send a test email user User management - watch-agents Watch agents (to run routinely) ``` -### Server cleandb - - -```man -Usage: argos server cleandb [OPTIONS] - - Clean the database (to run routinely) - - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. - -Options: - --max-results INTEGER Number of results per task to keep - --max-lock-seconds INTEGER The number of seconds after which a lock is - considered stale, must be higher than 60 (the - checks have a timeout value of 60 seconds) - --config TEXT Path of the configuration file. If ARGOS_YAML_FILE - environment variable is set, its value will be - used instead. Default value: argos-config.yaml and - /etc/argos/config.yaml as fallback. - --help Show this message and exit. -``` - - - -### Server watch-agents - - - -```man -Usage: argos server cleandb [OPTIONS] - - Clean the database (to run routinely) - - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. - -Options: - --max-results INTEGER Number of results per task to keep - --max-lock-seconds INTEGER The number of seconds after which a lock is - considered stale, must be higher than 60 (the - checks have a timeout value of 60 seconds) - --config TEXT Path of the configuration file. If ARGOS_YAML_FILE - environment variable is set, its value will be - used instead. Default value: argos-config.yaml and - /etc/argos/config.yaml as fallback. - --help Show this message and exit. -``` - - - ### Server reload-config