diff --git a/CHANGELOG.md b/CHANGELOG.md index 991756a..deabd01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - ✨ — Allow to customize agent User-Agent header (#78) - 📝 — Document how to add data to requests (#77) +- ✨ — No need cron tasks for DB cleaning anymore (#74 and #75) ## 0.7.4 diff --git a/argos/commands.py b/argos/commands.py index 97c244e..3ac9555 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -140,73 +140,19 @@ def start(host, port, config, reload): uvicorn.run("argos.server:app", host=host, port=port, reload=reload) -def validate_max_lock_seconds(ctx, param, value): - if value <= 60: - raise click.BadParameter("Should be strictly higher than 60") - return value - - -def validate_max_results(ctx, param, value): +def validate_time_without_agent(ctx, param, value): if value <= 0: raise click.BadParameter("Should be a positive integer") return value -@server.command() -@click.option( - "--max-results", - default=100, - help="Number of results per task to keep", - callback=validate_max_results, -) -@click.option( - "--max-lock-seconds", - default=100, - help=( - "The number of seconds after which a lock is " - "considered stale, must be higher than 60 " - "(the checks have a timeout value of 60 seconds)" - ), - callback=validate_max_lock_seconds, -) -@click.option( - "--config", - default="argos-config.yaml", - help="Path of the configuration file. " - "If ARGOS_YAML_FILE environment variable is set, its value will be used instead. " - "Default value: argos-config.yaml and /etc/argos/config.yaml as fallback.", - envvar="ARGOS_YAML_FILE", - callback=validate_config_access, -) -@coroutine -async def cleandb(max_results, max_lock_seconds, config): - """Clean the database (to run routinely) - - \b - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. - """ - # It’s mandatory to do it before the imports - os.environ["ARGOS_YAML_FILE"] = config - - # The imports are made here otherwise the agent will need server configuration files. - from argos.server import queries - - db = await get_db() - removed = await queries.remove_old_results(db, max_results) - updated = await queries.release_old_locks(db, max_lock_seconds) - - click.echo(f"{removed} results removed") - click.echo(f"{updated} locks released") - - @server.command() @click.option( "--time-without-agent", default=5, help="Time without seeing an agent after which a warning will be issued, in minutes. " "Default is 5 minutes.", - callback=validate_max_results, + callback=validate_time_without_agent, ) @click.option( "--config", diff --git a/argos/config-example.yaml b/argos/config-example.yaml index 5508bff..2008a3e 100644 --- a/argos/config-example.yaml +++ b/argos/config-example.yaml @@ -81,6 +81,12 @@ general: # To disable the IPv6 check of domains: # ipv6: false + # Argos root path + # If not present, default value is "" + # Set it to /foo if you want to use argos at /foo/ instead of / + # on your web server + # root_path: "/foo" + # Which way do you want to be warned when a check goes to that severity? # "local" emits a message in the server log # You’ll need to configure mail, gotify or apprise below to be able to use @@ -96,11 +102,6 @@ general: - local unknown: - local - # Argos root path - # If not present, default value is "" - # Set it to /foo if you want to use argos at /foo/ instead of / - # on your web server - # root_path: "/foo" # Mail configuration is quite straight-forward # mail: # mailfrom: no-reply@example.org @@ -144,6 +145,16 @@ ssl: - "1d": critical - "5d": warning +# Argos will do some cleaning in the background for you +# every 2 minutes and needs some configuration for that +cleaning: + # Max number of results per tasks you want to keep + # Minimum value is 1, default is 100 + max_results: 100 + # Max number of seconds a task can be locked + # Minimum value is 61, default is 100 + max_lock_seconds: 100 + # It's also possible to define the checks in another file # with the include syntax: # diff --git a/argos/logging.py b/argos/logging.py index 5aba551..e67bfb9 100644 --- a/argos/logging.py +++ b/argos/logging.py @@ -14,9 +14,10 @@ logger = logging.getLogger(__name__) # XXX Does not work ? -def set_log_level(log_level): +def set_log_level(log_level: str, quiet: bool = False): level = getattr(logging, log_level.upper(), None) if not isinstance(level, int): raise ValueError(f"Invalid log level: {log_level}") logger.setLevel(level=level) - logger.info("Log level set to %s", log_level) + if not quiet: + logger.info("Log level set to %s", log_level) diff --git a/argos/schemas/config.py b/argos/schemas/config.py index 1baa7ed..68dd3ca 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -48,6 +48,27 @@ class SSL(BaseModel): thresholds: List[Annotated[Tuple[int, Severity], BeforeValidator(parse_threshold)]] +class Cleaning(BaseModel): + max_results: int + max_lock_seconds: int + + @field_validator("max_results", mode="before") + def parse_max_results(cls, value): + """Ensure that max_results is higher than 0""" + if value >= 1: + return value + + return 100 + + @field_validator("max_lock_seconds", mode="before") + def parse_max_lock_seconds(cls, value): + """Ensure that max_lock_seconds is higher or equal to agent’s requests timeout (60)""" + if value > 60: + return value + + return 100 + + class WebsiteCheck(BaseModel): key: str value: str | List[str] | Dict[str, str] @@ -264,4 +285,5 @@ class Config(BaseModel): general: General service: Service ssl: SSL + cleaning: Cleaning websites: List[Website] diff --git a/argos/server/main.py b/argos/server/main.py index 0543182..28dd21f 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -6,11 +6,12 @@ from pathlib import Path from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from fastapi_login import LoginManager +from fastapi_utils.tasks import repeat_every from pydantic import ValidationError from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker -from argos.logging import logger +from argos.logging import logger, set_log_level from argos.server import models, routes, queries from argos.server.exceptions import NotAuthenticatedException, auth_exception_handler from argos.server.settings import read_yaml_config @@ -126,8 +127,24 @@ def create_manager(cookie_secret: str) -> LoginManager: ) +@repeat_every(seconds=120, logger=logger) +async def cleanup() -> None: + set_log_level("info", quiet=True) + logger.info("Start DB cleanup tasks.") + with app.state.SessionLocal() as db: + removed = await queries.remove_old_results( + db, app.state.config.cleaning.max_results + ) + updated = await queries.release_old_locks( + db, app.state.config.cleaning.max_lock_seconds + ) + + logger.info("%i results removed", removed) + logger.info("%i locks released", updated) + + @asynccontextmanager -async def lifespan(appli): +async def lifespan(appli: FastAPI): """Server start and stop actions Setup database connection then close it at shutdown. @@ -142,6 +159,7 @@ async def lifespan(appli): "There is no tasks in the database. " 'Please launch the command "argos server reload-config"' ) + await cleanup() yield diff --git a/docs/cli.md b/docs/cli.md index e524a8c..ffccc03 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -84,7 +84,6 @@ Options: --help Show this message and exit. Commands: - cleandb Clean the database (to run routinely) generate-config Output a self-documented example config file. generate-token Generate a token for agents migrate Run database migrations @@ -152,60 +151,28 @@ Options: --> -### Server cleandb - - -```man -Usage: argos server cleandb [OPTIONS] - - Clean the database (to run routinely) - - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. - -Options: - --max-results INTEGER Number of results per task to keep - --max-lock-seconds INTEGER The number of seconds after which a lock is - considered stale, must be higher than 60 (the - checks have a timeout value of 60 seconds) - --config TEXT Path of the configuration file. If ARGOS_YAML_FILE - environment variable is set, its value will be - used instead. Default value: argos-config.yaml and - /etc/argos/config.yaml as fallback. - --help Show this message and exit. -``` - - - ### Server watch-agents ```man -Usage: argos server cleandb [OPTIONS] +Usage: argos server watch-agents [OPTIONS] - Clean the database (to run routinely) + Watch agents (to run routinely) - - Removes old results from the database. - - Removes locks from tasks that have been locked for too long. + Issues a warning if no agent has been seen by the server for a given time. Options: - --max-results INTEGER Number of results per task to keep - --max-lock-seconds INTEGER The number of seconds after which a lock is - considered stale, must be higher than 60 (the - checks have a timeout value of 60 seconds) - --config TEXT Path of the configuration file. If ARGOS_YAML_FILE - environment variable is set, its value will be - used instead. Default value: argos-config.yaml and - /etc/argos/config.yaml as fallback. - --help Show this message and exit. + --time-without-agent INTEGER Time without seeing an agent after which a + warning will be issued, in minutes. Default is 5 + minutes. + --config TEXT Path of the configuration file. If + ARGOS_YAML_FILE environment variable is set, its + value will be used instead. + --help Show this message and exit. ```