diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e41199..6c9e25c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - ✨ — Add command to warn if it’s been long since last viewing an agent (fix #49) - 💥 — Change default config file path to argos-config.yaml (fix #36) - 📝 — New documentation URL: doc is now on https://argos-monitoring.framasoft.org/ +- 💥 — Remove env vars and only use the configuration file ## 0.1.1 diff --git a/argos/commands.py b/argos/commands.py index bc5cc92..ce518ff 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -243,15 +243,12 @@ async def reload_config(config): # The imports are made here otherwise the agent will need server configuration files. from argos.server import queries - from argos.server.main import get_application, read_config - from argos.server.settings import get_app_settings + from argos.server.main import read_config - appli = get_application() - settings = get_app_settings() - config = read_config(appli, settings) + _config = read_config(config) db = await get_db() - changed = await queries.update_from_config(db, config) + changed = await queries.update_from_config(db, _config) click.echo(f"{changed['added']} tasks added") click.echo(f"{changed['vanished']} tasks deleted") @@ -274,13 +271,13 @@ async def migrate(config): os.environ["ARGOS_YAML_FILE"] = config # The imports are made here otherwise the agent will need server configuration files. - from argos.server.settings import get_app_settings + from argos.server.settings import read_yaml_config - settings = get_app_settings() + settings = read_yaml_config(config) current_dir = Path(__file__).resolve().parent alembic_cfg = Config(current_dir / "server" / "migrations" / "alembic.ini") - alembic_cfg.set_main_option("sqlalchemy.url", settings.database_url) + alembic_cfg.set_main_option("sqlalchemy.url", str(settings.general.db.url)) command.upgrade(alembic_cfg, "head") diff --git a/argos/config-example.yaml b/argos/config-example.yaml index d36ad85..f45645a 100644 --- a/argos/config-example.yaml +++ b/argos/config-example.yaml @@ -1,4 +1,15 @@ general: + db: + # The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls + # Example for SQLite: "sqlite:////tmp/argos.db" + url: "postgresql://argos:argos@localhost/argos" + # You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?) + # See https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size for details + pool_size: 10 + max_overflow: 20 + # Can be "production", "dev", "test". + # If not present, default value is "production" + env: "production" frequency: "1m" # Run checks every minute. # Which way do you want to be warned when a check goes to that severity? # "local" emits a message in the server log diff --git a/argos/logging.py b/argos/logging.py index 071b003..2ba9c0e 100644 --- a/argos/logging.py +++ b/argos/logging.py @@ -3,7 +3,7 @@ import logging LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # Print level before message -logging.basicConfig(format="%(levelname)s: %(message)s") +logging.basicConfig(format="%(levelname)-9s %(message)s") # XXX We probably want different loggers for client and server. logger = logging.getLogger(__name__) diff --git a/argos/schemas/config.py b/argos/schemas/config.py index de69a05..dae487f 100644 --- a/argos/schemas/config.py +++ b/argos/schemas/config.py @@ -8,17 +8,27 @@ from pydantic import ( BaseModel, ConfigDict, HttpUrl, + PostgresDsn, StrictBool, EmailStr, PositiveInt, field_validator, ) from pydantic.functional_validators import BeforeValidator +from pydantic.networks import UrlConstraints +from pydantic_core import Url from typing_extensions import Annotated from argos.schemas.utils import string_to_duration Severity = Literal["warning", "error", "critical", "unknown"] +Environment = Literal["dev", "test", "production"] +SQLiteDsn = Annotated[ + Url, + UrlConstraints( + allowed_schemes=["sqlite"], + ), +] def parse_threshold(value): @@ -134,10 +144,18 @@ class GotifyUrl(BaseModel): tokens: List[str] +class DbSettings(BaseModel): + url: PostgresDsn | SQLiteDsn + pool_size: int = 10 + max_overflow: int = 20 + + class General(BaseModel): """Frequency for the checks and alerts""" frequency: int + db: DbSettings + env: Environment = "production" alerts: Alert mail: Optional[Mail] = None gotify: Optional[List[GotifyUrl]] = None diff --git a/argos/server/main.py b/argos/server/main.py index e6afac5..bf9cecb 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path @@ -9,20 +10,18 @@ from sqlalchemy.orm import sessionmaker from argos.logging import logger from argos.server import models, routes, queries -from argos.server.settings import get_app_settings, read_yaml_config +from argos.server.settings import read_yaml_config def get_application() -> FastAPI: """Spawn Argos FastAPI server""" - settings = get_app_settings() appli = FastAPI() + config_file = os.environ["ARGOS_YAML_FILE"] - config = read_config(appli, settings) + config = read_config(config_file) - # Settings is the pydantic settings object # Config is the argos config object (built from yaml) appli.state.config = config - appli.state.settings = settings appli.add_event_handler( "startup", @@ -79,10 +78,9 @@ def create_stop_app_handler(appli): return stop_app -def read_config(appli, settings): +def read_config(yaml_file): try: - config = read_yaml_config(settings.yaml_file) - appli.state.config = config + config = read_yaml_config(yaml_file) return config except ValidationError as err: logger.error("Errors where found while reading configuration:") @@ -92,25 +90,26 @@ def read_config(appli, settings): def setup_database(appli): - settings = appli.state.settings + config = appli.state.config + db_url = str(config.general.db.url) + logger.debug("Using database URL %s", db_url) # For sqlite, we need to add connect_args={"check_same_thread": False} - logger.debug("Using database URL %s", settings.database_url) - if settings.database_url.startswith("sqlite:////tmp"): + if config.general.env == "production" and db_url.startswith("sqlite:////tmp"): logger.warning("Using sqlite in /tmp is not recommended for production") extra_settings = {} - if settings.db_pool_size: - extra_settings.setdefault("pool_size", settings.db_pool_size) + if config.general.db.pool_size: + extra_settings.setdefault("pool_size", config.general.db.pool_size) - if settings.db_max_overflow: - extra_settings.setdefault("max_overflow", settings.db_max_overflow) + if config.general.db.max_overflow: + extra_settings.setdefault("max_overflow", config.general.db.max_overflow) - engine = create_engine(settings.database_url, **extra_settings) + engine = create_engine(db_url, **extra_settings) def _fk_pragma_on_connect(dbapi_con, con_record): dbapi_con.execute("pragma foreign_keys=ON") - if settings.database_url.startswith("sqlite:////"): + if db_url.startswith("sqlite:///"): event.listen(engine, "connect", _fk_pragma_on_connect) appli.state.SessionLocal = sessionmaker( diff --git a/argos/server/settings.py b/argos/server/settings.py index 01861cb..160e609 100644 --- a/argos/server/settings.py +++ b/argos/server/settings.py @@ -1,75 +1,12 @@ """Pydantic schemas for server""" -from functools import lru_cache -from os import environ from pathlib import Path -from typing import Optional, Union import yaml -from pydantic_settings import BaseSettings, SettingsConfigDict from yamlinclude import YamlIncludeConstructor from argos.schemas.config import Config -class Settings(BaseSettings): - model_config = SettingsConfigDict(env_prefix="argos_", env_file=".env") - app_env: str - database_url: str - yaml_file: str - db_pool_size: Optional[int] - db_max_overflow: Optional[int] - - -class DevSettings(Settings): - """Settings for dev environment. - - Uses argos-config.yaml as config file. - Uses a SQLite database.""" - - app_env: str = "dev" - db_pool_size: Optional[int] = None - db_max_overflow: Optional[int] = None - database_url: str = "sqlite:////tmp/argos.db" - - -class TestSettings(Settings): - """Settings for test environment. - - Uses tests/config.yaml as config file. - Uses a SQLite database.""" - - app_env: str = "test" - yaml_file: str = "tests/config.yaml" - database_url: str = "sqlite:////tmp/test-argos.db" - db_pool_size: Optional[int] = None - db_max_overflow: Optional[int] = None - - -class ProdSettings(Settings): - """Settings for prod environment.""" - - app_env: str = "prod" - db_pool_size: Optional[int] = 10 - db_max_overflow: Optional[int] = 20 - - -environments = { - "dev": DevSettings, - "prod": ProdSettings, - "test": TestSettings, -} - - -@lru_cache() -def get_app_settings() -> Union[None, Settings]: - """Load settings depending on the environment""" - app_env = environ.get("ARGOS_APP_ENV", "dev") - settings = environments.get(app_env) - if settings is not None: - return settings() - return None - - def read_yaml_config(filename): parsed = _load_yaml(filename) return Config(**parsed) diff --git a/conf/default-argos-server b/conf/default-argos-server index 6411fc6..724b42e 100644 --- a/conf/default-argos-server +++ b/conf/default-argos-server @@ -1,5 +1,4 @@ ARGOS_YAML_FILE="/etc/argos/config.yaml" -ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos" ARGOS_SERVER_WORKERS=4 ARGOS_SERVER_SOCKET=127.0.0.1:8000 # Comma separated list of IP addresses of the web proxy (usually Nginx) diff --git a/docs/cli.md b/docs/cli.md index d976b5e..194baf9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -191,7 +191,8 @@ Options: checks have a timeout value of 60 seconds) --config TEXT Path of the configuration file. If ARGOS_YAML_FILE environment variable is set, its value will be - used instead. + used instead. Default value: argos-config.yaml and + /etc/argos/config.yaml as fallback. --help Show this message and exit. ``` diff --git a/docs/conf.py b/docs/conf.py index a6dffa6..b0a05bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,6 +5,7 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +# pylint: disable-msg=invalid-name,redefined-builtin import argos project = "Argos" diff --git a/docs/configuration.md b/docs/configuration.md index 8d93f1a..4c819a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,51 +1,6 @@ # Configuration -There are actually two configuration files: one for the service and one for the checks. - -## Server configuration - -The server configuration is done using environment variables. You can put them in a `.env` file at the root of the project. -Here is a list of the useful variables, in the `.env` format: - -```{literalinclude} ../conf/.env.example ---- -caption: .env ---- -``` - -### Environment variables - -Here are the environment variables you can define to configure how the service will behave : - -#### ARGOS_YAML_FILE - -The path to the yaml configuration file, defining the checks. - -If you do not provide the environment variable, Argos will try to read `argos-config.yaml` in the current directory, then `/etc/config/argos.yaml`. - -#### ARGOS_DATABASE_URL - -The database url, as defined [in SQLAlchemy docs](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls). - -For instance, to connect to a postgres database on localhost with user, pass and dbname "argos": - -``` -ARGOS_DATABASE_URL="postgresql://argos:argos@localhost/argos" -``` - -#### DB_POOL_SIZE -#### DB_MAX_OVERFLOW - -You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?) These are documented [in the SQLAlchemy docs in greater details](https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size) - -```bash -DB_POOL_SIZE=10 -DB_MAX_OVERFLOW=20 -``` - -## Argos "checks" configuration - -Argos uses a YAML configuration file to define the websites to monitor and the checks to run on these websites. +Argos uses a simple YAML configuration file to define the server’s configuration, the websites to monitor and the checks to run on these websites. Here is a simple configuration file: @@ -54,5 +9,4 @@ Here is a simple configuration file: --- caption: argos-config.yaml --- - ``` diff --git a/docs/installation/getting-started.md b/docs/installation/getting-started.md index 6e498e6..b9876eb 100644 --- a/docs/installation/getting-started.md +++ b/docs/installation/getting-started.md @@ -7,6 +7,21 @@ NB: if you want a quick-installation guide, we [got you covered](tl-dr.md). - Python 3.11+ - PostgreSQL 13+ (for production) +## Recommendation + +Create a dedicated user for argos: + +```bash +adduser --home /opt/argos --disabled-login --disabled-password --system argos +``` + +Do all the manipulations below in `/opt/argos/`, with the user `argos`. +Either use `sudo` or login as `argos` with the following command: + +```bash +su argos -s /bin/bash +``` + ## Install with pip ```bash @@ -24,7 +39,7 @@ pip install argos-monitoring For production, we recommend the use of [Gunicorn](https://gunicorn.org/), which you can install at the same time as Argos: ```bash -pip install argos-monitoring[gunicorn] +pip install "argos-monitoring[gunicorn]" ``` ## Install from sources @@ -37,6 +52,8 @@ source venv/bin/activate pip install -e . ``` +To install gunicorn, use `pip install -e ".[gunicorn]"` instead of `pip install -e .` + ## Configure The quickest way to get started is to generate the configuration file from argos and edit it: @@ -47,14 +64,18 @@ argos server generate-config > argos-config.yaml You can read more about the configuration in the [configuration section](../configuration.md). -### Configure the server +For production, we suggest to put your config in `/etc/argos/config.yaml` and restricts the file’s permissions. +As root: +```bash +mkdir /etc/argos +chown argos: /etc/argos +chmod 700 /etc/argos +``` -Environment variables are used to configure the server. You can also put them in an `.env` file: - -```{literalinclude} ../../conf/.env.example ---- -caption: .env ---- +Then, as `argos`: +```bash +argos server generate-config > /etc/argos/config.yaml +chmod 600 /etc/argos/config.yaml ``` Please note that the only supported database engines are SQLite for development and PostgreSQL for production. @@ -69,7 +90,7 @@ argos server migrate ## Inject tasks into the database -Argos keeps tasks’ configuration in database, take from the config file. +Argos keeps tasks’ configuration in database, taken from the config file. Populate the database with the tasks: @@ -85,8 +106,6 @@ Then you can start the server: argos server start ``` -The server reads the `yaml` file at startup, and populates the tasks queue with the checks defined in the configuration. - This way to start the server is not suitable for production, use it only for developing or testing. ## Starting the server for production @@ -96,7 +115,7 @@ For production, you can use [Gunicorn](https://gunicorn.org/) to start the serve To install Gunicorn in the virtualenv, if you didn’t already install Argos that way: ```bash -pip install argos-monitoring[gunicorn] +pip install "argos-monitoring[gunicorn]" ``` To start the server: @@ -121,6 +140,8 @@ Gunicorn has a lot of other options, have a look at `gunicorn --help`. Argos uses FastAPI, so you can use other ways to start the server. See (but Gunicorn is recommended). +See [here](../deployment/systemd.md#server) for a systemd service example and [here](../deployment/nginx.md) for a nginx configuration example. + ## Generating a token The agent needs an authentication token to be able to communicate with the server. @@ -141,7 +162,7 @@ service: ## Running the agent You can run the agent on the same machine as the server, or on a different machine. -The only requirement is that the agent can reach the server. +The only requirement is that the agent can reach the server through HTTP or HTTPS. ```bash argos agent http://localhost:8000 "auth-token" diff --git a/docs/installation/tl-dr.md b/docs/installation/tl-dr.md index 9a1e356..88c1981 100644 --- a/docs/installation/tl-dr.md +++ b/docs/installation/tl-dr.md @@ -13,13 +13,17 @@ cd /tmp/argos python3 -m venv venv source venv/bin/activate pip install argos-monitoring -argos server generate-config > argos-config.yaml +argos server generate-config | + sed -e "s@production@test@" \ + -e "s@url: .postgresql.*@url: \"sqlite:////tmp/argos.db\"@" > argos-config.yaml argos server migrate -argos server generate-token +ARGOS_TOKEN=$(argos server generate-token) +sed -e "s@# - secret_token@- $ARGOS_TOKEN@" -i argos-config.yaml +echo "The agent token is $ARGOS_TOKEN" ``` Edit `argos-config.yaml`. -Add the generated token in it, as long as some real web sites to test. +Add some real web sites to test. Then: @@ -48,16 +52,14 @@ sudo -u postgres psql -c "ALTER DATABASE argos SET TIMEZONE TO 'UTC';" adduser --home /opt/argos --disabled-login --disabled-password --system argos cd /opt/argos -python3 -m venv venv -chown argos: -R venv -sudo -u argos bash -c "source venv/bin/activate && pip install argos-monitoring[gunicorn]" +sudo -u argos python3 -m venv venv +sudo -u argos bash -c 'source venv/bin/activate && pip install "argos-monitoring[gunicorn]"' mkdir /etc/argos /opt/argos/venv/bin/argos server generate-config > /etc/argos/config.yaml cat < /etc/default/argos-server ARGOS_YAML_FILE="/etc/argos/config.yaml" -ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos" ARGOS_SERVER_WORKERS=4 ARGOS_SERVER_SOCKET=127.0.0.1:8000 # Comma separated list of IP addresses of the web proxy (usually Nginx) @@ -119,21 +121,18 @@ WantedBy=multi-user.target EOF chown -R argos: /etc/default/argos-* /etc/argos/ +chmod 700 /etc/argos +chmod 600 /etc/argos/config.yaml systemctl daemon-reload ``` -Then, edit `/etc/default/argos-server` to put your database password in it and change the other settings to suit your needs. +Then, edit `/etc/argos/config.yaml` to put your database password in it and change the other settings to suit your needs. Create a token for your agent : ```bash -su - argos -s /bin/bash -. /etc/default/argos-server -export ARGOS_DATABASE_URL -export ARGOS_YAML_FILE -/opt/argos/venv/bin/argos server generate-token -exit +sudo -u argos /opt/argos/venv/bin/argos server generate-token ``` Edit `/etc/default/argos-agent` to put the generated token in it and change the other settings to suit your needs. @@ -150,7 +149,7 @@ systemctl status argos-server.service argos-agent.service If all works well, you have to put some cron tasks in `argos` crontab: ```bash -echo -e "*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos - +echo -e "*/10 * * * * /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos - ``` See the [this page](../deployment/nginx.md) for using Nginx as reverse proxy. diff --git a/tests/config.yaml b/tests/config.yaml index d248e88..1242298 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -1,4 +1,8 @@ general: + db: + # The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls + url: "sqlite:////tmp/test-argos.db" + env: test frequency: "1m" alerts: ok: diff --git a/tests/conftest.py b/tests/conftest.py index d07ea1a..9b6f3aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy.orm import Session -os.environ["ARGOS_APP_ENV"] = "test" +os.environ["ARGOS_YAML_FILE"] = "tests/config.yaml" @pytest.fixture @@ -45,10 +45,6 @@ def _create_app() -> FastAPI: ) app = get_application() - # Hardcode the database url and the yaml file for testing purpose - # Otherwise, the app will try to read the .env file or the environment variables - app.state.settings.database_url = "sqlite:////tmp/test-argos.db" - app.state.settings.yaml_file = "tests/config.yaml" setup_database(app) asyncio.run(connect_to_db(app)) diff --git a/tests/test_queries.py b/tests/test_queries.py index 0fc7c26..0300b23 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -172,6 +172,7 @@ def task(db): def empty_config(): return schemas.config.Config( general=schemas.config.General( + db=schemas.config.DbSettings(url="sqlite:////tmp/test-argos.db"), frequency="1m", alerts=schemas.config.Alert( ok=["", ""],