💥 — Remove env vars and only use the configuration file (fix #47)

CHANGELOG.md

@@ -10,6 +10,7 @@
 - ✨ — Add command to warn if it’s been long since last viewing an agent (fix #49)
 - 💥 — Change default config file path to argos-config.yaml (fix #36)
 - 📝 — New documentation URL: doc is now on https://argos-monitoring.framasoft.org/
+- 💥 — Remove env vars and only use the configuration file
 
 ## 0.1.1
 

argos/commands.py

@@ -243,15 +243,12 @@ async def reload_config(config):
 
     # The imports are made here otherwise the agent will need server configuration files.
     from argos.server import queries
-    from argos.server.main import get_application, read_config
+    from argos.server.main import read_config
-    from argos.server.settings import get_app_settings
 
-    appli = get_application()
+    _config = read_config(config)
-    settings = get_app_settings()
-    config = read_config(appli, settings)
 
     db = await get_db()
-    changed = await queries.update_from_config(db, config)
+    changed = await queries.update_from_config(db, _config)
 
     click.echo(f"{changed['added']} tasks added")
     click.echo(f"{changed['vanished']} tasks deleted")
@@ -274,13 +271,13 @@ async def migrate(config):
     os.environ["ARGOS_YAML_FILE"] = config
 
     # The imports are made here otherwise the agent will need server configuration files.
-    from argos.server.settings import get_app_settings
+    from argos.server.settings import read_yaml_config
 
-    settings = get_app_settings()
+    settings = read_yaml_config(config)
 
     current_dir = Path(__file__).resolve().parent
     alembic_cfg = Config(current_dir / "server" / "migrations" / "alembic.ini")
-    alembic_cfg.set_main_option("sqlalchemy.url", settings.database_url)
+    alembic_cfg.set_main_option("sqlalchemy.url", str(settings.general.db.url))
     command.upgrade(alembic_cfg, "head")
 
 

argos/config-example.yaml

@@ -1,4 +1,15 @@
 general:
+  db:
+    # The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls
+    # Example for SQLite: "sqlite:////tmp/argos.db"
+    url: "postgresql://argos:argos@localhost/argos"
+    # You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?)
+    # See https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size for details
+    pool_size: 10
+    max_overflow: 20
+  # Can be "production", "dev", "test".
+  # If not present, default value is "production"
+  env: "production"
   frequency: "1m" # Run checks every minute.
   # Which way do you want to be warned when a check goes to that severity?
   # "local" emits a message in the server log

argos/logging.py

@@ -3,7 +3,7 @@ import logging
 LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 
 # Print level before message
-logging.basicConfig(format="%(levelname)s: %(message)s")
+logging.basicConfig(format="%(levelname)-9s %(message)s")
 
 # XXX We probably want different loggers for client and server.
 logger = logging.getLogger(__name__)

argos/schemas/config.py

@@ -8,17 +8,27 @@ from pydantic import (
     BaseModel,
     ConfigDict,
     HttpUrl,
+    PostgresDsn,
     StrictBool,
     EmailStr,
     PositiveInt,
     field_validator,
 )
 from pydantic.functional_validators import BeforeValidator
+from pydantic.networks import UrlConstraints
+from pydantic_core import Url
 from typing_extensions import Annotated
 
 from argos.schemas.utils import string_to_duration
 
 Severity = Literal["warning", "error", "critical", "unknown"]
+Environment = Literal["dev", "test", "production"]
+SQLiteDsn = Annotated[
+    Url,
+    UrlConstraints(
+        allowed_schemes=["sqlite"],
+    ),
+]
 
 
 def parse_threshold(value):
@@ -134,10 +144,18 @@ class GotifyUrl(BaseModel):
     tokens: List[str]
 
 
+class DbSettings(BaseModel):
+    url: PostgresDsn | SQLiteDsn
+    pool_size: int = 10
+    max_overflow: int = 20
+
+
 class General(BaseModel):
     """Frequency for the checks and alerts"""
 
     frequency: int
+    db: DbSettings
+    env: Environment = "production"
     alerts: Alert
     mail: Optional[Mail] = None
     gotify: Optional[List[GotifyUrl]] = None

argos/server/main.py

@@ -1,3 +1,4 @@
+import os
 import sys
 from pathlib import Path
 
@@ -9,20 +10,18 @@ from sqlalchemy.orm import sessionmaker
 
 from argos.logging import logger
 from argos.server import models, routes, queries
-from argos.server.settings import get_app_settings, read_yaml_config
+from argos.server.settings import read_yaml_config
 
 
 def get_application() -> FastAPI:
     """Spawn Argos FastAPI server"""
-    settings = get_app_settings()
     appli = FastAPI()
+    config_file = os.environ["ARGOS_YAML_FILE"]
 
-    config = read_config(appli, settings)
+    config = read_config(config_file)
 
-    # Settings is the pydantic settings object
     # Config is the argos config object (built from yaml)
     appli.state.config = config
-    appli.state.settings = settings
 
     appli.add_event_handler(
         "startup",
@@ -79,10 +78,9 @@ def create_stop_app_handler(appli):
     return stop_app
 
 
-def read_config(appli, settings):
+def read_config(yaml_file):
     try:
-        config = read_yaml_config(settings.yaml_file)
+        config = read_yaml_config(yaml_file)
-        appli.state.config = config
         return config
     except ValidationError as err:
         logger.error("Errors where found while reading configuration:")
@@ -92,25 +90,26 @@ def read_config(appli, settings):
 
 
 def setup_database(appli):
-    settings = appli.state.settings
+    config = appli.state.config
+    db_url = str(config.general.db.url)
+    logger.debug("Using database URL %s", db_url)
     # For sqlite, we need to add connect_args={"check_same_thread": False}
-    logger.debug("Using database URL %s", settings.database_url)
+    if config.general.env == "production" and db_url.startswith("sqlite:////tmp"):
-    if settings.database_url.startswith("sqlite:////tmp"):
         logger.warning("Using sqlite in /tmp is not recommended for production")
 
     extra_settings = {}
-    if settings.db_pool_size:
+    if config.general.db.pool_size:
-        extra_settings.setdefault("pool_size", settings.db_pool_size)
+        extra_settings.setdefault("pool_size", config.general.db.pool_size)
 
-    if settings.db_max_overflow:
+    if config.general.db.max_overflow:
-        extra_settings.setdefault("max_overflow", settings.db_max_overflow)
+        extra_settings.setdefault("max_overflow", config.general.db.max_overflow)
 
-    engine = create_engine(settings.database_url, **extra_settings)
+    engine = create_engine(db_url, **extra_settings)
 
     def _fk_pragma_on_connect(dbapi_con, con_record):
         dbapi_con.execute("pragma foreign_keys=ON")
 
-    if settings.database_url.startswith("sqlite:////"):
+    if db_url.startswith("sqlite:///"):
         event.listen(engine, "connect", _fk_pragma_on_connect)
 
     appli.state.SessionLocal = sessionmaker(

argos/server/settings.py

@@ -1,75 +1,12 @@
 """Pydantic schemas for server"""
-from functools import lru_cache
-from os import environ
 from pathlib import Path
-from typing import Optional, Union
 
 import yaml
-from pydantic_settings import BaseSettings, SettingsConfigDict
 from yamlinclude import YamlIncludeConstructor
 
 from argos.schemas.config import Config
 
 
-class Settings(BaseSettings):
-    model_config = SettingsConfigDict(env_prefix="argos_", env_file=".env")
-    app_env: str
-    database_url: str
-    yaml_file: str
-    db_pool_size: Optional[int]
-    db_max_overflow: Optional[int]
-
-
-class DevSettings(Settings):
-    """Settings for dev environment.
-
-    Uses argos-config.yaml as config file.
-    Uses a SQLite database."""
-
-    app_env: str = "dev"
-    db_pool_size: Optional[int] = None
-    db_max_overflow: Optional[int] = None
-    database_url: str = "sqlite:////tmp/argos.db"
-
-
-class TestSettings(Settings):
-    """Settings for test environment.
-
-    Uses tests/config.yaml as config file.
-    Uses a SQLite database."""
-
-    app_env: str = "test"
-    yaml_file: str = "tests/config.yaml"
-    database_url: str = "sqlite:////tmp/test-argos.db"
-    db_pool_size: Optional[int] = None
-    db_max_overflow: Optional[int] = None
-
-
-class ProdSettings(Settings):
-    """Settings for prod environment."""
-
-    app_env: str = "prod"
-    db_pool_size: Optional[int] = 10
-    db_max_overflow: Optional[int] = 20
-
-
-environments = {
-    "dev": DevSettings,
-    "prod": ProdSettings,
-    "test": TestSettings,
-}
-
-
-@lru_cache()
-def get_app_settings() -> Union[None, Settings]:
-    """Load settings depending on the environment"""
-    app_env = environ.get("ARGOS_APP_ENV", "dev")
-    settings = environments.get(app_env)
-    if settings is not None:
-        return settings()
-    return None
-
-
 def read_yaml_config(filename):
     parsed = _load_yaml(filename)
     return Config(**parsed)

conf/default-argos-server

@@ -1,5 +1,4 @@
 ARGOS_YAML_FILE="/etc/argos/config.yaml"
-ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos"
 ARGOS_SERVER_WORKERS=4
 ARGOS_SERVER_SOCKET=127.0.0.1:8000
 # Comma separated list of IP addresses of the web proxy (usually Nginx)

docs/cli.md

@@ -191,7 +191,8 @@ Options:
                               checks have a timeout value of 60 seconds)
   --config TEXT               Path of the configuration file. If ARGOS_YAML_FILE
                               environment variable is set, its value will be
-                              used instead.
+                              used instead. Default value: argos-config.yaml and
+                              /etc/argos/config.yaml as fallback.
   --help                      Show this message and exit.
 ```
 

docs/conf.py

@@ -5,6 +5,7 @@
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+# pylint: disable-msg=invalid-name,redefined-builtin
 import argos
 
 project = "Argos"

docs/configuration.md

@@ -1,51 +1,6 @@
 # Configuration
 
-There are actually two configuration files: one for the service and one for the checks.
+Argos uses a simple YAML configuration file to define the server’s configuration, the websites to monitor and the checks to run on these websites.
-
-## Server configuration
-
-The server configuration is done using environment variables. You can put them in a `.env` file at the root of the project.
-Here is a list of the useful variables, in the `.env` format:
-
-```{literalinclude} ../conf/.env.example
----
-caption: .env
----
-```
-
-### Environment variables
-
-Here are the environment variables you can define to configure how the service will behave :
-
-#### ARGOS_YAML_FILE
-
-The path to the yaml configuration file, defining the checks.
-
-If you do not provide the environment variable, Argos will try to read `argos-config.yaml` in the current directory, then `/etc/config/argos.yaml`.
-
-#### ARGOS_DATABASE_URL
-
-The database url, as defined [in SQLAlchemy docs](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls).
-
-For instance, to connect to a postgres database on localhost with user, pass and dbname "argos":
-
-```
-ARGOS_DATABASE_URL="postgresql://argos:argos@localhost/argos"
-```
-
-#### DB_POOL_SIZE
-#### DB_MAX_OVERFLOW
-
-You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?) These are documented [in the SQLAlchemy docs in greater details](https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size)
-
-```bash
-DB_POOL_SIZE=10
-DB_MAX_OVERFLOW=20
-```
-
-## Argos "checks" configuration
-
-Argos uses a YAML configuration file to define the websites to monitor and the checks to run on these websites.
 
 Here is a simple configuration file:
 
@@ -54,5 +9,4 @@ Here is a simple configuration file:
 ---
 caption: argos-config.yaml
 ---
-
 ```

docs/installation/getting-started.md

@@ -7,6 +7,21 @@ NB: if you want a quick-installation guide, we [got you covered](tl-dr.md).
 - Python 3.11+
 - PostgreSQL 13+ (for production)
 
+## Recommendation
+
+Create a dedicated user for argos:
+
+```bash
+adduser --home /opt/argos --disabled-login --disabled-password --system argos
+```
+
+Do all the manipulations below in `/opt/argos/`, with the user `argos`.
+Either use `sudo` or login as `argos` with the following command:
+
+```bash
+su argos -s /bin/bash
+```
+
 ## Install with pip
 
 ```bash
@@ -24,7 +39,7 @@ pip install argos-monitoring
 For production, we recommend the use of [Gunicorn](https://gunicorn.org/), which you can install at the same time as Argos:
 
 ```bash
-pip install argos-monitoring[gunicorn]
+pip install "argos-monitoring[gunicorn]"
 ```
 
 ## Install from sources
@@ -37,6 +52,8 @@ source venv/bin/activate
 pip install -e .
 ```
 
+To install gunicorn, use `pip install -e ".[gunicorn]"` instead of `pip install -e .`
+
 ## Configure
 
 The quickest way to get started is to generate the configuration file from argos and edit it:
@@ -47,14 +64,18 @@ argos server generate-config > argos-config.yaml
 
 You can read more about the configuration in the [configuration section](../configuration.md).
 
-### Configure the server
+For production, we suggest to put your config in `/etc/argos/config.yaml` and restricts the file’s permissions.
+As root:
+```bash
+mkdir /etc/argos
+chown argos: /etc/argos
+chmod 700 /etc/argos
+```
 
-Environment variables are used to configure the server. You can also put them in an `.env` file:
+Then, as `argos`:
-
+```bash
-```{literalinclude} ../../conf/.env.example
+argos server generate-config > /etc/argos/config.yaml
----
+chmod 600 /etc/argos/config.yaml
-caption: .env
----
 ```
 
 Please note that the only supported database engines are SQLite for development and PostgreSQL for production.
@@ -69,7 +90,7 @@ argos server migrate
 
 ## Inject tasks into the database
 
-Argos keeps tasks’ configuration in database, take from the config file.
+Argos keeps tasks’ configuration in database, taken from the config file.
 
 Populate the database with the tasks:
 
@@ -85,8 +106,6 @@ Then you can start the server:
 argos server start
 ```
 
-The server reads the `yaml` file at startup, and populates the tasks queue with the checks defined in the configuration.
-
 This way to start the server is not suitable for production, use it only for developing or testing.
 
 ## Starting the server for production
@@ -96,7 +115,7 @@ For production, you can use [Gunicorn](https://gunicorn.org/) to start the serve
 To install Gunicorn in the virtualenv, if you didn’t already install Argos that way:
 
 ```bash
-pip install argos-monitoring[gunicorn]
+pip install "argos-monitoring[gunicorn]"
 ```
 
 To start the server:
@@ -121,6 +140,8 @@ Gunicorn has a lot of other options, have a look at `gunicorn --help`.
 Argos uses FastAPI, so you can use other ways to start the server.
 See <https://fastapi.tiangolo.com/deployment/manually/#asgi-servers> (but Gunicorn is recommended).
 
+See [here](../deployment/systemd.md#server) for a systemd service example and [here](../deployment/nginx.md) for a nginx configuration example.
+
 ## Generating a token 
 
 The agent needs an authentication token to be able to communicate with the server.
@@ -141,7 +162,7 @@ service:
 ## Running the agent
 
 You can run the agent on the same machine as the server, or on a different machine.
-The only requirement is that the agent can reach the server.
+The only requirement is that the agent can reach the server through HTTP or HTTPS.
 
 ```bash
 argos agent http://localhost:8000 "auth-token"

docs/installation/tl-dr.md

@@ -13,13 +13,17 @@ cd /tmp/argos
 python3 -m venv venv
 source venv/bin/activate
 pip install argos-monitoring
-argos server generate-config > argos-config.yaml
+argos server generate-config |
+    sed -e "s@production@test@" \
+        -e "s@url: .postgresql.*@url: \"sqlite:////tmp/argos.db\"@" > argos-config.yaml
 argos server migrate
-argos server generate-token
+ARGOS_TOKEN=$(argos server generate-token)
+sed -e "s@# - secret_token@- $ARGOS_TOKEN@" -i argos-config.yaml
+echo "The agent token is $ARGOS_TOKEN"
 ```
 
 Edit `argos-config.yaml`.
-Add the generated token in it, as long as some real web sites to test.
+Add some real web sites to test.
 
 Then:
 
@@ -48,16 +52,14 @@ sudo -u postgres psql -c "ALTER DATABASE argos SET TIMEZONE TO 'UTC';"
 adduser --home /opt/argos --disabled-login --disabled-password --system argos
 
 cd /opt/argos
-python3 -m venv venv
+sudo -u argos python3 -m venv venv
-chown argos: -R venv
+sudo -u argos bash -c 'source venv/bin/activate && pip install "argos-monitoring[gunicorn]"'
-sudo -u argos bash -c "source venv/bin/activate && pip install argos-monitoring[gunicorn]"
 
 mkdir /etc/argos
 /opt/argos/venv/bin/argos server generate-config > /etc/argos/config.yaml
 
 cat <<EOF > /etc/default/argos-server
 ARGOS_YAML_FILE="/etc/argos/config.yaml"
-ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos"
 ARGOS_SERVER_WORKERS=4
 ARGOS_SERVER_SOCKET=127.0.0.1:8000
 # Comma separated list of IP addresses of the web proxy (usually Nginx)
@@ -119,21 +121,18 @@ WantedBy=multi-user.target
 EOF
 
 chown -R argos: /etc/default/argos-* /etc/argos/
+chmod 700 /etc/argos
+chmod 600 /etc/argos/config.yaml
 
 systemctl daemon-reload
 ```
 
-Then, edit `/etc/default/argos-server` to put your database password in it and change the other settings to suit your needs.
+Then, edit `/etc/argos/config.yaml` to put your database password in it and change the other settings to suit your needs.
 
 Create a token for your agent :
 
 ```bash
-su - argos -s /bin/bash
+sudo -u argos /opt/argos/venv/bin/argos server generate-token
-. /etc/default/argos-server
-export ARGOS_DATABASE_URL
-export ARGOS_YAML_FILE
-/opt/argos/venv/bin/argos server generate-token
-exit
 ```
 
 Edit `/etc/default/argos-agent` to put the generated token in it and change the other settings to suit your needs.
@@ -150,7 +149,7 @@ systemctl status argos-server.service argos-agent.service
 If all works well, you have to put some cron tasks in `argos` crontab:
 
 ```bash
-echo -e "*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos -
+echo -e "*/10 * * * * /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos -
 ```
 
 See the [this page](../deployment/nginx.md) for using Nginx as reverse proxy.

tests/config.yaml

@@ -1,4 +1,8 @@
 general:
+  db:
+    # The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls
+    url: "sqlite:////tmp/test-argos.db"
+  env: test
   frequency: "1m"
   alerts:
     ok:

tests/conftest.py

@@ -6,7 +6,7 @@ from fastapi import FastAPI
 from fastapi.testclient import TestClient
 from sqlalchemy.orm import Session
 
-os.environ["ARGOS_APP_ENV"] = "test"
+os.environ["ARGOS_YAML_FILE"] = "tests/config.yaml"
 
 
 @pytest.fixture
@@ -45,10 +45,6 @@ def _create_app() -> FastAPI:
     )
 
     app = get_application()
-    # Hardcode the database url and the yaml file for testing purpose
-    # Otherwise, the app will try to read the .env file or the environment variables
-    app.state.settings.database_url = "sqlite:////tmp/test-argos.db"
-    app.state.settings.yaml_file = "tests/config.yaml"
 
     setup_database(app)
     asyncio.run(connect_to_db(app))

tests/test_queries.py

@@ -172,6 +172,7 @@ def task(db):
 def empty_config():
     return schemas.config.Config(
         general=schemas.config.General(
+            db=schemas.config.DbSettings(url="sqlite:////tmp/test-argos.db"),
             frequency="1m",
             alerts=schemas.config.Alert(
                 ok=["", ""],