almet/argos
1
0
Fork 0
mirror of https://framagit.org/framasoft/framaspace/argos.git synced 2025-04-28 18:02:41 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

🔀 Merge branch 'config-file-only' into 'develop'

Browse source 
💥 — Remove env vars and only use the configuration file (fix #47)

See merge request framasoft/framaspace/argos!59
This commit is contained in:
Luc Didry 2024-06-20 15:48:58 +00:00
commit 29d839dc4d
16 changed files with 111 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -10,6 +10,7 @@
- ✨ — Add command to warn if it’s been long since last viewing an agent (fix #49)
- 💥 — Change default config file path to argos-config.yaml (fix #36)
- 📝 — New documentation URL: doc is now on https://argos-monitoring.framasoft.org/
- 💥 — Remove env vars and only use the configuration file
## 0.1.1

15
argos/commands.py
View file

@ -243,15 +243,12 @@ async def reload_config(config):
# The imports are made here otherwise the agent will need server configuration files.
from argos.server import queries
from argos.server.main import get_application, read_config
from argos.server.settings import get_app_settings
from argos.server.main import read_config
appli = get_application()
settings = get_app_settings()
config = read_config(appli, settings)
_config = read_config(config)
db = await get_db()
changed = await queries.update_from_config(db, config)
changed = await queries.update_from_config(db, _config)
click.echo(f"{changed['added']} tasks added")
click.echo(f"{changed['vanished']} tasks deleted")
@ -274,13 +271,13 @@ async def migrate(config):
os.environ["ARGOS_YAML_FILE"] = config
# The imports are made here otherwise the agent will need server configuration files.
from argos.server.settings import get_app_settings
from argos.server.settings import read_yaml_config
settings = get_app_settings()
settings = read_yaml_config(config)
current_dir = Path(__file__).resolve().parent
alembic_cfg = Config(current_dir / "server" / "migrations" / "alembic.ini")
alembic_cfg.set_main_option("sqlalchemy.url", settings.database_url)
alembic_cfg.set_main_option("sqlalchemy.url", str(settings.general.db.url))
command.upgrade(alembic_cfg, "head")

11
argos/config-example.yaml
View file

@ -1,4 +1,15 @@
general:
db:
# The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls
# Example for SQLite: "sqlite:////tmp/argos.db"
url: "postgresql://argos:argos@localhost/argos"
# You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?)
# See https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size for details
pool_size: 10
max_overflow: 20
# Can be "production", "dev", "test".
# If not present, default value is "production"
env: "production"
frequency: "1m" # Run checks every minute.
# Which way do you want to be warned when a check goes to that severity?
# "local" emits a message in the server log

2
argos/logging.py
View file

@ -3,7 +3,7 @@ import logging
LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
# Print level before message
logging.basicConfig(format="%(levelname)s: %(message)s")
logging.basicConfig(format="%(levelname)-9s %(message)s")
# XXX We probably want different loggers for client and server.
logger = logging.getLogger(__name__)

18
argos/schemas/config.py
View file

@ -8,17 +8,27 @@ from pydantic import (
BaseModel,
ConfigDict,
HttpUrl,
PostgresDsn,
StrictBool,
EmailStr,
PositiveInt,
field_validator,
)
from pydantic.functional_validators import BeforeValidator
from pydantic.networks import UrlConstraints
from pydantic_core import Url
from typing_extensions import Annotated
from argos.schemas.utils import string_to_duration
Severity = Literal["warning", "error", "critical", "unknown"]
Environment = Literal["dev", "test", "production"]
SQLiteDsn = Annotated[
Url,
UrlConstraints(
allowed_schemes=["sqlite"],
),
]
def parse_threshold(value):
@ -134,10 +144,18 @@ class GotifyUrl(BaseModel):
tokens: List[str]
class DbSettings(BaseModel):
url: PostgresDsn | SQLiteDsn
pool_size: int = 10
max_overflow: int = 20
class General(BaseModel):
"""Frequency for the checks and alerts"""
frequency: int
db: DbSettings
env: Environment = "production"
alerts: Alert
mail: Optional[Mail] = None
gotify: Optional[List[GotifyUrl]] = None

33
argos/server/main.py
View file

@ -1,3 +1,4 @@
import os
import sys
from pathlib import Path
@ -9,20 +10,18 @@ from sqlalchemy.orm import sessionmaker
from argos.logging import logger
from argos.server import models, routes, queries
from argos.server.settings import get_app_settings, read_yaml_config
from argos.server.settings import read_yaml_config
def get_application() -> FastAPI:
"""Spawn Argos FastAPI server"""
settings = get_app_settings()
appli = FastAPI()
config_file = os.environ["ARGOS_YAML_FILE"]
config = read_config(appli, settings)
config = read_config(config_file)
# Settings is the pydantic settings object
# Config is the argos config object (built from yaml)
appli.state.config = config
appli.state.settings = settings
appli.add_event_handler(
"startup",
@ -79,10 +78,9 @@ def create_stop_app_handler(appli):
return stop_app
def read_config(appli, settings):
def read_config(yaml_file):
try:
config = read_yaml_config(settings.yaml_file)
appli.state.config = config
config = read_yaml_config(yaml_file)
return config
except ValidationError as err:
logger.error("Errors where found while reading configuration:")
@ -92,25 +90,26 @@ def read_config(appli, settings):
def setup_database(appli):
settings = appli.state.settings
config = appli.state.config
db_url = str(config.general.db.url)
logger.debug("Using database URL %s", db_url)
# For sqlite, we need to add connect_args={"check_same_thread": False}
logger.debug("Using database URL %s", settings.database_url)
if settings.database_url.startswith("sqlite:////tmp"):
if config.general.env == "production" and db_url.startswith("sqlite:////tmp"):
logger.warning("Using sqlite in /tmp is not recommended for production")
extra_settings = {}
if settings.db_pool_size:
extra_settings.setdefault("pool_size", settings.db_pool_size)
if config.general.db.pool_size:
extra_settings.setdefault("pool_size", config.general.db.pool_size)
if settings.db_max_overflow:
extra_settings.setdefault("max_overflow", settings.db_max_overflow)
if config.general.db.max_overflow:
extra_settings.setdefault("max_overflow", config.general.db.max_overflow)
engine = create_engine(settings.database_url, **extra_settings)
engine = create_engine(db_url, **extra_settings)
def _fk_pragma_on_connect(dbapi_con, con_record):
dbapi_con.execute("pragma foreign_keys=ON")
if settings.database_url.startswith("sqlite:////"):
if db_url.startswith("sqlite:///"):
event.listen(engine, "connect", _fk_pragma_on_connect)
appli.state.SessionLocal = sessionmaker(

63
argos/server/settings.py
View file

@ -1,75 +1,12 @@
"""Pydantic schemas for server"""
from functools import lru_cache
from os import environ
from pathlib import Path
from typing import Optional, Union
import yaml
from pydantic_settings import BaseSettings, SettingsConfigDict
from yamlinclude import YamlIncludeConstructor
from argos.schemas.config import Config
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="argos_", env_file=".env")
app_env: str
database_url: str
yaml_file: str
db_pool_size: Optional[int]
db_max_overflow: Optional[int]
class DevSettings(Settings):
"""Settings for dev environment.
Uses argos-config.yaml as config file.
Uses a SQLite database."""
app_env: str = "dev"
db_pool_size: Optional[int] = None
db_max_overflow: Optional[int] = None
database_url: str = "sqlite:////tmp/argos.db"
class TestSettings(Settings):
"""Settings for test environment.
Uses tests/config.yaml as config file.
Uses a SQLite database."""
app_env: str = "test"
yaml_file: str = "tests/config.yaml"
database_url: str = "sqlite:////tmp/test-argos.db"
db_pool_size: Optional[int] = None
db_max_overflow: Optional[int] = None
class ProdSettings(Settings):
"""Settings for prod environment."""
app_env: str = "prod"
db_pool_size: Optional[int] = 10
db_max_overflow: Optional[int] = 20
environments = {
"dev": DevSettings,
"prod": ProdSettings,
"test": TestSettings,
}
@lru_cache()
def get_app_settings() -> Union[None, Settings]:
"""Load settings depending on the environment"""
app_env = environ.get("ARGOS_APP_ENV", "dev")
settings = environments.get(app_env)
if settings is not None:
return settings()
return None
def read_yaml_config(filename):
parsed = _load_yaml(filename)
return Config(**parsed)

1
conf/default-argos-server
View file

@ -1,5 +1,4 @@
ARGOS_YAML_FILE="/etc/argos/config.yaml"
ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos"
ARGOS_SERVER_WORKERS=4
ARGOS_SERVER_SOCKET=127.0.0.1:8000
# Comma separated list of IP addresses of the web proxy (usually Nginx)

3
docs/cli.md
View file

@ -191,7 +191,8 @@ Options:
checks have a timeout value of 60 seconds)
--config TEXT Path of the configuration file. If ARGOS_YAML_FILE
environment variable is set, its value will be
used instead.
used instead. Default value: argos-config.yaml and
/etc/argos/config.yaml as fallback.
--help Show this message and exit.
```

1
docs/conf.py
View file

@ -5,6 +5,7 @@
# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
# pylint: disable-msg=invalid-name,redefined-builtin
import argos
project = "Argos"

48
docs/configuration.md
View file

@ -1,51 +1,6 @@
# Configuration
There are actually two configuration files: one for the service and one for the checks.
## Server configuration
The server configuration is done using environment variables. You can put them in a `.env` file at the root of the project.
Here is a list of the useful variables, in the `.env` format:
```{literalinclude} ../conf/.env.example
---
caption: .env
---
```
### Environment variables
Here are the environment variables you can define to configure how the service will behave :
#### ARGOS_YAML_FILE
The path to the yaml configuration file, defining the checks.
If you do not provide the environment variable, Argos will try to read `argos-config.yaml` in the current directory, then `/etc/config/argos.yaml`.
#### ARGOS_DATABASE_URL
The database url, as defined [in SQLAlchemy docs](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls).
For instance, to connect to a postgres database on localhost with user, pass and dbname "argos":
```
ARGOS_DATABASE_URL="postgresql://argos:argos@localhost/argos"
```
#### DB_POOL_SIZE
#### DB_MAX_OVERFLOW
You configure the size of the database pool of connection, and the max overflow (until when new connections are accepted ?) These are documented [in the SQLAlchemy docs in greater details](https://docs.sqlalchemy.org/en/20/core/pooling.html#sqlalchemy.pool.QueuePool.params.pool_size)
```bash
DB_POOL_SIZE=10
DB_MAX_OVERFLOW=20
```
## Argos "checks" configuration
Argos uses a YAML configuration file to define the websites to monitor and the checks to run on these websites.
Argos uses a simple YAML configuration file to define the server’s configuration, the websites to monitor and the checks to run on these websites.
Here is a simple configuration file:
@ -54,5 +9,4 @@ Here is a simple configuration file:
---
caption: argos-config.yaml
---
```

47
docs/installation/getting-started.md
View file

@ -7,6 +7,21 @@ NB: if you want a quick-installation guide, we [got you covered](tl-dr.md).
- Python 3.11+
- PostgreSQL 13+ (for production)
## Recommendation
Create a dedicated user for argos:
```bash
adduser --home /opt/argos --disabled-login --disabled-password --system argos
```
Do all the manipulations below in `/opt/argos/`, with the user `argos`.
Either use `sudo` or login as `argos` with the following command:
```bash
su argos -s /bin/bash
```
## Install with pip
```bash
@ -24,7 +39,7 @@ pip install argos-monitoring
For production, we recommend the use of [Gunicorn](https://gunicorn.org/), which you can install at the same time as Argos:
```bash
pip install argos-monitoring[gunicorn]
pip install "argos-monitoring[gunicorn]"
```
## Install from sources
@ -37,6 +52,8 @@ source venv/bin/activate
pip install -e .
```
To install gunicorn, use `pip install -e ".[gunicorn]"` instead of `pip install -e .`
## Configure
The quickest way to get started is to generate the configuration file from argos and edit it:
@ -47,14 +64,18 @@ argos server generate-config > argos-config.yaml
You can read more about the configuration in the [configuration section](../configuration.md).
### Configure the server
For production, we suggest to put your config in `/etc/argos/config.yaml` and restricts the file’s permissions.
As root:
```bash
mkdir /etc/argos
chown argos: /etc/argos
chmod 700 /etc/argos
```
Environment variables are used to configure the server. You can also put them in an `.env` file:
```{literalinclude} ../../conf/.env.example
---
caption: .env
---
Then, as `argos`:
```bash
argos server generate-config > /etc/argos/config.yaml
chmod 600 /etc/argos/config.yaml
```
Please note that the only supported database engines are SQLite for development and PostgreSQL for production.
@ -69,7 +90,7 @@ argos server migrate
## Inject tasks into the database
Argos keeps tasks’ configuration in database, take from the config file.
Argos keeps tasks’ configuration in database, taken from the config file.
Populate the database with the tasks:
@ -85,8 +106,6 @@ Then you can start the server:
argos server start
```
The server reads the `yaml` file at startup, and populates the tasks queue with the checks defined in the configuration.
This way to start the server is not suitable for production, use it only for developing or testing.
## Starting the server for production
@ -96,7 +115,7 @@ For production, you can use [Gunicorn](https://gunicorn.org/) to start the serve
To install Gunicorn in the virtualenv, if you didn’t already install Argos that way:
```bash
pip install argos-monitoring[gunicorn]
pip install "argos-monitoring[gunicorn]"
```
To start the server:
@ -121,6 +140,8 @@ Gunicorn has a lot of other options, have a look at `gunicorn --help`.
Argos uses FastAPI, so you can use other ways to start the server.
See <https://fastapi.tiangolo.com/deployment/manually/#asgi-servers> (but Gunicorn is recommended).
See [here](../deployment/systemd.md#server) for a systemd service example and [here](../deployment/nginx.md) for a nginx configuration example.
## Generating a token
The agent needs an authentication token to be able to communicate with the server.
@ -141,7 +162,7 @@ service:
## Running the agent
You can run the agent on the same machine as the server, or on a different machine.
The only requirement is that the agent can reach the server.
The only requirement is that the agent can reach the server through HTTP or HTTPS.
```bash
argos agent http://localhost:8000 "auth-token"

29
docs/installation/tl-dr.md
View file

@ -13,13 +13,17 @@ cd /tmp/argos
python3 -m venv venv
source venv/bin/activate
pip install argos-monitoring
argos server generate-config > argos-config.yaml
argos server generate-config |
sed -e "s@production@test@" \
-e "s@url: .postgresql.*@url: \"sqlite:////tmp/argos.db\"@" > argos-config.yaml
argos server migrate
argos server generate-token
ARGOS_TOKEN=$(argos server generate-token)
sed -e "s@# - secret_token@- $ARGOS_TOKEN@" -i argos-config.yaml
echo "The agent token is $ARGOS_TOKEN"
```
Edit `argos-config.yaml`.
Add the generated token in it, as long as some real web sites to test.
Add some real web sites to test.
Then:
@ -48,16 +52,14 @@ sudo -u postgres psql -c "ALTER DATABASE argos SET TIMEZONE TO 'UTC';"
adduser --home /opt/argos --disabled-login --disabled-password --system argos
cd /opt/argos
python3 -m venv venv
chown argos: -R venv
sudo -u argos bash -c "source venv/bin/activate && pip install argos-monitoring[gunicorn]"
sudo -u argos python3 -m venv venv
sudo -u argos bash -c 'source venv/bin/activate && pip install "argos-monitoring[gunicorn]"'
mkdir /etc/argos
/opt/argos/venv/bin/argos server generate-config > /etc/argos/config.yaml
cat <<EOF > /etc/default/argos-server
ARGOS_YAML_FILE="/etc/argos/config.yaml"
ARGOS_DATABASE_URL="postgresql://argos:THE_DB_PASSWORD@localhost/argos"
ARGOS_SERVER_WORKERS=4
ARGOS_SERVER_SOCKET=127.0.0.1:8000
# Comma separated list of IP addresses of the web proxy (usually Nginx)
@ -119,21 +121,18 @@ WantedBy=multi-user.target
EOF
chown -R argos: /etc/default/argos-* /etc/argos/
chmod 700 /etc/argos
chmod 600 /etc/argos/config.yaml
systemctl daemon-reload
```
Then, edit `/etc/default/argos-server` to put your database password in it and change the other settings to suit your needs.
Then, edit `/etc/argos/config.yaml` to put your database password in it and change the other settings to suit your needs.
Create a token for your agent :
```bash
su - argos -s /bin/bash
. /etc/default/argos-server
export ARGOS_DATABASE_URL
export ARGOS_YAML_FILE
/opt/argos/venv/bin/argos server generate-token
exit
sudo -u argos /opt/argos/venv/bin/argos server generate-token
```
Edit `/etc/default/argos-agent` to put the generated token in it and change the other settings to suit your needs.
@ -150,7 +149,7 @@ systemctl status argos-server.service argos-agent.service
If all works well, you have to put some cron tasks in `argos` crontab:
```bash
echo -e "*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * . /etc/default/argos-server && export ARGOS_DATABASE_URL && export ARGOS_YAML_FILE && cd /opt/argos/ && /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos -
echo -e "*/10 * * * * /opt/argos/venv/bin/argos server cleandb --max-lock-seconds 120 --max-results 1200\n*/10 * * * * /opt/argos/venv/bin/argos server watch-agents --time-without-agent 10" | crontab -u argos -
```
See the [this page](../deployment/nginx.md) for using Nginx as reverse proxy.

4
tests/config.yaml
View file

@ -1,4 +1,8 @@
general:
db:
# The database URL, as defined in SQLAlchemy docs : https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls
url: "sqlite:////tmp/test-argos.db"
env: test
frequency: "1m"
alerts:
ok:

6
tests/conftest.py
View file

@ -6,7 +6,7 @@ from fastapi import FastAPI
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session
os.environ["ARGOS_APP_ENV"] = "test"
os.environ["ARGOS_YAML_FILE"] = "tests/config.yaml"
@pytest.fixture
@ -45,10 +45,6 @@ def _create_app() -> FastAPI:
)
app = get_application()
# Hardcode the database url and the yaml file for testing purpose
# Otherwise, the app will try to read the .env file or the environment variables
app.state.settings.database_url = "sqlite:////tmp/test-argos.db"
app.state.settings.yaml_file = "tests/config.yaml"
setup_database(app)
asyncio.run(connect_to_db(app))

1
tests/test_queries.py
View file

@ -172,6 +172,7 @@ def task(db):
def empty_config():
return schemas.config.Config(
general=schemas.config.General(
db=schemas.config.DbSettings(url="sqlite:////tmp/test-argos.db"),
frequency="1m",
alerts=schemas.config.Alert(
ok=["", ""],