diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7e408a3 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +ARGOS_YAML_FILE = "my-config.yaml" +ARGOS_DATABASE_URL = "postgresql://argos:argos@localhost/argos" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 72a0c53..e695a0f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__ *.egg-info .vscode venv -.env \ No newline at end of file +.env +public diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba3b8db..1e15523 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,14 +2,25 @@ image: python:3.11 stages: - test + - deploy before_script: - - python -V - - python -m venv venv - - ./venv/bin/pip install -e ".[dev]" - + - make install pytest: stage: test script: - - ./venv/bin/pytest \ No newline at end of file + - make tests + +pages: + stage: deploy + + script: + - pwd + - ls + - make docs + artifacts: + paths: + - public/ + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a3ae9f1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## [Unreleased] + +- First working version \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5c3edae --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +venv: + python3 -m venv venv +install: venv + venv/bin/pip install -e ".[dev,docs]" +docs: cog + venv/bin/sphinx-build docs public +cog: + venv/bin/cog -r docs/*.md +tests: install + venv/bin/pytest diff --git a/README.md b/README.md index eea33f2..1a47ddc 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,6 @@ Todo: - [ ] Add a way to specify the severity of the alerts in the config - [ ] Add a command to generate new authentication token -Features : - -- [x] Uses `.yaml` files for configuration ; -- [x] Read the configuration file and convert it to tasks ; -- [x] Store tasks in a database ; -- [x] Multiple paths per websites can be tested ; -- [x] Handle jobs failures on the clients -- [x] Exposes an HTTP API that can be consumed by other systems ; -- [x] Checks can be distributed on the network thanks to a job queue ; Implemented checks : @@ -35,96 +26,6 @@ Implemented checks : - [x] Returned body matches what you expect ; - [x] SSL certificate expires in more than X days ; -## How to run ? - -To install it, create a virtualenv and install the dependencies: - -```bash -python3 -m venv venv -source venv/bin/activate -pip install -e . -``` - -Prepare a configuration file, you can copy the `config-example.yaml` file and edit it: - -```bash -cp config-example.yaml config.yaml -``` - -Then, you can run the server: - -```bash -argos server run -``` - -You can specify the environment variables to configure the server, or you can put them in an `.env` file: - -```bash -ARGOS_DATABASE_URL=postgresql://localhost/argos -ARGOS_YAML_FILE=config.yaml -``` - -The server will read a `yaml` file at startup, and will populate the tasks specified in it. See the configuration section below for more information on how to configure the checks you want to run. - -And here is how to run the agent: - -```bash -argos agent http://localhost:8000 "" -``` - -You also need to run cleaning tasks periodically. `argos server clean --help` will give you more information on how to do that. - -Here is a crontab example: - -```bash -# Run the cleaning tasks every hour (at minute 7) -7 * * * * argos server clean --max-results 100000 --max-lock-seconds 3600 -``` - - -## Configuration - -Here is a simple configuration file: - -```yaml -general: - frequency: "1m" # Run checks every minute. - alerts: - error: - - local - warning: - - local - alert: - - local -service: - secrets: - # Secrets can be generated using `openssl rand -base64 32`. - # DO NOT REUSE THESE ONES. - - "O4kt8Max9/k0EmHaEJ0CGGYbBNFmK8kOZNIoUk3Kjwc" - - "x1T1VZR51pxrv5pQUyzooMG4pMUvHNMhA5y/3cUsYVs=" - -ssl: - thresholds: - - "1d": critical - - "5d": warning - -# It's also possible to define the checks in another file -# with the include syntax: -# -# websites: !include websites.yaml -# -websites: - - domain: "https://mypads.framapad.org" - paths: - - path: "/mypads/" - checks: - - status-is: "200" - - body-contains: '
' - - ssl-certificate-expiration: "on-check" - - path: "/admin/" - checks: - - status-is: "401" -``` ## Development notes diff --git a/config-example.yaml b/config-example.yaml index 150b043..7738cb6 100644 --- a/config-example.yaml +++ b/config-example.yaml @@ -10,8 +10,6 @@ general: service: secrets: # Secrets can be generated using `openssl rand -base64 32`. - - "O4kt8Max9/k0EmHaEJ0CGGYbBNFmK8kOZNIoUk3Kjwc" - - "x1T1VZR51pxrv5pQUyzooMG4pMUvHNMhA5y/3cUsYVs=" ssl: thresholds: diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/capture.png b/docs/capture.png new file mode 100644 index 0000000..81a8071 Binary files /dev/null and b/docs/capture.png differ diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..8261b35 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,2 @@ +```{include} ../CHANGELOG.md +``` \ No newline at end of file diff --git a/docs/checks.md b/docs/checks.md new file mode 100644 index 0000000..4a910c9 --- /dev/null +++ b/docs/checks.md @@ -0,0 +1,9 @@ +# Checks + +At its core, argos runs checks and return the results to the service. Here are the implemented checks, with a description of what they do and how to configure them. + +| Check | Description | Configuration | +| --- | --- | --- | +| `status-is` | Check that the returned status code matches what you expect. | `status-is: "200"` | +| `body-contains` | Check that the returned body contains a given string. | `body-contains: "Hello world"` | +| `ssl-certificate-expiration` | Check that the SSL certificate expires in more than X days. | `ssl-certificate-expiration: "30d"` | \ No newline at end of file diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..cdfa659 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,124 @@ +# Command-line interface + + + + +## The argos cli + + +```man +Usage: argos [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + agent Get and run tasks to the provided server. + server +``` + + + +## Agent commands + + +```man +Usage: argos agent [OPTIONS] SERVER AUTH + + Get and run tasks to the provided server. Will wait for new tasks. + + Usage: argos agent https://argos.server "auth-token-here" + +Options: + --max-tasks INTEGER Number of concurrent tasks this agent can run + --wait-time INTEGER Waiting time between two polls on the server + (seconds) + --log-level [DEBUG|INFO|WARNING|ERROR|CRITICAL] + --help Show this message and exit. +``` + + + +## Server commands + + +```man +Usage: argos server [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + clean Clean the database (to run routinely) + start Starts the server. +``` + + + +### Server start + + +```man +Usage: argos server start [OPTIONS] + + Starts the server. + +Options: + --host TEXT Host to bind + --port INTEGER Port to bind + --reload Enable hot reloading + --log-config TEXT Path to the logging configuration file + --help Show this message and exit. +``` + + + +### Server clean + + +```man +Usage: argos server clean [OPTIONS] + + Clean the database (to run routinely) + + - Removes old results from the database. - Removes locks from tasks that have + been locked for too long. + +Options: + --max-results INTEGER Maximum number of results to keep + --max-lock-seconds INTEGER The number of seconds after which a lock is + considered stale + --help Show this message and exit. +``` + + \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..266955b --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,31 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Argos" +copyright = "2023, Alexis Métaireau" +author = "Alexis Métaireau" +release = "0.0.1" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx_design", +] +myst_enable_extensions = ["colon_fence"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "shibuya" +html_static_path = ["_static"] diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..43be971 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,28 @@ +# Configuration + +There are actually two configuration files: one for the service and one for the checks. + +## Server configuration + +The server configuration is done using environment variables. You can put them in a `.env` file at the root of the project. +Here is a list of the useful variables, in the `.env` format: + +```{literalinclude} ../.env.example +--- +caption: .env +--- +``` + +## Checks configuration + +Argos uses a YAML configuration file to define the websites to monitor and the checks to run on these websites. + +Here is a simple configuration file: + + +```{literalinclude} ../config-example.yaml +--- +caption: config.yaml +--- + +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..81d700a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,59 @@ +# Argos + +Argos is an HTTP monitoring service. + +1. Define a list of websites to monitor +2. Specify a list of checks to run on these websites. +3. Argos will run the checks periodically and alert you if something goes wrong. + +Internally, a HTTP API is exposed, and a job queue is used to distribute the checks on the network. + +## Features + +- **Made for large systems**: It's meant to supervise a large number of websites. +- **Server/Agent architecture**: The server is responsible for storing the configuration and the results of the checks. The agent is responsible for running the checks and sending the results to the server. +- **Extensible**: You can add new checks using pure python. +- **HTTP API**: You can use the HTTP API to get the results of the checks. + +```{code-block} bash +pip install argos-monitoring +argos server run +argos agent http://localhost:8000 "" +``` + +```{literalinclude} ../config-example.yaml +--- +caption: config.yaml +--- +``` + +## Next + +::::{grid} 2 +:::{grid-item-card} Getting started +:link: installation/getting-started.html +The best way to get started with argos. +::: +:::{grid-item-card} Developper docs +:link: /developper +You want to know more about the internals ?. +::: +:::: + + +```{toctree} +:caption: Getting started +:hidden: + +installation/getting-started +installation/postgresql +configuration +cli +``` + +```{toctree} +:caption: Checks +:hidden: + +checks +``` \ No newline at end of file diff --git a/docs/installation/getting-started.md b/docs/installation/getting-started.md new file mode 100644 index 0000000..0fb41c9 --- /dev/null +++ b/docs/installation/getting-started.md @@ -0,0 +1,65 @@ +# Installation + +So, you want to get started :-) Nothing complicated here, but here is a step by step guide. + +## Requirements + +- Python 3.11+ +- PostgreSQL 13+ + +## Getting dependencies + +Once you got the source locally, create a virtualenv and install the dependencies: + +```bash +python3 -m venv venv +source venv/bin/activate +pip install -e . +``` + +## Configure + +Prepare a configuration file, you can copy the `config-example.yaml` file and edit it: + +```bash +cp config-example.yaml config.yaml +``` + +You can read more about the configuration in the [configuration section](configuration.md). + +## Starting the server + +You need to specify environment variables to configure the server, or you can put them in an `.env` file: + +```{literalinclude} ../../.env.example +--- +caption: .env +--- +``` + +Then you can start the server: + +```bash +argos server run +``` + +The server will read a `yaml` file at startup, and will populate the tasks specified in it. + +## Running the agent + +And here is how to run the agent: + +```bash +argos agent http://localhost:8000 "" +``` + +## Running cleaning tasks + +You also need to run cleaning tasks periodically. `argos server clean --help` will give you more information on how to do that. + +Here is a crontab example: + +```bash +# Run the cleaning tasks every hour (at minute 7) +7 * * * * argos server clean --max-results 100000 --max-lock-seconds 3600 +``` \ No newline at end of file diff --git a/docs/installation/postgresql.md b/docs/installation/postgresql.md new file mode 100644 index 0000000..69eb324 --- /dev/null +++ b/docs/installation/postgresql.md @@ -0,0 +1,35 @@ +# Installing PostgreSQL + +If you intend to use argos with Postgresql, you can install it with the following commands: + +```bash +pip install argos-monitoring[postgresql] +``` + +Here are a few steps for you to install postgresql on your system: + +## Debian + +```bash +sudo apt install postgresql +``` + +By default, the `postgres` user has no password and can hence only connect if ran by the `postgres` system user. The following command will assign it: + +```bash +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" +``` + +```bash +psql postgres -c "CREATE DATABASE argos;" +psql postgres -c "ALTER DATABASE argos SET TIMEZONE TO 'UTC';" +``` + +## MacOS + +```bash +brew install postgresql@14 +brew services start postgresql@14 +createuser argos +createdb argos -O argos +``` \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/pyproject.toml b/pyproject.toml index 1f58aee..6e18cc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" [project] -name = "argos" +name = "argos-monitoring" version = "0.1.0" description = "Distributed supervision tool for HTTP." authors = [ @@ -31,6 +31,27 @@ dependencies = [ "tenacity>=8.2,<9", ] +[project.optional-dependencies] +dev = [ + "black==23.3.0", + "isort==5.11.5", + "pytest>=6.2.5", + "pytest-asyncio>=0.21,<1", + "ipython>=8.16,<9", + "ipdb>=0.13,<0.14", + "sphinx-autobuild", +] +postgres = [ + "psycopg2-binary>=2.9,<3", +] +docs = [ + "sphinx>=7,<8", + "myst-parser>=2.0,<3", + "shibuya", + "sphinx-design", + "cogapp", +] + [project.urls] homepage = "https://framagit.org/framasoft/framaspace/argos" repository = "https://framagit.org/framasoft/framaspace/argos" @@ -40,19 +61,6 @@ repository = "https://framagit.org/framasoft/framaspace/argos" [tool.setuptools] packages = ["argos"] -[project.optional-dependencies] -dev = [ - "black==23.3.0", - "isort==5.11.5", - "pytest>=6.2.5", - "pytest-asyncio>=0.21,<1", - "ipython>=8.16,<9", - "ipdb>=0.13,<0.14", -] -postgres = [ - "psycopg2-binary>=2.9,<3", -] - [project.scripts] argos = "argos.commands:cli" @@ -63,5 +71,4 @@ testpaths = [ "tests", "argos" ] -fixture_path = "tests/fixtures.py" pythonpath = "." \ No newline at end of file