From e3b1b714b3508cf9ae69080db25218a633c6e517 Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Thu, 14 Mar 2024 12:14:04 +0100 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=90=9B=20=E2=80=94=20Delete=20tasks?= =?UTF-8?q?=20which=20are=20not=20in=20config=20file=20anymore=20(fix=20#1?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- argos/server/models.py | 13 +++++++- argos/server/queries.py | 71 ++++++++++++++++++++++++++++++++++++++++- tests/test_queries.py | 20 ++++++++++-- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/argos/server/models.py b/argos/server/models.py index 451eb6b..0637e7e 100644 --- a/argos/server/models.py +++ b/argos/server/models.py @@ -93,7 +93,8 @@ class Result(Base): __tablename__ = "results" id: Mapped[int] = mapped_column(primary_key=True) task_id: Mapped[int] = mapped_column(ForeignKey("tasks.id")) - task: Mapped["Task"] = relationship(back_populates="results") + task: Mapped["Task"] = relationship(back_populates="results", + cascade="save-update, merge, delete") agent_id: Mapped[str] = mapped_column(nullable=True) submitted_at: Mapped[datetime] = mapped_column() @@ -112,3 +113,13 @@ class Result(Base): def __str__(self): return f"DB Result {self.id} - {self.status} - {self.context}" + +class ConfigCache(Base): + """Contains some informations on the previous config state + + Used to quickly determine if we need to update the tasks + """ + __tablename__ = "config_cache" + name: Mapped[str] = mapped_column(primary_key=True) + val: Mapped[str] = mapped_column() + updated_at: Mapped[datetime] = mapped_column() diff --git a/argos/server/queries.py b/argos/server/queries.py index 271b2c9..345f480 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -1,5 +1,7 @@ """Functions to ease SQL queries management""" from datetime import datetime, timedelta +from hashlib import sha256 +from typing import List from urllib.parse import urljoin from sqlalchemy import desc, func @@ -7,7 +9,7 @@ from sqlalchemy.orm import Session from argos import schemas from argos.logging import logger -from argos.server.models import Result, Task +from argos.server.models import Result, Task, ConfigCache async def list_tasks(db: Session, agent_id: str, limit: int = 100): @@ -60,10 +62,64 @@ async def count_results(db: Session): return db.query(Result).count() +async def is_config_unchanged(db: Session, config: schemas.Config) -> bool: + """Check if websites config has changed by using a hashsum and a config cache""" + websites_hash = sha256(str(config.websites).encode()).hexdigest() + conf_caches = ( + db.query(ConfigCache) + .all() + ) + same_config = True + if conf_caches: + for conf in conf_caches: + if not same_config: + break + + if conf.name == 'websites_hash': + same_config = conf.val == websites_hash + elif conf.name == 'general_frequency': + same_config = conf.val == str(config.general.frequency) + + if same_config: + return True + + for conf in conf_caches: + if conf.name == 'websites_hash': + conf.val = websites_hash + elif conf.name == 'general_frequency': + conf.val = config.general.frequency + conf.updated_at = datetime.now() + else: + web_hash = ConfigCache( + name='websites_hash', + val=websites_hash, + updated_at=datetime.now() + ) + gen_freq = ConfigCache( + name='general_frequency', + val=str(config.general.frequency), + updated_at=datetime.now() + ) + db.add(web_hash) + db.add(gen_freq) + db.commit() + + return False + + async def update_from_config(db: Session, config: schemas.Config): """Update tasks from config file""" + config_unchanged = await is_config_unchanged(db, config) + if config_unchanged: + return None + + max_task_id = ( + db.query(func.max(Task.id).label('max_id')) # pylint: disable-msg=not-callable + .all() + )[0].max_id tasks = [] unique_properties = [] + seen_tasks: List[int] = [] for website in config.websites: domain = str(website.domain) frequency = website.frequency or config.general.frequency @@ -83,6 +139,7 @@ async def update_from_config(db: Session, config: schemas.Config): ) if existing_tasks: existing_task = existing_tasks[0] + seen_tasks.append(existing_task.id) if frequency != existing_task.frequency: existing_task.frequency = frequency @@ -107,6 +164,18 @@ async def update_from_config(db: Session, config: schemas.Config): db.add_all(tasks) db.commit() + # Delete vanished tasks + if max_task_id: + vanished_tasks = ( + db.query(Task) + .filter( + Task.id <= max_task_id, + Task.id.not_in(seen_tasks) + ).delete() + ) + db.commit() + logger.info("%i tasks has been removed since not in config file anymore", vanished_tasks) + async def get_severity_counts(db: Session) -> dict: """Get the severities (ok, warning, critical…) and their count""" diff --git a/tests/test_queries.py b/tests/test_queries.py index 8770d98..d92c750 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -78,7 +78,7 @@ async def test_update_from_config_with_duplicate_tasks(db, empty_config): @pytest.mark.asyncio -async def test_update_from_config_db_can_handle_already_present_duplicates( +async def test_update_from_config_db_can_remove_duplicates_and_old_tasks( db, empty_config, task ): # Add a duplicate in the db @@ -99,12 +99,28 @@ async def test_update_from_config_db_can_handle_already_present_duplicates( dict( path="https://another-example.com", checks=[{task.check: task.expected}] ), + dict( + path=task.url, checks=[{task.check: task.expected}] + ), ], ) empty_config.websites = [website] await queries.update_from_config(db, empty_config) - assert db.query(Task).count() == 3 + assert db.query(Task).count() == 2 + + website = schemas.config.Website( + domain=task.domain, + paths=[ + dict( + path="https://another-example.com", checks=[{task.check: task.expected}] + ), + ], + ) + empty_config.websites = [website] + + await queries.update_from_config(db, empty_config) + assert db.query(Task).count() == 1 @pytest.mark.asyncio From f976905433658ad5f2633b6737e4446c30681537 Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Thu, 14 Mar 2024 12:18:05 +0100 Subject: [PATCH 2/7] =?UTF-8?q?=E2=99=BB=20=E2=80=94=20Move=20tasks=20conf?= =?UTF-8?q?ig=20(re)loading=20into=20a=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../1a3497f9f71b_adding_configcache_model.py | 35 +++++++++++++++++++ argos/commands.py | 21 +++++++++++ argos/server/main.py | 11 +++--- argos/server/queries.py | 5 ++- argos/server/routes/api.py | 2 +- conf/systemd-server.service | 2 ++ tests/conftest.py | 3 ++ tests/test_api.py | 4 +++ 8 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 alembic/versions/1a3497f9f71b_adding_configcache_model.py diff --git a/alembic/versions/1a3497f9f71b_adding_configcache_model.py b/alembic/versions/1a3497f9f71b_adding_configcache_model.py new file mode 100644 index 0000000..2f92fd3 --- /dev/null +++ b/alembic/versions/1a3497f9f71b_adding_configcache_model.py @@ -0,0 +1,35 @@ +"""Adding ConfigCache model + +Revision ID: 1a3497f9f71b +Revises: 7d480e6f1112 +Create Date: 2024-03-13 15:28:09.185377 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1a3497f9f71b' +down_revision: Union[str, None] = '7d480e6f1112' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('config_cache', + sa.Column('name', sa.String(), nullable=False), + sa.Column('val', sa.String(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('name') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('config_cache') + # ### end Alembic commands ### diff --git a/argos/commands.py b/argos/commands.py index 9c3ccd6..e53a93d 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -131,5 +131,26 @@ async def cleandb(max_results, max_lock_seconds): click.echo(f"{updated} locks released") +@server.command() +@coroutine +async def reload_config(): + """Read tasks config and add/delete tasks in database if needed + """ + # The imports are made here otherwise the agent will need server configuration files. + from argos.server import queries + from argos.server.main import get_application, read_config + from argos.server.settings import get_app_settings + + appli = get_application() + settings = get_app_settings() + config = read_config(appli, settings) + + db = await get_db() + changed = await queries.update_from_config(db, config) + + click.echo(f"{changed['added']} tasks added") + click.echo(f"{changed['vanished']} tasks deleted") + + if __name__ == "__main__": cli() diff --git a/argos/server/main.py b/argos/server/main.py index ce3f534..d3e175e 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -7,7 +7,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from argos.logging import logger -from argos.server import models, queries, routes +from argos.server import models, routes from argos.server.settings import get_app_settings, read_yaml_config @@ -39,15 +39,14 @@ def get_application() -> FastAPI: def create_start_app_handler(appli): """Warmup the server: - setup database connection and update the tasks in it before making it available + setup database connection """ - async def read_config_and_populate_db(): + async def connect_db_at_startup(): setup_database(appli) - db = await connect_to_db(appli) - await queries.update_from_config(db, appli.state.config) + return await connect_to_db(appli) - return read_config_and_populate_db + return connect_db_at_startup async def connect_to_db(appli): diff --git a/argos/server/queries.py b/argos/server/queries.py index 345f480..6d4d790 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -111,7 +111,7 @@ async def update_from_config(db: Session, config: schemas.Config): """Update tasks from config file""" config_unchanged = await is_config_unchanged(db, config) if config_unchanged: - return None + return {'added': 0, 'vanished': 0} max_task_id = ( db.query(func.max(Task.id).label('max_id')) # pylint: disable-msg=not-callable @@ -175,6 +175,9 @@ async def update_from_config(db: Session, config: schemas.Config): ) db.commit() logger.info("%i tasks has been removed since not in config file anymore", vanished_tasks) + return {'added': len(tasks), 'vanished': vanished_tasks} + + return {'added': len(tasks), 'vanished': 0} async def get_severity_counts(db: Session) -> dict: diff --git a/argos/server/routes/api.py b/argos/server/routes/api.py index 432660b..4a316b7 100644 --- a/argos/server/routes/api.py +++ b/argos/server/routes/api.py @@ -62,7 +62,7 @@ async def create_results( # XXX Use a job queue or make it async handle_alert(config, result, task, severity, last_severity, request) - db_results.append(result) + db_results.append(result) db.commit() return {"result_ids": [r.id for r in db_results]} diff --git a/conf/systemd-server.service b/conf/systemd-server.service index 698e637..0c69536 100644 --- a/conf/systemd-server.service +++ b/conf/systemd-server.service @@ -8,6 +8,8 @@ PartOf=postgresql.service [Service] User=www-data WorkingDirectory=/var/www/argos/ +ExecStartPre=/var/www/argos/venv/bin/alembic upgrade head +ExecStartPre=/var/www/argos/venv/bin/argos server reload-config ExecStart=/var/www/argos/venv/bin/argos server start ExecReload=/var/www/argos/venv/bin/argos server reload SyslogIdentifier=argos-server diff --git a/tests/conftest.py b/tests/conftest.py index 84e1a44..d07ea1a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import asyncio import os import pytest @@ -40,6 +41,7 @@ def _create_app() -> FastAPI: from argos.server.main import ( # local import for testing purpose get_application, setup_database, + connect_to_db, ) app = get_application() @@ -49,4 +51,5 @@ def _create_app() -> FastAPI: app.state.settings.yaml_file = "tests/config.yaml" setup_database(app) + asyncio.run(connect_to_db(app)) return app diff --git a/tests/test_api.py b/tests/test_api.py index 53ca7a7..21c87d1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,8 +1,11 @@ +import asyncio + import pytest from fastapi.testclient import TestClient from argos.schemas import AgentResult, SerializableException from argos.server import models +from argos.server.queries import update_from_config def test_read_tasks_requires_auth(app): @@ -12,6 +15,7 @@ def test_read_tasks_requires_auth(app): def test_tasks_retrieval_and_results(authorized_client, app): + asyncio.run(update_from_config(app.state.db, app.state.config)) with authorized_client as client: response = client.get("/api/tasks") assert response.status_code == 200 From 8875c704ee1166c7be87e8dfab12c970312c4e98 Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Thu, 14 Mar 2024 14:57:17 +0100 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=92=84=20=E2=80=94=20Use=20an=20argos?= =?UTF-8?q?=20server=20command=20to=20do=20alembic=20migrations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- argos/commands.py | 17 +++++++++++++++++ conf/systemd-server.service | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/argos/commands.py b/argos/commands.py index e53a93d..b90057a 100644 --- a/argos/commands.py +++ b/argos/commands.py @@ -4,6 +4,8 @@ from functools import wraps import click import uvicorn +from alembic import command +from alembic.config import Config from argos import logging from argos.agent import ArgosAgent @@ -152,5 +154,20 @@ async def reload_config(): click.echo(f"{changed['vanished']} tasks deleted") +@server.command() +@coroutine +async def migrate(): + """Run database migrations + """ + # The imports are made here otherwise the agent will need server configuration files. + from argos.server.settings import get_app_settings + + settings = get_app_settings() + + alembic_cfg = Config("alembic.ini") + alembic_cfg.set_main_option("sqlalchemy.url", settings.database_url) + command.upgrade(alembic_cfg, "head") + + if __name__ == "__main__": cli() diff --git a/conf/systemd-server.service b/conf/systemd-server.service index 0c69536..f5116a9 100644 --- a/conf/systemd-server.service +++ b/conf/systemd-server.service @@ -8,7 +8,7 @@ PartOf=postgresql.service [Service] User=www-data WorkingDirectory=/var/www/argos/ -ExecStartPre=/var/www/argos/venv/bin/alembic upgrade head +ExecStartPre=/var/www/argos/venv/bin/argos server migrate ExecStartPre=/var/www/argos/venv/bin/argos server reload-config ExecStart=/var/www/argos/venv/bin/argos server start ExecReload=/var/www/argos/venv/bin/argos server reload From cf609eae6b1b3285b34134a4fe2d24e96017da22 Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Mon, 18 Mar 2024 09:01:39 +0100 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=8E=A8=20=E2=80=94=20Improve=20code?= =?UTF-8?q?=20structure=20following=20review=20of=20MR=20!25?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 ++ .../1a3497f9f71b_adding_configcache_model.py | 14 +++----- argos/server/models.py | 8 ++++- argos/server/queries.py | 36 +++++++++---------- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index f42e098..9f61d37 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,8 @@ djlint: venv ## Format the templates venv/bin/djlint --ignore=H030,H031,H006 --profile jinja --lint argos/server/templates/*html pylint: venv ## Runs pylint on the code venv/bin/pylint argos +pylint-alembic: venv ## Runs pylint on alembic migration files + venv/bin/pylint --disable invalid-name,no-member alembic/versions/*.py lint: djlint pylint help: @python3 -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) diff --git a/alembic/versions/1a3497f9f71b_adding_configcache_model.py b/alembic/versions/1a3497f9f71b_adding_configcache_model.py index 2f92fd3..645bdc7 100644 --- a/alembic/versions/1a3497f9f71b_adding_configcache_model.py +++ b/alembic/versions/1a3497f9f71b_adding_configcache_model.py @@ -7,8 +7,8 @@ Create Date: 2024-03-13 15:28:09.185377 """ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. @@ -19,17 +19,13 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### op.create_table('config_cache', - sa.Column('name', sa.String(), nullable=False), - sa.Column('val', sa.String(), nullable=False), - sa.Column('updated_at', sa.DateTime(), nullable=False), - sa.PrimaryKeyConstraint('name') + sa.Column('name', sa.String(), nullable=False), + sa.Column('val', sa.String(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('name') ) - # ### end Alembic commands ### def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### op.drop_table('config_cache') - # ### end Alembic commands ### diff --git a/argos/server/models.py b/argos/server/models.py index 0637e7e..75ed8ab 100644 --- a/argos/server/models.py +++ b/argos/server/models.py @@ -117,7 +117,13 @@ class Result(Base): class ConfigCache(Base): """Contains some informations on the previous config state - Used to quickly determine if we need to update the tasks + Used to quickly determine if we need to update the tasks. + There is currently two cached settings: + - general_frequency: the content of general.frequency setting, in minutes + ex: 5 + - websites_hash: the sha256sum of websites setting, to allow a quick + comparison without looping through all websites + ex: 8b886e7db7b553fe99f6d5437f31745987e243c77b2109b84cf9a7f8bf7d75b1 """ __tablename__ = "config_cache" name: Mapped[str] = mapped_column(primary_key=True) diff --git a/argos/server/queries.py b/argos/server/queries.py index 6d4d790..742c516 100644 --- a/argos/server/queries.py +++ b/argos/server/queries.py @@ -62,7 +62,7 @@ async def count_results(db: Session): return db.query(Result).count() -async def is_config_unchanged(db: Session, config: schemas.Config) -> bool: +async def has_config_changed(db: Session, config: schemas.Config) -> bool: """Check if websites config has changed by using a hashsum and a config cache""" websites_hash = sha256(str(config.websites).encode()).hexdigest() conf_caches = ( @@ -72,24 +72,24 @@ async def is_config_unchanged(db: Session, config: schemas.Config) -> bool: same_config = True if conf_caches: for conf in conf_caches: - if not same_config: - break + match (conf.name): + case 'websites_hash': + if conf.val != websites_hash: + same_config = False + conf.val = websites_hash + conf.updated_at = datetime.now() + case 'general_frequency': + if conf.val != str(config.general.frequency): + same_config = False + conf.val = config.general.frequency + conf.updated_at = datetime.now() - if conf.name == 'websites_hash': - same_config = conf.val == websites_hash - elif conf.name == 'general_frequency': - same_config = conf.val == str(config.general.frequency) + db.commit() if same_config: - return True + return False - for conf in conf_caches: - if conf.name == 'websites_hash': - conf.val = websites_hash - elif conf.name == 'general_frequency': - conf.val = config.general.frequency - conf.updated_at = datetime.now() - else: + else: # no config cache found web_hash = ConfigCache( name='websites_hash', val=websites_hash, @@ -104,13 +104,13 @@ async def is_config_unchanged(db: Session, config: schemas.Config) -> bool: db.add(gen_freq) db.commit() - return False + return True async def update_from_config(db: Session, config: schemas.Config): """Update tasks from config file""" - config_unchanged = await is_config_unchanged(db, config) - if config_unchanged: + config_changed = await has_config_changed(db, config) + if not config_changed: return {'added': 0, 'vanished': 0} max_task_id = ( From 064e43dc01d4edbad715a3fc9c1477f4b1fbf7ad Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Mon, 18 Mar 2024 15:59:41 +0100 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=97=83=20=E2=80=94=20Add=20ON=20DELET?= =?UTF-8?q?E=20CASCADE=20to=20results=E2=80=99=20task=5Fid=20constraint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- alembic/env.py | 6 +++- .../7d480e6f1112_initial_migrations.py | 1 + ...dd_on_delete_cascade_to_results_task_id.py | 33 +++++++++++++++++++ argos/server/main.py | 9 ++++- argos/server/models.py | 9 ++--- 5 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/defda3f2952d_add_on_delete_cascade_to_results_task_id.py diff --git a/alembic/env.py b/alembic/env.py index 347ff3f..4f34ce8 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -28,6 +28,7 @@ def run_migrations_offline() -> None: context.configure( url=url, target_metadata=target_metadata, + render_as_batch=True, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) @@ -50,7 +51,10 @@ def run_migrations_online() -> None: ) with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + context.configure(connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) with context.begin_transaction(): context.run_migrations() diff --git a/alembic/versions/7d480e6f1112_initial_migrations.py b/alembic/versions/7d480e6f1112_initial_migrations.py index 87bb415..3382fe1 100644 --- a/alembic/versions/7d480e6f1112_initial_migrations.py +++ b/alembic/versions/7d480e6f1112_initial_migrations.py @@ -53,6 +53,7 @@ def upgrade() -> None: sa.ForeignKeyConstraint( ["task_id"], ["tasks.id"], + name="results_task_id_fkey", ), sa.PrimaryKeyConstraint("id"), ) diff --git a/alembic/versions/defda3f2952d_add_on_delete_cascade_to_results_task_id.py b/alembic/versions/defda3f2952d_add_on_delete_cascade_to_results_task_id.py new file mode 100644 index 0000000..777cfa2 --- /dev/null +++ b/alembic/versions/defda3f2952d_add_on_delete_cascade_to_results_task_id.py @@ -0,0 +1,33 @@ +"""Add ON DELETE CASCADE to results’ task_id + +Revision ID: defda3f2952d +Revises: 1a3497f9f71b +Create Date: 2024-03-18 15:09:34.544573 + +""" +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = 'defda3f2952d' +down_revision: Union[str, None] = '1a3497f9f71b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table('results', schema=None) as batch_op: + batch_op.drop_constraint('results_task_id_fkey', type_='foreignkey') + batch_op.create_foreign_key('results_task_id_fkey', + 'tasks', + ['task_id'], + ['id'], + ondelete='CASCADE') + + +def downgrade() -> None: + with op.batch_alter_table('results', schema=None) as batch_op: + batch_op.drop_constraint('results_task_id_fkey', type_='foreignkey') + batch_op.create_foreign_key('results_task_id_fkey', 'tasks', ['task_id'], ['id']) diff --git a/argos/server/main.py b/argos/server/main.py index d3e175e..78eff8d 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -3,7 +3,7 @@ import sys from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from pydantic import ValidationError -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker from argos.logging import logger @@ -94,6 +94,13 @@ def setup_database(appli): settings.database_url, **extra_settings ) + + def _fk_pragma_on_connect(dbapi_con, con_record): + dbapi_con.execute('pragma foreign_keys=ON') + + if settings.database_url.startswith("sqlite:////"): + event.listen(engine, 'connect', _fk_pragma_on_connect) + appli.state.SessionLocal = sessionmaker( autocommit=False, autoflush=False, bind=engine ) diff --git a/argos/server/models.py b/argos/server/models.py index 75ed8ab..866ccd6 100644 --- a/argos/server/models.py +++ b/argos/server/models.py @@ -47,7 +47,9 @@ class Task(Base): ) last_severity_update: Mapped[datetime] = mapped_column(nullable=True) - results: Mapped[List["Result"]] = relationship(back_populates="task") + results: Mapped[List["Result"]] = relationship(back_populates="task", + cascade="all, delete", + passive_deletes=True,) def __str__(self): return f"DB Task {self.url} - {self.check} - {self.expected}" @@ -92,9 +94,8 @@ class Result(Base): """ __tablename__ = "results" id: Mapped[int] = mapped_column(primary_key=True) - task_id: Mapped[int] = mapped_column(ForeignKey("tasks.id")) - task: Mapped["Task"] = relationship(back_populates="results", - cascade="save-update, merge, delete") + task_id: Mapped[int] = mapped_column(ForeignKey("tasks.id", ondelete="CASCADE")) + task: Mapped["Task"] = relationship(back_populates="results") agent_id: Mapped[str] = mapped_column(nullable=True) submitted_at: Mapped[datetime] = mapped_column() From d19622060f8527bf233fe94b9bb3a8262271151d Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Mon, 18 Mar 2024 16:05:00 +0100 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=8E=A8=20=E2=80=94=20Rename=20connect?= =?UTF-8?q?=5Fdb=5Fat=5Fstartup=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- argos/server/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argos/server/main.py b/argos/server/main.py index 78eff8d..45c2c5f 100644 --- a/argos/server/main.py +++ b/argos/server/main.py @@ -41,12 +41,12 @@ def create_start_app_handler(appli): """Warmup the server: setup database connection """ - async def connect_db_at_startup(): + async def _get_db(): setup_database(appli) return await connect_to_db(appli) - return connect_db_at_startup + return _get_db async def connect_to_db(appli): From 6c90543392b1f0b3df2c46607b8c6963d04272f9 Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Thu, 21 Mar 2024 15:52:17 +0100 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=A9=B9=20=E2=80=94=20Fix=20Alembic=20?= =?UTF-8?q?version=20after=20pull=20rebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- alembic/versions/1a3497f9f71b_adding_configcache_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/1a3497f9f71b_adding_configcache_model.py b/alembic/versions/1a3497f9f71b_adding_configcache_model.py index 645bdc7..66befb1 100644 --- a/alembic/versions/1a3497f9f71b_adding_configcache_model.py +++ b/alembic/versions/1a3497f9f71b_adding_configcache_model.py @@ -1,7 +1,7 @@ """Adding ConfigCache model Revision ID: 1a3497f9f71b -Revises: 7d480e6f1112 +Revises: e99bc35702c9 Create Date: 2024-03-13 15:28:09.185377 """ @@ -13,7 +13,7 @@ from alembic import op # revision identifiers, used by Alembic. revision: str = '1a3497f9f71b' -down_revision: Union[str, None] = '7d480e6f1112' +down_revision: Union[str, None] = 'e99bc35702c9' branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None