diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee7bd73..48ddd94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,33 +75,8 @@ jobs: name: container.tar path: share/container.tar - download-tessdata: - name: Download and cache Tesseract data - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Cache Tessdata - id: cache-tessdata - uses: actions/cache@v4 - with: - path: share/tessdata/ - key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} - enableCrossOsArchive: true - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Download Tessdata - run: |- - if [ -f "share/tessdata" ]; then - echo "Already cached, skipping" - else - python3 ./install/common/download-tessdata.py - fi - windows: runs-on: windows-latest - needs: - - download-tessdata env: DUMMY_CONVERSION: 1 steps: @@ -111,13 +86,18 @@ jobs: python-version: "3.12" - run: pip install poetry - run: poetry install - - name: Restore cached tessdata - uses: actions/cache/restore@v4 + - name: Cache inventory + uses: actions/cache@v4 with: path: share/tessdata/ - enableCrossOsArchive: true - fail-on-cache-miss: true - key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} + key: v1-inventory-windows-${{ hashFiles('./inventory.lock') }} + - name: Sync inventory + run: |- + if [ -f "share/tessdata" ]; then + echo "Already cached, skipping" + else + poetry run python3 ./dev_scripts/inventory.py sync + fi - name: Run CLI tests run: poetry run make test - name: Set up .NET CLI environment @@ -142,8 +122,6 @@ jobs: macOS: name: "macOS (${{ matrix.arch }})" runs-on: ${{ matrix.runner }} - needs: - - download-tessdata strategy: matrix: include: @@ -158,15 +136,20 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Restore cached tessdata - uses: actions/cache/restore@v4 - with: - path: share/tessdata/ - enableCrossOsArchive: true - fail-on-cache-miss: true - key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} - run: pip install poetry - run: poetry install + - name: Cache inventory + uses: actions/cache@v4 + with: + path: share/tessdata/ + key: v1-inventory-darwin-${{ matrix.arch }}-${{ hashFiles('./inventory.lock') }} + - name: Sync inventory + run: |- + if [ -f "share/tessdata" ]; then + echo "Already cached, skipping" + else + poetry run python3 ./dev_scripts/inventory.py sync + fi - name: Run CLI tests run: poetry run make test - name: Build macOS app @@ -206,7 +189,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" - name: Login to GHCR run: | @@ -379,7 +362,6 @@ jobs: runs-on: ubuntu-latest needs: - build-container-image - - download-tessdata strategy: matrix: include: @@ -410,7 +392,27 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" + + - name: Install inventory dependencies + run: | + sudo apt install pipx + pipx install poetry + poetry install + + - name: Cache inventory + uses: actions/cache@v4 + with: + path: share/tessdata/ + key: v1-inventory-linux-${{ hashFiles('./inventory.lock') }} + + - name: Sync inventory + run: |- + if [ -f "share/tessdata" ]; then + echo "Already cached, skipping" + else + poetry run python3 ./dev_scripts/inventory.py sync + fi - name: Login to GHCR run: | @@ -436,14 +438,6 @@ jobs: share/image-id.txt fail-on-cache-miss: true - - name: Restore cached tessdata - uses: actions/cache/restore@v4 - with: - path: share/tessdata/ - enableCrossOsArchive: true - fail-on-cache-miss: true - key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} - - name: Setup xvfb (Linux) run: | sudo apt update @@ -473,7 +467,7 @@ jobs: # file successfully. xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \ bash -c 'cd dangerzone; poetry run make test' - + - name: Upload PDF diffs uses: actions/upload-artifact@v4 with: diff --git a/BUILD.md b/BUILD.md index 3d95ba5..2a2563b 100644 --- a/BUILD.md +++ b/BUILD.md @@ -75,10 +75,10 @@ Build the latest container: python3 ./install/common/build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 ./install/common/download-tessdata.py +poetry run python3 ./dev_scripts/inventory.py sync ``` Run from source tree: @@ -137,10 +137,10 @@ Build the latest container: python3 ./install/common/build-image.py ``` -Download the OCR language data: +Download the OCR language data and other useful assets: ```sh -python3 ./install/common/download-tessdata.py +python3 ./dev_scripts/inventory sync ``` Run from source tree: @@ -342,10 +342,10 @@ Build the dangerzone container image: python3 ./install/common/build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 ./install/common/download-tessdata.py +poetry run python3 ./dev_scripts/inventory.py sync ``` Run from source tree: @@ -409,10 +409,10 @@ Build the dangerzone container image: python3 .\install\common\build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 .\install\common\download-tessdata.py +poetry run python3 .\dev_scripts\inventory.py sync ``` After that you can launch dangerzone during development with: diff --git a/QA.md b/QA.md index 0f0a760..d1bee95 100644 --- a/QA.md +++ b/QA.md @@ -26,7 +26,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Build and run the Dangerzone .exe - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -35,7 +35,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create and run an app bundle. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -44,7 +44,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create and run an app bundle. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -53,7 +53,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create a .deb package and install it system-wide. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -62,7 +62,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create an .rpm package and install it system-wide. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). diff --git a/RELEASE.md b/RELEASE.md index 135b9d8..2750122 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -145,11 +145,11 @@ Here is what you need to do: poetry sync ``` -- [ ] Build the container image and the OCR language data +- [ ] Build the container image and download the necessary assets ```bash poetry run ./install/common/build-image.py - poetry run ./install/common/download-tessdata.py + poetry run ./dev_scripts/inventory.py sync # Copy the container image to the assets folder cp share/container.tar ~dz/release-assets/$VERSION/dangerzone-$VERSION-arm64.tar @@ -230,6 +230,7 @@ The Windows release is performed in a Windows 11 virtual machine (as opposed to - [ ] Copy the container image into the VM > [!IMPORTANT] > Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar` and `share/image-id.txt` from the host into the `share` folder in the VM. +- [ ] Download the necessary assets with `poetry run .\dev_scripts\inventory.py sync` - [ ] Run `poetry run .\install\windows\build-app.bat` - [ ] When you're done you will have `dist\Dangerzone.msi` diff --git a/dev_scripts/qa.py b/dev_scripts/qa.py index 86d33df..826e751 100755 --- a/dev_scripts/qa.py +++ b/dev_scripts/qa.py @@ -45,7 +45,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Build and run the Dangerzone .exe - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -54,7 +54,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create and run an app bundle. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -63,7 +63,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create and run an app bundle. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -72,7 +72,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create a .deb package and install it system-wide. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -81,7 +81,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version} - [ ] Create a new development environment with Poetry. - [ ] Build the container image and ensure the development environment uses the new image. - - [ ] Download the OCR language data using `./install/common/download-tessdata.py` + - [ ] Download the necessary assets using `./dev_scripts/inventory.py sync` - [ ] Run the Dangerzone tests. - [ ] Create an .rpm package and install it system-wide. - [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below). @@ -292,10 +292,10 @@ Build the latest container: python3 ./install/common/build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 ./install/common/download-tessdata.py +poetry run python3 ./dev_scripts/inventory.py sync ``` Run from source tree: @@ -355,10 +355,10 @@ Build the latest container: python3 ./install/common/build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 ./install/common/download-tessdata.py +poetry run python3 ./dev_scripts/inventory.py sync ``` Run from source tree: @@ -419,10 +419,10 @@ Build the dangerzone container image: python3 .\install\common\build-image.py ``` -Download the OCR language data: +Download the necessary assets: ```sh -python3 .\install\common\download-tessdata.py +poetry run python3 .\dev_scripts\inventory.py sync ``` After that you can launch dangerzone during development with: @@ -758,9 +758,11 @@ class QABase(abc.ABC): self.prompt("Does it pass?", choices=["y", "n"]) logger.info("Successfully completed QA scenarios") - @task("Download Tesseract data", auto=True) - def download_tessdata(self): - self.run("python", str(Path("install", "common", "download-tessdata.py"))) + @task("Download the necessary assets", auto=True) + def sync_inventory(self): + self.run( + "poetry", "run", "python", str(Path("dev_scripts", "inventory.py")), "sync" + ) @classmethod @abc.abstractmethod @@ -862,7 +864,7 @@ class QAWindows(QABase): self.install_docker() self.install_poetry() self.build_image() - self.download_tessdata() + self.sync_inventory() self.run_tests() self.build_dangerzone_exe() @@ -954,7 +956,7 @@ class QALinux(QABase): def start(self): self.build_dev_image() self.build_container_image() - self.download_tessdata() + self.sync_inventory() self.run_tests() self.build_package() self.build_qa_image() diff --git a/dodo.py b/dodo.py index 1a7eda3..af2230b 100644 --- a/dodo.py +++ b/dodo.py @@ -46,8 +46,8 @@ def list_language_data(): return targets -TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"] -TESSDATA_TARGETS = list_language_data() +INVENTORY_DEPS = ["dev_scripts/inventory.py", "inventory.lock"] +INVENTORY_TARGETS = list_language_data() IMAGE_DEPS = [ "Dockerfile", @@ -67,7 +67,7 @@ PYTHON_DEPS = ["poetry.lock", "pyproject.toml"] DMG_DEPS = [ *list_files("install/macos"), - *TESSDATA_TARGETS, + *INVENTORY_TARGETS, *IMAGE_TARGETS, *PYTHON_DEPS, *SOURCE_DEPS, @@ -174,12 +174,12 @@ def task_init_release_dir(): } -def task_download_tessdata(): - """Download the Tesseract data using ./install/common/download-tessdata.py""" +def task_sync_inventory(): + """Download the necessary assets using ./dev_scripts/inventory.py sync""" return { - "actions": ["python install/common/download-tessdata.py"], - "file_dep": TESSDATA_DEPS, - "targets": TESSDATA_TARGETS, + "actions": ["poetry run python3 dev_scripts/inventory.py sync"], + "file_dep": INVENTORY_DEPS, + "targets": INVENTORY_TARGETS, "clean": True, } @@ -233,7 +233,7 @@ def task_macos_build_dmg(): "macos_check_system", "init_release_dir", "poetry_install", - "download_tessdata", + "inventory", ], "targets": [dmg_src, dmg_dst], "clean": True,