mirror of
https://github.com/freedomofpress/dangerzone.git
synced 2025-05-08 22:41:50 +02:00
Replace download-tessdata with inventory.py sync
This commit is contained in:
Alex Pyrgiotis
parent
05deb6c464
commit
57f59c77d3
6 changed files with 88 additions and 91 deletions
96
.github/workflows/ci.yml
vendored
96
.github/workflows/ci.yml
vendored
|
|
@ -75,33 +75,8 @@ jobs:
|
|||
name: container.tar
|
||||
path: share/container.tar
|
||||
|
||||
download-tessdata:
|
||||
name: Download and cache Tesseract data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache Tessdata
|
||||
id: cache-tessdata
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
enableCrossOsArchive: true
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Download Tessdata
|
||||
run: |-
|
||||
if [ -f "share/tessdata" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
python3 ./install/common/download-tessdata.py
|
||||
fi
|
||||
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
needs:
|
||||
- download-tessdata
|
||||
env:
|
||||
DUMMY_CONVERSION: 1
|
||||
steps:
|
||||
|
|
@ -111,13 +86,18 @@ jobs:
|
|||
python-version: "3.12"
|
||||
- run: pip install poetry
|
||||
- run: poetry install
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
- name: Cache inventory
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
key: v1-inventory-windows-${{ hashFiles('./inventory.lock') }}
|
||||
- name: Sync inventory
|
||||
run: |-
|
||||
if [ -f "share/tessdata" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
fi
|
||||
- name: Run CLI tests
|
||||
run: poetry run make test
|
||||
- name: Set up .NET CLI environment
|
||||
|
|
@ -142,8 +122,6 @@ jobs:
|
|||
macOS:
|
||||
name: "macOS (${{ matrix.arch }})"
|
||||
runs-on: ${{ matrix.runner }}
|
||||
needs:
|
||||
- download-tessdata
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
|
@ -158,15 +136,20 @@ jobs:
|
|||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
- run: pip install poetry
|
||||
- run: poetry install
|
||||
- name: Cache inventory
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
key: v1-inventory-darwin-${{ matrix.arch }}-${{ hashFiles('./inventory.lock') }}
|
||||
- name: Sync inventory
|
||||
run: |-
|
||||
if [ -f "share/tessdata" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
fi
|
||||
- name: Run CLI tests
|
||||
run: poetry run make test
|
||||
- name: Build macOS app
|
||||
|
|
@ -206,7 +189,7 @@ jobs:
|
|||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Login to GHCR
|
||||
run: |
|
||||
|
|
@ -379,7 +362,6 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-container-image
|
||||
- download-tessdata
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
|
|
@ -410,7 +392,27 @@ jobs:
|
|||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install inventory dependencies
|
||||
run: |
|
||||
sudo apt install pipx
|
||||
pipx install poetry
|
||||
poetry install
|
||||
|
||||
- name: Cache inventory
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
key: v1-inventory-linux-${{ hashFiles('./inventory.lock') }}
|
||||
|
||||
- name: Sync inventory
|
||||
run: |-
|
||||
if [ -f "share/tessdata" ]; then
|
||||
echo "Already cached, skipping"
|
||||
else
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
fi
|
||||
|
||||
- name: Login to GHCR
|
||||
run: |
|
||||
|
|
@ -436,14 +438,6 @@ jobs:
|
|||
share/image-id.txt
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Restore cached tessdata
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: share/tessdata/
|
||||
enableCrossOsArchive: true
|
||||
fail-on-cache-miss: true
|
||||
key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }}
|
||||
|
||||
- name: Setup xvfb (Linux)
|
||||
run: |
|
||||
sudo apt update
|
||||
|
|
@ -473,7 +467,7 @@ jobs:
|
|||
# file successfully.
|
||||
xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \
|
||||
bash -c 'cd dangerzone; poetry run make test'
|
||||
|
||||
|
||||
- name: Upload PDF diffs
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
|
|
|
|||
16
BUILD.md
16
BUILD.md
|
|
@ -75,10 +75,10 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
|
@ -137,10 +137,10 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the OCR language data and other useful assets:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
python3 ./dev_scripts/inventory sync
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
|
@ -342,10 +342,10 @@ Build the dangerzone container image:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
|
@ -409,10 +409,10 @@ Build the dangerzone container image:
|
|||
python3 .\install\common\build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 .\install\common\download-tessdata.py
|
||||
poetry run python3 .\dev_scripts\inventory.py sync
|
||||
```
|
||||
|
||||
After that you can launch dangerzone during development with:
|
||||
|
|
|
|||
10
QA.md
10
QA.md
|
|
@ -26,7 +26,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Build and run the Dangerzone .exe
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -35,7 +35,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -44,7 +44,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -53,7 +53,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a .deb package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -62,7 +62,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create an .rpm package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
|
|||
|
|
@ -145,11 +145,11 @@ Here is what you need to do:
|
|||
poetry sync
|
||||
```
|
||||
|
||||
- [ ] Build the container image and the OCR language data
|
||||
- [ ] Build the container image and download the necessary assets
|
||||
|
||||
```bash
|
||||
poetry run ./install/common/build-image.py
|
||||
poetry run ./install/common/download-tessdata.py
|
||||
poetry run ./dev_scripts/inventory.py sync
|
||||
|
||||
# Copy the container image to the assets folder
|
||||
cp share/container.tar ~dz/release-assets/$VERSION/dangerzone-$VERSION-arm64.tar
|
||||
|
|
@ -230,6 +230,7 @@ The Windows release is performed in a Windows 11 virtual machine (as opposed to
|
|||
- [ ] Copy the container image into the VM
|
||||
> [!IMPORTANT]
|
||||
> Instead of running `python .\install\windows\build-image.py` in the VM, run the build image script on the host (making sure to build for `linux/amd64`). Copy `share/container.tar` and `share/image-id.txt` from the host into the `share` folder in the VM.
|
||||
- [ ] Download the necessary assets with `poetry run .\dev_scripts\inventory.py sync`
|
||||
- [ ] Run `poetry run .\install\windows\build-app.bat`
|
||||
- [ ] When you're done you will have `dist\Dangerzone.msi`
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Build and run the Dangerzone .exe
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -54,7 +54,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -63,7 +63,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create and run an app bundle.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -72,7 +72,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create a .deb package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -81,7 +81,7 @@ poetry run ./dev_scripts/qa.py {distro}-{version}
|
|||
- [ ] Create a new development environment with Poetry.
|
||||
- [ ] Build the container image and ensure the development environment uses
|
||||
the new image.
|
||||
- [ ] Download the OCR language data using `./install/common/download-tessdata.py`
|
||||
- [ ] Download the necessary assets using `./dev_scripts/inventory.py sync`
|
||||
- [ ] Run the Dangerzone tests.
|
||||
- [ ] Create an .rpm package and install it system-wide.
|
||||
- [ ] Test some QA scenarios (see [Scenarios](#Scenarios) below).
|
||||
|
|
@ -292,10 +292,10 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
|
@ -355,10 +355,10 @@ Build the latest container:
|
|||
python3 ./install/common/build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 ./install/common/download-tessdata.py
|
||||
poetry run python3 ./dev_scripts/inventory.py sync
|
||||
```
|
||||
|
||||
Run from source tree:
|
||||
|
|
@ -419,10 +419,10 @@ Build the dangerzone container image:
|
|||
python3 .\install\common\build-image.py
|
||||
```
|
||||
|
||||
Download the OCR language data:
|
||||
Download the necessary assets:
|
||||
|
||||
```sh
|
||||
python3 .\install\common\download-tessdata.py
|
||||
poetry run python3 .\dev_scripts\inventory.py sync
|
||||
```
|
||||
|
||||
After that you can launch dangerzone during development with:
|
||||
|
|
@ -758,9 +758,11 @@ class QABase(abc.ABC):
|
|||
self.prompt("Does it pass?", choices=["y", "n"])
|
||||
logger.info("Successfully completed QA scenarios")
|
||||
|
||||
@task("Download Tesseract data", auto=True)
|
||||
def download_tessdata(self):
|
||||
self.run("python", str(Path("install", "common", "download-tessdata.py")))
|
||||
@task("Download the necessary assets", auto=True)
|
||||
def sync_inventory(self):
|
||||
self.run(
|
||||
"poetry", "run", "python", str(Path("dev_scripts", "inventory.py")), "sync"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@abc.abstractmethod
|
||||
|
|
@ -862,7 +864,7 @@ class QAWindows(QABase):
|
|||
self.install_docker()
|
||||
self.install_poetry()
|
||||
self.build_image()
|
||||
self.download_tessdata()
|
||||
self.sync_inventory()
|
||||
self.run_tests()
|
||||
self.build_dangerzone_exe()
|
||||
|
||||
|
|
@ -954,7 +956,7 @@ class QALinux(QABase):
|
|||
def start(self):
|
||||
self.build_dev_image()
|
||||
self.build_container_image()
|
||||
self.download_tessdata()
|
||||
self.sync_inventory()
|
||||
self.run_tests()
|
||||
self.build_package()
|
||||
self.build_qa_image()
|
||||
|
|
|
|||
18
dodo.py
18
dodo.py
|
|
@ -46,8 +46,8 @@ def list_language_data():
|
|||
return targets
|
||||
|
||||
|
||||
TESSDATA_DEPS = ["install/common/download-tessdata.py", "share/ocr-languages.json"]
|
||||
TESSDATA_TARGETS = list_language_data()
|
||||
INVENTORY_DEPS = ["dev_scripts/inventory.py", "inventory.lock"]
|
||||
INVENTORY_TARGETS = list_language_data()
|
||||
|
||||
IMAGE_DEPS = [
|
||||
"Dockerfile",
|
||||
|
|
@ -67,7 +67,7 @@ PYTHON_DEPS = ["poetry.lock", "pyproject.toml"]
|
|||
|
||||
DMG_DEPS = [
|
||||
*list_files("install/macos"),
|
||||
*TESSDATA_TARGETS,
|
||||
*INVENTORY_TARGETS,
|
||||
*IMAGE_TARGETS,
|
||||
*PYTHON_DEPS,
|
||||
*SOURCE_DEPS,
|
||||
|
|
@ -174,12 +174,12 @@ def task_init_release_dir():
|
|||
}
|
||||
|
||||
|
||||
def task_download_tessdata():
|
||||
"""Download the Tesseract data using ./install/common/download-tessdata.py"""
|
||||
def task_sync_inventory():
|
||||
"""Download the necessary assets using ./dev_scripts/inventory.py sync"""
|
||||
return {
|
||||
"actions": ["python install/common/download-tessdata.py"],
|
||||
"file_dep": TESSDATA_DEPS,
|
||||
"targets": TESSDATA_TARGETS,
|
||||
"actions": ["poetry run python3 dev_scripts/inventory.py sync"],
|
||||
"file_dep": INVENTORY_DEPS,
|
||||
"targets": INVENTORY_TARGETS,
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
|
|
@ -233,7 +233,7 @@ def task_macos_build_dmg():
|
|||
"macos_check_system",
|
||||
"init_release_dir",
|
||||
"poetry_install",
|
||||
"download_tessdata",
|
||||
"inventory",
|
||||
],
|
||||
"targets": [dmg_src, dmg_dst],
|
||||
"clean": True,
|
||||
|
|
|
|||
Loading…
Reference in a new issue