diff --git a/debian/source/options b/debian/source/options
index 57156b1..b394f06 100644
--- a/debian/source/options
+++ b/debian/source/options
@@ -1,4 +1,7 @@
 compression = "gzip"
 tar-ignore = "dev_scripts"
 tar-ignore = ".*"
-tar-ignore = "__pycache__"
\ No newline at end of file
+tar-ignore = "__pycache__"
+# Ignore the 'share/tessdata' dir, since it slows down the process, and we
+# install Tesseract data via Debian packages anyway.
+tar-ignore = "share/tessdata"
diff --git a/install/linux/build-rpm.py b/install/linux/build-rpm.py
index 56f5ab2..1cec7d2 100755
--- a/install/linux/build-rpm.py
+++ b/install/linux/build-rpm.py
@@ -64,10 +64,15 @@ def build(build_dir, qubes=False):
     os.symlink(dist_path, srpm_dir)
 
     print("* Creating a Python sdist")
+    tessdata = root / "share" / "tessdata"
+    tessdata_bak = root / "tessdata.bak"
     container_tar_gz = root / "share" / "container.tar.gz"
     container_tar_gz_bak = root / "container.tar.gz.bak"
+
+    if tessdata.exists():
+        tessdata.rename(tessdata_bak)
     stash_container = qubes and container_tar_gz.exists()
-    if stash_container:
+    if stash_container and container_tar_gz.exists():
         container_tar_gz.rename(container_tar_gz_bak)
     try:
         subprocess.run(["poetry", "build", "-f", "sdist"], cwd=root, check=True)
@@ -77,7 +82,9 @@ def build(build_dir, qubes=False):
         shutil.copy2(sdist_path, build_dir / "SOURCES" / sdist_name)
         sdist_path.unlink()
     finally:
-        if stash_container:
+        if tessdata_bak.exists():
+            tessdata_bak.rename(tessdata)
+        if stash_container and container_tar_gz_bak.exists():
             container_tar_gz_bak.rename(container_tar_gz)
 
     print("* Building RPM package")