From 4a85b63f2873b09dc539458a996de2aa67629415 Mon Sep 17 00:00:00 2001
From: "s0wlz (Matthias Puchstein)" <matthias@puchstein.lu>
Date: Mon, 15 Dec 2025 21:23:43 +0100
Subject: [PATCH] Restore DOP download helpers in raw layouts

---
 .gitignore            | 13 +++++++++++--
 README.md             |  4 ++--
 export_ortho_tiles.py |  4 +++-
 raw/dop/dlscript.sh   | 28 ++++++++++++++++++++++++++++
 raw/dop/filelist.txt  |  3 +++
 raw_dop/dlscript.sh   | 28 ++++++++++++++++++++++++++++
 raw_dop/filelist.txt  |  3 +++
 7 files changed, 78 insertions(+), 5 deletions(-)
 create mode 100755 raw/dop/dlscript.sh
 create mode 100644 raw/dop/filelist.txt
 create mode 100755 raw_dop/dlscript.sh
 create mode 100644 raw_dop/filelist.txt

diff --git a/.gitignore b/.gitignore
index f579b85..99dc961 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,7 +39,13 @@ export_unity/buildings_obj/
 
 # --- Raw downloaded source data (usually too big for git) ---
 *.zip
-raw/
+# Prefer raw/ layout; keep data ignored but allow helper files
+raw/**
+!raw/dop/
+raw/dop/**
+!raw/dop/dlscript.sh
+!raw/dop/filelist.txt
+raw/dop/jp2/**
 archives/
 
 # If you DO want to keep small sample tiles in git for CI/tests,
@@ -53,7 +59,10 @@ tmp/
 temp/
 .cache/
 *.gfs
-raw_dop/
+raw_dop/**
+!raw_dop/dlscript.sh
+!raw_dop/filelist.txt
+raw_dop/jp2/**
 raw_dgm1/
 raw_3dgeb_lod1/
 raw_3dgeb_lod2/
diff --git a/README.md b/README.md
index e816de2..a49e2f1 100644
--- a/README.md
+++ b/README.md
@@ -44,11 +44,11 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 - `_tmp.tif` files in `work/` are transient; you can delete `work/` to force a clean rebuild.
 - Keep file names stable to avoid churn in Unity scenes; re-exports overwrite in place.
 - Large raw datasets are intentionally excluded from version control—document download sources or scripts instead of committing data.
-- Additional inputs: `raw_dop/` contains an HTTPS download helper (`raw_dop/dlscript.sh`) that fetches JP2/J2W/XML orthophotos listed in `filelist.txt`; `3dgeblod1/` and `3dgeblod2/` hold zipped 3D building tiles with object lists for future use.
+- Additional inputs: download helpers live in `raw/dop/dlscript.sh` (legacy: `raw_dop/dlscript.sh`) which fetch JP2/J2W/XML orthophotos listed in `filelist.txt`; `3dgeblod1/` and `3dgeblod2/` hold zipped 3D building tiles with object lists for future use.
 - Handoff to Unity: copy/sync `export_unity/height_png16/` and `export_unity/tile_index.csv` into `DTrierFlood/Assets/GeoData/` before running the Unity-side importer. Keep `OUT_RES` aligned with the importer’s expected resolution (currently 1025).
 
 ### Orthophotos (textures)
-1. Ensure DOP JP2s are present in `raw/dop/jp2/` (legacy: `raw_dop/jp2/`; download helper currently lives in `raw_dop/dlscript.sh`).
+1. Ensure DOP JP2s are present in `raw/dop/jp2/` (legacy: `raw_dop/jp2/`; download helper lives in `raw/dop/dlscript.sh` or `raw_dop/dlscript.sh`).
 2. From `GeoData/`, run:
    ```bash
    uv run python export_ortho_tiles.py
diff --git a/export_ortho_tiles.py b/export_ortho_tiles.py
index 20e416d..dc28d4d 100644
--- a/export_ortho_tiles.py
+++ b/export_ortho_tiles.py
@@ -49,7 +49,9 @@ def export_orthos(args: argparse.Namespace) -> int:
 
     jp2_paths = sorted(glob.glob(os.path.join(raw_ortho_dir, "*.jp2")))
     if not jp2_paths:
-        raise SystemExit(f"No JP2 files found in {raw_ortho_dir}. Run raw_dop/dlscript.sh first.")
+        raise SystemExit(
+            f"No JP2 files found in {raw_ortho_dir}. Run raw/dop/dlscript.sh (or legacy raw_dop/dlscript.sh) first."
+        )
 
     build_vrt(args.vrt_path, jp2_paths)
     vrt_ds = open_dataset(args.vrt_path, f"Could not open VRT at {args.vrt_path}")
diff --git a/raw/dop/dlscript.sh b/raw/dop/dlscript.sh
new file mode 100755
index 0000000..e88048c
--- /dev/null
+++ b/raw/dop/dlscript.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Download DOP JP2/J2W/XML tiles listed in filelist.txt.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIST="$ROOT/filelist.txt"
+OUT="$ROOT/jp2"
+
+if [[ ! -f "$LIST" ]]; then
+  echo "Missing filelist: $LIST" >&2
+  exit 1
+fi
+
+mkdir -p "$OUT"
+
+while IFS= read -r url; do
+  [[ -z "$url" || "$url" =~ ^# ]] && continue
+  fname="${url##*/}"
+  dest="$OUT/$fname"
+  if [[ -f "$dest" ]]; then
+    echo "Exists: $fname"
+    continue
+  fi
+  echo "Downloading $fname"
+  curl -fL "$url" -o "$dest"
+done < "$LIST"
+
+echo "Done. Files in $OUT"
diff --git a/raw/dop/filelist.txt b/raw/dop/filelist.txt
new file mode 100644
index 0000000..2d9b454
--- /dev/null
+++ b/raw/dop/filelist.txt
@@ -0,0 +1,3 @@
+# List JP2 download URLs, one per line. Lines starting with # are ignored.
+# Example:
+# https://example.com/path/to/dop20rgb_32_328_5512_2_rp_2023.jp2
diff --git a/raw_dop/dlscript.sh b/raw_dop/dlscript.sh
new file mode 100755
index 0000000..e88048c
--- /dev/null
+++ b/raw_dop/dlscript.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Download DOP JP2/J2W/XML tiles listed in filelist.txt.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIST="$ROOT/filelist.txt"
+OUT="$ROOT/jp2"
+
+if [[ ! -f "$LIST" ]]; then
+  echo "Missing filelist: $LIST" >&2
+  exit 1
+fi
+
+mkdir -p "$OUT"
+
+while IFS= read -r url; do
+  [[ -z "$url" || "$url" =~ ^# ]] && continue
+  fname="${url##*/}"
+  dest="$OUT/$fname"
+  if [[ -f "$dest" ]]; then
+    echo "Exists: $fname"
+    continue
+  fi
+  echo "Downloading $fname"
+  curl -fL "$url" -o "$dest"
+done < "$LIST"
+
+echo "Done. Files in $OUT"
diff --git a/raw_dop/filelist.txt b/raw_dop/filelist.txt
new file mode 100644
index 0000000..2d9b454
--- /dev/null
+++ b/raw_dop/filelist.txt
@@ -0,0 +1,3 @@
+# List JP2 download URLs, one per line. Lines starting with # are ignored.
+# Example:
+# https://example.com/path/to/dop20rgb_32_328_5512_2_rp_2023.jp2