From cb0ba6d6b5a30a841359e8f370554ee9ac6f760c Mon Sep 17 00:00:00 2001
From: "s0wlz (Matthias Puchstein)" <matthias@puchstein.lu>
Date: Mon, 15 Dec 2025 21:05:58 +0100
Subject: [PATCH] Adopt raw/archives layout with path-compatible exporters

---
 .gitignore            |  2 ++
 AGENTS.md             |  8 ++++----
 README.md             | 13 ++++++-------
 export_heightmaps.py  | 17 +++++++++++++++--
 export_ortho_tiles.py | 20 ++++++++++++++++----
 gdal_utils.py         |  9 +++++++++
 6 files changed, 52 insertions(+), 17 deletions(-)

diff --git a/.gitignore b/.gitignore
index e170e00..f579b85 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,6 +39,8 @@ export_unity/buildings_obj/
 
 # --- Raw downloaded source data (usually too big for git) ---
 *.zip
+raw/
+archives/
 
 # If you DO want to keep small sample tiles in git for CI/tests,
 # comment out raw_dgm1/ and add a whitelist like:
diff --git a/AGENTS.md b/AGENTS.md
index 5f7f0ce..10c31fd 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -3,15 +3,15 @@
 ## Project Structure & Module Organization
 - `export_heightmaps.py` is the main pipeline: builds `work/dgm.vrt`, scales heights, and writes `export_unity/height_png16/*.png` plus `export_unity/tile_index.csv`.
 - `export_ortho_tiles.py` exports orthophotos into `export_unity/ortho_jpg/` using the terrain manifest.
-- Raw inputs (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/`) and `work/` intermediates are intentionally untracked; do not commit them.
-- `export_unity/` is safe to sync to Unity projects; treat it as generated output.
+- Preferred raw layout: `raw/dgm1/`, `raw/dop/jp2/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`; legacy paths (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod*/`) still work. `archives/` can hold untouched downloads/zips.
+- `export_unity/` is safe to sync to Unity projects; treat it as generated output. `work/` holds intermediates and is disposable.
 
 ## Build, Test, and Development Commands
 - Create a venv: `uv venv && source .venv/bin/activate`. Install deps: `uv sync` (generates `uv.lock`).
 - If wheels fail, install system GDAL first (e.g., `brew install gdal` or `apt-get install gdal-bin libgdal-dev`), then rerun `uv sync`.
 - Heightmap export (rebuilds VRT if absent): `uv run python export_heightmaps.py`.
-- Orthophoto export: `uv run python export_ortho_tiles.py` (requires JP2s under `raw_dop/jp2/`).
-- Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw_dgm1/*.tif`.
+- Orthophoto export: `uv run python export_ortho_tiles.py` (requires JP2s under `raw/dop/jp2/`; legacy `raw_dop/jp2/` still works).
+- Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif` (legacy: `raw_dgm1/*.tif`).
 - Inspect a result: `gdalinfo export_unity/height_png16/<tile>.png | head` to sanity-check bounds and scaling.
 - Expected warning: `Computed -srcwin ... falls partially outside source raster extent` means the DOP coverage is slightly smaller than the tile footprint; edge pixels will be filled with NoData/zeros. Add adjacent JP2s or shrink the requested window if you need to silence it.
 - Scripts accept CLI overrides (e.g., `--out-dir`, `--jpeg-quality`, `--resample`); run `uv run python <script> -h` to see options.
diff --git a/README.md b/README.md
index e4046a9..15678e1 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,8 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 ### Prerequisites
 - GDAL installed with Python bindings (`osgeo` importable).
 - Python 3.9+ available on PATH.
-- DGM1 source tiles placed in `raw_dgm1/` as `dgm1_<utm_zone>_<easting>_<northing>.tif` (with matching `.tfw` files).
-- Raw inputs (`raw_dop/`, `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/`) are **kept out of git**; keep them locally or document how to fetch/regenerate.
+- DGM1 source tiles placed under `raw/dgm1/` (legacy `raw_dgm1/` still works) as `dgm1_<utm_zone>_<easting>_<northing>.tif` with matching `.tfw` files.
+- Raw inputs (`raw/dop/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`) are **kept out of git**; keep them locally or document how to fetch/regenerate. Legacy `raw_dop/` and `raw_3dgeb_lod*/` remain ignored/compatible.
 
 ### Environment setup (uv)
 - Create a project venv: `uv venv && source .venv/bin/activate`.
@@ -14,9 +14,8 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 - If wheels fail to resolve, ensure system GDAL is present (e.g., `brew install gdal` or `apt-get install gdal-bin libgdal-dev`), then rerun `uv sync`.
 
 ### Repository Layout
-- `raw_dgm1/` — input rasters (not versioned).
-- `raw_dop/` — raw orthophoto downloads (JP2/J2W/XML), ignored in git.
-- `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/` — CityGML building tiles, ignored in git.
+- `raw/` — preferred working inputs (not versioned): `raw/dgm1/`, `raw/dop/jp2/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`. Legacy directories (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/`) are still honored by the scripts.
+- `archives/` — optional offline storage for untouched downloads (e.g., zipped DOP/CityGML tiles) to keep raw inputs separated from working copies.
 - `work/` — intermediates such as `dgm.vrt` and `_tmp.tif` files; safe to delete/regenerate.
 - `export_unity/height_png16/` — final 16-bit PNG heightmaps for Unity import.
 - `export_unity/tile_index.csv` — manifest mapping tile IDs to world bounds and global min/max used for scaling.
@@ -34,7 +33,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 3. Import the PNGs into Unity Terrains using `tile_index.csv` for placement and consistent height scaling (0–65535).
 
 ### Key Commands
-- Refresh VRT: `gdalbuildvrt work/dgm.vrt raw_dgm1/*.tif`
+- Refresh VRT: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif` (legacy: `raw_dgm1/*.tif`)
 - Run export pipeline: `uv run python export_heightmaps.py`
 - Inspect an output tile: `gdalinfo export_unity/height_png16/<tile>.png | head`
 - Override defaults (e.g., orthophoto out dir): `uv run python export_ortho_tiles.py --out-dir export_unity/ortho_jpg` (see `-h` on each script for tunables).
@@ -48,7 +47,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 - Handoff to Unity: copy/sync `export_unity/height_png16/` and `export_unity/tile_index.csv` into `DTrierFlood/Assets/GeoData/` before running the Unity-side importer. Keep `OUT_RES` aligned with the importer’s expected resolution (currently 1025).
 
 ### Orthophotos (textures)
-1. Ensure DOP JP2s are present in `raw_dop/jp2/` (use `raw_dop/dlscript.sh` if needed).
+1. Ensure DOP JP2s are present in `raw/dop/jp2/` (legacy: `raw_dop/jp2/`; download helper currently lives in `raw_dop/dlscript.sh`).
 2. From `GeoData/`, run:
    ```bash
    uv run python export_ortho_tiles.py
diff --git a/export_heightmaps.py b/export_heightmaps.py
index 49991e9..09edeec 100644
--- a/export_heightmaps.py
+++ b/export_heightmaps.py
@@ -15,6 +15,7 @@ from gdal_utils import (
     ensure_dir,
     ensure_parent,
     open_dataset,
+    resolve_first_existing,
     safe_remove,
 )
 
@@ -24,6 +25,12 @@ gdal.UseExceptions()
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Export heightmaps and manifest from DGM tiles.")
     parser.add_argument("--raw-dir", default="raw_dgm1", help="Directory containing input DGM GeoTIFFs.")
+    parser.add_argument(
+        "--raw-dir-new",
+        dest="raw_dir_new",
+        default="raw/dgm1",
+        help="Preferred directory containing input DGM GeoTIFFs (new layout).",
+    )
     parser.add_argument("--vrt-path", default="work/dgm.vrt", help="Path to build/read the DGM VRT.")
     parser.add_argument("--out-dir", default="export_unity/height_png16", help="Output directory for PNG heightmaps.")
     parser.add_argument(
@@ -61,7 +68,13 @@ def export_heightmaps(args: argparse.Namespace) -> int:
     ensure_dir(args.out_dir)
     ensure_parent(args.manifest_path)
 
-    tif_paths = sorted(glob.glob(os.path.join(args.raw_dir, "*.tif")))
+    raw_dir = (
+        resolve_first_existing([args.raw_dir_new, args.raw_dir], "DGM input directory")
+        if args.raw_dir_new
+        else args.raw_dir
+    )
+
+    tif_paths = sorted(glob.glob(os.path.join(raw_dir, "*.tif")))
     build_vrt(args.vrt_path, tif_paths)
     ds = open_dataset(args.vrt_path, f"Could not open {args.vrt_path} after attempting to build it.")
 
@@ -136,7 +149,7 @@ def export_heightmaps(args: argparse.Namespace) -> int:
     print(f"Manifest: {args.manifest_path}")
     print(f"Summary: wrote {written} tiles; skipped {skipped}.")
     if not args.skip_cleanup:
-        removed = cleanup_aux_files(build_patterns(args.raw_dir))
+        removed = cleanup_aux_files(build_patterns(raw_dir))
         print(f"Cleanup removed {removed} temporary files/sidecars.")
 
     return 1 if skipped else 0
diff --git a/export_ortho_tiles.py b/export_ortho_tiles.py
index 9714676..20e416d 100644
--- a/export_ortho_tiles.py
+++ b/export_ortho_tiles.py
@@ -9,14 +9,20 @@ import os
 
 from osgeo import gdal
 
-from gdal_utils import build_vrt, ensure_dir, ensure_parent, open_dataset
+from gdal_utils import build_vrt, ensure_dir, ensure_parent, open_dataset, resolve_first_existing
 
 gdal.UseExceptions()
 
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Export orthophoto tiles aligned to the terrain grid.")
-    parser.add_argument("--raw-ortho-dir", default="raw_dop/jp2", help="Directory containing JP2 orthophoto tiles.")
+    parser.add_argument("--raw-ortho-dir", default="raw_dop/jp2", help="Legacy directory containing JP2 orthophoto tiles.")
+    parser.add_argument(
+        "--raw-ortho-dir-new",
+        dest="raw_ortho_dir_new",
+        default="raw/dop/jp2",
+        help="Preferred directory containing JP2 orthophoto tiles (new layout).",
+    )
     parser.add_argument("--vrt-path", default="work/dop.vrt", help="Path to build/read the orthophoto VRT.")
     parser.add_argument("--tile-index", default="export_unity/tile_index.csv", help="Tile manifest from heightmap export.")
     parser.add_argument("--out-dir", default="export_unity/ortho_jpg", help="Output directory for cropped orthophotos.")
@@ -35,9 +41,15 @@ def export_orthos(args: argparse.Namespace) -> int:
     ensure_dir(args.out_dir)
     ensure_parent(args.vrt_path)
 
-    jp2_paths = sorted(glob.glob(os.path.join(args.raw_ortho_dir, "*.jp2")))
+    raw_ortho_dir = (
+        resolve_first_existing([args.raw_ortho_dir_new, args.raw_ortho_dir], "orthophoto input directory")
+        if args.raw_ortho_dir_new
+        else args.raw_ortho_dir
+    )
+
+    jp2_paths = sorted(glob.glob(os.path.join(raw_ortho_dir, "*.jp2")))
     if not jp2_paths:
-        raise SystemExit(f"No JP2 files found in {args.raw_ortho_dir}. Run raw_dop/dlscript.sh first.")
+        raise SystemExit(f"No JP2 files found in {raw_ortho_dir}. Run raw_dop/dlscript.sh first.")
 
     build_vrt(args.vrt_path, jp2_paths)
     vrt_ds = open_dataset(args.vrt_path, f"Could not open VRT at {args.vrt_path}")
diff --git a/gdal_utils.py b/gdal_utils.py
index 6d0c4ed..f3a8a1c 100644
--- a/gdal_utils.py
+++ b/gdal_utils.py
@@ -69,3 +69,12 @@ def cleanup_aux_files(patterns: Iterable[str]) -> int:
             if safe_remove(match):
                 removed += 1
     return removed
+
+
+def resolve_first_existing(paths: Sequence[str], label: str) -> str:
+    """Return the first existing path in order; fail with a clear message."""
+    for path in paths:
+        if path and os.path.exists(path):
+            return path
+    tried = ", ".join(paths)
+    raise SystemExit(f"No existing path found for {label}. Checked: {tried}")