Align dop20 filelist handling with archive source

AGENTS.md

@@ -16,6 +16,7 @@
 - Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif`.
 - Inspect a result: `gdalinfo export_unity/height_png16/<tile>.png | head` to sanity-check bounds and scaling.
 - Populate raw data from archives: `uv run python geodata_to_unity.py --use-archive --export all` (unzips `archive/*` and copies dop20 filelist).
+- Rebuild VRTs after moving data: add `--force-vrt`.
 - Expected warning: `Computed -srcwin ... falls partially outside source raster extent` means the DOP coverage is slightly smaller than the tile footprint; edge pixels will be filled with NoData/zeros. Add adjacent JP2s or shrink the requested window if you need to silence it.
 - Scripts accept CLI overrides (e.g., `--config`, `--raw-dgm1-path`, `--raw-dop20-path`, `--export`); run `uv run python geodata_to_unity.py -h` to see options.
 

README.md

@@ -43,6 +43,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height
 - Inspect an output tile: `gdalinfo export_unity/height_png16/<tile>.png | head`
 - Override config paths: use `--config <path>`, `--raw-dgm1-path <dir>`, `--raw-dop20-path <dir>`.
 - Use archives to populate raws: `uv run python geodata_to_unity.py --use-archive --export all` (unzips `archive/*` and copies dop20 filelist).
+- Rebuild VRTs after moving data: add `--force-vrt`.
 
 ### Workflow Notes
 - The pipeline computes a global min/max from the VRT to scale all tiles consistently; adjust `heightmap.out_res` or `heightmap.resample` in `geodata_config.json` if your AOI or target resolution changes.

geodata_pipeline/config.py

@@ -21,7 +21,6 @@ class RawConfig:
 class ArchiveConfig:
     dgm1_dir: str = "archive/dgm1"
     dop20_dir: str = "archive/dop20"
-    dop20_filelist: str = "archive/dop20/filelist.txt"
     citygml_lod1_dir: str = "archive/citygml/lod1"
     citygml_lod2_dir: str = "archive/citygml/lod2"
 

geodata_pipeline/gdal_utils.py

@@ -29,9 +29,14 @@ def open_dataset(path: str, purpose: str):
     return ds
 
 
-def build_vrt(vrt_path: str, sources: Sequence[str]) -> bool:
+def build_vrt(vrt_path: str, sources: Sequence[str], force: bool = False) -> bool:
     if os.path.exists(vrt_path):
-        return False
+        if not force:
+            return False
+        try:
+            os.remove(vrt_path)
+        except OSError as exc:
+            raise SystemExit(f"Could not remove existing VRT {vrt_path}: {exc}") from exc
     if not sources:
         raise SystemExit(f"No sources available to build VRT {vrt_path}.")
     ensure_parent(vrt_path)

geodata_pipeline/heightmaps.py

@@ -21,13 +21,13 @@ def _cleanup_patterns(raw_dir: str) -> Iterable[str]:
     ]
 
 
-def export_heightmaps(cfg: Config) -> int:
+def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int:
     ensure_dir(cfg.work.work_dir)
     ensure_dir(cfg.export.heightmap_dir)
     ensure_parent(cfg.export.manifest_path)
 
     tif_paths = sorted(glob.glob(os.path.join(cfg.raw.dgm1_dir, "*.tif")))
-    build_vrt(cfg.work.heightmap_vrt, tif_paths)
+    build_vrt(cfg.work.heightmap_vrt, tif_paths, force=force_vrt)
     ds = open_dataset(cfg.work.heightmap_vrt, f"Could not open {cfg.work.heightmap_vrt} after attempting to build it.")
 
     band = ds.GetRasterBand(1)

geodata_pipeline/orthophotos.py

@@ -12,7 +12,7 @@ from .gdal_utils import build_vrt, ensure_dir, ensure_parent, open_dataset
 gdal.UseExceptions()
 
 
-def export_orthophotos(cfg: Config) -> int:
+def export_orthophotos(cfg: Config, *, force_vrt: bool = False) -> int:
     ensure_dir(cfg.work.work_dir)
     ensure_dir(cfg.export.ortho_dir)
     ensure_parent(cfg.work.ortho_vrt)
@@ -21,7 +21,7 @@ def export_orthophotos(cfg: Config) -> int:
     if not jp2_paths:
         raise SystemExit(f"No JP2 files found in {cfg.raw.dop20_dir}. Run scripts/dlscript_dop20.sh first.")
 
-    build_vrt(cfg.work.ortho_vrt, jp2_paths)
+    build_vrt(cfg.work.ortho_vrt, jp2_paths, force=force_vrt)
     vrt_ds = open_dataset(cfg.work.ortho_vrt, f"Could not open VRT at {cfg.work.ortho_vrt}")
 
     if not os.path.exists(cfg.export.manifest_path):

geodata_pipeline/setup_helpers.py

@@ -37,7 +37,10 @@ def materialize_archives(cfg: Config) -> None:
     _unpack_all(cfg.archives.citygml_lod1_dir, cfg.raw.citygml_lod1_dir)
     _unpack_all(cfg.archives.citygml_lod2_dir, cfg.raw.citygml_lod2_dir)
     _unpack_all(cfg.archives.dop20_dir, cfg.raw.dop20_dir)
-    _copy_filelist(cfg.archives.dop20_filelist, os.path.join(os.path.dirname(cfg.raw.dop20_dir), "filelist.txt"))
+    _copy_filelist(
+        os.path.join(cfg.archives.dop20_dir, "filelist.txt"),
+        os.path.join(os.path.dirname(cfg.raw.dop20_dir), "filelist.txt"),
+    )
 
 
 def _unpack_all(archive_dir: str, dest_dir: str) -> None:

geodata_to_unity.py

@@ -38,6 +38,11 @@ def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace:
         action="store_true",
         help="Create default directory structure and config if missing; still runs export unless --export is omitted.",
     )
+    parser.add_argument(
+        "--force-vrt",
+        action="store_true",
+        help="Rebuild VRTs even if present (useful after moving raw data).",
+    )
     parser.add_argument(
         "--no-export",
         action="store_true",
@@ -72,9 +77,9 @@ def main(argv: Iterable[str] | None = None) -> int:
 
     exit_codes = []
     if args.export in ("heightmap", "all"):
-        exit_codes.append(export_heightmaps(cfg))
+        exit_codes.append(export_heightmaps(cfg, force_vrt=args.force_vrt))
     if args.export in ("textures", "all"):
-        exit_codes.append(export_orthophotos(cfg))
+        exit_codes.append(export_orthophotos(cfg, force_vrt=args.force_vrt))
 
     return max(exit_codes) if exit_codes else 0
 

scripts/dlscript_dop20.sh

@@ -4,20 +4,27 @@ set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 DOP_ROOT="$ROOT/raw/dop20"
-LIST="${LIST:-$DOP_ROOT/filelist.txt}"
-OUT="$DOP_ROOT/jp2"
+LIST="${LIST:-$ROOT/archive/dop20/filelist.txt}"
+OUT_JP2="$DOP_ROOT/jp2"
+OUT_J2W="$DOP_ROOT/j2w"
+OUT_META="$DOP_ROOT/meta"
 
 if [[ ! -f "$LIST" ]]; then
   echo "Missing filelist: $LIST" >&2
   exit 1
 fi
 
-mkdir -p "$OUT"
+mkdir -p "$OUT_JP2" "$OUT_J2W" "$OUT_META"
 
 while IFS= read -r url; do
   [[ -z "$url" || "$url" =~ ^# ]] && continue
   fname="${url##*/}"
-  dest="$OUT/$fname"
+  case "$fname" in
+    *.jp2) dest="$OUT_JP2/$fname" ;;
+    *.j2w) dest="$OUT_J2W/$fname" ;;
+    *_meta.xml) dest="$OUT_META/$fname" ;;
+    *) echo "Skipping unknown asset type: $fname" >&2; continue ;;
+  esac
   if [[ -f "$dest" ]]; then
     echo "Exists: $fname"
     continue
@@ -26,4 +33,4 @@ while IFS= read -r url; do
   curl -fL "$url" -o "$dest"
 done < "$LIST"
 
-echo "Done. Files in $OUT"
+echo "Done. Files in $OUT_JP2, $OUT_J2W, $OUT_META"

scripts/setup_dirs.sh

@@ -7,6 +7,8 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 DIRS=(
   "$ROOT/raw/dgm1"
   "$ROOT/raw/dop20/jp2"
+  "$ROOT/raw/dop20/j2w"
+  "$ROOT/raw/dop20/meta"
   "$ROOT/raw/citygml/lod1"
   "$ROOT/raw/citygml/lod2"
   "$ROOT/archive/dgm1"