From 7345278ed19697d1e51d96939c1fee5ed641f26d Mon Sep 17 00:00:00 2001 From: "s0wlz (Matthias Puchstein)" Date: Mon, 15 Dec 2025 21:46:09 +0100 Subject: [PATCH] Switch DOP download helpers to raw/dop20 and update paths --- .gitignore | 5 +++++ AGENTS.md | 4 ++-- README.md | 8 ++++---- export_ortho_tiles.py | 17 ++++++++++------- raw/dop/dlscript.sh | 32 -------------------------------- raw/dop/filelist.txt | 3 --- raw/dop20/dlscript.sh | 28 ++++++++++++++++++++++++++++ raw/dop20/filelist.txt | 10 ++++++++++ raw_dop/dlscript.sh | 32 -------------------------------- raw_dop/filelist.txt | 3 --- setup_dirs.sh | 2 ++ 11 files changed, 61 insertions(+), 83 deletions(-) delete mode 100755 raw/dop/dlscript.sh delete mode 100644 raw/dop/filelist.txt create mode 100755 raw/dop20/dlscript.sh create mode 100644 raw/dop20/filelist.txt delete mode 100755 raw_dop/dlscript.sh delete mode 100644 raw_dop/filelist.txt diff --git a/.gitignore b/.gitignore index 99dc961..14e9f13 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,11 @@ export_unity/buildings_obj/ *.zip # Prefer raw/ layout; keep data ignored but allow helper files raw/** +!raw/dop20/ +raw/dop20/** +!raw/dop20/dlscript.sh +!raw/dop20/filelist.txt +raw/dop20/jp2/** !raw/dop/ raw/dop/** !raw/dop/dlscript.sh diff --git a/AGENTS.md b/AGENTS.md index cd70552..5c5818f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ ## Project Structure & Module Organization - `export_heightmaps.py` is the main pipeline: builds `work/dgm.vrt`, scales heights, and writes `export_unity/height_png16/*.png` plus `export_unity/tile_index.csv`. - `export_ortho_tiles.py` exports orthophotos into `export_unity/ortho_jpg/` using the terrain manifest. -- Preferred raw layout: `raw/dgm1/`, `raw/dop/jp2/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`; legacy paths (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod*/`) still work. `archives/` can hold untouched downloads/zips. +- Preferred raw layout: `raw/dgm1/`, `raw/dop20/jp2/` (or `raw/dop/jp2/`), `raw/citygml/lod1/`, `raw/citygml/lod2/`; legacy paths (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod*/`) still work. `archives/` can hold untouched downloads/zips. - `export_unity/` is safe to sync to Unity projects; treat it as generated output. `work/` holds intermediates and is disposable. ## Build, Test, and Development Commands @@ -11,7 +11,7 @@ - If wheels fail, install system GDAL first (e.g., `brew install gdal` or `apt-get install gdal-bin libgdal-dev`), then rerun `uv sync`. - Prepare the directory tree (raw/archives/work/exports): `bash setup_dirs.sh`. - Heightmap export (rebuilds VRT if absent): `uv run python export_heightmaps.py`. -- Orthophoto export: `uv run python export_ortho_tiles.py` (requires JP2s under `raw/dop/jp2/`; legacy `raw_dop/jp2/` still works). +- Orthophoto export: `uv run python export_ortho_tiles.py` (requires JP2s under `raw/dop20/jp2/`; alt `raw/dop/jp2/` and legacy `raw_dop/jp2/` still work). - Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif` (legacy: `raw_dgm1/*.tif`). - Inspect a result: `gdalinfo export_unity/height_png16/.png | head` to sanity-check bounds and scaling. - Expected warning: `Computed -srcwin ... falls partially outside source raster extent` means the DOP coverage is slightly smaller than the tile footprint; edge pixels will be filled with NoData/zeros. Add adjacent JP2s or shrink the requested window if you need to silence it. diff --git a/README.md b/README.md index a49e2f1..4e61c3d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height - GDAL installed with Python bindings (`osgeo` importable). - Python 3.9+ available on PATH. - DGM1 source tiles placed under `raw/dgm1/` (legacy `raw_dgm1/` still works) as `dgm1___.tif` with matching `.tfw` files. -- Raw inputs (`raw/dop/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`) are **kept out of git**; keep them locally or document how to fetch/regenerate. Legacy `raw_dop/` and `raw_3dgeb_lod*/` remain ignored/compatible. +- Raw inputs (`raw/dop20/`, `raw/dop/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`) are **kept out of git**; keep them locally or document how to fetch/regenerate. Legacy `raw_dop/` and `raw_3dgeb_lod*/` remain ignored/compatible. ### Environment setup (uv) - Create a project venv: `uv venv && source .venv/bin/activate`. @@ -15,7 +15,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height - Create the default directory tree (inputs/archives/outputs): `bash setup_dirs.sh`. ### Repository Layout -- `raw/` — preferred working inputs (not versioned): `raw/dgm1/`, `raw/dop/jp2/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`. Legacy directories (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/`) are still honored by the scripts. +- `raw/` — preferred working inputs (not versioned): `raw/dgm1/`, `raw/dop20/jp2/` (or `raw/dop/jp2/`), `raw/citygml/lod1/`, `raw/citygml/lod2/`. Legacy directories (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod1/`, `raw_3dgeb_lod2/`) are still honored by the scripts. - `archives/` — optional offline storage for untouched downloads (e.g., zipped DOP/CityGML tiles) to keep raw inputs separated from working copies. - `work/` — intermediates such as `dgm.vrt` and `_tmp.tif` files; safe to delete/regenerate. - `export_unity/height_png16/` — final 16-bit PNG heightmaps for Unity import. @@ -44,11 +44,11 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height - `_tmp.tif` files in `work/` are transient; you can delete `work/` to force a clean rebuild. - Keep file names stable to avoid churn in Unity scenes; re-exports overwrite in place. - Large raw datasets are intentionally excluded from version control—document download sources or scripts instead of committing data. -- Additional inputs: download helpers live in `raw/dop/dlscript.sh` (legacy: `raw_dop/dlscript.sh`) which fetch JP2/J2W/XML orthophotos listed in `filelist.txt`; `3dgeblod1/` and `3dgeblod2/` hold zipped 3D building tiles with object lists for future use. +- Additional inputs: download helper lives in `raw/dop20/dlscript.sh` (fallback layouts still work) and pulls JP2/J2W/XML orthophotos listed in `filelist.txt` (one URL per line); `3dgeblod1/` and `3dgeblod2/` hold zipped 3D building tiles with object lists for future use. - Handoff to Unity: copy/sync `export_unity/height_png16/` and `export_unity/tile_index.csv` into `DTrierFlood/Assets/GeoData/` before running the Unity-side importer. Keep `OUT_RES` aligned with the importer’s expected resolution (currently 1025). ### Orthophotos (textures) -1. Ensure DOP JP2s are present in `raw/dop/jp2/` (legacy: `raw_dop/jp2/`; download helper lives in `raw/dop/dlscript.sh` or `raw_dop/dlscript.sh`). +1. Ensure DOP JP2s are present in `raw/dop20/jp2/` (alt: `raw/dop/jp2/`, legacy: `raw_dop/jp2/`); use `raw/dop20/dlscript.sh` to fetch JP2/J2W/XML entries listed in `filelist.txt` (one URL per line). 2. From `GeoData/`, run: ```bash uv run python export_ortho_tiles.py diff --git a/export_ortho_tiles.py b/export_ortho_tiles.py index dc28d4d..80c749a 100644 --- a/export_ortho_tiles.py +++ b/export_ortho_tiles.py @@ -20,9 +20,15 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--raw-ortho-dir-new", dest="raw_ortho_dir_new", - default="raw/dop/jp2", + default="raw/dop20/jp2", help="Preferred directory containing JP2 orthophoto tiles (new layout).", ) + parser.add_argument( + "--raw-ortho-dir-alt", + dest="raw_ortho_dir_alt", + default="raw/dop/jp2", + help="Alternate directory containing JP2 orthophoto tiles (intermediate layout).", + ) parser.add_argument("--vrt-path", default="work/dop.vrt", help="Path to build/read the orthophoto VRT.") parser.add_argument("--tile-index", default="export_unity/tile_index.csv", help="Tile manifest from heightmap export.") parser.add_argument("--out-dir", default="export_unity/ortho_jpg", help="Output directory for cropped orthophotos.") @@ -41,16 +47,13 @@ def export_orthos(args: argparse.Namespace) -> int: ensure_dir(args.out_dir) ensure_parent(args.vrt_path) - raw_ortho_dir = ( - resolve_first_existing([args.raw_ortho_dir_new, args.raw_ortho_dir], "orthophoto input directory") - if args.raw_ortho_dir_new - else args.raw_ortho_dir - ) + candidates = [args.raw_ortho_dir_new, args.raw_ortho_dir_alt, args.raw_ortho_dir] + raw_ortho_dir = resolve_first_existing(candidates, "orthophoto input directory") jp2_paths = sorted(glob.glob(os.path.join(raw_ortho_dir, "*.jp2"))) if not jp2_paths: raise SystemExit( - f"No JP2 files found in {raw_ortho_dir}. Run raw/dop/dlscript.sh (or legacy raw_dop/dlscript.sh) first." + f"No JP2 files found in {raw_ortho_dir}. Run raw/dop20/dlscript.sh (or raw/dop/dlscript.sh, legacy raw_dop/dlscript.sh) first." ) build_vrt(args.vrt_path, jp2_paths) diff --git a/raw/dop/dlscript.sh b/raw/dop/dlscript.sh deleted file mode 100755 index 27d1870..0000000 --- a/raw/dop/dlscript.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Download DOP JP2/J2W/XML tiles listed in filelist.txt. -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -LIST="$ROOT/filelist.txt" -OUT="$ROOT/jp2" - -if [[ ! -f "$LIST" ]]; then - echo "Missing filelist: $LIST" >&2 - exit 1 -fi - -mkdir -p "$OUT" - -while IFS= read -r url; do - [[ -z "$url" || "$url" =~ ^# ]] && continue - for ext in .jp2 .j2w .xml; do - base="${url%.*}" - target_url="${base}${ext}" - fname="${target_url##*/}" - dest="$OUT/$fname" - if [[ -f "$dest" ]]; then - echo "Exists: $fname" - continue - fi - echo "Downloading $fname" - curl -fL "$target_url" -o "$dest" - done -done < "$LIST" - -echo "Done. Files in $OUT" diff --git a/raw/dop/filelist.txt b/raw/dop/filelist.txt deleted file mode 100644 index 2d9b454..0000000 --- a/raw/dop/filelist.txt +++ /dev/null @@ -1,3 +0,0 @@ -# List JP2 download URLs, one per line. Lines starting with # are ignored. -# Example: -# https://example.com/path/to/dop20rgb_32_328_5512_2_rp_2023.jp2 diff --git a/raw/dop20/dlscript.sh b/raw/dop20/dlscript.sh new file mode 100755 index 0000000..ed79f05 --- /dev/null +++ b/raw/dop20/dlscript.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Download DOP assets (JP2/J2W/XML) listed line-by-line in filelist.txt. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIST="$ROOT/filelist.txt" +OUT="$ROOT/jp2" + +if [[ ! -f "$LIST" ]]; then + echo "Missing filelist: $LIST" >&2 + exit 1 +fi + +mkdir -p "$OUT" + +while IFS= read -r url; do + [[ -z "$url" || "$url" =~ ^# ]] && continue + fname="${url##*/}" + dest="$OUT/$fname" + if [[ -f "$dest" ]]; then + echo "Exists: $fname" + continue + fi + echo "Downloading $fname" + curl -fL "$url" -o "$dest" +done < "$LIST" + +echo "Done. Files in $OUT" diff --git a/raw/dop20/filelist.txt b/raw/dop20/filelist.txt new file mode 100644 index 0000000..f1af17b --- /dev/null +++ b/raw/dop20/filelist.txt @@ -0,0 +1,10 @@ +# List DOP asset URLs (JP2, J2W, XML), one per line. Lines starting with # are ignored. +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5510_2_rp_2023.jp2 +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5510_2_rp_2023.j2w +https://geobasis-rlp.de/data/dop20rgb/current/metadata/dop20rgb_32_328_5510_2_rp_2023_meta.xml +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5512_2_rp_2023.jp2 +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5512_2_rp_2023.j2w +https://geobasis-rlp.de/data/dop20rgb/current/metadata/dop20rgb_32_328_5512_2_rp_2023_meta.xml +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5514_2_rp_2023.jp2 +https://geobasis-rlp.de/data/dop20rgb/current/jp2/dop20rgb_32_328_5514_2_rp_2023.j2w +https://geobasis-rlp.de/data/dop20rgb/current/metadata/dop20rgb_32_328_5514_2_rp_2023_meta.xml diff --git a/raw_dop/dlscript.sh b/raw_dop/dlscript.sh deleted file mode 100755 index 27d1870..0000000 --- a/raw_dop/dlscript.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Download DOP JP2/J2W/XML tiles listed in filelist.txt. -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -LIST="$ROOT/filelist.txt" -OUT="$ROOT/jp2" - -if [[ ! -f "$LIST" ]]; then - echo "Missing filelist: $LIST" >&2 - exit 1 -fi - -mkdir -p "$OUT" - -while IFS= read -r url; do - [[ -z "$url" || "$url" =~ ^# ]] && continue - for ext in .jp2 .j2w .xml; do - base="${url%.*}" - target_url="${base}${ext}" - fname="${target_url##*/}" - dest="$OUT/$fname" - if [[ -f "$dest" ]]; then - echo "Exists: $fname" - continue - fi - echo "Downloading $fname" - curl -fL "$target_url" -o "$dest" - done -done < "$LIST" - -echo "Done. Files in $OUT" diff --git a/raw_dop/filelist.txt b/raw_dop/filelist.txt deleted file mode 100644 index 2d9b454..0000000 --- a/raw_dop/filelist.txt +++ /dev/null @@ -1,3 +0,0 @@ -# List JP2 download URLs, one per line. Lines starting with # are ignored. -# Example: -# https://example.com/path/to/dop20rgb_32_328_5512_2_rp_2023.jp2 diff --git a/setup_dirs.sh b/setup_dirs.sh index 76fea74..fc89d46 100644 --- a/setup_dirs.sh +++ b/setup_dirs.sh @@ -6,10 +6,12 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DIRS=( "$ROOT/raw/dgm1" + "$ROOT/raw/dop20/jp2" "$ROOT/raw/dop/jp2" "$ROOT/raw/citygml/lod1" "$ROOT/raw/citygml/lod2" "$ROOT/archives/dgm1" + "$ROOT/archives/dop20" "$ROOT/archives/dop" "$ROOT/archives/citygml/lod1" "$ROOT/archives/citygml/lod2"