diff --git a/AGENTS.md b/AGENTS.md index a204db3..27fdca3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ - Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif`. - Inspect a result: `gdalinfo export_unity/height_png16/.png | head` to sanity-check bounds and scaling. - Populate raw data from archives: `uv run python geodata_to_unity.py --use-archive --export all` (unzips `archive/*` and copies dop20 filelist). +- Rebuild VRTs after moving data: add `--force-vrt`. - Expected warning: `Computed -srcwin ... falls partially outside source raster extent` means the DOP coverage is slightly smaller than the tile footprint; edge pixels will be filled with NoData/zeros. Add adjacent JP2s or shrink the requested window if you need to silence it. - Scripts accept CLI overrides (e.g., `--config`, `--raw-dgm1-path`, `--raw-dop20-path`, `--export`); run `uv run python geodata_to_unity.py -h` to see options. diff --git a/README.md b/README.md index 8110820..bcdf5c0 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ This repository converts DGM1 elevation tiles into Unity-ready 16-bit PNG height - Inspect an output tile: `gdalinfo export_unity/height_png16/.png | head` - Override config paths: use `--config `, `--raw-dgm1-path `, `--raw-dop20-path `. - Use archives to populate raws: `uv run python geodata_to_unity.py --use-archive --export all` (unzips `archive/*` and copies dop20 filelist). +- Rebuild VRTs after moving data: add `--force-vrt`. ### Workflow Notes - The pipeline computes a global min/max from the VRT to scale all tiles consistently; adjust `heightmap.out_res` or `heightmap.resample` in `geodata_config.json` if your AOI or target resolution changes. diff --git a/geodata_pipeline/config.py b/geodata_pipeline/config.py index e117e17..45ed17d 100644 --- a/geodata_pipeline/config.py +++ b/geodata_pipeline/config.py @@ -21,7 +21,6 @@ class RawConfig: class ArchiveConfig: dgm1_dir: str = "archive/dgm1" dop20_dir: str = "archive/dop20" - dop20_filelist: str = "archive/dop20/filelist.txt" citygml_lod1_dir: str = "archive/citygml/lod1" citygml_lod2_dir: str = "archive/citygml/lod2" diff --git a/geodata_pipeline/gdal_utils.py b/geodata_pipeline/gdal_utils.py index 66ca373..cdf8a0b 100644 --- a/geodata_pipeline/gdal_utils.py +++ b/geodata_pipeline/gdal_utils.py @@ -29,9 +29,14 @@ def open_dataset(path: str, purpose: str): return ds -def build_vrt(vrt_path: str, sources: Sequence[str]) -> bool: +def build_vrt(vrt_path: str, sources: Sequence[str], force: bool = False) -> bool: if os.path.exists(vrt_path): - return False + if not force: + return False + try: + os.remove(vrt_path) + except OSError as exc: + raise SystemExit(f"Could not remove existing VRT {vrt_path}: {exc}") from exc if not sources: raise SystemExit(f"No sources available to build VRT {vrt_path}.") ensure_parent(vrt_path) diff --git a/geodata_pipeline/heightmaps.py b/geodata_pipeline/heightmaps.py index 04fb4da..0013dbe 100644 --- a/geodata_pipeline/heightmaps.py +++ b/geodata_pipeline/heightmaps.py @@ -21,13 +21,13 @@ def _cleanup_patterns(raw_dir: str) -> Iterable[str]: ] -def export_heightmaps(cfg: Config) -> int: +def export_heightmaps(cfg: Config, *, force_vrt: bool = False) -> int: ensure_dir(cfg.work.work_dir) ensure_dir(cfg.export.heightmap_dir) ensure_parent(cfg.export.manifest_path) tif_paths = sorted(glob.glob(os.path.join(cfg.raw.dgm1_dir, "*.tif"))) - build_vrt(cfg.work.heightmap_vrt, tif_paths) + build_vrt(cfg.work.heightmap_vrt, tif_paths, force=force_vrt) ds = open_dataset(cfg.work.heightmap_vrt, f"Could not open {cfg.work.heightmap_vrt} after attempting to build it.") band = ds.GetRasterBand(1) diff --git a/geodata_pipeline/orthophotos.py b/geodata_pipeline/orthophotos.py index c361cb2..3e73480 100644 --- a/geodata_pipeline/orthophotos.py +++ b/geodata_pipeline/orthophotos.py @@ -12,7 +12,7 @@ from .gdal_utils import build_vrt, ensure_dir, ensure_parent, open_dataset gdal.UseExceptions() -def export_orthophotos(cfg: Config) -> int: +def export_orthophotos(cfg: Config, *, force_vrt: bool = False) -> int: ensure_dir(cfg.work.work_dir) ensure_dir(cfg.export.ortho_dir) ensure_parent(cfg.work.ortho_vrt) @@ -21,7 +21,7 @@ def export_orthophotos(cfg: Config) -> int: if not jp2_paths: raise SystemExit(f"No JP2 files found in {cfg.raw.dop20_dir}. Run scripts/dlscript_dop20.sh first.") - build_vrt(cfg.work.ortho_vrt, jp2_paths) + build_vrt(cfg.work.ortho_vrt, jp2_paths, force=force_vrt) vrt_ds = open_dataset(cfg.work.ortho_vrt, f"Could not open VRT at {cfg.work.ortho_vrt}") if not os.path.exists(cfg.export.manifest_path): diff --git a/geodata_pipeline/setup_helpers.py b/geodata_pipeline/setup_helpers.py index 6d24105..d432552 100644 --- a/geodata_pipeline/setup_helpers.py +++ b/geodata_pipeline/setup_helpers.py @@ -37,7 +37,10 @@ def materialize_archives(cfg: Config) -> None: _unpack_all(cfg.archives.citygml_lod1_dir, cfg.raw.citygml_lod1_dir) _unpack_all(cfg.archives.citygml_lod2_dir, cfg.raw.citygml_lod2_dir) _unpack_all(cfg.archives.dop20_dir, cfg.raw.dop20_dir) - _copy_filelist(cfg.archives.dop20_filelist, os.path.join(os.path.dirname(cfg.raw.dop20_dir), "filelist.txt")) + _copy_filelist( + os.path.join(cfg.archives.dop20_dir, "filelist.txt"), + os.path.join(os.path.dirname(cfg.raw.dop20_dir), "filelist.txt"), + ) def _unpack_all(archive_dir: str, dest_dir: str) -> None: diff --git a/geodata_to_unity.py b/geodata_to_unity.py index 027f44c..4fa4082 100644 --- a/geodata_to_unity.py +++ b/geodata_to_unity.py @@ -38,6 +38,11 @@ def parse_args(argv: Iterable[str] | None = None) -> argparse.Namespace: action="store_true", help="Create default directory structure and config if missing; still runs export unless --export is omitted.", ) + parser.add_argument( + "--force-vrt", + action="store_true", + help="Rebuild VRTs even if present (useful after moving raw data).", + ) parser.add_argument( "--no-export", action="store_true", @@ -72,9 +77,9 @@ def main(argv: Iterable[str] | None = None) -> int: exit_codes = [] if args.export in ("heightmap", "all"): - exit_codes.append(export_heightmaps(cfg)) + exit_codes.append(export_heightmaps(cfg, force_vrt=args.force_vrt)) if args.export in ("textures", "all"): - exit_codes.append(export_orthophotos(cfg)) + exit_codes.append(export_orthophotos(cfg, force_vrt=args.force_vrt)) return max(exit_codes) if exit_codes else 0 diff --git a/scripts/dlscript_dop20.sh b/scripts/dlscript_dop20.sh index 7412bfa..9f05173 100644 --- a/scripts/dlscript_dop20.sh +++ b/scripts/dlscript_dop20.sh @@ -4,20 +4,27 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" DOP_ROOT="$ROOT/raw/dop20" -LIST="${LIST:-$DOP_ROOT/filelist.txt}" -OUT="$DOP_ROOT/jp2" +LIST="${LIST:-$ROOT/archive/dop20/filelist.txt}" +OUT_JP2="$DOP_ROOT/jp2" +OUT_J2W="$DOP_ROOT/j2w" +OUT_META="$DOP_ROOT/meta" if [[ ! -f "$LIST" ]]; then echo "Missing filelist: $LIST" >&2 exit 1 fi -mkdir -p "$OUT" +mkdir -p "$OUT_JP2" "$OUT_J2W" "$OUT_META" while IFS= read -r url; do [[ -z "$url" || "$url" =~ ^# ]] && continue fname="${url##*/}" - dest="$OUT/$fname" + case "$fname" in + *.jp2) dest="$OUT_JP2/$fname" ;; + *.j2w) dest="$OUT_J2W/$fname" ;; + *_meta.xml) dest="$OUT_META/$fname" ;; + *) echo "Skipping unknown asset type: $fname" >&2; continue ;; + esac if [[ -f "$dest" ]]; then echo "Exists: $fname" continue @@ -26,4 +33,4 @@ while IFS= read -r url; do curl -fL "$url" -o "$dest" done < "$LIST" -echo "Done. Files in $OUT" +echo "Done. Files in $OUT_JP2, $OUT_J2W, $OUT_META" diff --git a/scripts/setup_dirs.sh b/scripts/setup_dirs.sh index b9624be..4693d59 100644 --- a/scripts/setup_dirs.sh +++ b/scripts/setup_dirs.sh @@ -7,6 +7,8 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" DIRS=( "$ROOT/raw/dgm1" "$ROOT/raw/dop20/jp2" + "$ROOT/raw/dop20/j2w" + "$ROOT/raw/dop20/meta" "$ROOT/raw/citygml/lod1" "$ROOT/raw/citygml/lod2" "$ROOT/archive/dgm1"