From 1cea248a6a5c451cb433275ca66312d1ce91192f Mon Sep 17 00:00:00 2001 From: "s0wlz (Matthias Puchstein)" Date: Mon, 15 Dec 2025 22:28:21 +0100 Subject: [PATCH] Refactor to config-driven CLI and library pipeline --- .gitignore | 13 +- AGENTS.md | 22 +-- README.md | 44 ++--- export_heightmaps.py | 161 +----------------- export_ortho_tiles.py | 107 +----------- geodata_config.example.json | 34 ++++ geodata_pipeline/__init__.py | 1 + geodata_pipeline/config.py | 107 ++++++++++++ .../gdal_utils.py | 17 -- geodata_pipeline/heightmaps.py | 103 +++++++++++ geodata_pipeline/orthophotos.py | 66 +++++++ geodata_pipeline/setup_helpers.py | 59 +++++++ geodata_to_unity.py | 83 +++++++++ pyproject.toml | 9 +- 14 files changed, 505 insertions(+), 321 deletions(-) create mode 100644 geodata_config.example.json create mode 100644 geodata_pipeline/__init__.py create mode 100644 geodata_pipeline/config.py rename gdal_utils.py => geodata_pipeline/gdal_utils.py (67%) create mode 100644 geodata_pipeline/heightmaps.py create mode 100644 geodata_pipeline/orthophotos.py create mode 100644 geodata_pipeline/setup_helpers.py create mode 100644 geodata_to_unity.py diff --git a/.gitignore b/.gitignore index 365d9b2..ef6aaa0 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ export_unity/buildings_obj/ # Keep the manifest by default (remove this line if you want to ignore it too) !export_unity/tile_index.csv +geodata_config.json # --- Raw downloaded source data (usually too big for git) --- *.zip @@ -45,10 +46,7 @@ raw/** raw/dop20/** !raw/dop20/filelist.txt raw/dop20/jp2/** -!raw/dop/ -raw/dop/** -raw/dop/jp2/** -archives/ +archive/ # If you DO want to keep small sample tiles in git for CI/tests, # comment out raw_dgm1/ and add a whitelist like: @@ -61,10 +59,3 @@ tmp/ temp/ .cache/ *.gfs -raw_dop/** -!raw_dop/dlscript.sh -!raw_dop/filelist.txt -raw_dop/jp2/** -raw_dgm1/ -raw_3dgeb_lod1/ -raw_3dgeb_lod2/ diff --git a/AGENTS.md b/AGENTS.md index 1d4f648..a204db3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,26 +1,28 @@ # Repository Guidelines ## Project Structure & Module Organization -- `export_heightmaps.py` is the main pipeline: builds `work/dgm.vrt`, scales heights, and writes `export_unity/height_png16/*.png` plus `export_unity/tile_index.csv`. -- `export_ortho_tiles.py` exports orthophotos into `export_unity/ortho_jpg/` using the terrain manifest. -- Preferred raw layout: `raw/dgm1/`, `raw/dop20/jp2/` (or `raw/dop/jp2/`), `raw/citygml/lod1/`, `raw/citygml/lod2/`; legacy paths (`raw_dgm1/`, `raw_dop/`, `raw_3dgeb_lod*/`) still work. `archives/` can hold untouched downloads/zips. -- `export_unity/` is safe to sync to Unity projects; treat it as generated output. `work/` holds intermediates and is disposable. +- `geodata_to_unity.py` is the main CLI; library code lives in `geodata_pipeline/` (`heightmaps.py`, `orthophotos.py`, `config.py`, `setup_helpers.py`). +- Working inputs (ignored): `raw/dgm1/`, `raw/dop20/jp2/`, `raw/citygml/lod1/`, `raw/citygml/lod2/`. +- Archives (ignored): `archive/dgm1/`, `archive/dop20/`, `archive/citygml/lod1/`, `archive/citygml/lod2/` (zip storage + dop20 filelist). +- Config: `geodata_config.json` (generated) or `geodata_config.example.json` for defaults. +- `export_unity/` is generated output (heightmaps, orthophotos, manifest). `work/` holds intermediates and is disposable. ## Build, Test, and Development Commands - Create a venv: `uv venv && source .venv/bin/activate`. Install deps: `uv sync` (generates `uv.lock`). - If wheels fail, install system GDAL first (e.g., `brew install gdal` or `apt-get install gdal-bin libgdal-dev`), then rerun `uv sync`. -- Prepare the directory tree (raw/archives/work/exports): `bash scripts/setup_dirs.sh`. -- Heightmap export (rebuilds VRT if absent): `uv run python export_heightmaps.py`. -- Orthophoto export: `uv run python export_ortho_tiles.py` (requires JP2s under `raw/dop20/jp2/`; alt `raw/dop/jp2/` and legacy `raw_dop/jp2/` still work). Use `bash scripts/dlscript_dop20.sh` to fetch JP2/J2W/XML listed in `raw/dop20/filelist.txt`. -- Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif` (legacy: `raw_dgm1/*.tif`). +- Prepare the directory tree and config: `uv run python geodata_to_unity.py --setup` (or `bash scripts/setup_dirs.sh` for directories only). +- Heightmap export: `uv run python geodata_to_unity.py --export heightmap`. +- Orthophoto export: `uv run python geodata_to_unity.py --export textures` (requires JP2s under `raw/dop20/jp2/`; use `bash scripts/dlscript_dop20.sh` to fetch JP2/J2W/XML listed in `raw/dop20/filelist.txt`). +- Refresh VRT manually if needed: `gdalbuildvrt work/dgm.vrt raw/dgm1/*.tif`. - Inspect a result: `gdalinfo export_unity/height_png16/.png | head` to sanity-check bounds and scaling. +- Populate raw data from archives: `uv run python geodata_to_unity.py --use-archive --export all` (unzips `archive/*` and copies dop20 filelist). - Expected warning: `Computed -srcwin ... falls partially outside source raster extent` means the DOP coverage is slightly smaller than the tile footprint; edge pixels will be filled with NoData/zeros. Add adjacent JP2s or shrink the requested window if you need to silence it. -- Scripts accept CLI overrides (e.g., `--out-dir`, `--jpeg-quality`, `--resample`); run `uv run python