diff --git a/README.md b/README.md index d475743..b51d837 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Minimum versions supported: * Postgres 12 * PostGIS 3.0 -* osm2pgsql 1.5.0 +* osm2pgsql 1.6.0 ## Minimum Hardware @@ -162,7 +162,6 @@ psql -h localhost -p 5433 -d pgosm -U postgres -c "SELECT COUNT(*) FROM osm.road - See [more in docs/DOCKER-RUN.md](docs/DOCKER-RUN.md) about other ways to customize how PgOSM Flex runs. diff --git a/docs/APPEND-MODE.md b/docs/APPEND-MODE.md new file mode 100644 index 0000000..2361fb8 --- /dev/null +++ b/docs/APPEND-MODE.md @@ -0,0 +1,84 @@ +# Running osm2pgsql in append mode + +---- + +> WARNING: The process described here is experimental and details will probably change! + +---- + +## Using manual steps + +This page documents differences from [MANUAL-STEPS-RUN.md](MANUAL-STEPS-RUN.md) + + +Setup - Need to use Python venv, osmium is a requirement. Something like... + +```bash +python -m venv venv && source venv/bin/activate +cd ~/git/pgosm-flex && pip install -r requirements.txt +``` + + +Run osm2pgsql. Must use `--slim` mode without drop. + + +```bash +cd pgosm-flex/flex-config + +osm2pgsql --output=flex --style=./run.lua \ + --slim \ + -d $PGOSM_CONN \ + ~/pgosm-data/district-of-columbia-latest.osm.pbf +``` + +Run the normal post-processing as you normally would. + + +`osm2pgsql-replication` is bundled with osm2pgsql install. + +https://osm2pgsql.org/doc/manual.html#keeping-the-database-up-to-date-with-osm2pgsql-replication + + +Initialize replication. + + +```bash +osm2pgsql-replication init -d $PGOSM_CONN \ + --osm-file ~/pgosm-data/district-of-columbia-latest.osm.pbf +``` + + + +Refresh the data. First clear out data that might violate foreign keys. Packaged +in convenient procedure. + + +```sql +CALL osm.append_data_start(); +``` + +Update the data. + + +```bash +osm2pgsql-replication update -d $PGOSM_CONN \ + -- \ + --output=flex --style=./run.lua \ + --slim \ + -d $PGOSM_CONN +``` + +Refresh Mat views, rebuilds nested place polygon data. + + +```sql +CALL osm.append_data_finish(); +``` + + +Skip nested: + +```sql +CALL osm.append_data_finish(skip_nested := True); +``` + diff --git a/docs/MANUAL-STEPS-RUN.md b/docs/MANUAL-STEPS-RUN.md index 36803cf..d657eac 100644 --- a/docs/MANUAL-STEPS-RUN.md +++ b/docs/MANUAL-STEPS-RUN.md @@ -214,6 +214,7 @@ The post-processing SQL scripts create a procedure to calculate the nested place ```bash +psql -d $PGOSM_CONN -c "CALL osm.populate_place_polygon_nested();" psql -d $PGOSM_CONN -c "CALL osm.build_nested_admin_polygons();" ``` diff --git a/flex-config/sql/pgosm-meta.sql b/flex-config/sql/pgosm-meta.sql index ed41b73..4dad979 100644 --- a/flex-config/sql/pgosm-meta.sql +++ b/flex-config/sql/pgosm-meta.sql @@ -11,3 +11,47 @@ COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_version IS 'Version of osm2pgsql used COMMENT ON COLUMN osm.pgosm_flex.region IS 'Region specified at run time via env var PGOSM_REGION.'; COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE. Empty string when not defined.'; COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_mode IS 'Indicates which osm2pgsql mode was used, create or append.'; + + + +-- Helper Procedures for allowing updates via osm2pgsql-replication or similar + + +CREATE OR REPLACE PROCEDURE osm.append_data_start() + LANGUAGE plpgsql + AS $$ + + BEGIN + + RAISE NOTICE 'Truncating table osm.place_polygon_nested;'; + TRUNCATE TABLE osm.place_polygon_nested; + +END $$; + + + + +CREATE OR REPLACE PROCEDURE osm.append_data_finish(skip_nested BOOLEAN = False) + LANGUAGE plpgsql + AS $$ + BEGIN + + REFRESH MATERIALIZED VIEW osm.vplace_polygon; + REFRESH MATERIALIZED VIEW osm.vplace_polygon_subdivide; + REFRESH MATERIALIZED VIEW osm.vpoi_all; + + IF $1 = False THEN + RAISE NOTICE 'Populating nested place table'; + CALL osm.populate_place_polygon_nested(); + RAISE NOTICE 'Calculating nesting of place polygons'; + CALL osm.build_nested_admin_polygons(); + + END IF; + + +END $$; + + +COMMENT ON PROCEDURE osm.append_data_start() IS 'Prepares PgOSM Flex database for running osm2pgsql in append mode. Removes records from place_polygon_nested if they existed.'; +COMMENT ON PROCEDURE osm.append_data_finish() IS 'Finalizes PgOSM Flex after osm2pgsql-replication. Refreshes materialized view and (optionally) processes the place_polygon_nested data.'; + diff --git a/flex-config/sql/place.sql b/flex-config/sql/place.sql index d74c1b3..5fdc1b5 100644 --- a/flex-config/sql/place.sql +++ b/flex-config/sql/place.sql @@ -146,17 +146,31 @@ COMMENT ON COLUMN osm.place_polygon_nested.osm_type IS 'Values from place if a p COMMENT ON COLUMN osm.place_polygon_nested.geom IS 'Geometry loaded by osm2pgsql.'; -INSERT INTO osm.place_polygon_nested (osm_id, name, osm_type, admin_level, geom) -SELECT p.osm_id, p.name, p.osm_type, - COALESCE(p.admin_level::INT, 99) AS admin_level, - geom - FROM osm.vplace_polygon p - WHERE (p.boundary = 'administrative' - OR p.osm_type IN ('neighborhood', 'city', 'suburb', 'town', 'admin_level', 'locality') - ) - AND p.name IS NOT NULL -; +CREATE OR REPLACE PROCEDURE osm.populate_place_polygon_nested() +LANGUAGE sql +AS $$ + + + INSERT INTO osm.place_polygon_nested (osm_id, name, osm_type, admin_level, geom) + SELECT p.osm_id, p.name, p.osm_type, + COALESCE(p.admin_level::INT, 99) AS admin_level, + geom + FROM osm.vplace_polygon p + WHERE (p.boundary = 'administrative' + OR p.osm_type IN ('neighborhood', 'city', 'suburb', 'town', 'admin_level', 'locality') + ) + AND p.name IS NOT NULL + AND NOT EXISTS ( + SELECT osm_id + FROM osm.place_polygon_nested n + WHERE n.osm_id = p.osm_id + ) + ; + +$$; + +CALL osm.populate_place_polygon_nested(); CREATE OR REPLACE PROCEDURE osm.build_nested_admin_polygons( diff --git a/flex-config/style/infrastructure.lua b/flex-config/style/infrastructure.lua index 7de4f22..7a9c656 100644 --- a/flex-config/style/infrastructure.lua +++ b/flex-config/style/infrastructure.lua @@ -101,6 +101,9 @@ local function get_osm_type_subtype(object) elseif object.tags.utility then osm_type_table['osm_type'] = 'utility' osm_type_table['osm_subtype'] = nil + elseif object.tags.aeroway then + osm_type_table['osm_type'] = 'aeroway' + osm_type_table['osm_subtype'] = object.tags.aeroway else osm_type_table['osm_type'] = 'unknown' osm_type_table['osm_subtype'] = nil @@ -210,4 +213,4 @@ else nested(object) infrastructure_process_way(object_copy) end -end \ No newline at end of file +end diff --git a/requirements.txt b/requirements.txt index bdaf3ac..280778c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ click>=8.0.1 coverage>=6.1.2 osm2pgsql-tuner==0.0.4 +osmium==3.2.0 psycopg>=3.0.3 psycopg-binary>=3.0.3 sh>=1.14.2 diff --git a/tests/expected/infrastructure_line_osm_type_subtype_count.out b/tests/expected/infrastructure_line_osm_type_subtype_count.out index df269d1..5267e8e 100644 --- a/tests/expected/infrastructure_line_osm_type_subtype_count.out +++ b/tests/expected/infrastructure_line_osm_type_subtype_count.out @@ -1,3 +1,5 @@ +aeroway|runway|2 +aeroway|taxiway|1 power|cable|6 power|line|11 power|minor_line|2 diff --git a/tests/expected/infrastructure_point_osm_type_subtype_count.out b/tests/expected/infrastructure_point_osm_type_subtype_count.out index 9b24357..c8cc8fb 100644 --- a/tests/expected/infrastructure_point_osm_type_subtype_count.out +++ b/tests/expected/infrastructure_point_osm_type_subtype_count.out @@ -1,3 +1,5 @@ +aeroway|helipad|9 +aeroway|navigationaid|1 communications_tower||1 emergency_access||1 emergency_phone||16 diff --git a/tests/expected/infrastructure_polygon_osm_type_subtype_count.out b/tests/expected/infrastructure_polygon_osm_type_subtype_count.out index 6d17157..3afd20a 100644 --- a/tests/expected/infrastructure_polygon_osm_type_subtype_count.out +++ b/tests/expected/infrastructure_polygon_osm_type_subtype_count.out @@ -1,3 +1,9 @@ +aeroway|aerodrome|2 +aeroway|apron|1 +aeroway|hangar|2 +aeroway|helipad|12 +aeroway|heliport|2 +aeroway|runway|3 power|generator|9 power|plant|1 power|substation|15