From 634d077f81d0582385531cd13c06b50bbb0799f3 Mon Sep 17 00:00:00 2001 From: Hugh Saalmans Date: Wed, 15 May 2024 08:55:06 +1000 Subject: [PATCH 1/2] Updates for 202405 release --- README.md | 26 +++++------ create_concordance_file.py | 4 +- data/boundary_concordance_score.csv | 36 +++++++-------- .../00_import_concordance_file.sql | 16 +++---- .../02_create_residential_address_table.sql | 6 +-- .../data-prep/03_copy_to_gda2020_schema.sql | 46 +++++++++---------- .../example-usage/02_join_pc_and_lga_data.sql | 10 ++-- postgres-scripts/xx_test_results.sql | 8 ++-- postgres-scripts/xx_testing.sql | 20 ++++---- testing/01_create_concordance_table.sql | 6 +-- ..._create_postcode_lga_concordance_table.sql | 18 ++++---- testing/01a_abs_bdy_concordances.sql | 2 +- testing/xx_geoscape_testing.sql | 2 +- testing/xx_lga_testing.sql | 6 +-- testing/xx_testing.sql | 18 ++++---- xx_create_concordance_file_and_copy_to_s3.sh | 10 ++-- 16 files changed, 117 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index ba5f01c..290d32e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Concord -A [CSV file](https://minus34.com/opendata/geoscape-202402/boundary_concordance.csv) and supporting scripts for converting data between Australian boundaries. +A [CSV file](https://minus34.com/opendata/geoscape-202405/boundary_concordance.csv) and supporting scripts for converting data between Australian boundaries. It solves the problem of trying to merge 2 or more datasets based on different census or administrative boundaries such as statistical areas or postcodes. @@ -8,7 +8,7 @@ It does this by providing a list of **_concordances_** between pairs of boundari In this context, **_concordance_** describes what % of residential addresses in a "from" boundary fit within a "to" boundary. -Download & import the ~50Mb [concordance file](https://minus34.com/opendata/geoscape-202402/boundary_concordance.csv) into your database or reporting tool to [get started](#get-started). A [script](/postgres-scripts/00_import_concordance_file.sql) for importing into Postgres is also provided. +Download & import the ~50Mb [concordance file](https://minus34.com/opendata/geoscape-202405/boundary_concordance.csv) into your database or reporting tool to [get started](#get-started). A [script](/postgres-scripts/00_import_concordance_file.sql) for importing into Postgres is also provided. ### Example Use Cases @@ -42,7 +42,7 @@ In the [score](/data/boundary_concordance_score.csv) file, the **_error_** measu The concordance file is generated by the following process: -1. Tag all GNAF addresses with 2016 & 2021 ABS Census boundaries and geoscape 202402 Administrative boundaries +1. Tag all GNAF addresses with 2016 & 2021 ABS Census boundaries and geoscape 202405 Administrative boundaries 2. Remove all addresses in non-residential ABS Census 2021 meshblocks 3. Aggregate all residential addresses by a set of _**from**_ boundary and _**to**_ boundary pairs (e.g. postcode to LGA) 4. Determine the % overlap of residential addresses between both boundary types for all boundary pairs @@ -63,7 +63,7 @@ There are 2 options to get the data: #### 1. Download and Import -1. Download the [concordance file](https://minus34.com/opendata/geoscape-202402/boundary_concordance.csv) +1. Download the [concordance file](https://minus34.com/opendata/geoscape-202405/boundary_concordance.csv) 2. Import it into your database/reporting tool of choice. If using Postgres: 1. Edit the file path, schema name & table owner in `00_import_concordance_file.sql` in the [postgres-scripts](/postgres-scripts) folder 2. Run the SQL script to import the file @@ -75,9 +75,9 @@ This requires a knowledge of Python, Postgres & pg_restore. BTW - if the boundary combination you want isn't in the default concordance file - you need to edit the `settings.py` file before running `create_concordance_file.py`. If this is too hard - raise an [issue](https://github.com/iag-geo/concord/issues) and we may be able to generate it for you; noting you shouldn't convert data to a smaller boundary due to the increase in data errors. **Running the script only needs to be done for 3 reasons:** -1. The boundary from/to combination you need isn't in the standard [concordances file](https://minus34.com/opendata/geoscape-202402/boundary_concordance.csv) +1. The boundary from/to combination you need isn't in the standard [concordances file](https://minus34.com/opendata/geoscape-202405/boundary_concordance.csv) 2. It's now the future and we've been too lazy to update the concordances file with the latest boundary data from the ABS and/or Geoscape -3. You have a license of [Geoscape Buildings](https://geoscape.com.auhttps://minus34.com/opendata/geoscape-202402/boundary_concordance.csv/buildings/) or [Geoscape Land Parcels](https://geoscape.com.auhttps://minus34.com/opendata/geoscape-202402/boundary_concordance.csv/land-parcels/) and want to use the _planning zone_ data in those products to: +3. You have a license of [Geoscape Buildings](https://geoscape.com.auhttps://minus34.com/opendata/geoscape-202405/boundary_concordance.csv/buildings/) or [Geoscape Land Parcels](https://geoscape.com.auhttps://minus34.com/opendata/geoscape-202405/boundary_concordance.csv/land-parcels/) and want to use the _planning zone_ data in those products to: 1. Use a more accurate list of residential addresses to determine the data apportionment percentages (see **note** below); or 2. Use a different set of addresses to apportion your data; e.g. industrial or commercial addresses @@ -88,8 +88,8 @@ BTW - if the boundary combination you want isn't in the default concordance file Running the script requires the following open data, available as Postgres dump files, as well as the optional licensed Geoscape data mentioned above: 1. ABS Census 2016 boundaries ([download](https://minus34.com/opendata/census-2016/census_2016_bdys.dmp)) 2. ABS Census 2021 boundaries ([download](https://minus34.com/opendata/census-2021/census_2021_bdys_gda94.dmp)) -3. GNAF from gnaf-loader ([download](https://minus34.com/opendata/geoscape-202402/gnaf-202402.dmp)) -4. Geoscape Administrative Boundaries from gnaf-loader ([download](https://minus34.com/opendata/geoscape-202402/admin-bdys-202402.dmp)) +3. GNAF from gnaf-loader ([download](https://minus34.com/opendata/geoscape-202405/gnaf-202405.dmp)) +4. Geoscape Administrative Boundaries from gnaf-loader ([download](https://minus34.com/opendata/geoscape-202405/admin-bdys-202405.dmp)) 5. ABS Census 2016 data - used to generate error rates only ([download](https://minus34.com/opendata/census-2016/census_2016_data.dmp)) #### Process @@ -117,7 +117,7 @@ The behaviour of the Python script can be controlled by specifying various comma * `--pgpassword` password for accessing the Postgres server. This defaults to the `PGPASSWORD` environment variable if set, otherwise `password`. ##### Optional Arguments -* `--geoscape-version` Geoscape version number in YYYYMM format. Defaults to current year and last release month. e.g. `202402`. +* `--geoscape-version` Geoscape version number in YYYYMM format. Defaults to current year and last release month. e.g. `202405`. * `--gnaf-schema` input schema name to store final GNAF tables in. Also the **output schema** for the concordance table. Defaults to `gnaf_`. * `--admin-schema` input schema name to store final admin boundary tables in. Defaults to `admin_bdys_`. * `--output-table` name of both output concordance table and file. Defaults to `boundary_concordance`. @@ -147,8 +147,8 @@ WITH pc_data AS ( con.to_name AS lga_name, sum(pc.cases::float * con.address_percent / 100.0)::integer AS cases FROM testing.nsw_covid_cases_20220503_postcode AS pc - INNER JOIN gnaf_202402.boundary_concordance AS con ON pc.postcode = con.from_id - WHERE con.from_source = 'geoscape 202402' + INNER JOIN gnaf_202405.boundary_concordance AS con ON pc.postcode = con.from_id + WHERE con.from_source = 'geoscape 202405' AND con.from_bdy = 'postcode' AND con.to_source = 'abs 2016' AND con.to_bdy = 'lga' @@ -167,9 +167,9 @@ FROM testing.nsw_covid_tests_20220503_lga AS lga ## Data Licenses -Incorporates or developed using G-NAF © [Geoscape Australia](https://geoscape.com.au/legalhttps://minus34.com/opendata/geoscape-202402/boundary_concordance.csv-copyright-and-disclaimer/) licensed by the Commonwealth of Australia under the [Open Geo-coded National Address File (G-NAF) End User Licence Agreement](https:/https://minus34.com/opendata/geoscape-202402/boundary_concordance.csv.gov.auhttps://minus34.com/opendata/geoscape-202402/boundary_concordance.csvset/ds-dga-19432f89-dc3a-4ef3-b943-5326ef1dbecc/distribution/dist-dga-09f74802-08b1-4214-a6ea-3591b2753d30/details?q=). +Incorporates or developed using G-NAF © [Geoscape Australia](https://geoscape.com.au/legalhttps://minus34.com/opendata/geoscape-202405/boundary_concordance.csv-copyright-and-disclaimer/) licensed by the Commonwealth of Australia under the [Open Geo-coded National Address File (G-NAF) End User Licence Agreement](https:/https://minus34.com/opendata/geoscape-202405/boundary_concordance.csv.gov.auhttps://minus34.com/opendata/geoscape-202405/boundary_concordance.csvset/ds-dga-19432f89-dc3a-4ef3-b943-5326ef1dbecc/distribution/dist-dga-09f74802-08b1-4214-a6ea-3591b2753d30/details?q=). -Incorporates or developed using Administrative Boundaries © [Geoscape Australia](https://geoscape.com.au/legalhttps://minus34.com/opendata/geoscape-202402/boundary_concordance.csv-copyright-and-disclaimer/) licensed by the Commonwealth of Australia under [Creative Commons Attribution 4.0 International licence (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). +Incorporates or developed using Administrative Boundaries © [Geoscape Australia](https://geoscape.com.au/legalhttps://minus34.com/opendata/geoscape-202405/boundary_concordance.csv-copyright-and-disclaimer/) licensed by the Commonwealth of Australia under [Creative Commons Attribution 4.0 International licence (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). Based on [Australian Bureau of Statistics](https://www.abs.gov.au/websitedbs/d3310114.nsf/Home/Attributing+ABS+Material) data, licensed by the Commonwealth of Australia under [Creative Commons Attribution 4.0 International licence (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). diff --git a/create_concordance_file.py b/create_concordance_file.py index 8544dd0..2fd6f48 100644 --- a/create_concordance_file.py +++ b/create_concordance_file.py @@ -202,7 +202,7 @@ def add_asgs_concordances(pg_cur): count(*) as address_count, 100.0 as address_percent from census_2016_bdys.mb_2016_aust as mb - inner join gnaf_202402.address_principals as gnaf on gnaf.mb_2016_code::text = mb.mb_code16 + inner join gnaf_202405.address_principals as gnaf on gnaf.mb_2016_code::text = mb.mb_code16 group by from_id, from_name, to_id, @@ -250,7 +250,7 @@ def add_asgs_concordances(pg_cur): count(*) as address_count, 100.0 as address_percent from census_2021_bdys_gda94.mb_2021_aust_gda94 as mb - inner join gnaf_202402.address_principals as gnaf + inner join gnaf_202405.address_principals as gnaf on gnaf.mb_2021_code::text = mb.mb_code_2021 group by from_id, from_name, diff --git a/data/boundary_concordance_score.csv b/data/boundary_concordance_score.csv index bc61d14..a34f7be 100644 --- a/data/boundary_concordance_score.csv +++ b/data/boundary_concordance_score.csv @@ -17,7 +17,7 @@ abs 2016,sa2,abs 2016,poa,79,4.4 abs 2016,sa2,abs 2016,sa3,100,0.0 abs 2016,sa2,abs 2016,sa4,100,0.0 abs 2016,sa2,abs 2021,sa2,92, -abs 2016,sa2,geoscape 202402,postcode,78, +abs 2016,sa2,geoscape 202405,postcode,78, abs 2016,sa3,abs 2016,gcc,100,0.0 abs 2016,sa3,abs 2016,lga,83,3.0 abs 2016,sa3,abs 2016,sa4,100,0.0 @@ -39,24 +39,24 @@ abs 2021,sa2,abs 2021,lga,98, abs 2021,sa2,abs 2021,poa,84, abs 2021,sa2,abs 2021,sa3,100, abs 2021,sa2,abs 2021,sa4,100, -abs 2021,sa2,geoscape 202402,postcode,84, +abs 2021,sa2,geoscape 202405,postcode,84, abs 2021,sa3,abs 2021,gccsa,100, abs 2021,sa3,abs 2021,lga,85, abs 2021,sa3,abs 2021,sa4,100, abs 2021,sa4,abs 2021,gccsa,100, -geoscape 202402,lga,abs 2016,gccsa,99, -geoscape 202402,lga,abs 2016,lga,100, -geoscape 202402,lga,abs 2021,gccsa,100, -geoscape 202402,lga,abs 2021,lga,100, -geoscape 202402,locality,abs 2016,lga,98, -geoscape 202402,locality,abs 2016,sa2,93, -geoscape 202402,locality,abs 2016,sa3,99, -geoscape 202402,locality,abs 2021,lga,98, -geoscape 202402,locality,abs 2021,sa2,90, -geoscape 202402,locality,abs 2021,sa3,99, -geoscape 202402,locality,geoscape 202402,lga,98, -geoscape 202402,postcode,abs 2016,lga,93, -geoscape 202402,postcode,abs 2016,sa3,92, -geoscape 202402,postcode,abs 2021,lga,94, -geoscape 202402,postcode,abs 2021,sa3,93, -geoscape 202402,postcode,geoscape 202402,lga,93, +geoscape 202405,lga,abs 2016,gccsa,99, +geoscape 202405,lga,abs 2016,lga,100, +geoscape 202405,lga,abs 2021,gccsa,100, +geoscape 202405,lga,abs 2021,lga,100, +geoscape 202405,locality,abs 2016,lga,98, +geoscape 202405,locality,abs 2016,sa2,93, +geoscape 202405,locality,abs 2016,sa3,99, +geoscape 202405,locality,abs 2021,lga,98, +geoscape 202405,locality,abs 2021,sa2,90, +geoscape 202405,locality,abs 2021,sa3,99, +geoscape 202405,locality,geoscape 202405,lga,98, +geoscape 202405,postcode,abs 2016,lga,93, +geoscape 202405,postcode,abs 2016,sa3,92, +geoscape 202405,postcode,abs 2021,lga,94, +geoscape 202405,postcode,abs 2021,sa3,93, +geoscape 202405,postcode,geoscape 202405,lga,93, diff --git a/postgres-scripts/00_import_concordance_file.sql b/postgres-scripts/00_import_concordance_file.sql index fe1e21e..c56b1e3 100644 --- a/postgres-scripts/00_import_concordance_file.sql +++ b/postgres-scripts/00_import_concordance_file.sql @@ -1,7 +1,7 @@ -- create table -drop table if exists gnaf_202402.boundary_concordance; -create table gnaf_202402.boundary_concordance +drop table if exists gnaf_202405.boundary_concordance; +create table gnaf_202405.boundary_concordance ( from_source text not null, from_bdy text not null, @@ -14,21 +14,21 @@ create table gnaf_202402.boundary_concordance address_count integer, address_percent numeric(4, 1) ); -alter table gnaf_202402.boundary_concordance owner to postgres; +alter table gnaf_202405.boundary_concordance owner to postgres; -- import CSV file -- 586,977 rows affected in 1 s 365 ms -COPY gnaf_202402.boundary_concordance +COPY gnaf_202405.boundary_concordance FROM '/Users/minus34/Downloads/boundary_concordance.csv' WITH (HEADER, DELIMITER ',', FORMAT CSV); -analyse gnaf_202402.boundary_concordance; +analyse gnaf_202405.boundary_concordance; -- add primary key (faster if done after import) -- completed in 8 s 496 ms -alter table gnaf_202402.boundary_concordance add constraint boundary_concordance_pkey +alter table gnaf_202405.boundary_concordance add constraint boundary_concordance_pkey primary key (from_source, from_bdy, from_id, to_source, to_bdy, to_id); -- add index on required fields for converting data -create index boundary_concordance_combo_idx on gnaf_202402.boundary_concordance +create index boundary_concordance_combo_idx on gnaf_202405.boundary_concordance using btree (from_source, from_bdy, to_source, to_bdy); -alter table gnaf_202402.boundary_concordance cluster on boundary_concordance_combo_idx; \ No newline at end of file +alter table gnaf_202405.boundary_concordance cluster on boundary_concordance_combo_idx; \ No newline at end of file diff --git a/postgres-scripts/data-prep/02_create_residential_address_table.sql b/postgres-scripts/data-prep/02_create_residential_address_table.sql index 0541b78..0001e0d 100644 --- a/postgres-scripts/data-prep/02_create_residential_address_table.sql +++ b/postgres-scripts/data-prep/02_create_residential_address_table.sql @@ -27,9 +27,9 @@ select gnaf.gnaf_pid, gnaf.mb_2021_code, lower(mb21.mb_cat) as mb_category_2021, gnaf.geom -from gnaf_202402.address_principals as gnaf - inner join admin_bdys_202402.abs_2016_mb as mb16 on mb16.mb_16code = gnaf.mb_2016_code - inner join admin_bdys_202402.abs_2021_mb as mb21 on mb21.mb21_code = gnaf.mb_2021_code +from gnaf_202405.address_principals as gnaf + inner join admin_bdys_202405.abs_2016_mb as mb16 on mb16.mb_16code = gnaf.mb_2016_code + inner join admin_bdys_202405.abs_2021_mb as mb21 on mb21.mb21_code = gnaf.mb_2021_code left outer join blg on blg.gnaf_pid = gnaf.gnaf_pid ; analyse geoscape_202203.address_principals_buildings; diff --git a/postgres-scripts/data-prep/03_copy_to_gda2020_schema.sql b/postgres-scripts/data-prep/03_copy_to_gda2020_schema.sql index 61be395..501cf3c 100644 --- a/postgres-scripts/data-prep/03_copy_to_gda2020_schema.sql +++ b/postgres-scripts/data-prep/03_copy_to_gda2020_schema.sql @@ -1,34 +1,34 @@ -CREATE SCHEMA IF NOT EXISTS gnaf_202402_gda2020 AUTHORIZATION postgres; +CREATE SCHEMA IF NOT EXISTS gnaf_202405_gda2020 AUTHORIZATION postgres; -- create tables -drop table if exists gnaf_202402_gda2020.address_principal_census_2016_boundaries; -create table gnaf_202402_gda2020.address_principal_census_2016_boundaries as -select * from gnaf_202402.address_principal_census_2016_boundaries; -analyse gnaf_202402_gda2020.address_principal_census_2016_boundaries; - -drop table if exists gnaf_202402_gda2020.address_principal_census_2021_boundaries; -create table gnaf_202402_gda2020.address_principal_census_2021_boundaries as -select * from gnaf_202402.address_principal_census_2021_boundaries; -analyse gnaf_202402_gda2020.address_principal_census_2021_boundaries; - -drop table if exists gnaf_202402_gda2020.boundary_concordance_score; -create table gnaf_202402_gda2020.boundary_concordance_score as -select * from gnaf_202402.boundary_concordance_score; -analyse gnaf_202402_gda2020.boundary_concordance_score; - -drop table if exists gnaf_202402_gda2020.boundary_concordance; -create table gnaf_202402_gda2020.boundary_concordance as -select * from gnaf_202402.boundary_concordance; -analyse gnaf_202402_gda2020.boundary_concordance; +drop table if exists gnaf_202405_gda2020.address_principal_census_2016_boundaries; +create table gnaf_202405_gda2020.address_principal_census_2016_boundaries as +select * from gnaf_202405.address_principal_census_2016_boundaries; +analyse gnaf_202405_gda2020.address_principal_census_2016_boundaries; + +drop table if exists gnaf_202405_gda2020.address_principal_census_2021_boundaries; +create table gnaf_202405_gda2020.address_principal_census_2021_boundaries as +select * from gnaf_202405.address_principal_census_2021_boundaries; +analyse gnaf_202405_gda2020.address_principal_census_2021_boundaries; + +drop table if exists gnaf_202405_gda2020.boundary_concordance_score; +create table gnaf_202405_gda2020.boundary_concordance_score as +select * from gnaf_202405.boundary_concordance_score; +analyse gnaf_202405_gda2020.boundary_concordance_score; + +drop table if exists gnaf_202405_gda2020.boundary_concordance; +create table gnaf_202405_gda2020.boundary_concordance as +select * from gnaf_202405.boundary_concordance; +analyse gnaf_202405_gda2020.boundary_concordance; -- add primary key (faster if done after import) -- completed in 8 s 496 ms -alter table gnaf_202402_gda2020.boundary_concordance add constraint boundary_concordance_gda2020_pkey +alter table gnaf_202405_gda2020.boundary_concordance add constraint boundary_concordance_gda2020_pkey primary key (from_source, from_bdy, from_id, to_source, to_bdy, to_id); -- add index on required fields for converting data -create index boundary_concordance_gda2020_combo_idx on gnaf_202402_gda2020.boundary_concordance +create index boundary_concordance_gda2020_combo_idx on gnaf_202405_gda2020.boundary_concordance using btree (from_source, from_bdy, to_source, to_bdy); -alter table gnaf_202402_gda2020.boundary_concordance cluster on boundary_concordance_gda2020_combo_idx; +alter table gnaf_202405_gda2020.boundary_concordance cluster on boundary_concordance_gda2020_combo_idx; diff --git a/postgres-scripts/example-usage/02_join_pc_and_lga_data.sql b/postgres-scripts/example-usage/02_join_pc_and_lga_data.sql index 50d57bf..5f3d12b 100644 --- a/postgres-scripts/example-usage/02_join_pc_and_lga_data.sql +++ b/postgres-scripts/example-usage/02_join_pc_and_lga_data.sql @@ -19,7 +19,7 @@ -- select * from testing.nsw_covid_tests_20220503_lga; -- -- -- concordance table --- select * from gnaf_202402.boundary_concordance; +-- select * from gnaf_202405.boundary_concordance; WITH pc_data AS ( SELECT con.to_id AS lga_id, @@ -28,8 +28,8 @@ WITH pc_data AS ( sum(pc.cases) as qa_count, count(*) as postcode_count FROM testing.nsw_covid_cases_20220503_postcode AS pc - INNER JOIN gnaf_202402.boundary_concordance AS con ON pc.postcode = con.from_id - WHERE con.from_source = 'geoscape 202402' + INNER JOIN gnaf_202405.boundary_concordance AS con ON pc.postcode = con.from_id + WHERE con.from_source = 'geoscape 202405' AND con.from_bdy = 'postcode' AND con.to_source = 'abs 2016' AND con.to_bdy = 'lga' @@ -49,8 +49,8 @@ INNER JOIN pc_data on pc_data.lga_id = lga.lga_code19; -- example of poor concordance due to number of partial postcodes that intersect with one LGA select * -from gnaf_202402.boundary_concordance -WHERE from_source = 'geoscape 202402' +from gnaf_202405.boundary_concordance +WHERE from_source = 'geoscape 202405' AND from_bdy = 'postcode' AND to_source = 'abs 2016' AND to_bdy = 'lga' diff --git a/postgres-scripts/xx_test_results.sql b/postgres-scripts/xx_test_results.sql index 88483fd..f8b4947 100644 --- a/postgres-scripts/xx_test_results.sql +++ b/postgres-scripts/xx_test_results.sql @@ -2,7 +2,7 @@ -- 592552 rows select count(*) -from gnaf_202402.boundary_concordance; +from gnaf_202405.boundary_concordance; @@ -14,7 +14,7 @@ select count(*) as bdy_pair_count, max(cor.ratio_from_to * 100.0 - bdy.address_percent)::smallint as max_delta, (sum(abs(cor.ratio_from_to * 100.0 - bdy.address_percent) * address_count) / 100.0)::integer as address_count from census_2021_bdys_gda94.correspondences_sa2 as cor - inner join gnaf_202402.boundary_concordance as bdy on bdy.from_id = cor.sa2_maincode_2016 + inner join gnaf_202405.boundary_concordance as bdy on bdy.from_id = cor.sa2_maincode_2016 and bdy.to_id = cor.sa2_code_2021 where abs(cor.ratio_from_to * 100.0 - bdy.address_percent) > 5.0 ; @@ -99,8 +99,8 @@ with agg as ( sa2_code_2021::text as to_id, sa2_name_2021 as to_name, count(*) as address_count - from gnaf_202402.address_principal_census_2016_boundaries as f - inner join gnaf_202402.address_principal_census_2021_boundaries as t on t.gnaf_pid = f.gnaf_pid + from gnaf_202405.address_principal_census_2016_boundaries as f + inner join gnaf_202405.address_principal_census_2021_boundaries as t on t.gnaf_pid = f.gnaf_pid where sa2_16main = '101021011' and mb_category = 'RESIDENTIAL' and mb_category_2021 = 'Residential' diff --git a/postgres-scripts/xx_testing.sql b/postgres-scripts/xx_testing.sql index 82864bf..bedddef 100644 --- a/postgres-scripts/xx_testing.sql +++ b/postgres-scripts/xx_testing.sql @@ -1,20 +1,20 @@ select count(*) -from admin_bdys_202402.abs_2021_sa2; +from admin_bdys_202405.abs_2021_sa2; select * -from gnaf_202402.boundary_concordance +from gnaf_202405.boundary_concordance where from_bdy = 'postcode' and to_bdy = 'poa'; select * -from admin_bdys_202402.abs_2016_sa1; +from admin_bdys_202405.abs_2016_sa1; select * -from gnaf_202402.address_principal_census_2016_boundaries; +from gnaf_202405.address_principal_census_2016_boundaries; select * @@ -30,11 +30,11 @@ from census_2016_bdys.sa2_2016_aust as from_bdy select * -from gnaf_202402; +from gnaf_202405; select * -from admin_bdys_202402.abs_2016_mb as adm +from admin_bdys_202405.abs_2016_mb as adm full outer join census_2016_bdys.mb_2016_aust as abs on adm.mb_16code::text = abs.mb_code16 where not st_equals(adm.geom, abs.geom) ; @@ -51,7 +51,7 @@ select from_id as postcode, to_name as lga_name, address_count, address_percent -from gnaf_202402.boundary_concordance +from gnaf_202405.boundary_concordance where from_bdy = 'postcode' and to_source = 'abs 2016' and to_bdy = 'lga' @@ -63,8 +63,8 @@ where from_bdy = 'postcode' drop table if exists testing.temp_mb; create table testing.temp_mb as select mb.* -from admin_bdys_202402.abs_2021_mb as mb -inner join admin_bdys_202402.postcode_bdys as pc on st_intersects(mb.geom, pc.geom) --- inner join admin_bdys_202402.postcode_bdys as pc on st_intersects(st_centroid(mb.geom), pc.geom) +from admin_bdys_202405.abs_2021_mb as mb +inner join admin_bdys_202405.postcode_bdys as pc on st_intersects(mb.geom, pc.geom) +-- inner join admin_bdys_202405.postcode_bdys as pc on st_intersects(st_centroid(mb.geom), pc.geom) and postcode in ('2050', '2042'); diff --git a/testing/01_create_concordance_table.sql b/testing/01_create_concordance_table.sql index 7880774..4f1e6f7 100644 --- a/testing/01_create_concordance_table.sql +++ b/testing/01_create_concordance_table.sql @@ -16,7 +16,7 @@ with mb as ( state as source_state from mb inner join census_2016_bdys.lga_2016_aust as abs_lga on st_intersects(mb.geom, abs_lga.geom) - inner join gnaf_202402.address_principals as gnaf on gnaf.mb_2016_code = mb.mb_2016_code + inner join gnaf_202405.address_principals as gnaf on gnaf.mb_2016_code = mb.mb_2016_code ) select source.gnaf_pid, source_id, @@ -25,7 +25,7 @@ select source.gnaf_pid, lga_pid as target_id, lga_name as target_name, state as target_state -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma inner join source on source.gnaf_pid = psma.gnaf_pid ; analyse temp_bdy_concordance; @@ -81,7 +81,7 @@ with source_counts as ( ), target as ( select lga_pid, st_collect(geom) as geom - from admin_bdys_202402.local_government_areas + from admin_bdys_202405.local_government_areas group by lga_pid ) select final.*, diff --git a/testing/01_create_postcode_lga_concordance_table.sql b/testing/01_create_postcode_lga_concordance_table.sql index 2ee97ed..2b63d9b 100644 --- a/testing/01_create_postcode_lga_concordance_table.sql +++ b/testing/01_create_postcode_lga_concordance_table.sql @@ -13,8 +13,8 @@ select source.gnaf_pid, target.lga_pid as target_id, target.lga_name as target_name, target.state as target_state -from gnaf_202402.address_principal_admin_boundaries as target -inner join gnaf_202402.address_principals as source on source.gnaf_pid = target.gnaf_pid +from gnaf_202405.address_principal_admin_boundaries as target +inner join gnaf_202405.address_principals as source on source.gnaf_pid = target.gnaf_pid ; analyse temp_bdy_concordance; @@ -31,7 +31,7 @@ where target_state = 'ACT' update temp_bdy_concordance as tmp set target_id = 'lgaot9999991', target_name = 'Unincorporated - Norfolk Island' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = 'locc15e0d2d6f2a' and target_id is null; @@ -39,7 +39,7 @@ where psma.gnaf_pid = tmp.gnaf_pid update temp_bdy_concordance as tmp set target_id = 'lgaot9999992', target_name = 'Unincorporated - Jervis Bay' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = 'loced195c315de9' and target_id is null; @@ -47,7 +47,7 @@ where psma.gnaf_pid = tmp.gnaf_pid update temp_bdy_concordance as tmp set target_id = 'lgasa9999991', target_name = 'Unincorporated - Thistle Island' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = '250190776' and target_id is null; @@ -56,7 +56,7 @@ where psma.gnaf_pid = tmp.gnaf_pid update temp_bdy_concordance as tmp set target_id = 'lgacbffb11990f2', target_name = 'Hobart City' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = 'loc0f7a581b85b7' and target_id is null; @@ -65,7 +65,7 @@ where psma.gnaf_pid = tmp.gnaf_pid update temp_bdy_concordance as tmp set target_id = 'lgaa8d127fa14e7', target_name = 'Ceduna' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = 'loccf8be9dcdacd' and target_id is null; @@ -74,7 +74,7 @@ where psma.gnaf_pid = tmp.gnaf_pid update temp_bdy_concordance as tmp set target_id = 'lga7872e04f6637', target_name = 'Tenterfield' -from gnaf_202402.address_principal_admin_boundaries as psma +from gnaf_202405.address_principal_admin_boundaries as psma where psma.gnaf_pid = tmp.gnaf_pid and locality_pid = 'loc552bd3aef1b8' and target_id is null; @@ -90,7 +90,7 @@ analyse temp_bdy_concordance; -- locality_name, -- postcode, -- state --- from temp_bdy_concordance as tmp, gnaf_202402.address_principal_admin_boundaries as psma +-- from temp_bdy_concordance as tmp, gnaf_202405.address_principal_admin_boundaries as psma -- where psma.gnaf_pid = tmp.gnaf_pid -- and target_id is null -- group by locality_name, diff --git a/testing/01a_abs_bdy_concordances.sql b/testing/01a_abs_bdy_concordances.sql index 3243d06..6ff58cf 100644 --- a/testing/01a_abs_bdy_concordances.sql +++ b/testing/01a_abs_bdy_concordances.sql @@ -28,7 +28,7 @@ with agg as ( lga_16name as to_name, -- state as to_state, count(*) as address_count - from gnaf_202402.address_principal_census_2016_boundaries + from gnaf_202405.address_principal_census_2016_boundaries group by from_id, from_name, -- from_state, diff --git a/testing/xx_geoscape_testing.sql b/testing/xx_geoscape_testing.sql index cb9baff..27d8b61 100644 --- a/testing/xx_geoscape_testing.sql +++ b/testing/xx_geoscape_testing.sql @@ -12,7 +12,7 @@ with blg as ( blg.gnaf_pid as blg_gnaf_pid, is_residential, building_count - from gnaf_202402.address_principals as gnaf + from gnaf_202405.address_principals as gnaf full outer join blg on blg.gnaf_pid = gnaf.gnaf_pid ) select is_residential, diff --git a/testing/xx_lga_testing.sql b/testing/xx_lga_testing.sql index 9e5e6a1..bf54d7a 100644 --- a/testing/xx_lga_testing.sql +++ b/testing/xx_lga_testing.sql @@ -29,7 +29,7 @@ -- ), gnaf_2022 as ( -- select mb_2016_code, -- count(*) as address_count --- from gnaf_202402.address_principals +-- from gnaf_202405.address_principals -- group by mb_2016_code -- ) -- select mb.mb_2016_code, @@ -103,7 +103,7 @@ with psma_lga as ( select lga_pid, name as psma_name, st_collect(geom) as geom - from admin_bdys_202402.local_government_areas + from admin_bdys_202405.local_government_areas group by lga_pid, name ), psma_lga_pnt as ( @@ -181,7 +181,7 @@ with psma_lga as ( select lga_pid, name as psma_name, sum(st_area(st_transform(geom, 3577))) as area_m2 - from admin_bdys_202402.local_government_areas + from admin_bdys_202405.local_government_areas group by lga_pid, name ), psma_lga_pnt as ( diff --git a/testing/xx_testing.sql b/testing/xx_testing.sql index 79abbd5..bad158b 100644 --- a/testing/xx_testing.sql +++ b/testing/xx_testing.sql @@ -55,8 +55,8 @@ select bdy.state, bdy.postcode, bdy.locality_name, count(bdy.gnaf_pid) as addr_count -from gnaf_202402.address_principal_admin_boundaries as bdy - inner join gnaf_202402.address_principals as gnaf on gnaf.gnaf_pid = bdy.gnaf_pid +from gnaf_202405.address_principal_admin_boundaries as bdy + inner join gnaf_202405.address_principals as gnaf on gnaf.gnaf_pid = bdy.gnaf_pid group by bdy.state, bdy.lga_name, bdy.postcode, @@ -82,9 +82,9 @@ select bdy.state, bdy.postcode, bdy.locality_name, count(bdy.gnaf_pid) as addr_count -from gnaf_202402.address_principal_admin_boundaries as bdy -inner join gnaf_202402.address_principals as gnaf on gnaf.gnaf_pid = bdy.gnaf_pid -inner join admin_bdys_202402.abs_2016_mb as mb on mb.mb_16code = gnaf.mb_2016_code +from gnaf_202405.address_principal_admin_boundaries as bdy +inner join gnaf_202405.address_principals as gnaf on gnaf.gnaf_pid = bdy.gnaf_pid +inner join admin_bdys_202405.abs_2016_mb as mb on mb.mb_16code = gnaf.mb_2016_code where mb.mb_category = 'RESIDENTIAL' group by bdy.state, bdy.lga_name, @@ -101,18 +101,18 @@ LIMIT 50 select * -from gnaf_202402.address_principals +from gnaf_202405.address_principals ; select * -from gnaf_202402.address_alias_admin_boundaries +from gnaf_202405.address_alias_admin_boundaries ; select * -from admin_bdys_202402.abs_2016_mb +from admin_bdys_202405.abs_2016_mb ; -select * from admin_bdys_202402.local_government_areas +select * from admin_bdys_202405.local_government_areas order by name, state; diff --git a/xx_create_concordance_file_and_copy_to_s3.sh b/xx_create_concordance_file_and_copy_to_s3.sh index 0471776..76e7030 100644 --- a/xx_create_concordance_file_and_copy_to_s3.sh +++ b/xx_create_concordance_file_and_copy_to_s3.sh @@ -3,8 +3,8 @@ conda activate geo AWS_PROFILE="minus34" -OUTPUT_FOLDER="/Users/$(whoami)/tmp/geoscape_202402" -OUTPUT_FOLDER_2020="/Users/$(whoami)/tmp/geoscape_202402_gda2020" +OUTPUT_FOLDER="/Users/$(whoami)/tmp/geoscape_202405" +OUTPUT_FOLDER_2020="/Users/$(whoami)/tmp/geoscape_202405_gda2020" mkdir -p "${OUTPUT_FOLDER}" mkdir -p "${OUTPUT_FOLDER_2020}" @@ -12,11 +12,11 @@ mkdir -p "${OUTPUT_FOLDER_2020}" # Process using GDA94 boundaries python3 /Users/$(whoami)/git/iag_geo/concord/create_concordance_file.py --pgdb=geo --output-path=${OUTPUT_FOLDER} # Process using GDA2020 boundaries -python3 /Users/$(whoami)/git/iag_geo/concord/create_concordance_file.py --pgdb=geo --admin-schema="admin_bdys_202402_gda2020" --gnaf-schema="gnaf_202402_gda2020" --output-path=${OUTPUT_FOLDER_2020} +python3 /Users/$(whoami)/git/iag_geo/concord/create_concordance_file.py --pgdb=geo --admin-schema="admin_bdys_202405_gda2020" --gnaf-schema="gnaf_202405_gda2020" --output-path=${OUTPUT_FOLDER_2020} # copy concordance files to GDA94 & GDA2020 folders as GDA2020 would be the same as the GDA94 files -aws --profile=${AWS_PROFILE} s3 sync ${OUTPUT_FOLDER} s3://minus34.com/opendata/geoscape-202402 --exclude "*" --include "*.csv" --acl public-read -aws --profile=${AWS_PROFILE} s3 sync ${OUTPUT_FOLDER_2020} s3://minus34.com/opendata/geoscape-202402-gda2020 --exclude "*" --include "*.csv" --acl public-read +aws --profile=${AWS_PROFILE} s3 sync ${OUTPUT_FOLDER} s3://minus34.com/opendata/geoscape-202405 --exclude "*" --include "*.csv" --acl public-read +aws --profile=${AWS_PROFILE} s3 sync ${OUTPUT_FOLDER_2020} s3://minus34.com/opendata/geoscape-202405-gda2020 --exclude "*" --include "*.csv" --acl public-read # copy GDA94 score file to GitHub repo local files cp ${OUTPUT_FOLDER}/boundary_concordance_score.csv /Users/$(whoami)/git/iag_geo/concord/data/ From 6400e6e379f18b7d12bc4e29627ba515f661ef7b Mon Sep 17 00:00:00 2001 From: Hugh Saalmans Date: Mon, 20 May 2024 20:53:46 +1000 Subject: [PATCH 2/2] Updates for 202405 release --- data/boundary_concordance_score.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/boundary_concordance_score.csv b/data/boundary_concordance_score.csv index a34f7be..eec1a29 100644 --- a/data/boundary_concordance_score.csv +++ b/data/boundary_concordance_score.csv @@ -2,7 +2,7 @@ from_source,from_bdy,to_source,to_bdy,concordance_percent,error_percent abs 2016,gcc,abs 2021,gccsa,100, abs 2016,lga,abs 2016,gccsa,99,0.1 abs 2016,lga,abs 2016,ra,92,1.2 -abs 2016,lga,abs 2016,sa3,73,4.8 +abs 2016,lga,abs 2016,sa3,73,4.9 abs 2016,lga,abs 2016,ste,100,0.2 abs 2016,poa,abs 2016,lga,93,1.4 abs 2016,poa,abs 2016,sa2,62,8.9 @@ -13,7 +13,7 @@ abs 2016,sa1,abs 2016,sa4,100,0.0 abs 2016,sa1,abs 2021,sa1,92, abs 2016,sa2,abs 2016,gcc,100,0.0 abs 2016,sa2,abs 2016,lga,97,0.5 -abs 2016,sa2,abs 2016,poa,79,4.4 +abs 2016,sa2,abs 2016,poa,79,4.5 abs 2016,sa2,abs 2016,sa3,100,0.0 abs 2016,sa2,abs 2016,sa4,100,0.0 abs 2016,sa2,abs 2021,sa2,92,