From b351e95c4a38e10971098755667ea59a5804f8f8 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 26 May 2023 12:12:49 +1000 Subject: [PATCH 01/13] [QOLDEV-424] add unit test for parsing CSV file with commas inside quotes --- .../tests/samples/sample_with_quoted_commas.csv | 4 ++++ ckanext/xloader/tests/test_loader.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 ckanext/xloader/tests/samples/sample_with_quoted_commas.csv diff --git a/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv b/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv new file mode 100644 index 00000000..7fe94e5b --- /dev/null +++ b/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv @@ -0,0 +1,4 @@ +Funding agency,Program title,Opening date,Service ID +DTIS,"Department of Employment, Small Business and Training",23/03/2023,63039 +DTIS,"Foo, baz, meh",22/03/2023,63040 +,,,63041 diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 68452d11..1ab79524 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -624,6 +624,18 @@ def test_with_blanks(self, Session): ) assert len(self._get_records(Session, "test1")) == 3 + def test_with_quoted_commas(self, Session): + csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") + resource_id = "test1" + factories.Resource(id=resource_id) + loader.load_csv( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, "test1")) == 3 + def test_with_mixed_types(self, Session): csv_filepath = get_sample_filepath("mixed_numeric_string_sample.csv") resource_id = "test1" From 83e1b86675ad8c6f22b2252e0568b8a36f66b5d1 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 26 May 2023 12:24:25 +1000 Subject: [PATCH 02/13] [QOLDEV-424] add unit test for parsing CSV file with commas inside quotes using tabulator --- ckanext/xloader/tests/test_loader.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 1ab79524..1b4a2ec5 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -1171,3 +1171,15 @@ def test_no_entries(self): mimetype="csv", logger=logger, ) + + def test_with_quoted_commas(self, Session): + csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") + resource_id = "test1" + factories.Resource(id=resource_id) + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, "test1")) == 3 From 9ae1b26e307c03e9b376bec36ef0be735b4c73cc Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 29 May 2023 09:59:15 +1000 Subject: [PATCH 03/13] [QOLDEV-424] add unit test for parsing CSV file with a mixture of single and double quotes --- .../samples/sample_with_mixed_quotes.csv | 136 ++++++++++++++++++ ckanext/xloader/tests/test_loader.py | 24 ++++ 2 files changed, 160 insertions(+) create mode 100644 ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv diff --git a/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv b/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv new file mode 100644 index 00000000..8408a155 --- /dev/null +++ b/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv @@ -0,0 +1,136 @@ +Category,Category name,Priority,Initiative name,Investment objectives,Primary digital priority,Initiative stage,Actual start date,Approved end date,Date data current at,Percentage complete,Overall status,Project commencement allocation,Approved expenditure,Actual cost to date,Scope change event,Cost re-evaluation event,Delivery delay event,Project journey and reasons for variance,Learn more (URL) +DESBT,"Department of Employment, Small Business and Training",High,Business Launchpad Project - Stage 2,"This BLP initiative has a customer-journey based platform for businesses located throughout Queensland. The aim is to assist businesses to better understand their start up and compliance requirements, with a view to streamlining the complex regulatory environment that may delay or impede businesses from starting, growing, and employing. As at 1 July 2022, Business Launchpad Stage 2 has approval to extend beyond the SBRR scope of deliverables to focus on a revised user journey, inclusion of additional industries, and a broader coverage of more than 95% of the Queensland population, to be completed by 30 June 2023.",Collaboration,Delivery,01/07/2022,30/06/2023,31/03/2023,41,G,5633000,5739000,2352000,N,N,N,"As at 31 March 2023 +- Overall 'green' (on track) status +- Revised user journey following results of BLP UX/UI testing +- Transition to support progressing with documentation and walk-through of the solution. +- Ongoing high levels of BLP usage reflecting the success of search engine marketing. BLP focused campaign to further increase awareness and usage is being finalised. + +As at 28 February 2023 +- Overall 'green' (on track) status +- Results of BLP UX/UI testing is guiding development of the revised user journey. +- BLP transition to BAU support continuing with workshops, showcases and handover documentation. +- BLP usage is increasing + +As at 31 January 2023 +- Continued amber status [closely monitored] with risks under management +- Search Engine Marketing -'Always On' yielding good results with continued increase in users and the proportion benefitting from BLP +- Good progress on development of revised BLP user journey. + +As at 31 December 2022 +Status AMBER [Closely monitored] +- Search Engine Marketing commenced 19 December 2022 and already showing increased users and proportion of customers benefitting from BLP +- External assurance review completed and reported 'green' rating for confidence of delivery. + +As at 30 November 2022 +- Continued amber status pending risk management +- Marketing to commence to increase awareness of platform +- Good progress on development of revised user journey + +As at 31 October 2022 +Status AMBER [Closely monitored] +- BLP Stage 2 continue reporting amber status reflective of ongoing high-level risks associated with demand-driven labour-market conditions and planned transition to support. +- Communications and engagement are in progress. +- The revised user journey continues development and testing. This is planned to be ready for release in the first quarter of 2023. As at 30 September 2022 +Status AMBER [Closely monitored] +Project journey events: +- A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness continues to progress. +- BLP industries expanded to include all industries. +- Engagement with agencies continues, to heighten BLP awareness and complete validation following recent expansion to encompass all industries. + +As at 31 August 2022 +Status GREEN [On track] +The project is reporting green overall. Ongoing resourcing risk will continue to be monitored and managed for the life of the project, due to a tight labour market. +Project journey events: +- A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness continues to progress. +- Further analysis of June/July 2022 marketing campaign has offered recommendations for consideration, to improve target audience awareness and BLP uptake. +- BLP industries expanded to include Retail Trade, Accommodation and Non-residential Construction industries finalised. +- Engagement with agencies continues, to heighten BLP awareness and complete validation following recent expansion with three additional industries. + +As at 31 July 2022 +Status AMBER [Closely monitored] +The project is continuing to report amber overall mainly due to ongoing resourcing challenges. +Project journey events: +- A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness, is progressing. +- Analysis of a major marketing campaign conducted in June/July 2022 showed a significant step-up in number of BLP users. +- The target of 95% of Queensland population coverage was met in June 2022 with 100% of Queensland population now covered on BLP. +- Agency engagement for extension industries has commenced. + +As at 1 July 2022 +BLP commenced work on expanding industries to include Retail Trade, Accommodation and Non-residential Construction industries. + +As at June 2022 +Stage 2 of the project is commencing and will build up the solution delivered in BLP Stage 1. Customer journey will be revised in line with outcome of customer testing. The increased coverage target of at least 95% of the Queensland population was met in June 2022, with all local governments included on BLP. Benefits realisation through marketing and promotion of BLP.",https://www.business.qld.gov.au/starting-business/planning/launchpad +DESBT,"Department of Employment, Small Business and Training",High,VET Modernisation and Transformation Program - Tranche 1,"The Vocational Education and Training (VET) Modernisation and Transformation (VMT) Program seeks to reduce the risks associated with department legacy systems by delivering contemporary, consolidated, integrated, user-friendly applications to support delivery of VET outcomes. To optimise the technical capabilities of the new solutions, engagement with business teams in the review and development of business processes is a priority. ",Trust,Delivery,01/07/2021,31/08/2023,28/02/2023,52,G,8692200,9614968,4961147,Y,Y,Y,"As at 28 February 2023 +- Tranche 1 VMT projects continue on schedule and on budget for Tranche 1 completion by 31 August 2023. +- Customer Engagement and Contract Establishment projects continue to progress focusing on delivery activities for new CRM and Portal enhancements. +- VMT Tranche 2 Business Case tracking for completion April 2023. + +As at 31 January 2023 +- VMT Projects continue to track to schedule and on budget for Tranche 1 completion 31 August 2023. +- Customer Engagement and Contract Establishment Projects progressing well with delivery activities for new CRM and Portal enhancements. + +As at 31 December 2022 +Status GREEN +- VMT projects continuing to track to board endorsed updated schedule and on budget for Tranche 1 completion on 31 August 2023. +- Customer Engagement and Contract Establishment projects completed partner onboarding and delivery activities underway. +- Planning in progress for Tranche 2, focusing on remaining legacy systems for planned commencement at completion of Tranch 1. + +As at 30 November 2022 +Status GREEN +- Tranche 1 delivery date extended to 31 August 2023 due to CRM vendor procurement delays and subsequent additional time requirements for build completion and testing of new CRM. +- All projects maintaining momentum and progressing to revised schedule within budget. + +As at 31 October 2022 +Status GREEN +-New 'Partner Portal' Digital Channel continues to perform well with 3516 registered, active, external users from 634 different organisations. Update release being planned for January 2023. +-SkillsCRM (CEP Project) delivery partner on-boarded and formal delivery stage commenced. +-Contract Establishment and Variation (CEV PRoject) continuing delivery partner select with a view to commencing prior to end of December 2022. + +As at 30 September 2022 Status GREEN. +The VMT 'Partner Portal' solution was successfully launched on the 17 August 2022. The decommissioning of the outdated legacy application, 'DETConnect', has completed. Work is now increasing on the next VET systems to be replaced, SkillsCRM (via the Customer Engagement Project) and Policy on Line (via the Contract Establishment and Variation Project). +Project Journey Events: +- Partner Portal. After the successful launch of Partner Portal and decommissioning of DETConnect, the transition to BAU is underway with the Project team continuing to support business until BAU transition is completed. +- Data, Infrastructure and Reporting. +New 'Data Lake' infrastructure built. Data ingestion processes being trialled. QTS report requirement gathering underway which will showcase new capability once completed. Compliance tool SMCM successfully launched September 30. +-Customer Engagement Project (CEP). Completed assurance reviews successfully. Delivery partner selection completed. Partner and formal delivery stage due to start 18 October 2022. Ramp up of activities continuing with business demonstrations of CRM proof of concept. +-Contract Establishment and Variation (CEV). +Requirements gathering completed. Delivery partner selection process commenced. 'As is' process documentation underway. + +As at 31 August 2022 +Status GREEN. The project remains on track. Successful launch of new secure 'Partner Portal' Digital Channel for VET related organisations occurred 17 August 2022. + +Current Projects underway: +- Partner Portal. Go-live occurred on track 17 August 2022. All registered VET organisations now able to use the portal to access key applications and send information to DESBT via secure channel. Enhanced support being provided for 6 weeks. Legacy system decommissioning underway. +- Data, Infrastructure and Reporting. Build of initial Data Lake (centralised, quality, information source) continuing and requirement gathering of first report planned to use new capabilites commenced. +- Customer Services Hub (CRM). Implementation partner selection complete. Solution delivery activities due to start by end September 2022. +- Contract Engagement and Variation. Requirements gathering complete and partner selection process to commence by end September 2022. + +As at 31 July 2022 +Status GREEN + +Project journey events: +Implementation of next changes to VMT applications remain on track for August 2022 with full launch of new secure Partner Portal Digital Channel for VET related organisations. +VMT Program scope adjusted to include additional at risk system decommission activties during this financial year. Approved expenditure updated to align with revised scope. + +Current Projects underway +- Partner Portal. Opened for registrations 4 July 2022. Majority of VET related organisation now registered. Full access (go-live) on track to commence 17 August 2022. Legacy system to be disabled and decommissioned September 2022. +- Data, Infrastructure and Reporting. Build of initial Data Lake (centralised, quality, information source) underway with population and work on first report to commence in September. +- Customer Services Hub (CRM). Requirements confirmed and partner selection underway. Work on legacy CRM replacement due to start September/October 2022. +- Contract Engagement and Variation. Requirements gathering and new process design activities in progress. + +15 May 2022 Update +Status GREEN + +Implementation of next changes to VET applications on track for August 2022 with introduction of new secure 'Patner Portal' Digital Channel for VET related organisations. + +Projects Completed +-Database consolidation - key databases transitioned to supported versions and platforms. Completed November 2021. +-System to System Integration platform. Completed 9 May 2022. + +Current projects underway +-Partner Portal secure digital channel, in final testing. Pilot successfully complete and on track for release in August 2022. +Projects in startup +-Data, Infrastructure and Reporting, planning underway. +-Customer Services Hub (CRM), planning underway. +-Contract Engagement and Variation, planning underway. +-Planning continues for Tranche 2.",https://portal.desbt.qld.gov.au/ diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 1b4a2ec5..451c42ae 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -636,6 +636,18 @@ def test_with_quoted_commas(self, Session): ) assert len(self._get_records(Session, "test1")) == 3 + def test_with_mixed_quotes(self, Session): + csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") + resource_id = "test1" + factories.Resource(id=resource_id) + loader.load_csv( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, "test1")) == 2 + def test_with_mixed_types(self, Session): csv_filepath = get_sample_filepath("mixed_numeric_string_sample.csv") resource_id = "test1" @@ -1183,3 +1195,15 @@ def test_with_quoted_commas(self, Session): logger=logger, ) assert len(self._get_records(Session, "test1")) == 3 + + def test_with_mixed_quotes(self, Session): + csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") + resource_id = "test1" + factories.Resource(id=resource_id) + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, "test1")) == 2 From 08298013eb4f4b9fbc514e449dd829a8048ef4a0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 29 May 2023 10:19:16 +1000 Subject: [PATCH 04/13] [QOLDEV-424] reuse sample size constant for both loading methods --- ckanext/xloader/loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 55c9cab5..15783021 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -16,7 +16,7 @@ import ckan.plugins as p from .job_exceptions import FileCouldNotBeLoadedError, LoaderError -from .parser import XloaderCSVParser +from .parser import CSV_SAMPLE_LINES, XloaderCSVParser from .utils import headers_guess, type_guess from ckan.plugins.toolkit import config @@ -36,12 +36,12 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # Determine the header row try: file_format = os.path.splitext(csv_filepath)[1].strip('.') - with Stream(csv_filepath, format=file_format) as stream: + with Stream(csv_filepath, format=file_format, sample_size=CSV_SAMPLE_LINES) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] - with Stream(csv_filepath, format=file_format) as stream: + with Stream(csv_filepath, format=file_format, sample_size=CSV_SAMPLE_LINES) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException as e: raise LoaderError('Tabulator error: {}'.format(e)) From b444c6c147cea8a0f0e4ad721abe52677c415b49 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 29 May 2023 10:19:42 +1000 Subject: [PATCH 05/13] [QOLDEV-424] increase CSV sample size to better match Messytables behaviour --- ckanext/xloader/loader.py | 2 +- ckanext/xloader/parser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 15783021..7ab76ca5 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -72,7 +72,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): logger.info('Ensuring character coding is UTF8') f_write = tempfile.NamedTemporaryFile(suffix=file_format, delete=False) try: - with Stream(csv_filepath, format=file_format, skip_rows=skip_rows) as stream: + with Stream(csv_filepath, format=file_format, skip_rows=skip_rows, sample_size=CSV_SAMPLE_LINES) as stream: stream.save(target=f_write.name, format='csv', encoding='utf-8', delimiter=delimiter) csv_filepath = f_write.name diff --git a/ckanext/xloader/parser.py b/ckanext/xloader/parser.py index b52c59a3..82539f4d 100644 --- a/ckanext/xloader/parser.py +++ b/ckanext/xloader/parser.py @@ -12,7 +12,7 @@ from ckan.plugins.toolkit import config -CSV_SAMPLE_LINES = 100 +CSV_SAMPLE_LINES = 1000 class XloaderCSVParser(Parser): From 37a2a5428ab6fd886e8bd93bf78743ed9a6e8daa Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 29 May 2023 11:21:32 +1000 Subject: [PATCH 06/13] [QOLDEV-424] set default CSV sample size in config - This is more efficient than setting it on each call, and applies even to code that just reads the config without accepting an override. --- ckanext/xloader/loader.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 7ab76ca5..2060a9ef 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -10,7 +10,7 @@ import psycopg2 from six.moves import zip -from tabulator import Stream, TabulatorException +from tabulator import config as tabulator_config, Stream, TabulatorException from unidecode import unidecode import ckan.plugins as p @@ -28,6 +28,7 @@ _drop_indexes = datastore_db._drop_indexes MAX_COLUMN_LENGTH = 63 +tabulator_config.CSV_SAMPLE_LINES = CSV_SAMPLE_LINES def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): @@ -36,12 +37,12 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # Determine the header row try: file_format = os.path.splitext(csv_filepath)[1].strip('.') - with Stream(csv_filepath, format=file_format, sample_size=CSV_SAMPLE_LINES) as stream: + with Stream(csv_filepath, format=file_format) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] - with Stream(csv_filepath, format=file_format, sample_size=CSV_SAMPLE_LINES) as stream: + with Stream(csv_filepath, format=file_format) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException as e: raise LoaderError('Tabulator error: {}'.format(e)) @@ -72,7 +73,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): logger.info('Ensuring character coding is UTF8') f_write = tempfile.NamedTemporaryFile(suffix=file_format, delete=False) try: - with Stream(csv_filepath, format=file_format, skip_rows=skip_rows, sample_size=CSV_SAMPLE_LINES) as stream: + with Stream(csv_filepath, format=file_format, skip_rows=skip_rows) as stream: stream.save(target=f_write.name, format='csv', encoding='utf-8', delimiter=delimiter) csv_filepath = f_write.name From 51fffade18e76eb6ec41a29ae885f5c00aac16a2 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 29 May 2023 11:56:38 +1000 Subject: [PATCH 07/13] [QOLDEV-424] alter sample mixed-quotes file to use generic data --- .../samples/sample_with_mixed_quotes.csv | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv b/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv index 8408a155..a9527cf7 100644 --- a/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv +++ b/ckanext/xloader/tests/samples/sample_with_mixed_quotes.csv @@ -1,136 +1,136 @@ Category,Category name,Priority,Initiative name,Investment objectives,Primary digital priority,Initiative stage,Actual start date,Approved end date,Date data current at,Percentage complete,Overall status,Project commencement allocation,Approved expenditure,Actual cost to date,Scope change event,Cost re-evaluation event,Delivery delay event,Project journey and reasons for variance,Learn more (URL) -DESBT,"Department of Employment, Small Business and Training",High,Business Launchpad Project - Stage 2,"This BLP initiative has a customer-journey based platform for businesses located throughout Queensland. The aim is to assist businesses to better understand their start up and compliance requirements, with a view to streamlining the complex regulatory environment that may delay or impede businesses from starting, growing, and employing. As at 1 July 2022, Business Launchpad Stage 2 has approval to extend beyond the SBRR scope of deliverables to focus on a revised user journey, inclusion of additional industries, and a broader coverage of more than 95% of the Queensland population, to be completed by 30 June 2023.",Collaboration,Delivery,01/07/2022,30/06/2023,31/03/2023,41,G,5633000,5739000,2352000,N,N,N,"As at 31 March 2023 +DDSSHHESW,"Department of Defence, Social Security, Health, Housing, Education, and Silly Walks",High,Silly Walks project - Stage 2,"Lorum ipsum.",Collaboration,Delivery,01/07/1970,30/06/1971,31/03/1971,41,G,5633000,5739000,2352000,N,N,N,"As at 31 March 1971 - Overall 'green' (on track) status -- Revised user journey following results of BLP UX/UI testing +- Revised user journey following results of Silly Walk UX/UI testing - Transition to support progressing with documentation and walk-through of the solution. -- Ongoing high levels of BLP usage reflecting the success of search engine marketing. BLP focused campaign to further increase awareness and usage is being finalised. +- Ongoing high levels of silly walk usage reflecting the success of search engine marketing. Silly walk focused campaign to further increase awareness and usage is being finalised. -As at 28 February 2023 +As at 28 February 1971 - Overall 'green' (on track) status -- Results of BLP UX/UI testing is guiding development of the revised user journey. -- BLP transition to BAU support continuing with workshops, showcases and handover documentation. -- BLP usage is increasing +- Results of Silly Walk UX/UI testing is guiding development of the revised user journey. +- Silly Walk transition to BAU support continuing with workshops, showcases and handover documentation. +- Silly Walk usage is increasing -As at 31 January 2023 +As at 31 January 1971 - Continued amber status [closely monitored] with risks under management -- Search Engine Marketing -'Always On' yielding good results with continued increase in users and the proportion benefitting from BLP -- Good progress on development of revised BLP user journey. +- Search Engine Marketing -'Always On' yielding good results with continued increase in users and the proportion benefitting from Silly Walk +- Good progress on development of revised Silly Walk user journey. -As at 31 December 2022 +As at 31 December 1970 Status AMBER [Closely monitored] -- Search Engine Marketing commenced 19 December 2022 and already showing increased users and proportion of customers benefitting from BLP +- Search Engine Marketing commenced 19 December 1970 and already showing increased users and proportion of customers benefitting from Silly Walk - External assurance review completed and reported 'green' rating for confidence of delivery. -As at 30 November 2022 +As at 30 November 1970 - Continued amber status pending risk management - Marketing to commence to increase awareness of platform - Good progress on development of revised user journey -As at 31 October 2022 +As at 31 October 1970 Status AMBER [Closely monitored] -- BLP Stage 2 continue reporting amber status reflective of ongoing high-level risks associated with demand-driven labour-market conditions and planned transition to support. +- Silly Walk Stage 2 continue reporting amber status reflective of ongoing high-level risks associated with demand-driven labour-market conditions and planned transition to support. - Communications and engagement are in progress. -- The revised user journey continues development and testing. This is planned to be ready for release in the first quarter of 2023. As at 30 September 2022 +- The revised user journey continues development and testing. This is planned to be ready for release in the first quarter of 1971. As at 30 September 1970 Status AMBER [Closely monitored] Project journey events: - A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness continues to progress. -- BLP industries expanded to include all industries. -- Engagement with agencies continues, to heighten BLP awareness and complete validation following recent expansion to encompass all industries. +- Silly Walk industries expanded to include all industries. +- Engagement with agencies continues, to heighten Silly Walk awareness and complete validation following recent expansion to encompass all industries. -As at 31 August 2022 +As at 31 August 1970 Status GREEN [On track] The project is reporting green overall. Ongoing resourcing risk will continue to be monitored and managed for the life of the project, due to a tight labour market. Project journey events: - A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness continues to progress. -- Further analysis of June/July 2022 marketing campaign has offered recommendations for consideration, to improve target audience awareness and BLP uptake. -- BLP industries expanded to include Retail Trade, Accommodation and Non-residential Construction industries finalised. -- Engagement with agencies continues, to heighten BLP awareness and complete validation following recent expansion with three additional industries. +- Further analysis of June/July 1970 marketing campaign has offered recommendations for consideration, to improve target audience awareness and Silly Walk uptake. +- Silly Walk industries expanded to include Retail Trade, Accommodation and Non-residential Construction industries finalised. +- Engagement with agencies continues, to heighten Silly Walk awareness and complete validation following recent expansion with three additional industries. -As at 31 July 2022 +As at 31 July 1970 Status AMBER [Closely monitored] The project is continuing to report amber overall mainly due to ongoing resourcing challenges. Project journey events: - A revised customer journey in line with outcomes of customer testing and retesting to validate solution usefulness, is progressing. -- Analysis of a major marketing campaign conducted in June/July 2022 showed a significant step-up in number of BLP users. -- The target of 95% of Queensland population coverage was met in June 2022 with 100% of Queensland population now covered on BLP. +- Analysis of a major marketing campaign conducted in June/July 1970 showed a significant step-up in number of Silly Walk users. +- The target of 95% of Circus population coverage was met in June 1970 with 100% of Circus population now covered on Silly Walk. - Agency engagement for extension industries has commenced. -As at 1 July 2022 -BLP commenced work on expanding industries to include Retail Trade, Accommodation and Non-residential Construction industries. +As at 1 July 1970 +Silly Walk commenced work on expanding industries to include Retail Trade, Accommodation and Non-residential Construction industries. -As at June 2022 -Stage 2 of the project is commencing and will build up the solution delivered in BLP Stage 1. Customer journey will be revised in line with outcome of customer testing. The increased coverage target of at least 95% of the Queensland population was met in June 2022, with all local governments included on BLP. Benefits realisation through marketing and promotion of BLP.",https://www.business.qld.gov.au/starting-business/planning/launchpad -DESBT,"Department of Employment, Small Business and Training",High,VET Modernisation and Transformation Program - Tranche 1,"The Vocational Education and Training (VET) Modernisation and Transformation (VMT) Program seeks to reduce the risks associated with department legacy systems by delivering contemporary, consolidated, integrated, user-friendly applications to support delivery of VET outcomes. To optimise the technical capabilities of the new solutions, engagement with business teams in the review and development of business processes is a priority. ",Trust,Delivery,01/07/2021,31/08/2023,28/02/2023,52,G,8692200,9614968,4961147,Y,Y,Y,"As at 28 February 2023 -- Tranche 1 VMT projects continue on schedule and on budget for Tranche 1 completion by 31 August 2023. +As at June 1970 +Stage 2 of the project is commencing and will build up the solution delivered in Silly Walk Stage 1. Customer journey will be revised in line with outcome of customer testing. The increased coverage target of at least 95% of the Circus population was met in June 1970, with all local governments included on Silly Walk. Benefits realisation through marketing and promotion of Silly Walk.",https://example.com +DDSSHHESW,"Department of Defence, Social Security, Health, Housing, Education, and Silly Walks",High,Flying Circus Modernisation and Transformation Program - Tranche 1,"The Flying Circus Modernisation and Transformation (FCMT) Program seeks to reduce the risks associated with department legacy systems by delivering contemporary, consolidated, integrated, user-friendly applications to support delivery of Flying Circus outcomes. To optimise the technical capabilities of the new solutions, engagement with business teams in the review and development of business processes is a priority. ",Trust,Delivery,01/07/1969,31/08/1971,28/02/1971,52,G,8692200,9614968,4961147,Y,Y,Y,"As at 28 February 1971 +- Tranche 1 FCMT projects continue on schedule and on budget for Tranche 1 completion by 31 August 1971. - Customer Engagement and Contract Establishment projects continue to progress focusing on delivery activities for new CRM and Portal enhancements. -- VMT Tranche 2 Business Case tracking for completion April 2023. +- FCMT Tranche 2 Business Case tracking for completion April 1971. -As at 31 January 2023 -- VMT Projects continue to track to schedule and on budget for Tranche 1 completion 31 August 2023. +As at 31 January 1971 +- FCMT Projects continue to track to schedule and on budget for Tranche 1 completion 31 August 1971. - Customer Engagement and Contract Establishment Projects progressing well with delivery activities for new CRM and Portal enhancements. -As at 31 December 2022 +As at 31 December 1970 Status GREEN -- VMT projects continuing to track to board endorsed updated schedule and on budget for Tranche 1 completion on 31 August 2023. +- FCMT projects continuing to track to board endorsed updated schedule and on budget for Tranche 1 completion on 31 August 1971. - Customer Engagement and Contract Establishment projects completed partner onboarding and delivery activities underway. - Planning in progress for Tranche 2, focusing on remaining legacy systems for planned commencement at completion of Tranch 1. -As at 30 November 2022 +As at 30 November 1970 Status GREEN -- Tranche 1 delivery date extended to 31 August 2023 due to CRM vendor procurement delays and subsequent additional time requirements for build completion and testing of new CRM. +- Tranche 1 delivery date extended to 31 August 1971 due to CRM vendor procurement delays and subsequent additional time requirements for build completion and testing of new CRM. - All projects maintaining momentum and progressing to revised schedule within budget. -As at 31 October 2022 +As at 31 October 1970 Status GREEN --New 'Partner Portal' Digital Channel continues to perform well with 3516 registered, active, external users from 634 different organisations. Update release being planned for January 2023. +-New 'Partner Portal' Digital Channel continues to perform well with 3516 registered, active, external users from 634 different organisations. Update release being planned for January 1971. -SkillsCRM (CEP Project) delivery partner on-boarded and formal delivery stage commenced. --Contract Establishment and Variation (CEV PRoject) continuing delivery partner select with a view to commencing prior to end of December 2022. +-Contract Establishment and Variation (CEV PRoject) continuing delivery partner select with a view to commencing prior to end of December 1970. -As at 30 September 2022 Status GREEN. -The VMT 'Partner Portal' solution was successfully launched on the 17 August 2022. The decommissioning of the outdated legacy application, 'DETConnect', has completed. Work is now increasing on the next VET systems to be replaced, SkillsCRM (via the Customer Engagement Project) and Policy on Line (via the Contract Establishment and Variation Project). +As at 30 September 1970 Status GREEN. +The FCMT 'Partner Portal' solution was successfully launched on the 17 August 1970. The decommissioning of the outdated legacy application, 'WalkConnect', has completed. Work is now increasing on the next Flying Circus systems to be replaced, SkillsCRM (via the Customer Engagement Project) and Policy on Line (via the Contract Establishment and Variation Project). Project Journey Events: -- Partner Portal. After the successful launch of Partner Portal and decommissioning of DETConnect, the transition to BAU is underway with the Project team continuing to support business until BAU transition is completed. +- Partner Portal. After the successful launch of Partner Portal and decommissioning of WalkConnect, the transition to BAU is underway with the Project team continuing to support business until BAU transition is completed. - Data, Infrastructure and Reporting. New 'Data Lake' infrastructure built. Data ingestion processes being trialled. QTS report requirement gathering underway which will showcase new capability once completed. Compliance tool SMCM successfully launched September 30. --Customer Engagement Project (CEP). Completed assurance reviews successfully. Delivery partner selection completed. Partner and formal delivery stage due to start 18 October 2022. Ramp up of activities continuing with business demonstrations of CRM proof of concept. +-Customer Engagement Project (CEP). Completed assurance reviews successfully. Delivery partner selection completed. Partner and formal delivery stage due to start 18 October 1970. Ramp up of activities continuing with business demonstrations of CRM proof of concept. -Contract Establishment and Variation (CEV). Requirements gathering completed. Delivery partner selection process commenced. 'As is' process documentation underway. -As at 31 August 2022 -Status GREEN. The project remains on track. Successful launch of new secure 'Partner Portal' Digital Channel for VET related organisations occurred 17 August 2022. +As at 31 August 1970 +Status GREEN. The project remains on track. Successful launch of new secure 'Partner Portal' Digital Channel for Flying Circus related organisations occurred 17 August 1970. Current Projects underway: -- Partner Portal. Go-live occurred on track 17 August 2022. All registered VET organisations now able to use the portal to access key applications and send information to DESBT via secure channel. Enhanced support being provided for 6 weeks. Legacy system decommissioning underway. +- Partner Portal. Go-live occurred on track 17 August 1970. All registered Flying Circus organisations now able to use the portal to access key applications and send information to DDSSHHESW via secure channel. Enhanced support being provided for 6 weeks. Legacy system decommissioning underway. - Data, Infrastructure and Reporting. Build of initial Data Lake (centralised, quality, information source) continuing and requirement gathering of first report planned to use new capabilites commenced. -- Customer Services Hub (CRM). Implementation partner selection complete. Solution delivery activities due to start by end September 2022. -- Contract Engagement and Variation. Requirements gathering complete and partner selection process to commence by end September 2022. +- Customer Services Hub (CRM). Implementation partner selection complete. Solution delivery activities due to start by end September 1970. +- Contract Engagement and Variation. Requirements gathering complete and partner selection process to commence by end September 1970. -As at 31 July 2022 +As at 31 July 1970 Status GREEN Project journey events: -Implementation of next changes to VMT applications remain on track for August 2022 with full launch of new secure Partner Portal Digital Channel for VET related organisations. -VMT Program scope adjusted to include additional at risk system decommission activties during this financial year. Approved expenditure updated to align with revised scope. +Implementation of next changes to FCMT applications remain on track for August 1970 with full launch of new secure Partner Portal Digital Channel for Flying Circus related organisations. +FCMT Program scope adjusted to include additional at risk system decommission activties during this financial year. Approved expenditure updated to align with revised scope. Current Projects underway -- Partner Portal. Opened for registrations 4 July 2022. Majority of VET related organisation now registered. Full access (go-live) on track to commence 17 August 2022. Legacy system to be disabled and decommissioned September 2022. +- Partner Portal. Opened for registrations 4 July 1970. Majority of Flying Circus related organisation now registered. Full access (go-live) on track to commence 17 August 1970. Legacy system to be disabled and decommissioned September 1970. - Data, Infrastructure and Reporting. Build of initial Data Lake (centralised, quality, information source) underway with population and work on first report to commence in September. -- Customer Services Hub (CRM). Requirements confirmed and partner selection underway. Work on legacy CRM replacement due to start September/October 2022. +- Customer Services Hub (CRM). Requirements confirmed and partner selection underway. Work on legacy CRM replacement due to start September/October 1970. - Contract Engagement and Variation. Requirements gathering and new process design activities in progress. -15 May 2022 Update +15 May 1970 Update Status GREEN -Implementation of next changes to VET applications on track for August 2022 with introduction of new secure 'Patner Portal' Digital Channel for VET related organisations. +Implementation of next changes to Flying Circus applications on track for August 1970 with introduction of new secure 'Silly Portal' Digital Channel for Flying Circus related organisations. Projects Completed --Database consolidation - key databases transitioned to supported versions and platforms. Completed November 2021. --System to System Integration platform. Completed 9 May 2022. +-Database consolidation - key databases transitioned to supported versions and platforms. Completed November 1969. +-System to System Integration platform. Completed 9 May 1970. Current projects underway --Partner Portal secure digital channel, in final testing. Pilot successfully complete and on track for release in August 2022. +-Partner Portal secure digital channel, in final testing. Pilot successfully complete and on track for release in August 1970. Projects in startup -Data, Infrastructure and Reporting, planning underway. -Customer Services Hub (CRM), planning underway. -Contract Engagement and Variation, planning underway. --Planning continues for Tranche 2.",https://portal.desbt.qld.gov.au/ +-Planning continues for Tranche 2.",https://example.com From e7c46815c0b8c781261f64796b10f8239d0eadab Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 13 Jun 2023 17:09:34 +0000 Subject: [PATCH 08/13] feat(templates): conditional DataStore tab; - Added helper for valid xloader formats. - Added condition to display the DataStore tab in Resource Edit. --- ckanext/xloader/helpers.py | 6 ++++++ ckanext/xloader/plugin.py | 1 + ckanext/xloader/templates/package/resource_edit_base.html | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ckanext/xloader/helpers.py b/ckanext/xloader/helpers.py index 8c94387a..e4aadad0 100644 --- a/ckanext/xloader/helpers.py +++ b/ckanext/xloader/helpers.py @@ -25,3 +25,9 @@ def xloader_status_description(status): return captions.get(status['status'], status['status'].capitalize()) else: return _('Not Uploaded Yet') + + +def is_xloader_format(resource_format): + from ckanext.xloader.plugin import XLoaderFormats + + return XLoaderFormats.is_it_an_xloader_format(resource_format) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index dbde8ed5..397d15a0 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -211,4 +211,5 @@ def get_helpers(self): return { "xloader_status": xloader_helpers.xloader_status, "xloader_status_description": xloader_helpers.xloader_status_description, + "is_xloader_format": xloader_helpers.is_xloader_format, } diff --git a/ckanext/xloader/templates/package/resource_edit_base.html b/ckanext/xloader/templates/package/resource_edit_base.html index 34403521..29d715a9 100644 --- a/ckanext/xloader/templates/package/resource_edit_base.html +++ b/ckanext/xloader/templates/package/resource_edit_base.html @@ -2,5 +2,7 @@ {% block inner_primary_nav %} {{ super() }} - {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} + {% if h.is_xloader_format(res.format) or res.datastore_active %} + {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} + {% endif %} {% endblock %} From 899e8220b6a53a879fafd2417467ba7c127452fd Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 14 Jun 2023 15:05:13 +0000 Subject: [PATCH 09/13] feat(templates): added more conditions for DataStore tab, added DataStore link in resource read; - Added condition for `datastore_rw_resource_url_types`. - Added DataStore action link to resource read if perms are correct. --- .../templates/package/resource_edit_base.html | 7 +++++-- ckanext/xloader/templates/package/resource_read.html | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 ckanext/xloader/templates/package/resource_read.html diff --git a/ckanext/xloader/templates/package/resource_edit_base.html b/ckanext/xloader/templates/package/resource_edit_base.html index 29d715a9..40221d7f 100644 --- a/ckanext/xloader/templates/package/resource_edit_base.html +++ b/ckanext/xloader/templates/package/resource_edit_base.html @@ -2,7 +2,10 @@ {% block inner_primary_nav %} {{ super() }} - {% if h.is_xloader_format(res.format) or res.datastore_active %} - {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} + {% + if (h.is_xloader_format(res.format) or res.datastore_active) + and res.url_type not in h.datastore_rw_resource_url_types() + %} + {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id, icon='cloud-upload') }} {% endif %} {% endblock %} diff --git a/ckanext/xloader/templates/package/resource_read.html b/ckanext/xloader/templates/package/resource_read.html new file mode 100644 index 00000000..192a9cb5 --- /dev/null +++ b/ckanext/xloader/templates/package/resource_read.html @@ -0,0 +1,12 @@ +{% ckan_extends %} + +{% block action_manage %} + {{ super() }} + {% + if (h.is_xloader_format(res.format) or res.datastore_active) + and h.check_access('package_update', {'id':pkg.id }) + and res.url_type not in h.datastore_rw_resource_url_types() + %} +
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}
    • + {% endif %} +{% endblock %} From 8ff2e9fb00a36bf2c49620dcca090a9a6f2abc81 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Thu, 15 Jun 2023 17:24:33 +0000 Subject: [PATCH 10/13] feat(templates): added conditional DataStore action to more templates, added new helper; - Added the conditional DataStore action link to more templates. - Added another new helper to handle possible non-existant core helper. --- ckanext/xloader/helpers.py | 7 +++++++ ckanext/xloader/plugin.py | 1 + .../templates/package/resource_edit_base.html | 2 +- ckanext/xloader/templates/package/resource_read.html | 4 ++-- .../templates/package/snippets/resource_item.html | 12 ++++++++++++ .../templates/package/snippets/resources.html | 12 ++++++++++++ 6 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 ckanext/xloader/templates/package/snippets/resource_item.html create mode 100644 ckanext/xloader/templates/package/snippets/resources.html diff --git a/ckanext/xloader/helpers.py b/ckanext/xloader/helpers.py index e4aadad0..d434c9e5 100644 --- a/ckanext/xloader/helpers.py +++ b/ckanext/xloader/helpers.py @@ -31,3 +31,10 @@ def is_xloader_format(resource_format): from ckanext.xloader.plugin import XLoaderFormats return XLoaderFormats.is_it_an_xloader_format(resource_format) + + +def is_xloader_type(resource_url_type): + try: + return resource_url_type not in toolkit.h.datastore_rw_resource_url_types() + except AttributeError: + return (resource_url_type == 'upload' or resource_url_type == '') diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 397d15a0..2298cd47 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -212,4 +212,5 @@ def get_helpers(self): "xloader_status": xloader_helpers.xloader_status, "xloader_status_description": xloader_helpers.xloader_status_description, "is_xloader_format": xloader_helpers.is_xloader_format, + "is_xloader_type": xloader_helpers.is_xloader_type, } diff --git a/ckanext/xloader/templates/package/resource_edit_base.html b/ckanext/xloader/templates/package/resource_edit_base.html index 40221d7f..3d325861 100644 --- a/ckanext/xloader/templates/package/resource_edit_base.html +++ b/ckanext/xloader/templates/package/resource_edit_base.html @@ -4,7 +4,7 @@ {{ super() }} {% if (h.is_xloader_format(res.format) or res.datastore_active) - and res.url_type not in h.datastore_rw_resource_url_types() + and h.is_xloader_type(res.url_type) %} {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id, icon='cloud-upload') }} {% endif %} diff --git a/ckanext/xloader/templates/package/resource_read.html b/ckanext/xloader/templates/package/resource_read.html index 192a9cb5..05296fad 100644 --- a/ckanext/xloader/templates/package/resource_read.html +++ b/ckanext/xloader/templates/package/resource_read.html @@ -1,11 +1,11 @@ {% ckan_extends %} -{% block action_manage %} +{% block action_manage_inner %} {{ super() }} {% if (h.is_xloader_format(res.format) or res.datastore_active) and h.check_access('package_update', {'id':pkg.id }) - and res.url_type not in h.datastore_rw_resource_url_types() + and h.is_xloader_type(res.url_type) %}
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}
    • {% endif %} diff --git a/ckanext/xloader/templates/package/snippets/resource_item.html b/ckanext/xloader/templates/package/snippets/resource_item.html new file mode 100644 index 00000000..04b38f04 --- /dev/null +++ b/ckanext/xloader/templates/package/snippets/resource_item.html @@ -0,0 +1,12 @@ +{% ckan_extends %} + +{% block resource_item_explore_inner %} + {{ super() }} + {% + if (h.is_xloader_format(res.format) or res.datastore_active) + and h.check_access('package_update', {'id':pkg.id }) + and h.is_xloader_type(res.url_type) + %} +
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='cloud-upload' %}
    • + {% endif %} +{% endblock %} diff --git a/ckanext/xloader/templates/package/snippets/resources.html b/ckanext/xloader/templates/package/snippets/resources.html new file mode 100644 index 00000000..bc3e75aa --- /dev/null +++ b/ckanext/xloader/templates/package/snippets/resources.html @@ -0,0 +1,12 @@ +{% ckan_extends %} + +{% block resources_list_edit_dropdown_inner %} + {{ super() }} + {% + if (h.is_xloader_format(resource.format) or resource.datastore_active) + and h.check_access('package_update', {'id':pkg.id }) + and h.is_xloader_type(resource.url_type) + %} +
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=resource.id, class_='dropdown-item', icon='cloud-upload' %}
    • + {% endif %} +{% endblock %} From 9e05f22ff931b58cbcebd823a1c76f01221fe28b Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Fri, 16 Jun 2023 15:17:03 +0000 Subject: [PATCH 11/13] feat(dev): combined helpers, moved XLoaderFormats class; - Combined helpers and conditions into one helper. - Moved `XLoaderFormats` class to utils script to prevent circular imports. --- ckanext/xloader/command.py | 3 +- ckanext/xloader/helpers.py | 20 ++++++----- ckanext/xloader/plugin.py | 34 ++----------------- .../templates/package/resource_edit_base.html | 5 +-- .../templates/package/resource_read.html | 6 +--- .../package/snippets/resource_item.html | 6 +--- .../templates/package/snippets/resources.html | 6 +--- ckanext/xloader/utils.py | 30 ++++++++++++++++ 8 files changed, 48 insertions(+), 62 deletions(-) diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index 7f2c000a..64b79754 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -3,6 +3,7 @@ import sys import logging import ckan.plugins.toolkit as tk +from ckanext.xloader.utils import XLoaderFormats class XloaderCmd: @@ -84,8 +85,6 @@ def _submit_resource(self, resource, user, indent=0): '''resource: resource dictionary ''' indentation = ' ' * indent - # import here, so that that loggers are setup - from ckanext.xloader.plugin import XLoaderFormats if not XLoaderFormats.is_it_an_xloader_format(resource['format']): print(indentation diff --git a/ckanext/xloader/helpers.py b/ckanext/xloader/helpers.py index d434c9e5..3c071028 100644 --- a/ckanext/xloader/helpers.py +++ b/ckanext/xloader/helpers.py @@ -1,4 +1,5 @@ import ckan.plugins.toolkit as toolkit +from ckanext.xloader.utils import XLoaderFormats def xloader_status(resource_id): @@ -27,14 +28,15 @@ def xloader_status_description(status): return _('Not Uploaded Yet') -def is_xloader_format(resource_format): - from ckanext.xloader.plugin import XLoaderFormats - - return XLoaderFormats.is_it_an_xloader_format(resource_format) - - -def is_xloader_type(resource_url_type): +def is_resource_supported_by_xloader(res_dict, check_access = True): + is_supported_format = XLoaderFormats.is_it_an_xloader_format(res_dict.get('format')) + is_datastore_active = res_dict.get('datastore_active', False) + if check_access: + user_has_access = toolkit.h.check_access('package_update', {'id':res_dict.get('package_id')}) + else: + user_has_access = True try: - return resource_url_type not in toolkit.h.datastore_rw_resource_url_types() + is_supported_url_type = res_dict.get('url_type') not in toolkit.h.datastore_rw_resource_url_types() except AttributeError: - return (resource_url_type == 'upload' or resource_url_type == '') + is_supported_url_type = (res_dict.get('url_type') == 'upload' or res_dict.get('url_type') == '') + return (is_supported_format or is_datastore_active) and user_has_access and is_supported_url_type diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 2298cd47..6791f2d8 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -7,6 +7,7 @@ from . import action, auth, helpers as xloader_helpers, utils from .loader import fulltext_function_exists, get_write_engine +from ckanext.xloader.utils import XLoaderFormats try: config_declarations = toolkit.blanket.config_declarations @@ -19,36 +20,6 @@ def config_declarations(cls): log = logging.getLogger(__name__) -# resource.formats accepted by ckanext-xloader. Must be lowercase here. -DEFAULT_FORMATS = [ - "csv", - "application/csv", - "xls", - "xlsx", - "tsv", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "ods", - "application/vnd.oasis.opendocument.spreadsheet", -] - - -class XLoaderFormats(object): - formats = None - - @classmethod - def is_it_an_xloader_format(cls, format_): - if cls.formats is None: - cls._formats = toolkit.config.get("ckanext.xloader.formats") - if cls._formats is not None: - cls._formats = cls._formats.lower().split() - else: - cls._formats = DEFAULT_FORMATS - if not format_: - return False - return format_.lower() in cls._formats - - @config_declarations class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IConfigurer) @@ -211,6 +182,5 @@ def get_helpers(self): return { "xloader_status": xloader_helpers.xloader_status, "xloader_status_description": xloader_helpers.xloader_status_description, - "is_xloader_format": xloader_helpers.is_xloader_format, - "is_xloader_type": xloader_helpers.is_xloader_type, + "is_resource_supported_by_xloader": xloader_helpers.is_resource_supported_by_xloader, } diff --git a/ckanext/xloader/templates/package/resource_edit_base.html b/ckanext/xloader/templates/package/resource_edit_base.html index 3d325861..5c02815a 100644 --- a/ckanext/xloader/templates/package/resource_edit_base.html +++ b/ckanext/xloader/templates/package/resource_edit_base.html @@ -2,10 +2,7 @@ {% block inner_primary_nav %} {{ super() }} - {% - if (h.is_xloader_format(res.format) or res.datastore_active) - and h.is_xloader_type(res.url_type) - %} + {% if h.is_resource_supported_by_xloader(res) %} {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id, icon='cloud-upload') }} {% endif %} {% endblock %} diff --git a/ckanext/xloader/templates/package/resource_read.html b/ckanext/xloader/templates/package/resource_read.html index 05296fad..6d5f5ff2 100644 --- a/ckanext/xloader/templates/package/resource_read.html +++ b/ckanext/xloader/templates/package/resource_read.html @@ -2,11 +2,7 @@ {% block action_manage_inner %} {{ super() }} - {% - if (h.is_xloader_format(res.format) or res.datastore_active) - and h.check_access('package_update', {'id':pkg.id }) - and h.is_xloader_type(res.url_type) - %} + {% if h.is_resource_supported_by_xloader(res) %}
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}
    • {% endif %} {% endblock %} diff --git a/ckanext/xloader/templates/package/snippets/resource_item.html b/ckanext/xloader/templates/package/snippets/resource_item.html index 04b38f04..37ed457c 100644 --- a/ckanext/xloader/templates/package/snippets/resource_item.html +++ b/ckanext/xloader/templates/package/snippets/resource_item.html @@ -2,11 +2,7 @@ {% block resource_item_explore_inner %} {{ super() }} - {% - if (h.is_xloader_format(res.format) or res.datastore_active) - and h.check_access('package_update', {'id':pkg.id }) - and h.is_xloader_type(res.url_type) - %} + {% if h.is_resource_supported_by_xloader(res) %}
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='cloud-upload' %}
    • {% endif %} {% endblock %} diff --git a/ckanext/xloader/templates/package/snippets/resources.html b/ckanext/xloader/templates/package/snippets/resources.html index bc3e75aa..e04dde4d 100644 --- a/ckanext/xloader/templates/package/snippets/resources.html +++ b/ckanext/xloader/templates/package/snippets/resources.html @@ -2,11 +2,7 @@ {% block resources_list_edit_dropdown_inner %} {{ super() }} - {% - if (h.is_xloader_format(resource.format) or resource.datastore_active) - and h.check_access('package_update', {'id':pkg.id }) - and h.is_xloader_type(resource.url_type) - %} + {% if h.is_resource_supported_by_xloader(resource) %}
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=resource.id, class_='dropdown-item', icon='cloud-upload' %}
    • {% endif %} {% endblock %} diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index cbffaa2f..03af2bdb 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -9,6 +9,36 @@ from decimal import Decimal import ckan.plugins as p +from ckan.plugins.toolkit import config + +# resource.formats accepted by ckanext-xloader. Must be lowercase here. +DEFAULT_FORMATS = [ + "csv", + "application/csv", + "xls", + "xlsx", + "tsv", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "ods", + "application/vnd.oasis.opendocument.spreadsheet", +] + + +class XLoaderFormats(object): + formats = None + + @classmethod + def is_it_an_xloader_format(cls, format_): + if cls.formats is None: + cls._formats = config.get("ckanext.xloader.formats") + if cls._formats is not None: + cls._formats = cls._formats.lower().split() + else: + cls._formats = DEFAULT_FORMATS + if not format_: + return False + return format_.lower() in cls._formats def resource_data(id, resource_id): From 4656f063f833cd0b771223b3860ad8e37791d076 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 28 Jul 2023 14:48:09 +1000 Subject: [PATCH 12/13] [QOLDEV-424] fix tests to avoid hardcoding resource IDs - Our test IDs don't have the right format, and we should avoid hardcoding IDs anyway --- ckanext/xloader/helpers.py | 4 +- ckanext/xloader/tests/test_loader.py | 192 +++++++++++++-------------- 2 files changed, 98 insertions(+), 98 deletions(-) diff --git a/ckanext/xloader/helpers.py b/ckanext/xloader/helpers.py index 3c071028..829b7b74 100644 --- a/ckanext/xloader/helpers.py +++ b/ckanext/xloader/helpers.py @@ -28,11 +28,11 @@ def xloader_status_description(status): return _('Not Uploaded Yet') -def is_resource_supported_by_xloader(res_dict, check_access = True): +def is_resource_supported_by_xloader(res_dict, check_access=True): is_supported_format = XLoaderFormats.is_it_an_xloader_format(res_dict.get('format')) is_datastore_active = res_dict.get('datastore_active', False) if check_access: - user_has_access = toolkit.h.check_access('package_update', {'id':res_dict.get('package_id')}) + user_has_access = toolkit.h.check_access('package_update', {'id': res_dict.get('package_id')}) else: user_has_access = True try: diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 451c42ae..8cc69a06 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -85,8 +85,8 @@ def _get_column_types(self, Session, table_name): class TestLoadCsv(TestLoadBase): def test_simple(self, Session): csv_filepath = get_sample_filepath("simple.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -95,7 +95,7 @@ def test_simple(self, Session): ) assert self._get_records( - Session, "test1", limit=1, exclude_full_text_column=False + Session, resource_id, limit=1, exclude_full_text_column=False ) == [ ( 1, @@ -105,7 +105,7 @@ def test_simple(self, Session): u"Galway", ) ] - assert self._get_records(Session, "test1") == [ + assert self._get_records(Session, resource_id) == [ (1, u"2011-01-01", u"1", u"Galway"), (2, u"2011-01-02", u"-1", u"Galway"), (3, u"2011-01-03", u"0", u"Galway"), @@ -113,14 +113,14 @@ def test_simple(self, Session): (5, None, None, u"Berkeley"), (6, u"2011-01-03", u"5", None), ] - assert self._get_column_names(Session, "test1") == [ + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"date", u"temperature", u"place", ] - assert self._get_column_types(Session, "test1") == [ + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", u"text", @@ -130,8 +130,8 @@ def test_simple(self, Session): def test_simple_with_indexing(self, Session): csv_filepath = get_sample_filepath("simple.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] fields = loader.load_csv( csv_filepath, resource_id=resource_id, @@ -144,7 +144,7 @@ def test_simple_with_indexing(self, Session): assert ( self._get_records( - Session, "test1", limit=1, exclude_full_text_column=False + Session, resource_id, limit=1, exclude_full_text_column=False )[0][1] == "'-01':2,3 '1':4 '2011':1 'galway':5" ) @@ -155,8 +155,8 @@ def test_boston_311_complete(self): # to get the test file: # curl -o ckanext/xloader/tests/samples/boston_311.csv https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/2968e2c0-d479-49ba-a884-4ef523ada3c0/download/311.csv # noqa csv_filepath = get_sample_filepath("boston_311.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] import time t0 = time.time() @@ -179,8 +179,8 @@ def test_boston_311_sample5(self): # to create the test file: # head -n 100001 ckanext/xloader/tests/samples/boston_311.csv > ckanext/xloader/tests/samples/boston_311_sample5.csv csv_filepath = get_sample_filepath("boston_311_sample5.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] import time t0 = time.time() @@ -199,8 +199,8 @@ def test_boston_311_sample5(self): def test_boston_311(self, Session): csv_filepath = get_sample_filepath("boston_311_sample.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -208,7 +208,7 @@ def test_boston_311(self, Session): logger=logger, ) - records = self._get_records(Session, "test1") + records = self._get_records(Session, resource_id) print(records) assert records == [ ( @@ -308,8 +308,8 @@ def test_boston_311(self, Session): u"Citizens Connect App", ), ] # noqa - print(self._get_column_names(Session, "test1")) - assert self._get_column_names(Session, "test1") == [ + print(self._get_column_names(Session, resource_id)) + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"CASE_ENQUIRY_ID", @@ -342,16 +342,16 @@ def test_boston_311(self, Session): u"Longitude", u"Source", ] # noqa - print(self._get_column_types(Session, "test1")) - assert self._get_column_types(Session, "test1") == [ + print(self._get_column_types(Session, resource_id)) + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", ] + [u"text"] * (len(records[0]) - 1) def test_brazilian(self, Session): csv_filepath = get_sample_filepath("brazilian_sample.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -359,7 +359,7 @@ def test_brazilian(self, Session): logger=logger, ) - records = self._get_records(Session, "test1") + records = self._get_records(Session, resource_id) print(records) assert records[0] == ( 1, @@ -459,8 +459,8 @@ def test_brazilian(self, Session): None, None, ) # noqa - print(self._get_column_names(Session, "test1")) - assert self._get_column_names(Session, "test1") == [ + print(self._get_column_names(Session, resource_id)) + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"NU_ANO_CENSO", @@ -559,16 +559,16 @@ def test_brazilian(self, Session): u"PROVA_MEAN_MAT_I_MUN", u"PROVA_MEAN_MAT_T_MUN", ] # noqa - print(self._get_column_types(Session, "test1")) - assert self._get_column_types(Session, "test1") == [ + print(self._get_column_types(Session, resource_id)) + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", ] + [u"text"] * (len(records[0]) - 1) def test_german(self, Session): csv_filepath = get_sample_filepath("german_sample.csv") - resource_id = "test_german" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -576,7 +576,7 @@ def test_german(self, Session): logger=logger, ) - records = self._get_records(Session, "test_german") + records = self._get_records(Session, resource_id) print(records) assert records[0] == ( 1, @@ -591,8 +591,8 @@ def test_german(self, Session): u"24221", u"672", ) - print(self._get_column_names(Session, "test_german")) - assert self._get_column_names(Session, "test_german") == [ + print(self._get_column_names(Session, resource_id)) + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"Stadtname", @@ -606,64 +606,64 @@ def test_german(self, Session): u"Schuler_Berufsausbildung_2010/2011", u"Schuler_andere allgemeinbildende Schulen_2010/2011", ] - print(self._get_column_types(Session, "test_german")) - assert self._get_column_types(Session, "test_german") == [ + print(self._get_column_types(Session, resource_id)) + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", ] + [u"text"] * (len(records[0]) - 1) def test_with_blanks(self, Session): csv_filepath = get_sample_filepath("sample_with_blanks.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 3 + assert len(self._get_records(Session, resource_id)) == 3 def test_with_quoted_commas(self, Session): csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 3 + assert len(self._get_records(Session, resource_id)) == 3 def test_with_mixed_quotes(self, Session): csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 2 + assert len(self._get_records(Session, resource_id)) == 2 def test_with_mixed_types(self, Session): csv_filepath = get_sample_filepath("mixed_numeric_string_sample.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 2 + assert len(self._get_records(Session, resource_id)) == 2 def test_reload(self, Session): csv_filepath = get_sample_filepath("simple.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -679,15 +679,15 @@ def test_reload(self, Session): logger=logger, ) - assert len(self._get_records(Session, "test1")) == 6 - assert self._get_column_names(Session, "test1") == [ + assert len(self._get_records(Session, resource_id)) == 6 + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"date", u"temperature", u"place", ] - assert self._get_column_types(Session, "test1") == [ + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", u"text", @@ -701,8 +701,8 @@ def test_reload(self, Session): ) def test_reload_with_overridden_types(self, Session): csv_filepath = get_sample_filepath("simple.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -732,15 +732,15 @@ def test_reload_with_overridden_types(self, Session): fields=fields, resource_id=resource_id, logger=logger ) - assert len(self._get_records(Session, "test1")) == 6 - assert self._get_column_names(Session, "test1") == [ + assert len(self._get_records(Session, resource_id)) == 6 + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"date", u"temperature", u"place", ] - assert self._get_column_types(Session, "test1") == [ + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", u"timestamp", @@ -750,7 +750,7 @@ def test_reload_with_overridden_types(self, Session): # check that rows with nulls are indexed correctly records = self._get_records( - Session, "test1", exclude_full_text_column=False + Session, resource_id, exclude_full_text_column=False ) print(records) assert records[4][1] == "'berkeley':1" @@ -775,8 +775,8 @@ def test_encode_headers(self): def test_column_names(self, Session): csv_filepath = get_sample_filepath("column_names.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( csv_filepath, resource_id=resource_id, @@ -784,12 +784,12 @@ def test_column_names(self, Session): logger=logger, ) - assert self._get_column_names(Session, "test1")[2:] == [ + assert self._get_column_names(Session, resource_id)[2:] == [ u"d@t$e", u"t^e&m*pe!r(a)t?u:r%%e", r"p\l/a[c{e%", ] - assert self._get_records(Session, "test1")[0] == ( + assert self._get_records(Session, resource_id)[0] == ( 1, u"2011-01-01", u"1", @@ -800,8 +800,8 @@ def test_column_names(self, Session): class TestLoadUnhandledTypes(TestLoadBase): def test_kml(self): filepath = get_sample_filepath("polling_locations.kml") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] with pytest.raises(LoaderError) as exception: loader.load_csv( filepath, @@ -817,8 +817,8 @@ def test_kml(self): def test_geojson(self): filepath = get_sample_filepath("polling_locations.geojson") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] with pytest.raises(LoaderError) as exception: loader.load_csv( filepath, @@ -839,8 +839,8 @@ def test_geojson(self): ) def test_shapefile_zip_python2(self): filepath = get_sample_filepath("polling_locations.shapefile.zip") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] with pytest.raises(LoaderError): loader.load_csv( filepath, @@ -859,8 +859,8 @@ def test_shapefile_zip_python3(self, Session): # finds, 'Polling_Locations.cpg'. This file only contains the # following data: `UTF-8`. filepath = get_sample_filepath("polling_locations.shapefile.zip") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_csv( filepath, resource_id=resource_id, @@ -868,8 +868,8 @@ def test_shapefile_zip_python3(self, Session): logger=logger, ) - assert self._get_records(Session, "test1") == [] - assert self._get_column_names(Session, "test1") == [ + assert self._get_records(Session, resource_id) == [] + assert self._get_column_names(Session, resource_id) == [ '_id', '_full_text', 'UTF-8' @@ -879,8 +879,8 @@ def test_shapefile_zip_python3(self, Session): class TestLoadTabulator(TestLoadBase): def test_simple(self, Session): csv_filepath = get_sample_filepath("simple.xls") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_table( csv_filepath, resource_id=resource_id, @@ -891,7 +891,7 @@ def test_simple(self, Session): assert ( "'galway':" in self._get_records( - Session, "test1", limit=1, exclude_full_text_column=False + Session, resource_id, limit=1, exclude_full_text_column=False )[0][1] ) # Indexed record looks like this (depending on CKAN version?): @@ -899,7 +899,7 @@ def test_simple(self, Session): # "'-01':4,5 '00':6,7,8 '1':1 '2011':3 'galway':2" # "'-01':2,3 '00':5,6 '1':7 '2011':1 'galway':8 't00':4" - assert self._get_records(Session, "test1") == [ + assert self._get_records(Session, resource_id) == [ (1, datetime.datetime(2011, 1, 1, 0, 0), Decimal("1"), u"Galway",), ( 2, @@ -927,14 +927,14 @@ def test_simple(self, Session): u"Berkeley", ), ] - assert self._get_column_names(Session, "test1") == [ + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"date", u"temperature", u"place", ] - assert self._get_column_types(Session, "test1") == [ + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", u"timestamp", @@ -948,8 +948,8 @@ def test_boston_311_complete(self): # to get the test file: # curl -o ckanext/xloader/tests/samples/boston_311.csv https://data.boston.gov/dataset/8048697b-ad64-4bfc-b090-ee00169f2323/resource/2968e2c0-d479-49ba-a884-4ef523ada3c0/download/311.csv # noqa csv_filepath = get_sample_filepath("boston_311.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] import time t0 = time.time() @@ -972,8 +972,8 @@ def test_boston_311_sample5(self): # to create the test file: # head -n 100001 ckanext/xloader/tests/samples/boston_311.csv > ckanext/xloader/tests/samples/boston_311_sample5.csv csv_filepath = get_sample_filepath("boston_311_sample5.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] import time t0 = time.time() @@ -992,8 +992,8 @@ def test_boston_311_sample5(self): def test_boston_311(self, Session): csv_filepath = get_sample_filepath("boston_311_sample.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_table( csv_filepath, resource_id=resource_id, @@ -1001,7 +1001,7 @@ def test_boston_311(self, Session): logger=logger, ) - records = self._get_records(Session, "test1") + records = self._get_records(Session, resource_id) print(records) assert records == [ ( @@ -1101,8 +1101,8 @@ def test_boston_311(self, Session): u"Citizens Connect App", ), ] # noqa - print(self._get_column_names(Session, "test1")) - assert self._get_column_names(Session, "test1") == [ + print(self._get_column_names(Session, resource_id)) + assert self._get_column_names(Session, resource_id) == [ u"_id", u"_full_text", u"CASE_ENQUIRY_ID", @@ -1135,8 +1135,8 @@ def test_boston_311(self, Session): u"Longitude", u"Source", ] # noqa - print(self._get_column_types(Session, "test1")) - assert self._get_column_types(Session, "test1") == [ + print(self._get_column_types(Session, resource_id)) + assert self._get_column_types(Session, resource_id) == [ u"int4", u"tsvector", u"numeric", @@ -1174,8 +1174,8 @@ def test_no_entries(self): csv_filepath = get_sample_filepath("no_entries.csv") # no datastore table is created - we need to except, or else # datastore_active will be set on a non-existent datastore table - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] with pytest.raises(LoaderError): loader.load_table( csv_filepath, @@ -1186,24 +1186,24 @@ def test_no_entries(self): def test_with_quoted_commas(self, Session): csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_table( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 3 + assert len(self._get_records(Session, resource_id)) == 3 def test_with_mixed_quotes(self, Session): csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") - resource_id = "test1" - factories.Resource(id=resource_id) + resource = factories.Resource() + resource_id = resource['id'] loader.load_table( csv_filepath, resource_id=resource_id, mimetype="text/csv", logger=logger, ) - assert len(self._get_records(Session, "test1")) == 2 + assert len(self._get_records(Session, resource_id)) == 2 From b23b22c9426f0a7436cdad8bcb5b745bea4230ca Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 28 Jul 2023 15:44:31 +1000 Subject: [PATCH 13/13] [QOLDEV-424] ensure consistent column name ordering in tests - This is intermittently breaking the boston_311 test, when columns load in an unexpected order --- ckanext/xloader/tests/test_loader.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 8cc69a06..d55ec949 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -64,8 +64,12 @@ def _get_column_names(self, Session, table_name): # SELECT column_name FROM information_schema.columns WHERE table_name='test1'; c = Session.connection() sql = ( - "SELECT column_name FROM information_schema.columns " - "WHERE table_name='{}';".format(table_name) + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name='{}' + ORDER BY ordinal_position; + """.format(table_name) ) results = c.execute(sql) records = results.fetchall() @@ -74,8 +78,12 @@ def _get_column_names(self, Session, table_name): def _get_column_types(self, Session, table_name): c = Session.connection() sql = ( - "SELECT udt_name FROM information_schema.columns " - "WHERE table_name='{}';".format(table_name) + """ + SELECT udt_name + FROM information_schema.columns + WHERE table_name='{}' + ORDER BY ordinal_position; + """.format(table_name) ) results = c.execute(sql) records = results.fetchall()