diff --git "a/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" "b/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" new file mode 100644 index 00000000..4879119e --- /dev/null +++ "b/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" @@ -0,0 +1,11 @@ +SET_ID,SET_NAME,INDIKATOR_ID,INDIKATOR_NAME,INDIKATOR_BESCHREIB,KOERPERSCHAFT_ID,KOERPERSCHAFT_NAME,JAHR,VALUE +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",433,"Berufswahlschule Bezirk Horgen",1995,192230 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",440,"Schulpsychologischer Dienst des Bezirks Horgen",1995,97576 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",456,"Gemeindezentrum Brüelmatt",1995,128003 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",474,"Schulzweckverband Bezirk Affoltern",1995,1074294 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",510,"Berufswahlschule Limmattal (BWL)",1995,338128 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",841,"Fürsorgeverband Weiningen",1996,335003 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",6,"Friedhofverband Weiningen",1996,287025 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",7,"Seniorenzentrum Im Morgen Weiningen",1996,899498 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",10,"Regionale Tierkörpersammelstelle Regensdorf",1996,32213 + diff --git a/ckanext/xloader/tests/test_jobs.py b/ckanext/xloader/tests/test_jobs.py index edca32d4..0058a3d5 100644 --- a/ckanext/xloader/tests/test_jobs.py +++ b/ckanext/xloader/tests/test_jobs.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + import os import json import random @@ -21,47 +23,49 @@ SOURCE_URL = 'http://www.example.com/static/file' -def mock_actions(func): - ''' - Decorator that mocks actions used by these tests - Based on ckan.test.helpers.mock_action - ''' - def wrapper(*args, **kwargs): - # Mock CKAN's resource_show API - from ckan.logic import get_action as original_get_action - - def side_effect(called_action_name): - if called_action_name == 'resource_show': - def mock_resource_show(context, data_dict): - return { - 'id': data_dict['id'], - 'name': 'short name', - 'url': SOURCE_URL, - 'format': 'CSV', - 'package_id': 'test-pkg', - } - return mock_resource_show - elif called_action_name == 'package_show': - def mock_package_show(context, data_dict): - return { - 'id': data_dict['id'], - 'name': 'pkg-name', - } - return mock_package_show - else: - return original_get_action(called_action_name) - try: - with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action: - mock_get_action.side_effect = side_effect - - return_value = func(*args, **kwargs) - finally: - pass - # Make sure to stop the mock, even with an exception - # mock_action.stop() - return return_value - - return make_decorator(func)(wrapper) +def mock_actions(resource_url=SOURCE_URL): + def decorator(func): + ''' + Decorator that mocks actions used by these tests + Based on ckan.test.helpers.mock_action + ''' + def wrapper(*args, **kwargs): + # Mock CKAN's resource_show API + from ckan.logic import get_action as original_get_action + + def side_effect(called_action_name): + if called_action_name == 'resource_show': + def mock_resource_show(context, data_dict): + return { + 'id': data_dict['id'], + 'name': 'short name', + 'url': resource_url, + 'format': 'CSV', + 'package_id': 'test-pkg', + } + return mock_resource_show + elif called_action_name == 'package_show': + def mock_package_show(context, data_dict): + return { + 'id': data_dict['id'], + 'name': 'pkg-name', + } + return mock_package_show + else: + return original_get_action(called_action_name) + try: + with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action: + mock_get_action.side_effect = side_effect + + return_value = func(*args, **kwargs) + finally: + pass + # Make sure to stop the mock, even with an exception + # mock_action.stop() + return return_value + + return make_decorator(func)(wrapper) + return decorator class TestxloaderDataIntoDatastore(util.PluginsMixin): @@ -87,7 +91,8 @@ def teardown_class(cls): connection.close() def register_urls(self, filename='simple.csv', - content_type='application/csv'): + content_type='application/csv' + resource_url=SOURCE_URL): """Mock some test URLs with responses. Mocks some URLs related to a data file and a CKAN resource that @@ -101,7 +106,7 @@ def register_urls(self, filename='simple.csv', responses.add_passthru(config['solr_url']) # A URL that just returns a static file - responses.add(responses.GET, SOURCE_URL, + responses.add(responses.GET, resource_url, body=get_sample_file(filename), content_type=content_type) @@ -157,7 +162,7 @@ def get_load_logs(self, task_id): .where(logs.c.job_id == task_id)) return Logs(result.fetchall()) - @mock_actions + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_simple_csv(self): # Test not only the load and xloader_hook is called at the end @@ -216,7 +221,7 @@ def test_simple_csv(self): eq_(job['status'], u'complete') eq_(job['error'], None) - @mock_actions + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_umlaut_and_extra_comma(self): self.register_urls(filename='umlaut_and_extra_comma.csv') @@ -260,7 +265,49 @@ def test_umlaut_and_extra_comma(self): eq_(job['status'], u'complete') eq_(job['error'], None) - @mock_actions + @mock_actions(resource_url='http://example.com/umlaut_name_äöü.csv') + @responses.activate + def test_resource_url_with_umlaut(self): + # test that xloader can handle URLs with umlauts + # e.g. http://www.web.statistik.zh.ch/ogd/data/KANTON_ZUERICH_gpfi_Jahresrechung_Zweckverbände.csv + self.register_urls(filename='umlaut_name_äöü.csv') + data = { + 'api_key': self.api_key, + 'job_type': 'xloader_to_datastore', + 'result_url': self.callback_url, + 'metadata': { + 'ckan_url': 'http://%s/' % self.host, + 'resource_id': self.resource_id + } + } + job_id = 'test{}'.format(random.randint(0, 1e5)) + + with mock.patch('ckanext.xloader.jobs.set_datastore_active_flag') \ + as mocked_set_datastore_active_flag: + # in tests we call jobs directly, rather than use rq, so mock + # get_current_job() + with mock.patch('ckanext.xloader.jobs.get_current_job', + return_value=mock.Mock(id=job_id)): + result = jobs.xloader_data_into_datastore(data) + assert result is None, jobs_db.get_job(job_id)['error']['message'] + + # Check it said it was successful + eq_(responses.calls[-1].request.url, 'http://www.ckan.org/api/3/action/xloader_hook') + job_dict = json.loads(responses.calls[-1].request.body) + assert job_dict['status'] == u'complete', job_dict + eq_(job_dict, + {u'metadata': {u'ckan_url': u'http://www.ckan.org/', + u'resource_id': u'foo-bar-42'}, + u'status': u'complete'}) + + logs = self.get_load_logs(job_id) + logs.assert_no_errors() + + job = jobs_db.get_job(job_id) + eq_(job['status'], u'complete') + eq_(job['error'], None) + + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_first_request_is_202_pending_response(self): # when you first get the CSV it returns this 202 response, which is