Skip to content

Commit

Permalink
Add test for umlaut CSV url
Browse files Browse the repository at this point in the history
  • Loading branch information
Stefan Oderbolz committed Nov 7, 2018
1 parent 9f1729f commit 3120e94
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 46 deletions.
11 changes: 11 additions & 0 deletions ckanext/xloader/tests/samples/umlaut_name_äöü.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
SET_ID,SET_NAME,INDIKATOR_ID,INDIKATOR_NAME,INDIKATOR_BESCHREIB,KOERPERSCHAFT_ID,KOERPERSCHAFT_NAME,JAHR,VALUE
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",433,"Berufswahlschule Bezirk Horgen",1995,192230
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",440,"Schulpsychologischer Dienst des Bezirks Horgen",1995,97576
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",456,"Gemeindezentrum Brüelmatt",1995,128003
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",474,"Schulzweckverband Bezirk Affoltern",1995,1074294
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",510,"Berufswahlschule Limmattal (BWL)",1995,338128
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",841,"Fürsorgeverband Weiningen",1996,335003
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",6,"Friedhofverband Weiningen",1996,287025
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",7,"Seniorenzentrum Im Morgen Weiningen",1996,899498
6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",10,"Regionale Tierkörpersammelstelle Regensdorf",1996,32213

142 changes: 96 additions & 46 deletions ckanext/xloader/tests/test_jobs.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# encoding: utf-8

import os
import json
import random
Expand All @@ -21,47 +23,49 @@

SOURCE_URL = 'http://www.example.com/static/file'

def mock_actions(func):
'''
Decorator that mocks actions used by these tests
Based on ckan.test.helpers.mock_action
'''
def wrapper(*args, **kwargs):
# Mock CKAN's resource_show API
from ckan.logic import get_action as original_get_action

def side_effect(called_action_name):
if called_action_name == 'resource_show':
def mock_resource_show(context, data_dict):
return {
'id': data_dict['id'],
'name': 'short name',
'url': SOURCE_URL,
'format': 'CSV',
'package_id': 'test-pkg',
}
return mock_resource_show
elif called_action_name == 'package_show':
def mock_package_show(context, data_dict):
return {
'id': data_dict['id'],
'name': 'pkg-name',
}
return mock_package_show
else:
return original_get_action(called_action_name)
try:
with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action:
mock_get_action.side_effect = side_effect

return_value = func(*args, **kwargs)
finally:
pass
# Make sure to stop the mock, even with an exception
# mock_action.stop()
return return_value

return make_decorator(func)(wrapper)
def mock_actions(resource_url=SOURCE_URL):
def decorator(func):
'''
Decorator that mocks actions used by these tests
Based on ckan.test.helpers.mock_action
'''
def wrapper(*args, **kwargs):
# Mock CKAN's resource_show API
from ckan.logic import get_action as original_get_action

def side_effect(called_action_name):
if called_action_name == 'resource_show':
def mock_resource_show(context, data_dict):
return {
'id': data_dict['id'],
'name': 'short name',
'url': resource_url,
'format': 'CSV',
'package_id': 'test-pkg',
}
return mock_resource_show
elif called_action_name == 'package_show':
def mock_package_show(context, data_dict):
return {
'id': data_dict['id'],
'name': 'pkg-name',
}
return mock_package_show
else:
return original_get_action(called_action_name)
try:
with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action:
mock_get_action.side_effect = side_effect

return_value = func(*args, **kwargs)
finally:
pass
# Make sure to stop the mock, even with an exception
# mock_action.stop()
return return_value

return make_decorator(func)(wrapper)
return decorator


class TestxloaderDataIntoDatastore(util.PluginsMixin):
Expand All @@ -87,7 +91,8 @@ def teardown_class(cls):
connection.close()

def register_urls(self, filename='simple.csv',
content_type='application/csv'):
content_type='application/csv',
resource_url=None):
"""Mock some test URLs with responses.
Mocks some URLs related to a data file and a CKAN resource that
Expand All @@ -98,10 +103,13 @@ def register_urls(self, filename='simple.csv',
resource_show URL for the resource that contains the data file
"""
if not resource_url:
resource_url = SOURCE_URL

responses.add_passthru(config['solr_url'])

# A URL that just returns a static file
responses.add(responses.GET, SOURCE_URL,
responses.add(responses.GET, resource_url,
body=get_sample_file(filename),
content_type=content_type)

Expand Down Expand Up @@ -157,7 +165,7 @@ def get_load_logs(self, task_id):
.where(logs.c.job_id == task_id))
return Logs(result.fetchall())

@mock_actions
@mock_actions(resource_url=SOURCE_URL)
@responses.activate
def test_simple_csv(self):
# Test not only the load and xloader_hook is called at the end
Expand Down Expand Up @@ -216,7 +224,7 @@ def test_simple_csv(self):
eq_(job['status'], u'complete')
eq_(job['error'], None)

@mock_actions
@mock_actions(resource_url=SOURCE_URL)
@responses.activate
def test_umlaut_and_extra_comma(self):
self.register_urls(filename='umlaut_and_extra_comma.csv')
Expand Down Expand Up @@ -260,7 +268,49 @@ def test_umlaut_and_extra_comma(self):
eq_(job['status'], u'complete')
eq_(job['error'], None)

@mock_actions
@mock_actions(resource_url='http://example.com/umlaut_name_äöü.csv')
@responses.activate
def test_resource_url_with_umlaut(self):
# test that xloader can handle URLs with umlauts
# e.g. http://www.web.statistik.zh.ch/ogd/data/KANTON_ZUERICH_gpfi_Jahresrechung_Zweckverbände.csv
self.register_urls(filename='umlaut_name_äöü.csv')
data = {
'api_key': self.api_key,
'job_type': 'xloader_to_datastore',
'result_url': self.callback_url,
'metadata': {
'ckan_url': 'http://%s/' % self.host,
'resource_id': self.resource_id
}
}
job_id = 'test{}'.format(random.randint(0, 1e5))

with mock.patch('ckanext.xloader.jobs.set_datastore_active_flag') \
as mocked_set_datastore_active_flag:
# in tests we call jobs directly, rather than use rq, so mock
# get_current_job()
with mock.patch('ckanext.xloader.jobs.get_current_job',
return_value=mock.Mock(id=job_id)):
result = jobs.xloader_data_into_datastore(data)
assert result is None, jobs_db.get_job(job_id)['error']['message']

# Check it said it was successful
eq_(responses.calls[-1].request.url, 'http://www.ckan.org/api/3/action/xloader_hook')
job_dict = json.loads(responses.calls[-1].request.body)
assert job_dict['status'] == u'complete', job_dict
eq_(job_dict,
{u'metadata': {u'ckan_url': u'http://www.ckan.org/',
u'resource_id': u'foo-bar-42'},
u'status': u'complete'})

logs = self.get_load_logs(job_id)
logs.assert_no_errors()

job = jobs_db.get_job(job_id)
eq_(job['status'], u'complete')
eq_(job['error'], None)

@mock_actions(resource_url=SOURCE_URL)
@responses.activate
def test_first_request_is_202_pending_response(self):
# when you first get the CSV it returns this 202 response, which is
Expand Down

0 comments on commit 3120e94

Please sign in to comment.