diff --git a/datapusher/jobs.py b/datapusher/jobs.py index 188ee6a..4964f7d 100644 --- a/datapusher/jobs.py +++ b/datapusher/jobs.py @@ -360,10 +360,13 @@ def push_to_datastore(task_id, input, dry_run=False): response.raise_for_status() cl = response.headers.get('content-length') - if cl and int(cl) > MAX_CONTENT_LENGTH: - raise util.JobError( - 'Resource too large to download: {cl} > max ({max_cl}).' - .format(cl=cl, max_cl=MAX_CONTENT_LENGTH)) + try: + if cl and int(cl) > MAX_CONTENT_LENGTH: + raise util.JobError( + 'Resource too large to download: {cl} > max ({max_cl}).' + .format(cl=cl, max_cl=MAX_CONTENT_LENGTH)) + except ValueError: + pass tmp = tempfile.TemporaryFile() length = 0 diff --git a/requirements-dev.txt b/requirements-dev.txt index c6aa38c..18832f8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ -r requirements.txt -httpretty>=0.6 +httpretty==0.9.4 nose==1.2.1 diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index 4c79761..4e5871f 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -106,7 +106,7 @@ def register_urls(self, filename='simple.csv', format='CSV', @httpretty.activate @raises(util.JobError) def test_too_large_content_length(self): - """It should raise JobError if the returned Content-Length header + """It should raise JobError if the returned Content-Length header is too large. If the returned header is larger than MAX_CONTENT_LENGTH then the async @@ -169,7 +169,32 @@ def test_too_large_file(self): body='a' * size, content_type='application/json', forcing_headers={ - 'content-length': '' + 'content-length': None + }) + + jobs.push_to_datastore('fake_id', data, True) + + @httpretty.activate + def test_content_length_string(self): + """If the Content-Length header value is a string, just ignore it. + """ + self.register_urls() + data = { + 'api_key': self.api_key, + 'job_type': 'push_to_datastore', + 'metadata': { + 'ckan_url': 'http://%s/' % self.host, + 'resource_id': self.resource_id + } + } + + source_url = 'http://www.source.org/static/file' + httpretty.register_uri( + httpretty.GET, source_url, + body='aaaaa', + content_type='application/json', + forcing_headers={ + 'Content-Length': 'some string' }) jobs.push_to_datastore('fake_id', data, True)