diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..b5158981 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +registries: + python-index-pypi-org: + type: python-index + url: https://pypi.org/ + replaces-base: true + username: "${{secrets.PYTHON_INDEX_PYPI_ORG_USERNAME}}" + password: "${{secrets.PYTHON_INDEX_PYPI_ORG_PASSWORD}}" + +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + time: "19:00" + open-pull-requests-limit: 10 + registries: + - python-index-pypi-org diff --git a/README.rst b/README.rst index 0586f336..95c3015e 100644 --- a/README.rst +++ b/README.rst @@ -196,6 +196,12 @@ This setting is shared with other plugins that download resource files, such as ckan.download_proxy = http://my-proxy:1234/ +You may also wish to configure the database to use your preferred date input style on COPY. +For example, to make [PostgreSQL](https://www.postgresql.org/docs/current/runtime-config-client.html#RUNTIME-CONFIG-CLIENT-FORMAT) +expect European (day-first) dates, you could add to ``postgresql.conf``: + + datestyle=ISO,DMY + ------------------------ Developer installation ------------------------ diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 3fa26803..aabc8148 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -152,10 +152,17 @@ def xloader_submit(context, data_dict): 'original_url': resource_dict.get('url'), } } - timeout = config.get('ckanext.xloader.job_timeout', '3600') + # Expand timeout for resources that have to be type-guessed + timeout = config.get( + 'ckanext.xloader.job_timeout', + '3600' if utils.datastore_resource_exists(res_id) else '10800') + log.debug("Timeout for XLoading resource %s is %s", res_id, timeout) + try: job = enqueue_job( - jobs.xloader_data_into_datastore, [data], rq_kwargs=dict(timeout=timeout) + jobs.xloader_data_into_datastore, [data], + title="xloader_submit: package: {} resource: {}".format(resource_dict.get('package_id'), res_id), + rq_kwargs=dict(timeout=timeout) ) except Exception: log.exception('Unable to enqueued xloader res_id=%s', res_id) diff --git a/ckanext/xloader/config_declaration.yaml b/ckanext/xloader/config_declaration.yaml index b31f12e2..feb1cc9c 100644 --- a/ckanext/xloader/config_declaration.yaml +++ b/ckanext/xloader/config_declaration.yaml @@ -29,9 +29,7 @@ groups: default: 1_000_000_000 example: 100000 description: | - The connection string for the jobs database used by XLoader. The - default of an sqlite file is fine for development. For production use a - Postgresql database. + The maximum file size that XLoader will attempt to load. type: int required: false - key: ckanext.xloader.use_type_guessing @@ -48,6 +46,15 @@ groups: type: bool required: false legacy_key: ckanext.xloader.just_load_with_messytables + - key: ckanext.xloader.max_type_guessing_length + default: 0 + example: 100000 + description: | + The maximum file size that will be passed to Tabulator if the + use_type_guessing flag is enabled. Larger files will use COPY even if + the flag is set. Defaults to 1/10 of the maximum content length. + type: int + required: false - key: ckanext.xloader.parse_dates_dayfirst default: False example: False diff --git a/ckanext/xloader/db.py b/ckanext/xloader/db.py index a3078ea4..a93eb0d8 100644 --- a/ckanext/xloader/db.py +++ b/ckanext/xloader/db.py @@ -191,9 +191,7 @@ def add_pending_job(job_id, job_type, api_key, if not metadata: metadata = {} - conn = ENGINE.connect() - trans = conn.begin() - try: + with ENGINE.begin() as conn: conn.execute(JOBS_TABLE.insert().values( job_id=job_id, job_type=job_type, @@ -225,12 +223,6 @@ def add_pending_job(job_id, job_type, api_key, ) if inserts: conn.execute(METADATA_TABLE.insert(), inserts) - trans.commit() - except Exception: - trans.rollback() - raise - finally: - conn.close() class InvalidErrorObjectError(Exception): diff --git a/ckanext/xloader/helpers.py b/ckanext/xloader/helpers.py index 6c4b8b9b..90c70933 100644 --- a/ckanext/xloader/helpers.py +++ b/ckanext/xloader/helpers.py @@ -28,13 +28,17 @@ def xloader_status_description(status): return _('Not Uploaded Yet') -def is_resource_supported_by_xloader(res_dict, check_access = True): +def is_resource_supported_by_xloader(res_dict, check_access=True): is_supported_format = XLoaderFormats.is_it_an_xloader_format(res_dict.get('format')) is_datastore_active = res_dict.get('datastore_active', False) user_has_access = not check_access or toolkit.h.check_access('package_update', {'id':res_dict.get('package_id')}) - try: - is_supported_url_type = res_dict.get('url_type') not in toolkit.h.datastore_rw_resource_url_types() - except AttributeError: - is_supported_url_type = (res_dict.get('url_type') == 'upload' or not res_dict.get('url_type')) + url_type = res_dict.get('url_type') + if url_type: + try: + is_supported_url_type = url_type not in toolkit.h.datastore_rw_resource_url_types() + except AttributeError: + is_supported_url_type = (url_type == 'upload') + else: + is_supported_url_type = True return (is_supported_format or is_datastore_active) and user_has_access and is_supported_url_type diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 7b96b993..8393c970 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -7,6 +7,7 @@ import tempfile import json import datetime +import os import traceback import sys @@ -21,7 +22,7 @@ from . import db, loader from .job_exceptions import JobError, HTTPError, DataTooBigError, FileCouldNotBeLoadedError -from .utils import set_resource_metadata +from .utils import datastore_resource_exists, set_resource_metadata try: from ckan.lib.api_token import get_user_from_token @@ -35,10 +36,13 @@ requests.packages.urllib3.disable_warnings() MAX_CONTENT_LENGTH = int(config.get('ckanext.xloader.max_content_length') or 1e9) +# Don't try Tabulator load on large files +MAX_TYPE_GUESSING_LENGTH = int(config.get('ckanext.xloader.max_type_guessing_length') or MAX_CONTENT_LENGTH / 10) MAX_EXCERPT_LINES = int(config.get('ckanext.xloader.max_excerpt_lines') or 0) CHUNK_SIZE = 16 * 1024 # 16kb DOWNLOAD_TIMEOUT = 30 +MAX_RETRIES = 1 RETRYABLE_ERRORS = ( errors.DeadlockDetected, errors.LockNotAvailable, @@ -89,18 +93,21 @@ def xloader_data_into_datastore(input): db.mark_job_as_errored(job_id, str(e)) job_dict['status'] = 'error' job_dict['error'] = str(e) - log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc())) + log.error('xloader error: %s, %s', e, traceback.format_exc()) errored = True except Exception as e: if isinstance(e, RETRYABLE_ERRORS): tries = job_dict['metadata'].get('tries', 0) - if tries == 0: + if tries < MAX_RETRIES: + tries = tries + 1 log.info("Job %s failed due to temporary error [%s], retrying", job_id, e) job_dict['status'] = 'pending' - job_dict['metadata']['tries'] = tries + 1 + job_dict['metadata']['tries'] = tries enqueue_job( xloader_data_into_datastore, [input], + title="retry xloader_data_into_datastore: resource: {} attempt {}".format( + job_dict['metadata']['resource_id'], tries), rq_kwargs=dict(timeout=RETRIED_JOB_TIMEOUT) ) return None @@ -109,7 +116,7 @@ def xloader_data_into_datastore(input): job_id, traceback.format_tb(sys.exc_info()[2])[-1] + repr(e)) job_dict['status'] = 'error' job_dict['error'] = str(e) - log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc())) + log.error('xloader error: %s, %s', e, traceback.format_exc()) errored = True finally: # job_dict is defined in xloader_hook's docstring @@ -226,11 +233,12 @@ def tabulator_load(): logger.info('Loading CSV') # If ckanext.xloader.use_type_guessing is not configured, fall back to # deprecated ckanext.xloader.just_load_with_messytables - use_type_guessing = asbool(config.get( - 'ckanext.xloader.use_type_guessing', config.get( - 'ckanext.xloader.just_load_with_messytables', False))) - logger.info("'use_type_guessing' mode is: %s", - use_type_guessing) + use_type_guessing = asbool( + config.get('ckanext.xloader.use_type_guessing', config.get( + 'ckanext.xloader.just_load_with_messytables', False))) \ + and not datastore_resource_exists(resource['id']) \ + and os.path.getsize(tmp_file.name) <= MAX_TYPE_GUESSING_LENGTH + logger.info("'use_type_guessing' mode is: %s", use_type_guessing) try: if use_type_guessing: tabulator_load() @@ -558,8 +566,7 @@ def __init__(self, task_id, input): self.input = input def emit(self, record): - conn = db.ENGINE.connect() - try: + with db.ENGINE.connect() as conn: # Turn strings into unicode to stop SQLAlchemy # "Unicode type received non-unicode bind param value" warnings. message = str(record.getMessage()) @@ -575,8 +582,6 @@ def emit(self, record): module=module, funcName=funcName, lineno=record.lineno)) - finally: - conn.close() class DatetimeJsonEncoder(json.JSONEncoder): diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index e64802e8..85be3f34 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -9,15 +9,16 @@ from decimal import Decimal import psycopg2 +from chardet.universaldetector import UniversalDetector from six.moves import zip -from tabulator import config as tabulator_config, Stream, TabulatorException +from tabulator import config as tabulator_config, EncodingError, Stream, TabulatorException from unidecode import unidecode import ckan.plugins as p from .job_exceptions import FileCouldNotBeLoadedError, LoaderError -from .parser import CSV_SAMPLE_LINES, XloaderCSVParser -from .utils import headers_guess, type_guess +from .parser import CSV_SAMPLE_LINES, TypeConverter +from .utils import datastore_resource_exists, headers_guess, type_guess from ckan.plugins.toolkit import config @@ -30,6 +31,52 @@ MAX_COLUMN_LENGTH = 63 tabulator_config.CSV_SAMPLE_LINES = CSV_SAMPLE_LINES +SINGLE_BYTE_ENCODING = 'cp1252' + + +class UnknownEncodingStream(object): + """ Provides a context manager that wraps a Tabulator stream + and tries multiple encodings if one fails. + + This is particularly relevant in cases like Latin-1 encoding, + which is usually ASCII and thus the sample could be sniffed as UTF-8, + only to run into problems later in the file. + """ + + def __init__(self, filepath, file_format, decoding_result, **kwargs): + self.filepath = filepath + self.file_format = file_format + self.stream_args = kwargs + self.decoding_result = decoding_result # {'encoding': 'EUC-JP', 'confidence': 0.99} + + def __enter__(self): + try: + + if (self.decoding_result and self.decoding_result['confidence'] and self.decoding_result['confidence'] > 0.7): + self.stream = Stream(self.filepath, format=self.file_format, encoding=self.decoding_result['encoding'], + ** self.stream_args).__enter__() + else: + self.stream = Stream(self.filepath, format=self.file_format, ** self.stream_args).__enter__() + + except (EncodingError, UnicodeDecodeError): + self.stream = Stream(self.filepath, format=self.file_format, + encoding=SINGLE_BYTE_ENCODING, **self.stream_args).__enter__() + return self.stream + + def __exit__(self, *args): + return self.stream.__exit__(*args) + + +def detect_encoding(file_path): + detector = UniversalDetector() + with open(file_path, 'rb') as file: + for line in file: + detector.feed(line) + if detector.done: + break + detector.close() + return detector.result # e.g. {'encoding': 'EUC-JP', 'confidence': 0.99} + def _fields_match(fields, existing_fields, logger): ''' Check whether all columns have the same names and types as previously, @@ -77,15 +124,17 @@ def _clear_datastore_resource(resource_id): def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): '''Loads a CSV into DataStore. Does not create the indexes.''' + decoding_result = detect_encoding(csv_filepath) + logger.info("load_csv: Decoded encoding: %s", decoding_result) # Determine the header row try: file_format = os.path.splitext(csv_filepath)[1].strip('.') - with Stream(csv_filepath, format=file_format) as stream: + with UnknownEncodingStream(csv_filepath, file_format, decoding_result) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] - with Stream(csv_filepath, format=file_format) as stream: + with UnknownEncodingStream(csv_filepath, file_format, decoding_result) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException as e: raise LoaderError('Tabulator error: {}'.format(e)) @@ -116,10 +165,16 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): logger.info('Ensuring character coding is UTF8') f_write = tempfile.NamedTemporaryFile(suffix=file_format, delete=False) try: - with Stream(csv_filepath, format=file_format, skip_rows=skip_rows) as stream: - stream.save(target=f_write.name, format='csv', encoding='utf-8', - delimiter=delimiter) - csv_filepath = f_write.name + save_args = {'target': f_write.name, 'format': 'csv', 'encoding': 'utf-8', 'delimiter': delimiter} + try: + with UnknownEncodingStream(csv_filepath, file_format, decoding_result, + skip_rows=skip_rows) as stream: + stream.save(**save_args) + except (EncodingError, UnicodeDecodeError): + with Stream(csv_filepath, format=file_format, encoding=SINGLE_BYTE_ENCODING, + skip_rows=skip_rows) as stream: + stream.save(**save_args) + csv_filepath = f_write.name # datastore db connection engine = get_write_engine() @@ -287,16 +342,18 @@ def load_table(table_filepath, resource_id, mimetype='text/csv', logger=None): # Determine the header row logger.info('Determining column names and types') + decoding_result = detect_encoding(table_filepath) + logger.info("load_table: Decoded encoding: %s", decoding_result) try: file_format = os.path.splitext(table_filepath)[1].strip('.') - with Stream(table_filepath, format=file_format, - custom_parsers={'csv': XloaderCSVParser}) as stream: + with UnknownEncodingStream(table_filepath, file_format, decoding_result, + post_parse=[TypeConverter().convert_types]) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] - with Stream(table_filepath, format=file_format, - custom_parsers={'csv': XloaderCSVParser}) as stream: + with UnknownEncodingStream(table_filepath, file_format, decoding_result, + post_parse=[TypeConverter().convert_types]) as stream: header_offset, headers = headers_guess(stream.sample) except TabulatorException as e: raise LoaderError('Tabulator error: {}'.format(e)) @@ -332,9 +389,11 @@ def load_table(table_filepath, resource_id, mimetype='text/csv', logger=None): for t, h in zip(types, headers)] headers = [header.strip()[:MAX_COLUMN_LENGTH] for header in headers if header.strip()] + type_converter = TypeConverter(types=types) - with Stream(table_filepath, format=file_format, skip_rows=skip_rows, - custom_parsers={'csv': XloaderCSVParser}) as stream: + with UnknownEncodingStream(table_filepath, file_format, decoding_result, + skip_rows=skip_rows, + post_parse=[type_converter.convert_types]) as stream: def row_iterator(): for row in stream: data_row = {} @@ -457,17 +516,6 @@ def send_resource_to_datastore(resource_id, headers, records): .format(str(e))) -def datastore_resource_exists(resource_id): - from ckan import model - context = {'model': model, 'ignore_auth': True} - try: - response = p.toolkit.get_action('datastore_search')(context, dict( - id=resource_id, limit=0)) - except p.toolkit.ObjectNotFound: - return False - return response or {'fields': []} - - def delete_datastore_resource(resource_id): from ckan import model context = {'model': model, 'user': '', 'ignore_auth': True} diff --git a/ckanext/xloader/parser.py b/ckanext/xloader/parser.py index 82539f4d..11e756cd 100644 --- a/ckanext/xloader/parser.py +++ b/ckanext/xloader/parser.py @@ -1,161 +1,73 @@ # -*- coding: utf-8 -*- -import csv +import datetime from decimal import Decimal, InvalidOperation -from itertools import chain +import re +import six from ckan.plugins.toolkit import asbool -from dateutil.parser import isoparser, parser -from dateutil.parser import ParserError - -from tabulator import helpers -from tabulator.parser import Parser +from dateutil.parser import isoparser, parser, ParserError from ckan.plugins.toolkit import config CSV_SAMPLE_LINES = 1000 +DATE_REGEX = re.compile(r'''^\d{1,4}[-/.\s]\S+[-/.\s]\S+''') -class XloaderCSVParser(Parser): - """Extends tabulator CSVParser to detect datetime and numeric values. +class TypeConverter: + """ Post-process table cells to convert strings into numbers and timestamps + as desired. """ - # Public - - options = [ - 'delimiter', - 'doublequote', - 'escapechar', - 'quotechar', - 'quoting', - 'skipinitialspace', - 'lineterminator' - ] - - def __init__(self, loader, force_parse=False, **options): - super(XloaderCSVParser, self).__init__(loader, force_parse, **options) - # Set attributes - self.__loader = loader - self.__options = options - self.__force_parse = force_parse - self.__extended_rows = None - self.__encoding = None - self.__dialect = None - self.__chars = None - - @property - def closed(self): - return self.__chars is None or self.__chars.closed - - def open(self, source, encoding=None): - # Close the character stream, if necessary, before reloading it. - self.close() - self.__chars = self.__loader.load(source, encoding=encoding) - self.__encoding = getattr(self.__chars, 'encoding', encoding) - if self.__encoding: - self.__encoding.lower() - self.reset() - - def close(self): - if not self.closed: - self.__chars.close() - - def reset(self): - helpers.reset_stream(self.__chars) - self.__extended_rows = self.__iter_extended_rows() - - @property - def encoding(self): - return self.__encoding - - @property - def dialect(self): - if self.__dialect: - dialect = { - 'delimiter': self.__dialect.delimiter, - 'doubleQuote': self.__dialect.doublequote, - 'lineTerminator': self.__dialect.lineterminator, - 'quoteChar': self.__dialect.quotechar, - 'skipInitialSpace': self.__dialect.skipinitialspace, - } - if self.__dialect.escapechar is not None: - dialect['escapeChar'] = self.__dialect.escapechar - return dialect - - @property - def extended_rows(self): - return self.__extended_rows - - # Private - - def __iter_extended_rows(self): - - def type_value(value): - """Returns numeric values as Decimal(). Uses dateutil to parse - date values. Otherwise, returns values as it receives them - (strings). - """ - if value in ('', None): - return '' - - try: - return Decimal(value) - except InvalidOperation: - pass - - try: - i = isoparser() - return i.isoparse(value) - except ValueError: - pass - - try: - p = parser() - yearfirst = asbool(config.get( - 'ckanext.xloader.parse_dates_yearfirst', False)) - dayfirst = asbool(config.get( - 'ckanext.xloader.parse_dates_dayfirst', False)) - return p.parse(value, yearfirst=yearfirst, dayfirst=dayfirst) - except ParserError: - pass - - return value - - sample, dialect = self.__prepare_dialect(self.__chars) - items = csv.reader(chain(sample, self.__chars), dialect=dialect) - for row_number, item in enumerate(items, start=1): - values = [] - for value in item: - value = type_value(value) - values.append(value) - yield row_number, None, list(values) - - def __prepare_dialect(self, stream): - - # Get sample - sample = [] - while True: - try: - sample.append(next(stream)) - except StopIteration: - break - if len(sample) >= CSV_SAMPLE_LINES: - break - - # Get dialect + def __init__(self, types=None): + self.types = types + + def convert_types(self, extended_rows): + """ Try converting cells to numbers or timestamps if applicable. + If a list of types was supplied, use that. + If not, then try converting each column to numeric first, + then to a timestamp. If both fail, just keep it as a string. + """ + for row_number, headers, row in extended_rows: + for cell_index, cell_value in enumerate(row): + if cell_value is None: + row[cell_index] = '' + if not cell_value: + continue + cell_type = self.types[cell_index] if self.types else None + if cell_type in [Decimal, None]: + converted_value = to_number(cell_value) + # Can't do a simple truthiness check, + # because 0 is a valid numeric result. + if converted_value is not None: + row[cell_index] = converted_value + continue + if cell_type in [datetime.datetime, None]: + converted_value = to_timestamp(cell_value) + if converted_value: + row[cell_index] = converted_value + yield (row_number, headers, row) + + +def to_number(value): + if not isinstance(value, six.string_types): + return None + try: + return Decimal(value) + except InvalidOperation: + return None + + +def to_timestamp(value): + if not isinstance(value, six.string_types) or not DATE_REGEX.search(value): + return None + try: + i = isoparser() + return i.isoparse(value) + except ValueError: try: - separator = '' - delimiter = self.__options.get('delimiter', ',\t;|') - dialect = csv.Sniffer().sniff(separator.join(sample), delimiter) - if not dialect.escapechar: - dialect.doublequote = True - except csv.Error: - class dialect(csv.excel): - pass - for key, value in self.__options.items(): - setattr(dialect, key, value) - # https://github.com/frictionlessdata/FrictionlessDarwinCore/issues/1 - if getattr(dialect, 'quotechar', None) == '': - setattr(dialect, 'quoting', csv.QUOTE_NONE) - - self.__dialect = dialect - return sample, dialect + p = parser() + yearfirst = asbool(config.get('ckanext.xloader.parse_dates_yearfirst', False)) + dayfirst = asbool(config.get('ckanext.xloader.parse_dates_dayfirst', False)) + return p.parse(value, yearfirst=yearfirst, dayfirst=dayfirst) + except ParserError: + return None diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 392b1cf5..6e65e466 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -78,8 +78,8 @@ def notify(self, entity, operation): See: ckan/model/modification.py.DomainObjectModificationExtension """ if operation != DomainObjectOperation.changed \ - or not isinstance(entity, Resource) \ - or not getattr(entity, 'url_changed', False): + or not isinstance(entity, Resource) \ + or not getattr(entity, 'url_changed', False): return context = { "ignore_auth": True, diff --git a/ckanext/xloader/templates/xloader/resource_data.html b/ckanext/xloader/templates/xloader/resource_data.html index 74a5f715..98027508 100644 --- a/ckanext/xloader/templates/xloader/resource_data.html +++ b/ckanext/xloader/templates/xloader/resource_data.html @@ -23,7 +23,7 @@ {% set delete_action = h.url_for('xloader.delete_datastore_table', id=pkg.id, resource_id=res.id) %}
{{ h.csrf_input() if 'csrf_input' in h }} - ","477.05","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","3206" +"206681442214","MR DAVID SHEARER","3.79","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2213" +"206681442215","MRS M SHONK + MR E T SHONK ","10.3","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2093" +"206681442216","MS AGATHA SKOURTIS","108.42","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","3025" +"206681442217","MR JAMES SMITH","108.42","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","4811" +"206681442218","MRS JILLIAN MELINDA SMITH","602.27","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2752" +"206681442219","MISS JESSICA SARAH STEAD","174.01","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2040" +"206681442220","MISS CHAU DONG MINH TANG","542.1","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","3065" +"206681442221","MR TROY TAYLOR","240.69","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","4000" +"206681442222","MR ANDREW PHILIP THOMPSON","2.17","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2204" +"206681442223","MR IVAN CONRAD TIMBS","702.02","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","2612" +"206681442224","MR J WAJNTRAUB + MRS S WAJNTRAUB ","542.1","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","3205" +"206681442225","MR HOWARD GRENVILLE WEBBER","400.61","VIRGIN AUSTRALIA HOLDINGS LIMITED","2012-02-28 00:00:00","4556" +"206681442226","JANI ILARI KALLA","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","6157" +"206681442227","GARY JOHN & DESLEY L CAHILL","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4816" +"206681442228","CARMEL ANASTASIA MEAGLIA","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2205" +"206681442229","ASHLEY & ANNIE BRUGGEMANN","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4671" +"206681442230","TERRY & MARY RITCHIE","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442231","BODY CORPORATE VILLAGE WAY CTS 19459","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4214" +"206681442232","MATHEW JOHN SHORTLAND","10","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2573" +"206681442233","TANYA MARIE TOWNSON","10.01","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442234","VENEE ELVA RUSSELL","10.02","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4035" +"206681442235","ELIZABETH FERNANCE","10.03","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4223" +"206681442236","CHARLES JOHN & OLWYN MARTIN","10.04","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4121" +"206681442237","ALFRED BRETT SEILER","10.05","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4129" +"206681442238","LOUISE WOODHAM & NATHAN FREY","10.07","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4400" +"206681442239","MITRA KHAKBAZ","10.09","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4005" +"206681442240","ALLAN EDWARD KILCULLEN","10.1","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4817" +"206681442241","BEVAN JOHN LISTON","10.11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442242","KRIS MICHAEL KANKAHAINEN","10.11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4107" +"206681442243","MICHAEL LYNN","10.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4860" +"206681442244","ALAN RAYMOND & GERAL BURKITT","10.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4228" +"206681442245","JENNIFER & NEVILLE MARXSEN","10.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4680" +"206681442246","DARREN MAIN GRANT & LISA MARIE GROSSKOPF","10.2","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4504" +"206681442247","PEARSON AUTOMOTIVE","10.23","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4064" +"206681442248","MR SHANE HOPE & MISS YVONNE HILTON","10.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4173" +"206681442249","CARMEL LESLEY NEILSON & WAYNE MERVYN NEILSON &","10.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4650" +"206681442250","STEPHEN KENNETH ROBERTSON","10.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4740" +"206681442251","SHIH CHE LIN","10.26","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4214" +"206681442252","DAVID BRETT BROWNE","10.29","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4558" +"206681442253","NEVILLE COLIN WOODHOUSE","10.32","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442254","DARRYN GREGORY & PET ROBIN","10.34","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4178" +"206681442255","DUDLEY JESSER","10.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442256","MURRAY JOHN & SANDRA DIXON","10.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4870" +"206681442257","SHATHISO JOHNSON BAREKI","10.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4215" +"206681442258","ARTHUR EDWARD & MAUR MACDONALD","10.39","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4390" +"206681442259","GARY GOLDBERG","10.4","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2480" +"206681442260","PHUONG VAN NGO","10.41","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4508" +"206681442261","JACQUELYN WILSON","10.42","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3046" +"206681442262","GARTH TURTON","10.42","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4051" +"206681442263","DAVID JAMES & ANNE M O'ROURKE","10.43","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4701" +"206681442264","ROBERT RUSSELL & VER MCKENZIE","10.45","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4503" +"206681442265","ESTATE OF DULCIE L SYKES","10.48","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4215" +"206681442266","LEESA GAYE OSMOND","10.51","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4671" +"206681442267","DAVID JOHN & ROSEMAR GILES","10.54","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4303" +"206681442268","SALLY & AQEEL AHMED","10.56","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442269","JUDITH MARJORY BURGESS","10.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3101" +"206681442270","TROY ANTONY EWART","10.61","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4301" +"206681442271","RODULFO MANOY & GEORGE HAJEK","10.62","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4152" +"206681442272","GLEN DUNSTAN","10.66","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3196" +"206681442273","ANNE RALSTON WRIGHT","10.73","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4825" +"206681442274","ALAN & NICOLE MAREE JACKSON","10.74","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4720" +"206681442275","DANIEL MALCOLM BROWN","10.81","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4501" +"206681442276","JENNIFER DEMERAL","10.82","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4214" +"206681442277","DARREN & LISA GARRETT","10.83","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4165" +"206681442278","LORRAINE & PETER JACKSON","10.84","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4740" +"206681442279","CHERYL MADELINE CAMPBELL","10.86","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4824" +"206681442280","OLAF PETER PRILL","10.89","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4305" +"206681442281","AJAY GIDH","10.9","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4051" +"206681442282","DEBRA JOANNE PRINDABLE","10.9","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4178" +"206681442283","MATTHEW WILLIAM CLARKE","10.96","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2914" +"206681442284","MARK STANLEY MCKENZIE","11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4207" +"206681442285","TREVOR & JANICE GARWOOD","11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4610" +"206681442286","LISA ANNE BRATINA","11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4228" +"206681442287","MICHAEL GEORGE KIRKWOOD","11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4561" +"206681442288","STEPHAN & JULIE BAWDEN","11.04","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4114" +"206681442289","PETER JOHN BOURKE","11.04","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4118" +"206681442290","TYRONE PAGE & ULRIKE","11.07","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4301" +"206681442291","SIMON ROBERT GRAY","11.08","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4006" +"206681442292","ALLAN NICHOLAS SCHWARZROCK","11.12","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4650" +"206681442293","IVAN J BLAKE & JAINE RIGTER","11.12","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4220" +"206681442294","DAVID MATTHEW REGINA CHRISTIE","11.12","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4151" +"206681442295","GEOFFREY WAYNE & EVAN GRIGG","11.14","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4720" +"206681442296","KYLIE JANELLE HARDCASTLE","11.14","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4013" +"206681442297","PAMELA ANN WELLER","11.15","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4655" +"206681442298","JASON PATRICK & ELIZ MURPHY","11.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4511" +"206681442299","MLADEN & VESNA SAJKO","11.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4157" +"206681442300","DEAN STEPHEN BROCKENSHIRE","11.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2571" +"206681442301","LISA CHRISTOBEL BOWKER","11.22","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4066" +"206681442302","MATTHEW RAY EBBAGE","11.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4101" +"206681442303","BRIAN & GEORGINA WHITLEY","11.25","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4521" +"206681442304","HAYLEY WESTON","11.25","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4159" +"206681442305","JAMES PATRICK HOCKING","11.28","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4127" +"206681442306","ROBERT ANDREW & SARA BROWNHALL","11.29","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442307","EDWARD JAMES DODGSON","11.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442308","MELISSA JOY DODD","11.32","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442309","JOSHUA CALVIN BEGENT","11.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4306" +"206681442311","DORATHY AMANDA WALTERS","11.4","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4744" +"206681442312","RICHARD ROBERTS & KYM RALEIGH","11.4","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4053" +"206681442313","SAMARA INSOLL","11.48","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4212" +"206681442314","NEIL GREGORY FLESSER","11.49","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4800" +"206681442315","EUNICE GLADYS WILBRAHAM","11.51","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4570" +"206681442316","KARA NICOLE MCINNES","11.57","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4503" +"206681442317","DAVID BLYTH","11.58","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4078" +"206681442318","KEVIN & MARION KEIR","11.58","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4216" +"206681442319","FRANCES & CHARLES KEEBLE","11.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4500" +"206681442320","LYNETTE ANNE & PETER NISSEN","11.6","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442321","DANIEL PETER JOHNSON","11.61","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4051" +"206681442322","ALLAN & EUNICE DELLAWAY","11.62","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4122" +"206681442323","CHRISTOPHER JOHN BEEM","11.63","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4101" +"206681442324","DAVID JAMES & KELLIE POULTON","11.64","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442325","MAVIS CAROLIN SCOTT","11.64","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4018" +"206681442326","REEGAN & ADAM MARTIN","11.68","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2153" +"206681442327","DENYSE B BONNEY","11.7","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4811" +"206681442328","JAMES ANDERSON","11.71","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4220" +"206681442329","SUSANNAH PINTER","11.72","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4744" +"206681442330","BRENTON MARK & KAREN GARNETT","11.78","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4306" +"206681442331","PL CAMELOT VENTURES AS TRUSTEE FOR K F T TRUST NO","11.82","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4215" +"206681442332","RON HENRY SCHMIDT","11.84","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","830" +"206681442333","ROSS COCKBURN & AUDREY KILL","11.86","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4010" +"206681442334","BENJAMIN CLARK","11.88","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4306" +"206681442335","IRIS LEAH TERESA BAKER","11.9","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2170" +"206681442336","MARK JOHN DEEBLE","11.94","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4740" +"206681442337","CHRISTINE & BARRY RIGBY","11.94","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2485" +"206681442338","NATASHA ANN WOODWARD","11.97","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4869" +"206681442339","BENJAMIN JOHN CANSDALE","11.98","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4064" +"206681442340","PETER HERALD","11.98","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4184" +"206681442341","SIMON CUSHWAY","11.99","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4154" +"206681442342","ANTHONY & MICHELLE JOHNSTON","12","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4551" +"206681442343","PAUL HAUCK","12.03","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4000" +"206681442344","RONALD ALBERT & PEAR NORTHILL","12.03","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4413" +"206681442345","ROBYN ELLEN SOMERS","12.03","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4178" +"206681442346","ROSE ANN HODGMAN","12.06","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4068" +"206681442347","JOHN & MARDI BOLTON","12.09","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4165" +"206681442348","KRYSTYNA RENNIE","12.09","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4053" +"206681442349","JOANNE BARSBY","12.12","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442350","BRENDAN JAMES FELSCHOW","12.14","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4508" +"206681442351","MARTIN WILLIAM HARRISON","12.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4870" +"206681442352","PATRICK HEINEMANN","12.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4870" +"206681442353","ELEKRA & SPENCER RORIE","12.17","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4211" +"206681442354","ROBERT CLIVE & NOELE CROCKER","12.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4211" +"206681442355","DANIEL JOSEPH & DAVI CARMICHAEL","12.21","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4160" +"206681442356","WENBO JIANG & XIU FAN CHEN","12.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4303" +"206681442357","NOEL JEFFREY BRADY","12.27","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4550" +"206681442358","DARREN RICHARD GOSSNER & MATTHEW JOHN ANDERSON","12.29","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4500" +"206681442359","STEPHEN MICHAEL & MA JOLLY","12.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442360","SHONA & ARCHIE WALLACE","12.34","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4504" +"206681442361","ZOFIA HYS","12.34","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4300" +"206681442362","PIROSKA KING","12.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4154" +"206681442363","ARVIN CHAND & AMITA MOHINI","12.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4503" +"206681442364","WIETSKE GERARDINA & GAUNT","12.38","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4309" +"206681442365","MARK REGINALD MATTHEWS","12.39","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4811" +"206681442366","SHARP ARLEEN & CLINTON","12.4","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","6020" +"206681442367","EMOKE & LASZLO & MAR ZSOLDOS","12.41","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4005" +"206681442368","MARK & KARON KELLER","12.42","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4702" +"206681442369","JODIE KATRINA & TONY MCLACHLAN","12.43","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442370","ALAN WARWICK & LINDA LEWIS","12.45","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4670" +"206681442371","ADRIAN WAYNE LORRAWAY","12.5","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4702" +"206681442372","NICHOLE KRISTY MIKLOS","12.53","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4152" +"206681442373","NATASHA LEANNE HAYES","12.54","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4017" +"206681442374","KAREN LEE & DARREN J SHEEHAN","12.55","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4516" +"206681442375","RACHAEL MAY COLLINS-COOK","12.58","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4211" +"206681442376","TAMARA JUNE WEIGHT & SUSANNE ELIZABETH DEVINE","12.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442377","RODNEY GATES","12.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","7015" +"206681442378","REBECCA & LEE-ANNE SMITH","12.61","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","830" +"206681442379","ADAM WILLIAM JOHNSON","12.62","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4069" +"206681442380","ZAC ASHLEY & ALEXAND MORGAN","12.63","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4165" +"206681442381","HILARY SEALY","12.64","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4211" +"206681442382","NAOMI JOHNSTONE & SCOTT LENAN","12.68","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4207" +"206681442383","WAYNE FLICKER","12.7","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2756" +"206681442384","BRENDA ANDERSON","12.71","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4811" +"206681442385","MATTHEW JAMES ALLEN","12.71","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4002" +"206681442386","MARIA-THERESIA ALTENHOFEN-CROSS & JOHN ERI CROSS","12.72","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4570" +"206681442387","MELODIE ZYLSTRA","12.72","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4570" +"206681442388","AMANDA & GRAHAM SWALLOW","12.75","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4720" +"206681442389","GRAEME ROBERT & ROBI DOHERTY","12.75","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4214" +"206681442390","GILLIAN LEIGH O'SULLIVAN","12.79","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4817" +"206681442391","JULIA MELLICK","12.84","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4178" +"206681442392","TOLISIALE & HAMAKO MAHINA","12.87","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4300" +"206681442393","SIMON JOHN STEVENS","12.89","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4305" +"206681442394","MICHAEL ANTHONY & DE SNELSON","12.89","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4817" +"206681442395","QUERIDA JO LOFTES","12.89","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4812" +"206681442396","LORRAINE VICTORIA DIAS","12.89","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4000" +"206681442397","JOHN MICHAEL TRAVIS LINLEY","12.92","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4051" +"206681442398","CAROLINE HENDRY & RICHARD HOPKINS","12.93","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4067" +"206681442399","JOSH EAGLE","12.95","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4030" +"206681442400","MARK SHAWN FROST & BELINDA JEAN MARSHALL","12.95","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4221" +"206681442401","BRENT & GABRIELLE ANTHONY","12.96","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4213" +"206681442402","RICHARD SADLER","12.98","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4065" +"206681442403","GROVE FRUIT JUICE PTY LTD","13","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4122" +"206681442404","LEAH SPARKS","13","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4561" +"206681442405","JAMES MAURICE & PATR GORDON","13","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4870" +"206681442406","MARK JOSEPH SEARS","13","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4565" +"206681442407","SOPHIE VICTORIA STEWART & TREVOR MATTHEW ROWE","13","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4055" +"206681442408","BOBBY JAMES & SIMONE TAYLOR","13.02","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","6254" +"206681442409","PATRICK MICHAEL & ME REEVES","13.08","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4101" +"206681442410","MAURICE GROGNUZ","13.09","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4670" +"206681442411","ALAN PIGOTT & ALAN CONDER","13.11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2025" +"206681442412","SAMANTHA & CAMERON SCHELBACH","13.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4309" +"206681442413","SHERIDAN ANNE ST CLAIR","13.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4164" +"206681442414","ANDREW CHRISTIE","13.17","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4521" +"206681442415","MARK ANDREW & MELISS VINTON","13.17","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4508" +"206681442416","IRWIN DOUGLAS & MARI SORENSEN","13.2","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4305" +"206681442417","CARLY SUSAN BENNETTS","13.23","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4034" +"206681442418","RYAN THORNTON","13.24","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2560" +"206681442419","RICHARD BAILEY","13.26","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3850" +"206681442420","DAVID IAN & EMILY RU PRYOR","13.27","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4120" +"206681442421","WILLIAM SINCLAIR","13.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4514" +"206681442422","CATHERINE LUCILLE VALENTINE & ROBERT WAREING","13.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4165" +"206681442423","RAYMOND JAMES JONES","13.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4170" +"206681442424","ANDREW STEWART T/A AWE COMMUNICATIONS","13.3","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4207" +"206681442425","TONY RONALD OSBOURNE","13.35","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4311" +"206681442426","MARK JOHN & LENY FIG O'HARA","13.35","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4825" +"206681442427","CECILIA ASHLEY & DAV BUTLER","13.35","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4816" +"206681442428","WILLIAM LEATHAM","13.36","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4170" +"206681442429","MAXWELL RAYMOND MATHERS & DENISE MAREE MELLARE","13.44","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4129" +"206681442430","RENE & JACQUELINE WASSERFUHR","13.44","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4556" +"206681442431","MICHAEL LEIGH KENNEDY","13.48","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4610" +"206681442432","MEDECO MEDICAL CENTRE BEENLEIGH","13.5","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4207" +"206681442433","GARY PAUL & GAYE SHELLEY","13.5","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4510" +"206681442434","STEVE & BRENDA GEIGER","13.53","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4740" +"206681442435","GREGORY BERNARD JAMES","13.53","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4051" +"206681442436","ROBBIE DEEBLE","13.56","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4740" +"206681442437","OWEN TRAYNOR","13.56","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","6076" +"206681442438","TONI MICHELLE & SHAN MORGAN","13.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4717" +"206681442439","NICOLAS VAN HORTON","13.59","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4220" +"206681442440","IAN BOWDEN","13.6","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4886" +"206681442441","QUEENSLAND COUNTRY CREDIT UNION - JIMBOOMBA","13.61","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442442","ALANA FELLINGHAM","13.62","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4551" +"206681442443","ALLAN JOHN & CARMEL BETHEL","13.62","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4871" +"206681442444","PETER WILLIAM & ODET NORMAN","13.63","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442445","EMILY & MATTHEW PARSLOW","13.68","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4173" +"206681442446","JAMES OI YUEN GOCK","13.69","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2049" +"206681442447","JODIE ELIZABETH MORRISON","13.7","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4280" +"206681442448","BELINDA JANE HARNETT-PETERS & RANDALL NEI PETERS","13.74","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4017" +"206681442449","JULIEN & CHRISTIAN JUVIGNY","13.78","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4215" +"206681442450","SUSAN JOY MURRAY & THOMAS HOGAN","13.79","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4217" +"206681442451","PATRICK COLIN & HEAT HARRIS","13.8","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4506" +"206681442452","LINDY BOTHA","13.84","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4154" +"206681442453","PATRICIA LORETTA & D KNIGHT","13.85","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4650" +"206681442454","COWBURN CONSULTING PTY LTD","13.87","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4000" +"206681442455","SPENCER JAMES HAMILTON","13.9","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4300" +"206681442456","ANNA LOUISE ROSS","13.95","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4170" +"206681442457","JOHN HUGH & BOB SUTHERLAND","13.98","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4509" +"206681442458","ROBERTA MARY MACNEE","13.99","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4567" +"206681442459","MATTHEW CHRISTENSEN","14.03","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4152" +"206681442460","TROY & KIRSTY JEFFRIES","14.04","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4370" +"206681442461","WILLIAM GEORGE BALSDON","14.05","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4878" +"206681442462","JAIME LISA CAMPBELL & DANIEL BEVERIDGE","14.07","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4216" +"206681442463","NANCY JOHANNESSON","14.11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4505" +"206681442464","JOSHUA FRANK SEIDL","14.11","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4122" +"206681442465","DAVID LESTER","14.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4817" +"206681442466","MATHIAS DONALD","14.16","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4103" +"206681442467","GLEN EVAN & HAYLEE L MARTIN","14.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4350" +"206681442468","JOHN GORDON EVANS","14.19","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4814" +"206681442469","DIANA NOYCE & LAURENCE VIZER T/A","14.2","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4502" +"206681442470","GREIG MANLEY","14.22","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3040" +"206681442471","BRENDON ANSELL","14.23","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4171" +"206681442472","CATHERINE A ROBERTSON & PAUL BROMILEY","14.27","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4064" +"206681442473","ADAM LEE & SAMANTHA RANKIN","14.28","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4132" +"206681442474","BERNICE BOYS","14.34","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4011" +"206681442475","HAYLEY MICHELLE BURROW","14.34","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2153" +"206681442476","SIONE FAUMUINA","14.42","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4815" +"206681442477","GERARD JARMAN","14.44","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","3337" +"206681442478","DOUGLAS CECIL GOOLEY","14.48","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","2481" +"206681442479","ANTHONY AUGUSTO HENRIQUES T/A CAFÚ VILA FRANCA","14.5","SUNCORP GENERAL INSURANCE","2012-03-12 00:00:00","4020" diff --git a/ckanext/xloader/tests/samples/sample_with_blanks.csv b/ckanext/xloader/tests/samples/sample_with_blanks.csv index 2b7c415c..b53b25db 100644 --- a/ckanext/xloader/tests/samples/sample_with_blanks.csv +++ b/ckanext/xloader/tests/samples/sample_with_blanks.csv @@ -1,4 +1,4 @@ -Funding agency,Program title,Opening date,Service ID -DTIS,Visitor First Experiences Fund,23/03/2023,63039 -DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040 -,,,63041 +Funding agency,Program title,Opening date,Service ID +DTIS,Visitor First Experiences Fund,23/03/2023,63039 +DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040 +,,,63041 diff --git a/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv b/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv new file mode 100644 index 00000000..7fe94e5b --- /dev/null +++ b/ckanext/xloader/tests/samples/sample_with_quoted_commas.csv @@ -0,0 +1,4 @@ +Funding agency,Program title,Opening date,Service ID +DTIS,"Department of Employment, Small Business and Training",23/03/2023,63039 +DTIS,"Foo, baz, meh",22/03/2023,63040 +,,,63041 diff --git a/ckanext/xloader/tests/samples/simple-large.csv b/ckanext/xloader/tests/samples/simple-large.csv index 53d3fb24..46c6c3b9 100644 --- a/ckanext/xloader/tests/samples/simple-large.csv +++ b/ckanext/xloader/tests/samples/simple-large.csv @@ -1,4 +1,5 @@ id,text +0,- 1,a 2,b 3,c @@ -49997,4 +49998,4 @@ id,text 49996,x 49997,y 49998,z -49999,a \ No newline at end of file +49999,a diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index 4c4e2820..a692eecc 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -632,6 +632,18 @@ def test_with_blanks(self, Session): ) assert len(self._get_records(Session, resource_id)) == 3 + def test_with_quoted_commas(self, Session): + csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_csv( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 3 + def test_with_mixed_quotes(self, Session): csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") resource = factories.Resource() @@ -938,6 +950,23 @@ def test_simple(self, Session): u"text", ] + def test_simple_large_file(self, Session): + csv_filepath = get_sample_filepath("simple-large.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert self._get_column_types(Session, resource_id) == [ + u"int4", + u"tsvector", + u"numeric", + u"text", + ] + # test disabled by default to avoid adding large file to repo and slow test @pytest.mark.skip def test_boston_311_complete(self): @@ -1180,6 +1209,30 @@ def test_no_entries(self): logger=logger, ) + def test_with_quoted_commas(self, Session): + csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 3 + + def test_with_iso_8859_1(self, Session): + csv_filepath = get_sample_filepath("non_utf8_sample.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 266 + def test_with_mixed_quotes(self, Session): csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") resource = factories.Resource() @@ -1191,3 +1244,24 @@ def test_with_mixed_quotes(self, Session): logger=logger, ) assert len(self._get_records(Session, resource_id)) == 2 + + def test_preserving_time_ranges(self, Session): + """ Time ranges should not be treated as timestamps + """ + csv_filepath = get_sample_filepath("non_timestamp_sample.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert self._get_records(Session, resource_id) == [ + (1, "Adavale", 4474, Decimal("-25.9092582"), Decimal("144.5975769"), + "8:00", "16:00", datetime.datetime(2018, 7, 19)), + (2, "Aramac", 4726, Decimal("-22.971298"), Decimal("145.241481"), + "9:00-13:00", "14:00-16:45", datetime.datetime(2018, 7, 17)), + (3, "Barcaldine", 4725, Decimal("-23.55327901"), Decimal("145.289156"), + "9:00-12:30", "13:30-16:30", datetime.datetime(2018, 7, 20)) + ] diff --git a/ckanext/xloader/tests/test_parser.py b/ckanext/xloader/tests/test_parser.py index 67929d9f..ac4047dd 100644 --- a/ckanext/xloader/tests/test_parser.py +++ b/ckanext/xloader/tests/test_parser.py @@ -6,7 +6,7 @@ from datetime import datetime from tabulator import Stream -from ckanext.xloader.parser import XloaderCSVParser +from ckanext.xloader.parser import TypeConverter csv_filepath = os.path.abspath( os.path.join(os.path.dirname(__file__), "samples", "date_formats.csv") @@ -16,7 +16,7 @@ class TestParser(object): def test_simple(self): with Stream(csv_filepath, format='csv', - custom_parsers={'csv': XloaderCSVParser}) as stream: + post_parse=[TypeConverter().convert_types]) as stream: assert stream.sample == [ [ 'date', @@ -49,7 +49,7 @@ def test_simple(self): def test_dayfirst(self): print('test_dayfirst') with Stream(csv_filepath, format='csv', - custom_parsers={'csv': XloaderCSVParser}) as stream: + post_parse=[TypeConverter().convert_types]) as stream: assert stream.sample == [ [ 'date', @@ -82,7 +82,7 @@ def test_dayfirst(self): def test_yearfirst(self): print('test_yearfirst') with Stream(csv_filepath, format='csv', - custom_parsers={'csv': XloaderCSVParser}) as stream: + post_parse=[TypeConverter().convert_types]) as stream: assert stream.sample == [ [ 'date', @@ -115,7 +115,7 @@ def test_yearfirst(self): @pytest.mark.ckan_config("ckanext.xloader.parse_dates_yearfirst", True) def test_yearfirst_dayfirst(self): with Stream(csv_filepath, format='csv', - custom_parsers={'csv': XloaderCSVParser}) as stream: + post_parse=[TypeConverter().convert_types]) as stream: assert stream.sample == [ [ 'date', diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index 3e29509a..073a8091 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -247,3 +247,13 @@ def type_guess(rows, types=TYPES, strict=False): guesses_tuples = [(t, guess[t]) for t in types if t in guess] _columns.append(max(guesses_tuples, key=lambda t_n: t_n[1])[0]) return _columns + + +def datastore_resource_exists(resource_id): + context = {'model': model, 'ignore_auth': True} + try: + response = p.toolkit.get_action('datastore_search')(context, dict( + id=resource_id, limit=0)) + except p.toolkit.ObjectNotFound: + return False + return response or {'fields': []} diff --git a/ckanext/xloader/views.py b/ckanext/xloader/views.py index 399d6ad6..1988b6d6 100644 --- a/ckanext/xloader/views.py +++ b/ckanext/xloader/views.py @@ -1,4 +1,4 @@ -from flask import Blueprint, request +from flask import Blueprint from ckan.plugins.toolkit import _, h, g, render, request, abort, NotAuthorized, get_action, ObjectNotFound diff --git a/requirements.txt b/requirements.txt index 58540beb..fe92b6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ six>=1.12.0 tabulator==1.53.5 Unidecode==1.0.22 python-dateutil>=2.8.2 +chardet==5.2.0 \ No newline at end of file