From 35f17573079c5213a5282207376944270b6a9a08 Mon Sep 17 00:00:00 2001 From: Guido Scicolone Date: Fri, 23 Aug 2019 15:03:27 +0200 Subject: [PATCH] colon separator support added --- .gitignore | 1 + README.md | 9 + bin/comma_splice | 13 +- lib/comma_splice/file_corrector.rb | 19 +- lib/comma_splice/helpers/comma_calculator.rb | 5 + lib/comma_splice/helpers/content_finder.rb | 12 +- lib/comma_splice/helpers/line.rb | 7 +- .../helpers/variable_column_finder.rb | 14 +- lib/comma_splice/line_corrector.rb | 13 +- spec/basic_spec.rb | 176 ++++++++++++++++-- spec/content_finder_spec.rb | 69 ++++++- spec/test_csvs/10000-maniacs-colons-fixed.csv | 29 +++ spec/test_csvs/10000-maniacs-colons.csv | 29 +++ spec/test_csvs/2015-07-17-colons.txt | 26 +++ spec/test_csvs/test-colons.csv | 3 + ...scaped-colons-and-non-csv-header-fixed.csv | 55 ++++++ .../unescaped-colons-and-non-csv-header.csv | 55 ++++++ spec/test_csvs/unescaped-colons-fixed.csv | 40 ++++ spec/test_csvs/unescaped-colons.csv | 40 ++++ .../unescaped-quotes-colons-fixed.csv | 43 +++++ spec/test_csvs/unescaped-quotes-colons.csv | 43 +++++ spec/variable_column_finder_spec.rb | 37 +++- 22 files changed, 683 insertions(+), 55 deletions(-) create mode 100644 spec/test_csvs/10000-maniacs-colons-fixed.csv create mode 100644 spec/test_csvs/10000-maniacs-colons.csv create mode 100644 spec/test_csvs/2015-07-17-colons.txt create mode 100644 spec/test_csvs/test-colons.csv create mode 100644 spec/test_csvs/unescaped-colons-and-non-csv-header-fixed.csv create mode 100644 spec/test_csvs/unescaped-colons-and-non-csv-header.csv create mode 100644 spec/test_csvs/unescaped-colons-fixed.csv create mode 100644 spec/test_csvs/unescaped-colons.csv create mode 100644 spec/test_csvs/unescaped-quotes-colons-fixed.csv create mode 100644 spec/test_csvs/unescaped-quotes-colons.csv diff --git a/.gitignore b/.gitignore index 6fd3af4..b163798 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ .DS_Store .byebug_history test-file.txt +.idea/ diff --git a/README.md b/README.md index f7c148c..a849451 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o ```ruby CommaSplice::FileCorrector.new(file_path).bad_lines.size + + #you can specify another separator + CommaSplice::FileCorrector.new(file_path, separator: ';').bad_lines.size ``` ``` comma_splice bad_line_count /path/to/file.csv @@ -95,6 +98,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o ##### Display the fixed contents ```ruby CommaSplice::FileCorrector.new(file_path).corrected + + #you can specify another separator + CommaSplice::FileCorrector.new(file_path, separator: ';').corrected ``` ```bash comma_splice correct /path/to/file.csv @@ -103,6 +109,9 @@ You can use this in a ruby program by using installing the `comma_splice` gem, o ##### Process a file and save the fixed version ```ruby CommaSplice::FileCorrector.new(file_path).save(save_path) + + #you can specify another separator + CommaSplice::FileCorrector.new(file_path, separator: ';').save(save_path) ``` ```bash comma_splice fix /path/to/file.csv /path/to/save diff --git a/bin/comma_splice b/bin/comma_splice index 085e025..8402418 100755 --- a/bin/comma_splice +++ b/bin/comma_splice @@ -8,6 +8,7 @@ class CommaSpliceCLI < Thor class_option :start_line, type: :numeric, default: nil class_option :end_line, type: :numeric, default: nil class_option :debug, type: :boolean, default: false + class_option :separator, type: :string, default: ',' desc 'version', 'print the current comma_splice version' def version @@ -21,7 +22,8 @@ class CommaSpliceCLI < Thor file_corrector = CommaSplice::FileCorrector.new( file_path, start_line: options[:start_line], - end_line: options[:end_line] + end_line: options[:end_line], + separator: options[:separator] ) puts file_corrector.corrected @@ -34,7 +36,8 @@ class CommaSpliceCLI < Thor file_corrector = CommaSplice::FileCorrector.new( file_path, start_line: options[:start_line], - end_line: options[:end_line] + end_line: options[:end_line], + separator: options[:separator] ) file_corrector.save(fix_path) @@ -47,7 +50,8 @@ class CommaSpliceCLI < Thor file_corrector = CommaSplice::FileCorrector.new( file_path, start_line: options[:start_line], - end_line: options[:end_line] + end_line: options[:end_line], + separator: options[:separator] ) puts file_corrector.bad_lines @@ -60,7 +64,8 @@ class CommaSpliceCLI < Thor file_corrector = CommaSplice::FileCorrector.new( file_path, start_line: options[:start_line], - end_line: options[:end_line] + end_line: options[:end_line], + separator: options[:separator] ) puts file_corrector.bad_lines.size diff --git a/lib/comma_splice/file_corrector.rb b/lib/comma_splice/file_corrector.rb index 5dd9bcf..2cbc633 100644 --- a/lib/comma_splice/file_corrector.rb +++ b/lib/comma_splice/file_corrector.rb @@ -1,21 +1,21 @@ module CommaSplice class FileCorrector - attr_reader :file_contents, :csv_content, :start_line, :end_line, :start_column, :end_column + attr_reader :file_contents, :csv_content, :start_line, :end_line, :start_column, :end_column, :separator - def initialize(file_path, start_line: nil, end_line:nil, start_column: nil, end_column: nil) + def initialize(file_path, start_line: nil, end_line: nil, start_column: nil, end_column: nil, separator: ',') @file_path = file_path @file_contents = File.read(file_path, encoding: 'utf-8') - @content_finder = ContentFinder.new(@file_contents, start_line, end_line) + @content_finder = ContentFinder.new(@file_contents, start_line, end_line, separator) @csv_content = @content_finder.content @start_line = @content_finder.start_line @end_line = @content_finder.end_line - + @separator = @content_finder.separator if start_column && end_column @start_column = start_column @end_column = end_column else - finder = VariableColumnFinder.new(@csv_content[0], @csv_content[1..-1]) + finder = VariableColumnFinder.new(@csv_content[0], @csv_content[1..-1], @separator) @start_column = finder.start_column @end_column = finder.end_column end @@ -24,7 +24,7 @@ def initialize(file_path, start_line: nil, end_line:nil, start_column: nil, end_ end def header - @header ||= Line.new(csv_content.first) + @header ||= Line.new(csv_content.first, separator) end def bad_lines @@ -60,7 +60,7 @@ def save(path) end end - def to_json + def to_json(*_args) @content_finder.parsed.try(:to_json) end @@ -68,12 +68,15 @@ def to_json def line_correctors @line_correctors ||= csv_content.collect do |line| - LineCorrector.new(header, Line.new(line), @start_column, @end_column) + LineCorrector.new(header, Line.new(line, separator), @start_column, @end_column, separator) end end def corrected_lines line_correctors.collect do |line| + # if line.original.include?('http://www.affaritaliani.it/economia/tim-fulvio-conti-resta-smentita-ipotesi-di-passaggio-a-open-fiber-597756.html') + # puts line + # end if line.needs_correcting? line.corrected else diff --git a/lib/comma_splice/helpers/comma_calculator.rb b/lib/comma_splice/helpers/comma_calculator.rb index fc4bf24..cd6358a 100644 --- a/lib/comma_splice/helpers/comma_calculator.rb +++ b/lib/comma_splice/helpers/comma_calculator.rb @@ -8,6 +8,7 @@ class CommaCalculator def initialize(headers, values) raise StandardError, "Determining all the possibilities to fit #{values.size} values into the #{headers.size} headers #{headers.inspect} is computationally expensive. Please specify the columns where commas might be." if headers.size > 10 && values.size > 10 + @separator = separator @headers = headers @values = values @longest_header = @headers.max_by(&:length) @@ -66,6 +67,10 @@ def print_all_options ranked_options.each_with_index do |option, index| print_option(option, index) end + private + + def quoted_values(values) + "\"#{values.join(@separator).gsub(/(? 2 + Line.new(line, separator).values.size > 2 end relative_end_line = @file_contents.lines[@start_line..-1].find_index do |line| - Line.new(line).values.size < 2 + Line.new(line, separator).values.size < 2 end @end_line = if relative_end_line @@ -40,7 +40,7 @@ def find_content def parsed quote_chars = %w[" | ~ ^ & *] begin - CSV.parse(@content.join("\n"), quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true) + CSV.parse(@content.join('\n'), col_sep: separator, quote_char: quote_chars.shift, headers: :first_row, liberal_parsing: true) rescue CSV::MalformedCSVError quote_chars.empty? ? raise : retry end diff --git a/lib/comma_splice/helpers/line.rb b/lib/comma_splice/helpers/line.rb index 123ceaf..549d0ba 100644 --- a/lib/comma_splice/helpers/line.rb +++ b/lib/comma_splice/helpers/line.rb @@ -1,9 +1,10 @@ module CommaSplice class Line - attr_reader :values, :line + attr_reader :values, :line, :separator - def initialize(line) + def initialize(line, separator = ',') @line = line + @separator = separator @values = parse_csv_content(line).first end @@ -12,7 +13,7 @@ def initialize(line) def parse_csv_content(content, headers = false) quote_chars = %w[" | ~ ^ & *] begin - CSV.parse(content.mb_chars.tidy_bytes.to_s, quote_char: quote_chars.shift, headers: headers, liberal_parsing: true) + CSV.parse(content.mb_chars.tidy_bytes.to_s, col_sep: separator, quote_char: quote_chars.shift, headers: headers, liberal_parsing: true) rescue CSV::MalformedCSVError quote_chars.empty? ? raise : retry end diff --git a/lib/comma_splice/helpers/variable_column_finder.rb b/lib/comma_splice/helpers/variable_column_finder.rb index 860d725..3b7e761 100644 --- a/lib/comma_splice/helpers/variable_column_finder.rb +++ b/lib/comma_splice/helpers/variable_column_finder.rb @@ -18,12 +18,12 @@ module CommaSplice # 17385094,,,01-27-2019 @ 13:47:00,KIng Tubby Meets The Upsetter,King And The Upsetter At Spanish Town,KIng Tubby Meets The Upsetter,Celluloid,post,live,y, class VariableColumnFinder - attr_reader :start_column, :end_column + attr_reader :start_column, :end_column, :separator - def initialize(header_line, value_lines) + def initialize(header_line, value_lines, separator = ',') @values = value_lines @header = header_line - + @separator = separator find_variable_column_boundaries end @@ -44,9 +44,9 @@ def find_variable_column_boundaries def left_to_right_index left_to_right_index = [] - @header.split(',').size.times do |time| + @header.split(separator).size.times do |time| left_to_right_index.push(@values.map do |value_line| - value_line.split(',')[time].to_s.size + value_line.split(separator)[time].to_s.size end.uniq.size == 1) end @@ -55,9 +55,9 @@ def left_to_right_index def right_to_left_index right_to_left_index = [] - @header.split(',').size.times do |time| + @header.split(separator).size.times do |time| right_to_left_index.unshift(@values.map do |value_line| - value_line.split(',')[-time].to_s.size + value_line.split(separator)[-time].to_s.size end.uniq.size == 1) end diff --git a/lib/comma_splice/line_corrector.rb b/lib/comma_splice/line_corrector.rb index 41673ee..c920817 100644 --- a/lib/comma_splice/line_corrector.rb +++ b/lib/comma_splice/line_corrector.rb @@ -1,10 +1,10 @@ module CommaSplice class LineCorrector - attr_reader :headers, :values, :header_line, :value_line, :right_bounds, :left_bounds + attr_reader :headers, :values, :header_line, :value_line, :right_bounds, :left_bounds, :separator - def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1) - header_line = Line.new(header_line) unless header_line.is_a?(Line) - value_line = Line.new(value_line) unless value_line.is_a?(Line) + def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1, separator = ',') + header_line = Line.new(header_line, separator) unless header_line.is_a?(Line) + value_line = Line.new(value_line, separator) unless value_line.is_a?(Line) @header_line = header_line @value_line = value_line @@ -12,6 +12,7 @@ def initialize(header_line, value_line, left_bounds = 0, right_bounds = -1) @values = value_line.values @left_bounds = left_bounds @right_bounds = right_bounds + @separator = separator raise 'right bounds must be negative' unless right_bounds.negative? raise 'left bounds must be not be negative' if left_bounds.negative? @@ -59,11 +60,11 @@ def print_all_options protected def corrector - CommaCalculator.new(selected_headers, selected_values) + CommaCalculator.new(selected_headers, selected_values, separator) end def generate_csv_line(values) - CSV.generate_line(values) + CSV.generate_line(values, col_sep: @separator) end def selected_headers diff --git a/spec/basic_spec.rb b/spec/basic_spec.rb index 25cf117..a57634b 100644 --- a/spec/basic_spec.rb +++ b/spec/basic_spec.rb @@ -2,16 +2,168 @@ require 'spec_helper' describe CommaSplice do - context 'unescaped-commas-and-non-header' do - subject do - CommaSplice::FileCorrector.new(test_csv_path('unescaped-commas-and-non-csv-header.csv')) + context 'with no delimiter specified' do + context 'unescaped-commas-and-non-header' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-commas-and-non-csv-header.csv')) + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-commas-and-non-csv-header-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end end - it 'should make proper corrections' do - fixed_contents = read_test_csv('unescaped-commas-and-non-csv-header-fixed.csv') - subject.save('test-file.txt') + context 'unescaped-commas' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-commas.csv')) + end - expect(File.read('test-file.txt')).to eq(fixed_contents) + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-commas-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unescaped-quotes' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-quotes.csv')) + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-quotes-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unknown combo should prompt for option' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('10000-maniacs.csv')) + end + + it 'should prompt for correction' do + expect($stdin).to receive(:gets).and_return('4') + + fixed_contents = read_test_csv('10000-maniacs-fixed.csv') + subject.save('test-file.txt') + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + end + + context 'with comma as delimiter' do + context 'unescaped-commas-and-non-header' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-commas-and-non-csv-header.csv'), separator: ',') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-commas-and-non-csv-header-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unescaped-commas' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-commas.csv'), separator: ',') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-commas-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unescaped-quotes' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-quotes.csv'), separator: ',') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-quotes-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unknown combo should prompt for option' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('10000-maniacs.csv'), separator: ',') + end + + it 'should prompt for correction' do + expect($stdin).to receive(:gets).and_return('4') + + fixed_contents = read_test_csv('10000-maniacs-fixed.csv') + subject.save('test-file.txt') + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + end + + context 'with colon as delimiter' do + context 'unescaped-commas-and-non-header' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-colons-and-non-csv-header.csv'), separator: ';') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-colons-and-non-csv-header-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unescaped-commas' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-colons.csv'), separator: ';') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-colons-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unescaped-quotes' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('unescaped-quotes-colons.csv'), separator: ';') + end + + it 'should make proper corrections' do + fixed_contents = read_test_csv('unescaped-quotes-colons-fixed.csv') + subject.save('test-file.txt') + + expect(File.read('test-file.txt')).to eq(fixed_contents) + end + end + + context 'unknown combo should prompt for option' do + subject do + CommaSplice::FileCorrector.new(test_csv_path('10000-maniacs-colons.csv'), separator: ';') + end + + it 'should prompt for correction' do + expect($stdin).to receive(:gets).and_return('4') + + fixed_contents = read_test_csv('10000-maniacs-colons-fixed.csv') + subject.save('test-file.txt') + expect(File.read('test-file.txt')).to eq(fixed_contents) + end end end @@ -56,21 +208,21 @@ context 'equal columns' do it 'should throw error' do - expect { + expect do CommaSplice::FileCorrector.new(test_csv_path('equal-columns.csv')).corrected - }.to raise_error StandardError + end.to raise_error StandardError end it 'should not throw error when columns are supplied' do - expect { + expect do CommaSplice::FileCorrector.new(test_csv_path('equal-columns.csv'), start_column: 4, end_column: -5).corrected - }.to_not raise_error + end.to_not raise_error end end context 'specified content' do subject do - CommaSplice::FileCorrector.new(test_csv_path('find-content.csv'), start_line: 15, end_line:-1) + CommaSplice::FileCorrector.new(test_csv_path('find-content.csv'), start_line: 15, end_line: -1) end it 'finds the csv content bounds' do diff --git a/spec/content_finder_spec.rb b/spec/content_finder_spec.rb index 934a061..8075acf 100644 --- a/spec/content_finder_spec.rb +++ b/spec/content_finder_spec.rb @@ -2,15 +2,29 @@ require 'spec_helper' describe CommaSplice::ContentFinder do - describe 'non-csv header' do - subject do - file = read_test_csv('unescaped-commas-and-non-csv-header.csv') - CommaSplice::ContentFinder.new(file) + context 'with no delimiter specified' do + describe 'non-csv header' do + subject do + file = read_test_csv('unescaped-commas-and-non-csv-header.csv') + CommaSplice::ContentFinder.new(file) + end + + it 'finds the csv content bounds' do + expect(subject.start_line).to eq(15) + expect(subject.end_line).to eq(-1) + end end - it 'finds the csv content bounds' do - expect(subject.start_line).to eq(15) - expect(subject.end_line).to eq(-1) + describe 'no header' do + subject do + file = read_test_csv('unescaped-commas.csv') + CommaSplice::ContentFinder.new(file) + end + + it 'finds the csv content bounds' do + expect(subject.start_line).to eq(0) + expect(subject.end_line).to eq(-1) + end end end @@ -29,7 +43,7 @@ describe 'no header' do subject do file = read_test_csv('unescaped-commas.csv') - CommaSplice::ContentFinder.new(file) + CommaSplice::ContentFinder.new(file, nil, nil, ',') end it 'finds the csv content bounds' do @@ -38,6 +52,20 @@ end end + context 'with comma as delimiter' do + describe 'non-csv header' do + subject do + file = read_test_csv('unescaped-commas-and-non-csv-header.csv') + CommaSplice::ContentFinder.new(file, nil, nil, ',') + end + + it 'finds the csv content bounds' do + expect(subject.start_line).to eq(15) + expect(subject.end_line).to eq(-1) + end + end + end + describe 'short content' do subject do file = read_test_csv('equal-columns.csv') @@ -50,4 +78,29 @@ end end + context 'with colon as delimiter' do + describe 'non-csv header' do + subject do + file = read_test_csv('unescaped-colons-and-non-csv-header.csv') + CommaSplice::ContentFinder.new(file, nil, nil, ';') + end + + it 'finds the csv content bounds' do + expect(subject.start_line).to eq(15) + expect(subject.end_line).to eq(-1) + end + end + + describe 'no header' do + subject do + file = read_test_csv('unescaped-colons.csv') + CommaSplice::ContentFinder.new(file, nil, nil, ';') + end + + it 'finds the csv content bounds' do + expect(subject.start_line).to eq(0) + expect(subject.end_line).to eq(-1) + end + end + end end diff --git a/spec/test_csvs/10000-maniacs-colons-fixed.csv b/spec/test_csvs/10000-maniacs-colons-fixed.csv new file mode 100644 index 0000000..8fe15c5 --- /dev/null +++ b/spec/test_csvs/10000-maniacs-colons-fixed.csv @@ -0,0 +1,29 @@ + +Playlist download for: + +Station: KOOP +Show date: 12-09-2017 + +Show name: Graveside Service +Show DJ(s): +Rush Mr Harris Tim Chris + +Number of entries: 13 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +16851097;;;12-09-2017 @ 09:57:00;"10;000 Maniacs and Michael Stipe";To Sir with Love;Campfire Songs;Rhino;post;live;y; +16851096;;;12-09-2017 @ 09:44:00;Fran Jeffries;Mine Eyes;Fran Can Really Hang You Up the Most;Warwick;post;live;y; +16851095;;;12-09-2017 @ 09:39:00;Fran Jeffries;Sex and the Single Girl;Fran Can Really Hang You Up the Most;Warwick;post;live;y; +16851094;;;12-09-2017 @ 09:35:00;Stereolab;Op Hop Detonation;Cobra and Phases Group PLay Voltage in the Milky Night;Electra;post;live;y; +16851093;;;12-09-2017 @ 09:30:00;Stereolab;Spiracles;Cobra and Phases Group PLay Voltage in the Milky Night;Electra;post;live;y; +16851092;;;12-09-2017 @ 09:27:00;Gerry & The Pacemakers;I LIke It;Gerrry Cross the Mersey;Razor and Ties;post;live;y; +16851091;;;12-09-2017 @ 09:24:00;Gerry & The Pacemakers;You're the reason;Gerrry Cross the Mersey;Razor and Ties;post;live;y; +16851090;;;12-09-2017 @ 09:21:00;The Waitresses;Christmas Wrapping;Best of;Polydor;post;live;y; +16851089;;;12-09-2017 @ 09:17:00;The Waitresses;I Know What Boys Like;Best of;Polydor;post;live;y; +16851088;;;12-09-2017 @ 09:13:00;The Waitresses;Wasn't Tomorrow Wonderful?;Best of;Polydor;post;live;y; +16851087;;;12-09-2017 @ 09:09:00;Flipper;Love Canal;Blowin' Chunks;Def American;post;live;y; +16851086;;;12-09-2017 @ 09:04:00;Esma redzepova & the Ensemble Teodosievsky;Ahsin;Belly Dance music;Essential Media Group;post;live;y; +16851085;;;12-09-2017 @ 09:01:00;Vivian Blaine;Aselaide's Lament;American Songbook;;post;live;y; diff --git a/spec/test_csvs/10000-maniacs-colons.csv b/spec/test_csvs/10000-maniacs-colons.csv new file mode 100644 index 0000000..22daf72 --- /dev/null +++ b/spec/test_csvs/10000-maniacs-colons.csv @@ -0,0 +1,29 @@ + +Playlist download for: + +Station: KOOP +Show date: 12-09-2017 + +Show name: Graveside Service +Show DJ(s): +Rush Mr Harris Tim Chris + +Number of entries: 13 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +16851097;;;12-09-2017 @ 09:57:00;10;000 Maniacs and Michael Stipe;To Sir with Love;Campfire Songs;Rhino;post;live;y; +16851096;;;12-09-2017 @ 09:44:00;Fran Jeffries;Mine Eyes;Fran Can Really Hang You Up the Most;Warwick;post;live;y; +16851095;;;12-09-2017 @ 09:39:00;Fran Jeffries;Sex and the Single Girl;Fran Can Really Hang You Up the Most;Warwick;post;live;y; +16851094;;;12-09-2017 @ 09:35:00;Stereolab;Op Hop Detonation;Cobra and Phases Group PLay Voltage in the Milky Night;Electra;post;live;y; +16851093;;;12-09-2017 @ 09:30:00;Stereolab;Spiracles;Cobra and Phases Group PLay Voltage in the Milky Night;Electra;post;live;y; +16851092;;;12-09-2017 @ 09:27:00;Gerry & The Pacemakers;I LIke It;Gerrry Cross the Mersey;Razor and Ties;post;live;y; +16851091;;;12-09-2017 @ 09:24:00;Gerry & The Pacemakers;You're the reason;Gerrry Cross the Mersey;Razor and Ties;post;live;y; +16851090;;;12-09-2017 @ 09:21:00;The Waitresses;Christmas Wrapping;Best of;Polydor;post;live;y; +16851089;;;12-09-2017 @ 09:17:00;The Waitresses;I Know What Boys Like;Best of;Polydor;post;live;y; +16851088;;;12-09-2017 @ 09:13:00;The Waitresses;Wasn't Tomorrow Wonderful?;Best of;Polydor;post;live;y; +16851087;;;12-09-2017 @ 09:09:00;Flipper;Love Canal;Blowin' Chunks;Def American;post;live;y; +16851086;;;12-09-2017 @ 09:04:00;Esma redzepova & the Ensemble Teodosievsky;Ahsin;Belly Dance music;Essential Media Group;post;live;y; +16851085;;;12-09-2017 @ 09:01:00;Vivian Blaine;Aselaide's Lament;American Songbook;;post;live;y; diff --git a/spec/test_csvs/2015-07-17-colons.txt b/spec/test_csvs/2015-07-17-colons.txt new file mode 100644 index 0000000..89a0212 --- /dev/null +++ b/spec/test_csvs/2015-07-17-colons.txt @@ -0,0 +1,26 @@ + +Playlist download for: + +Station: KOOP +Show date: 07-17-2015 + +Show name: Youth Spin +Show DJ(s): +na + +Number of entries: 10 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +14998046;;;07-17-2015 @ 18:59:00;Boris;Just Abandoned Myself;Pink;Southern Lord;;live;y; +14998007;;;07-17-2015 @ 18:42:00;Bobby Brown;Every Little Step;Don't Be Cruel;MCA;;live;y; +14997998;;;07-17-2015 @ 18:37:00;Mark Morrison;Return of the Mack;Return of the Mack;WEA;;live;y; +14997992;;;07-17-2015 @ 18:35:00;Prince;Rasberry Beret;The Hits/The B-Sides;Paisley Park;;live;y; +14997991;;;07-17-2015 @ 18:34:00;Deftones;U;U;D;D;L;R;L;R;Select;Start;Saturday Night Wrist;Maverick;;live;y; +14997950;;;07-17-2015 @ 18:18:00;Yo Lo Tengo;Let's Save Tony Orlando's House;...And Then Nothing Turned Itself Inside Out;Matador;;live;y; +14997941;;;07-17-2015 @ 18:15:00;Kate Bush;The Big Sky;Hounds of Love;Fish People;;live;y; +14997940;;;07-17-2015 @ 18:10:00;Hall and Oates;You Make My Dreams;The Essential Daryl Hall and John Oates;Arista;;live;y; +14997939;;;07-17-2015 @ 18:05:00;The Verve;Blue;A Storm in Heaven;Hut;;live;y; +14997938;;;07-17-2015 @ 18:02:00;REM;Harbor Code;Reckoning;IRS;;live;y; diff --git a/spec/test_csvs/test-colons.csv b/spec/test_csvs/test-colons.csv new file mode 100644 index 0000000..1d8037c --- /dev/null +++ b/spec/test_csvs/test-colons.csv @@ -0,0 +1,3 @@ +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17385060;;;01-27-2019 @ 12:34:00;"Lester Sterling; Lynn Taitt & The Jets";Check Point Charlie;Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968;Dub Store;post;live;y; +17385059;;;01-27-2019 @ 12:31:00;Lester Sterling;Lester Sterling Special;Merritone Rock Steady 2: This Music Got Soul 1966-1967;Dub Store;post;live;y; diff --git a/spec/test_csvs/unescaped-colons-and-non-csv-header-fixed.csv b/spec/test_csvs/unescaped-colons-and-non-csv-header-fixed.csv new file mode 100644 index 0000000..c1ebcb9 --- /dev/null +++ b/spec/test_csvs/unescaped-colons-and-non-csv-header-fixed.csv @@ -0,0 +1,55 @@ + +Playlist download for: + +Station: KOOP +Show date: 01-27-2019 + +Show name: Jamaican Gold +Show DJ(s): +Art Baker Rush DJ Jeff DJ Lippy Sean Mason Erik DJ Dale DJ Dangit + +Number of entries: 39 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17385098;;;01-27-2019 @ 13:58:00;Niney & Soul Syndicate;So Long Dub;Dub Box Set Vol. 2;Trojan;post;live;y; +17385097;;;01-27-2019 @ 13:57:00;King Tubby;Love Thy Neighbor;Jesus Dread;Blood & Fire;post;live;y; +17385096;;;01-27-2019 @ 13:53:00;King Tubby / The Aggrovators;Declaration Of Dub;Dub From The Roots;Charly;post;live;y; +17385095;;;01-27-2019 @ 13:50:00;Harry Mudie / King Tubby;Dub With A Difference;In Dub Conference Vol. 1;Moodisc;post;live;y; +17385094;;;01-27-2019 @ 13:47:00;KIng Tubby Meets The Upsetter;King And The Upsetter At Spanish Town;KIng Tubby Meets The Upsetter;Celluloid;post;live;y; +17385093;;;01-27-2019 @ 13:44:00;King Tubby & The Aggrovators;Hideaway;Dubbing In The Backyard;Pressure Sounds;post;live;y; +17385092;;;01-27-2019 @ 13:42:00;King Tubby;Tubby Get Smart;Dub Like Dirt;Blood & Fire;post;live;y; +17385091;;;01-27-2019 @ 13:36:00;Yabby You Meets Trinity;Di Gun Gun Fever;At (King Tubby's) Dub Station;Prophet;post;live;y; +17385090;;;01-27-2019 @ 13:32:00;Glen Brown / King Tubby;Universal Dub;Termination Dub;Blood & Fire;post;live;y; +17385089;;;01-27-2019 @ 13:28:00;Glen Brown;Daughter A Whole Lot Of Sugar Down Deh!;Glen Brown: Dubble Attack - The Orginal Dee-Jay Intrumental Collection 1972-74;VP;post;live;y; +17385088;;;01-27-2019 @ 13:22:00;Sylford Walker;My Father's Homeland;Lamb's Bread;Shanachie;post;live;y; +17385087;;;01-27-2019 @ 13:18:00;Sylford Walker;Jah Golden Pen;Reggae Anthology: Joe Gibbs - Scorchers From The Mighty Two;VP;post;live;y; +17385086;;;01-27-2019 @ 13:15:00;Leroy Sibbles;What Is Mine Version;Strictly Roots;Micron;post;live;y; +17385085;;;01-27-2019 @ 13:11:00;The Heptones;Mistry Babylon;The Trojan: Roots Collection;Trojan;post;live;y; +17385083;;;01-27-2019 @ 13:04:00;The Heptones;Revolution;Packin House;Blood & Fire;post;live;y; +17385082;;;01-27-2019 @ 13:01:00;Wanachi;Black Root;Reggae Goodies Vol. 1 & 2;Wackie's;post;live;y; +17385069;;;01-27-2019 @ 13:00:00;Skatalites;Latin Goes Ska;Latin Goes Ska;Jet Set;post;live;y; +17385079;;;01-27-2019 @ 13:00:00;Wane Wade;Lord Of Lords;Jesus Dread;Blood & Fire;post;live;y; +17385080;;;01-27-2019 @ 13:00:00;Earl Morgan;Jamaican Hammer Horror;D.C. Dub Connection;Tele-Tech;post;live;y; +17385081;;;01-27-2019 @ 13:00:00;Mad Professor;Listen To Your Radio;Unreleased;Mossburg;post;live;y; +17385084;;;01-27-2019 @ 13:00:00;Leroy Sibbles;Total Destruction;Packin House;Blood & Fire;post;live;y; +17385068;;;01-27-2019 @ 12:58:00;Ernest Ranglin;Flamingo;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385067;;;01-27-2019 @ 12:56:00;Stranger Cole;Last Flight To Reggae City;Bangarang!;Trojan;post;live;y; +17385066;;;01-27-2019 @ 12:53:00;Lester Sterling;Forest Gate Rock (Barbados);Essential Rocksteady Vol 2 Platinum Edition;Culture Town;post;live;y; +17385065;;;01-27-2019 @ 12:50:00;Stranger Cole & Lester Sterling;Bangarang;Unearthed Gold of Rocksteady Vol. 2;ERNI;post;live;y; +17385064;;;01-27-2019 @ 12:46:00;Ernest Ranglin;Summertime;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385063;;;01-27-2019 @ 12:43:00;The Uniques;Trying To Find A Home;Absolutely Rocksteady;Pressure Sounds;post;live;y; +17385062;;;01-27-2019 @ 12:41:00;Stranger Cole;We Shall Overcome;Bangarang!;Trojan;post;live;y; +17385061;;;01-27-2019 @ 12:37:00;Stranger Cole;Down The Trainline;Bangarang!;Trojan;post;live;y; +17385060;;;01-27-2019 @ 12:34:00;"Lester Sterling; Lynn Taitt & The Jets";Check Point Charlie;Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968;Dub Store;post;live;y; +17385059;;;01-27-2019 @ 12:31:00;Lester Sterling;Lester Sterling Special;Merritone Rock Steady 2: This Music Got Soul 1966-1967;Dub Store;post;live;y; +17385058;;;01-27-2019 @ 12:28:00;Roland Alphonso;You Can Depend On Me;Latin Goes Ska;Jet Set;post;live;y; +17385057;;;01-27-2019 @ 12:21:00;Skatalites;James Bond;Ball Of Fire;Island;post;live;y; +17385056;;;01-27-2019 @ 12:14:00;Skatalites;Swing Easy;Ball Of Fire;Island;post;live;y; +17385055;;;01-27-2019 @ 12:12:00;SKA-J;The Pink Panther;Ska-J Plays Standars;SKA-J;post;live;y; +17385054;;;01-27-2019 @ 12:08:00;Skatalites;Hanging Tree;Foundation Ska;Heartbeat;post;live;y; +17385053;;;01-27-2019 @ 12:06:00;Lester Stering;Man About Town;Man About Ska Town;King Edwards;post;live;y; +17385052;;;01-27-2019 @ 12:03:00;"Lester Sterling; Johnny Moore; & Tommy McCook";Settle Down;Ska Bonanza;Heartbeat;post;live;y; +17385044;;;01-27-2019 @ 12:00:00;Raymond Harper;Safari;Gaz Mayall's Top Ska Tunes;Trojan;post;live;y; diff --git a/spec/test_csvs/unescaped-colons-and-non-csv-header.csv b/spec/test_csvs/unescaped-colons-and-non-csv-header.csv new file mode 100644 index 0000000..b43b8c1 --- /dev/null +++ b/spec/test_csvs/unescaped-colons-and-non-csv-header.csv @@ -0,0 +1,55 @@ + +Playlist download for: + +Station: KOOP +Show date: 01-27-2019 + +Show name: Jamaican Gold +Show DJ(s): +Art Baker Rush DJ Jeff DJ Lippy Sean Mason Erik DJ Dale DJ Dangit + +Number of entries: 39 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17385098;;;01-27-2019 @ 13:58:00;Niney & Soul Syndicate;So Long Dub;Dub Box Set Vol. 2;Trojan;post;live;y; +17385097;;;01-27-2019 @ 13:57:00;King Tubby;Love Thy Neighbor;Jesus Dread;Blood & Fire;post;live;y; +17385096;;;01-27-2019 @ 13:53:00;King Tubby / The Aggrovators;Declaration Of Dub;Dub From The Roots;Charly;post;live;y; +17385095;;;01-27-2019 @ 13:50:00;Harry Mudie / King Tubby;Dub With A Difference;In Dub Conference Vol. 1;Moodisc;post;live;y; +17385094;;;01-27-2019 @ 13:47:00;KIng Tubby Meets The Upsetter;King And The Upsetter At Spanish Town;KIng Tubby Meets The Upsetter;Celluloid;post;live;y; +17385093;;;01-27-2019 @ 13:44:00;King Tubby & The Aggrovators;Hideaway;Dubbing In The Backyard;Pressure Sounds;post;live;y; +17385092;;;01-27-2019 @ 13:42:00;King Tubby;Tubby Get Smart;Dub Like Dirt;Blood & Fire;post;live;y; +17385091;;;01-27-2019 @ 13:36:00;Yabby You Meets Trinity;Di Gun Gun Fever;At (King Tubby's) Dub Station;Prophet;post;live;y; +17385090;;;01-27-2019 @ 13:32:00;Glen Brown / King Tubby;Universal Dub;Termination Dub;Blood & Fire;post;live;y; +17385089;;;01-27-2019 @ 13:28:00;Glen Brown;Daughter A Whole Lot Of Sugar Down Deh!;Glen Brown: Dubble Attack - The Orginal Dee-Jay Intrumental Collection 1972-74;VP;post;live;y; +17385088;;;01-27-2019 @ 13:22:00;Sylford Walker;My Father's Homeland;Lamb's Bread;Shanachie;post;live;y; +17385087;;;01-27-2019 @ 13:18:00;Sylford Walker;Jah Golden Pen;Reggae Anthology: Joe Gibbs - Scorchers From The Mighty Two;VP;post;live;y; +17385086;;;01-27-2019 @ 13:15:00;Leroy Sibbles;What Is Mine Version;Strictly Roots;Micron;post;live;y; +17385085;;;01-27-2019 @ 13:11:00;The Heptones;Mistry Babylon;The Trojan: Roots Collection;Trojan;post;live;y; +17385083;;;01-27-2019 @ 13:04:00;The Heptones;Revolution;Packin House;Blood & Fire;post;live;y; +17385082;;;01-27-2019 @ 13:01:00;Wanachi;Black Root;Reggae Goodies Vol. 1 & 2;Wackie's;post;live;y; +17385069;;;01-27-2019 @ 13:00:00;Skatalites;Latin Goes Ska;Latin Goes Ska;Jet Set;post;live;y; +17385079;;;01-27-2019 @ 13:00:00;Wane Wade;Lord Of Lords;Jesus Dread;Blood & Fire;post;live;y; +17385080;;;01-27-2019 @ 13:00:00;Earl Morgan;Jamaican Hammer Horror;D.C. Dub Connection;Tele-Tech;post;live;y; +17385081;;;01-27-2019 @ 13:00:00;Mad Professor;Listen To Your Radio;Unreleased;Mossburg;post;live;y; +17385084;;;01-27-2019 @ 13:00:00;Leroy Sibbles;Total Destruction;Packin House;Blood & Fire;post;live;y; +17385068;;;01-27-2019 @ 12:58:00;Ernest Ranglin;Flamingo;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385067;;;01-27-2019 @ 12:56:00;Stranger Cole;Last Flight To Reggae City;Bangarang!;Trojan;post;live;y; +17385066;;;01-27-2019 @ 12:53:00;Lester Sterling;Forest Gate Rock (Barbados);Essential Rocksteady Vol 2 Platinum Edition;Culture Town;post;live;y; +17385065;;;01-27-2019 @ 12:50:00;Stranger Cole & Lester Sterling;Bangarang;Unearthed Gold of Rocksteady Vol. 2;ERNI;post;live;y; +17385064;;;01-27-2019 @ 12:46:00;Ernest Ranglin;Summertime;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385063;;;01-27-2019 @ 12:43:00;The Uniques;Trying To Find A Home;Absolutely Rocksteady;Pressure Sounds;post;live;y; +17385062;;;01-27-2019 @ 12:41:00;Stranger Cole;We Shall Overcome;Bangarang!;Trojan;post;live;y; +17385061;;;01-27-2019 @ 12:37:00;Stranger Cole;Down The Trainline;Bangarang!;Trojan;post;live;y; +17385060;;;01-27-2019 @ 12:34:00;Lester Sterling; Lynn Taitt & The Jets;Check Point Charlie;Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968;Dub Store;post;live;y; +17385059;;;01-27-2019 @ 12:31:00;Lester Sterling;Lester Sterling Special;Merritone Rock Steady 2: This Music Got Soul 1966-1967;Dub Store;post;live;y; +17385058;;;01-27-2019 @ 12:28:00;Roland Alphonso;You Can Depend On Me;Latin Goes Ska;Jet Set;post;live;y; +17385057;;;01-27-2019 @ 12:21:00;Skatalites;James Bond;Ball Of Fire;Island;post;live;y; +17385056;;;01-27-2019 @ 12:14:00;Skatalites;Swing Easy;Ball Of Fire;Island;post;live;y; +17385055;;;01-27-2019 @ 12:12:00;SKA-J;The Pink Panther;Ska-J Plays Standars;SKA-J;post;live;y; +17385054;;;01-27-2019 @ 12:08:00;Skatalites;Hanging Tree;Foundation Ska;Heartbeat;post;live;y; +17385053;;;01-27-2019 @ 12:06:00;Lester Stering;Man About Town;Man About Ska Town;King Edwards;post;live;y; +17385052;;;01-27-2019 @ 12:03:00;Lester Sterling; Johnny Moore; & Tommy McCook;Settle Down;Ska Bonanza;Heartbeat;post;live;y; +17385044;;;01-27-2019 @ 12:00:00;Raymond Harper;Safari;Gaz Mayall's Top Ska Tunes;Trojan;post;live;y; diff --git a/spec/test_csvs/unescaped-colons-fixed.csv b/spec/test_csvs/unescaped-colons-fixed.csv new file mode 100644 index 0000000..3e7b539 --- /dev/null +++ b/spec/test_csvs/unescaped-colons-fixed.csv @@ -0,0 +1,40 @@ +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17385098;;;01-27-2019 @ 13:58:00;Niney & Soul Syndicate;So Long Dub;Dub Box Set Vol. 2;Trojan;post;live;y; +17385097;;;01-27-2019 @ 13:57:00;King Tubby;Love Thy Neighbor;Jesus Dread;Blood & Fire;post;live;y; +17385096;;;01-27-2019 @ 13:53:00;King Tubby / The Aggrovators;Declaration Of Dub;Dub From The Roots;Charly;post;live;y; +17385095;;;01-27-2019 @ 13:50:00;Harry Mudie / King Tubby;Dub With A Difference;In Dub Conference Vol. 1;Moodisc;post;live;y; +17385094;;;01-27-2019 @ 13:47:00;KIng Tubby Meets The Upsetter;King And The Upsetter At Spanish Town;KIng Tubby Meets The Upsetter;Celluloid;post;live;y; +17385093;;;01-27-2019 @ 13:44:00;King Tubby & The Aggrovators;Hideaway;Dubbing In The Backyard;Pressure Sounds;post;live;y; +17385092;;;01-27-2019 @ 13:42:00;King Tubby;Tubby Get Smart;Dub Like Dirt;Blood & Fire;post;live;y; +17385091;;;01-27-2019 @ 13:36:00;Yabby You Meets Trinity;Di Gun Gun Fever;At (King Tubby's) Dub Station;Prophet;post;live;y; +17385090;;;01-27-2019 @ 13:32:00;Glen Brown / King Tubby;Universal Dub;Termination Dub;Blood & Fire;post;live;y; +17385089;;;01-27-2019 @ 13:28:00;Glen Brown;Daughter A Whole Lot Of Sugar Down Deh!;Glen Brown: Dubble Attack - The Orginal Dee-Jay Intrumental Collection 1972-74;VP;post;live;y; +17385088;;;01-27-2019 @ 13:22:00;Sylford Walker;My Father's Homeland;Lamb's Bread;Shanachie;post;live;y; +17385087;;;01-27-2019 @ 13:18:00;Sylford Walker;Jah Golden Pen;Reggae Anthology: Joe Gibbs - Scorchers From The Mighty Two;VP;post;live;y; +17385086;;;01-27-2019 @ 13:15:00;Leroy Sibbles;What Is Mine Version;Strictly Roots;Micron;post;live;y; +17385085;;;01-27-2019 @ 13:11:00;The Heptones;Mistry Babylon;The Trojan: Roots Collection;Trojan;post;live;y; +17385083;;;01-27-2019 @ 13:04:00;The Heptones;Revolution;Packin House;Blood & Fire;post;live;y; +17385082;;;01-27-2019 @ 13:01:00;Wanachi;Black Root;Reggae Goodies Vol. 1 & 2;Wackie's;post;live;y; +17385069;;;01-27-2019 @ 13:00:00;Skatalites;Latin Goes Ska;Latin Goes Ska;Jet Set;post;live;y; +17385079;;;01-27-2019 @ 13:00:00;Wane Wade;Lord Of Lords;Jesus Dread;Blood & Fire;post;live;y; +17385080;;;01-27-2019 @ 13:00:00;Earl Morgan;Jamaican Hammer Horror;D.C. Dub Connection;Tele-Tech;post;live;y; +17385081;;;01-27-2019 @ 13:00:00;Mad Professor;Listen To Your Radio;Unreleased;Mossburg;post;live;y; +17385084;;;01-27-2019 @ 13:00:00;Leroy Sibbles;Total Destruction;Packin House;Blood & Fire;post;live;y; +17385068;;;01-27-2019 @ 12:58:00;Ernest Ranglin;Flamingo;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385067;;;01-27-2019 @ 12:56:00;Stranger Cole;Last Flight To Reggae City;Bangarang!;Trojan;post;live;y; +17385066;;;01-27-2019 @ 12:53:00;Lester Sterling;Forest Gate Rock (Barbados);Essential Rocksteady Vol 2 Platinum Edition;Culture Town;post;live;y; +17385065;;;01-27-2019 @ 12:50:00;Stranger Cole & Lester Sterling;Bangarang;Unearthed Gold of Rocksteady Vol. 2;ERNI;post;live;y; +17385064;;;01-27-2019 @ 12:46:00;Ernest Ranglin;Summertime;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385063;;;01-27-2019 @ 12:43:00;The Uniques;Trying To Find A Home;Absolutely Rocksteady;Pressure Sounds;post;live;y; +17385062;;;01-27-2019 @ 12:41:00;Stranger Cole;We Shall Overcome;Bangarang!;Trojan;post;live;y; +17385061;;;01-27-2019 @ 12:37:00;Stranger Cole;Down The Trainline;Bangarang!;Trojan;post;live;y; +17385060;;;01-27-2019 @ 12:34:00;"Lester Sterling; Lynn Taitt & The Jets";Check Point Charlie;Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968;Dub Store;post;live;y; +17385059;;;01-27-2019 @ 12:31:00;Lester Sterling;Lester Sterling Special;Merritone Rock Steady 2: This Music Got Soul 1966-1967;Dub Store;post;live;y; +17385058;;;01-27-2019 @ 12:28:00;Roland Alphonso;You Can Depend On Me;Latin Goes Ska;Jet Set;post;live;y; +17385057;;;01-27-2019 @ 12:21:00;Skatalites;James Bond;Ball Of Fire;Island;post;live;y; +17385056;;;01-27-2019 @ 12:14:00;Skatalites;Swing Easy;Ball Of Fire;Island;post;live;y; +17385055;;;01-27-2019 @ 12:12:00;SKA-J;The Pink Panther;Ska-J Plays Standars;SKA-J;post;live;y; +17385054;;;01-27-2019 @ 12:08:00;Skatalites;Hanging Tree;Foundation Ska;Heartbeat;post;live;y; +17385053;;;01-27-2019 @ 12:06:00;Lester Stering;Man About Town;Man About Ska Town;King Edwards;post;live;y; +17385052;;;01-27-2019 @ 12:03:00;"Lester Sterling; Johnny Moore; & Tommy McCook";Settle Down;Ska Bonanza;Heartbeat;post;live;y; +17385044;;;01-27-2019 @ 12:00:00;Raymond Harper;Safari;Gaz Mayall's Top Ska Tunes;Trojan;post;live;y; diff --git a/spec/test_csvs/unescaped-colons.csv b/spec/test_csvs/unescaped-colons.csv new file mode 100644 index 0000000..af03dcc --- /dev/null +++ b/spec/test_csvs/unescaped-colons.csv @@ -0,0 +1,40 @@ +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17385098;;;01-27-2019 @ 13:58:00;Niney & Soul Syndicate;So Long Dub;Dub Box Set Vol. 2;Trojan;post;live;y; +17385097;;;01-27-2019 @ 13:57:00;King Tubby;Love Thy Neighbor;Jesus Dread;Blood & Fire;post;live;y; +17385096;;;01-27-2019 @ 13:53:00;King Tubby / The Aggrovators;Declaration Of Dub;Dub From The Roots;Charly;post;live;y; +17385095;;;01-27-2019 @ 13:50:00;Harry Mudie / King Tubby;Dub With A Difference;In Dub Conference Vol. 1;Moodisc;post;live;y; +17385094;;;01-27-2019 @ 13:47:00;KIng Tubby Meets The Upsetter;King And The Upsetter At Spanish Town;KIng Tubby Meets The Upsetter;Celluloid;post;live;y; +17385093;;;01-27-2019 @ 13:44:00;King Tubby & The Aggrovators;Hideaway;Dubbing In The Backyard;Pressure Sounds;post;live;y; +17385092;;;01-27-2019 @ 13:42:00;King Tubby;Tubby Get Smart;Dub Like Dirt;Blood & Fire;post;live;y; +17385091;;;01-27-2019 @ 13:36:00;Yabby You Meets Trinity;Di Gun Gun Fever;At (King Tubby's) Dub Station;Prophet;post;live;y; +17385090;;;01-27-2019 @ 13:32:00;Glen Brown / King Tubby;Universal Dub;Termination Dub;Blood & Fire;post;live;y; +17385089;;;01-27-2019 @ 13:28:00;Glen Brown;Daughter A Whole Lot Of Sugar Down Deh!;Glen Brown: Dubble Attack - The Orginal Dee-Jay Intrumental Collection 1972-74;VP;post;live;y; +17385088;;;01-27-2019 @ 13:22:00;Sylford Walker;My Father's Homeland;Lamb's Bread;Shanachie;post;live;y; +17385087;;;01-27-2019 @ 13:18:00;Sylford Walker;Jah Golden Pen;Reggae Anthology: Joe Gibbs - Scorchers From The Mighty Two;VP;post;live;y; +17385086;;;01-27-2019 @ 13:15:00;Leroy Sibbles;What Is Mine Version;Strictly Roots;Micron;post;live;y; +17385085;;;01-27-2019 @ 13:11:00;The Heptones;Mistry Babylon;The Trojan: Roots Collection;Trojan;post;live;y; +17385083;;;01-27-2019 @ 13:04:00;The Heptones;Revolution;Packin House;Blood & Fire;post;live;y; +17385082;;;01-27-2019 @ 13:01:00;Wanachi;Black Root;Reggae Goodies Vol. 1 & 2;Wackie's;post;live;y; +17385069;;;01-27-2019 @ 13:00:00;Skatalites;Latin Goes Ska;Latin Goes Ska;Jet Set;post;live;y; +17385079;;;01-27-2019 @ 13:00:00;Wane Wade;Lord Of Lords;Jesus Dread;Blood & Fire;post;live;y; +17385080;;;01-27-2019 @ 13:00:00;Earl Morgan;Jamaican Hammer Horror;D.C. Dub Connection;Tele-Tech;post;live;y; +17385081;;;01-27-2019 @ 13:00:00;Mad Professor;Listen To Your Radio;Unreleased;Mossburg;post;live;y; +17385084;;;01-27-2019 @ 13:00:00;Leroy Sibbles;Total Destruction;Packin House;Blood & Fire;post;live;y; +17385068;;;01-27-2019 @ 12:58:00;Ernest Ranglin;Flamingo;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385067;;;01-27-2019 @ 12:56:00;Stranger Cole;Last Flight To Reggae City;Bangarang!;Trojan;post;live;y; +17385066;;;01-27-2019 @ 12:53:00;Lester Sterling;Forest Gate Rock (Barbados);Essential Rocksteady Vol 2 Platinum Edition;Culture Town;post;live;y; +17385065;;;01-27-2019 @ 12:50:00;Stranger Cole & Lester Sterling;Bangarang;Unearthed Gold of Rocksteady Vol. 2;ERNI;post;live;y; +17385064;;;01-27-2019 @ 12:46:00;Ernest Ranglin;Summertime;Mr. Ranglin With Soul;Dub Store;post;live;y; +17385063;;;01-27-2019 @ 12:43:00;The Uniques;Trying To Find A Home;Absolutely Rocksteady;Pressure Sounds;post;live;y; +17385062;;;01-27-2019 @ 12:41:00;Stranger Cole;We Shall Overcome;Bangarang!;Trojan;post;live;y; +17385061;;;01-27-2019 @ 12:37:00;Stranger Cole;Down The Trainline;Bangarang!;Trojan;post;live;y; +17385060;;;01-27-2019 @ 12:34:00;Lester Sterling; Lynn Taitt & The Jets;Check Point Charlie;Merritone Rock Steady 3: Bang Bang Rock Steady 1966-1968;Dub Store;post;live;y; +17385059;;;01-27-2019 @ 12:31:00;Lester Sterling;Lester Sterling Special;Merritone Rock Steady 2: This Music Got Soul 1966-1967;Dub Store;post;live;y; +17385058;;;01-27-2019 @ 12:28:00;Roland Alphonso;You Can Depend On Me;Latin Goes Ska;Jet Set;post;live;y; +17385057;;;01-27-2019 @ 12:21:00;Skatalites;James Bond;Ball Of Fire;Island;post;live;y; +17385056;;;01-27-2019 @ 12:14:00;Skatalites;Swing Easy;Ball Of Fire;Island;post;live;y; +17385055;;;01-27-2019 @ 12:12:00;SKA-J;The Pink Panther;Ska-J Plays Standars;SKA-J;post;live;y; +17385054;;;01-27-2019 @ 12:08:00;Skatalites;Hanging Tree;Foundation Ska;Heartbeat;post;live;y; +17385053;;;01-27-2019 @ 12:06:00;Lester Stering;Man About Town;Man About Ska Town;King Edwards;post;live;y; +17385052;;;01-27-2019 @ 12:03:00;Lester Sterling; Johnny Moore; & Tommy McCook;Settle Down;Ska Bonanza;Heartbeat;post;live;y; +17385044;;;01-27-2019 @ 12:00:00;Raymond Harper;Safari;Gaz Mayall's Top Ska Tunes;Trojan;post;live;y; diff --git a/spec/test_csvs/unescaped-quotes-colons-fixed.csv b/spec/test_csvs/unescaped-quotes-colons-fixed.csv new file mode 100644 index 0000000..f745ba3 --- /dev/null +++ b/spec/test_csvs/unescaped-quotes-colons-fixed.csv @@ -0,0 +1,43 @@ + +Playlist download for: + +Station: KOOP +Show date: 03-16-2019 + +Show name: The Lounge Show +Show DJ(s): +Miss Novak Miss Novak Tom Ashburn Steve The Swinger Dennis Campa Rush Tim Jay Robillard DJ Del Sur rocky + +Number of entries: 27 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17494020;Vinyl Record;Lounge;03-16-2019 @ 11:19:00;George Martin;And I Love Her;7" single;United Artists;post;live;y; +17494014;Vinyl Record;Lounge;03-16-2019 @ 11:16:00;Juan Garcia Esquivel;Mah-Na Mah-Na;7" single;Orefon;post;live;y; +17494013;Vinyl Record;Lounge;03-16-2019 @ 11:13:00;Peter Nero;Be-In (Hare Krishna);7" single;Columbia;post;live;y; +17494012;Vinyl Record;Lounge;03-16-2019 @ 11:10:00;Trini Lopez;Unchain My Heart;7" single;Reprise;post;live;y; +17494011;Vinyl Record;Lounge;03-16-2019 @ 11:07:00;Aretha Franklin;It Won't Be Long;7" single;Columbia;post;live;y; +17494010;Vinyl Record;Lounge;03-16-2019 @ 11:04:00;Ramsey Lewis;Do What You Wanna;7" single;Chess;post;live;y; +17494007;Vinyl Record;Lounge;03-16-2019 @ 11:01:00;Rolf Harris;Sun Arise;7" single;Epic;post;live;y; +17494006;Vinyl Record;Lounge;03-16-2019 @ 10:58:00;John McFarland Sextet;The Chimp & The Bumble Bee;7" single;United Artists;post;live;y; +17494005;Compact Disc;Lounge;03-16-2019 @ 10:55:00;Herb Alpert And The Tijuana Brass;Mexican Shuffle;Greatest Hits;Shout;post;live;y; +17494004;Vinyl Record;Lounge;03-16-2019 @ 10:53:00;Katie Lee;Real Sick Sounds;Songs Of Couch & Consultation;Reprise;post;live;y; +17494003;Vinyl Record;Lounge;03-16-2019 @ 10:50:00;Henry Mancini Orch.;Experiment In Terror;7" single;RCA Victor;post;live;y; +17494000;Vinyl Record;Lounge;03-16-2019 @ 10:48:00;Nilsson;Everybody's Talkin';7" single;RCA Victor;post;live;y; +17493999;Vinyl Record;Lounge;03-16-2019 @ 10:45:00;Robert Colbert;Missy;7" single;Roulette;post;live;y; +17493998;Vinyl Record;Lounge;03-16-2019 @ 10:42:00;Hugo Montenegro & his Orch.;"Theme from The Good; The Bad; & The Ugly";7" single;RCA Victor;post;live;y; +17493997;Vinyl Record;Lounge;03-16-2019 @ 10:39:00;Ann-Margaret & Lee Hazlewood;Chico;7" single;LHI;post;live;y; +17493996;Vinyl Record;Lounge;03-16-2019 @ 10:36:00;Dean Martin;Come On Down;7" single;Reprise;post;live;y; +17493991;Vinyl Record;Lounge;03-16-2019 @ 10:33:00;Bert Kaempfert;Strangers In The Night;Music From A Man Could Get Killed;Decca;post;live;y; +17493990;Compact Disc;Lounge;03-16-2019 @ 10:29:00;Martin Denny;Hypnotique;Hypnotique;Scamp;post;live;y; +17493989;Compact Disc;Lounge;03-16-2019 @ 10:26:00;Nat King Cole;Nature Boy;His Greatest Hits;Capitol;post;live;y;y +17493988;Vinyl Record;Lounge;03-16-2019 @ 10:22:00;Deana Martin;Bottom Of My Mind;7" single;Reprise;post;live;y; +17493987;Vinyl Record;Lounge;03-16-2019 @ 10:19:00;Bert Kaempfert;Bye Bye Blues;7" single;Decca;post;live;y;y +17493984;Vinyl Record;Lounge;03-16-2019 @ 10:16:00;The Leon Symphoniette with Hibari Girls Choirs;Deserted Home (Kojo-No Tsuki);Musical Memories Of Japan;London International;post;live;y; +17493983;Vinyl Record;Lounge;03-16-2019 @ 10:13:00;George Fischoff Keyboard Komplex;Ping Pong;7" single;Project 3;post;live;y; +17493982;Vinyl Record;Lounge;03-16-2019 @ 10:09:00;Francoise Hardy;Catch A Falling Star;7" single;Pye;post;live;y; +17493981;Vinyl Record;Lounge;03-16-2019 @ 10:07:00;The Puppets;Manhattan Safari;7" single;RCA Victor;post;live;y; +17493980;Vinyl Record;Lounge;03-16-2019 @ 10:04:00;Los Brasilios;Brasilian Beat;Brasil '67;Design;post;live;y; +17493974;Vinyl Record;Lounge;03-16-2019 @ 10:01:00;"Robert Maxwell; His Harp; & Orchestra";That Old Black Magic;7" single;Decca;post;live;y; diff --git a/spec/test_csvs/unescaped-quotes-colons.csv b/spec/test_csvs/unescaped-quotes-colons.csv new file mode 100644 index 0000000..2ef36b0 --- /dev/null +++ b/spec/test_csvs/unescaped-quotes-colons.csv @@ -0,0 +1,43 @@ + +Playlist download for: + +Station: KOOP +Show date: 03-16-2019 + +Show name: The Lounge Show +Show DJ(s): +Miss Novak Miss Novak Tom Ashburn Steve The Swinger Dennis Campa Rush Tim Jay Robillard DJ Del Sur rocky + +Number of entries: 27 + + +Fields are comma delimited; in the following header order: + +playid;playtype;genre;timestamp;artist;title;albumtitle;label;prepost;programtype;iswebcast;isrequest +17494020;Vinyl Record;Lounge;03-16-2019 @ 11:19:00;George Martin;And I Love Her;7" single;United Artists;post;live;y; +17494014;Vinyl Record;Lounge;03-16-2019 @ 11:16:00;Juan Garcia Esquivel;Mah-Na Mah-Na;7" single;Orefon;post;live;y; +17494013;Vinyl Record;Lounge;03-16-2019 @ 11:13:00;Peter Nero;Be-In (Hare Krishna);7" single;Columbia;post;live;y; +17494012;Vinyl Record;Lounge;03-16-2019 @ 11:10:00;Trini Lopez;Unchain My Heart;7" single;Reprise;post;live;y; +17494011;Vinyl Record;Lounge;03-16-2019 @ 11:07:00;Aretha Franklin;It Won't Be Long;7" single;Columbia;post;live;y; +17494010;Vinyl Record;Lounge;03-16-2019 @ 11:04:00;Ramsey Lewis;Do What You Wanna;7" single;Chess;post;live;y; +17494007;Vinyl Record;Lounge;03-16-2019 @ 11:01:00;Rolf Harris;Sun Arise;7" single;Epic;post;live;y; +17494006;Vinyl Record;Lounge;03-16-2019 @ 10:58:00;John McFarland Sextet;The Chimp & The Bumble Bee;7" single;United Artists;post;live;y; +17494005;Compact Disc;Lounge;03-16-2019 @ 10:55:00;Herb Alpert And The Tijuana Brass;Mexican Shuffle;Greatest Hits;Shout;post;live;y; +17494004;Vinyl Record;Lounge;03-16-2019 @ 10:53:00;Katie Lee;Real Sick Sounds;Songs Of Couch & Consultation;Reprise;post;live;y; +17494003;Vinyl Record;Lounge;03-16-2019 @ 10:50:00;Henry Mancini Orch.;Experiment In Terror;7" single;RCA Victor;post;live;y; +17494000;Vinyl Record;Lounge;03-16-2019 @ 10:48:00;Nilsson;Everybody's Talkin';7" single;RCA Victor;post;live;y; +17493999;Vinyl Record;Lounge;03-16-2019 @ 10:45:00;Robert Colbert;Missy;7" single;Roulette;post;live;y; +17493998;Vinyl Record;Lounge;03-16-2019 @ 10:42:00;Hugo Montenegro & his Orch.;Theme from The Good; The Bad; & The Ugly;7" single;RCA Victor;post;live;y; +17493997;Vinyl Record;Lounge;03-16-2019 @ 10:39:00;Ann-Margaret & Lee Hazlewood;Chico;7" single;LHI;post;live;y; +17493996;Vinyl Record;Lounge;03-16-2019 @ 10:36:00;Dean Martin;Come On Down;7" single;Reprise;post;live;y; +17493991;Vinyl Record;Lounge;03-16-2019 @ 10:33:00;Bert Kaempfert;Strangers In The Night;Music From A Man Could Get Killed;Decca;post;live;y; +17493990;Compact Disc;Lounge;03-16-2019 @ 10:29:00;Martin Denny;Hypnotique;Hypnotique;Scamp;post;live;y; +17493989;Compact Disc;Lounge;03-16-2019 @ 10:26:00;Nat King Cole;Nature Boy;His Greatest Hits;Capitol;post;live;y;y +17493988;Vinyl Record;Lounge;03-16-2019 @ 10:22:00;Deana Martin;Bottom Of My Mind;7" single;Reprise;post;live;y; +17493987;Vinyl Record;Lounge;03-16-2019 @ 10:19:00;Bert Kaempfert;Bye Bye Blues;7" single;Decca;post;live;y;y +17493984;Vinyl Record;Lounge;03-16-2019 @ 10:16:00;The Leon Symphoniette with Hibari Girls Choirs;Deserted Home (Kojo-No Tsuki);Musical Memories Of Japan;London International;post;live;y; +17493983;Vinyl Record;Lounge;03-16-2019 @ 10:13:00;George Fischoff Keyboard Komplex;Ping Pong;7" single;Project 3;post;live;y; +17493982;Vinyl Record;Lounge;03-16-2019 @ 10:09:00;Francoise Hardy;Catch A Falling Star;7" single;Pye;post;live;y; +17493981;Vinyl Record;Lounge;03-16-2019 @ 10:07:00;The Puppets;Manhattan Safari;7" single;RCA Victor;post;live;y; +17493980;Vinyl Record;Lounge;03-16-2019 @ 10:04:00;Los Brasilios;Brasilian Beat;Brasil '67;Design;post;live;y; +17493974;Vinyl Record;Lounge;03-16-2019 @ 10:01:00;Robert Maxwell; His Harp; & Orchestra;That Old Black Magic;7" single;Decca;post;live;y; diff --git a/spec/variable_column_finder_spec.rb b/spec/variable_column_finder_spec.rb index c059392..ff03def 100644 --- a/spec/variable_column_finder_spec.rb +++ b/spec/variable_column_finder_spec.rb @@ -2,7 +2,6 @@ require 'spec_helper' describe CommaSplice::VariableColumnFinder do - describe 'unescaped-commas' do subject do file = read_test_csv('unescaped-commas.csv') @@ -15,6 +14,18 @@ end end + describe 'with no delimiter specified' do + subject do + file = read_test_csv('unescaped-commas.csv') + CommaSplice::VariableColumnFinder.new(file.lines[0], file.lines[1..-1]) + end + + it 'should detect correct column bounds' do + expect(subject.start_column).to eq(4) + expect(subject.end_column).to eq(-5) + end + end + describe 'short content' do subject do file = read_test_csv('equal-columns.csv') @@ -26,4 +37,28 @@ expect(subject.end_column).to eq(-1) end end + + describe 'with comma as delimiter' do + subject do + file = read_test_csv('unescaped-commas.csv') + CommaSplice::VariableColumnFinder.new(file.lines[0], file.lines[1..-1], ',') + end + + it 'should detect correct column bounds' do + expect(subject.start_column).to eq(4) + expect(subject.end_column).to eq(-5) + end + end + + describe 'with colon as delimiter' do + subject do + file = read_test_csv('unescaped-colons.csv') + CommaSplice::VariableColumnFinder.new(file.lines[0], file.lines[1..-1], ';') + end + + it 'should detect correct column bounds' do + expect(subject.start_column).to eq(4) + expect(subject.end_column).to eq(-5) + end + end end