Merge pull request #342 from umts/werebus/gtfs-redo

Added new importers for routes, stops, and their join

Gemfile

@@ -5,6 +5,7 @@ ruby file: '.ruby-version'
 
 gem 'bootstrap', '~> 5.3'
 gem 'devise', '~> 4.9'
+gem 'gtfs'
 gem 'haml-rails'
 gem 'importmap-rails'
 gem 'irb'

Gemfile.lock

@@ -131,6 +131,10 @@ GEM
     ffi (1.17.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    gtfs (0.5.1)
+      multi_json
+      rake
+      rubyzip (~> 2.3)
     haml (5.2.1)
       temple (>= 0.8.0)
       tilt
@@ -181,6 +185,7 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.8)
     minitest (5.25.1)
+    multi_json (1.15.0)
     mysql2 (0.5.6)
     net-imap (0.5.0)
       date
@@ -388,6 +393,7 @@ DEPENDENCIES
   ed25519
   exception_notification
   factory_bot_rails
+  gtfs
   haml-rails
   importmap-rails
   irb

app/models/bus_stop/import.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class BusStop < ApplicationRecord
+  class Import
+    def initialize(source)
+      @source = source
+    end
+
+    def import!
+      @source.each_stop do |stop|
+        next if stop.location_type.present? && stop.location_type != '0'
+
+        BusStop.create_with(name: stop.name).find_or_create_by!(hastus_id: stop.id).tap do |s|
+          s.update! name: stop.name
+        end
+      end
+    end
+  end
+end

app/models/bus_stops_route/import.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+class BusStopsRoute < ApplicationRecord
+  class Import
+    def initialize(source)
+      @source = source
+    end
+
+    def combined_trip_data
+      trip_data.transform_values do |trips|
+        trips.max_by(&:length).tap do |longest_sequence|
+          trips.excluding(longest_sequence).each do |sequence|
+            ([nil] + sequence).each_cons(2) do |previous_stop_id, stop_id|
+              next if longest_sequence.include? stop_id
+
+              # after the previous stop in this sequence...
+              location_in_longest_sequence = longest_sequence.index(previous_stop_id) ||
+                                             # ...or before the first common stop, or, failing all that, at the end
+                                             (common_index(longest_sequence, sequence) - 1)
+
+              longest_sequence.insert(location_in_longest_sequence + 1, stop_id)
+            end
+          end
+        end
+      end
+    end
+
+    def import!
+      combined_trip_data.each do |(route_name, direction), stops|
+        route = Route.find_by!(number: route_name)
+
+        BusStopsRoute.transaction do
+          route.bus_stops_routes.where(direction: direction).delete_all
+          stops.each.with_index(1) do |stop_id, sequence|
+            bus_stop = BusStop.find_by! hastus_id: stop_id
+            route.bus_stops_routes.create! bus_stop:, direction:, sequence:
+          end
+        end
+      end
+    end
+
+    private
+
+    def common_index(main, other)
+      main.index((main & other).first) || -1
+    end
+
+    def grouped_trips
+      @grouped_trips ||= @source.trips.group_by do |trip|
+        [route_name(trip.route_id), trip.direction_id]
+      end
+    end
+
+    def route_name(route_id)
+      @route_names ||= @source.routes.to_h { |route| [route.id, route.short_name] }
+      @route_names[route_id]
+    end
+
+    def stop_sequence(trip)
+      [].tap do |sequence|
+        stops_by_trip[trip.id].each do |stop_time|
+          sequence[stop_time.stop_sequence.to_i - 1] = stop_time.stop_id
+        end
+      end.compact.uniq
+    end
+
+    def stops_by_trip
+      @stops_by_trip ||= @source.stop_times.group_by(&:trip_id)
+    end
+
+    def trip_data
+      @trip_data ||= grouped_trips.transform_values do |trips|
+        trips.map { |trip| stop_sequence(trip) }.uniq
+      end
+    end
+  end
+end

app/models/route/import.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Route < ApplicationRecord
+  class Import
+    def initialize(source)
+      @source = source
+    end
+
+    def import!
+      @source.each_route do |route|
+        Route.find_or_create_by!(number: route.short_name).tap do |r|
+          r.update!(description: route.long_name)
+        end
+      end
+    end
+  end
+end

coverage/.last_run.json

@@ -1,5 +1,5 @@
 {
   "result": {
-    "line": 92.94
+    "line": 94.27
   }
 }

spec/factories/bus_stops.rb

@@ -3,7 +3,7 @@
 FactoryBot.define do
   factory :bus_stop do
     name { 'Test Stop' }
-    sequence :hastus_id
+    sequence(:hastus_id) { |n| "FAC-#{n}" }
 
     trait :pending do
       created_at { 2.days.ago }

spec/fixtures/files/routes.txt

@@ -0,0 +1,3 @@
+route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,network_id,route_sort_order
+1,ER,"Existing Route",,3,,FFFFFF,000000,Test,1
+2,NE,"Non-Existing Route",,3,,000000,111111,Test,2

spec/fixtures/files/stop_times.txt

@@ -0,0 +1,16 @@
+trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type,shape_dist_traveled,timepoint
+inbound-01,06:00:00,06:00:00,A,1,0,0,0.000,1
+inbound-01,06:01:00,06:01:00,B,2,0,0,0.100,0
+inbound-01,06:02:00,06:02:00,C,3,0,0,0.200,0
+inbound-01,06:03:00,06:03:00,D,4,0,0,0.300,0
+inbound-01,06:04:00,06:04:00,E,5,0,0,0.400,0
+inbound-01,06:05:00,06:05:00,F,6,0,0,0.500,1
+outbound-01,06:05:00,06:10:00,F,1,0,0,0.000,1
+outbound-01,06:11:00,06:11:00,E,2,0,0,0.100,0
+outbound-01,06:12:00,06:12:00,D,3,0,0,0.200,0
+outbound-01,06:13:00,06:13:00,C,4,0,0,0.300,0
+outbound-01,06:14:00,06:14:00,B,5,0,0,0.400,0
+outbound-01,06:15:00,06:15:00,A,6,0,0,0.500,1
+inbound-02,06:30:00,06:30:00,D1,1,0,0,0.000,1
+inbound-02,06:31:00,06:31:00,E1,2,0,0,0.100,0
+inbound-02,06:32:00,06:32:00,F,3,0,0,0.200,1

spec/fixtures/files/stops.txt

@@ -0,0 +1,12 @@
+stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,wheelchair_boarding
+1,1,"Existing Stop",,  42.477045, -72.607454,,,0,,1
+2,2,"New Stop",,  42.107338, -72.576106,,,0,Big_Station,1
+Big_Station,,"Big Station",,  42.086972, -72.562004,,,1,,0
+A,A,"Stop A",,  42.477045, -72.607454,,,0,,1
+B,B,"Stop B",,  42.107338, -72.576106,,,0,,1
+C,C,"Stop C",,  42.086972, -72.562004,,,0,,1
+D,D,"Stop D",,  42.477045, -72.607454,,,0,,1
+E,E,"Stop E",,  42.107338, -72.576106,,,0,,1
+F,F,"Stop F",,  42.086972, -72.562004,,,0,,1
+D1,D1,"Stop D1",,  42.477045, -72.607454,,,0,,1
+E1,E1,"Stop E1",,  42.107338, -72.576106,,,0,,1

spec/fixtures/files/trips.txt

@@ -0,0 +1,4 @@
+route_id,service_id,trip_id,trip_headsign,direction_id,block_id,shape_id,wheelchair_accessible,bikes_allowed
+1,Full-Service,inbound-01,"Inbound Bus",0,,,1,1
+1,Full-Service,inbound-02,"Inbound From Elsewhere",0,,,1,1
+1,Full-Service,outbound-01,"Outbound Bus",1,,,1,1

spec/models/bus_stop/import_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BusStop::Import do
+  describe '#import!' do
+    subject(:call) { described_class.new(source).import! }
+
+    include_context 'with a dummy source'
+
+    before { create :bus_stop, name: 'Old Name', hastus_id: '1' }
+
+    it 'imports stops' do
+      expect { call }.to change(BusStop, :count).by(stop_data.count - 2) # (The two cases below)
+    end
+
+    it 'updates existing stops' do
+      call
+      expect(BusStop.find_by(hastus_id: '1').name).to eq 'Existing Stop'
+    end
+
+    it 'skips stations' do
+      call
+      expect(BusStop.find_by(name: 'Big Station')).to be_nil
+    end
+  end
+end

spec/models/bus_stops_route/import_spec.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BusStopsRoute::Import do
+  describe '#combined_trip_data' do
+    subject(:call) { importer.combined_trip_data }
+
+    let(:importer) { described_class.new 'DUMMYSOURCE' }
+    let(:combined_data) { ->(seq) { { %w[RT D] => seq } } }
+
+    before do
+      allow(importer).to receive(:trip_data).and_return({ %w[RT D] => stop_sequences })
+    end
+
+    context 'when all stops in other variant are in longest variant' do
+      let(:stop_sequences) { [%w[A B C D E], %w[B C]] }
+
+      it 'preserves sequence of longest variant' do
+        expect(call).to eq combined_data[%w[A B C D E]]
+      end
+    end
+
+    context 'when a stop (but not the 1st stop) in another variant is not in the longest variant' do
+      context 'when one stop is in the other variant' do
+        let(:stop_sequences) { [%w[A B C D E], %w[C F]] }
+
+        it 'inserts that stop into longest variant after common stop' do
+          expect(call).to eq combined_data[%w[A B C F D E]]
+        end
+      end
+
+      context 'when multiple stops are in the other variant' do
+        let(:stop_sequences) { [%w[A B C D E], %w[C F G H]] }
+
+        it 'inserts those stops into longest variant after common stop' do
+          expect(call).to eq combined_data[%w[A B C F G H D E]]
+        end
+      end
+    end
+
+    context 'when the first stop in another variant is not in the longest variant' do
+      context 'when one stop is in the other variant' do
+        let(:stop_sequences) { [%w[A B C D E], %w[F D E]] }
+
+        it 'inserts that stop into longest variant before common stop' do
+          expect(call).to eq combined_data[%w[A B C F D E]]
+        end
+      end
+
+      context 'when multiple stops are in the other variant' do
+        let(:stop_sequences) { [%w[A B C D E], %w[F G H D E]] }
+
+        it 'inserts those stops into longest variant before common stop' do
+          expect(call).to eq combined_data[%w[A B C F G H D E]]
+        end
+      end
+    end
+
+    context 'when no stop in the other variant is in the longest variant' do
+      let(:stop_sequences) { [%w[A B C D E], %w[F G H I]] }
+
+      it 'appends the other variant to the end of the longest variant' do
+        expect(call).to eql combined_data[%w[A B C D E F G H I]]
+      end
+    end
+  end
+
+  describe '#import!' do
+    subject(:call) { described_class.new(source).import! }
+
+    include_context 'with a dummy source'
+
+    let!(:route) { create :route, number: 'ER' }
+
+    before { BusStop::Import.new(source).import! }
+
+    it 'destroys existing bus stops routes for the route and direction' do
+      bus_stop = create(:bus_stops_route, route: route, direction: '0').bus_stop
+      call
+      expect(BusStopsRoute.find_by(bus_stop:, route:)).to be_nil
+    end
+
+    it 'does not destroy bus stops routes for other routes' do
+      route = create :route, number: '55'
+      bus_stop = create(:bus_stops_route, route: route, direction: '0').bus_stop
+      call
+      expect(BusStopsRoute.find_by(bus_stop:, route:)).to be_present
+    end
+
+    it 'does not destroy bus stops routes for other directions' do
+      bus_stop = create(:bus_stops_route, route: route, direction: 'SKYWARD').bus_stop
+      call
+      expect(BusStopsRoute.find_by(bus_stop:, route:)).to be_present
+    end
+
+    it 'creates bus stops routes for the route' do
+      call
+      expect(route.bus_stops_routes.count).to eq(stop_times_data.count - 1) # One common stop between two trips
+    end
+
+    it 'creates bus stops routes with the correct directions' do
+      call
+      expect(route.bus_stops_routes.pluck(:direction).uniq).to eq %w[0 1]
+    end
+
+    it 'creates bus stops routes with the correct combined stop sequence' do
+      call
+      bus_stops_routes = route.bus_stops_routes.order(:sequence).where(direction: '0')
+      expect(bus_stops_routes.map { |bsr| bsr.bus_stop.hastus_id }).to eq %w[A B C D E D1 E1 F]
+    end
+
+    it 'creates bus stops routes with the correct sequence numbers' do
+      call
+      bus_stops_routes = route.bus_stops_routes.order(:sequence).where(direction: '1')
+      expect(bus_stops_routes.pluck(:sequence)).to eq (1..6).to_a
+    end
+  end
+end

spec/models/route/import_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Route::Import do
+  describe '#import!' do
+    subject(:call) { described_class.new(source).import! }
+
+    include_context 'with a dummy source'
+
+    before { create :route, number: 'ER', description: 'Old description' }
+
+    it 'imports routes' do
+      expect { call }.to change(Route, :count).by(1)
+    end
+
+    it 'updates existing routes' do
+      call
+      expect(Route.find_by(number: 'ER').description).to eq 'Existing Route'
+    end
+  end
+end

spec/support/import_shared_context.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'gtfs'
+
+RSpec.shared_context 'with a dummy source' do
+  let(:source) { instance_double GTFS::Source }
+
+  let(:stop_data) { GTFS::Stop.parse_stops file_fixture('stops.txt').read }
+  let(:route_data) { GTFS::Route.parse_routes file_fixture('routes.txt').read }
+  let(:trip_data) { GTFS::Trip.parse_trips file_fixture('trips.txt').read }
+  let(:stop_times_data) { GTFS::StopTime.parse_stop_times file_fixture('stop_times.txt').read }
+
+  before do
+    allow(source).to receive(:each_route) { |&block| route_data.each(&block) }
+    allow(source).to receive(:each_stop) { |&block| stop_data.each(&block) }
+    allow(source).to receive_messages(trips: trip_data, routes: route_data, stop_times: stop_times_data)
+  end
+end