From 0ae974e7547dbead01d76b5732170edf0fdab4eb Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Tue, 18 Jul 2023 12:24:03 +0100 Subject: [PATCH 01/41] feat: Add CITES processes public downloads. Adds tab and toggle options to the downloads modal, and the ability to filter results that are exported by RST and Captive Breeding types --- .../downloads_controller.js.coffee | 5 +- ...s_for_cites_processes_controller.js.coffee | 46 +++++++++++++++++++ .../templates/downloads_for_cites.handlebars | 43 ++++++++++++++++- app/assets/stylesheets/species/all.scss | 24 ++++++++++ app/controllers/species/exports_controller.rb | 2 + app/models/species/cites_processes_export.rb | 12 +++++ 6 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee diff --git a/app/assets/javascripts/species/controllers/downloads_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_controller.js.coffee index 7682ecd1ee..f488c2dfe8 100644 --- a/app/assets/javascripts/species/controllers/downloads_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/downloads_controller.js.coffee @@ -1,7 +1,7 @@ Species.DownloadsController = Ember.Controller.extend Species.Spinner, needs: [ 'downloadsForCmsListings', - 'downloadsForCitesListings', 'downloadsForCitesRestrictions', + 'downloadsForCitesListings', 'downloadsForCitesRestrictions', 'downloadsForCitesProcesses', 'downloadsForEuListings', 'downloadsForEuDecisions' ] downloadsPopupVisible: false @@ -23,6 +23,9 @@ Species.DownloadsController = Ember.Controller.extend Species.Spinner, legislationIsCitesRestrictions: ( -> @get('citesLegislation') == 'restrictions' ).property('citesLegislation') + legislationIsCitesProcesses: ( -> + @get('citesLegislation') == 'processes' + ).property('citesLegislation') legislationIsEuListings: ( -> @get('euLegislation') == 'listings' ).property('euLegislation') diff --git a/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee new file mode 100644 index 0000000000..d626283443 --- /dev/null +++ b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee @@ -0,0 +1,46 @@ +Species.DownloadsForCitesProcessesController = Ember.Controller.extend + designation: 'cites' + + needs: ['downloads'] + + processType: 'Both' + + toParams: ( -> + { + data_type: 'Processes' + filters: + process_type: @get('processType') + csv_separator: @get('controllers.downloads.csvSeparator') + } + ).property( + 'processType', + 'controllers.downloads.csvSeparator' + ) + + downloadUrl: ( -> + '/species/exports/download?' + $.param(@get('toParams')) + ).property('toParams') + + actions: + startDownload: () -> + @set('downloadInProgress', true) + @set('downloadMessage', 'Downloading...') + $.ajax({ + type: 'GET' + dataType: 'json' + url: @get('downloadUrl') + }).done((data) => + @set('downloadInProgress', false) + if data.total > 0 + @set('downloadMessage', null) + ga('send', { + hitType: 'event', + eventCategory: 'Downloads: ' + @get('processType'), + eventAction: 'Format: CSV', + eventLabel: @get('controllers.downloads.csvSeparator') + }) + window.location = @get('downloadUrl') + return + else + @set('downloadMessage', 'No results') + ) diff --git a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars index 14df63ce5d..def1d69168 100644 --- a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars +++ b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars @@ -4,11 +4,16 @@ LISTINGS {{/view}} -
  • +
  • {{#view Species.ToggleButton option="restrictions" valueBinding="controller.citesLegislation"}} QUOTAS/SUSPENSIONS {{/view}}
    • +
  • + {{#view Species.ToggleButton option="processes" valueBinding="controller.citesLegislation"}} + Processes + {{/view}} +
    • {{#view Species.DownloadsForLegislation isVisibleBinding="controller.legislationIsCitesListings" @@ -136,3 +141,39 @@ {{/view}} +{{#view Species.DownloadsForLegislation + isVisibleBinding="controller.legislationIsCitesProcesses" + controllerBinding="controllers.downloadsForCitesProcesses" +}} +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + {{view Species.StartDownloadButton controllerBinding="controller"}} +
    + {{#if controller.downloadInProgress}} + + {{/if}} + {{controller.downloadMessage}} +
    +
    +{{/view}} \ No newline at end of file diff --git a/app/assets/stylesheets/species/all.scss b/app/assets/stylesheets/species/all.scss index bc037efabc..9d7232fc32 100755 --- a/app/assets/stylesheets/species/all.scss +++ b/app/assets/stylesheets/species/all.scss @@ -651,6 +651,30 @@ body.inner #footer .holder{ width: 140px; } .doc-type .typy-columns .col .row { margin: 0 0 5px } + +// +.process-type { + overflow: hidden; + padding: 16px 0 13px 20px; +} +.process-type .heading { + color: $color-navy-dark; + font-weight: bold; + display: block; + margin: 0 0 4px; + text-transform: uppercase; +} +.process-type .heading.more { margin: 0 0 4px 11px; } +.process-type .typy-columns { overflow: hidden; } +.process-type .typy-columns .col { + float: left; + width: 25%; +} +.process-type .typy-columns .col-wide { + float: left; + width: 50%; +} + .info-columns { width: 100%; padding: 20px 0 16px; diff --git a/app/controllers/species/exports_controller.rb b/app/controllers/species/exports_controller.rb index cc83073f9b..4a2b3c7798 100644 --- a/app/controllers/species/exports_controller.rb +++ b/app/controllers/species/exports_controller.rb @@ -16,6 +16,8 @@ def download result = Species::ListingsExportFactory.new(@filters).export when 'EuDecisions' result = Species::EuDecisionsExport.new(@filters).export + when 'Processes' + result = Species::CitesProcessesExport.new(@filters).export end respond_to do |format| format.html { diff --git a/app/models/species/cites_processes_export.rb b/app/models/species/cites_processes_export.rb index a4a74a1e77..83f73cdc45 100644 --- a/app/models/species/cites_processes_export.rb +++ b/app/models/species/cites_processes_export.rb @@ -5,12 +5,24 @@ def query .joins("LEFT JOIN taxon_concepts ON taxon_concept_id = taxon_concepts.id LEFT JOIN geo_entities ON geo_entity_id = geo_entities.id LEFT JOIN events ON start_event_id = events.id") + .where(resolution: resolution) .order('taxon_concepts.full_name','cites_processes.type DESC','geo_entities.name_en') rel.select(sql_columns) end private + def resolution + case @filters['process_type'] + when 'Rst' + ['Significant Trade'] + when 'CaptiveBreeding' + ['Captive Breeding'] + else + ['Significant Trade', 'Captive Breeding'] + end + end + def resource_name 'cites_processes' end From 1e82fc559d80fc9aff27b175991fa0805b0717dd Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Tue, 18 Jul 2023 16:17:06 +0100 Subject: [PATCH 02/41] feat: add additional filters to the processes download tab (current/all, years, taxon, location) --- ...ds_for_cites_listings_controller.js.coffee | 4 +- ...s_for_cites_processes_controller.js.coffee | 85 ++++++++++++++++++- .../templates/downloads_for_cites.handlebars | 47 +++++++++- app/models/species/cites_processes_export.rb | 26 +++++- 4 files changed, 153 insertions(+), 9 deletions(-) diff --git a/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee index 120ea591d8..5f8f9a993e 100644 --- a/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee @@ -25,7 +25,7 @@ Species.DownloadsForCitesListingsController = Ember.Controller.extend } ).filter((e) -> e.taxonConcepts.length > 0 - ) + ) else @get('higherTaxaController.contentByRank') ).property('higherTaxaController.contentByRank.@each', 'taxonConceptQuery') @@ -103,4 +103,4 @@ Species.DownloadsForCitesListingsController = Ember.Controller.extend @set('selectedTaxonConcepts', []) deleteGeoEntitySelection: (context) -> - @get('selectedGeoEntities').removeObject(context) \ No newline at end of file + @get('selectedGeoEntities').removeObject(context) diff --git a/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee index d626283443..8012dcbd8a 100644 --- a/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee @@ -1,20 +1,88 @@ Species.DownloadsForCitesProcessesController = Ember.Controller.extend designation: 'cites' - needs: ['downloads'] + needs: ['geoEntities','higherTaxaCitesEu', 'downloads'] + higherTaxaController: ( -> + @get('controllers.higherTaxaCitesEu') + ).property() + + geoEntityQuery: null + taxonConceptQuery: null + selectedGeoEntities: [] + selectedTaxonConcepts: [] + timeScope: 'current' + timeScopeIsCurrent: ( -> + @get('timeScope') == 'current' + ).property('timeScope') + years: [1975..new Date().getFullYear()] + selectedYears: [] processType: 'Both' + documentTypeIsCitesSuspensions: ( -> + @get('documentType') == 'CitesSuspensions' + ).property('documentType') + + autoCompleteTaxonConcepts: ( -> + if @get('taxonConceptQuery') && @get('taxonConceptQuery').length > 0 + re = new RegExp("^"+@get('taxonConceptQuery'),"i") + @get('higherTaxaController.contentByRank') + .map((e) => + { + rankName: e.rankName + taxonConcepts: e.taxonConcepts.filter((item) => + re.test item.get('fullName') + ) + } + ).filter((e) -> + e.taxonConcepts.length > 0 + ) + else + @get('higherTaxaController.contentByRank') + ).property('higherTaxaController.contentByRank.@each', 'taxonConceptQuery') + + autoCompleteRegions: ( -> + if @get('geoEntityQuery') && @get('geoEntityQuery').length > 0 + re = new RegExp("(^|\\(| )"+@get('geoEntityQuery'),"i") + @get('controllers.geoEntities.regions') + .filter (item, index, enumerable) => + re.test item.get('name') + else + @get('controllers.geoEntities.regions') + ).property('controllers.geoEntities.regions.@each', 'geoEntityQuery') + + autoCompleteCountries: ( -> + if @get('geoEntityQuery') && @get('geoEntityQuery').length > 0 + re = new RegExp("(^|\\(| )"+@get('geoEntityQuery'),"i") + @get('controllers.geoEntities.countries') + .filter (item, index, enumerable) => + re.test item.get('name') + else + @get('controllers.geoEntities.countries') + ).property('controllers.geoEntities.countries.@each', 'geoEntityQuery') + + selectedGeoEntitiesIds: ( -> + @get('selectedGeoEntities').mapProperty('id') + ).property('selectedGeoEntities.@each') + + selectedTaxonConceptsIds: ( -> + @get('selectedTaxonConcepts').mapProperty('id') + ).property('selectedTaxonConcepts.@each') toParams: ( -> { data_type: 'Processes' - filters: + filters: process_type: @get('processType') + designation: @get('designation') + geo_entities_ids: @get('selectedGeoEntitiesIds') + taxon_concepts_ids: @get('selectedTaxonConceptsIds') + set: @get('timeScope') + years: @get('selectedYears') csv_separator: @get('controllers.downloads.csvSeparator') } ).property( - 'processType', - 'controllers.downloads.csvSeparator' + 'selectedGeoEntitiesIds.@each', 'selectedTaxonConceptsIds.@each', + 'timeScope', 'selectedYears.@each', 'processType', 'controllers.downloads.csvSeparator' ) downloadUrl: ( -> @@ -44,3 +112,12 @@ Species.DownloadsForCitesProcessesController = Ember.Controller.extend else @set('downloadMessage', 'No results') ) + + deleteTaxonConceptSelection: (context) -> + @set('selectedTaxonConcepts', []) + + deleteGeoEntitySelection: (context) -> + @get('selectedGeoEntities').removeObject(context) + + deleteYearSelection: (context) -> + @get('selectedYears').removeObject(Number(context)) diff --git a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars index def1d69168..b30c018532 100644 --- a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars +++ b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars @@ -167,6 +167,51 @@ +
    +
    +
      +
    • + {{#view Species.ToggleButton option="current" valueBinding="controller.timeScope"}} + CURRENT + {{/view}} +
      • +
    • + {{#view Species.ToggleButton option="all" valueBinding="controller.timeScope"}} + ALL + {{/view}} +
      • +
    +
    +
    + {{#if timeScopeIsCurrent}} + Only current data has
    been selected. + {{else}} + {{#unless documentTypeIsCitesSuspensions}} +
    + {{view Species.YearSearchButton selectedYearsBinding="controller.selectedYears"}} + {{view Species.YearSearchDropdown controllerBinding="controller"}} +
    + {{/unless}} + {{/if}} +
    +
    +
    +
    +
    + {{view Species.HigherTaxaSearchButton controllerBinding="controller"}} + {{view Species.HigherTaxaSearchDropdown controllerBinding="controller"}} +
    +
    +
    +
    + {{view Species.GeoEntitiesSearchButton + selectedGeoEntitiesBinding="controller.selectedGeoEntities" + loadedBinding="controller.loaded" + }} + {{view Species.GeoEntitiesSearchDropdown controllerBinding="controller"}} +
    +
    +
    {{view Species.StartDownloadButton controllerBinding="controller"}}
    @@ -176,4 +221,4 @@ {{controller.downloadMessage}}
    -{{/view}} \ No newline at end of file +{{/view}} diff --git a/app/models/species/cites_processes_export.rb b/app/models/species/cites_processes_export.rb index 83f73cdc45..6df8af062b 100644 --- a/app/models/species/cites_processes_export.rb +++ b/app/models/species/cites_processes_export.rb @@ -5,13 +5,35 @@ def query .joins("LEFT JOIN taxon_concepts ON taxon_concept_id = taxon_concepts.id LEFT JOIN geo_entities ON geo_entity_id = geo_entities.id LEFT JOIN events ON start_event_id = events.id") - .where(resolution: resolution) - .order('taxon_concepts.full_name','cites_processes.type DESC','geo_entities.name_en') + rel = apply_filters(rel) + rel = rel.order('taxon_concepts.full_name','cites_processes.type DESC','geo_entities.name_en') + rel.select(sql_columns) end private + def apply_filters(rel) + rel = rel.where(resolution: resolution) unless @filters['process_type'] == 'Both' + rel = rel.where("DATE_PART('YEAR', start_date) IN (?)", @filters['years']) if @filters['years']&.any? + rel = rel.where("status != 'Closed'") if @filters['set'] == 'current' + rel = rel.where('geo_entities.id IN (?)', geo_entities_ids) if @filters['geo_entities_ids']&.any? + + # Query 'data' json field for taxon concepts that have the submitted taxon_concept_id in their ancestry, + # or are the taxon_concept indicated by the txon_concept_id. + if @filters['taxon_concepts_ids']&.any? + taxon_concept = TaxonConcept.find(@filters['taxon_concepts_ids'].first) + rel = rel.where("taxon_concepts.data -> :rank_id_key = :taxon_concept_id OR taxon_concept_id = :taxon_concept_id", + rank_id_key: "#{taxon_concept.rank.name.downcase}_id", taxon_concept_id: taxon_concept.id.to_s) + end + + rel + end + + def geo_entities_ids + GeoEntity.nodes_and_descendants(@filters['geo_entities_ids']).pluck(:id) + end + def resolution case @filters['process_type'] when 'Rst' From 943b2fb45e12b6d48fd0a6848000033914e86cc4 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Wed, 19 Jul 2023 11:41:19 +0100 Subject: [PATCH 03/41] fix: account for species csv generation where no appendix is submitted, and move private method to public so class calling it has access to it. This allows downloads to be generated for species lists by year and by taxonomy from the cites compliance tool --- lib/modules/trade/download_data_retriever.rb | 23 +++++++++++----- lib/modules/trade/grouping/base.rb | 28 ++++++++++---------- lib/modules/trade/grouping/compliance.rb | 6 ++++- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/lib/modules/trade/download_data_retriever.rb b/lib/modules/trade/download_data_retriever.rb index f18a149980..3e7e414e6f 100644 --- a/lib/modules/trade/download_data_retriever.rb +++ b/lib/modules/trade/download_data_retriever.rb @@ -48,13 +48,22 @@ def self.search_download(params) SQL when 'species' appendix = params[:appendix] - <<-SQL - SELECT #{ATTRIBUTES.join(',')} - FROM non_compliant_shipments_view - WHERE year = #{year} - AND taxon_concept_id IN (#{id}) - AND appendix = '#{appendix}' - SQL + if appendix.present? + <<-SQL + SELECT #{ATTRIBUTES.join(',')} + FROM non_compliant_shipments_view + WHERE year = #{year} + AND taxon_concept_id IN (#{id}) + AND appendix = '#{appendix}' + SQL + else + <<-SQL + SELECT #{ATTRIBUTES.join(',')} + FROM non_compliant_shipments_view + WHERE year = #{year} + AND taxon_concept_id IN (#{id}) + SQL + end when 'commodity' <<-SQL SELECT #{ATTRIBUTES.join(',')} diff --git a/lib/modules/trade/grouping/base.rb b/lib/modules/trade/grouping/base.rb index 9943824abb..d105b9147e 100644 --- a/lib/modules/trade/grouping/base.rb +++ b/lib/modules/trade/grouping/base.rb @@ -35,6 +35,20 @@ def json_by_attribute(data, opts={}) raise NotImplementedError end + def read_taxonomy_conversion + conversion = {} + taxonomy = CSV.read(TAXONOMIC_GROUPING, {headers: true}) + taxonomy.each do |csv| + conversion[csv['group']] ||= [] + data = { + taxon_name: csv['taxon_name'], + rank: csv['taxonomic_level'] + } + conversion[csv['group']] << data + end + conversion + end + protected def shipments_table @@ -80,20 +94,6 @@ def limit private - def read_taxonomy_conversion - conversion = {} - taxonomy = CSV.read(TAXONOMIC_GROUPING, {headers: true}) - taxonomy.each do |csv| - conversion[csv['group']] ||= [] - data = { - taxon_name: csv['taxon_name'], - rank: csv['taxonomic_level'] - } - conversion[csv['group']] << data - end - conversion - end - def sanitise_group(group) return nil unless group attributes[group.to_sym] diff --git a/lib/modules/trade/grouping/compliance.rb b/lib/modules/trade/grouping/compliance.rb index d2a1a581e3..5ecef93c4f 100644 --- a/lib/modules/trade/grouping/compliance.rb +++ b/lib/modules/trade/grouping/compliance.rb @@ -271,7 +271,11 @@ def only_importer_countries(importers, keys, year) end def group_query - columns = @attributes.compact.uniq.join(',') + columns = if @attributes + @attributes.compact.uniq.join(',') + else + attributes.values.join(',') + end <<-SQL SELECT #{columns}, COUNT(*) AS cnt, 100.0*COUNT(*)/(SUM(COUNT(*)) OVER (PARTITION BY year)) AS percent FROM non_compliant_shipments_view From 2fa82067dab1216b0156f5e2333313bda2050774 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Wed, 26 Jul 2023 10:17:09 +0100 Subject: [PATCH 04/41] fix: update links on erb version of dropdown, to match updated version in handlebars template --- app/views/layouts/pages.html.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/views/layouts/pages.html.erb b/app/views/layouts/pages.html.erb index 271162f06a..946b12ac73 100644 --- a/app/views/layouts/pages.html.erb +++ b/app/views/layouts/pages.html.erb @@ -56,7 +56,8 @@
  • Species+ / CITES Checklist API
  • EU Captive breeding database
  • EU Wildlife Trade Legislation
    • -
  • EU analysis
    • +
  • EU analysis
    • +
  • CITES Wildlife TradeView
    • From e7a1a3e47dcd04c69a2c243084b643e8c949a4f1 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Mon, 31 Jul 2023 17:29:34 +0100 Subject: [PATCH 05/41] fix: delete cites rst decisions that have been removed in the external API --- lib/modules/import/rst/importer.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/modules/import/rst/importer.rb b/lib/modules/import/rst/importer.rb index b9f0980457..21e39283af 100644 --- a/lib/modules/import/rst/importer.rb +++ b/lib/modules/import/rst/importer.rb @@ -2,6 +2,9 @@ module Import::Rst::Importer class << self def import(data) + # Array of CitesRstProcesses in current import we can use to + # destroy records no longer being returned from the RST API. + active_ids = [] data.map do |item| taxon_concept = map_taxon_concept(item) unless taxon_concept @@ -29,7 +32,10 @@ def import(data) start_date: item['startDate'], document: "https://rst.cites.org/public/case-details/#{item['id']}" ) + active_ids << rst_process.id end + + destroy_invalid_rst_processes(active_ids) end private @@ -49,5 +55,16 @@ def map_event(item) Rails.logger.info "Event #{item['meeting']['name']} for case #{item['id']} not found" unless event event end + + def destroy_invalid_rst_processes(active_ids) + CitesRstProcess.where.not(id: active_ids).find_each do |rst_process| + case_id = rst_process.case_id + if rst_process.destroy + Rails.logger.info "RST process with case_id #{case_id} destroyed" + else + Rails.logger.info "RST process with case_id #{case_id} could not be destroyed" + end + end + end end end From 73ef91fed89743a57088ba1be8d09e86051008a7 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Wed, 2 Aug 2023 16:47:17 +0100 Subject: [PATCH 06/41] feat: Add EU region to response for checklist app, add EU region to drop downs for listing changes in admin section --- app/controllers/admin/taxon_listing_changes_controller.rb | 2 +- app/models/geo_entity_type.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/admin/taxon_listing_changes_controller.rb b/app/controllers/admin/taxon_listing_changes_controller.rb index b287a6f416..cb13ddd8bd 100644 --- a/app/controllers/admin/taxon_listing_changes_controller.rb +++ b/app/controllers/admin/taxon_listing_changes_controller.rb @@ -95,7 +95,7 @@ def load_change_types find_by_name(ChangeType::EXCEPTION) @species_listings = @designation.species_listings.order(:abbreviation) @geo_entities = GeoEntity.order(:name_en).joins(:geo_entity_type). - where(:is_current => true, :geo_entity_types => { :name => 'COUNTRY' }) + where(:is_current => true, :geo_entity_types => { :name => ['COUNTRY', 'REGION'] }) @hash_annotations = if @designation.is_eu? Annotation.for_eu diff --git a/app/models/geo_entity_type.rb b/app/models/geo_entity_type.rb index 02c5b6aa2d..98d31533da 100644 --- a/app/models/geo_entity_type.rb +++ b/app/models/geo_entity_type.rb @@ -18,7 +18,7 @@ class GeoEntityType < ActiveRecord::Base DEFAULT_SET = '3' SETS = { "1" => [CITES_REGION], # CITES Checklist - "2" => [COUNTRY, TERRITORY], # CITES Checklist + "2" => [COUNTRY, REGION, TERRITORY], # CITES Checklist "3" => [CITES_REGION, COUNTRY, TERRITORY], # Species+ "4" => [COUNTRY, REGION, TERRITORY, TRADE_ENTITY], # CITES Trade "5" => [COUNTRY, TERRITORY] # E-library From 6321675bc4510ba9fe30e5e1beb1ccf3d629eab4 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Wed, 2 Aug 2023 17:01:26 +0100 Subject: [PATCH 07/41] fix: add dummy ga function to handle google analytics on development and staging --- app/assets/javascripts/cites_trade/application.js | 15 ++++++--------- app/views/layouts/cites_trade.html.erb | 6 ++++++ app/views/layouts/pages.html.erb | 6 ++++++ app/views/layouts/species.html.erb | 6 ++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/app/assets/javascripts/cites_trade/application.js b/app/assets/javascripts/cites_trade/application.js index 9e7336df88..e722056765 100644 --- a/app/assets/javascripts/cites_trade/application.js +++ b/app/assets/javascripts/cites_trade/application.js @@ -786,15 +786,12 @@ $(document).ready(function(){ $.cookie('cites_trade.csv_separator', csv_separator) query += '&filters[csv_separator]=' + csv_separator; - // google analytics function only defined on production - if (typeof(ga) === 'function') { - ga('send', { - hitType: 'event', - eventCategory: 'Downloads: ' + report_type, - eventAction: 'Format: CSV', - eventLabel: csv_separator - }); - } + ga('send', { + hitType: 'event', + eventCategory: 'Downloads: ' + report_type, + eventAction: 'Format: CSV', + eventLabel: csv_separator + }); downloadResults( decodeURIComponent( query ) ); return } diff --git a/app/views/layouts/cites_trade.html.erb b/app/views/layouts/cites_trade.html.erb index e51f6f6483..acb608b660 100644 --- a/app/views/layouts/cites_trade.html.erb +++ b/app/views/layouts/cites_trade.html.erb @@ -26,6 +26,12 @@ <%= render "cites_trade/hotjar" %> + <% else %> + <% end %> diff --git a/app/views/layouts/pages.html.erb b/app/views/layouts/pages.html.erb index 946b12ac73..8e4b108d85 100644 --- a/app/views/layouts/pages.html.erb +++ b/app/views/layouts/pages.html.erb @@ -36,6 +36,12 @@ <%= render "species/hotjar" %> + <% else %> + <% end %> diff --git a/app/views/layouts/species.html.erb b/app/views/layouts/species.html.erb index 51f969d627..c78b690c8a 100644 --- a/app/views/layouts/species.html.erb +++ b/app/views/layouts/species.html.erb @@ -33,6 +33,12 @@ <%= render "species/hotjar" %> + <% else %> + <% end %> From 93832806fafb22854099a46f59e170a9f6990d27 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Thu, 3 Aug 2023 16:31:48 +0100 Subject: [PATCH 08/41] fix: use ILIKE instead of LIKE in species search queries --- app/models/species/taxon_concept_prefix_matcher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/species/taxon_concept_prefix_matcher.rb b/app/models/species/taxon_concept_prefix_matcher.rb index f0d7a5874f..7c1a831ec7 100644 --- a/app/models/species/taxon_concept_prefix_matcher.rb +++ b/app/models/species/taxon_concept_prefix_matcher.rb @@ -92,7 +92,7 @@ def initialize_query if @taxon_concept_query @query = @query.where( ActiveRecord::Base.send(:sanitize_sql_array, [ - "name_for_matching LIKE :sci_name_prefix", + "name_for_matching ILIKE :sci_name_prefix", :sci_name_prefix => "#{@taxon_concept_query}%" ]) ) From 7d9aa6047e9152a8c7e69252922293c2aa9f4790 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Thu, 3 Aug 2023 19:24:01 +0100 Subject: [PATCH 09/41] feat: remove filtering by EN, ES, and FR in autocomplete view, so all common names can be queried --- ..._all_countries_to_species_automcomplete.rb | 15 ++ .../20230803181442.sql | 199 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb create mode 100644 db/views/auto_complete_taxon_concepts_view/20230803181442.sql diff --git a/db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb b/db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb new file mode 100644 index 0000000000..39f1392a57 --- /dev/null +++ b/db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb @@ -0,0 +1,15 @@ +class AddAllCountriesToSpeciesAutomcomplete < ActiveRecord::Migration + def up + execute "DROP VIEW IF EXISTS auto_complete_taxon_concepts_view" + execute "CREATE VIEW auto_complete_taxon_concepts_view AS #{view_sql('20230803181442', 'auto_complete_taxon_concepts_view')}" + execute File.read(File.expand_path('../../mviews/011_rebuild_auto_complete_taxon_concepts_mview.sql', __FILE__)) + execute "SELECT * FROM rebuild_auto_complete_taxon_concepts_mview()" + end + + def down + execute "DROP VIEW IF EXISTS auto_complete_taxon_concepts_view" + execute "CREATE VIEW auto_complete_taxon_concepts_view AS #{view_sql('20220808123526', 'auto_complete_taxon_concepts_view')}" + execute File.read(File.expand_path('../../mviews/011_rebuild_auto_complete_taxon_concepts_mview.sql', __FILE__)) + execute "SELECT * FROM rebuild_auto_complete_taxon_concepts_mview()" + end +end diff --git a/db/views/auto_complete_taxon_concepts_view/20230803181442.sql b/db/views/auto_complete_taxon_concepts_view/20230803181442.sql new file mode 100644 index 0000000000..65ba0f159b --- /dev/null +++ b/db/views/auto_complete_taxon_concepts_view/20230803181442.sql @@ -0,0 +1,199 @@ +WITH synonyms_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + atc.id, + atc.full_name, + atc.author_year, + tc.id, + tc.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(tc.full_name, ' ')) + FROM taxon_concepts tc + JOIN taxon_relationships tr + ON tr.other_taxon_concept_id = tc.id + JOIN taxon_relationship_types trt + ON trt.id = tr.taxon_relationship_type_id + AND trt.name = 'HAS_SYNONYM' + JOIN taxon_concepts atc + ON atc.id = tr.taxon_concept_id + WHERE tc.name_status = 'S' AND atc.name_status = 'A' +), scientific_names_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + id, + taxon_concepts.full_name, + taxon_concepts.author_year, + id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(full_name, ' ')) + FROM taxon_concepts +), unlisted_subspecies_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + parents.id, + parents.full_name, + parents.author_year, + taxon_concepts.id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_concepts.full_name, ' ')) + FROM taxon_concepts + JOIN ranks ON ranks.id = taxon_concepts.rank_id + AND ranks.name IN ('SUBSPECIES', 'VARIETY') + JOIN taxon_concepts parents + ON parents.id = taxon_concepts.parent_id + WHERE taxon_concepts.name_status NOT IN ('S', 'T', 'N') + AND parents.name_status = 'A' + + EXCEPT + + SELECT + parents.id, + parents.full_name, + parents.author_year, + taxon_concepts.id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_concepts.full_name, ' ')) + FROM taxon_concepts + JOIN ranks ON ranks.id = taxon_concepts.rank_id + AND ranks.name IN ('SUBSPECIES') -- VARIETY not here on purpose + JOIN taxon_concepts parents + ON parents.id = taxon_concepts.parent_id + JOIN taxonomies ON taxonomies.id = taxon_concepts.taxonomy_id + WHERE taxon_concepts.name_status NOT IN ('S', 'T', 'N') + AND parents.name_status = 'A' + AND CASE + WHEN taxonomies.name = 'CMS' + THEN (taxon_concepts.listing->'cms_historically_listed')::BOOLEAN + ELSE (taxon_concepts.listing->'cites_historically_listed')::BOOLEAN + OR (taxon_concepts.listing->'eu_historically_listed')::BOOLEAN + END +), taxon_common_names AS ( + SELECT + taxon_commons.*, + common_names.name + FROM taxon_commons + JOIN common_names + ON common_names.id = taxon_commons.common_name_id +), common_names_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + taxon_concept_id, + taxon_concepts.full_name, + taxon_concepts.author_year, + NULL::INT, + taxon_common_names.name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_common_names.name, E'\\s|''')) + FROM taxon_common_names + JOIN taxon_concepts + ON taxon_common_names.taxon_concept_id = taxon_concepts.id +), taxon_common_names_dehyphenated AS ( + SELECT + taxon_concept_id, + taxon_concepts.full_name, + taxon_concepts.author_year, + NULL::INT, + taxon_common_names.name, + UPPER(REPLACE(taxon_common_names.name, '-', ' ')) + FROM taxon_common_names + JOIN taxon_concepts + ON taxon_common_names.taxon_concept_id = taxon_concepts.id + WHERE STRPOS(taxon_common_names.name, '-') > 0 +), common_names_segmented_dehyphenated AS ( + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment + FROM common_names_segmented + UNION + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, REGEXP_SPLIT_TO_TABLE(matched_name_segment, E'-') + FROM common_names_segmented + WHERE STRPOS(matched_name_segment, '-') > 0 + UNION + SELECT * FROM taxon_common_names_dehyphenated +), all_names_segmented_cleaned AS ( + SELECT * FROM ( + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, + CASE + WHEN POSITION(matched_name_segment IN + UPPER(matched_name) + ) = 1 THEN UPPER(matched_name) + ELSE matched_name_segment + END, type_of_match + FROM ( + SELECT *, 'SELF' AS type_of_match + FROM scientific_names_segmented + UNION + SELECT *, 'SYNONYM' + FROM synonyms_segmented + UNION + SELECT *, 'SUBSPECIES' + FROM unlisted_subspecies_segmented + UNION + SELECT *, 'COMMON_NAME' + FROM common_names_segmented_dehyphenated + ) all_names_segmented + ) all_names_segmented_no_prefixes + WHERE LENGTH(matched_name_segment) >= 3 +), taxa_with_visibility_flags AS ( + SELECT taxon_concepts.id, + CASE + WHEN taxonomies.name = 'CITES_EU' THEN TRUE + ELSE FALSE + END AS taxonomy_is_cites_eu, + name_status, + ranks.name AS rank_name, + ranks.display_name_en AS rank_display_name_en, + ranks.display_name_es AS rank_display_name_es, + ranks.display_name_fr AS rank_display_name_fr, + ranks.taxonomic_position AS rank_order, + taxon_concepts.taxonomic_position, + CASE + WHEN + name_status = 'A' + AND ( + ranks.name != 'SUBSPECIES' + AND ranks.name != 'VARIETY' + OR taxonomies.name = 'CITES_EU' + AND ( + (listing->'cites_historically_listed')::BOOLEAN + OR (listing->'eu_historically_listed')::BOOLEAN + ) + OR taxonomies.name = 'CMS' + AND (listing->'cms_historically_listed')::BOOLEAN + ) + THEN TRUE + ELSE FALSE + END AS show_in_species_plus_ac, + CASE + WHEN + name_status = 'A' + AND ( + ranks.name != 'SUBSPECIES' + AND ranks.name != 'VARIETY' + OR (listing->'cites_show')::BOOLEAN + ) + THEN TRUE + ELSE FALSE + END AS show_in_checklist_ac, + CASE + WHEN + taxonomies.name = 'CITES_EU' + AND ARRAY['A', 'H', 'N']::VARCHAR[] && ARRAY[name_status] + THEN TRUE + ELSE FALSE + END AS show_in_trade_ac, + CASE + WHEN + taxonomies.name = 'CITES_EU' + AND ARRAY['A', 'H', 'N', 'T']::VARCHAR[] && ARRAY[name_status] + THEN TRUE + ELSE FALSE + END AS show_in_trade_internal_ac + FROM taxon_concepts + JOIN ranks ON ranks.id = rank_id + JOIN taxonomies ON taxonomies.id = taxon_concepts.taxonomy_id +) +SELECT + t1.*, + matched_name_segment AS name_for_matching, + matched_taxon_concept_id AS matched_id, + matched_name, + full_name, + author_year, + type_of_match +FROM taxa_with_visibility_flags t1 +JOIN all_names_segmented_cleaned t2 +ON t1.id = t2.taxon_concept_id +WHERE LENGTH(matched_name_segment) >= 3; From 19541b20a2b6802b2cf797042ebe5fc04f51db1e Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Fri, 4 Aug 2023 13:53:58 +0100 Subject: [PATCH 10/41] feat: use ILIKE instead of LIKE in MTaxonFilterByScientificNameWithDescendants --- ...cept_filter_by_scientific_name_with_descendants.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb b/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb index 7d1e522f4c..d4ddb43b22 100644 --- a/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb +++ b/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb @@ -11,14 +11,21 @@ def initialize(relation, scientific_name, match_options = {}) def relation types_of_match = ['SELF'] types_of_match << 'SYNONYM' if @match_synonyms - types_of_match << 'COMMON_NAME' if @match_common_names types_of_match << 'SUBSPECIES' if @match_subspecies + + name_for_matching_query = if @match_common_names + "name_for_matching LIKE :sci_name_prefix AND (type_of_match IN (:types_of_match) OR \ + name_for_matching ILIKE :sci_name_prefix AND type_of_match = 'COMMON_NAME')" + else + "name_for_matching LIKE :sci_name_prefix AND type_of_match IN (:types_of_match)" + end + subquery = MAutoCompleteTaxonConcept.select( 'id, ARRAY_AGG_NOTNULL(matched_name) AS matched_names_ary' ). where( ActiveRecord::Base.send(:sanitize_sql_array, [ - "name_for_matching LIKE :sci_name_prefix AND type_of_match IN (:types_of_match)", + name_for_matching_query, sci_name_prefix: "#{@scientific_name}%", types_of_match: types_of_match ]) From 216c083b3493c92ff3a2e730bc60f3fb74e7162f Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Fri, 4 Aug 2023 17:39:53 +0100 Subject: [PATCH 11/41] feat: replace ILIKe with mb_chars --- ...pt_filter_by_scientific_name_with_descendants.rb | 13 +++---------- app/models/species/taxon_concept_prefix_matcher.rb | 2 +- lib/modules/search_param_sanitiser.rb | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb b/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb index d4ddb43b22..a3dac71f63 100644 --- a/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb +++ b/app/models/m_taxon_concept_filter_by_scientific_name_with_descendants.rb @@ -2,7 +2,7 @@ class MTaxonConceptFilterByScientificNameWithDescendants def initialize(relation, scientific_name, match_options = {}) @relation = relation || MTaxonConcept.all - @scientific_name = scientific_name.upcase.strip + @scientific_name = scientific_name.mb_chars.upcase.strip @match_synonyms = match_options[:synonyms] || true @match_common_names = match_options[:common_names] || false @match_subspecies = match_options[:subspecies] || false @@ -11,21 +11,14 @@ def initialize(relation, scientific_name, match_options = {}) def relation types_of_match = ['SELF'] types_of_match << 'SYNONYM' if @match_synonyms + types_of_match << 'COMMON_NAME' if @match_common_names types_of_match << 'SUBSPECIES' if @match_subspecies - - name_for_matching_query = if @match_common_names - "name_for_matching LIKE :sci_name_prefix AND (type_of_match IN (:types_of_match) OR \ - name_for_matching ILIKE :sci_name_prefix AND type_of_match = 'COMMON_NAME')" - else - "name_for_matching LIKE :sci_name_prefix AND type_of_match IN (:types_of_match)" - end - subquery = MAutoCompleteTaxonConcept.select( 'id, ARRAY_AGG_NOTNULL(matched_name) AS matched_names_ary' ). where( ActiveRecord::Base.send(:sanitize_sql_array, [ - name_for_matching_query, + "name_for_matching LIKE :sci_name_prefix AND type_of_match IN (:types_of_match)", sci_name_prefix: "#{@scientific_name}%", types_of_match: types_of_match ]) diff --git a/app/models/species/taxon_concept_prefix_matcher.rb b/app/models/species/taxon_concept_prefix_matcher.rb index 7c1a831ec7..f0d7a5874f 100644 --- a/app/models/species/taxon_concept_prefix_matcher.rb +++ b/app/models/species/taxon_concept_prefix_matcher.rb @@ -92,7 +92,7 @@ def initialize_query if @taxon_concept_query @query = @query.where( ActiveRecord::Base.send(:sanitize_sql_array, [ - "name_for_matching ILIKE :sci_name_prefix", + "name_for_matching LIKE :sci_name_prefix", :sci_name_prefix => "#{@taxon_concept_query}%" ]) ) diff --git a/lib/modules/search_param_sanitiser.rb b/lib/modules/search_param_sanitiser.rb index 0faf45074c..2a04e21415 100644 --- a/lib/modules/search_param_sanitiser.rb +++ b/lib/modules/search_param_sanitiser.rb @@ -6,7 +6,7 @@ def sanitise_string(s) end def sanitise_upcase_string(s) - s && s.strip.upcase + s && s.strip.mb_chars.upcase end def sanitise_symbol(s, default = nil) From 80430d8599b1f81ac6c08bf56712974630a57812 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Fri, 4 Aug 2023 17:51:17 +0100 Subject: [PATCH 12/41] chore: rename db migration --- ...1442_remove_language_restrictions_from_autocomplete_view.rb} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename db/migrate/{20230803181442_add_all_countries_to_species_automcomplete.rb => 20230803181442_remove_language_restrictions_from_autocomplete_view.rb} (91%) diff --git a/db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb b/db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb similarity index 91% rename from db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb rename to db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb index 39f1392a57..192839d594 100644 --- a/db/migrate/20230803181442_add_all_countries_to_species_automcomplete.rb +++ b/db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb @@ -1,4 +1,4 @@ -class AddAllCountriesToSpeciesAutomcomplete < ActiveRecord::Migration +class RemoveLanguageRestrictionsFromAutocompleteView < ActiveRecord::Migration def up execute "DROP VIEW IF EXISTS auto_complete_taxon_concepts_view" execute "CREATE VIEW auto_complete_taxon_concepts_view AS #{view_sql('20230803181442', 'auto_complete_taxon_concepts_view')}" From ceeff59657a5284ca0641bed11051daf30aad975 Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Mon, 14 Aug 2023 15:40:46 +0100 Subject: [PATCH 13/41] fix: add non-public document options for signed_in users to allow correct parameter building --- .../controllers/elibrary_search_controller.js.coffee | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee b/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee index 0ebb2e3508..846b0ab3dd 100644 --- a/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee @@ -33,7 +33,8 @@ Species.ElibrarySearchController = Ember.Controller.extend Species.Spinner, allDocumentTypes = @get('controllers.events.documentTypes') .concat @get('controllers.events.interSessionalDocumentTypes') .concat @get('controllers.events.identificationDocumentTypes') - + if @get('isSignedIn') + allDocumentTypes = allDocumentTypes.concat(@get('controllers.events.interSessionalNonPublicDocumentTypes')) @set('selectedDocumentType', allDocumentTypes.findBy('id', filtersHash.document_type)) general_subtype_type = @get_general_subtype_type(filtersHash) @@ -71,10 +72,10 @@ Species.ElibrarySearchController = Ember.Controller.extend Species.Spinner, general_subtype: isGeneralSubType } - getDocTypeParam: -> + getDocTypeParam: -> id = @get('selectedDocumentType.id') - if id != '__all__' then id else null + if id != '__all__' then id else null filteredDocumentTypes: ( -> if @get('selectedEventType') From 39423baee7ed76b3a558a5c8294b30b30b0cb6cc Mon Sep 17 00:00:00 2001 From: sergio marrocoli Date: Wed, 30 Aug 2023 14:33:40 +0100 Subject: [PATCH 14/41] fix: add line height to authors part of species pages --- .../javascripts/species/templates/taxon_concept.handlebars | 2 +- app/assets/stylesheets/species/all.scss | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/species/templates/taxon_concept.handlebars b/app/assets/javascripts/species/templates/taxon_concept.handlebars index e403f38597..47d8e14822 100644 --- a/app/assets/javascripts/species/templates/taxon_concept.handlebars +++ b/app/assets/javascripts/species/templates/taxon_concept.handlebars @@ -8,7 +8,7 @@ {{/if}}

    {{fullName}}

    -

    {{authorYear}}

    +

    {{authorYear}}