Merge branch 'develop' into epic/CV2-4111-workspace-analytics

Gemfile.lock

@@ -47,7 +47,7 @@ GIT
 
 GIT
   remote: https://github.com/meedan/rails-i18n.git
-  revision: f8dd347402d9de9125af4db05f94f6933147b18f
+  revision: e3004b7793e5fd759c5b2134e021a29c6d0a6488
   branch: rails-6-x
   specs:
     rails-i18n (6.0.0)

app/lib/check_cached_fields.rb

@@ -97,12 +97,13 @@ def index_cached_field(options, value, name, obj)
           update_pg: options[:update_pg],
           pg_field_name: options[:pg_field_name],
         }
-        self.delay_for(1.second).index_cached_field_bg(index_options, value, name, obj)
+        self.delay_for(1.second).index_cached_field_bg(index_options, value, name, obj.class.name, obj.id)
       end
     end
 
-    def index_cached_field_bg(index_options, value, name, obj)
-      self.index_and_pg_cached_field(index_options, value, name, obj)
+    def index_cached_field_bg(index_options, value, name, klass, id)
+      obj = klass.constantize.find_by_id id
+      self.index_and_pg_cached_field(index_options, value, name, obj) unless obj.nil?
     end
 
     def update_pg_cache_field(options, value, name, target)

app/lib/check_elastic_search.rb

@@ -63,7 +63,7 @@ def update_elasticsearch_doc_bg(options)
       create_doc_if_not_exists(options)
       sleep 1
       client = $repository.client
-      client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3, body: { doc: fields }
+      client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { doc: fields }
     end
   end
 
@@ -98,7 +98,7 @@ def create_update_nested_obj_bg(options)
     end
     values = store_elasticsearch_data(options[:keys], options[:data])
     client = $repository.client
-    client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3,
+    client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id],
             body: { script: { source: source, params: { value: values, id: values['id'] } } }
   end
 
@@ -178,7 +178,7 @@ def destroy_elasticsearch_doc_nested(options)
       begin
         client = $repository.client
         source = "for (int i = 0; i < ctx._source.#{nested_type}.size(); i++) { if(ctx._source.#{nested_type}[i].id == params.id){ctx._source.#{nested_type}.remove(i);}}"
-        client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3,
+        client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id],
                  body: { script: { source: source, params: { id: options[:model_id] } } }
       rescue
         Rails.logger.info "[ES destroy] doc with id #{options[:doc_id]} not exists"

app/models/bot/smooch.rb

@@ -62,28 +62,32 @@ def suggestion_accepted?
     def self.inherit_status_and_send_report(rid)
       relationship = Relationship.find_by_id(rid)
       unless relationship.nil?
-        # A relationship created by the Smooch Bot or Alegre Bot is related to search results, so the user has already received the report as a search result - unless it's a suggestion
-        return if [BotUser.smooch_user&.id, BotUser.alegre_user&.id].include?(relationship.user_id) && !relationship.confirmed_by
         target = relationship.target
         parent = relationship.source
+
+        # Always inherit status for confirmed matches
         if ::Bot::Smooch.team_has_smooch_bot_installed(target) && relationship.is_confirmed?
           s = target.annotations.where(annotation_type: 'verification_status').last&.load
           status = parent.last_verification_status
           if !s.nil? && s.status != status
             s.status = status
             s.save
           end
-          ::Bot::Smooch.send_report_from_parent_to_child(parent.id, target.id)
+
+          # A relationship created by the Smooch Bot or Alegre Bot is related to search results (unless it's a suggestion that was confirmed), so the user has already received the report as a search result... no need to send another report
+          # Only send a report for (1) Confirmed matches created manually OR (2) Suggestions accepted
+          created_by_bot = [BotUser.smooch_user&.id, BotUser.alegre_user&.id].include?(relationship.user_id)
+          ::Bot::Smooch.send_report_from_parent_to_child(parent.id, target.id) if !created_by_bot || relationship.confirmed_by
         end
       end
     end
 
     after_create do
-      self.class.delay_for(1.seconds, { queue: 'smooch_priority'}).inherit_status_and_send_report(self.id)
+      self.class.delay_for(2.seconds, { queue: 'smooch_priority'}).inherit_status_and_send_report(self.id)
     end
 
     after_update do
-      self.class.delay_for(1.seconds, { queue: 'smooch_priority'}).inherit_status_and_send_report(self.id) if self.suggestion_accepted?
+      self.class.delay_for(2.seconds, { queue: 'smooch_priority'}).inherit_status_and_send_report(self.id) if self.suggestion_accepted?
     end
 
     after_destroy do

app/models/concerns/smooch_messages.rb

@@ -400,8 +400,7 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req
         end
         unless associated.nil?
           self.smoooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj)
-          # Check if message contains caption then create an item and force relationship
-          self.relate_item_and_caption(message, associated, app_id, author, request_type, associated_obj) unless message['caption'].blank?
+          self.smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj)
         end
       end
     end
@@ -416,21 +415,39 @@ def smoooch_post_save_message_actions(message, associated, app_id, author, reque
       self.send_report_to_user(message['authorId'], message, associated, message['language'], 'fact_check_report') if self.should_try_to_send_report?(request_type, associated)
     end
 
-    def relate_item_and_caption(message, associated, app_id, author, request_type, associated_obj)
+    def smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj)
+      if !message['caption'].blank?
+        # Check if message contains caption then create an item and force relationship
+        self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.suggested_type)
+      elsif message['type'] == 'text' && associated.class.name == 'ProjectMedia' && associated.media.type == 'Link'
+        # Check if message of type text contain a link and long text
+        # Text words equal the number of words - 1(which is the link size)
+        text_words = ::Bot::Alegre.get_number_of_words(message['text']) - 1
+        if text_words > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer)
+          # Remove link from text
+          link = self.extract_url(message['text'])
+          message['text'] = message['text'].remove(link.url)
+          self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.confirmed_type)
+        end
+      end
+    end
+
+    def relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, relationship_type)
       message['_id'] = SecureRandom.hex
       message['type'] = 'text'
       message['request_body'] = message['text']
-      message['text'] = message['caption']
+      message['text'] = message['caption'] unless message['caption'].nil?
       message.delete('caption')
       message.delete('mediaUrl')
       target = self.create_project_media_from_message(message)
       unless target.nil?
         smoooch_post_save_message_actions(message, target, app_id, author, request_type, associated_obj)
-        Relationship.create_unless_exists(associated.id, target.id, Relationship.suggested_type)
+        Relationship.create_unless_exists(associated.id, target.id, relationship_type)
       end
     end
 
     def smooch_save_tipline_request(message, associated, app_id, author, request_type, associated_obj)
+      text = message['text']
       message['text'] = message['request_body'] unless message['request_body'].blank?
       message.delete('request_body')
       fields = { smooch_data: message.merge({ app_id: app_id }) }
@@ -450,6 +467,8 @@ def smooch_save_tipline_request(message, associated, app_id, author, request_typ
           associated.save!
         end
       end
+      # Back message text to original one
+      message['text'] = text
     end
 
     def create_tipline_requests(associated, author, fields)

app/models/concerns/smooch_zendesk.rb

@@ -74,6 +74,7 @@ def zendesk_send_message_to_user(uid, text, extra = {}, _force = false, preview_
       response_body = nil
       response_code = 0
       begin
+        Rails.logger.info("[Smooch Bot] [Zendesk] Sending message to #{uid} for app #{app_id}: #{params.to_json}")
         response_body = api_instance.post_message(app_id, uid, message_post_body) # It will raise an exception if message can't be sent
         response_code = 200
       rescue SmoochApi::ApiError => e

app/models/relationship.rb

@@ -18,7 +18,7 @@ class Relationship < ApplicationRecord
   validate :cant_be_related_to_itself
   validates :relationship_type, uniqueness: { scope: [:source_id, :target_id], message: :already_exists }, on: :create
 
-  before_create :set_confirmed, :destroy_same_suggested_item, if: proc { |r| r.is_confirmed? }
+  before_create :destroy_same_suggested_item, if: proc { |r| r.is_confirmed? }
   after_create :move_to_same_project_as_main, prepend: true
   after_create :point_targets_to_new_source, :update_counters, prepend: true
   after_update :reset_counters, prepend: true

app/workers/generic_worker.rb

@@ -3,6 +3,7 @@ class GenericWorker
   include Sidekiq::Worker
 
   sidekiq_options retry: 3
+  sidekiq_retry_in { |_count, _exception| 3 }
 
   def perform(klass_name, klass_method, *method_args)
     klass = klass_name.constantize

config/tipline_strings.yml

@@ -981,29 +981,44 @@ es:
   unsubscribe_button_label: Cancelar suscripción
   unsubscribed: Actualmente no estás suscrita(o) a nuestro boletín.
 tl:
-  add_more_details_state_button_label: ''
-  ask_if_ready_state_button_label: ''
-  cancelled: ''
-  confirm_preferred_language: ''
-  invalid_format: ''
-  keep_subscription_button_label: ''
-  languages: ''
-  languages_and_privacy_title: ''
-  main_menu: ''
-  main_state_button_label: ''
-  navigation_button: ''
-  no_results_in_language: ''
-  privacy_and_purpose: ''
-  privacy_statement: ''
-  privacy_title: ''
-  report_updated: ''
-  search_result_is_not_relevant_button_label: ''
-  search_result_is_relevant_button_label: ''
-  search_state_button_label: ''
-  subscribe_button_label: ''
-  subscribed: ''
-  unsubscribe_button_label: ''
-  unsubscribed: ''
+  add_more_details_state_button_label: Magdagdag
+  ask_if_ready_state_button_label: I-cancel
+  cancelled: Ok!
+  confirm_preferred_language: Pumili ng wika
+  invalid_format: "Hindi suportado ang format ng ipinadala mo."
+  keep_subscription_button_label: Manatiling konektado
+  languages: Mga wika
+  languages_and_privacy_title: Wika at Privacy
+  main_menu: Main menu
+  main_state_button_label: I-cancel
+  navigation_button: Gamitin ang mga button para mag-navigate
+  no_results_in_language: "Walang resulta na nahanap sa %{language}. Narito ang ilang resulta sa ibang wika na maaaring konektado sa hinahanap mo."
+  privacy_and_purpose: |-
+    PRIVACY AT LAYUNIN
+
+    Welcome sa %{team} %{channel} tipline.
+
+    Puwede mong gamitin ang tip line na ito para magpadala ng mga bagay na gusto mong ipa-fact check.
+
+    Ligtas ang data mo rito. Seryoso kami sa aming responsibilidad na alagaan ang iyong personal na impormasyon at panatilihing ang %{channel} ay pribado at ligtas; hindi kailanman ipamamahagi, ibebenta o gagamitin sa ibang layunin ang impormasyon na nakapagtutukoy sa iyo (personally identifiable information o PII) maliban para maibigay at mapabuti ang serbisyong ito.
+
+    Para ma-detect sa lalong mabilis na paraan ang kumakalat na misinformation, posibleng ibahagi namin ang content na hindi PII mula sa tip line na ito sa mga mananaliksik at fact-checking partner na pumasa sa aming pagsusuri.
+
+    Paalala na ang mga website na aming ili-link ay may kanya-kanyang privacy policy.
+
+    Kung hindi mo nais na magamit ang mga isinumite mong impormasyon sa gawaing ito, mangyaring huwag mag-ambag sa aming system.
+  privacy_statement: Privacy statement
+  privacy_title: Privacy
+  report_updated: "*In-update* ang fact check na ito dahil sa bagong impormasyon:"
+  search_result_is_not_relevant_button_label: "Hindi"
+  search_result_is_relevant_button_label: "Oo"
+  search_state_button_label: I-submit
+  subscribe_button_label: Mag-subscribe
+  subscribed: Kasalukuyan kang naka-subscribe sa aming newsletter.
+  unsubscribe_button_label: Mag-unsubscribe
+  unsubscribed: Hindi ka kasalukuyang naka-subscribe sa aming newsletter.
+  nlu_disambiguation: "Pumili ng isa sa mga option na ito:"
+  nlu_cancel: "Bumalik sa menu"
 ta:
   add_more_details_state_button_label: கூடுதல் தகவல்ககள்
   ask_if_ready_state_button_label: ரத்து செய்

test/models/bot/smooch_3_test.rb

@@ -191,6 +191,139 @@ def teardown
     end
   end
 
+  test "should bundle message with link and text" do
+    uid = random_string
+    payload = {
+      trigger: 'message:appUser',
+      app: {
+        '_id': @app_id
+      },
+      version: 'v1.1',
+      messages: [message],
+      appUser: {
+        '_id': random_string,
+        'conversationStarted': true
+      }
+    }
+    Sidekiq::Testing.fake! do
+      # 1) Send link and short text
+      message = {
+        '_id': random_string,
+        authorId: uid,
+        type: 'text',
+        source: { type: "whatsapp" },
+        text: "#{@link_url} short text",
+      }
+      payload[:messages] = [message]
+      Bot::Smooch.run(payload.to_json)
+      sleep 1
+      assert_difference 'ProjectMedia.count' do
+        assert_no_difference 'Claim.count' do
+          assert_difference 'Link.count' do
+            Sidekiq::Worker.drain_all
+          end
+        end
+      end
+      # Clean up created items to start other cases with same link
+      ProjectMedia.last.destroy
+      Link.last.destroy
+      # 2) Send link with long text
+      long_text = []
+      15.times{ long_text << random_string }
+      link_long_text = @link_url.concat(' ').concat(long_text.join(' '))
+      message = {
+        '_id': random_string,
+        authorId: uid,
+        type: 'text',
+        source: { type: "whatsapp" },
+        text: link_long_text,
+      }
+      payload[:messages] = [message]
+      Bot::Smooch.run(payload.to_json)
+      sleep 1
+      pm_id = ProjectMedia.last.id
+      assert_difference 'ProjectMedia.count', 2 do
+        assert_difference 'Claim.count' do
+          assert_difference 'Link.count' do
+            assert_difference 'Relationship.count' do
+              Sidekiq::Worker.drain_all
+            end
+          end
+        end
+      end
+      l1 = Link.last
+      c1 = Claim.last
+      pm_l1 = l1.project_medias.last
+      pm_c1 = c1.project_medias.last
+      r = Relationship.last
+      assert_equal [pm_l1.id, pm_c1.id].sort, [r.source_id, r.target_id].sort
+      # 3) Same message multiple times (re-send message in step 2)
+      message['_id'] = random_string
+      payload[:messages] = [message]
+      Bot::Smooch.run(payload.to_json)
+      sleep 1
+      assert_no_difference 'ProjectMedia.count' do
+        assert_no_difference 'Relationship.count' do
+          assert_difference 'TiplineRequest.count', 2 do
+            Sidekiq::Worker.drain_all
+          end
+        end
+      end
+      assert_equal 2, pm_l1.tipline_requests.count
+      assert_equal 2, pm_c1.tipline_requests.count
+      # 4) Send different messages with the same link
+      long_text2 = []
+      15.times{ long_text2 << random_string }
+      link_long_text2 = long_text2.join(' ').concat(' ').concat(@link_url)
+      message = {
+        '_id': random_string,
+        authorId: uid,
+        type: 'text',
+        source: { type: "whatsapp" },
+        text: link_long_text2,
+      }
+      payload[:messages] = [message]
+      Bot::Smooch.run(payload.to_json)
+      sleep 1
+      assert_difference 'ProjectMedia.count' do
+        assert_difference 'Relationship.count' do
+          assert_difference 'Claim.count' do
+            assert_no_difference 'Link.count' do
+              Sidekiq::Worker.drain_all
+            end
+          end
+        end
+      end
+      pm = ProjectMedia.last
+      r = Relationship.last
+      assert_equal [pm_l1.id, pm.id].sort, [r.source_id, r.target_id].sort
+      # 5) Send two messages with the same text but different links
+      link_long_text3 = @link_url_2.concat(' ').concat(long_text.join(' '))
+      message = {
+        '_id': random_string,
+        authorId: uid,
+        type: 'text',
+        source: { type: "whatsapp" },
+        text: link_long_text3,
+      }
+      payload[:messages] = [message]
+      Bot::Smooch.run(payload.to_json)
+      sleep 1
+      assert_difference 'ProjectMedia.count' do
+        assert_difference 'Relationship.count' do
+          assert_difference 'Link.count' do
+            assert_no_difference 'Claim.count' do
+              Sidekiq::Worker.drain_all
+            end
+          end
+        end
+      end
+      pm = ProjectMedia.last
+      r = Relationship.last
+      assert_equal [pm_c1.id, pm.id].sort, [r.source_id, r.target_id].sort
+    end
+  end
+
   test "should force relationship between media and caption text" do
     long_text = []
     15.times{ long_text << random_string }