Fixing conflicts

.gitlab-ci.yml

@@ -36,6 +36,7 @@ deploy_qa:
     AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
+    - pip install setuptools==68.0.0
     - pip install urllib3==2.0.6
     - pip install botocore==1.31.62
     - pip install boto3==1.28.62
@@ -107,6 +108,7 @@ deploy_live:
     AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
     GITHUB_TOKEN: $GITHUB_TOKEN
   script:
+    - pip install setuptools==68.0.0
     - pip install urllib3==2.0.6
     - pip install botocore==1.31.62
     - pip install boto3==1.28.62

Gemfile

@@ -83,7 +83,7 @@ gem 'request_store'
 gem 'geojsonlint'
 gem 'jsonpath', '~> 0.8.8'
 gem 'graphql-batch', '~> 0.4.2'
-gem 'google-protobuf', '3.19.6'
+gem 'google-protobuf', '3.25.5'
 gem 'concurrent-ruby', require: 'concurrent'
 gem 'rubyzip', '1.3.0'
 gem 'rack-protection', '2.2.0'

Gemfile.lock

@@ -328,7 +328,7 @@ GEM
       google-apis-core (>= 0.4, < 2.a)
     google-apis-sheets_v4 (0.12.0)
       google-apis-core (>= 0.4, < 2.a)
-    google-protobuf (3.19.6)
+    google-protobuf (3.25.5)
     google_drive (3.0.7)
       google-apis-drive_v3 (>= 0.5.0, < 1.0.0)
       google-apis-sheets_v4 (>= 0.4.0, < 1.0.0)
@@ -446,7 +446,7 @@ GEM
     net-smtp (0.4.0.1)
       net-protocol
     netrc (0.11.0)
-    nio4r (2.7.0)
+    nio4r (2.7.3)
     nokogiri (1.16.5)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
@@ -627,7 +627,7 @@ GEM
       phony (>= 2.18.12)
     promise.rb (0.7.4)
     public_suffix (4.0.7)
-    puma (5.6.8)
+    puma (5.6.9)
       nio4r (~> 2.0)
     pusher (2.0.2)
       httpclient (~> 2.8)
@@ -726,7 +726,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
-    rexml (3.3.3)
+    rexml (3.3.6)
       strscan
     rotp (6.3.0)
     rqrcode (2.1.1)
@@ -873,7 +873,7 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.7.0)
+    webrick (1.8.2)
     webrobots (0.1.2)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
@@ -920,7 +920,7 @@ DEPENDENCIES
   gem-licenses
   geocoder (= 1.6.4)
   geojsonlint
-  google-protobuf (= 3.19.6)
+  google-protobuf (= 3.25.5)
   google_drive
   graphiql-rails!
   graphql (~> 1.11.0)

app/controllers/test_controller.rb

@@ -1,7 +1,8 @@
 require 'sample_data'
 
 class TestController < ApplicationController
-  before_action :check_environment
+  before_action :check_environment, :init_bot_events
+  after_action :trigger_bot_events
 
   include SampleData
 
@@ -250,4 +251,12 @@ def new_media(type)
   def check_environment
     (render(plain: 'Only available in test mode', status: 400) and return) unless Rails.env === 'test'
   end
+
+  def init_bot_events
+    BotUser.init_event_queue if Rails.env.test?
+  end
+
+  def trigger_bot_events
+    BotUser.trigger_events if Rails.env.test?
+  end
 end

app/graph/mutations/explainer_mutations.rb

@@ -20,6 +20,8 @@ class Create < Mutations::CreateMutation
 
   class Update < Mutations::UpdateMutation
     include SharedCreateAndUpdateFields
+
+    argument :trashed, GraphQL::Types::Boolean, required: false
   end
 
   class Destroy < Mutations::DestroyMutation; end

app/graph/mutations/fact_check_mutations.rb

@@ -26,6 +26,7 @@ class Update < Mutations::UpdateMutation
 
     argument :title, GraphQL::Types::String, required: false
     argument :summary, GraphQL::Types::String, required: false
+    argument :trashed, GraphQL::Types::Boolean, required: false
   end
 
   class Destroy < Mutations::DestroyMutation; end

app/graph/types/explainer_type.rb

@@ -13,4 +13,5 @@ class ExplainerType < DefaultObject
   field :user, UserType, null: true
   field :team, PublicTeamType, null: true
   field :tags, [GraphQL::Types::String, null: true], null: true
+  field :trashed, GraphQL::Types::Boolean, null: true
 end

app/graph/types/fact_check_type.rb

@@ -14,4 +14,5 @@ class FactCheckType < DefaultObject
   field :rating, GraphQL::Types::String, null: true
   field :imported, GraphQL::Types::Boolean, null: true
   field :report_status, GraphQL::Types::String, null: true
+  field :trashed, GraphQL::Types::Boolean, null: true
 end

app/graph/types/project_media_type.rb

@@ -206,7 +206,7 @@ def flags
   field :tags, TagType.connection_type, null: true
 
   def tags
-    object.get_annotations('tag').map(&:load)
+    object.get_annotations('tag').map(&:load).sort_by { |tag| tag.tag_text.downcase }
   end
 
   field :comments, CommentType.connection_type, null: true

app/graph/types/team_type.rb

@@ -224,7 +224,7 @@ def sources(keyword: nil)
   end
 
   def tag_texts(keyword: nil)
-    object.tag_texts_by_keyword(keyword)
+    object.tag_texts_by_keyword(keyword).order('text ASC')
   end
 
   field :tag_texts_count, GraphQL::Types::Int, null: true do
@@ -306,6 +306,7 @@ def tipline_messages(uid:)
     argument :rating, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :imported, GraphQL::Types::Boolean, required: false, camelize: false # Only for fact-checks
     argument :target_id, GraphQL::Types::Int, required: false, camelize: false # Exclude articles already applied to the `ProjectMedia` with this ID
+    argument :trashed, GraphQL::Types::Boolean, required: false, camelize: false, default_value: false
   end
 
   def articles(**args)
@@ -336,6 +337,7 @@ def articles(**args)
     argument :rating, [GraphQL::Types::String, null: true], required: false, camelize: false
     argument :imported, GraphQL::Types::Boolean, required: false, camelize: false # Only for fact-checks
     argument :target_id, GraphQL::Types::Int, required: false, camelize: false # Exclude articles already applied to the `ProjectMedia` with this ID
+    argument :trashed, GraphQL::Types::Boolean, required: false, camelize: false, default_value: false
   end
 
   def articles_count(**args)

app/lib/check_cached_fields.rb

@@ -66,8 +66,10 @@ def index_and_pg_cached_field(options, value, name, target)
       if should_update_cached_field?(options, target)
         update_index = options[:update_es] || false
         value = target.send(update_index, value) if update_index.is_a?(Symbol) && target.respond_to?(update_index)
-        field_name = options[:es_field_name] || name
-        es_options = { keys: [field_name], data: { field_name => value } }
+        field_name = options[:es_field_name].nil? ? [name] : [options[:es_field_name]]&.flatten
+        data = {}
+        field_name.each{ |f| data[f] = value }
+        es_options = { keys: field_name, data: data }
         es_options[:pm_id] = target.id if target.class.name == 'ProjectMedia'
         model = { klass: target.class.name, id: target.id }
         ElasticSearchWorker.new.perform(YAML::dump(model), YAML::dump(es_options), 'update_doc')

app/lib/generic_worker_helpers.rb

@@ -0,0 +1,9 @@
+module GenericWorkerHelpers
+  def run_later(klass_method, *method_args)
+    GenericWorker.perform_async(self.to_s, klass_method, *method_args)
+  end
+
+  def run_later_in(time, klass_method, *method_args)
+    GenericWorker.perform_in(time, self.to_s, klass_method, *method_args)
+  end
+end

app/models/annotations/tag.rb

@@ -1,5 +1,6 @@
 class Tag < ApplicationRecord
   include AnnotationBase
+  extend TagHelpers
 
   # "tag" is a reference to a TagText object
   field :tag, GraphQL::Types::Int, presence: true
@@ -91,6 +92,18 @@ def hit_nested_objects_limit?
     ret
   end
 
+  def self.create_project_media_tags(project_media_id, tags_json)
+    project_media = ProjectMedia.find_by_id(project_media_id)
+
+    if !project_media.nil?
+      tags = JSON.parse(tags_json)
+      clean_tags(tags).each { |tag| Tag.create! annotated: project_media, tag: tag.strip, skip_check_ability: true }
+    else
+      error = StandardError.new("[ProjectMedia] Exception creating project media's tags in background. Project media is nil.")
+      CheckSentry.notify(error, project_media_id: project_media_id)
+    end
+  end
+
   private
 
   def get_tag_text_reference

app/models/api_key.rb

@@ -16,7 +16,9 @@ class ApiKey < ApplicationRecord
 
   validate :validate_team_api_keys_limit, on: :create
 
-  has_one :bot_user, dependent: :destroy
+  has_one :bot_user
+
+  after_destroy :delete_bot_user
 
   # Reimplement this method in your application
   def self.applications
@@ -52,6 +54,14 @@ def create_bot_user
     end
   end
 
+  def delete_bot_user
+    if bot_user.present? && bot_user.owns_media?
+      bot_user.update(api_key_id: nil)
+    else
+      bot_user.destroy
+    end
+  end
+
   def set_user_and_team
     self.user = User.current unless User.current.nil?
     self.team = Team.current unless Team.current.nil?

app/models/bot/alegre.rb

@@ -621,7 +621,6 @@ def self.create_relationship(source, target, pm_id_scores, relationship_type, or
         target.save!
       end
       r = self.fill_in_new_relationship(source, target, pm_id_scores, relationship_type, original_source, original_relationship_type)
-      r.save!
       self.report_exception_if_bad_relationship(r, pm_id_scores, relationship_type)
       Rails.logger.info "[Alegre Bot] [ProjectMedia ##{target.id}] [Relationships 5/6] Created new relationship for relationship ID Of #{r.id}"
     elsif self.is_relationship_upgrade?(r, relationship_type)
@@ -640,26 +639,23 @@ def self.is_relationship_upgrade?(relationship, relationship_type)
 
   def self.fill_in_new_relationship(source, target, pm_id_scores, relationship_type, original_source, original_relationship_type)
     score_with_context = pm_id_scores[source.id] || {}
-    r = Relationship.new
-    r.skip_check_ability = true
-    r.relationship_type = relationship_type
-    r.model = self.get_indexing_model(source, score_with_context)
-    r.weight = score_with_context[:score]
-    r.details = score_with_context[:context]
-    r.source_id = source.id
-    r.target_id = target.id
-    r.source_field = score_with_context[:source_field]
-    r.target_field = score_with_context[:target_field]
-    if original_source
-      r.original_weight = pm_id_scores[original_source.id][:score]
-      r.original_details = pm_id_scores[original_source.id][:context]
-      r.original_relationship_type = original_relationship_type
-      r.original_model = self.get_indexing_model(original_source, pm_id_scores[original_source.id])
-      r.original_source_id = original_source.id
-      r.original_source_field = pm_id_scores[original_source.id][:source_field]
-    end
-    r.user_id ||= BotUser.alegre_user&.id
-    r
+    options = {
+      model: self.get_indexing_model(source, score_with_context),
+      weight: score_with_context[:score],
+      details: score_with_context[:context],
+      source_field: score_with_context[:source_field],
+      target_field: score_with_context[:target_field],
+    }
+    options.merge!({
+      original_weight: pm_id_scores[original_source.id][:score],
+      original_details: pm_id_scores[original_source.id][:context],
+      original_relationship_type: original_relationship_type,
+      original_model: self.get_indexing_model(original_source, pm_id_scores[original_source.id]),
+      original_source_id: original_source.id,
+      original_source_field: pm_id_scores[original_source.id][:source_field],
+    }) if original_source
+    options[:user_id] = BotUser.alegre_user&.id
+    Relationship.create_unless_exists(source.id, target.id, relationship_type, options)
   end
 
   def self.can_create_relationship?(source, target, relationship_type)

app/models/bot/smooch.rb

@@ -852,7 +852,7 @@ def self.save_text_message(message)
   end
 
   def self.create_project_media(message, type, extra)
-    extra.merge!({ archived: message['archived'] })
+    extra.merge!({ archived: message['archived'] }) unless message['archived'].blank?
     channel_value = self.get_smooch_channel(message)
     extra.merge!({ channel: {main: channel_value }}) unless channel_value.nil?
     pm = ProjectMedia.create!({ media_type: type, smooch_message: message }.merge(extra))

app/models/claim_description.rb

@@ -14,8 +14,10 @@ class ClaimDescription < ApplicationRecord
 
   validates_presence_of :team
   validates_uniqueness_of :project_media_id, allow_nil: true
+  validate :cant_apply_article_to_item_if_article_is_in_the_trash
   after_commit :update_fact_check, on: [:update]
   after_update :update_report_status
+  after_update :reset_item_rating_if_removed
   after_update :replace_media, unless: proc { |cd| cd.disable_replace_media }
   after_update :migrate_claim_and_fact_check_logs, if: proc { |cd| cd.saved_change_to_project_media_id? && !cd.project_media_id.nil? }
 
@@ -115,4 +117,22 @@ def migrate_claim_and_fact_check_logs
       .where.not(event: 'create').update_all(associated_id: self.project_media_id)
     end
   end
+
+  def cant_apply_article_to_item_if_article_is_in_the_trash
+    errors.add(:base, I18n.t(:cant_apply_article_to_item_if_article_is_in_the_trash)) if self.project_media && self.fact_check&.trashed
+  end
+
+  # If claim/fact-check is detached from item, reset the item status/rating back to the default one (unstarted, undetermined, etc.)
+  def reset_item_rating_if_removed
+    if self.project_media_id.nil? && !self.project_media_id_before_last_save.nil?
+      old_pm = ProjectMedia.find_by_id(self.project_media_id_before_last_save)
+      return if old_pm.nil?
+      status = old_pm.last_status_obj
+      default_status = old_pm.team.verification_statuses('media')[:default]
+      if status && status.status != default_status
+        status.status = default_status
+        status.save
+      end
+    end
+  end
 end

app/models/concerns/article.rb

@@ -14,6 +14,7 @@ module Article
     after_commit :update_elasticsearch_data, :send_to_alegre, :notify_bots, on: [:create, :update]
     after_commit :destroy_elasticsearch_data, on: :destroy
     after_save :create_tag_texts_if_needed
+    after_update :schedule_for_permanent_deletion_if_sent_to_trash, if: proc { |obj| obj.is_a?(FactCheck) || obj.is_a?(Explainer) }
   end
 
   def text_fields
@@ -68,6 +69,13 @@ def create_tag_texts_if_needed
     self.class.delay.create_tag_texts_if_needed(self.team_id, self.tags) if self.respond_to?(:tags) && !self.tags.blank?
   end
 
+  def schedule_for_permanent_deletion_if_sent_to_trash
+    if self.trashed && !self.trashed_before_last_save
+      interval = CheckConfig.get('empty_trash_interval', 30, :integer)
+      self.class.delay_for(interval.days, { queue: 'trash', retry: 0 }).delete_permanently(self.id)
+    end
+  end
+
   module ClassMethods
     def create_tag_texts_if_needed(team_id, tags)
       tags.to_a.map(&:strip).each do |tag|
@@ -87,5 +95,13 @@ def send_to_alegre(id)
         ::Bot::Alegre.send_field_to_similarity_index(obj.project_media, field)
       end unless obj.nil?
     end
+
+    def delete_permanently(id)
+      obj = self.find_by_id(id)
+      if obj && obj.trashed
+        obj.destroy!
+        obj.claim_description.destroy! if obj.is_a?(FactCheck)
+      end
+    end
   end
 end