diff --git a/ruby/lib/ci/queue/redis/build_record.rb b/ruby/lib/ci/queue/redis/build_record.rb index b1329ff9..2942a290 100644 --- a/ruby/lib/ci/queue/redis/build_record.rb +++ b/ruby/lib/ci/queue/redis/build_record.rb @@ -73,6 +73,11 @@ def record_success(id, stats: nil, skip_flaky_record: false) error_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline| pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY)) pipeline.hget(key('requeues-count'), id.b) + pipeline.sadd( + key('success-reports'), + id.dup.force_encoding(Encoding::BINARY), + ) + pipeline.expire(key('success-reports'), config.redis_ttl) record_stats(stats, pipeline: pipeline) end record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0) diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index 00013b5d..c80e8e20 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -55,6 +55,19 @@ def poll sleep 0.05 end end + + # check we executed all tests + # only the master should perform this check because otherwise we will DDoS Redis when all workers + # try to fetch the processed tests at the same time + if master? + missing_tests = @index.keys - (redis.smembers(key('success-reports')) + redis.hkeys(key('error-reports'))) + puts "missing_tests: #{missing_tests.size}" + if exhausted? && missing_tests.size > 0 + puts "We have not processed all tests!" + puts missing_tests.inspect + end + end + redis.pipelined do |pipeline| pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl) pipeline.expire(key('processed'), config.redis_ttl)