Specify language for text extraction

config/app.yml

@@ -29,4 +29,7 @@ libreoffice_path: <%= ENV['LIBREOFFICE_PATH'] %>
 tesseract_path: <%= ENV['TESSERACT_PATH'] %>
 tika_path: <%= ENV['TIKA_PATH'] %>
 wkhtmltopdf_path: <%= ENV['WKHTMLTOPDF_PATH'] %>
+
+# Other settings
+tika_config_directory: <%= ENV['TIKA_CONFIG_DIRECTORY'] %>
 wkhtmltopdf_params: '-d 100 --encoding UTF-8'

lib/colore.rb

@@ -9,3 +9,4 @@
 require_relative 'document'
 require_relative 'heathen'
 require_relative 'sidekiq_workers'
+require_relative 'tika_config'

lib/config.rb

@@ -41,6 +41,8 @@ class C_
     attr_accessor :tika_path
     # @return [String] Path to the wkhtmltopdf binary. Defaults to `"wkhtmltopdf"`
     attr_accessor :wkhtmltopdf_path
+    # @return [String] Relative path to the writable tika config directory. Defaults to `"../tmp/tika"`
+    attr_accessor :tika_config_directory
     # @return [String] Params for wkhtmltopdf
     attr_accessor :wkhtmltopdf_params
 
@@ -65,6 +67,8 @@ def self.config
         c.tesseract_path = yaml['tesseract_path'] || 'tesseract'
         c.tika_path = yaml['tika_path'] || 'tika'
         c.wkhtmltopdf_path = yaml['wkhtmltopdf_path'] || 'wkhtmltopdf'
+
+        c.tika_config_directory = yaml['tika_config_directory'] || '../tmp/tika'
         c.wkhtmltopdf_params = yaml['wkhtmltopdf_params'] || ''
 
         c

lib/heathen/processor_methods/libreoffice.rb

@@ -55,6 +55,7 @@ def libreoffice(format:)
       if to_suffix == 'txt'
         executioner.execute(
           Colore::C_.tika_path,
+          "--config=#{Colore::TikaConfig.path_for(job.language)}",
           '--text',
           job.content_file,
           binary: true

lib/heathen/processor_methods/pdftotext.rb

@@ -7,6 +7,7 @@ def pdftotext
 
       executioner.execute(
         Colore::C_.tika_path,
+        "--config=#{Colore::TikaConfig.path_for(job.language)}",
         '--text',
         job.content_file,
         binary: true

lib/tika_config.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'pathname'
+
+module Colore
+  # The Colore Tika is a module to help with Tika-related configuration files.
+  module TikaConfig
+    # The configuration template version
+    VERSION = 'v1'
+
+    # The default language to use when the language has not been found
+    DEFAULT_LANGUAGE = 'eng'
+
+    # Config template
+    TEMPLATE = <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <properties>
+        <parsers>
+          <parser class="org.apache.tika.parser.DefaultParser"></parser>
+          <parser class="org.apache.tika.parser.ocr.TesseractOCRParser">
+            <params>
+              <param name="language" type="string">%<language_alpha3>s</param>
+            </params>
+          </parser>
+        </parsers>
+      </properties>
+    XML
+
+    class << self
+      private
+
+      def tika_config_path
+        Pathname.new File.expand_path(Colore::C_.tika_config_directory, __dir__)
+      end
+
+      def path_for!(language_alpha3)
+        file = tika_config_path.join('ocr', VERSION, "tika.#{language_alpha3}.xml")
+        return file if file.file?
+
+        FileUtils.mkdir_p(tika_config_path.join('ocr', VERSION))
+        File.write(file, format(TEMPLATE, language_alpha3: language_alpha3))
+        file
+      end
+    end
+
+    # Returns the file path of the Tika configuration for performing OCR
+    # detection in a specified language.
+    #
+    # @param [String] language The language code in either ISO 639-1 (two-letter) or ISO 639-2 (three-letter) format.
+    #                          Supported languages are those with corresponding Tika configuration files.
+    #
+    # @return [Pathname] The path to the Tika configuration file for the specified language or
+    #                    the configuration file for DEFAULT_LANGUAGE if the language is not found.
+    def self.path_for(language)
+      language_alpha3 = Colore::Utils.language_alpha3(language) || DEFAULT_LANGUAGE
+
+      path_for!(language_alpha3)
+    end
+  end
+end

spec/fixtures/heathen/quickfox.ar.txt

@@ -0,0 +1 @@
+الثعلب البني السريع مفتون بالكلاب الكسولة

spec/heathen/processor_methods/pdftotext_spec.rb

@@ -4,7 +4,8 @@
 
 RSpec.describe Heathen::Processor do
   let(:content) { fixture('heathen/quickfox.pdf').read }
-  let(:job) { Heathen::Job.new 'foo', content, 'en' }
+  let(:job) { Heathen::Job.new 'foo', content, language }
+  let(:language) { 'en' }
   let(:processor) { described_class.new job: job, logger: spec_logger }
 
   after do
@@ -14,7 +15,19 @@
   describe '#pdftotext' do
     it 'converts PDF to TXT' do
       processor.pdftotext
+      expect(job.content).to eq 'The quick brown fox jumps lazily over the dog'
       expect(job.content.mime_type).to eq 'text/plain; charset=us-ascii'
     end
+
+    context 'with Arabic files' do
+      let(:content) { fixture('heathen/quickfox.ar.pdf').read }
+      let(:language) { 'ar' }
+
+      it 'extracts Arabic text from images' do
+        processor.pdftotext
+        expect(job.content).to eq fixture('heathen/quickfox.ar.txt').read.strip.force_encoding(Encoding::ASCII_8BIT)
+        expect(job.content.mime_type).to eq 'text/plain; charset=utf-8'
+      end
+    end
   end
 end

spec/lib/tika_config_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'fileutils'
+require 'pathname'
+
+RSpec.describe Colore::TikaConfig do
+  let(:tika_config_directory) { '../tmp/tika-test' }
+  let(:tika_test_config_path) { Pathname.new(File.expand_path('../../tmp/tika-test', __dir__)) }
+
+  before do
+    allow(Colore::C_.config).to receive(:tika_config_directory).and_return tika_config_directory
+    FileUtils.mkdir_p tika_test_config_path
+    FileUtils.rm_rf tika_test_config_path
+  end
+
+  after do
+    FileUtils.rm_rf tika_test_config_path
+  end
+
+  describe '.path_for' do
+    subject(:path_for) { described_class.path_for(language) }
+
+    context 'when the language is found' do
+      let(:language) { 'fr' }
+
+      before do
+        allow(Colore::Utils).to receive(:language_alpha3).with('fr').and_return('fra')
+      end
+
+      it 'returns the correct configuration file path' do
+        expect(path_for).to eq tika_test_config_path.join('ocr', described_class::VERSION, 'tika.fra.xml')
+      end
+    end
+
+    context 'when the language is not found' do
+      let(:language) { 'unknown' }
+
+      it 'returns the default configuration file path' do
+        expect(path_for).to eq tika_test_config_path.join('ocr', described_class::VERSION, "tika.#{described_class::DEFAULT_LANGUAGE}.xml")
+      end
+    end
+
+    context 'when the configuration file is already present' do
+      let(:language) { 'en' }
+
+      before do
+        allow(File).to receive(:write)
+          .with(tika_test_config_path.join('ocr', described_class::VERSION, 'tika.eng.xml'), an_instance_of(String))
+          .and_call_original
+      end
+
+      it 'does not overwrite it' do
+        2.times { described_class.path_for(language) }
+        expect(File).to have_received(:write).once
+      end
+    end
+  end
+end