From a3c5b0371438fc6d9ccbafab9c927351ce9924f8 Mon Sep 17 00:00:00 2001
From: Ryan Rotter <rrotter@users.noreply.github.com>
Date: Thu, 2 Nov 2023 16:45:11 -0400
Subject: [PATCH] create nebula::role::hathitrust::solr::lss

- managed /etc/hosts
- uses new profile nebula::profile::hathitrust::lss
  - mounts nfs for lss
  - handles dependencies and config for lss
  - manage all local (i.e. not on nfs) solr config, including:
    - solr unit file
    - solr.xml
    - log4j.properties
  - lss index release cron
  - manages firewall openings for solr port
  - notable changes from previous solr configs:
    - uses java 11
    - remove many java flags (mostly concerning GC logging) that are unsupported in java 11
    - add After=htsolr-lss.mount to unit file so we don't start solr without NFS
---
 manifests/profile/hathitrust/lss.pp           | 131 ++++++++++++++++++
 manifests/role/hathitrust/solr/lss.pp         |  24 ++++
 spec/classes/profile/hathitrust/lss_spec.rb   |  74 ++++++++++
 spec/classes/role/ht_lss_spec.rb              |  17 +++
 spec/fixtures/hiera/hathitrust.yaml           |  10 ++
 .../solr_lss/index-release-lss.sh.erb         |   2 +-
 6 files changed, 257 insertions(+), 1 deletion(-)
 create mode 100644 manifests/profile/hathitrust/lss.pp
 create mode 100644 manifests/role/hathitrust/solr/lss.pp
 create mode 100644 spec/classes/profile/hathitrust/lss_spec.rb
 create mode 100644 spec/classes/role/ht_lss_spec.rb

diff --git a/manifests/profile/hathitrust/lss.pp b/manifests/profile/hathitrust/lss.pp
new file mode 100644
index 000000000..f50158d2f
--- /dev/null
+++ b/manifests/profile/hathitrust/lss.pp
@@ -0,0 +1,131 @@
+# HathiTrust solr large scale search server
+#
+# @example
+#   include nebula::profile::hathitrust::lss
+class nebula::profile::hathitrust::lss (
+  String $jdk_version = '11',
+  String $solr_home = '/var/lib/solr',
+  String $java_home = '/usr/lib/jvm/default-java',
+  String $heap = '32G',
+  String $timezone = 'America/Detroit',
+  String $port = '8081',
+  String $solr_bin = '/opt/solr/bin/solr',
+  String $snapshot_name = 'htsolr-lss',
+  Boolean $is_primary_site = false,
+  Boolean $is_primary_node = false,
+  String $solr_core,
+  String $mirror_site_ip,
+  String $mail_recipient,
+){
+  package { "openjdk-${jdk_version}-jre-headless": }
+  package { "solr": }
+
+  include nebula::profile::dns::smartconnect;
+
+  # mount solr index, htapps
+  file {
+    default:
+      ensure => "directory",
+      owner  => "root",
+      mode   => "755",
+    ;
+    "/htsolr":;
+    "/htsolr/serve":;
+  }
+  nebula::nfs_mount {
+    default:
+      tag             => "smartconnect",
+      private_network => true,
+      monitored       => true,
+      before          => Service["solr"],
+    ;
+    "/htsolr/lss": remote_target => "nas-${::datacenter}.sc:/ifs/htsolr/lss";
+    "/htapps":     remote_target => "nas-${::datacenter}.sc:/ifs/htapps";
+  }
+  # symlinks into solr nfs mount expected by core configs
+  file {
+    default:
+      ensure => "link",
+      before => Service["solr"],
+    ;
+    "/htsolr/serve/lss-shared":       target => "/htsolr/current_snap/shared";
+    "/htsolr/serve/lss-${solr_core}": target => "/htsolr/current_snap/cores/${solr_core}";
+  }
+
+  include nebula::profile::users
+  realize User["solr"]
+
+  # lss solr conf
+  file {
+    default:
+      ensure => "directory",
+      owner  => "solr",
+      group  => "htprod",
+      mode   => "2775",
+      before => Service["solr"],
+    ;
+    $solr_home:;
+    "${solr_home}/logs":;
+  }
+  file {
+    default:
+      owner  => "root",
+      mode   => "644",
+      notify => Service["solr"],
+    ;
+    "${solr_home}/log4j.properties": content => template("nebula/profile/hathitrust/solr_lss/log4j.properties.erb");
+    "${solr_home}/solr.in.sh":       content => template("nebula/profile/hathitrust/solr_lss/solr.in.sh.erb");
+    "${solr_home}/solr.xml":         content => template("nebula/profile/hathitrust/solr_lss/solr.xml.erb");
+  }
+  # core configs appear to require jars to be available at ../lib; this is otherwise redundant with `/htsolr/serve/lss-shared`
+  file { "${solr_home}/lib":
+    ensure => "link",
+    target => "/htsolr/current_snap/shared/lib",
+    before => Service["solr"],
+  }
+  # core served by _this host_
+  file { "Solr LSS Core":
+    path   => "${solr_home}/${solr_core}",
+    ensure => "link",
+    target => "/htsolr/current_snap/cores/${solr_core}",
+    notify => Service["solr"],
+  }
+
+  # lss service
+  file { "/etc/systemd/system/solr.service":
+    content => template("nebula/profile/hathitrust/solr_lss/solr.service.erb"),
+    notify  => Service["solr"],
+  }
+  service { "solr":
+    ensure  => "running",
+    enable  => true,
+    require => [Package["solr"], File["/etc/systemd/system/solr.service"]],
+  }
+
+  # allow access to solr port servers, staff
+  nebula::exposed_port {
+    default: port => $port;
+    "200 Solr - Private": block => "hathitrust::networks::private_all";
+    "200 Solr - Staff":   block => "hathitrust::networks::staff";
+  }
+
+  # lss release script
+  file { "/usr/local/bin/index-release-lss":
+    owner   => "root",
+    mode    => "755",
+    content => template("nebula/profile/hathitrust/solr_lss/index-release-lss.sh.erb"),
+  }
+  if ($is_primary_site) {
+    cron { "lss solr index release":
+      hour    => 6,
+      minute  => 0,
+      command => "/usr/local/bin/index-release-lss > /tmp/index-release-lss.log 2>&1 || /usr/bin/mail -s '${facts['networking']['hostname']} lss index release problem' ${mail_recipient} < /tmp/index-release-lss.log",
+    }
+  } else {
+    cron { "lss solr index release":
+      hour    => 5,
+      minute  => 55,
+      command => "/usr/local/bin/index-release-lss > /tmp/index-release-lss.log 2>&1 || /usr/bin/mail -s '${facts['networking']['hostname']} lss index release problem' ${mail_recipient} < /tmp/index-release-lss.log",
+    }
+  }
+}
diff --git a/manifests/role/hathitrust/solr/lss.pp b/manifests/role/hathitrust/solr/lss.pp
new file mode 100644
index 000000000..8fa4f2c17
--- /dev/null
+++ b/manifests/role/hathitrust/solr/lss.pp
@@ -0,0 +1,24 @@
+# HathiTrust solr large scale search server
+#
+# @example
+#   include nebula::role::hathitrust::solr::lss
+class nebula::role::hathitrust::solr::lss {
+  include nebula::role::minimum
+
+  include nebula::profile::krb5
+  include nebula::profile::duo
+  include nebula::profile::exim4
+  include nebula::profile::grub
+  include nebula::profile::ntp
+  include nebula::profile::users
+  include nebula::profile::networking
+
+  include nebula::profile::hathitrust::networking
+  include nebula::profile::dns::smartconnect
+  include nebula::profile::hathitrust::hosts
+
+  include nebula::profile::elastic::metricbeat
+  include nebula::profile::elastic::filebeat::prospectors::ulib
+
+  include nebula::profile::hathitrust::lss
+}
diff --git a/spec/classes/profile/hathitrust/lss_spec.rb b/spec/classes/profile/hathitrust/lss_spec.rb
new file mode 100644
index 000000000..a02ffe5ef
--- /dev/null
+++ b/spec/classes/profile/hathitrust/lss_spec.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe 'nebula::profile::hathitrust::lss' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:hiera_config) { 'spec/fixtures/hiera/hathitrust_config.yaml' }
+
+      it { is_expected.to compile }
+      # solr and dependencies
+      it { is_expected.to contain_package('openjdk-11-jre-headless') }
+      it { is_expected.to contain_package('solr') }
+      it { is_expected.to contain_user('solr') }
+      it { is_expected.to contain_service('solr') }
+
+      # solr config
+      it { is_expected.to contain_file('/etc/systemd/system/solr.service').with_content(%r{SOLR_INCLUDE=/s0lr/h0me/solr.in.sh}) }
+      it { is_expected.to contain_file('/s0lr/h0me/log4j.properties').with_content(%r{solr.log=/s0lr/h0me/logs}) }
+      it { is_expected.to contain_file('/s0lr/h0me/solr.xml') }
+      it { is_expected.to contain_file('/s0lr/h0me/solr.in.sh').with_content(/SOLR_PORT=2525/).with_content(%r{SOLR_HOME="/s0lr/h0me"}) }
+      it { is_expected.to contain_file('Solr LSS Core').with(ensure: 'link', path: '/s0lr/h0me/foobar9000', target: '/htsolr/current_snap/cores/foobar9000') }
+      it { is_expected.to contain_file('/s0lr/h0me/lib').with(ensure: 'link', target: '/htsolr/current_snap/shared/lib') }
+
+      it { is_expected.to contain_file('/htsolr/serve/lss-shared').with(ensure: 'link', target: '/htsolr/current_snap/shared') }
+      it { is_expected.to contain_file('/htsolr/serve/lss-foobar9000').with(ensure: 'link', target: '/htsolr/current_snap/cores/foobar9000') }
+
+      # release script
+      it { is_expected.to contain_file('/usr/local/bin/index-release-lss')
+        .with_content(%r{^TARGET=/htsolr/lss/.snapshot/htsolr-lss_\$\{TODAY\}$})
+        .with_content(%r{babel.hathitrust.org:443:5.4.3.2})
+        .with_content(%r{ls \$\{TARGET\}/cores/foobar9000/core-foobar9000x/data/index/\*\.fdt})
+      }
+      it { is_expected.to contain_cron('lss solr index release')
+        .with(command: "/usr/local/bin/index-release-lss > /tmp/index-release-lss.log 2>&1 || /usr/bin/mail -s 'foo lss index release problem' nobody@default.invalid < /tmp/index-release-lss.log")
+      }
+
+      context 'on primary site' do
+        let(:params) do
+          { is_primary_site: true }
+        end
+        it { is_expected.to contain_file('/usr/local/bin/index-release-lss').with_content(%r{^if ! curl -A SOLR -s --retry 5 --fail https://babel.hathitrust.org}) }
+        it { is_expected.to contain_cron('lss solr index release')
+          .with(hour: 6, minute: 0)
+        }
+      end
+      context 'on mirror site' do
+        let(:params) do
+          { is_primary_site: false }
+        end
+        it { is_expected.to contain_file('/usr/local/bin/index-release-lss').with_content(%r{^#if ! curl -A SOLR -s --retry 5 --fail https://babel.hathitrust.org}) }
+        it { is_expected.to contain_cron('lss solr index release')
+          .with(hour: 5, minute: 55)
+        }
+      end
+
+      context 'on primary node' do
+        let(:params) do
+          { is_primary_node: true }
+        end
+        it { is_expected.to contain_file('/usr/local/bin/index-release-lss').with_content(%r{^touch /htapps/babel/flags/web/lss-release-\$\{TODAY\}$}) }
+      end
+      context 'on non-primary node' do
+        let(:params) do
+          { is_primary_node: false }
+        end
+        it { is_expected.to contain_file('/usr/local/bin/index-release-lss').with_content(%r{^#touch /htapps/babel/flags/web/lss-release-\$\{TODAY\}$}) }
+      end
+
+      it { is_expected.to contain_firewall('200 Solr - Private: foobar net').with(source: '192.168.99.0/24') }
+      it { is_expected.to contain_firewall('200 Solr - Staff: Net Two').with(source: '10.0.2.0/24') }
+    end
+  end
+end
diff --git a/spec/classes/role/ht_lss_spec.rb b/spec/classes/role/ht_lss_spec.rb
new file mode 100644
index 000000000..7781e7bc8
--- /dev/null
+++ b/spec/classes/role/ht_lss_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe 'nebula::role::hathitrust::solr::lss' do
+  on_supported_os.each do |os, os_facts|
+    context "on #{os}" do
+      let(:facts) { os_facts }
+      let(:hiera_config) { 'spec/fixtures/hiera/hathitrust_config.yaml' }
+
+      it { is_expected.to compile }
+      it { is_expected.to contain_package('solr') }
+      it { is_expected.to contain_user('solr') }
+      it { is_expected.to contain_service('solr') }
+      it { is_expected.not_to contain_package('openafs-client') }
+    end
+  end
+end
diff --git a/spec/fixtures/hiera/hathitrust.yaml b/spec/fixtures/hiera/hathitrust.yaml
index 1b60da7aa..b668b3618 100644
--- a/spec/fixtures/hiera/hathitrust.yaml
+++ b/spec/fixtures/hiera/hathitrust.yaml
@@ -59,6 +59,10 @@ hathitrust::networks::staff:
   - "%{alias('networks::one')}"
   - "%{alias('networks::two')}"
 
+hathitrust::networks::private_all:
+  - name: 'foobar net'
+    block: '192.168.99.0/24'
+
 nebula::usergroup::membership:
   htprod: ['htingest', 'htweb']
   htingest: ['htingest']
@@ -135,3 +139,9 @@ nebula::profile::hathitrust::secure_rsync::datasets:
   secure_dataset:
     comment: "Secure Dataset"
     path: "/datasets/secure_dataset"
+
+nebula::profile::hathitrust::lss::solr_home: "/s0lr/h0me"
+nebula::profile::hathitrust::lss::solr_core: "foobar9000"
+nebula::profile::hathitrust::lss::port: "2525"
+nebula::profile::hathitrust::lss::mirror_site_ip: "5.4.3.2"
+nebula::profile::hathitrust::lss::mail_recipient: nobody@default.invalid
diff --git a/templates/profile/hathitrust/solr_lss/index-release-lss.sh.erb b/templates/profile/hathitrust/solr_lss/index-release-lss.sh.erb
index 6c7e0a689..bead9dade 100755
--- a/templates/profile/hathitrust/solr_lss/index-release-lss.sh.erb
+++ b/templates/profile/hathitrust/solr_lss/index-release-lss.sh.erb
@@ -64,7 +64,7 @@ fi
 # run the first query to initialize solr
 INITQUERY=0
 if ! wget -q -T 60 -t 5 -O - "http://solr-sdr-search-<%= @solr_core %>:8081/solr/core-<%= @solr_core %>x/select/?q=aardvark&version=2.2&start=0&rows=10&indent=on" >& /dev/null; then
-  /bin/echo "WARN: failed to perform initial query against core <%= @solr_core %> after successful release"
+  /bin/echo "WARN: failed to perform initial query against core <%= @solr_core %>x after successful release"
   INITQUERY=1
 fi
 if ! wget -q -T 60 -t 5 -O - "http://solr-sdr-search-<%= @solr_core %>:8081/solr/core-<%= @solr_core %>y/select?indent=on&q=*:*&rows=0" >& /dev/null; then