Improve build into master

.dockerignore

@@ -0,0 +1,10 @@
+/solr-*
+/unpacked-war/
+/build/
+/specify_exports/*
+!/specify_exports/README
+
+/.lastupdate
+/example.crontab
+
+.DS_Store

.gitignore

@@ -5,4 +5,6 @@
 !/specify_exports/README
 
 /.lastupdate
-/example.crontab
+/example.crontab
+
+.DS_Store

Dockerfile

@@ -0,0 +1,60 @@
+# Build it like this:
+# docker build --tag webportal-service:improve-build .
+
+# Run it like this:
+# docker run -p 80:80 -v /absolute/location/of/your/export.zip:/home/specify/webportal-installer/specify_exports/export.zip webportal-service:improve-build
+
+FROM ubuntu:18.04
+
+LABEL maintainer="Specify Collections Consortium <github.com/specify>"
+
+# Get Ubuntu packages
+RUN apt-get update && apt-get -y install \
+        nginx \
+        unzip \
+        curl \
+        wget \
+        python \
+        python-lxml \
+        make \
+        lsof \
+        default-jre \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create a user and group for the application
+RUN groupadd -g 999 specify && \
+    useradd -r -u 999 -g specify specify
+
+# Create the application directory and set ownership
+RUN mkdir -p /home/specify/webportal-installer && chown specify:specify -R /home/specify
+
+# Switch to the specify user
+USER specify
+
+# Copy the application files into the container
+COPY --chown=specify:specify . /home/specify/webportal-installer
+WORKDIR /home/specify/webportal-installer
+
+# Expose the port for the web portal
+EXPOSE 80
+
+# Switch back to root user for further configuration
+USER root
+
+# Configure nginx to proxy the Solr requests and serve the static files
+COPY webportal-nginx.conf /etc/nginx/sites-available/webportal-nginx.conf
+
+# Disable the default nginx site and enable the portal site
+RUN rm /etc/nginx/sites-enabled/default \
+        && ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \
+        && service nginx stop
+
+# Redirect nginx logs to stdout and stderr
+RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log
+
+# Build the Solr app
+# Run Solr in foreground
+# Wait for Solr to load
+# Import data from the .zip file
+# Run Docker in foreground
+CMD ["sh", "-c", "make clean-all && make build-all && ./build/bin/solr start -force && sleep 20 && curl -v \"http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\\\"&escape=\\&header=true\" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' && nginx -g 'daemon off;'"]

Makefile

@@ -1,32 +1,218 @@
 # Mirror for downloading Apache Solr.
-SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr
+SOLR_MIRROR := https://archive.apache.org/dist/lucene/solr
+
+# Use 'schema.xml' if solr will be used to create the core
+# Use 'managed-schema' if pre-configuring core
+SCHEMA_FILE := managed-schema
+
+#location of default settings files in solr dist
+DEFAULT_SETS := server/solr/configsets/_default
+
+PYTHON := python2
 
 # Use latest available version of Solr 4.
-export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr_vers.py)
+SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py)
 
-export SOLR_DIST := solr-$(SOLR_VERSION)
-export TOPDIR := $(shell pwd)
+SOLR_DIST := solr-$(SOLR_VERSION)
 
-all: build
+INPUTS := $(wildcard specify_exports/*.zip)
+COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS))
+PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles)
+WEBAPPS := $(addprefix build/html/, $(COLLECTIONS))
+SETTING_TEMPLATES := $(addprefix build/setting_templates/, $(COLLECTIONS))
+SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS))
 
-clean:
-	rm -rf build/
+###### Usage Information #####
 
-realclean: clean
-	rm -rf solr-* unpacked-war
+.PHONY: usage
+usage:
+	@echo Usage:
+	@echo
+	@echo make build-all    -- Same as build-html build-cores.
+	@echo make build-html   -- Builds the web app part of the web portal.
+	@echo make build-cores  -- Builds the solr cores for all collections.
+	@echo make load-data    -- Loads the collection data into solr.
+	@echo make load-data-COLLECTION  -- Loads the data for COLLECTION into solr.
+	@echo make force-load-data    -- Loads the collection data into solr even if up-to-date.
+	@echo make force-load-data-COLLECTION  -- Loads the data for COLLECTION into solr even if up-to-date.
+	@echo
+	@echo The build-all and build-cores targets should only be used when solr is
+	@echo not running. The build-html and load-data* targets maybe used freely.
+	@echo All build and load targets will only have effect if there are changes in
+	@echo the specify_exports or custom_settings directories.
 
-build: $(SOLR_DIST) build.make specify_exports specify_exports/*.zip
-	cp -r $(SOLR_DIST)/ build
-	$(MAKE) -f $(TOPDIR)/build.make -C build
-	touch $@
+##### Main targets #####
+
+.PHONY: build-all build-cores build-html build-setting-templates load-data
+build-all: build-cores build-html build-setting-templates
+build-cores: $(SOLR_CORES)
+build-html: $(WEBAPPS) build/html/index.html
+build-setting-templates: $(SETTING_TEMPLATES)
+load-data: $(addprefix load-data-, $(COLLECTIONS))
+force-load-data: $(addprefix force-load-data-, $(COLLECTIONS))
+
+##### Cleaning targets #####
+
+.PHONY: clean-all clean-build clean-solr clean-cores clean-html clean-*-html clean-*-core clean-setting-templates
+
+clean-all: clean-build clean-solr
+
+clean-build:
+	rm -rf build
+
+clean-solr:
+	rm -rf solr-*
+
+clean-cores:
+	rm -rf $(SOLR_CORES)
+
+clean-html:
+	rm -rf build/html
+
+clean-%-html:
+	rm -rf build/html/$*
+
+clean-%-core:
+	rm -rf build/server/solr/$*
+
+clean-setting-templates:
+	rm -rf build/setting-templates
+
+##### Common building steps #####
 
 $(SOLR_DIST).tgz:
-	# Fetching Solr distribution tar ball.
+	@printf "\n\n### Fetching Solr distribution tar ball.\n\n"
 	wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@
 
 $(SOLR_DIST): $(SOLR_DIST).tgz
-	# Unpacking Solr distribution.
+	@printf "\n\n### Unpacking Solr distribution.\n\n"
 	rm -rf $@
 	tar -zxf $<
+	touch $@
+
+build: $(SOLR_DIST)
+	@printf "\n\n### Copying solr to build directory.\n\n"
+	cp -r $(SOLR_DIST)/ build
+	$(PYTHON) patch_web_xml.py \
+		$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml\
+		> $@/server/solr-webapp/webapp/WEB-INF/web.xml
+
+build/col: | build
+	@printf "\n\n"
+	mkdir build/col
+
+.PRECIOUS: build/col/%/PortalFiles
+build/col/%/PortalFiles: specify_exports/%.zip | build/col
+	@printf "\n\n### Extracting $@.\n\n"
+	unzip -DD -qq -o -d build/col/$* $^
+
+##### Solr core building #####
+
+build/server/solr/%: build/col/%/SolrFldSchema.xml | build
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@/conf $@/data
 
-$(SOLR_DIST)/%: $(SOLR_DIST)
+	$(PYTHON) patch_solrconfig_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml \
+		> $@/conf/solrconfig.xml
+
+	$(PYTHON) patch_schema_xml.py \
+		$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \
+		build/col/$*/SolrFldSchema.xml \
+		> $@/conf/$(SCHEMA_FILE)
+
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt $@/conf/
+	cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt $@/conf/
+	cp -r $(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ $@/conf/
+
+	echo 'dataDir=data' > $@/core.properties
+	echo 'name=$*' >> $@/core.properties
+	echo 'config=conf/solrconfig.xml' >> $@/core.properties
+
+	touch $@
+
+build/col/%/SolrFldSchema.xml: build/col/%/PortalFiles
+	@printf "\n\n### Generating $@.\n\n"
+	echo '<?xml version="1.0" encoding="UTF-8" ?>' > $@
+	echo "<fields>" >> $@
+	cat $</SolrFldSchema.xml >> $@
+	echo "</fields>" >> $@
+
+##### Website building #####
+
+build/html: | build
+	@printf "\n\n"
+	mkdir build/html
+
+build/html/index.html: $(WEBAPPS) | build/html
+	@printf "\n\n### Generating $@.\n\n"
+	$(PYTHON) make_toplevel_index.py index_skel.html \
+		$(addsuffix /resources/config/settings.json, $(WEBAPPS)) \
+		> $@
+
+build/html/%: build/col/%/PortalFiles custom_settings/%/fldmodel.json custom_settings/%/settings.json | build/html
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	cp -r PortalApp/* $@
+	$(PYTHON) make_fldmodel_json.py \
+		build/col/$*/PortalFiles/*flds.json \
+		custom_settings/$*/fldmodel.json \
+		> $@/resources/config/fldmodel.json
+	$(PYTHON) patch_settings_json.py \
+		PortalApp/resources/config/settings.json \
+		custom_settings/$*/settings.json \
+		$* \
+		build/col/$*/PortalFiles/*Setting.json \
+		> $@/resources/config/settings.json
+	touch $@
+
+custom_settings/%/fldmodel.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
+custom_settings/%/settings.json:
+	@printf "\n\n### Generating empty $@.\n\n"
+	mkdir -p custom_settings/$*
+	touch $@
+
+##### Loading data #####
+
+.PHONY: load-data-%
+load-data-%: build/html/%/load-timestamp.txt ;
+
+.PRECIOUS: build/html/%/load-timestamp.txt
+build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/%
+	@printf "\n\n### Loading data into $*.\n\n"
+	curl -X POST "http://localhost:8983/solr/$*/update" \
+		-d '{ "delete": {"query":"*:*"} }' \
+		-H 'Content-Type: application/json' \
+		| grep '"status":0'
+	curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \
+		--data-binary @build/col/$*/PortalFiles/PortalData.csv \
+		-H 'Content-type:application/csv' \
+		| grep '"status":0'
+	date > $@
+
+.PHONY: force-load-data-%
+force-load-data-%:
+	rm -f build/html/$*/load-timestamp.txt
+	$(MAKE) build/html/$*/load-timestamp.txt
+
+##### Settings templates #####
+
+build/setting_templates: | build
+	@printf "\n\n"
+	mkdir $@
+
+build/setting_templates/%: build/html/% | build/setting_templates
+	@printf "\n\n### Generating $@.\n\n"
+	mkdir -p $@
+	$(PYTHON) make_settings_template.py \
+		PortalApp/resources/config/settings.json \
+		> $@/settings.json
+	$(PYTHON) make_fields_template.py \
+		build/html/$*/resources/config/fldmodel.json \
+		> $@/fldmodel.json
+	touch $@

README.md

@@ -1,9 +1,13 @@
 Specify Web Portal (Version 2.0)
 ================================
 
-The Specify Collections Consortium is funded by its member institutions. The Consortium web site is: [http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/)
+The Specify Collections Consortium is funded by its member
+institutions. The Consortium web site is:
+[http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/)
 
-Specify Web Portal Copyright © 2020 Specify Collections Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free software licensed under GNU General Public License 2 (GPL2).
+Specify Web Portal Copyright © 2020 Specify Collections
+Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free
+software licensed under GNU General Public License 2 (GPL2).
 
 ```
 Specify Collections Consortium
@@ -13,11 +17,22 @@ University of Kansas
 Lawrence, KS 66045 USA
 ```
 
+## Configuration Instructions
+
+Instructions on customizing the Web Portal configuration are available on the Specify Communtiy Forum:
+
+[Web Portal Configuration Instructions](https://discourse.specifysoftware.org/t/web-portal-configuration-instructions/1144)
+
 ## Developer Instructions
 
-After completing these instructions you will be able to run Specify Web Portal 2.0.
+After completing these instructions you will be able to run Specify
+Web Portal 2.0.
 
-These instructions are for deployment on a server running Ubuntu. An export file for a single collection is required for setting up the Specify Web Portal. This can be accomplished using the Schema Mappging tool tool within the Specify 6 application together with the stand alone Specify Data Export tool.
+These instructions are for deployment on a server running Ubuntu. An
+export file for a single collection is required for setting up the
+Specify Web Portal. This can be accomplished using the Schema Mappging
+tool tool within the Specify 6 application together with the stand
+alone Specify Data Export tool.
 
 Install system dependencies
 ------------
@@ -35,137 +50,194 @@ Installation Instructions
 -------------------------
 
 These instructions illustrate the fewest steps needed to install the
-web portal. For updating existing installations, se the 'Data Only Updates' section below.
-
-1. Clone the Web Portal 2.0 repository by clicking the green button (Clone or Download) at the top of the page and unpack this repository on your server.
-
-   This will install Solr on the server.
+web portal. 
+
+1. These instructions assume the Web Portal is being setup under a
+   user account with the name `specify`. If a different account is
+   used, the directory paths in these instructions will need to be
+   updated accordingly.
+
+   You will also need to update the `webportal-solr.service` file to use
+   the correct user and group.
+
+1. Clone the Web Portal 2.0 repository using git:
+
+    ```console
+    specify@wp:~$ git clone https://github.com/specify/webportal-installer.git
+    ```
+    
+    This will create the directory `webportal-installer`.
+    
+1. Configure nginx to proxy the Solr requests and serve the static
+   files by copying the provided `webportal-nginx.conf` to
+   `/etc/nginx/sites-available/`:
+   
+   ```console
+   root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-nginx.conf /etc/nginx/sites-available/
+   ```
+
+   N.B. this file will require changes if the `webportal-installer` is
+   in a different location.
+
+1. Disable the default nginx site and enable the portal site:
+   ```console
+   root@wp:~# rm /etc/nginx/sites-enabled/default
+   root@wp:~# ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/
+   root@wp:~# systemctl restart nginx
+   ```
 
 2. Use the Specify Data Export tool to create a Web Portal export zip
-   file (see the Specify 6 Data Export documentation) for each collection
-   to be hosted in the portal. If aggregated collections are desired, replace the single colleciton with the aggregated collections file after the initial Web Portal installation.
-
-3. Create a  `specify_exports` directory in the web portal directory and copy the zip files (step 2) into it. The copied files should be given names that are suitable for use in URLs; so no spaces, capital letters, slashes or
-   other problematic characters. E.g. `kufish.zip`
-
-4. Build the Solr app: `make clean && make`.
-
-5. Copy the Solr core to the Solr installation:
-
-   [CORENAME] - the name of the exported archive without the file extension. (E.g. `kufish`)
-
-   [SOLRVERSION] - the version of Solr. (E.g. `7.5.0`)
-
-```
-    mkdir solr-[SOLRVERSION]/server/solr/[CORENAME]
-    cp -r build/cores/[CORENAME]/core/* solr-[SOLRVERSION]/server/solr/[CORENAME]
-    cp build/cores/[CORENAME]/web.xml solr-[SOLRVERSION]/server/solr-webapp/webapp/WEB-INF/web.xml 
-    # Only necessary for the first core.
-```
-6. Restrict access to the solr admin web page. This can be done in solr 7.5 by editing solr:
-
-   `[SOLRVERSION]/server/etc/jetty-http.xml`
-
-   In the ServerConnector section replace: 
-
-   `<Set name="host"><Property name="jetty.host" /></Set>` with `<Set name="host">127.0.0.1</Set>`
-
-7. Start solr
-   `solr-[SOLRVERSION]/bin/solr start`
-
-   When completing this step the following warnings may be issued and can be safely ignored:
-
-   `*** [WARN] *** Your open file limit is currently 1024.`
-
-    `It should be set to 65000 to avoid operational disruption.` 
-    `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to  false in your profile or solr.in.sh` 
-   `*** [WARN] *** Your Max Processes Limit is currently 63590.` 
-    `It should be set to 65000 to avoid operational disruption.` 
-    `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to  false in your profile or solr.in.sh` 
-   `Waiting up to 180 seconds to see Solr running on port 8983 [|]` 
-   `Started Solr server on port 8983 (pid=15422). Happy searching!`
-
-8. Import the csv data:
-  `curl 'http://localhost:8983/solr/[CORENAME]/update/csv?commit=true&encapsulator="&escape=\&header=true' --data-binary @build/cores/[CORENAME]/PortalFiles/PortalData.csv -H 'Content-type:application/csv'`
-
-  When completing this step you may receive output similar to the following: 
-
-  `{`
-
-  `"responseHeader":{`
-
-  `"status":0,`
-
-  `"QTime":1153}}`
-
-9. Move the built webportal to convenient location:
-
-   [WPPATH] - the directory where the web portal fiels reside. (E.g. /home/specify/webportal)
-
-    `mv build [WPPATH]`
-
-10. Create the Nginx configuration file to serve the portal:
-
-     `sudo nano /etc/nginx/sites-available/webportal.conf`
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be hosted in the portal. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
+
+3. Copy the zip files from the Specify Data Export into the
+   `webportal-installer/specify_exports` directory. The copied files
+   should be given names that are suitable for use in URLs; so no
+   spaces, capital letters, slashes or other problematic
+   characters. E.g. `kufish.zip`
+
+4. Build the Solr app: 
+   ```console
+   specify@wp:~/webportal-installer$ make clean-all && make build-all
    ```
-   server {
-       listen 80 default_server;
-
-       rewrite ^/([^/]+)/select /solr/$1/select;
 
-       location ~ ^/solr/([^/]+)/select {
-         proxy_pass http://localhost:8983;
-       }
-
-       location / {
-                root [WPPATH]/html;
-        }
-   }
-   ```
-11. Remove the default Nginx site and enable the portal site:
-   ```
-   sudo rm /etc/nginx/sites-enabled/default
-   sudo ln -L /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/
+5. Copy the provided `webportal-solr.service` systemd unit file to
+   `/etc/systemd/system/webportal-solr.service` and reload the
+   systemd daemon.
+
+   ```console
+   root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-solr.service /etc/systemd/system/
+   root@wp:~# systemctl daemon-reload
    ```
-12. Restart Nginx: `sudo systemctl restart nginx`
+   N.B. this file will require changes if the `webportal-installer` is
+   in a different location.
 
-    The Web Portal is now running and can be tested.
+6. Start and enable the service:
 
-    The Web Portal can be customized by editing the settings in the settings.json file found at here: [WPPATH]/html/corename/resources/config/settings.json 
+    ```console
+    root@wp:~# systemctl start webportal-solr.service
+    root@wp:~# systemctl enable webportal-solr.service
+    ```
+    
+7. Load the Specify data into Solr:
 
+   ```console
+   specify@wp:~/webportal-installer$ make load-data
+   ```
+
+   When completing this step you will receive output consisting of
+   blocks of JSON. This is normal unless one of the blocks includes an
+   error message.
+
+8. The Web Portal is now running and can be tested by visiting the
+   server's address with a web browser.
+
+9. Be aware that the Solr admin page is available by default on
+   port 8983. This can be useful for troubleshooting but should be
+   blocked in production using firewall rules such as the following:
+
+    ```console
+    root@wp:~# ufw default deny incoming
+    root@wp:~# ufw default allow outgoing
+    root@wp:~# ufw allow ssh
+    root@wp:~# ufw allow http
+    root@wp:~# ufw --force enable
+    ```
+
+Customization
+-------------
+
+Certain customizations of the webportal are possible such as
+background images. To enable customizations, after the webportal is
+built with `make build-all`, copy the folders inside
+`webportal-installer/build/setting_templates/` into
+`webportal-installer/custom_settings/` and edit the files in each
+folder as desired. Afterwards, `make build-html` will incorporate the
+customizations into the portal without restarting any services.
+
+The provided `webportal-nginx.conf` serves the directory
+`/home/specify/custom-images/` at `/custom-images/` allowing files
+place there to be used in customization.
+
+Data-only Updates
+--------
+
+The portal can be updated with new data from exported collections
+without restarting the Solr service if the export mappings are the
+same.
+
+To update the Specify data in the webportal follow these steps.
 
-Data Only Updates
------------------
-
-If the fields used in a portal are unchanged and only data is being updated, delete the current contents of the solr core with:
 
-`curl 'http://localhost:8983/solr/hollow/update?commit=true&stream.body=<delete><query>*%3A*</query></delete>'`
-(You will probably need to add/edit a requestParsers block in the `solr-[SOLRVERSION]/server/solr/[CORENAME]/conf/solrconfig.xml` file for the core. Add it to the requestDispatcher block:
+2. Use the Specify Data Export tool to create a Web Portal export zip
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be updated. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
 
-```
-<requestDispatcher>
-    <requestParsers enableRemoteStreaming="true"
-                enableStreamBody="true"
-                multipartUploadLimitInKB="2048"
-                formdataUploadLimitInKB="2048"
-                addHttpRequestToContext="false" />
 
+3. Copy the zip files from the Specify Data Export into the
+   `webportal-installer/specify_exports` directory. The copied files
+   in the `specify_exports` directory must have the same names as the
+   original exports.
+   
+7. Load the Specify data into Solr:
 
-    <httpCaching never304="true" />
-</requestDispatcher>
-```
-(See http://lucene.apache.org/solr/guide/requestdispatcher-in-solrconfig.html)
+    ```console
+    specify@wp:~/webportal-installer$ make load-data
+    ```
+    
+8. Only export files newer than the last update will be loaded. The
+   target `make force-load-data` will force all collections to be
+   reloaded.
 
-Then use the curl CSV import command above to add the new data.
+Full Updates
+--------
 
+If new collections are to be added to the portal or mappings of
+existing collections change, the Solr service has to be stopped and
+the cores rebuilt.
 
-Schema Definition Updates
--------------------------
 
-In this case, you will need to stop Solr `(solr-[SOLRVERSION]/bin/solr stop)`, remove the cores to be updated from your Solr server directory, and follow all the installation steps besides the HTTP configuration.
+1. Stop the webportal-solr systemd service:
 
+    ```console
+    root@wp:~# systemctl stop webportal-solr.service
+    ```
 
-Web Portal Application Updates
-------------------------------
+2. Use the Specify Data Export tool to create a Web Portal export zip
+   file (see the Specify 6 Data Export documentation) for each
+   collection to be hosted in the portal. If aggregated collections
+   are desired, replace the single colleciton with the aggregated
+   collections file after the initial Web Portal installation.
+
+
+3. Copy the zip files from the Specify Data Export into the
+   `webportal-installer/specify_exports` directory. The copied files
+   should be given names that are suitable for use in URLs; so no
+   spaces, capital letters, slashes or other problematic
+   characters. E.g. `kufish.zip`
+   
+4. Build the Solr app: 
+
+    ```console
+    specify@wp:~/webportal-installer$ make build-all
+    ```
+
+6. Start the webportal-solr service:
+
+    ```console
+    root@wp:~# systemctl start webportal-solr.service
+    ```
+    
+7. Load the Specify data into Solr:
+
+    ```console
+    specify@wp:~/webportal-installer$ make load-data
+    ```
+    
+8. The Web Portal is now running and can be tested by visiting the
+   server's address with a web browser.
 
-If a new version of the Web Portal is being installed it will be necessary to perform step 8 after building.

index_skel.html

@@ -3,15 +3,41 @@
   <head>
     <title>Specify Web Portal</title>
     <meta name="robots" content="noindex" />
+    <style>
+      body {
+          font-size: 100%;
+          font-family: sans-serif;
+      }
+      #collections span {
+          font-size: 66%;
+          margin-left: 2em;
+      }
+      #collections span::before {
+          content: "last update: "
+      }
+    </style>
   </head>
   <body>
     <h2>The following collections are available:</h2>
     <ul id="collections">
       <li>Nothing here</li>
     </ul>
-    <p>
-      Last update:
-      <span id="update-time">N/A</span>
-    </p>
+    <script type="text/javascript">
+      document.querySelectorAll('#collections li').forEach(function(li) {
+          var a = li.querySelector('a');
+          if (a == null) return;
+
+          var xhr = new XMLHttpRequest();
+          xhr.open('GET', a.getAttribute('href') + '/load-timestamp.txt');
+          xhr.onload = function() {
+              if (xhr.status === 200) {
+                  var update = document.createElement('span');
+                  update.textContent = xhr.responseText;
+                  li.append(update);
+              }
+          };
+          xhr.send();
+      });
+    </script>
   </body>
 </html>

make_fldmodel_json.py

@@ -27,7 +27,11 @@
 
 if os.path.isfile(sys.argv[2]):
     with open(sys.argv[2]) as f:
-        custom = json.load(f, object_pairs_hook=OrderedDict)
+        data = f.read().strip()
+        if data:
+            custom = json.loads(data, object_pairs_hook=OrderedDict)
+        else:
+            custom = []
 else:
     custom = []
 

make_toplevel_index.py

@@ -50,15 +50,12 @@ def splitall(path):
     with open(settings_file) as f:
         settings = json.load(f)
 
-    core_dir = splitall(settings_file)[1]
+    core_dir = splitall(settings_file)[2]
     core_name = settings[0]['collectionName'] or core_dir
 
     li = ElementTree.SubElement(collections, 'li')
     a = ElementTree.SubElement(li, 'a')
     a.set('href', core_dir)
     a.text = core_name
 
-update_time = skel.find('.//span[@id="update-time"]')
-update_time.text = datetime.now().isoformat()
-
 skel.write(sys.stdout)

patch_settings_json.py

@@ -27,7 +27,11 @@
 
 if os.path.isfile(sys.argv[2]):
     with open(sys.argv[2]) as f:
-        custom_settings = json.load(f)
+        data = f.read().strip()
+        if data:
+            custom_settings = json.loads(data)
+        else:
+            custom_settings = {}
 else:
     custom_settings = {}
 

webportal-nginx.conf

@@ -0,0 +1,17 @@
+server {
+    listen 80 default_server;
+
+    rewrite ^/([^/]+)/select /solr/$1/select;
+
+    location ~ ^/solr/([^/]+)/select {
+      proxy_pass http://localhost:8983;
+    }
+
+    location / {
+      root /home/specify/webportal-installer/build/html;
+    }
+
+    location /custom-images/ {
+      alias /home/specify/custom-images/;
+    }
+}

webportal-solr.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Solr server for Specify Web Portal
+After=network.target
+
+[Service]
+User=specify
+Group=specify
+PIDFile=/home/specify/webportal-installer/build/bin/solr-8983.pid
+ExecStart=/home/specify/webportal-installer/build/bin/solr start
+ExecStop=/home/specify/webportal-installer/build/bin/solr stop
+LimitNOFILE=65000
+LimitNPROC=65000
+Restart=always
+
+[Install]
+WantedBy=multi-user.target