diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6d5988d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +/solr-* +/unpacked-war/ +/build/ +/specify_exports/* +!/specify_exports/README + +/.lastupdate +/example.crontab + +.DS_Store diff --git a/.gitignore b/.gitignore index b880781..6d5988d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ !/specify_exports/README /.lastupdate -/example.crontab \ No newline at end of file +/example.crontab + +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f2039e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# Build it like this: +# docker build --tag webportal-service:improve-build . + +# Run it like this: +# docker run -p 80:80 -v /absolute/location/of/your/export.zip:/home/specify/webportal-installer/specify_exports/export.zip webportal-service:improve-build + +FROM ubuntu:18.04 + +LABEL maintainer="Specify Collections Consortium " + +# Get Ubuntu packages +RUN apt-get update && apt-get -y install \ + nginx \ + unzip \ + curl \ + wget \ + python \ + python-lxml \ + make \ + lsof \ + default-jre \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Create a user and group for the application +RUN groupadd -g 999 specify && \ + useradd -r -u 999 -g specify specify + +# Create the application directory and set ownership +RUN mkdir -p /home/specify/webportal-installer && chown specify:specify -R /home/specify + +# Switch to the specify user +USER specify + +# Copy the application files into the container +COPY --chown=specify:specify . /home/specify/webportal-installer +WORKDIR /home/specify/webportal-installer + +# Expose the port for the web portal +EXPOSE 80 + +# Switch back to root user for further configuration +USER root + +# Configure nginx to proxy the Solr requests and serve the static files +COPY webportal-nginx.conf /etc/nginx/sites-available/webportal-nginx.conf + +# Disable the default nginx site and enable the portal site +RUN rm /etc/nginx/sites-enabled/default \ + && ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ \ + && service nginx stop + +# Redirect nginx logs to stdout and stderr +RUN ln -sf /dev/stderr /var/log/nginx/error.log && ln -sf /dev/stdout /var/log/nginx/access.log + +# Build the Solr app +# Run Solr in foreground +# Wait for Solr to load +# Import data from the .zip file +# Run Docker in foreground +CMD ["sh", "-c", "make clean-all && make build-all && ./build/bin/solr start -force && sleep 20 && curl -v \"http://localhost:8983/solr/export/update/csv?commit=true&encapsulator=\\\"&escape=\\&header=true\" --data-binary @./build/col/export/PortalFiles/PortalData.csv -H 'Content-type:application/csv' && nginx -g 'daemon off;'"] \ No newline at end of file diff --git a/Makefile b/Makefile index a95b9ec..a374f01 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,218 @@ # Mirror for downloading Apache Solr. -SOLR_MIRROR := http://archive.apache.org/dist/lucene/solr +SOLR_MIRROR := https://archive.apache.org/dist/lucene/solr + +# Use 'schema.xml' if solr will be used to create the core +# Use 'managed-schema' if pre-configuring core +SCHEMA_FILE := managed-schema + +#location of default settings files in solr dist +DEFAULT_SETS := server/solr/configsets/_default + +PYTHON := python2 # Use latest available version of Solr 4. -export SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | python2 get_latest_solr_vers.py) +SOLR_VERSION := $(shell curl -s $(SOLR_MIRROR)/ | $(PYTHON) get_latest_solr_vers.py) -export SOLR_DIST := solr-$(SOLR_VERSION) -export TOPDIR := $(shell pwd) +SOLR_DIST := solr-$(SOLR_VERSION) -all: build +INPUTS := $(wildcard specify_exports/*.zip) +COLLECTIONS := $(patsubst specify_exports/%.zip, %, $(INPUTS)) +PORTALFILES := $(foreach c, $(COLLECTIONS), build/col/$c/PortalFiles) +WEBAPPS := $(addprefix build/html/, $(COLLECTIONS)) +SETTING_TEMPLATES := $(addprefix build/setting_templates/, $(COLLECTIONS)) +SOLR_CORES := $(addprefix build/server/solr/, $(COLLECTIONS)) -clean: - rm -rf build/ +###### Usage Information ##### -realclean: clean - rm -rf solr-* unpacked-war +.PHONY: usage +usage: + @echo Usage: + @echo + @echo make build-all -- Same as build-html build-cores. + @echo make build-html -- Builds the web app part of the web portal. + @echo make build-cores -- Builds the solr cores for all collections. + @echo make load-data -- Loads the collection data into solr. + @echo make load-data-COLLECTION -- Loads the data for COLLECTION into solr. + @echo make force-load-data -- Loads the collection data into solr even if up-to-date. + @echo make force-load-data-COLLECTION -- Loads the data for COLLECTION into solr even if up-to-date. + @echo + @echo The build-all and build-cores targets should only be used when solr is + @echo not running. The build-html and load-data* targets maybe used freely. + @echo All build and load targets will only have effect if there are changes in + @echo the specify_exports or custom_settings directories. -build: $(SOLR_DIST) build.make specify_exports specify_exports/*.zip - cp -r $(SOLR_DIST)/ build - $(MAKE) -f $(TOPDIR)/build.make -C build - touch $@ +##### Main targets ##### + +.PHONY: build-all build-cores build-html build-setting-templates load-data +build-all: build-cores build-html build-setting-templates +build-cores: $(SOLR_CORES) +build-html: $(WEBAPPS) build/html/index.html +build-setting-templates: $(SETTING_TEMPLATES) +load-data: $(addprefix load-data-, $(COLLECTIONS)) +force-load-data: $(addprefix force-load-data-, $(COLLECTIONS)) + +##### Cleaning targets ##### + +.PHONY: clean-all clean-build clean-solr clean-cores clean-html clean-*-html clean-*-core clean-setting-templates + +clean-all: clean-build clean-solr + +clean-build: + rm -rf build + +clean-solr: + rm -rf solr-* + +clean-cores: + rm -rf $(SOLR_CORES) + +clean-html: + rm -rf build/html + +clean-%-html: + rm -rf build/html/$* + +clean-%-core: + rm -rf build/server/solr/$* + +clean-setting-templates: + rm -rf build/setting-templates + +##### Common building steps ##### $(SOLR_DIST).tgz: - # Fetching Solr distribution tar ball. + @printf "\n\n### Fetching Solr distribution tar ball.\n\n" wget $(SOLR_MIRROR)/$(SOLR_VERSION)/$@ $(SOLR_DIST): $(SOLR_DIST).tgz - # Unpacking Solr distribution. + @printf "\n\n### Unpacking Solr distribution.\n\n" rm -rf $@ tar -zxf $< + touch $@ + +build: $(SOLR_DIST) + @printf "\n\n### Copying solr to build directory.\n\n" + cp -r $(SOLR_DIST)/ build + $(PYTHON) patch_web_xml.py \ + $(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml\ + > $@/server/solr-webapp/webapp/WEB-INF/web.xml + +build/col: | build + @printf "\n\n" + mkdir build/col + +.PRECIOUS: build/col/%/PortalFiles +build/col/%/PortalFiles: specify_exports/%.zip | build/col + @printf "\n\n### Extracting $@.\n\n" + unzip -DD -qq -o -d build/col/$* $^ + +##### Solr core building ##### + +build/server/solr/%: build/col/%/SolrFldSchema.xml | build + @printf "\n\n### Generating $@.\n\n" + mkdir -p $@/conf $@/data -$(SOLR_DIST)/%: $(SOLR_DIST) + $(PYTHON) patch_solrconfig_xml.py \ + $(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml \ + > $@/conf/solrconfig.xml + + $(PYTHON) patch_schema_xml.py \ + $(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \ + build/col/$*/SolrFldSchema.xml \ + > $@/conf/$(SCHEMA_FILE) + + cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt $@/conf/ + cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt $@/conf/ + cp $(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt $@/conf/ + cp -r $(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ $@/conf/ + + echo 'dataDir=data' > $@/core.properties + echo 'name=$*' >> $@/core.properties + echo 'config=conf/solrconfig.xml' >> $@/core.properties + + touch $@ + +build/col/%/SolrFldSchema.xml: build/col/%/PortalFiles + @printf "\n\n### Generating $@.\n\n" + echo '' > $@ + echo "" >> $@ + cat $> $@ + echo "" >> $@ + +##### Website building ##### + +build/html: | build + @printf "\n\n" + mkdir build/html + +build/html/index.html: $(WEBAPPS) | build/html + @printf "\n\n### Generating $@.\n\n" + $(PYTHON) make_toplevel_index.py index_skel.html \ + $(addsuffix /resources/config/settings.json, $(WEBAPPS)) \ + > $@ + +build/html/%: build/col/%/PortalFiles custom_settings/%/fldmodel.json custom_settings/%/settings.json | build/html + @printf "\n\n### Generating $@.\n\n" + mkdir -p $@ + cp -r PortalApp/* $@ + $(PYTHON) make_fldmodel_json.py \ + build/col/$*/PortalFiles/*flds.json \ + custom_settings/$*/fldmodel.json \ + > $@/resources/config/fldmodel.json + $(PYTHON) patch_settings_json.py \ + PortalApp/resources/config/settings.json \ + custom_settings/$*/settings.json \ + $* \ + build/col/$*/PortalFiles/*Setting.json \ + > $@/resources/config/settings.json + touch $@ + +custom_settings/%/fldmodel.json: + @printf "\n\n### Generating empty $@.\n\n" + mkdir -p custom_settings/$* + touch $@ + +custom_settings/%/settings.json: + @printf "\n\n### Generating empty $@.\n\n" + mkdir -p custom_settings/$* + touch $@ + +##### Loading data ##### + +.PHONY: load-data-% +load-data-%: build/html/%/load-timestamp.txt ; + +.PRECIOUS: build/html/%/load-timestamp.txt +build/html/%/load-timestamp.txt: build/col/%/PortalFiles | build/html/% + @printf "\n\n### Loading data into $*.\n\n" + curl -X POST "http://localhost:8983/solr/$*/update" \ + -d '{ "delete": {"query":"*:*"} }' \ + -H 'Content-Type: application/json' \ + | grep '"status":0' + curl "http://localhost:8983/solr/$*/update/csv?commit=true&encapsulator=\"&escape=\&header=true" \ + --data-binary @build/col/$*/PortalFiles/PortalData.csv \ + -H 'Content-type:application/csv' \ + | grep '"status":0' + date > $@ + +.PHONY: force-load-data-% +force-load-data-%: + rm -f build/html/$*/load-timestamp.txt + $(MAKE) build/html/$*/load-timestamp.txt + +##### Settings templates ##### + +build/setting_templates: | build + @printf "\n\n" + mkdir $@ + +build/setting_templates/%: build/html/% | build/setting_templates + @printf "\n\n### Generating $@.\n\n" + mkdir -p $@ + $(PYTHON) make_settings_template.py \ + PortalApp/resources/config/settings.json \ + > $@/settings.json + $(PYTHON) make_fields_template.py \ + build/html/$*/resources/config/fldmodel.json \ + > $@/fldmodel.json + touch $@ diff --git a/README.md b/README.md index 8502beb..4ef7296 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ Specify Web Portal (Version 2.0) ================================ -The Specify Collections Consortium is funded by its member institutions. The Consortium web site is: [http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/) +The Specify Collections Consortium is funded by its member +institutions. The Consortium web site is: +[http://wwww.specifysoftware.org](http://wwww.specifysoftware.org/) -Specify Web Portal Copyright © 2020 Specify Collections Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free software licensed under GNU General Public License 2 (GPL2). +Specify Web Portal Copyright © 2020 Specify Collections +Consortium. Specify comes with ABSOLUTELY NO WARRANTY. This is free +software licensed under GNU General Public License 2 (GPL2). ``` Specify Collections Consortium @@ -13,11 +17,22 @@ University of Kansas Lawrence, KS 66045 USA ``` +## Configuration Instructions + +Instructions on customizing the Web Portal configuration are available on the Specify Communtiy Forum: + +[Web Portal Configuration Instructions](https://discourse.specifysoftware.org/t/web-portal-configuration-instructions/1144) + ## Developer Instructions -After completing these instructions you will be able to run Specify Web Portal 2.0. +After completing these instructions you will be able to run Specify +Web Portal 2.0. -These instructions are for deployment on a server running Ubuntu. An export file for a single collection is required for setting up the Specify Web Portal. This can be accomplished using the Schema Mappging tool tool within the Specify 6 application together with the stand alone Specify Data Export tool. +These instructions are for deployment on a server running Ubuntu. An +export file for a single collection is required for setting up the +Specify Web Portal. This can be accomplished using the Schema Mappging +tool tool within the Specify 6 application together with the stand +alone Specify Data Export tool. Install system dependencies ------------ @@ -35,137 +50,194 @@ Installation Instructions ------------------------- These instructions illustrate the fewest steps needed to install the -web portal. For updating existing installations, se the 'Data Only Updates' section below. - -1. Clone the Web Portal 2.0 repository by clicking the green button (Clone or Download) at the top of the page and unpack this repository on your server. - - This will install Solr on the server. +web portal. + +1. These instructions assume the Web Portal is being setup under a + user account with the name `specify`. If a different account is + used, the directory paths in these instructions will need to be + updated accordingly. + + You will also need to update the `webportal-solr.service` file to use + the correct user and group. + +1. Clone the Web Portal 2.0 repository using git: + + ```console + specify@wp:~$ git clone https://github.com/specify/webportal-installer.git + ``` + + This will create the directory `webportal-installer`. + +1. Configure nginx to proxy the Solr requests and serve the static + files by copying the provided `webportal-nginx.conf` to + `/etc/nginx/sites-available/`: + + ```console + root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-nginx.conf /etc/nginx/sites-available/ + ``` + + N.B. this file will require changes if the `webportal-installer` is + in a different location. + +1. Disable the default nginx site and enable the portal site: + ```console + root@wp:~# rm /etc/nginx/sites-enabled/default + root@wp:~# ln -s /etc/nginx/sites-available/webportal-nginx.conf /etc/nginx/sites-enabled/ + root@wp:~# systemctl restart nginx + ``` 2. Use the Specify Data Export tool to create a Web Portal export zip - file (see the Specify 6 Data Export documentation) for each collection - to be hosted in the portal. If aggregated collections are desired, replace the single colleciton with the aggregated collections file after the initial Web Portal installation. - -3. Create a `specify_exports` directory in the web portal directory and copy the zip files (step 2) into it. The copied files should be given names that are suitable for use in URLs; so no spaces, capital letters, slashes or - other problematic characters. E.g. `kufish.zip` - -4. Build the Solr app: `make clean && make`. - -5. Copy the Solr core to the Solr installation: - - [CORENAME] - the name of the exported archive without the file extension. (E.g. `kufish`) - - [SOLRVERSION] - the version of Solr. (E.g. `7.5.0`) - -``` - mkdir solr-[SOLRVERSION]/server/solr/[CORENAME] - cp -r build/cores/[CORENAME]/core/* solr-[SOLRVERSION]/server/solr/[CORENAME] - cp build/cores/[CORENAME]/web.xml solr-[SOLRVERSION]/server/solr-webapp/webapp/WEB-INF/web.xml - # Only necessary for the first core. -``` -6. Restrict access to the solr admin web page. This can be done in solr 7.5 by editing solr: - - `[SOLRVERSION]/server/etc/jetty-http.xml` - - In the ServerConnector section replace: - - `` with `127.0.0.1` - -7. Start solr - `solr-[SOLRVERSION]/bin/solr start` - - When completing this step the following warnings may be issued and can be safely ignored: - - `*** [WARN] *** Your open file limit is currently 1024.` - - `It should be set to 65000 to avoid operational disruption.` - `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh` - `*** [WARN] *** Your Max Processes Limit is currently 63590.` - `It should be set to 65000 to avoid operational disruption.` - `If you no longer wish to see this warning, set SOLR_ULIMIT_CHECKS to false in your profile or solr.in.sh` - `Waiting up to 180 seconds to see Solr running on port 8983 [|]` - `Started Solr server on port 8983 (pid=15422). Happy searching!` - -8. Import the csv data: - `curl 'http://localhost:8983/solr/[CORENAME]/update/csv?commit=true&encapsulator="&escape=\&header=true' --data-binary @build/cores/[CORENAME]/PortalFiles/PortalData.csv -H 'Content-type:application/csv'` - - When completing this step you may receive output similar to the following: - - `{` - - `"responseHeader":{` - - `"status":0,` - - `"QTime":1153}}` - -9. Move the built webportal to convenient location: - - [WPPATH] - the directory where the web portal fiels reside. (E.g. /home/specify/webportal) - - `mv build [WPPATH]` - -10. Create the Nginx configuration file to serve the portal: - - `sudo nano /etc/nginx/sites-available/webportal.conf` + file (see the Specify 6 Data Export documentation) for each + collection to be hosted in the portal. If aggregated collections + are desired, replace the single colleciton with the aggregated + collections file after the initial Web Portal installation. + +3. Copy the zip files from the Specify Data Export into the + `webportal-installer/specify_exports` directory. The copied files + should be given names that are suitable for use in URLs; so no + spaces, capital letters, slashes or other problematic + characters. E.g. `kufish.zip` + +4. Build the Solr app: + ```console + specify@wp:~/webportal-installer$ make clean-all && make build-all ``` - server { - listen 80 default_server; - - rewrite ^/([^/]+)/select /solr/$1/select; - location ~ ^/solr/([^/]+)/select { - proxy_pass http://localhost:8983; - } - - location / { - root [WPPATH]/html; - } - } - ``` -11. Remove the default Nginx site and enable the portal site: - ``` - sudo rm /etc/nginx/sites-enabled/default - sudo ln -L /etc/nginx/sites-available/webportal.conf /etc/nginx/sites-enabled/ +5. Copy the provided `webportal-solr.service` systemd unit file to + `/etc/systemd/system/webportal-solr.service` and reload the + systemd daemon. + + ```console + root@wp:~# install -o root -g root -m644 /home/specify/webportal-installer/webportal-solr.service /etc/systemd/system/ + root@wp:~# systemctl daemon-reload ``` -12. Restart Nginx: `sudo systemctl restart nginx` + N.B. this file will require changes if the `webportal-installer` is + in a different location. - The Web Portal is now running and can be tested. +6. Start and enable the service: - The Web Portal can be customized by editing the settings in the settings.json file found at here: [WPPATH]/html/corename/resources/config/settings.json + ```console + root@wp:~# systemctl start webportal-solr.service + root@wp:~# systemctl enable webportal-solr.service + ``` + +7. Load the Specify data into Solr: + ```console + specify@wp:~/webportal-installer$ make load-data + ``` + + When completing this step you will receive output consisting of + blocks of JSON. This is normal unless one of the blocks includes an + error message. + +8. The Web Portal is now running and can be tested by visiting the + server's address with a web browser. + +9. Be aware that the Solr admin page is available by default on + port 8983. This can be useful for troubleshooting but should be + blocked in production using firewall rules such as the following: + + ```console + root@wp:~# ufw default deny incoming + root@wp:~# ufw default allow outgoing + root@wp:~# ufw allow ssh + root@wp:~# ufw allow http + root@wp:~# ufw --force enable + ``` + +Customization +------------- + +Certain customizations of the webportal are possible such as +background images. To enable customizations, after the webportal is +built with `make build-all`, copy the folders inside +`webportal-installer/build/setting_templates/` into +`webportal-installer/custom_settings/` and edit the files in each +folder as desired. Afterwards, `make build-html` will incorporate the +customizations into the portal without restarting any services. + +The provided `webportal-nginx.conf` serves the directory +`/home/specify/custom-images/` at `/custom-images/` allowing files +place there to be used in customization. + +Data-only Updates +-------- + +The portal can be updated with new data from exported collections +without restarting the Solr service if the export mappings are the +same. + +To update the Specify data in the webportal follow these steps. -Data Only Updates ------------------ - -If the fields used in a portal are unchanged and only data is being updated, delete the current contents of the solr core with: -`curl 'http://localhost:8983/solr/hollow/update?commit=true&stream.body=*%3A*'` -(You will probably need to add/edit a requestParsers block in the `solr-[SOLRVERSION]/server/solr/[CORENAME]/conf/solrconfig.xml` file for the core. Add it to the requestDispatcher block: +2. Use the Specify Data Export tool to create a Web Portal export zip + file (see the Specify 6 Data Export documentation) for each + collection to be updated. If aggregated collections + are desired, replace the single colleciton with the aggregated + collections file after the initial Web Portal installation. -``` - - +3. Copy the zip files from the Specify Data Export into the + `webportal-installer/specify_exports` directory. The copied files + in the `specify_exports` directory must have the same names as the + original exports. + +7. Load the Specify data into Solr: - - -``` -(See http://lucene.apache.org/solr/guide/requestdispatcher-in-solrconfig.html) + ```console + specify@wp:~/webportal-installer$ make load-data + ``` + +8. Only export files newer than the last update will be loaded. The + target `make force-load-data` will force all collections to be + reloaded. -Then use the curl CSV import command above to add the new data. +Full Updates +-------- +If new collections are to be added to the portal or mappings of +existing collections change, the Solr service has to be stopped and +the cores rebuilt. -Schema Definition Updates -------------------------- -In this case, you will need to stop Solr `(solr-[SOLRVERSION]/bin/solr stop)`, remove the cores to be updated from your Solr server directory, and follow all the installation steps besides the HTTP configuration. +1. Stop the webportal-solr systemd service: + ```console + root@wp:~# systemctl stop webportal-solr.service + ``` -Web Portal Application Updates ------------------------------- +2. Use the Specify Data Export tool to create a Web Portal export zip + file (see the Specify 6 Data Export documentation) for each + collection to be hosted in the portal. If aggregated collections + are desired, replace the single colleciton with the aggregated + collections file after the initial Web Portal installation. + + +3. Copy the zip files from the Specify Data Export into the + `webportal-installer/specify_exports` directory. The copied files + should be given names that are suitable for use in URLs; so no + spaces, capital letters, slashes or other problematic + characters. E.g. `kufish.zip` + +4. Build the Solr app: + + ```console + specify@wp:~/webportal-installer$ make build-all + ``` + +6. Start the webportal-solr service: + + ```console + root@wp:~# systemctl start webportal-solr.service + ``` + +7. Load the Specify data into Solr: + + ```console + specify@wp:~/webportal-installer$ make load-data + ``` + +8. The Web Portal is now running and can be tested by visiting the + server's address with a web browser. -If a new version of the Web Portal is being installed it will be necessary to perform step 8 after building. diff --git a/build.make b/build.make deleted file mode 100644 index 1485a91..0000000 --- a/build.make +++ /dev/null @@ -1,59 +0,0 @@ - -#all: solr-home setting_templates html html/index.html -all: setting_templates html html/index.html - -cores: $(TOPDIR)/core.make $(TOPDIR)/$(SOLR_DIST) \ - $(TOPDIR)/specify_exports $(TOPDIR)/specify_exports/*.zip - # We build a Solr core and webapp instance for - # each subdir in specify_exports. - rm -rf cores - for zipfile in $(TOPDIR)/specify_exports/*.zip ; do \ - zipfile_name=`basename "$$zipfile"` ; \ - corename="$${zipfile_name%.*}" ; \ - mkdir -p "cores/$$corename" ; \ - unzip -d "cores/$$corename" "$$zipfile" ; \ - $(MAKE) CORENAME="$$corename" -f $(TOPDIR)/core.make -C "cores/$$corename" ; \ - done - -setting_templates: $(TOPDIR)/make_settings_template.py $(TOPDIR)/make_fields_template.py cores - mkdir -p $@ - for core in cores/* ; do \ - corename=`basename "$$core"` ; \ - mkdir -p "$@/$$corename" ; \ - python2 $(TOPDIR)/make_settings_template.py \ - $(TOPDIR)/PortalApp/resources/config/settings.json \ - > "$@/$$corename/settings.json" ; \ - python2 $(TOPDIR)/make_fields_template.py \ - "cores/$$corename/webapp/resources/config/fldmodel.json" \ - > "$@/$$corename/fldmodel.json" ; \ - done - - -html/index.html: $(TOPDIR)/make_toplevel_index.py $(TOPDIR)/index_skel.html cores html - python2 $(TOPDIR)/make_toplevel_index.py $(TOPDIR)/index_skel.html \ - cores/*/webapp/resources/config/settings.json > $@ - -html: cores - # Put the webapps in the html folder. - mkdir -p html - for core in cores/* ; do \ - cp -r $$core/webapp html/`basename $$core` ; \ - done - -#solr-home: $(TOPDIR)/$(SOLR_DIST) cores solr.xml -# # Build the Solr home directory. -# rm -rf solr-home -# cp -r $(TOPDIR)/$(SOLR_DIST)/example/multicore solr-home - # Copy each core into place. -# rm -rf solr-home/core* -# for core in cores/* ; do \ -# cp -r $$core/core solr-home/`basename $$core` ; \ -# done - # Copy top level Solr configuration into place. -# cp solr.xml solr-home/ - -#solr.xml: $(TOPDIR)/make_solr_xml.py $(TOPDIR)/$(SOLR_DIST)/example/multicore/solr.xml cores - # Generate top level Solr config that defines the available cores. - #python $(TOPDIR)/make_solr_xml.py $(TOPDIR)/$(SOLR_DIST)/example/multicore/solr.xml \ - # cores/* > $@ - diff --git a/core.make b/core.make deleted file mode 100644 index ebd627b..0000000 --- a/core.make +++ /dev/null @@ -1,75 +0,0 @@ -# -# Use 'schema.xml' if solr will be used to create the core -# Use 'managed-schema' if pre-configuring core -SCHEMA_FILE := managed-schema -#location of default settings files in solr dist -DEFAULT_SETS := server/solr/configsets/_default - -all: webapp core - -# The custom setting file should really be a dependency here, -# but I don't know how to handle the case that it doesn't exist. -settings.json: $(TOPDIR)/patch_settings_json.py \ - $(TOPDIR)/PortalApp/resources/config/settings.json - # Patch web app settings. - python2 $^ $(TOPDIR)/custom_settings/$(CORENAME)/settings.json \ - $(CORENAME) PortalFiles/*Setting.json > $@ - -SolrFldSchema.xml: PortalFiles/SolrFldSchema.xml - # Add a root element to the schema field list. - echo '' > $@ - echo "" >> $@ - cat $< >> $@ - echo "" >> $@ - -$(SCHEMA_FILE): $(TOPDIR)/patch_schema_xml.py \ - $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/$(SCHEMA_FILE) \ - SolrFldSchema.xml - # Patching Solr schema with fields from Specify export. - python2 $^ > $@ - -web.xml: $(TOPDIR)/patch_web_xml.py \ - $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml - # Patching solr server app for cross-domain access to enable extjs ajax stores to POST solr query params. - #python $^ > $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml - python2 $^ > $@ - sudo cp $@ $(TOPDIR)/$(SOLR_DIST)/server/solr-webapp/webapp/WEB-INF/web.xml - -solrconfig.xml: $(TOPDIR)/patch_solrconfig_xml.py \ - $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/solrconfig.xml - # Patching Solr config for use with Specify. - python2 $^ > $@ - -# The custom setting file should really be a dependency here, -# but I don't know how to handle the case that it doesn't exist. -fldmodel.json: $(TOPDIR)/make_fldmodel_json.py PortalFiles/*flds.json - # Patch any custom settings into the field definitions. - python2 $^ $(TOPDIR)/custom_settings/$(CORENAME)/fldmodel.json > $@ - -webapp: $(TOPDIR)/PortalApp settings.json fldmodel.json - # Setup web app instance for this core. - mkdir -p webapp - cp -r $(TOPDIR)/PortalApp/* webapp/ - - # Copy WebPortal field specs into place. - cp fldmodel.json webapp/resources/config/fldmodel.json - - # Copy patched settings into place. - cp settings.json webapp/resources/config/ - - # Fix Solr URL format in WebApp. - #sed -i "s,solrURL + ':' + solrPort + '/',solrURL," webapp/app/store/MainSolrStore.js - -core: $(TOPDIR)/$(SOLR_DIST) PortalFiles solrconfig.xml $(SCHEMA_FILE) web.xml - # Setup solr-home subdir for this core. - mkdir -p core/conf - cp solrconfig.xml $(SCHEMA_FILE) core/conf/ - #cp -r PortalFiles/solr core/data/index - cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/protwords.txt core/conf/ - cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/synonyms.txt core/conf/ - cp $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/stopwords.txt core/conf/ - cp -r $(TOPDIR)/$(SOLR_DIST)/$(DEFAULT_SETS)/conf/lang/ core/conf/ - echo 'dataDir=data' > core/core.properties - echo 'name=$(CORENAME)' >> core/core.properties - echo 'config=conf/solrconfig.xml' >> core/core.properties - mkdir core/data diff --git a/index_skel.html b/index_skel.html index 4176e36..f92f626 100644 --- a/index_skel.html +++ b/index_skel.html @@ -3,15 +3,41 @@ Specify Web Portal +

The following collections are available:

-

- Last update: - N/A -

+ diff --git a/log4j.properties b/log4j.properties deleted file mode 100644 index 1d7c991..0000000 --- a/log4j.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Logging level -solr.log=/var/log/tomcat7/ -log4j.rootLogger=INFO, file, CONSOLE - -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender - -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n - -#- size rotation with log cleanup. -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.MaxFileSize=4MB -log4j.appender.file.MaxBackupIndex=9 - -#- File to log to and log format -log4j.appender.file.File=${solr.log}/solr.log -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n - -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.hadoop=WARN - -# set to INFO to enable infostream log messages -log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF diff --git a/make_fldmodel_json.py b/make_fldmodel_json.py index 59193cc..679e87e 100644 --- a/make_fldmodel_json.py +++ b/make_fldmodel_json.py @@ -27,7 +27,11 @@ if os.path.isfile(sys.argv[2]): with open(sys.argv[2]) as f: - custom = json.load(f, object_pairs_hook=OrderedDict) + data = f.read().strip() + if data: + custom = json.loads(data, object_pairs_hook=OrderedDict) + else: + custom = [] else: custom = [] diff --git a/make_toplevel_index.py b/make_toplevel_index.py index 2343f42..b1b3ae1 100644 --- a/make_toplevel_index.py +++ b/make_toplevel_index.py @@ -50,7 +50,7 @@ def splitall(path): with open(settings_file) as f: settings = json.load(f) - core_dir = splitall(settings_file)[1] + core_dir = splitall(settings_file)[2] core_name = settings[0]['collectionName'] or core_dir li = ElementTree.SubElement(collections, 'li') @@ -58,7 +58,4 @@ def splitall(path): a.set('href', core_dir) a.text = core_name -update_time = skel.find('.//span[@id="update-time"]') -update_time.text = datetime.now().isoformat() - skel.write(sys.stdout) diff --git a/patch_settings_json.py b/patch_settings_json.py index 8314e88..81069bd 100644 --- a/patch_settings_json.py +++ b/patch_settings_json.py @@ -27,7 +27,11 @@ if os.path.isfile(sys.argv[2]): with open(sys.argv[2]) as f: - custom_settings = json.load(f) + data = f.read().strip() + if data: + custom_settings = json.loads(data) + else: + custom_settings = {} else: custom_settings = {} diff --git a/webportal-nginx.conf b/webportal-nginx.conf new file mode 100644 index 0000000..f93ece6 --- /dev/null +++ b/webportal-nginx.conf @@ -0,0 +1,17 @@ +server { + listen 80 default_server; + + rewrite ^/([^/]+)/select /solr/$1/select; + + location ~ ^/solr/([^/]+)/select { + proxy_pass http://localhost:8983; + } + + location / { + root /home/specify/webportal-installer/build/html; + } + + location /custom-images/ { + alias /home/specify/custom-images/; + } +} diff --git a/webportal-solr.service b/webportal-solr.service new file mode 100644 index 0000000..52b1640 --- /dev/null +++ b/webportal-solr.service @@ -0,0 +1,16 @@ +[Unit] +Description=Solr server for Specify Web Portal +After=network.target + +[Service] +User=specify +Group=specify +PIDFile=/home/specify/webportal-installer/build/bin/solr-8983.pid +ExecStart=/home/specify/webportal-installer/build/bin/solr start +ExecStop=/home/specify/webportal-installer/build/bin/solr stop +LimitNOFILE=65000 +LimitNPROC=65000 +Restart=always + +[Install] +WantedBy=multi-user.target