diff --git a/admin/templates/configuration/etc/log4cxx.index.properties b/admin/templates/configuration/etc/log4cxx.index.properties
new file mode 100644
index 0000000000..b34769a14e
--- /dev/null
+++ b/admin/templates/configuration/etc/log4cxx.index.properties
@@ -0,0 +1,18 @@
+#
+# Configuration file for log4cxx
+# can be used for unit test
+# by launching next command before unit tests:
+# export LSST_LOG_CONFIG=$HOME/.lsst/log4cxx.unittest.properties
+#
+
+log4j.rootLogger=INFO, CONSOLE
+#log4j.rootLogger=DEBUG, CONSOLE
+#log4j.rootLogger=WARN, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+#log4j.appender.CONSOLE.layout.ConversionPattern=[%d{yyyy-MM-ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%d{ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+
+# Tune log at the module level
+#log4j.logger.lsst.qserv.util=DEBUG
diff --git a/admin/templates/configuration/etc/log4cxx.index_master.properties b/admin/templates/configuration/etc/log4cxx.index_master.properties
new file mode 100644
index 0000000000..0a3b0c6da5
--- /dev/null
+++ b/admin/templates/configuration/etc/log4cxx.index_master.properties
@@ -0,0 +1,18 @@
+#
+# Configuration file for log4cxx
+# can be used for unit test
+# by launching next command before unit tests:
+# export LSST_LOG_CONFIG=$HOME/.lsst/log4cxx.unittest.properties
+#
+
+#log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
+#log4j.rootLogger=WARN, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+#log4j.appender.CONSOLE.layout.ConversionPattern=[%d{yyyy-MM-ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%d{ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+
+# Tune log at the module level
+#log4j.logger.lsst.qserv.util=DEBUG
diff --git a/admin/tools/docker/index/container/buildContainers.README b/admin/tools/docker/index/container/buildContainers.README
new file mode 100644
index 0000000000..8c412da52c
--- /dev/null
+++ b/admin/tools/docker/index/container/buildContainers.README
@@ -0,0 +1,22 @@
+Invoking the following command from the qserv directory should build and push the containers and is
+useful for breaking the build into smaller commands when there are problems.
+
+It helps to do a "rm -rf bin share build" before running as docker copies everything in
+the qserv directory and includes it in the base container. This saves a couple of GB
+in both the initial copy and pushing the containers.
+
+
+docker build -f admin/tools/docker/index/container/dev/Dockerfile -t qserv/indexbase:dev . && \
+cd admin/tools/docker/index/container/dev/worker/ && docker build -t qserv/indexworker:dev . && \
+cd ../master/ && docker build -t qserv/indexmaster:dev . && \
+cd ../clientNum/ && docker build -t qserv/indexclientnum:dev . && \
+cd ../../../../../../../../qserv
+docker push qserv/indexmaster:dev && docker push qserv/indexworker:dev && docker push qserv/indexclientnum:dev
+
+
+Useful kubernetes commands:
+kubectl apply -f admin/tools/docker/index/index-k8-m.yaml
+kubectl delete -f admin/tools/docker/index/index-k8-m.yaml
+kubectl get pods
+kubectl logs -f imaster-sts-0 | grep -i keycount
+kubectl logs -f iclientnum2-sts-0 | egrep "DONE|INSERT|LOOK"
diff --git a/admin/tools/docker/index/container/buildContainers.bash b/admin/tools/docker/index/container/buildContainers.bash
new file mode 100644
index 0000000000..bc47c659b1
--- /dev/null
+++ b/admin/tools/docker/index/container/buildContainers.bash
@@ -0,0 +1,24 @@
+#! /bin/bash
+
+set -e
+
+# qserv/admin/tools/docker/loader/container/buildContainers.bash
+# cd back to base qserv directory as the Dockerfile COPY needs the entire project
+# in the docker context.
+cd ../../../../../../qserv
+docker build -f admin/tools/docker/index/container/dev/Dockerfile -t qserv/indexbase:dev .
+
+# go to individual directories to minimize the size of docker's context copy
+# worker
+cd admin/tools/docker/index/container/dev/worker/ && docker build -t qserv/indexworker:dev .
+#docker build -f admin/tools/docker/index/container/dev/worker/Dockerfile -t qserv/indexworker:dev .
+
+# master
+cd ../master/ && docker build -t qserv/indexmaster:dev .
+#docker build -f admin/tools/docker/index/container/dev/master/Dockerfile -t qserv/indexmaster:dev .
+
+# clientNum
+cd ../clientNum/ && docker build -t qserv/indexclientnum:dev .
+#docker build -f admin/tools/docker/index/container/dev/clientNum/Dockerfile -t qserv/indexclientnum:dev .
+
+
diff --git a/admin/tools/docker/index/container/dev/Dockerfile b/admin/tools/docker/index/container/dev/Dockerfile
new file mode 100644
index 0000000000..a5fd8583d3
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/Dockerfile
@@ -0,0 +1,26 @@
+# docker build -f admin/tools/docker/index/container/dev/Dockerfile -t qserv/indexbase:dev .
+#
+# Using the development toolchain
+
+FROM qserv/qserv:dev
+
+USER 0
+
+#RUN mv /usr/bin/sh /usr/bin/sh.old && ln -s /usr/bin/bash /usr/bin/sh
+RUN yum update --assumeyes && yum install --assumeyes bind-utils gdb screen
+
+USER 1000
+
+RUN mkdir /home/qserv/dev/ && \
+ chown -R qserv:qserv /home/qserv
+
+COPY --chown=qserv:qserv . /home/qserv/dev/qserv
+
+RUN bash -lc "rm -rf /home/qserv/dev/qserv/build /home/qserv/dev/qserv/share /home/qserv/dev/qserv/bin && \
+ cd /qserv/stack/ && source ./loadLSST.bash && \
+ cd /home/qserv/dev/qserv && setup -r . -t qserv-dev && \
+ printenv && \
+ scons -j10 install && \
+ mkdir -p /home/qserv/run && \
+ qserv-configure.py --all -R /home/qserv/run"
+
diff --git a/admin/tools/docker/index/container/dev/clientNum/Dockerfile b/admin/tools/docker/index/container/dev/clientNum/Dockerfile
new file mode 100644
index 0000000000..b4618d743e
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/clientNum/Dockerfile
@@ -0,0 +1,13 @@
+#
+#
+# cd ~/work/qserv/admin/tools/docker/index/container/dev/clientNum
+# docker build -t qserv/indexclientnum:dev .
+FROM qserv/indexbase:dev
+
+USER 0
+
+RUN yum update --assumeyes && yum install --assumeyes bind-utils
+
+USER 1000
+
+ENTRYPOINT ["/home/qserv/dev/qserv/admin/tools/docker/index/container/dev/clientNum/appClientNum.bash"]
diff --git a/admin/tools/docker/index/container/dev/clientNum/appClientNum.bash b/admin/tools/docker/index/container/dev/clientNum/appClientNum.bash
new file mode 100755
index 0000000000..5553004e5f
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/clientNum/appClientNum.bash
@@ -0,0 +1,23 @@
+#! /bin/bash -l
+# admin/tools/docker/loader/container/dev/clientNum/appClientNum.bash
+
+_term() {
+ echo "Caught SIGTERM signal!"
+ kill -TERM "$child" 2>/dev/null
+}
+
+trap _term SIGTERM
+trap _term SIGKILL
+
+source /qserv/stack/loadLSST.bash
+cd /home/qserv/dev/qserv
+setup -r . -t qserv-dev
+
+export LSST_LOG_CONFIG=/home/qserv/dev/qserv/admin/templates/configuration/etc/log4cxx.index.properties
+
+echo appClientNum $1 $2 $3
+
+/home/qserv/dev/qserv/build/loader/appClientNum $1 $2 /home/qserv/dev/qserv/core/modules/loader/config/$3
+
+child=$!
+wait "$child"
diff --git a/admin/tools/docker/index/container/dev/clientNum/appClientNumScreen.bash b/admin/tools/docker/index/container/dev/clientNum/appClientNumScreen.bash
new file mode 100755
index 0000000000..734bb3e65d
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/clientNum/appClientNumScreen.bash
@@ -0,0 +1,9 @@
+#! /bin/bash -l
+# admin/tools/docker/loader/container/dev/clientNum/appClientNumScreen.bash
+
+echo appClientScreen $1 $2 $3
+
+screen -dm /home/qserv/dev/qserv/admin/tools/docker/index/container/dev/clientNum/appClientNum $1 $2 $3
+
+
+tail -f /dev/null
diff --git a/admin/tools/docker/index/container/dev/master/Dockerfile b/admin/tools/docker/index/container/dev/master/Dockerfile
new file mode 100644
index 0000000000..f048cda80e
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/master/Dockerfile
@@ -0,0 +1,13 @@
+# Run the following build command from the qserv base directory (could be ~/work/qserv or ~/development/qserv)
+# The COPY command can only access files below $PWD in the file tree.
+# cd ~/work/qserv/admin/tools/docker/index/container/dev/master
+# docker build -t qserv/indexmaster:dev .
+FROM qserv/indexbase:dev
+
+USER 0
+
+RUN yum update --assumeyes && yum install --assumeyes bind-utils
+
+USER 1000
+
+ENTRYPOINT ["/home/qserv/dev/qserv/admin/tools/docker/index/container/dev/master/appMaster.bash"]
diff --git a/admin/tools/docker/index/container/dev/master/appMaster.bash b/admin/tools/docker/index/container/dev/master/appMaster.bash
new file mode 100755
index 0000000000..3054ad04a0
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/master/appMaster.bash
@@ -0,0 +1,22 @@
+#! /bin/bash -l
+# admin/tools/docker/loader/container/dev/master/appMaster.bash
+
+_term() {
+ echo "Caught SIGTERM signal!"
+ kill -TERM "$child" 2>/dev/null
+}
+
+trap _term SIGTERM
+trap _term SIGKILL
+
+source /qserv/stack/loadLSST.bash
+cd /home/qserv/dev/qserv
+setup -r . -t qserv-dev
+
+export LSST_LOG_CONFIG=/home/qserv/dev/qserv/admin/templates/configuration/etc/log4cxx.index_master.properties
+
+/home/qserv/dev/qserv/build/loader/appMaster /home/qserv/dev/qserv/core/modules/loader/config/master.cnf
+
+child=$!
+echo "child ${child}"
+wait "$child"
diff --git a/admin/tools/docker/index/container/dev/worker/Dockerfile b/admin/tools/docker/index/container/dev/worker/Dockerfile
new file mode 100644
index 0000000000..523361d001
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/worker/Dockerfile
@@ -0,0 +1,13 @@
+#
+#
+# cd ~/work/qserv/admin/tools/docker/index/container/dev/worker
+# docker build -t qserv/indexworker:dev .
+FROM qserv/indexbase:dev
+
+USER 0
+
+RUN yum update --assumeyes && yum install --assumeyes bind-utils
+
+USER 1000
+
+ENTRYPOINT ["/home/qserv/dev/qserv/admin/tools/docker/index/container/dev/worker/appWorker.bash"]
diff --git a/admin/tools/docker/index/container/dev/worker/appWorker.bash b/admin/tools/docker/index/container/dev/worker/appWorker.bash
new file mode 100755
index 0000000000..324fec66ba
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/worker/appWorker.bash
@@ -0,0 +1,22 @@
+#! /bin/bash
+# admin/tools/docker/loader/container/dev/worker/appWorker.bash
+
+_term() {
+ echo "Caught SIGTERM signal!"
+ kill -TERM "$child" 2>/dev/null
+}
+
+trap _term SIGTERM
+trap _term SIGKILL
+
+source /qserv/stack/loadLSST.bash
+cd /home/qserv/dev/qserv
+setup -r . -t qserv-dev
+
+export LSST_LOG_CONFIG=/home/qserv/dev/qserv/admin/templates/configuration/etc/log4cxx.index.properties
+
+/home/qserv/dev/qserv/build/loader/appWorker /home/qserv/dev/qserv/core/modules/loader/config/worker-k8s-a.cnf
+
+child=$!
+echo "child ${child}"
+wait "$child"
diff --git a/admin/tools/docker/index/container/dev/worker/appWorkerScreen.bash b/admin/tools/docker/index/container/dev/worker/appWorkerScreen.bash
new file mode 100755
index 0000000000..3893b5fe6a
--- /dev/null
+++ b/admin/tools/docker/index/container/dev/worker/appWorkerScreen.bash
@@ -0,0 +1,7 @@
+#! /bin/bash
+# admin/tools/docker/loader/container/dev/worker/appWorkerScreen.bash
+
+
+screen -dm /home/qserv/dev/qserv/admin/tools/docker/index/container/dev/worker/appWorker.bash
+
+tail -f /dev/null
diff --git a/admin/tools/docker/index/index-k8-100m.yaml b/admin/tools/docker/index/index-k8-100m.yaml
new file mode 100644
index 0000000000..643c33978a
--- /dev/null
+++ b/admin/tools/docker/index/index-k8-100m.yaml
@@ -0,0 +1,206 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: imaster-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10042
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: imaster-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: imaster-sts
+ labels:
+ app: index
+spec:
+ serviceName: imaster-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: imaster-pod
+ template:
+ metadata:
+ labels:
+ app: imaster-pod
+ spec:
+ containers:
+ - name: imaster-ctr
+ image: qserv/indexmaster:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10042
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iworker-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10043
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iworker-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iworker-sts
+ labels:
+ app: index
+spec:
+ serviceName: iworker-svc
+ podManagementPolicy: Parallel
+ replicas: 3
+ selector:
+ matchLabels:
+ app: iworker-pod
+ template:
+ metadata:
+ labels:
+ app: iworker-pod
+ spec:
+ containers:
+ - name: iworker-ctr
+ image: qserv/indexworker:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10043
+ protocol: UDP
+ - containerPort: 10143
+ protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum-pod
+ spec:
+ containers:
+ - name: iclientnum-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["100000000", "1", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum2-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum2-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum2-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum2-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum2-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum2-pod
+ spec:
+ containers:
+ - name: iclientnum2-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["200000001", "300000001", "client-k8s-a2.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum3-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum3-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum3-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum3-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum3-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum3-pod
+ spec:
+ containers:
+ - name: iclientnum3-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["100000001", "200000000", "client-k8s-a3.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+
diff --git a/admin/tools/docker/index/index-k8-10m.yaml b/admin/tools/docker/index/index-k8-10m.yaml
new file mode 100644
index 0000000000..5b989a088c
--- /dev/null
+++ b/admin/tools/docker/index/index-k8-10m.yaml
@@ -0,0 +1,415 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: imaster-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10042
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: imaster-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: imaster-sts
+ labels:
+ app: index
+spec:
+ serviceName: imaster-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: imaster-pod
+ template:
+ metadata:
+ labels:
+ app: imaster-pod
+ spec:
+ containers:
+ - name: imaster-ctr
+ image: qserv/indexmaster:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10042
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iworker-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10043
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iworker-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iworker-sts
+ labels:
+ app: index
+spec:
+ serviceName: iworker-svc
+ podManagementPolicy: Parallel
+ replicas: 14
+ selector:
+ matchLabels:
+ app: iworker-pod
+ template:
+ metadata:
+ labels:
+ app: iworker-pod
+ spec:
+ containers:
+ - name: iworker-ctr
+ image: qserv/indexworker:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10043
+ protocol: UDP
+ - containerPort: 10143
+ protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum-pod
+ spec:
+ containers:
+ - name: iclientnum-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["10000000", "1", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum2-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum2-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum2-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum2-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum2-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum2-pod
+ spec:
+ containers:
+ - name: iclientnum2-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["20000001", "30000001", "client-k8s-a2.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum3-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum3-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum3-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum3-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum3-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum3-pod
+ spec:
+ containers:
+ - name: iclientnum3-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["10000001", "20000000", "client-k8s-a3.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum4-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum4-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum4-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum4-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum4-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum4-pod
+ spec:
+ containers:
+ - name: iclientnum3-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["40000001", "50000000", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum5-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum5-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum5-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum5-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum5-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum5-pod
+ spec:
+ containers:
+ - name: iclientnum5-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["50000001", "60000000", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum6-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum6-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum6-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum6-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum6-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum6-pod
+ spec:
+ containers:
+ - name: iclientnum6-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["60000001", "70000000", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum7-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum7-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum7-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum7-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum7-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum7-pod
+ spec:
+ containers:
+ - name: iclientnum7-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["70000001", "80000000", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum8-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum8-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum8-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum8-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum8-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum8-pod
+ spec:
+ containers:
+ - name: iclientnum8-ctr
+ image: qserv/indexclientnum:dev
+ imagePullPolicy: Always
+ args: ["80000001", "90000000", "client-k8s-a1.cnf"]
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
diff --git a/admin/tools/docker/index/index-k8-m.yaml b/admin/tools/docker/index/index-k8-m.yaml
new file mode 100644
index 0000000000..ac8d5f337b
--- /dev/null
+++ b/admin/tools/docker/index/index-k8-m.yaml
@@ -0,0 +1,206 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: imaster-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10042
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: imaster-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: imaster-sts
+ labels:
+ app: index
+spec:
+ serviceName: imaster-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: imaster-pod
+ template:
+ metadata:
+ labels:
+ app: imaster-pod
+ spec:
+ containers:
+ - name: imaster-ctr
+ image: qserv/indexmaster:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10042
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iworker-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10043
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iworker-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iworker-sts
+ labels:
+ app: index
+spec:
+ serviceName: iworker-svc
+ podManagementPolicy: Parallel
+ replicas: 3
+ selector:
+ matchLabels:
+ app: iworker-pod
+ template:
+ metadata:
+ labels:
+ app: iworker-pod
+ spec:
+ containers:
+ - name: iworker-ctr
+ image: qserv/indexworker:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10043
+ protocol: UDP
+ - containerPort: 10143
+ protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum-pod
+ spec:
+ containers:
+ - name: iclientnum-ctr
+ image: qserv/indexclientnum:dev
+ args: ["1", "1000000", "client-k8s-a1.cnf"]
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum2-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum2-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum2-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum2-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum2-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum2-pod
+ spec:
+ containers:
+ - name: iclientnum2-ctr
+ image: qserv/indexclientnum:dev
+ args: ["1000001", "2000000", "client-k8s-a2.cnf"]
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum3-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum3-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum3-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum3-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum3-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum3-pod
+ spec:
+ containers:
+ - name: iclientnum3-ctr
+ image: qserv/indexclientnum:dev
+ args: ["2000001", "3000000", "client-k8s-a3.cnf"]
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
+
diff --git a/admin/tools/docker/index/index-k8.yaml b/admin/tools/docker/index/index-k8.yaml
new file mode 100644
index 0000000000..2eeab49889
--- /dev/null
+++ b/admin/tools/docker/index/index-k8.yaml
@@ -0,0 +1,123 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: imaster-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10042
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: imaster-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: imaster-sts
+ labels:
+ app: index
+spec:
+ serviceName: imaster-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: imaster-pod
+ template:
+ metadata:
+ labels:
+ app: imaster-pod
+ spec:
+ containers:
+ - name: imaster-ctr
+ image: qserv/indexmaster:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10042
+ protocol: UDP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iworker-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10043
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iworker-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iworker-sts
+ labels:
+ app: index
+spec:
+ serviceName: iworker-svc
+ podManagementPolicy: Parallel
+ replicas: 3
+ selector:
+ matchLabels:
+ app: iworker-pod
+ template:
+ metadata:
+ labels:
+ app: iworker-pod
+ spec:
+ containers:
+ - name: iworker-ctr
+ image: qserv/indexworker:dev
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10043
+ protocol: UDP
+ - containerPort: 10143
+ protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: iclientnum-svc
+ labels:
+ app: index
+spec:
+ ports:
+ - port: 10050
+ protocol: UDP
+ clusterIP: None
+ selector:
+ app: iclientnum-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: iclientnum-sts
+ labels:
+ app: index
+spec:
+ serviceName: iclientnum-svc
+ podManagementPolicy: Parallel
+ replicas: 1
+ selector:
+ matchLabels:
+ app: iclientnum-pod
+ template:
+ metadata:
+ labels:
+ app: iclientnum-pod
+ spec:
+ containers:
+ - name: iclientnum-ctr
+ image: qserv/indexclientnum:dev
+ args: ["1", "500000", "client-k8s-a1.cnf"]
+ imagePullPolicy: Always
+ ports:
+ - containerPort: 10050
+ protocol: UDP
+
diff --git a/core/modules/SConscript b/core/modules/SConscript
index 0959632683..2ae5be9a57 100644
--- a/core/modules/SConscript
+++ b/core/modules/SConscript
@@ -284,6 +284,11 @@ shlibs["qmetaLib"] = dict(mods="qmeta:python",
libs="qserv_qmeta log",
SHLIBPREFIX='',
instDir="$python_prefix/lsst/qserv/qmeta")
+
+# library of tools for building binary applications of the loader subsystem
+shlibs["loader"] = dict(mods="""loader""",
+ libs="""qserv_common util protobuf boost_filesystem boost_system
+ log log4cxx""")
# get list of all modules
all_modules = sorted(str(d) for d in Glob('*', source=True) if os.path.isdir(d.srcnode().abspath))
diff --git a/core/modules/loader/BufferUdp.cc b/core/modules/loader/BufferUdp.cc
new file mode 100644
index 0000000000..02ec9c3698
--- /dev/null
+++ b/core/modules/loader/BufferUdp.cc
@@ -0,0 +1,202 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+// Class header
+#include "BufferUdp.h"
+
+// qserv headers
+#include "loader/LoaderMsg.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.BufferUdp");
+}
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+MsgElement::Ptr BufferUdp::readFromSocket(boost::asio::ip::tcp::socket& socket, std::string const& note) {
+ // Repeatedly read a socket until a valid MsgElement is read, eof, or an error occurs.
+ for (;;) {
+ LOGS(_log, LOG_LVL_DEBUG, note << " readFromSocket");
+ boost::system::error_code error;
+
+ // If there's something in the buffer already, get it and return.
+ // This can happen when the previous read of socket read multiple elements.
+ MsgElement::Ptr msgElem = _safeRetrieve("1readFromSocket" + note);
+ if (msgElem != nullptr) {
+ return msgElem;
+ }
+
+ size_t len = socket.read_some(boost::asio::buffer(_wCursor, getAvailableWriteLength()), error);
+ _wCursor += len; /// must advance the cursor.
+
+ // EOF is only a problem if no MsgElement was retrieved.
+ // ??? This is definitely the case in UDP, EOF as nothing more will show up.
+ // ??? But TCP is another issue as EOF is returned when the connection is still live but
+ // ??? there's no data (len=0). Why does read_some set error to eof before the tcp connection is closed?
+ if (error == boost::asio::error::eof) {
+ LOGS(_log, LOG_LVL_WARN, "readFromSocket eof");
+ break; // Connection closed cleanly by peer.
+ } else if (error && error != boost::asio::error::eof) {
+ throw LoaderMsgErr(ERR_LOC, "BufferUdp::readFromSocket note=" + note + " asio error=" + error.message());
+ }
+
+ /// Try to retrieve an element (there's no guarantee that an entire element got read in a single read.
+ // Store original cursor positions so they can be restored if the read fails.
+ msgElem = _safeRetrieve("2readFromSocket" + note);
+ if (msgElem != nullptr) {
+ return msgElem;
+ }
+ }
+ return nullptr;
+}
+
+
+bool BufferUdp::releaseOwnership() {
+ if (_ourBuffer) {
+ _ourBuffer = false;
+ return true;
+ }
+ return false;
+}
+
+
+bool BufferUdp::append(const void* in, size_t len) {
+ if (isAppendSafe(len)) {
+ memcpy(_wCursor, in, len);
+ _wCursor += len;
+ return true;
+ }
+ return false;
+}
+
+
+void BufferUdp::advanceWriteCursor(size_t len) {
+ _wCursor += len;
+ if (not isAppendSafe(0)) {
+ // The buffer has overflowed.
+ throw std::overflow_error("BufferUdp advanceWriteCursor beyond buffer len=" +
+ std::to_string(len));
+ }
+}
+
+
+void BufferUdp::advanceReadCursor(size_t len) {
+ _rCursor += len;
+ if (_rCursor > _end) {
+ // Something read data outside of the buffer range.
+ throw std::overflow_error("BufferUdp advanceReadCursor beyond buffer len=" +
+ std::to_string(len));
+ }
+}
+
+
+std::shared_ptr BufferUdp::_safeRetrieve(std::string const& note) {
+ auto wCursorOriginal = _wCursor;
+ auto rCursorOriginal = _rCursor;
+ // throwOnMissing=false since missing data is possible with TCP.
+ MsgElement::Ptr msgElem = MsgElement::retrieve(*this, note + " _safeRetrieve", false);
+ if (msgElem != nullptr) {
+ return msgElem;
+ } else {
+ _wCursor = wCursorOriginal;
+ _rCursor = rCursorOriginal;
+ }
+ return nullptr;
+}
+
+
+bool BufferUdp::isRetrieveSafe(size_t len) const {
+ auto newLen = (_rCursor + len);
+ return (newLen <= _end && newLen <= _wCursor);
+}
+
+
+bool BufferUdp::retrieve(void* out, size_t len) {
+ if (isRetrieveSafe(len)) {
+ memcpy(out, _rCursor, len);
+ _rCursor += len;
+ return true;
+ }
+ LOGS(_log, LOG_LVL_DEBUG, "BufferUdp::retrieve not safe len=" << len);
+ return false;
+}
+
+
+bool BufferUdp::retrieveString(std::string& out, size_t len) {
+ if (isRetrieveSafe(len)) {
+ const char* strEnd = _rCursor + len;
+ std::string str(_rCursor, strEnd);
+ _rCursor = strEnd;
+ out = str;
+ return true;
+ }
+ return false;
+}
+
+
+std::string BufferUdp::dumpStr(bool hexDump, bool charDump) const {
+ std::stringstream os;
+ dump(os, hexDump, charDump);
+ return os.str();
+}
+
+
+std::ostream& BufferUdp::dump(std::ostream &os, bool hexDump, bool charDump) const {
+ os << "maxLength=" << _length;
+ os << " buffer=" << (void*)_buffer;
+ os << " wCurLen=" << getAvailableWriteLength();
+ os << " wCursor=" << (void*)_wCursor;
+ os << " rCurLen=" << getBytesLeftToRead();
+ os << " rCursor=" << (void*)_rCursor;
+ os << " end=" << (void*)_end;
+
+ // hex dump
+ if (hexDump) {
+ os << "(";
+ for (const char* j=_buffer; j < _wCursor; ++j) {
+ os << std::hex << (int)*j << " ";
+ }
+ os << ")";
+ }
+
+ // character dump
+ if (charDump) {
+ os << "(" << std::string(_buffer, _wCursor) << ")";
+ }
+
+ return os;
+}
+
+
+std::ostream& operator<<(std::ostream& os, BufferUdp const& buf) {
+ return buf.dump(os, false, false);
+}
+
+}}} // namespace lsst:qserv:loader
diff --git a/core/modules/loader/BufferUdp.h b/core/modules/loader/BufferUdp.h
new file mode 100644
index 0000000000..b83d063614
--- /dev/null
+++ b/core/modules/loader/BufferUdp.h
@@ -0,0 +1,171 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_BUFFERUDP_H
+#define LSST_QSERV_LOADER_BUFFERUDP_H
+
+// system headers
+#include
+#include
+#include
+#include
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+class MsgElement;
+
+
+/// A buffer for reading and writing. Nothing can be read from the buffer until
+/// something has been written to it.
+/// TODO: rename BufferUdp is not really accurate anymore.
+class BufferUdp {
+public:
+ using Ptr = std::shared_ptr;
+
+ /// The absolute largest UDP message we would send.
+ /// Usually, they should be much smaller.
+ static size_t const MAX_MSG_SIZE_UDP = 6000;
+
+ /// These are also being used for TCP messages, which can be much larger.
+ static size_t const MAX_MSG_SIZE_TCP = 10000000;
+
+ /// Create the object with a new _buffer with 'length' bytes.
+ explicit BufferUdp(size_t length = MAX_MSG_SIZE_UDP)
+ : _buffer(new char[length]), _length(length), _ourBuffer(true) {
+ _setupBuffer();
+ }
+
+ /// Create a BufferUdp object that uses 'buf' as its buffer, with 'length'
+ /// indicating the number of bytes in the buffer. If 'buf' already
+ /// contains valid data, 'validBytes' must be set to how many bytes of the buffer
+ /// are valid.
+ /// If BufferUdp should take ownership of 'buf', i.e. delete 'buf' when it is done,
+ /// call makeOwnerOfBuffer().
+ BufferUdp(char* buf, size_t length, size_t validBytes) : _buffer(buf), _length(length) {
+ _setupBuffer();
+ advanceWriteCursor(validBytes);
+ }
+
+ BufferUdp(BufferUdp const&) = delete;
+ BufferUdp& operator=(BufferUdp const&) = delete;
+
+ ~BufferUdp() { if (_ourBuffer) { delete[] _buffer; } }
+
+ /// Resets the cursors in the buffer so it is effectively empty.
+ void reset() { _setupBuffer(); }
+
+ /// Return true only if this object owns the buffer.
+ bool releaseOwnership();
+
+ /// Make this object is responsible for deleting _buffer.
+ void makeOwnerOfBuffer() { _ourBuffer = true; }
+
+ /// Return true if there's at least 'len' room left in the buffer.
+ bool isAppendSafe(size_t len) const { return (_wCursor + len) <= _end; }
+
+ /// Append 'len' bytes at 'in' to the end of _buffer, but only if it is safe to do so.
+ bool append(const void* in, size_t len);
+
+ /// Advance the write cursor. This is usually needed after some other object has been
+ /// allowed to write directly to the buffer. (boost::asio)
+ void advanceWriteCursor(size_t len);
+
+ /// Advance the read cursor, which usually needs to be done after another object
+ /// has been allowed to read directly from the buffer. (boost::asio)
+ void advanceReadCursor(size_t len);
+
+ /// Repeatedly read a socket until a valid MsgElement is read, eof, or an error occurs.
+ /// Errors throw LoaderMsgErr
+ std::shared_ptr readFromSocket(boost::asio::ip::tcp::socket& socket, std::string const& note);
+
+ /// Return the total length of _buffer.
+ size_t getMaxLength() const { return _length; }
+
+ /// Returns the number of bytes left to be read from the buffer.
+ int getBytesLeftToRead() const { return _wCursor - _rCursor; }
+
+ /// Returns the amount of room left in the buffer after the write cursor.
+ size_t getAvailableWriteLength() const { return _end - _wCursor; }
+
+ /// Returns a char* pointing to data to be read from the buffer.
+ const char* getReadCursor() const { return _rCursor; }
+
+ /// Returns a char* pointing to where data should be written to the buffer.
+ char* getWriteCursor() const { return _wCursor; }
+
+ /// Returns true if retrieving 'len' bytes from the buffer will not violate the buffer rules.
+ bool isRetrieveSafe(size_t len) const;
+
+ /// Returns true if 'len' bytes could be copied to out without violating _buffer rules.
+ bool retrieve(void* out, size_t len);
+
+ /// Returns true if 'len' bytes could be copied to 'out' without violating _buffer rules.
+ bool retrieveString(std::string& out, size_t len);
+
+ /// Dumps basic data to a string. If 'hexDump' is true, include a complete dump of
+ /// _buffer in hex.
+ std::string dumpStr(bool hexDump=true) const { return dumpStr(hexDump, false); }
+
+ /// Dumps basic data to a string. If 'hexDump' is true, include a complete dump of
+ /// _buffer in hex. If 'charDump' is true, include a complete dump of the buffer
+ /// in ascii.
+ std::string dumpStr(bool hexDump, bool charDump) const;
+
+ std::ostream& dump(std::ostream &os, bool hexDump, bool charDump) const;
+
+private:
+ void _setupBuffer() {
+ _end = _buffer + _length;
+ _wCursor = _buffer;
+ _rCursor = _buffer;
+ }
+
+ /// Parse the valid portion of _buffer (starting at _rCursor) and see if one
+ /// MsgElement is available. If so, return the element and advance _rCursor.
+ /// Otherwise return nullptr.
+ /// If a message is not recovered, the buffer is left effectively unchanged.
+ std::shared_ptr _safeRetrieve(std::string const& note);
+
+
+ char* _buffer;
+ size_t _length; ///< Number of bytes in the array (total capacity of array).
+ char* _end; ///< Immediately after the last element in the array.
+ char* _wCursor; ///< Where new elements will be appended to the array.
+ const char* _rCursor; ///< Where data is read from the buffer.
+
+ bool _ourBuffer{false}; ///< true if this class object is responsible for deleting the buffer.
+};
+
+/// Print basic buffer information. Use BufferUdp::dump() directly if the buffer contents are needed.
+std::ostream& operator<<(std::ostream& os, BufferUdp const& buf);
+
+}}} // namespace lsst:qserv:loader
+
+#endif // LSST_QSERV_LOADER_BUFFERUDP_H
diff --git a/core/modules/loader/Central.cc b/core/modules/loader/Central.cc
new file mode 100644
index 0000000000..2490d2e275
--- /dev/null
+++ b/core/modules/loader/Central.cc
@@ -0,0 +1,93 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+
+// System headers
+#include
+
+// Class header
+#include "Central.h"
+
+// Third-party headers
+#include "boost/asio.hpp"
+
+// qserv headers
+#include "loader/LoaderMsg.h"
+#include "proto/loader.pb.h"
+#include "proto/ProtoImporter.h"
+
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.Central");
+}
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+void Central::_initialize() {
+ // Order is important here.
+ _pool = util::ThreadPool::newThreadPool(_threadPoolSize, _queue);
+
+ doList = std::make_shared(*this);
+
+ std::thread t([this](){ _checkDoList(); });
+ _checkDoListThread = std::move(t);
+}
+
+Central::~Central() {
+ _loop = false;
+ _pool->shutdownPool();
+ for (std::thread& thd : _ioServiceThreads) {
+ thd.join();
+ }
+}
+
+
+void Central::run() {
+ std::thread thd([this]() { ioService.run(); });
+ _ioServiceThreads.push_back(std::move(thd));
+}
+
+
+void Central::_checkDoList() {
+ while(_loop) {
+ // Run and then sleep for a second. TODO A more advanced timer should be used
+ doList->checkList();
+ usleep(_loopSleepTime);
+ }
+}
+
+
+std::ostream& operator<<(std::ostream& os, ChunkSubchunk const& csc) {
+ os << "chunk=" << csc.chunk << " subchunk=" << csc.subchunk;
+ return os;
+}
+
+
+}}} // namespace lsst::qserv::loader
diff --git a/core/modules/loader/Central.h b/core/modules/loader/Central.h
new file mode 100644
index 0000000000..94c2a21ac0
--- /dev/null
+++ b/core/modules/loader/Central.h
@@ -0,0 +1,174 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CENTRAL_H
+#define LSST_QSERV_LOADER_CENTRAL_H
+
+// system headers
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/ClientServer.h"
+#include "loader/MasterServer.h"
+#include "loader/MWorkerList.h"
+#include "loader/WorkerServer.h"
+#include "loader/WWorkerList.h"
+#include "proto/loader.pb.h"
+#include "util/ThreadPool.h"
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+/// TODO add fileId and row to this so it can be checked in _workerKeyInsertReq
+struct ChunkSubchunk {
+ ChunkSubchunk(int chunk_, int subchunk_) : chunk(chunk_), subchunk(subchunk_) {}
+ int const chunk;
+ int const subchunk;
+ friend std::ostream& operator<<(std::ostream& os, ChunkSubchunk const& csc);
+};
+
+
+/// This class is 'central' to the execution of the program, and must be around
+/// until the bitter end. As such, it can be accessed by normal pointers.
+/// This class is central to clients, workers and the master.
+/// It provides a DoList and a means to contact the master. The master
+/// needs to know its own address.
+class Central {
+public:
+ Central() = delete;
+ Central(Central const&) = delete;
+ Central& operator=(Central const&) = delete;
+
+ virtual ~Central();
+
+ std::string getMasterHostName() const { return _masterAddr.ip; }
+ int getMasterPort() const { return _masterAddr.port; }
+ NetworkAddress getMasterAddr() const { return _masterAddr; }
+
+ uint64_t getNextMsgId() { return _sequence++; }
+
+ int getErrCount() const { return _server->getErrCount(); }
+
+ /// Start the server on UDP and/or TCP ports. May throw boost::system::system_error
+ void start() {
+ startService();
+ startMonitoring();
+ }
+
+ /// Override with function to define and start the server.
+ void virtual startService() = 0;
+
+ /// Override with functions to add do list items.
+ void virtual startMonitoring() {};
+
+ /// Send the contents of 'sendBuf' to 'host:port'. This waits for the message to be
+ /// sent before returning.
+ /// @throw boost::system::system_error on failure.
+ void sendBufferTo(std::string const& host, int port, BufferUdp& sendBuf) {
+ _server->sendBufferTo(host, port, sendBuf);
+ }
+
+ /// Only allow tracked commands on the queue. The DoList has to able to tell
+ /// if a Command completed.
+ void queueCmd(util::CommandTracked::Ptr const& cmd) {
+ _queue->queCmd(cmd);
+ }
+
+ /// Add a DoListItem to the _doList which will run and
+ /// rerun the item until it is no longer needed.
+ bool addDoListItem(DoListItem::Ptr const& item) {
+ return doList->addItem(item);
+ }
+
+ /// Run the item immediately before adding it to _doList.
+ bool runAndAddDoListItem(DoListItem::Ptr const& item) {
+ doList->runItemNow(item);
+ return doList->addItem(item);
+ }
+
+ /// Run the server.
+ void runServer() {
+ for (; _runningIOThreads < _iOThreads; ++_runningIOThreads) {
+ run();
+ }
+ }
+
+ /// Provides a method for identifying different Central classes and
+ /// CentralWorkers in the log file.
+ virtual std::string getOurLogId() const { return "Central baseclass"; }
+
+protected:
+ Central(boost::asio::io_service& ioService_,
+ std::string const& masterHostName, int masterPort,
+ int threadPoolSize, int loopSleepTime,
+ int iOThreads)
+ : ioService(ioService_), _masterAddr(masterHostName, masterPort),
+ _threadPoolSize(threadPoolSize), _loopSleepTime(loopSleepTime),
+ _iOThreads(iOThreads) {
+ _initialize();
+ }
+
+ void run(); ///< Run a single asio thread.
+
+ boost::asio::io_service& ioService;
+
+ DoList::Ptr doList; ///< List of items to be checked at regular intervals.
+
+ ServerUdpBase::Ptr _server;
+
+private:
+ /// Repeatedly check the items on the _doList.
+ void _checkDoList();
+
+ void _initialize();///< Finish construction.
+
+ NetworkAddress _masterAddr; ///< Network address of the master node.
+
+ std::atomic _sequence{1};
+
+ util::CommandQueue::Ptr _queue{std::make_shared()}; // Must be defined before _pool
+
+ int _threadPoolSize{10}; ///< Number of threads _pool.
+ util::ThreadPool::Ptr _pool; ///< Thread pool.
+
+ bool _loop{true}; ///< continue looping through _checkDolList() while this is true.
+ int _loopSleepTime{100000}; ///< microseconds to sleep between each check of all list items.
+
+ std::vector _ioServiceThreads; ///< List of asio io threads created by this
+
+ std::thread _checkDoListThread; ///< Thread for running doList checks on DoListItems.
+
+ int _iOThreads{5}; ///< Number of asio IO threads to run, set by config file.
+ int _runningIOThreads{0}; ///< Number of asio IO threads started.
+};
+
+}}} // namespace lsst::qserv::loader
+
+
+#endif // LSST_QSERV_LOADER_CENTRAL_H
diff --git a/core/modules/loader/CentralClient.cc b/core/modules/loader/CentralClient.cc
new file mode 100644
index 0000000000..bb86b545f6
--- /dev/null
+++ b/core/modules/loader/CentralClient.cc
@@ -0,0 +1,398 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+// Class header
+#include "CentralClient.h"
+#include "ClientConfig.h"
+
+// system headers
+#include
+
+// Third-party headers
+#include "boost/asio.hpp"
+
+// qserv headers
+#include "loader/LoaderMsg.h"
+#include "loader/WWorkerList.h"
+#include "proto/loader.pb.h"
+#include "proto/ProtoImporter.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.CentralClient");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+CentralClient::CentralClient(boost::asio::io_service& ioService_,
+ std::string const& hostName, ClientConfig const& cfg)
+ : CentralFollower(ioService_, hostName, cfg.getMasterHost(), cfg.getMasterPortUdp(),
+ cfg.getThreadPoolSize(),cfg.getLoopSleepTime(), cfg.getIOThreads(), cfg.getClientPortUdp()),
+ _defWorkerHost(cfg.getDefWorkerHost()),
+ _defWorkerPortUdp(cfg.getDefWorkerPortUdp()),
+ _doListMaxLookups(cfg.getMaxLookups()),
+ _doListMaxInserts(cfg.getMaxInserts()),
+ _maxRequestSleepTime(cfg.getMaxRequestSleepTime()) {
+}
+
+
+void CentralClient::startService() {
+ _server = std::make_shared(ioService, getHostName(), getUdpPort(), this);
+}
+
+
+CentralClient::~CentralClient() {
+}
+
+
+void CentralClient::handleKeyLookup(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ LOGS(_log, LOG_LVL_DEBUG, "CentralClient::handleKeyLookup");
+
+ auto const sData = std::dynamic_pointer_cast(MsgElement::retrieve(*data, " CentralClient::handleKeyLookup "));
+ if (sData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralClient::handleKeyLookup Failed to parse list");
+ return;
+ }
+ auto protoData = sData->protoParse();
+ if (protoData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralClient::handleKeyLookup Failed to parse list");
+ return;
+ }
+
+ // TODO put in separate thread
+ _handleKeyLookup(inMsg, protoData);
+}
+
+
+void CentralClient::_handleKeyLookup(LoaderMsg const& inMsg, std::unique_ptr& protoBuf) {
+ std::unique_ptr protoData(std::move(protoBuf));
+
+ CompositeKey key(protoData->keyint(), protoData->keystr());
+ ChunkSubchunk chunkInfo(protoData->chunk(), protoData->subchunk());
+
+ LOGS(_log, LOG_LVL_DEBUG, "trying to remove oneShot for lookup key=" << key << " " << chunkInfo);
+ /// Locate the original one shot and mark it as done.
+ CentralClient::KeyLookupReqOneShot::Ptr keyLookupOneShot;
+ {
+ std::lock_guard lck(_waitingKeyLookupMtx);
+ auto iter = _waitingKeyLookupMap.find(key);
+ if (iter == _waitingKeyLookupMap.end()) {
+ LOGS(_log, LOG_LVL_WARN, "_handleKeyLookup could not find key=" << key);
+ return;
+ }
+ keyLookupOneShot = iter->second;
+ _waitingKeyLookupMap.erase(iter);
+ }
+ keyLookupOneShot->keyInfoComplete(key, chunkInfo.chunk, chunkInfo.subchunk, protoData->success());
+ LOGS(_log, LOG_LVL_INFO, "Successful KEY_LOOKUP key=" << key << " " << chunkInfo);
+}
+
+
+void CentralClient::handleKeyInsertComplete(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ LOGS(_log, LOG_LVL_DEBUG, "CentralClient::handleKeyInsertComplete");
+
+ auto sData = std::dynamic_pointer_cast(MsgElement::retrieve(*data, " CentralClient::handleKeyInsertComplete "));
+ if (sData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralClient::handleKeyInsertComplete Failed to retrieve element");
+ return;
+ }
+ auto protoData = sData->protoParse();
+ if (protoData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralClient::handleKeyInsertComplete Failed to parse list");
+ return;
+ }
+
+ // TODO put in separate thread
+ _handleKeyInsertComplete(inMsg, protoData);
+}
+
+
+void CentralClient::_handleKeyInsertComplete(LoaderMsg const& inMsg, std::unique_ptr& protoBuf) {
+ std::unique_ptr protoData(std::move(protoBuf));
+
+ CompositeKey key(protoData->keyint(), protoData->keystr());
+ ChunkSubchunk chunkInfo(protoData->chunk(), protoData->subchunk());
+
+ LOGS(_log, LOG_LVL_DEBUG, "trying to remove oneShot for key=" << key << " " << chunkInfo);
+ /// Locate the original one shot and mark it as done.
+ CentralClient::KeyInsertReqOneShot::Ptr keyInsertOneShot;
+ size_t mapSize;
+ {
+ std::lock_guard lck(_waitingKeyInsertMtx);
+ auto iter = _waitingKeyInsertMap.find(key);
+ if (iter == _waitingKeyInsertMap.end()) {
+ LOGS(_log, LOG_LVL_WARN, "handleKeyInsertComplete could not find key=" << key);
+ return;
+ }
+ keyInsertOneShot = iter->second;
+ _waitingKeyInsertMap.erase(iter);
+ mapSize = _waitingKeyInsertMap.size();
+ }
+ keyInsertOneShot->keyInsertComplete();
+ LOGS(_log, LOG_LVL_INFO, "Successful KEY_INSERT_COMPLETE key=" << key << " " << chunkInfo <<
+ " mapSize=" << mapSize);
+}
+
+
+/// Returns a pointer to a Tracker object that can be used to track job
+// completion and the status of the job. keyInsertOneShot will call
+// _keyInsertReq until it knows the task was completed via a call
+// to _handleKeyInsertComplete
+KeyInfoData::Ptr CentralClient::keyInsertReq(CompositeKey const& key, int chunk, int subchunk) {
+ // Insert a oneShot DoListItem to keep trying to add the key until
+ // we get word that it has been added successfully.
+ LOGS(_log, LOG_LVL_INFO, "Trying to insert key=" << key << " chunk=" << chunk <<
+ " subchunk=" << subchunk);
+ auto keyInsertOneShot = std::make_shared(this, key, chunk, subchunk);
+ {
+ std::unique_lock lck(_waitingKeyInsertMtx);
+ // Limit the number of concurrent inserts.
+ // If the key is already in the map, there is no point in blocking.
+ int loopCount = 0;
+ auto iter = _waitingKeyInsertMap.find(key);
+ while (_waitingKeyInsertMap.size() > _doListMaxInserts
+ && iter == _waitingKeyInsertMap.end()) {
+ size_t sz = _waitingKeyInsertMap.size();
+ lck.unlock();
+ if (loopCount % 100 == 0) {
+ LOGS(_log, LOG_LVL_DEBUG, "keyInsertReq waiting key=" << key <<
+ "size=" << sz << " loopCount=" << loopCount);
+ }
+ // Let the CPU do something else while waiting for some requests to finish.
+ usleep(_maxRequestSleepTime);
+ ++loopCount;
+ lck.lock();
+ iter = _waitingKeyInsertMap.find(key);
+ }
+
+ if (iter != _waitingKeyInsertMap.end()) {
+ // There is already an entry in the map and we can just use the existing entry,
+ // as long as it has the same chunk and subchunk numbers.
+ auto cData = iter->second->cmdData;
+ if (cData->chunk == chunk && cData->subchunk == subchunk) {
+ return cData;
+ } else {
+ // TODO This MUST go to some form of output for the end user as it is an input data error
+ // either here or when the caller gets a nullptr response
+ LOGS(_log, LOG_LVL_ERROR, "key:value does not match existing key:value key=" << key <<
+ " orignal(" << cData->chunk << "," << cData->subchunk <<
+ ") new(" << chunk << "," << subchunk << ")");
+ return nullptr;
+ }
+ }
+ // The key wasn't found and needs to be inserted.
+ _waitingKeyInsertMap[key] = keyInsertOneShot;
+ }
+ runAndAddDoListItem(keyInsertOneShot);
+ return keyInsertOneShot->cmdData;
+}
+
+
+void CentralClient::_keyInsertReq(CompositeKey const& key, int chunk, int subchunk) {
+ LOGS(_log, LOG_LVL_INFO, "CentralClient::_keyInsertReq trying key=" << key);
+ LoaderMsg msg(LoaderMsg::KEY_INSERT_REQ, getNextMsgId(), getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ // create the proto buffer
+ lsst::qserv::proto::KeyInfoInsert protoKeyInsert;
+ lsst::qserv::proto::LdrNetAddress* protoAddr = protoKeyInsert.mutable_requester();
+ protoAddr->set_ip(getHostName());
+ protoAddr->set_udpport(getUdpPort());
+ protoAddr->set_tcpport(getTcpPort());
+ lsst::qserv::proto::KeyInfo* protoKeyInfo = protoKeyInsert.mutable_keyinfo();
+ protoKeyInfo->set_keyint(key.kInt);
+ protoKeyInfo->set_keystr(key.kStr);
+ protoKeyInfo->set_chunk(chunk);
+ protoKeyInfo->set_subchunk(subchunk);
+ protoKeyInsert.set_hops(0);
+
+ StringElement strElem;
+ protoKeyInsert.SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+ try {
+ std::string ip;
+ int port = 0;
+ getWorkerForKey(key, ip, port);
+ sendBufferTo(ip, port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralClient::_keyInsertReq boost system_error=" << e.what() <<
+ " key=" << key << " chunk=" << chunk << " sub=" << subchunk);
+ }
+}
+
+
+KeyInfoData::Ptr CentralClient::keyLookupReq(CompositeKey const& key) {
+ // Returns a pointer to a Tracker object that can be used to track job
+ // completion and job status. keyInsertOneShot will call _keyInsertReq until
+ // it knows the task was completed. _handleKeyInfoComplete marks
+ // the jobs complete as the messages come in from workers.
+ // Insert a oneShot DoListItem to keep trying to add the key until
+ // we get word that it has been added successfully.
+ LOGS(_log, LOG_LVL_INFO, "Trying to lookup key=" << key);
+ auto keyLookupOneShot = std::make_shared(this, key);
+ {
+ std::unique_lock lck(_waitingKeyLookupMtx);
+ // Limit the number of concurrent lookups.
+ // If the key is already in the map, there is no point in blocking.
+ int loopCount = 0;
+ uint64_t sleptForMicroSec = 0;
+ uint64_t const tenSec = 10000000;
+ auto iter = _waitingKeyLookupMap.find(key);
+ while (_waitingKeyLookupMap.size() > _doListMaxLookups
+ && iter == _waitingKeyLookupMap.end()) {
+ size_t sz = _waitingKeyLookupMap.size();
+ lck.unlock();
+ // Log a message about this about once every 10 seconds.
+ if (sleptForMicroSec > tenSec) sleptForMicroSec = 0;
+ if (sleptForMicroSec == 0) {
+ LOGS(_log, LOG_LVL_INFO, "keyInfoReq waiting key=" << key <<
+ "size=" << sz << " loopCount=" << loopCount);
+ }
+ // Let the CPU do something else while waiting for some requests to finish.
+ usleep(_maxRequestSleepTime);
+ sleptForMicroSec += _maxRequestSleepTime;
+ ++loopCount;
+ lck.lock();
+ iter = _waitingKeyLookupMap.find(key);
+ }
+
+ // Use the existing lookup, if there is one.
+ if (iter != _waitingKeyLookupMap.end()) {
+ auto cData = iter->second->cmdData;
+ return cData;
+ }
+
+ _waitingKeyLookupMap[key] = keyLookupOneShot;
+ }
+ runAndAddDoListItem(keyLookupOneShot);
+ return keyLookupOneShot->cmdData;
+}
+
+
+void CentralClient::_keyLookupReq(CompositeKey const& key) {
+ LOGS(_log, LOG_LVL_INFO, "CentralClient::_keyLookupReq trying key=" << key);
+ LoaderMsg msg(LoaderMsg::KEY_LOOKUP_REQ, getNextMsgId(), getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ // create the proto buffer
+ lsst::qserv::proto::KeyInfoInsert protoKeyInsert;
+ lsst::qserv::proto::LdrNetAddress* protoAddr = protoKeyInsert.mutable_requester();
+ protoAddr->set_ip(getHostName());
+ protoAddr->set_udpport(getUdpPort());
+ protoAddr->set_tcpport(getTcpPort());
+ lsst::qserv::proto::KeyInfo* protoKeyInfo = protoKeyInsert.mutable_keyinfo();
+ protoKeyInfo->set_keyint(key.kInt);
+ protoKeyInfo->set_keystr(key.kStr);
+ protoKeyInfo->set_chunk(0);
+ protoKeyInfo->set_subchunk(0);
+ protoKeyInsert.set_hops(0);
+
+ StringElement strElem;
+ protoKeyInsert.SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+
+ try {
+ std::string ip;
+ int port = 0;
+ getWorkerForKey(key, ip, port);
+ sendBufferTo(ip, port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralClient::_keyInfoReq boost system_error=" << e.what() <<
+ " key=" << key);
+ }
+}
+
+
+void CentralClient::getWorkerForKey(CompositeKey const& key, std::string& ip, int& port) {
+ auto worker = getWorkerList()->findWorkerForKey(key);
+ if (worker != nullptr) {
+ auto nAddr = worker->getUdpAddress();
+ ip = nAddr.ip;
+ port = nAddr.port;
+ LOGS(_log, LOG_LVL_DEBUG, "getWorkerForKey " << key << " worker=" << *worker);
+ } else {
+ ip = getDefWorkerHost();
+ port = getDefWorkerPortUdp();
+ }
+}
+
+
+std::ostream& operator<<(std::ostream& os, KeyInfoData const& data) {
+ os << "key=" << data.key << "(" << data.chunk << "," << data.subchunk << ") " <<
+ "success=" << data.success;
+ return os;
+}
+
+
+util::CommandTracked::Ptr CentralClient::KeyInsertReqOneShot::createCommand() {
+ struct KeyInsertReqCmd : public util::CommandTracked {
+ KeyInsertReqCmd(KeyInfoData::Ptr& cd, CentralClient* cent_) : cData(cd), cent(cent_) {}
+ void action(util::CmdData*) override {
+ cent->_keyInsertReq(cData->key, cData->chunk, cData->subchunk);
+ }
+ KeyInfoData::Ptr cData;
+ CentralClient* cent;
+ };
+ return std::make_shared(cmdData, central);
+}
+
+
+void CentralClient::KeyInsertReqOneShot::keyInsertComplete() {
+ cmdData->success = true;
+ cmdData->setComplete();
+ infoReceived();
+}
+
+
+util::CommandTracked::Ptr CentralClient::KeyLookupReqOneShot::createCommand() {
+ struct KeyInfoReqCmd : public util::CommandTracked {
+ KeyInfoReqCmd(KeyInfoData::Ptr& cd, CentralClient* cent_) : cData(cd), cent(cent_) {}
+ void action(util::CmdData*) override {
+ cent->_keyLookupReq(cData->key);
+ }
+ KeyInfoData::Ptr cData;
+ CentralClient* cent;
+ };
+ return std::make_shared(cmdData, central);
+}
+
+
+void CentralClient::KeyLookupReqOneShot::keyInfoComplete(CompositeKey const& key,
+ int chunk, int subchunk, bool success) {
+ if (key == cmdData->key) {
+ cmdData->chunk = chunk;
+ cmdData->subchunk = subchunk;
+ cmdData->success = success;
+ }
+ cmdData->setComplete();
+ infoReceived();
+}
+
+
+}}} // namespace lsst::qserv::loader
diff --git a/core/modules/loader/CentralClient.h b/core/modules/loader/CentralClient.h
new file mode 100644
index 0000000000..67e5e2364d
--- /dev/null
+++ b/core/modules/loader/CentralClient.h
@@ -0,0 +1,174 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CENTRAL_CLIENT_H
+#define LSST_QSERV_LOADER_CENTRAL_CLIENT_H
+
+// system headers
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/Central.h"
+#include "loader/CentralFollower.h"
+#include "loader/ClientServer.h"
+#include "loader/WWorkerList.h"
+#include "util/Command.h"
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+class ClientConfig;
+
+/// This class is used to track the status and value of jobs inserting
+/// key-value pairs to the system or looking up key-value pairs. The
+/// Tracker base class provides a means of notifying other threads
+/// that the task is complete.
+class KeyInfoData : public util::Tracker {
+public:
+ using Ptr = std::shared_ptr;
+ KeyInfoData(CompositeKey const& key_, int chunk_, int subchunk_) :
+ key(key_), chunk(chunk_), subchunk(subchunk_) {}
+
+ CompositeKey key;
+ int chunk;
+ int subchunk;
+ bool success{false};
+
+ friend std::ostream& operator<<(std::ostream& os, KeyInfoData const& data);
+};
+
+/// This class is 'Central' to the client. The client maintains a UDP port
+/// so replies to its request can be sent directly back to it.
+/// 'Central' provides access to the master and a DoList for handling requests.
+class CentralClient : public CentralFollower {
+public:
+ /// The client needs to know the master's IP and its own IP.
+ CentralClient(boost::asio::io_service& ioService_,
+ std::string const& hostName, ClientConfig const& cfg);
+
+ void startService() override;
+
+ ~CentralClient() override;
+
+ /// @return the default worker's host name.
+ std::string getDefWorkerHost() const { return _defWorkerHost; }
+ /// @return the default worker's UDP port
+ int getDefWorkerPortUdp() const { return _defWorkerPortUdp; }
+
+ /// Try to get the correct address for the worker responsible for 'key'
+ /// and place the information in 'ip' and 'port'. If a reasonable worker is
+ /// not located, the default worker information is returned.
+ void getWorkerForKey(CompositeKey const& key, std::string& ip, int& port);
+
+ /// Asynchronously request a key value insert to the workers.
+ /// This can block if too many key insert requests are already in progress.
+ /// @return - a KeyInfoData object for checking the job's status or
+ /// nullptr if CentralClient is already trying to insert the key
+ /// but value doesn't match the existing value. This indicates
+ /// there is an input data error.
+ KeyInfoData::Ptr keyInsertReq(CompositeKey const& key, int chunk, int subchunk);
+ /// Handle a workers response to the keyInserReq call
+ void handleKeyInsertComplete(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ /// Asynchronously request a key value lookup from the workers. It returns a
+ /// KeyInfoData object to be used to track job status and get the value of the key.
+ /// This can block if too many key lookup requests are already in progress.
+ KeyInfoData::Ptr keyLookupReq(CompositeKey const& key);
+ /// Handle a workers response to the keyInfoReq call.
+ void handleKeyLookup(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ int getDoListMaxLookups() { return _doListMaxLookups; }
+ int getDoListMaxInserts() { return _doListMaxInserts; }
+
+ std::string getOurLogId() const override { return "client"; }
+
+private:
+ void _keyInsertReq(CompositeKey const& key, int chunk, int subchunk); ///< see keyInsertReq()
+ void _handleKeyInsertComplete(LoaderMsg const& inMsg, std::unique_ptr& protoBuf);
+
+ void _keyLookupReq(CompositeKey const& key); ///< see keyLookReq()
+ void _handleKeyLookup(LoaderMsg const& inMsg, std::unique_ptr& protoBuf);
+
+
+
+ /// Create commands to add a key to the index and track that they are done.
+ /// It should keep trying this until it works, and then drop it from _waitingKeyInsertMap.
+ struct KeyInsertReqOneShot : public DoListItem {
+ using Ptr = std::shared_ptr;
+
+ KeyInsertReqOneShot(CentralClient* central_, CompositeKey const& key_, int chunk_, int subchunk_) :
+ cmdData(std::make_shared(key_, chunk_, subchunk_)), central(central_) {
+ setOneShot(true);
+ }
+
+ util::CommandTracked::Ptr createCommand() override;
+
+ /// TODO Have this function take result codes (such as 'success') as arguments
+ /// and put them in cmdData.
+ void keyInsertComplete();
+
+ KeyInfoData::Ptr cmdData;
+ CentralClient* central;
+ };
+
+ /// Create commands to lookup a key in the index and get its value.
+ /// It should keep trying this until it works and then drop it from _waitingKeyInfoMap.
+ struct KeyLookupReqOneShot : public DoListItem {
+ using Ptr = std::shared_ptr;
+
+ KeyLookupReqOneShot(CentralClient* central_, CompositeKey const& key_) :
+ cmdData(std::make_shared(key_, -1, -1)), central(central_) { setOneShot(true); }
+
+ util::CommandTracked::Ptr createCommand() override;
+
+ // TODO Have this function take result codes as arguments and put them in cmdData.
+ void keyInfoComplete(CompositeKey const& key, int chunk, int subchunk, bool success);
+
+ KeyInfoData::Ptr cmdData;
+ CentralClient* central;
+ };
+
+ // If const is removed, these will need mutex protection.
+ const std::string _defWorkerHost; ///< Default worker host
+ const int _defWorkerPortUdp; ///< Default worker UDP port
+
+ size_t const _doListMaxLookups = 1000; ///< Maximum number of concurrent lookups in DoList, set by config
+ size_t const _doListMaxInserts = 1000; ///< Maximum number of concurrent inserts in DoList, set by config
+ /// Time to sleep between checking requests when at max length, set by config
+ int const _maxRequestSleepTime = 100000;
+
+ std::map _waitingKeyInsertMap; ///< Map of current insert requests.
+ std::mutex _waitingKeyInsertMtx; ///< protects _waitingKeyInsertMap, _doListMaxInserts
+
+ std::map _waitingKeyLookupMap; ///< Map of current look up requests.
+ std::mutex _waitingKeyLookupMtx; ///< protects _waitingKeyLookMap, _doListMaxLookups
+};
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CENTRAL_CLIENT_H
diff --git a/core/modules/loader/CentralFollower.cc b/core/modules/loader/CentralFollower.cc
new file mode 100644
index 0000000000..2ae6278fed
--- /dev/null
+++ b/core/modules/loader/CentralFollower.cc
@@ -0,0 +1,127 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+// Class header
+#include "loader/CentralFollower.h"
+
+// system headers
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// qserv headers
+#include "proto/loader.pb.h"
+#include "proto/ProtoImporter.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.CentralFollower");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+CentralFollower::~CentralFollower() {
+ // Members that contain pointers to this. Deleting while this != null.
+ // TODO: wait for reference count to drop to one on _wWorkerList
+ _destroy = true;
+ _wWorkerList.reset();
+}
+
+
+WWorkerList::Ptr const& CentralFollower::getWorkerList() {
+ // _wWorkerList == nullptr is extremely rare.
+ if (_wWorkerList == nullptr && !_destroy) {
+ std::lock_guard lg(_wListInitMtx);
+ if (_wWorkerList == nullptr && !_destroy) _wWorkerList = std::make_shared(this);
+ }
+ return _wWorkerList;
+}
+
+
+void CentralFollower::startMonitoring() {
+ LOGS(_log, LOG_LVL_INFO, "CentralFollower::startMonitoring");
+ doList->addItem(getWorkerList());
+}
+
+
+bool CentralFollower::workerInfoReceive(BufferUdp::Ptr const& data) {
+ // Open the data protobuffer and add it to our list.
+ StringElement::Ptr sData = std::dynamic_pointer_cast(MsgElement::retrieve(*data, "CentralFollower::workerInfoReceive"));
+ if (sData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralFollower::workerInfoRecieve Failed to parse list");
+ return false;
+ }
+ std::unique_ptr protoList = sData->protoParse();
+ if (protoList == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralFollower::workerInfoRecieve Failed to parse list");
+ return false;
+ }
+
+ // TODO: move this call to another thread
+ _workerInfoReceive(protoList);
+ return true;
+}
+
+
+void CentralFollower::_workerInfoReceive(std::unique_ptr& protoL) {
+ std::unique_ptr protoList(std::move(protoL));
+
+ // Check the information, if it is our network address, set or check our id.
+ // Then compare it with the map, adding new/changed information.
+ uint32_t wId = protoList->wid();
+ std::string ipUdp("");
+ int portUdp = 0;
+ int portTcp = 0;
+ if (protoList->has_address()) {
+ proto::LdrNetAddress protoAddr = protoList->address();
+ ipUdp = protoAddr.ip();
+ portUdp = protoAddr.udpport();
+ portTcp = protoAddr.tcpport();
+ }
+ KeyRange strRange;
+ if (protoList->has_range()) {
+ proto::WorkerRange protoRange = protoList->range();
+ bool valid = protoRange.valid();
+ if (valid) {
+ CompositeKey min(protoRange.minint(), protoRange.minstr());
+ CompositeKey max(protoRange.maxint(), protoRange.maxstr());
+ bool unlimited = protoRange.maxunlimited();
+ strRange.setMinMax(min, max, unlimited);
+ }
+ }
+
+ checkForThisWorkerValues(wId, ipUdp, portUdp, portTcp, strRange);
+
+ // Make/update entry in map.
+ getWorkerList()->updateEntry(wId, ipUdp, portUdp, portTcp, strRange);
+}
+
+
+}}} // namespace lsst::qserv::loader
diff --git a/core/modules/loader/CentralFollower.h b/core/modules/loader/CentralFollower.h
new file mode 100644
index 0000000000..1603bafcf8
--- /dev/null
+++ b/core/modules/loader/CentralFollower.h
@@ -0,0 +1,116 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2019 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CENTRAL_FOLLOWER_H
+#define LSST_QSERV_LOADER_CENTRAL_FOLLOWER_H
+
+// system headers
+#include
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/Central.h"
+#include "loader/DoList.h"
+#include "loader/Neighbor.h"
+#include "loader/ServerTcpBase.h"
+#include "loader/WorkerConfig.h"
+
+
+namespace lsst {
+namespace qserv {
+
+namespace proto {
+class WorkerKeysInfo;
+}
+
+namespace loader {
+
+/// This class is used as a base central class for servers that need to get
+/// lists of workers from the master.
+/// CentralFollower provides no service on its own. The derived classes must:
+/// call workerInfoReceive(data) to handle LoaderMsg::MAST_WORKER_INFO
+/// call getWorkerList()->workerListReceive(data) to handle LoaderMsg::MAST_WORKER_LIST
+/// messages sent from the master and call workerInfoReceive() as needed to
+/// handle LoaderMsg::MAST_WORKER_INFO.
+class CentralFollower : public Central {
+public:
+ typedef std::pair CompKeyPair;
+
+ CentralFollower(boost::asio::io_service& ioService,
+ std::string const& hostName_, std::string const& masterHost, int masterPortUdp,
+ int threadPoolSize, int loopSleepTime, int ioThreads, int fPortUdp)
+ : Central(ioService, masterHost, masterPortUdp, threadPoolSize, loopSleepTime, ioThreads),
+ _hostName(hostName_),
+ _udpPort(fPortUdp){
+ }
+
+ ~CentralFollower() override;
+
+ /// CentralFollower provides no service on its own. The derived classes must handle
+ /// messages sent from the master and call workerInfoReceive() as needed.
+ void startMonitoring() override;
+
+ std::string const& getHostName() const { return _hostName; }
+ int getUdpPort() const { return _udpPort; }
+
+ /// Only workers have TCP ports.
+ virtual int getTcpPort() const { return 0; }
+
+ /// Receive information about workers from the master.
+ bool workerInfoReceive(BufferUdp::Ptr const& data);
+
+ /// Returns a pointer to our worker list.
+ WWorkerList::Ptr const& getWorkerList();
+
+ std::string getOurLogId() const override { return "CentralFollower"; }
+
+protected:
+ /// Real workers need to check this for initial ranges.
+ virtual void checkForThisWorkerValues(uint32_t wId, std::string const& ip,
+ int portUdp, int portTcp, KeyRange& strRange) {};
+
+private:
+ const std::string _hostName; ///< our host name
+ const int _udpPort; ///< our UDP port
+
+ /// This function is needed to fill the map. On real workers, CentralWorker
+ /// needs to do additional work to set its own id.
+ void _workerInfoReceive(std::unique_ptr& protoBuf);
+
+ /// Maps of workers with their key ranges and network addresses. due to
+ /// lazy initialization.
+ /// This should be accessed through getWorkerList()
+ WWorkerList::Ptr _wWorkerList;
+ std::mutex _wListInitMtx; ///< mutex for initialization of _wWorkerList
+ std::atomic _destroy{false};
+};
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CENTRAL_FOLLOWER_H
+
+
diff --git a/core/modules/loader/CentralMaster.cc b/core/modules/loader/CentralMaster.cc
new file mode 100644
index 0000000000..87ef96889a
--- /dev/null
+++ b/core/modules/loader/CentralMaster.cc
@@ -0,0 +1,213 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+
+// Class header
+#include "loader/CentralMaster.h"
+
+// system headers
+#include
+
+// Third-party headers
+#include "boost/asio.hpp"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.CentralMaster");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+void CentralMaster::startService() {
+ _server = std::make_shared(ioService, getMasterHostName(), getMasterPort(), this);
+}
+
+
+void CentralMaster::addWorker(std::string const& ip, int udpPort, int tcpPort) {
+ auto item = _mWorkerList->addWorker(ip, udpPort, tcpPort);
+
+ if (item != nullptr) {
+ // If that was the first worker added, it gets unlimited range.
+ if (_firstWorkerRegistered.exchange(true) == false) {
+ LOGS(_log, LOG_LVL_INFO, "setAllInclusiveRange for name=" << item->getId());
+ item->setAllInclusiveRange();
+ }
+
+ item->addDoListItems(this);
+ LOGS(_log, LOG_LVL_INFO, "Master::addWorker " << *item);
+ }
+}
+
+
+void CentralMaster::updateWorkerInfo(uint32_t workerId, NeighborsInfo const& nInfo, KeyRange const& strRange) {
+ if (workerId == 0) {
+ return;
+ }
+ auto item = getWorkerWithId(workerId);
+ if (item == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralMaster::updateNeighbors nullptr for workerId=" << workerId);
+ return;
+ }
+ // TODO setting nInfo and strRange can be done in one call to reduce mutex locking.
+ item->setNeighborsInfo(nInfo);
+ item->setRangeString(strRange);
+ _assignNeighborIfNeeded(workerId, item);
+}
+
+
+void CentralMaster::setWorkerNeighbor(MWorkerListItem::WPtr const& target, int message, uint32_t neighborId) {
+ // Get the target worker's network address
+ auto targetWorker = target.lock();
+ if (targetWorker == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralMaster::setWorkerNeighbor nullptr for " << neighborId);
+ return;
+ }
+
+ LOGS(_log, LOG_LVL_DEBUG, "CentralMaster::setWorkerNeighbor " << neighborId << " " << *targetWorker);
+ // Build and send the message
+ LoaderMsg msg(message, getNextMsgId(), getMasterHostName(), getMasterPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ UInt32Element neighborIdElem(neighborId);
+ neighborIdElem.appendToData(msgData);
+ auto addr = targetWorker->getUdpAddress();
+ try {
+ sendBufferTo(addr.ip, addr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralMaster::setWorkerNeighbor boost system_error=" << e.what() <<
+ " targ=" << *targetWorker << " msg=" << message <<
+ " neighborId=" << neighborId);
+ }
+}
+
+
+void CentralMaster::_assignNeighborIfNeeded(uint32_t workerId, MWorkerListItem::Ptr const& wItem) {
+ // Go through the list and see if all the workers are full.
+ // If they are, assign a worker to the end (rightmost) worker
+ // and increase the maximum by an order of magnitude, max 10 million.
+ // TODO Make a better algorithm, insert workers at busiest worker.
+ std::string funcName("_assignNeighborIfNeeded");
+ LOGS(_log, LOG_LVL_DEBUG, funcName);
+ if (_addingWorkerId != 0 && _addingWorkerId != workerId) {
+ // Already in process of adding a worker, and the worker
+ // with new information wasn't the one added. Nothing to do.
+ // TODO Check if it failed. (May need to go in a timer thread instead)
+ return;
+ }
+ // Only one thread should ever be working on this logic at a time.
+ std::lock_guard lck(_assignMtx);
+ if (_addingWorkerId != 0) {
+ if (_addingWorkerId == workerId) {
+ auto rng = wItem->getRangeString();
+ if (rng.getValid()) {
+ wItem->setActive(true);
+ LOGS(_log, LOG_LVL_INFO, "Successfully activated wId=" << workerId <<
+ " range=" << rng);
+ _addingWorkerId = 0;
+ }
+ }
+ }
+
+ auto pair = _mWorkerList->getActiveInactiveWorkerLists();
+ std::vector& activeList = pair.first;
+ std::vector& inactiveList = pair.second;
+ if (inactiveList.empty() || _addingWorkerId != 0) { return; }
+ double sum = 0.0;
+ int max = 0;
+ uint32_t maxWId = 0;
+ uint32_t rightMostName = 0; // Name of the rightmost worker, unlimited upper range.
+ MWorkerListItem::Ptr rightMostItem;
+ for(auto& item : activeList) {
+ int keyCount = item->getKeyCount();
+ sum += keyCount;
+ if (keyCount > max) {
+ max = keyCount;
+ maxWId = item->getId();
+ }
+ auto range = item->getRangeString();
+ if (range.getValid() && range.getUnlimited()) {
+ if (rightMostName != 0) {
+ std::string eStr("Multiple rightMost workers name=");
+ eStr += rightMostName + " name=" + item->getId();
+ LOGS(_log, LOG_LVL_ERROR, "_assignNeighborIfNeeded " << eStr);
+ throw LoaderMsgErr(ERR_LOC, eStr);
+ }
+ rightMostName = item->getId();
+ rightMostItem = item;
+ }
+ }
+ if (rightMostItem == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, funcName << " no rightmost worker found when expected.");
+ return;
+ }
+ double avg = sum/(double)(activeList.size());
+ LOGS(_log, LOG_LVL_INFO, "max=" << max << " maxWId=" << maxWId << " avg=" << avg);
+ if (avg > getMaxKeysPerWorker()) {
+ // Assign a neighbor to the rightmost worker, if there are any unused nodes.
+ // TODO Probably better to assign a new neighbor next to the node with the most recent activity.
+ // but that's much more complicated.
+ LOGS(_log, LOG_LVL_INFO, "ADDING WORKER avg=" << avg);
+ auto inactiveItem = inactiveList.front();
+ if (inactiveItem == nullptr) {
+ throw LoaderMsgErr(ERR_LOC,"_assignNeighborIfNeeded unexpected inactiveList nullptr");
+ }
+ _addingWorkerId = inactiveItem->getId();
+ // Sequence of events goes something like
+ // 1) left item gets message from master that it is getting a right neighbor, writes it down.
+ // 2) Right item get message from master that it is getting a left neighbor, writes it down.
+ // 3) CentralWorker::_monitor() on the left node(rightmostItem) connects to the right
+ // node(inactiveItem), ranges are setup and shifts are started.
+ // 4) When message received from the new worker saying that it has a valid range,
+ // set _addingWorkerId to 0. This check happens earlier in this function.
+ //
+ // Steps 1 and 2
+ rightMostItem->setRightNeighbor(inactiveItem);
+ inactiveItem->setLeftNeighbor(rightMostItem);
+ }
+}
+
+MWorkerListItem::Ptr CentralMaster::getWorkerWithId(uint32_t id) {
+ return _mWorkerList->getWorkerWithId(id);
+}
+
+
+void CentralMaster::reqWorkerKeysInfo(uint64_t msgId, std::string const& targetIp, short targetPort,
+ std::string const& ourHostName, short ourPort) {
+ LoaderMsg reqMsg(LoaderMsg::WORKER_KEYS_INFO_REQ, msgId, ourHostName, ourPort);
+ BufferUdp data;
+ reqMsg.appendToData(data);
+ try {
+ sendBufferTo(targetIp, targetPort, data);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralMaster::reqWorkerKeysInfo boost system_error=" << e.what() <<
+ " msgId=" << msgId << " tIp=" << targetIp << " tPort=" << targetPort <<
+ " ourHost=" << ourHostName << " ourPort=" << ourPort);
+ }
+}
+
+}}} // namespace lsst::qserv::loader
diff --git a/core/modules/loader/CentralMaster.h b/core/modules/loader/CentralMaster.h
new file mode 100644
index 0000000000..24a934d8e4
--- /dev/null
+++ b/core/modules/loader/CentralMaster.h
@@ -0,0 +1,102 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CENTRALMASTER_H
+#define LSST_QSERV_LOADER_CENTRALMASTER_H
+
+// system headers
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/Central.h"
+#include "loader/MasterConfig.h"
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+/// Central Master is the central element of the master. It maintains a DoList
+/// and a list of all workers including their addresses, key ranges, and number of keys
+/// on each worker. The authoritative ranges come from the workers. The ranges in the
+/// master's list may be out of date but the system should handle it gracefully.
+///
+/// Workers register with the master when they start and are inactive until the master
+/// gives them a valid range or a neighbor. The first worker activated has a range
+/// covering all possible keys.
+class CentralMaster : public Central {
+public:
+ /// Base class basic constructor, copy constructor, and operator= set to delete.
+ CentralMaster(boost::asio::io_service& ioService_, std::string const& masterHostName_,
+ MasterConfig const& cfg)
+ : Central(ioService_, masterHostName_, cfg.getMasterPort(),
+ cfg.getThreadPoolSize(), cfg.getLoopSleepTime(), cfg.getIOThreads()),
+ _maxKeysPerWorker(cfg.getMaxKeysPerWorker()) {}
+
+ /// Open the UDP port. This can throw boost::system::system_error.
+ void startService() override;
+
+ ~CentralMaster() override { _mWorkerList.reset(); }
+
+ void setMaxKeysPerWorker(int val) { _maxKeysPerWorker = val; }
+ int getMaxKeysPerWorker() const { return _maxKeysPerWorker; }
+
+ void addWorker(std::string const& ip, int udpPort, int tcpPort); ///< Add a new worker to the system.
+ void updateWorkerInfo(uint32_t workerId, NeighborsInfo const& nInfo, KeyRange const& strRange);
+
+ MWorkerListItem::Ptr getWorkerWithId(uint32_t id);
+
+ MWorkerList::Ptr getWorkerList() const { return _mWorkerList; }
+
+ void reqWorkerKeysInfo(uint64_t msgId, std::string const& targetIp, short targetPort,
+ std::string const& ourHostName, short ourPort);
+
+ std::string getOurLogId() const override { return "master"; }
+
+ void setWorkerNeighbor(MWorkerListItem::WPtr const& target, int message, uint32_t neighborId);
+
+private:
+ /// Upon receiving new worker information, check if an inactive worker should be made active.
+ void _assignNeighborIfNeeded(uint32_t workerId, MWorkerListItem::Ptr const& wItem);
+
+ std::mutex _assignMtx; ///< Protects critical region where worker's can be set to active.
+
+ std::atomic _maxKeysPerWorker{1000}; // TODO load from config file.
+
+ MWorkerList::Ptr _mWorkerList{std::make_shared(this)}; ///< List of workers.
+
+ std::atomic _firstWorkerRegistered{false}; ///< True when one worker has been activated.
+
+ /// The id of the worker being added. '0' indicates no worker being added.
+ /// Its value can only be set to non-zero values within _assignNeighborIfNeeded(...).
+ std::atomic _addingWorkerId{0}; // TODO maybe move _addingWorkerId to MWorkerList.
+};
+
+}}} // namespace lsst::qserv::loader
+
+
+#endif // LSST_QSERV_LOADER_CENTRALMASTER_H
diff --git a/core/modules/loader/CentralWorker.cc b/core/modules/loader/CentralWorker.cc
new file mode 100644
index 0000000000..a6511ffa5a
--- /dev/null
+++ b/core/modules/loader/CentralWorker.cc
@@ -0,0 +1,1143 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 AURA/LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+
+// Class header
+#include "loader/CentralWorker.h"
+
+// system headers
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// qserv headers
+#include "loader/BufferUdp.h"
+#include "loader/CentralWorkerDoListItem.h"
+#include "loader/LoaderMsg.h"
+#include "loader/WorkerConfig.h"
+#include "proto/loader.pb.h"
+#include "proto/ProtoImporter.h"
+#include "util/Timer.h"
+
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.CentralWorker");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+CentralWorker::CentralWorker(boost::asio::io_service& ioService_, boost::asio::io_context& io_context_,
+ std::string const& hostName_, WorkerConfig const& cfg)
+ : CentralFollower(ioService_, hostName_, cfg.getMasterHost(), cfg.getMasterPortUdp(),
+ cfg.getThreadPoolSize(), cfg.getLoopSleepTime(), cfg.getIOThreads(),cfg.getWPortUdp()),
+ _tcpPort(cfg.getWPortTcp()),
+ _ioContext(io_context_),
+ _recentAddLimit(cfg.getRecentAddLimit()),
+ _thresholdNeighborShift(cfg.getThresholdNeighborShift()),
+ _maxKeysToShift(cfg.getMaxKeysToShift()) {
+}
+
+
+void CentralWorker::startService() {
+ _server = std::make_shared(ioService, getHostName(), getUdpPort(), this);
+ _tcpServer = std::make_shared(_ioContext, _tcpPort, this);
+ _tcpServer->runThread();
+}
+
+
+CentralWorker::~CentralWorker() {
+ // Members that contain pointers to this. Deleting while this != null.
+ // TODO: Wait for reference count to drop to one or less,
+ // although CentralWorker is never really shutdown.
+ _tcpServer.reset();
+}
+
+
+std::string CentralWorker::getOurLogId() const {
+ std::stringstream os;
+ os << "(w name=" << _ourId << " addr=" << getHostName() <<
+ ":udp=" << getUdpPort() << " tcp=" << _tcpPort << ")";
+ return os.str();
+}
+
+void CentralWorker::startMonitoring() {
+ CentralFollower::startMonitoring();
+ // Add _workerList to _doList so it starts checking new entries.
+ _centralWorkerDoListItem = std::make_shared(this);
+ doList->addItem(_centralWorkerDoListItem);
+}
+
+
+void CentralWorker::_monitor() {
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_monitor");
+
+ /// If our id is invalid, try registering with the master.
+ if (_isOurIdInvalid()) {
+ _registerWithMaster();
+ // Give the master a half second to answer.
+ usleep(500000);
+ return;
+ }
+
+ // If data gets shifted, check everything again as ranges will have
+ // changed and there may be a lot more data to shift.
+ bool dataShifted = false;
+ do {
+ // TODO Check if we've heard from left neighbor (possibly kill connection if nothing heard???)
+
+ // Check the right neighbor connection, kill and restart if needed.
+ // Check if data needs to be shifted with the right node
+ // This mutex is locked for a long time TODO break this up?
+ std::lock_guard rMtxLG(_rightMtx);
+ LOGS(_log, LOG_LVL_INFO, "_monitor " << _ourId <<
+ " checking right neighbor " << _neighborRight.getId());
+ if (_neighborRight.getId() != 0) {
+ try {
+ if (not _neighborRight.getEstablished()) {
+ LOGS(_log, LOG_LVL_INFO, "_monitor " << _ourId << " trying to connect");
+ auto nAddr = _neighborRight.getAddressTcp();
+ if (nAddr.ip == "") {
+ // look up the network address for the rightNeighbor
+ WWorkerListItem::Ptr nWorker =
+ getWorkerList()->getWorkerWithId(_neighborRight.getId());
+ if (nWorker != nullptr) {
+ auto addrTcp = nWorker->getTcpAddress();
+ auto addrUdp = nWorker->getUdpAddress();
+ if (addrTcp.ip.empty() || addrUdp.ip.empty()) {
+ throw LoaderMsgErr(ERR_LOC, "Missing valid address for neighbor=" +
+ std::to_string(_neighborRight.getId()));
+ }
+ LOGS(_log, LOG_LVL_INFO, "_monitor neighbor right " <<
+ _neighborRight.getId() << " T=" << addrTcp << " U=" << addrUdp);
+ _neighborRight.setAddressTcp(addrTcp);
+ _neighborRight.setAddressUdp(addrUdp);
+ }
+ }
+
+ LOGS(_log, LOG_LVL_INFO, "_monitor trying to establish TCP connection with " <<
+ _neighborRight.getId() << " " << _neighborRight.getAddressTcp());
+ _rightConnect(rMtxLG); // calls _determineRange() while establishing connection
+ } else {
+ LOGS(_log, LOG_LVL_INFO, "_monitor " << _ourId << " getting range info");
+ if (_determineRange()) {
+ _rangeChanged = true;
+ }
+ }
+ dataShifted = _shiftIfNeeded(rMtxLG);
+ } catch (LoaderMsgErr const& ex) {
+ LOGS(_log, LOG_LVL_ERROR, "_monitor() catching exception " << ex.what());
+ _rightDisconnect(rMtxLG, "_monitor msgErr ex");
+ } catch (boost::system::system_error const& ex) {
+ LOGS(_log, LOG_LVL_ERROR, "_monitor() catching boost exception " << ex.what());
+ _rightDisconnect(rMtxLG, "_monitor boost ex");
+ }
+ } else {
+ // If there is a connection, close it.
+ _rightDisconnect(rMtxLG, "_monitor closing since _neighborRight.getId()=0");
+ }
+ if (_rangeChanged) {
+ // Send new range to master so all clients and workers can be updated.
+ _rangeChanged = false;
+ LOGS(_log, LOG_LVL_INFO, "_monitor updating range with master");
+ NetworkAddress masterAddr(getMasterHostName(), getMasterPort());
+ _sendWorkerKeysInfo(masterAddr, getNextMsgId());
+ }
+ } while (dataShifted);
+}
+
+
+bool CentralWorker::_setOurId(uint32_t id) {
+ std::lock_guard lck(_ourIdMtx);
+ if (_ourIdInvalid) {
+ _ourId = id;
+ _ourIdInvalid = false;
+ return true;
+ } else {
+ /// TODO add error message, check if _ourId matches id
+ if (id == 0) {
+ _masterDisable();
+ } else if (id != _ourId) {
+ LOGS(_log, LOG_LVL_ERROR, "worker=" << _ourId <<
+ " id being changed by master!!! new id=" << id);
+ }
+ return false;
+ }
+}
+
+
+uint32_t CentralWorker::getOurId() const {
+ std::lock_guard lck(_ourIdMtx);
+ return _ourId;
+}
+
+
+void CentralWorker::_masterDisable() {
+ LOGS(_log, LOG_LVL_INFO, "worker=" << _ourId <<
+ " changed to 0, master shutting this down.");
+ _ourIdInvalid = true;
+ // Disconnect from right neighbor.
+ {
+ std::lock_guard rMtxLG(_rightMtx);
+ _rightDisconnect(rMtxLG, "_masterDisable");
+ _neighborRight.setId(0);
+ }
+ // Disconnect from left neighbor. TODO actively kill the left connection.
+ _neighborLeft.setId(0);
+ // TODO invalidate range and _keyValueMap
+}
+
+
+bool CentralWorker::_determineRange() {
+ std::string const funcName("CentralWorker::_determineRange");
+ bool rangeChanged = false;
+ BufferUdp data(2000);
+ {
+ data.reset();
+ UInt32Element imLeftKind(LoaderMsg::IM_YOUR_L_NEIGHBOR);
+ imLeftKind.appendToData(data);
+ // Send information about how many keys on this node and their range.
+ StringElement strElem;
+ std::unique_ptr protoWKI = workerKeysInfoBuilder();
+ protoWKI->SerializeToString(&(strElem.element));
+ UInt32Element bytesInMsg(strElem.transmitSize());
+ // Must send the number of bytes in the message so TCP server knows how many bytes to read.
+ bytesInMsg.appendToData(data);
+ strElem.appendToData(data);
+ ServerTcpBase::writeData(*_rightSocket, data);
+ }
+ // Get back their basic info
+ {
+ data.reset();
+ auto msgElem = data.readFromSocket(*_rightSocket, funcName + " - range bytes");
+ auto bytesInMsg = std::dynamic_pointer_cast(msgElem);
+ msgElem = data.readFromSocket(*_rightSocket, funcName + " - range info");
+ auto strWKI = std::dynamic_pointer_cast(msgElem);
+ auto protoItem = strWKI->protoParse();
+ if (protoItem == nullptr) {
+ LOGS(_log, LOG_LVL_ERROR, funcName << " protoItem parse issue!!!!!");
+ throw LoaderMsgErr(ERR_LOC, "protoItem parse issue!!!!!");
+ }
+ NeighborsInfo nInfoR;
+ auto workerId = protoItem->wid();
+ nInfoR.keyCount = protoItem->mapsize();
+ _neighborRight.setKeyCount(nInfoR.keyCount); // TODO add a timestamp to this data.
+ nInfoR.recentAdds = protoItem->recentadds();
+ proto::WorkerRange protoRange = protoItem->range();
+ LOGS(_log, LOG_LVL_INFO, funcName << " rightNeighbor workerId=" << workerId <<
+ " keyCount=" << nInfoR.keyCount << " recentAdds=" << nInfoR.recentAdds);
+ bool valid = protoRange.valid();
+ KeyRange rightRange;
+ if (valid) {
+ CompositeKey min(protoRange.minint(), protoRange.minstr());
+ CompositeKey max(protoRange.maxint(), protoRange.maxstr());
+ bool unlimited = protoRange.maxunlimited();
+ rightRange.setMinMax(min, max, unlimited);
+ LOGS(_log, LOG_LVL_INFO, funcName << " rightRange=" << rightRange);
+ _neighborRight.setRange(rightRange);
+ // Adjust our max range given the the right minimum information.
+ // Our maximum value is up to but not including the right minimum.
+ {
+ std::lock_guard lckMap(_idMapMtx);
+ auto origMax = _keyRange.getMax();
+ auto origUnlim = _keyRange.getUnlimited();
+ // Can't be unlimited anymore as there is a right neighbor.
+ _keyRange.setMax(min, false);
+ if (origUnlim != _keyRange.getUnlimited() ||
+ (!origUnlim && origMax != _keyRange.getMax())) {
+ rangeChanged = true;
+ }
+ }
+ }
+ proto::Neighbor protoLeftNeigh = protoItem->left();
+ nInfoR.neighborLeft->update(protoLeftNeigh.wid()); // Not really useful in this case.
+ proto::Neighbor protoRightNeigh = protoItem->right();
+ nInfoR.neighborRight->update(protoRightNeigh.wid()); // This should be our id
+ if (nInfoR.neighborLeft->get() != getOurId()) {
+ LOGS(_log, LOG_LVL_ERROR, "Our (" << getOurId() <<
+ ") right neighbor does not have our name as its left neighbor" );
+ }
+ }
+ return rangeChanged;
+}
+
+
+// must hold _rightMtx before calling
+bool CentralWorker::_shiftIfNeeded(std::lock_guard const& rightMtxLG) {
+ // There should be reasonably recent information from our neighbors. Use that
+ // and our status to ask the right neighbor to give us entries or we send entries
+ // to the right neighbor.
+ // If right connection is not established, return
+ if (not _neighborRight.getEstablished()) {
+ LOGS(_log, LOG_LVL_INFO, "_shiftIfNeeded no right neighbor, no shift.");
+ return false;
+ }
+ if (_shiftAsClientInProgress) {
+ LOGS(_log, LOG_LVL_INFO, "_shiftIfNeeded shift already in progress.");
+ return false;
+ }
+
+ // Get local copies of range and map info.
+ KeyRange range;
+ size_t mapSize;
+ {
+ std::lock_guard lck(_idMapMtx);
+ range = _keyRange;
+ mapSize = _keyValueMap.size();
+ }
+
+ // If this worker has more keys than the rightNeighbor, consider shifting keys to the right neighbor.
+ // If this worker has _thresholdAverage more keys than average or _thresholdNeighborShift more keys than the right neighbor
+ // send enough keys to the right to balance (min 1 key, max _maxShiftKeys, never shift more than 1/3 of our keys)
+ int rightKeyCount = 0;
+ KeyRange rightRange;
+ _neighborRight.getKeyData(rightKeyCount, rightRange);
+ if (range > rightRange) {
+ LOGS(_log, LOG_LVL_ERROR, "Right neighbor range is less than ours!!!! our=" << range << " right=" << rightRange);
+ return false;
+ }
+ int keysToShift = 0;
+ CentralWorker::Direction direction = NONE0;
+ int sourceSize = 0;
+ LOGS(_log, LOG_LVL_INFO, "_shiftIfNeeded _monitor thisSz=" << mapSize << " rightSz=" << rightKeyCount);
+ if (mapSize > rightKeyCount*_thresholdNeighborShift) { // TODO add average across workers check
+ keysToShift = (mapSize - rightKeyCount)/2; // Try for nearly equal number of keys on each.
+ direction = TORIGHT1;
+ sourceSize = mapSize;
+ } else if (mapSize*_thresholdNeighborShift < rightKeyCount) { // TODO add average across workers check
+ keysToShift = (rightKeyCount - mapSize)/2; // Try for nearly equal number of keys on each.
+ direction = FROMRIGHT2;
+ sourceSize = rightKeyCount;
+ } else {
+ LOGS(_log, LOG_LVL_INFO, "No reason to shift.");
+ return false;
+ }
+ if (keysToShift > _maxKeysToShift) keysToShift = _maxKeysToShift;
+ if (keysToShift > sourceSize/3) keysToShift = sourceSize/3;
+ if (keysToShift < 1) {
+ LOGS(_log, LOG_LVL_INFO, "Worker doesn't have enough keys to shift.");
+ return false;
+ }
+ _shiftAsClientInProgress = true;
+ LOGS(_log, LOG_LVL_INFO, "shift dir(TO1 FROM2)=" << direction << " keys=" << keysToShift <<
+ " szThis=" << mapSize << " szRight=" << rightKeyCount);
+ _shift(direction, keysToShift);
+ return true;
+}
+
+
+void CentralWorker::_shift(Direction direction, int keysToShift) {
+ std::string const fName("CentralWorker::_shift");
+ LOGS(_log, LOG_LVL_DEBUG, fName);
+ if (direction == FROMRIGHT2) {
+ BufferUdp data(1000000);
+ // Construct a message asking for keys to shift (it will shift its lowest keys, which will be our highest keys)
+ proto::KeyShiftRequest protoKeyShiftRequest;
+ protoKeyShiftRequest.set_keystoshift(keysToShift);
+ {
+ StringElement keyShiftReq;
+ protoKeyShiftRequest.SerializeToString(&(keyShiftReq.element));
+ // Send the message kind, followed by the transmit size, and then the protobuffer.
+ UInt32Element kindShiftFromRight(LoaderMsg::SHIFT_FROM_RIGHT);
+ UInt32Element bytesInMsg(keyShiftReq.transmitSize());
+ BufferUdp data(kindShiftFromRight.transmitSize() + bytesInMsg.transmitSize() + keyShiftReq.transmitSize());
+ kindShiftFromRight.appendToData(data);
+ bytesInMsg.appendToData(data);
+ keyShiftReq.appendToData(data);
+ LOGS(_log, LOG_LVL_INFO, fName << " FROMRIGHT " << keysToShift);
+ ServerTcpBase::writeData(*_rightSocket, data);
+ }
+ // Wait for the KeyList response
+ {
+ data.reset();
+ auto msgElem = data.readFromSocket(*_rightSocket,
+ fName + " waiting for FROMRIGHT KeyList");
+ auto keyListElem = std::dynamic_pointer_cast(msgElem);
+ if (keyListElem == nullptr) {
+ throw LoaderMsgErr(ERR_LOC, fName +" FROMRIGHT failure to get KeyList");
+ }
+ auto protoKeyList = keyListElem->protoParse();
+ if (protoKeyList == nullptr) {
+ throw LoaderMsgErr(ERR_LOC, fName + " FROMRIGHT failure to parse KeyList size=" +
+ std::to_string(keyListElem->element.size()));
+ }
+
+ // TODO This is very similar to code in TcpBaseConnection::_handleShiftToRight and they should be merged.
+ int sz = protoKeyList->keypair_size();
+ std::vector keyList;
+ for (int j=0; j < sz; ++j) {
+ proto::KeyInfo const& protoKI = protoKeyList->keypair(j);
+ ChunkSubchunk chSub(protoKI.chunk(), protoKI.subchunk());
+ keyList.push_back(std::make_pair(CompositeKey(protoKI.keyint(), protoKI.keystr()), chSub));
+ }
+ insertKeys(keyList, false);
+ }
+ // Send received message
+ data.reset();
+ UInt32Element elem(LoaderMsg::SHIFT_FROM_RIGHT_RECEIVED);
+ elem.appendToData(data);
+ ServerTcpBase::writeData(*_rightSocket, data);
+ LOGS(_log, LOG_LVL_INFO, fName << " direction=" << direction << " keys=" << keysToShift);
+
+ } else if (direction == TORIGHT1) {
+ LOGS(_log, LOG_LVL_INFO, fName << " TORIGHT " << keysToShift);
+ // TODO this is very similar to CentralWorker::buildKeyList() and should be merged with that.
+ // Construct a message with that many keys and send it (sending the highest keys)
+ proto::KeyList protoKeyList;
+ protoKeyList.set_keycount(keysToShift);
+ CompositeKey minKey = CompositeKey::minValue; // smallest key is sent to right neighbor
+ {
+ std::lock_guard lck(_idMapMtx);
+ if (not _transferListToRight.empty()) {
+ throw LoaderMsgErr(ERR_LOC, fName + " _transferList not empty");
+ }
+ for (int j=0; j < keysToShift && _keyValueMap.size() > 1; ++j) {
+ auto iter = _keyValueMap.end();
+ --iter; // rbegin() returns a reverse iterator which doesn't work with erase().
+ _transferListToRight.push_back(std::make_pair(iter->first, iter->second));
+ proto::KeyInfo* protoKI = protoKeyList.add_keypair();
+ minKey = iter->first;
+ protoKI->set_keyint(minKey.kInt);
+ protoKI->set_keystr(minKey.kStr);
+ protoKI->set_chunk(iter->second.chunk);
+ protoKI->set_subchunk(iter->second.subchunk);
+ _keyValueMap.erase(iter);
+ }
+ // Adjust our range;
+ _keyRange.setMax(minKey);
+ }
+ StringElement keyList;
+ protoKeyList.SerializeToString(&(keyList.element));
+ // Send the message kind, followed by the transmit size, and then the protobuffer.
+ UInt32Element kindShiftRight(LoaderMsg::SHIFT_TO_RIGHT);
+ UInt32Element bytesInMsg(keyList.transmitSize());
+ BufferUdp data(kindShiftRight.transmitSize() + bytesInMsg.transmitSize() + keyList.transmitSize());
+ if (data.getMaxLength() > TcpBaseConnection::getMaxBufSize()) {
+ std::string errMsg = fName + " SHIFT_TO_RIGHT FAILED message too big sz=" +
+ std::to_string(data.getMaxLength()) +
+ " max=" + std::to_string(TcpBaseConnection::getMaxBufSize());
+ LOGS(_log, LOG_LVL_ERROR, errMsg);
+ // This will keep getting thrown and never work, but at least it will show up
+ // in the logs.
+ // TODO Maybe create new exception, catch it and halve the number of keys to shift?
+ throw LoaderMsgErr(ERR_LOC, errMsg);
+ }
+ kindShiftRight.appendToData(data);
+ bytesInMsg.appendToData(data);
+ keyList.appendToData(data);
+
+ LOGS(_log, LOG_LVL_INFO, fName << " TORIGHT sending keys");
+ ServerTcpBase::writeData(*_rightSocket, data);
+
+ // read back LoaderMsg::SHIFT_TO_RIGHT_KEYS_RECEIVED
+ data.reset();
+ auto msgElem = data.readFromSocket(*_rightSocket,
+ fName + " SHIFT_TO_RIGHT_KEYS_RECEIVED");
+ UInt32Element::Ptr received = std::dynamic_pointer_cast(msgElem);
+ LOGS(_log, LOG_LVL_INFO, fName << " TORIGHT keys were received");
+ if (received == nullptr || received->element != LoaderMsg::SHIFT_TO_RIGHT_RECEIVED) {
+ throw LoaderMsgErr(ERR_LOC, fName +" receive failure");
+ }
+ _finishShiftToRight();
+ LOGS(_log, LOG_LVL_INFO, fName + " end direction=" << direction << " keys=" << keysToShift);
+ }
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_shift DumpKeys " << dumpKeysStr(2));
+ _shiftAsClientInProgress = false;
+}
+
+
+
+
+
+void CentralWorker::_finishShiftToRight() {
+ std::lock_guard lck(_idMapMtx);
+ _transferListToRight.clear();
+}
+
+
+void CentralWorker::finishShiftFromRight() {
+ std::lock_guard lck(_idMapMtx);
+ _transferListWithLeft.clear();
+}
+
+
+StringElement::UPtr CentralWorker::buildKeyList(int keysToShift) {
+ std::string funcName = "CentralWorker::buildKeyList";
+ proto::KeyList protoKeyList;
+ CompositeKey minKey = CompositeKey::minValue; // smallest key sent
+ CompositeKey maxKey = CompositeKey::minValue; // largest key sent
+ {
+ LOGS(_log, LOG_LVL_INFO, funcName);
+ std::lock_guard lck(_idMapMtx);
+ if (not _transferListWithLeft.empty()) {
+ throw LoaderMsgErr(ERR_LOC, "_shift _transferListFromRight not empty");
+ }
+ int maxKeysToShift = _keyValueMap.size()/3;
+ if (keysToShift > maxKeysToShift) keysToShift = maxKeysToShift;
+ protoKeyList.set_keycount(keysToShift);
+ bool firstPass = true;
+ for (int j=0; j < keysToShift && _keyValueMap.size() > 1; ++j) {
+ auto iter = _keyValueMap.begin();
+ if (firstPass) {
+ minKey = iter->first;
+ }
+ _transferListWithLeft.push_back(std::make_pair(iter->first, iter->second));
+ proto::KeyInfo* protoKI = protoKeyList.add_keypair();
+ maxKey = iter->first;
+ protoKI->set_keyint(maxKey.kInt);
+ protoKI->set_keystr(maxKey.kStr);
+ protoKI->set_chunk(iter->second.chunk);
+ protoKI->set_subchunk(iter->second.subchunk);
+ _keyValueMap.erase(iter);
+ }
+ // Adjust our range;
+ auto iter = _keyValueMap.begin();
+ auto minKey = _keyRange.getMin();
+ if (minKey != CompositeKey::minValue) {
+ if (iter->first != minKey) {
+ _keyRange.setMin(iter->first);
+ _rangeChanged = true;
+ }
+ }
+ }
+ StringElement::UPtr keyList(new StringElement());
+ protoKeyList.SerializeToString(&(keyList->element));
+ return keyList;
+}
+
+
+/// Must hold _rightMtx before calling
+void CentralWorker::_rightConnect(std::lock_guard const& rightMtxLG) {
+ std::string const funcName("CentralWorker::_rightConnect");
+ if(_rightConnectStatus == VOID0) {
+ LOGS(_log, LOG_LVL_INFO, funcName + " starting rightConnection");
+ _rightConnectStatus = STARTING1;
+ // Connect to the right neighbor server
+ AsioTcp::resolver resolver(_ioContext);
+ auto addr = _neighborRight.getAddressTcp();
+ AsioTcp::resolver::results_type endpoints = resolver.resolve(addr.ip, std::to_string(addr.port));
+ _rightSocket.reset(new AsioTcp::socket(_ioContext));
+ boost::system::error_code ec;
+ boost::asio::connect(*_rightSocket, endpoints, ec);
+ if (ec) {
+ _rightSocket.reset();
+ LOGS(_log, LOG_LVL_WARN, "failed to connect to " << _neighborRight.getId() << " " <<
+ addr << " ec=" << ec.value() << ":" << ec.message());
+ return;
+ }
+
+ // Get name from server
+ BufferUdp data(2000);
+ {
+ auto msgElem = data.readFromSocket(*_rightSocket, "CentralWorker::_rightConnect");
+ // First element should be UInt32Element with the other worker's name
+ UInt32Element::Ptr nghName = std::dynamic_pointer_cast(msgElem);
+ if (nghName == nullptr) {
+ throw LoaderMsgErr(ERR_LOC, std::string("first element wasn't correct type ") +
+ msgElem->getStringVal());
+ }
+
+ // Check if correct name
+ if (nghName->element != _neighborRight.getId()) {
+ throw LoaderMsgErr(ERR_LOC, std::string("wrong name expected ") +
+ std::to_string(_neighborRight.getId()) +
+ " got " + std::to_string(nghName->element));
+ }
+ }
+
+ // Send our basic key info so ranges can be determined.
+ _determineRange();
+
+ _rightConnectStatus = ESTABLISHED2;
+ LOGS(_log, LOG_LVL_INFO, funcName + " established rightConnection");
+ _neighborRight.setEstablished(true);
+ }
+}
+
+
+void CentralWorker::setNeighborInfoLeft(uint32_t wId, int keyCount, KeyRange const& range) {
+ if (wId != _neighborLeft.getId()) {
+ LOGS(_log, LOG_LVL_ERROR, "disconnecting left since setNeighborInfoLeft wId(" << wId <<
+ ") != neighborLeft.name(" << _neighborLeft.getId() << ")");
+ _neighborLeft.setEstablished(false);
+ return;
+ }
+ _neighborLeft.setKeyCount(keyCount);
+ _neighborLeft.setRange(range);
+ _neighborLeft.setEstablished(true);
+}
+
+
+/// Must hold _rightMtx before calling
+void CentralWorker::_rightDisconnect(std::lock_guard const& lg, std::string const& note) {
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_rightDisconnect " << note);
+ if (_rightSocket != nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralWorker::_rightDisconnect disconnecting");
+ _rightSocket->shutdown(boost::asio::ip::tcp::socket::shutdown_both);
+ _rightSocket->close();
+ _neighborRight.setEstablished(false);
+ }
+ _rightConnectStatus = VOID0;
+ _cancelShiftsWithRightNeighbor();
+}
+
+
+void CentralWorker::_cancelShiftsWithRightNeighbor() {
+ // Client side of connection, was sending largest keys right.
+ // If keys were being shifted from right, this node's map is still intact.
+ LOGS(_log, LOG_LVL_DEBUG, "_cancelShiftsWithRightNeighbor");
+ std::lock_guard lck(_idMapMtx);
+ if (_shiftAsClientInProgress.exchange(false)) {
+ LOGS(_log, LOG_LVL_WARN, "Canceling shiftToRight neighbor");
+ // Restore the transfer list to the id map
+ for (auto&& elem:_transferListToRight) {
+ auto res = _keyValueMap.insert(std::make_pair(elem.first, elem.second));
+ if (not res.second) {
+ LOGS(_log, LOG_LVL_WARN, "_cancelShiftsRightNeighbor Possible duplicate " <<
+ elem.first << ":" << elem.second);
+ }
+ }
+ _transferListToRight.clear();
+ // Leave the reduced range until fixed by our right neighbor.
+ }
+}
+
+
+void CentralWorker::cancelShiftsWithLeftNeighbor() {
+ // Server side of connection, was sending smallest keys left.
+ // If keys were being transfered from the left node, this node's map is still intact.
+ LOGS(_log, LOG_LVL_WARN, "cancelShiftsWithLeftNeighbor");
+ std::lock_guard lck(_idMapMtx);
+ if (not _transferListWithLeft.empty()) {
+ // Restore the transfer list to the id map
+ for (auto&& elem:_transferListWithLeft) {
+ auto res = _keyValueMap.insert(std::make_pair(elem.first, elem.second));
+ if (not res.second) {
+ LOGS(_log, LOG_LVL_WARN, "_cancelShiftsRightNeighbor Possible duplicate " <<
+ elem.first << ":" << elem.second);
+ }
+ }
+ _transferListWithLeft.clear();
+
+ // Fix the bottom of the range.
+ if (_keyRange.getMin() != CompositeKey::minValue) {
+ _keyRange.setMin(_keyValueMap.begin()->first);
+ }
+ }
+}
+
+
+void CentralWorker::checkForThisWorkerValues(uint32_t wId, std::string const& ip,
+ int portUdp, int portTcp, KeyRange& strRange) {
+ // If the address matches ours, check the name.
+ if (getHostName() == ip && getUdpPort() == portUdp) {
+ if (_isOurIdInvalid()) {
+ LOGS(_log, LOG_LVL_INFO, "Setting our name " << wId);
+ _setOurId(wId);
+ } else if (getOurId() != wId) {
+ LOGS(_log, LOG_LVL_ERROR, "Our wId doesn't match address from master! wId=" <<
+ getOurId() << " from master=" << wId);
+ }
+
+ // It is this worker. If there is a valid range in the message and our range is not valid,
+ // take the range given as our own.
+ if (strRange.getValid()) {
+ std::lock_guard lckM(_idMapMtx);
+ if (not _keyRange.getValid()) {
+ LOGS(_log, LOG_LVL_INFO, "Setting our range " << strRange);
+ _keyRange.setMinMax(strRange.getMin(), strRange.getMax(), strRange.getUnlimited());
+ }
+ }
+ }
+
+}
+
+
+KeyRange CentralWorker::updateRangeWithLeftData(KeyRange const& leftNeighborRange) {
+ // Update our range with data from our left neighbor. Our min is their max.
+ // If our range is invalid
+ // our min is their max incremented (stringRange increment function)
+ // if their max is unlimited, our max becomes unlimited
+ // else max = increment(min)
+ // send range to master
+ // return our new range
+ KeyRange newLeftNeighborRange(leftNeighborRange);
+ {
+ std::unique_lock lck(_idMapMtx);
+ if (not _keyRange.getValid()) {
+ // Our range has not been set, so base it on the range of the left neighbor.
+ auto min = KeyRange::increment(leftNeighborRange.getMax());
+ auto max = min;
+ _keyRange.setMinMax(min, max, leftNeighborRange.getUnlimited());
+ newLeftNeighborRange.setMax(max, false);
+ } else {
+ // Our range is valid already, it should be > than the left neighbor range.
+ if (_keyRange < leftNeighborRange) {
+ LOGS(_log, LOG_LVL_ERROR, "LeftNeighborRange(" << leftNeighborRange <<
+ ") is greater than our range(" << _keyRange << ")");
+ // TODO corrective action?
+ }
+ // The left neighbor's max should be the minimum value in our keymap, unless the
+ // map is empty.
+ if (_keyValueMap.empty()) {
+ // Don't do anything to left neighbor range.
+ } else {
+ auto min = _keyValueMap.begin()->first;
+ _keyRange.setMin(min);
+ newLeftNeighborRange.setMax(min, false);
+ }
+ }
+ }
+
+ return newLeftNeighborRange;
+}
+
+
+bool CentralWorker::workerKeyInsertReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ StringElement::Ptr sData = std::dynamic_pointer_cast(
+ MsgElement::retrieve(*data, " CentralWorker::workerKeyInsertReq"));
+ if (sData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralWorker::workerKeyInsertReq Failed to read list element");
+ return false;
+ }
+ auto protoData = sData->protoParse();
+ if (protoData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralWorker::workerKeyInsertReq Failed to parse list");
+ return false;
+ }
+
+ // TODO move this to another thread
+ _workerKeyInsertReq(inMsg, protoData);
+ return true;
+}
+
+
+void CentralWorker::_workerKeyInsertReq(LoaderMsg const& inMsg, std::unique_ptr& protoBuf) {
+ std::unique_ptr protoData(std::move(protoBuf));
+
+ // Get the source of the request
+ proto::LdrNetAddress protoAddr = protoData->requester();
+ NetworkAddress nAddr(protoAddr.ip(), protoAddr.udpport());
+
+ proto::KeyInfo protoKeyInfo = protoData->keyinfo();
+ CompositeKey key(protoKeyInfo.keyint(), protoKeyInfo.keystr());
+ ChunkSubchunk chunkInfo(protoKeyInfo.chunk(), protoKeyInfo.subchunk());
+
+ /// see if the key should be inserted into our map
+ std::unique_lock lck(_idMapMtx);
+ auto min = _keyRange.getMin();
+ auto leftAddress = _neighborLeft.getAddressUdp();
+ auto rightAddress = _neighborRight.getAddressUdp();
+ if (_keyRange.isInRange(key)) {
+ // insert into our map
+ auto res = _keyValueMap.insert(std::make_pair(key, chunkInfo));
+ lck.unlock();
+ if (not res.second) {
+ // Element already found, check file id and row number. Bad if not the same.
+ // TODO HIGH send back duplicate key mismatch message to the original requester and return
+ }
+ LOGS(_log, LOG_LVL_INFO, "Key inserted=" << key << "(" << chunkInfo << ")");
+ // TODO Send this item to the keyLogger (which would then send KEY_INSERT_COMPLETE back to the requester),
+ // for now this function will send the message back for proof of concept.
+ LoaderMsg msg(LoaderMsg::KEY_INSERT_COMPLETE, inMsg.msgId->element, getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ // protoKeyInfo should still be the same
+ proto::KeyInfo protoReply;
+ protoReply.set_keyint(key.kInt);
+ protoReply.set_keystr(key.kStr);
+ protoReply.set_chunk(chunkInfo.chunk);
+ protoReply.set_subchunk(chunkInfo.subchunk);
+ StringElement strElem;
+ protoReply.SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+ LOGS(_log, LOG_LVL_INFO, "sending complete " << key << " to " << nAddr << " from " << _ourId);
+ try {
+ sendBufferTo(nAddr.ip, nAddr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_workerKeyInsertReq boost system_error=" << e.what() <<
+ " msg=" << inMsg);
+ }
+ } else {
+ lck.unlock();
+ // Find the target range in the list and send the request there
+ auto targetWorker = getWorkerList()->findWorkerForKey(key);
+ if (targetWorker != nullptr && targetWorker->getId() != _ourId) {
+ _forwardKeyInsertRequest(targetWorker->getUdpAddress(), inMsg, protoData);
+ } else {
+ // Send request to left or right neighbor
+ if (key < min && leftAddress.ip != "") {
+ _forwardKeyInsertRequest(leftAddress, inMsg, protoData);
+ } else if (key > min && rightAddress.ip != "") {
+ _forwardKeyInsertRequest(rightAddress, inMsg, protoData);
+ }
+ }
+ }
+}
+
+
+void CentralWorker::_forwardKeyInsertRequest(NetworkAddress const& targetAddr, LoaderMsg const& inMsg,
+ std::unique_ptr& protoData) {
+ // Aside from hops, the proto buffer should be the same.
+ proto::KeyInfo protoKeyInfo = protoData->keyinfo();
+ CompositeKey key(protoKeyInfo.keyint(), protoKeyInfo.keystr());
+ // The proto buffer should be the same, just need a new message.
+ int hops = protoData->hops() + 1;
+ if (hops > 4) { // TODO replace magic number with variable set via config file.
+ LOGS(_log, LOG_LVL_WARN, "Too many hops, dropping insert request hops=" << hops << " key=" << key);
+ return;
+ }
+ LOGS(_log, LOG_LVL_INFO, "Forwarding key insert hops=" << hops << " key=" << key);
+ LoaderMsg msg(LoaderMsg::KEY_INSERT_REQ, inMsg.msgId->element, getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+
+ StringElement strElem;
+ protoData->SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+ try {
+ sendBufferTo(targetAddr.ip, targetAddr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_forwardKeyInsertRequest boost system_error=" << e.what() <<
+ " tAddr=" << targetAddr << " inMsg=" << inMsg);
+ }
+}
+
+
+bool CentralWorker::workerKeyInfoReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ LOGS(_log, LOG_LVL_DEBUG, "CentralWorker::workerKeyInfoReq");
+ StringElement::Ptr sData = std::dynamic_pointer_cast(
+ MsgElement::retrieve(*data, " CentralWorker::workerKeyInfoReq "));
+ if (sData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralWorker::workerKeyInfoReq Failed to read list element");
+ return false;
+ }
+ auto protoData = sData->protoParse();
+ if (protoData == nullptr) {
+ LOGS(_log, LOG_LVL_WARN, "CentralWorker::workerKeyInfoReq Failed to parse list");
+ return false;
+ }
+
+ // TODO move this to another thread
+ _workerKeyInfoReq(inMsg, protoData);
+ return true;
+}
+
+
+void CentralWorker::_workerKeyInfoReq(LoaderMsg const& inMsg, std::unique_ptr& protoBuf) {
+ std::unique_ptr protoData(std::move(protoBuf));
+
+ // Get the source of the request
+ proto::LdrNetAddress protoAddr = protoData->requester();
+ NetworkAddress nAddr(protoAddr.ip(), protoAddr.udpport());
+
+ proto::KeyInfo protoKeyInfo = protoData->keyinfo();
+ CompositeKey key(protoKeyInfo.keyint(), protoKeyInfo.keystr());
+
+ /// see if the key is in our map
+ std::unique_lock lck(_idMapMtx);
+ if (_keyRange.isInRange(key)) {
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_workerKeyInfoReq " << _ourId << " looking for key=" << key);
+ // check out map
+ auto iter = _keyValueMap.find(key);
+ lck.unlock();
+
+ // Key found or not, message will be returned.
+ LoaderMsg msg(LoaderMsg::KEY_LOOKUP, inMsg.msgId->element, getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ proto::KeyInfo protoReply;
+ protoReply.set_keyint(key.kInt);
+ protoReply.set_keystr(key.kStr);
+ if (iter == _keyValueMap.end()) {
+ // key not found message.
+ protoReply.set_chunk(0);
+ protoReply.set_subchunk(0);
+ protoReply.set_success(false);
+ LOGS(_log, LOG_LVL_INFO, "Key info not found key=" << key);
+ } else {
+ // key found message.
+ auto elem = iter->second;
+ protoReply.set_chunk(elem.chunk);
+ protoReply.set_subchunk(elem.subchunk);
+ protoReply.set_success(true);
+ LOGS(_log, LOG_LVL_INFO, "Key info lookup key=" << key <<
+ " (" << protoReply.chunk() << ", " << protoReply.subchunk() << ")");
+ }
+ StringElement strElem;
+ protoReply.SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+ LOGS(_log, LOG_LVL_INFO, "sending key lookup " << key << " to " << nAddr << " from " << _ourId);
+ try {
+ sendBufferTo(nAddr.ip, nAddr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_workerKeyInfoReq boost system_error=" << e.what() <<
+ " inMsg=" << inMsg);
+ }
+ } else {
+ // Find the target range in the list and send the request there
+ auto targetWorker = getWorkerList()->findWorkerForKey(key);
+ if (targetWorker == nullptr) {
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_workerKeyInfoReq " << _ourId <<
+ " could not forward key=" << key);
+ // TODO HIGH forward request to neighbor in case it was in recent shift.
+ return;
+ }
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker::_workerKeyInfoReq " << _ourId <<
+ " forwarding key=" << key << " to " << *targetWorker);
+ _forwardKeyInfoRequest(targetWorker, inMsg, protoData);
+ }
+}
+
+
+bool CentralWorker::workerWorkerSetRightNeighbor(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ auto msgElem = MsgElement::retrieve(*data, "CentralWorker::workerWorkerSetRightNeighbor");
+ UInt32Element::Ptr neighborName = std::dynamic_pointer_cast(msgElem);
+ if (neighborName == nullptr) {
+ return false;
+ }
+
+ LOGS(_log, LOG_LVL_INFO, "workerWorkerSetRightNeighbor ourName=" << _ourId << " rightN=" << neighborName->element);
+ // Just setting the name, so it can stay here. See CentralWorker::_monitor(), which establishes/maintains connections.
+ _neighborRight.setId(neighborName->element);
+ return true;
+}
+
+
+bool CentralWorker::workerWorkerSetLeftNeighbor(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ auto msgElem = MsgElement::retrieve(*data, "CentralWorker::workerWorkerSetLeftNeighbor");
+ UInt32Element::Ptr neighborName = std::dynamic_pointer_cast(msgElem);
+ if (neighborName == nullptr) {
+ return false;
+ }
+
+ LOGS(_log, LOG_LVL_INFO, "workerWorkerSetLeftNeighbor ourName=" << _ourId << " leftN=" << neighborName->element);
+ // TODO move to separate thread
+ _neighborLeft.setId(neighborName->element);
+ // Just setting the name. See CentralWorker::_monitor(), which establishes/maintains connections.
+ return true;
+}
+
+
+bool CentralWorker::workerWorkerKeysInfoReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data) {
+ // Send a message containing information about the range and number of keys handled by this worker back
+ // to the sender. Nothing in data
+
+ // TODO move this to another thread
+ _workerWorkerKeysInfoReq(inMsg);
+ return true;
+
+}
+
+
+void CentralWorker::_workerWorkerKeysInfoReq(LoaderMsg const& inMsg) {
+ // Use the address from inMsg as this kind of request is pointless to forward.
+ NetworkAddress nAddr(inMsg.senderHost->element, inMsg.senderPort->element);
+ uint64_t msgId = inMsg.msgId->element;
+ _sendWorkerKeysInfo(nAddr, msgId);
+}
+
+
+void CentralWorker::_sendWorkerKeysInfo(NetworkAddress const& nAddr, uint64_t msgId) {
+ // Build message containing Range, size of map, number of items added.
+ LoaderMsg msg(LoaderMsg::WORKER_KEYS_INFO, msgId, getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ std::unique_ptr protoWKI = workerKeysInfoBuilder();
+ StringElement strElem;
+ protoWKI->SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+ LOGS(_log, LOG_LVL_INFO, "sending WorkerKeysInfo name=" << _ourId <<
+ " mapsize=" << protoWKI->mapsize() << " recentAdds=" << protoWKI->recentadds() <<
+ " to " << nAddr);
+ try {
+ sendBufferTo(nAddr.ip, nAddr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_sendWorkerKeysInfo boost system_error=" << e.what() <<
+ " nAddr=" << nAddr << "msgId=" << msgId);
+ }
+}
+
+
+std::unique_ptr CentralWorker::workerKeysInfoBuilder() {
+ std::unique_ptr protoWKI(new proto::WorkerKeysInfo());
+ // Build message containing Range, size of map, number of items added.
+ // TODO this code is similar to code elsewhere, try to merge it.
+ KeyRange range;
+ size_t mapSize;
+ size_t recentAdds;
+ {
+ std::lock_guard lck(_idMapMtx);
+ range = _keyRange;
+ mapSize = _keyValueMap.size();
+ _removeOldEntries();
+ recentAdds = _recentAdds.size();
+ }
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker WorkerKeysInfo a name=" << _ourId <<
+ " keyCount=" << mapSize << " recentAdds=" << recentAdds);
+ protoWKI->set_wid(_ourId);
+ protoWKI->set_mapsize(mapSize);
+ protoWKI->set_recentadds(recentAdds);
+ proto::WorkerRange *protoRange = protoWKI->mutable_range();
+ range.loadProtoRange(*protoRange);
+ proto::Neighbor *protoLeft = protoWKI->mutable_left();
+ protoLeft->set_wid(_neighborLeft.getId());
+ proto::Neighbor *protoRight = protoWKI->mutable_right();
+ protoRight->set_wid(_neighborRight.getId());
+ LOGS(_log, LOG_LVL_INFO, "CentralWorker WorkerKeysInfo b name=" << _ourId <<
+ " keyCount=" << mapSize << " recentAdds=" << recentAdds);
+ return protoWKI;
+}
+
+
+// TODO This looks a lot like the other _forward*** functions, try to combine them.
+void CentralWorker::_forwardKeyInfoRequest(WWorkerListItem::Ptr const& target, LoaderMsg const& inMsg,
+ std::unique_ptr const& protoData) {
+ // The proto buffer should be the same, just need a new message.
+ LoaderMsg msg(LoaderMsg::KEY_LOOKUP_REQ, inMsg.msgId->element, getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+
+ StringElement strElem;
+ protoData->SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+
+ auto nAddr = target->getUdpAddress();
+ try {
+ sendBufferTo(nAddr.ip, nAddr.port, msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_forwardKeyInfoRequest boost system_error=" << e.what() <<
+ " target=" << target << " inMsg=" << inMsg);
+ }
+}
+
+
+void CentralWorker::_registerWithMaster() {
+ LoaderMsg msg(LoaderMsg::MAST_WORKER_ADD_REQ, getNextMsgId(), getHostName(), getUdpPort());
+ BufferUdp msgData;
+ msg.appendToData(msgData);
+ // create the proto buffer
+ lsst::qserv::proto::LdrNetAddress protoBuf;
+ protoBuf.set_ip(getHostName());
+ protoBuf.set_udpport(getUdpPort());
+ protoBuf.set_tcpport(getTcpPort());
+
+ StringElement strElem;
+ protoBuf.SerializeToString(&(strElem.element));
+ strElem.appendToData(msgData);
+
+ try {
+ sendBufferTo(getMasterHostName(), getMasterPort(), msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::_registerWithMaster boost system_error=" << e.what());
+ }
+}
+
+
+void CentralWorker::testSendBadMessage() {
+ uint16_t kind = 60200;
+ LoaderMsg msg(kind, getNextMsgId(), getHostName(), getUdpPort());
+ LOGS(_log, LOG_LVL_INFO, "testSendBadMessage msg=" << msg);
+ BufferUdp msgData(128);
+ msg.appendToData(msgData);
+ try {
+ sendBufferTo(getMasterHostName(), getMasterPort(), msgData);
+ } catch (boost::system::system_error const& e) {
+ LOGS(_log, LOG_LVL_ERROR, "CentralWorker::testSendBadMessage boost system_error=" << e.what());
+ throw e; // This would not be the expected error, re-throw so it is noticed.
+ }
+}
+
+
+void CentralWorker::_removeOldEntries() {
+ // _idMapMtx must be held when this is called.
+ auto now = std::chrono::system_clock::now();
+ auto then = now - _recentAddLimit;
+ while (_recentAdds.size() > 0 && _recentAdds.front() < then) {
+ _recentAdds.pop_front();
+ }
+}
+
+
+void CentralWorker::insertKeys(std::vector const& keyList, bool mustSetMin) {
+ std::unique_lock lck(_idMapMtx);
+ auto maxKey = _keyRange.getMax();
+ bool maxKeyChanged = false;
+ for (auto&& elem:keyList) {
+ auto const& key = elem.first;
+ auto res = _keyValueMap.insert(std::make_pair(key, elem.second));
+ if (key > maxKey) {
+ maxKey = key;
+ maxKeyChanged = true;
+ }
+ if (not res.second) {
+ LOGS(_log, LOG_LVL_WARN, "insertKeys Possible duplicate " <<
+ elem.first << ":" << elem.second);
+ }
+ }
+
+ // On all nodes except the left most, the minimum should be reset.
+ if (mustSetMin && _keyValueMap.size() > 0) {
+ auto minKeyPair = _keyValueMap.begin();
+ _keyRange.setMin(minKeyPair->first);
+ }
+
+ if (maxKeyChanged) {
+ // if unlimited is false, range will be slightly off until corrected by the right neighbor.
+ bool unlimited = _keyRange.getUnlimited();
+ _keyRange.setMax(maxKey, unlimited);
+ }
+}
+
+
+std::string CentralWorker::dumpKeysStr(unsigned int count) {
+ std::stringstream os;
+ std::lock_guard lck(_idMapMtx);
+ os << "name=" << getOurId() << " count=" << _keyValueMap.size() << " range("
+ << _keyRange << ") pairs: ";
+
+ if (count < 1 || _keyValueMap.size() < count*2) {
+ for (auto&& elem:_keyValueMap) {
+ os << elem.first << "{" << elem.second << "} ";
+ }
+ } else {
+ auto iter = _keyValueMap.begin();
+ for (size_t j=0; j < count && iter != _keyValueMap.end(); ++iter, ++j) {
+ os << iter->first << "{" << iter->second << "} ";
+ }
+ os << " ... ";
+ auto rIter = _keyValueMap.rbegin();
+ for (size_t j=0; j < count && rIter != _keyValueMap.rend(); ++rIter, ++j) {
+ os << rIter->first << "{" << rIter->second << "} ";
+ }
+
+ }
+ return os.str();
+}
+}}} // namespace lsst::qserv::loader
diff --git a/core/modules/loader/CentralWorker.h b/core/modules/loader/CentralWorker.h
new file mode 100644
index 0000000000..b11a8e4dcf
--- /dev/null
+++ b/core/modules/loader/CentralWorker.h
@@ -0,0 +1,266 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CENTRAL_WORKER_H
+#define LSST_QSERV_LOADER_CENTRAL_WORKER_H
+
+// system headers
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/CentralFollower.h"
+#include "loader/DoList.h"
+#include "loader/Neighbor.h"
+#include "loader/ServerTcpBase.h"
+#include "loader/WorkerConfig.h"
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+class CentralWorkerDoListItem;
+
+/// This class is central to the worker. In addition
+/// to maintaining lists of other workers it maintains a key-value
+/// store over a range of keys. The range can change over time
+/// as keys are shifted between this worker and its left and
+/// right neighbors. It connects to its neighbors using TCP and
+/// informs the master of its current key range using UDP.
+/// Key-value lookups and inserts are done using UDP.
+/// Workers will attempt to forward key lookups and inserts
+/// to the correct worker when the key is not in this worker's
+/// range.
+class CentralWorker : public CentralFollower {
+public:
+ typedef std::pair CompKeyPair;
+
+ enum SocketStatus {
+ VOID0 = 0,
+ STARTING1,
+ ESTABLISHED2
+ };
+
+ enum Direction {
+ NONE0 = 0,
+ TORIGHT1,
+ FROMRIGHT2
+ };
+
+ CentralWorker(boost::asio::io_service& ioService, boost::asio::io_context& io_context_,
+ std::string const& hostName_, WorkerConfig const& cfg);
+
+ /// Open the UDP and TCP ports and start monitoring. This can throw boost::system::system_error.
+ void startService() override;
+ void startMonitoring() override;
+
+ ~CentralWorker() override;
+
+ int getTcpPort() const override { return _tcpPort; }
+ uint32_t getOurId() const;
+
+ /// Insert the keys in keyList into _keyValueMap, adjusting ranges
+ /// as needed.
+ /// @parameter mustSetMin should be set true if this is not the left
+ /// most worker. It causes the minimum value to
+ /// be set to the smallest key in _keyValueMap.
+ void insertKeys(std::vector const& keyList, bool mustSetMin);
+
+ /// @Return a list of the smallest keys from _keyValueMap. The keys are removed from
+ /// from the map. Put keys are also put in _transferList in case the shift fails
+ /// and they need to be put back into _keyValueMap.
+ /// TODO add argument for smallest or largest and code to build list from smallest or largest keys.
+ StringElement::UPtr buildKeyList(int keysToShift);
+
+ /////////////////////////////////////////////////////////////////////////////////
+ /// Methods to handle messages received from other servers.
+ /// 'inMsg' contains information about the originator of a request
+ /// and the type of message.
+ /// 'data' contains the message data.
+
+ /// Update our range with data from our left neighbor.
+ /// Our minimum key is their maximum key(exclusive).
+ /// @returns what it thinks the range of the left neighbor should be.
+ KeyRange updateRangeWithLeftData(KeyRange const& strRange);
+
+ /// Receive a request to insert a key value pair.
+ /// If the key value pair could not be inserted, it tries to forward the request appropriately.
+ /// @Returns true if the request could be parsed.
+ bool workerKeyInsertReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ /// Receive a request to lookup a key value.
+ /// If the key is not within this worker's range, it tries to forward the request appropriately.
+ /// @Returns true if the request could be parsed.
+ bool workerKeyInfoReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ /// Receive a request for information about this worker's keys, how many key-value pairs are
+ /// stored and the range of keys the worker is responsible for.
+ /// @Returns true if the message could be parsed.
+ bool workerWorkerKeysInfoReq(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ /// Receive a message from the master providing the wId of our right neighbor.
+ bool workerWorkerSetRightNeighbor(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ /// Receive a message from the master providing the wId of our left neighbor.
+ bool workerWorkerSetLeftNeighbor(LoaderMsg const& inMsg, BufferUdp::Ptr const& data);
+
+ std::string getOurLogId() const override;
+
+ std::unique_ptr workerKeysInfoBuilder(); // TODO make private
+ void setNeighborInfoLeft(uint32_t wId, int keyCount, KeyRange const& range); // TODO make private
+
+ /// @Return a string describing the first and last 'count' keys. count=0 dumps all keys.
+ std::string dumpKeysStr(unsigned int count);
+
+ /// Called when our right neighbor indicates it is done with a shift FROMRIGHT
+ void finishShiftFromRight();
+
+ /// Called when there has been a problem with shifting with the left neighbor and changes
+ /// to _keyValueMap need to be undone.
+ void cancelShiftsWithLeftNeighbor();
+
+ /// Send a bad message for testing purposes.
+ void testSendBadMessage();
+
+ friend CentralWorkerDoListItem;
+
+protected:
+ void checkForThisWorkerValues(uint32_t wId, std::string const& ip,
+ int portUdp, int portTcp, KeyRange& strRange) override;
+private:
+ /// Contact the master so it can provide this worker with an id. The master
+ /// will activate this worker when it is needed at a later time.
+ void _registerWithMaster();
+
+ /// @return true if our worker id is not valid.
+ bool _isOurIdInvalid() const {
+ std::lock_guard lck(_ourIdMtx);
+ return _ourIdInvalid;
+ }
+
+ /// If ourId is invalid, set our id to id.
+ bool _setOurId(uint32_t id);
+
+ /// Disable this worker. Only to be used if the master has deemed
+ /// this worker as too unreliable and replaced it.
+ void _masterDisable();
+
+ /// This function is run to monitor this worker's status. It is used to
+ /// register with the master, connect and control shifting with the
+ /// the right neighbor.
+ void _monitor();
+
+ /// Use the information from our right neighbor to set our key range.
+ bool _determineRange();
+
+ /// If this worker has significantly more or fewer keys than its right neighbor,
+ /// shift keys between them to make a more even distribution.
+ /// @Return true if data was shifted with the right neighbor.
+ bool _shiftIfNeeded(std::lock_guard const& rightMtxLG);
+
+ /// Attempt to shift keys to or from the right neighbor.
+ /// @parameter keysToShift is number of keys to shift.
+ /// @parameter direction is TO or FROM the right neighbor.
+ void _shift(Direction direction, int keysToShift);
+
+ /// See workerKeyInsertReq(...)
+ void _workerKeyInsertReq(LoaderMsg const& inMsg, std::unique_ptr& protoBuf);
+ /// Forward a workerKeyInsertReq to an appropriate worker.
+ void _forwardKeyInsertRequest(NetworkAddress const& targetAddr, LoaderMsg const& inMsg,
+ std::unique_ptr& protoData);
+
+ /// See workerKeyInfoReq
+ void _workerKeyInfoReq(LoaderMsg const& inMsg, std::unique_ptr& protoBuf);
+ /// Forward a workerKeyInfoReq to an appropriate worker.
+ void _forwardKeyInfoRequest(WWorkerListItem::Ptr const& target, LoaderMsg const& inMsg,
+ std::unique_ptr const& protoData);
+
+ /// See workerWorkerKeysInfoReq(...)
+ void _workerWorkerKeysInfoReq(LoaderMsg const& inMsg);
+ /// Send information about our keys (range, number of pairs) to 'nAddr'.
+ void _sendWorkerKeysInfo(NetworkAddress const& nAddr, uint64_t msgId);
+
+ void _removeOldEntries(); ///< remove old entries from _recentAdds
+
+ /// Connect to the right neighbor. Must hold _rightMtx in the lock.
+ void _rightConnect(std::lock_guard const& rightMtxLG);
+ ///< Disconnect from the right neighbor. Must hold _rightMtx in the lock.
+ void _rightDisconnect(std::lock_guard const& rightMtxLG, std::string const& note);
+
+ void _cancelShiftsWithRightNeighbor(); ///< Cancel shifts to/from the right neighbor.
+ void _finishShiftToRight(); ///< The shift to the right neighbor is complete, cleanup.
+
+ const int _tcpPort;
+ boost::asio::io_context& _ioContext;
+
+ bool _ourIdInvalid{true}; ///< true until our id has been set by the master.
+ std::atomic _ourId{0}; ///< id given by the master, 0 is invalid id.
+ mutable std::mutex _ourIdMtx; ///< protects _ourIdInvalid, _ourId
+
+ KeyRange _keyRange; ///< range for this worker
+ std::atomic _rangeChanged{false};
+ std::map _keyValueMap;
+ std::deque _recentAdds; ///< track how many keys added recently.
+ std::chrono::milliseconds _recentAddLimit; ///< After this period of time, additions are no longer recent.
+ std::mutex _idMapMtx; ///< protects _strRange, _keyValueMap,
+ ///< _recentAdds, _transferListToRight, _transferListFromRight
+
+ Neighbor _neighborLeft{Neighbor::LEFT};
+ Neighbor _neighborRight{Neighbor::RIGHT};
+
+ ServerTcpBase::Ptr _tcpServer; // For our right neighbor to connect to us.
+
+ std::mutex _rightMtx;
+ SocketStatus _rightConnectStatus{VOID0};
+ std::shared_ptr _rightSocket;
+
+ std::atomic _shiftAsClientInProgress{false}; ///< True when shifting to or from right neighbor.
+
+ /// Shift if a node has % more than it's neighbor. the percentage threshold is expressed
+ /// as a decimal, so 1.1 would be 10% more than neighbor or 110%.
+ double _thresholdNeighborShift;
+
+ /// Maximum number of keys to shift in one iteration. 10000 may be reasonable.
+ /// An iteration would be transfer, insert, and verify range. During the
+ /// insert phase, the mutex is locked preventing key inserts and lookups.
+ /// Using smaller values locks the mutex for more periods of time but each
+ /// period is shorter and lookups can occur during the gaps.
+ /// Too big a value, and the maps will be paralyzed for a long time during inserts.
+ /// Too small and shift operations will take significantly longer.
+ int _maxKeysToShift;
+ std::vector _transferListToRight; ///< List of items being transfered to right
+ /// List of items being transfered to our left neighbor. (answering neighbor's FromRight request)
+ std::vector _transferListWithLeft;
+
+ /// The DoListItem that makes sure _monitor() is run.
+ std::shared_ptr _centralWorkerDoListItem;
+};
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CENTRAL_WORKER_H
+
diff --git a/core/modules/loader/CentralWorkerDoListItem.h b/core/modules/loader/CentralWorkerDoListItem.h
new file mode 100644
index 0000000000..6ab99228b8
--- /dev/null
+++ b/core/modules/loader/CentralWorkerDoListItem.h
@@ -0,0 +1,67 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+
+#ifndef LSST_QSERV_LOADER_CENTRALWORKERDOLISTITEM_H
+#define LSST_QSERV_LOADER_CENTRALWORKERDOLISTITEM_H
+
+// Qserv headers
+#include "loader/CentralWorker.h"
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+/// This class exists to regularly call the CentralWorker::_monitor() function, which
+/// does things like monitor TCP connections and control shifting with the right neighbor.
+class CentralWorkerDoListItem : public DoListItem {
+public:
+ CentralWorkerDoListItem() = delete;
+ explicit CentralWorkerDoListItem(CentralWorker* centralWorker) : _centralWorker(centralWorker) {
+ setTimeOut(std::chrono::seconds(15)); // TODO: DM-17453 set via config
+ }
+
+ util::CommandTracked::Ptr createCommand() override {
+ struct CWMonitorCmd : public util::CommandTracked {
+ CWMonitorCmd(CentralWorker* centralW) : centralWorker(centralW) {}
+ void action(util::CmdData*) override {
+ centralWorker->_monitor();
+ }
+ CentralWorker* centralWorker;
+ };
+ util::CommandTracked::Ptr cmd(std::make_shared(_centralWorker));
+ return cmd;
+ }
+
+private:
+ CentralWorker* _centralWorker;
+};
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CENTRAL_WORKER_DO_LIST_ITEM_H
+
+
+
+
+
diff --git a/core/modules/loader/ClientConfig.cc b/core/modules/loader/ClientConfig.cc
new file mode 100644
index 0000000000..66247a281e
--- /dev/null
+++ b/core/modules/loader/ClientConfig.cc
@@ -0,0 +1,62 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 AURA/LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+// Class header
+#include "loader/ClientConfig.h"
+
+// System headers
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+// Qserv headers
+#include "util/ConfigStore.h"
+#include "util/ConfigStoreError.h"
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.Config");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+ClientConfig::ClientConfig(util::ConfigStore const& configStore) {
+ try {
+ setFromConfig(configStore);
+ } catch (util::ConfigStoreError const& e) {
+ throw ConfigErr(ERR_LOC, std::string("ClientConfig ") + e.what());
+ }
+}
+
+
+std::ostream& ClientConfig::dump(std::ostream &os) const {
+ os << "(ClientConfig(" << header << ") ";
+ ConfigBase::dump(os);
+ os << ")";
+ return os;
+}
+
+}}} // namespace lsst::qserv::css
+
diff --git a/core/modules/loader/ClientConfig.h b/core/modules/loader/ClientConfig.h
new file mode 100644
index 0000000000..6256a7e2ab
--- /dev/null
+++ b/core/modules/loader/ClientConfig.h
@@ -0,0 +1,104 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CLIENTCONFIG_H
+#define LSST_QSERV_LOADER_CLIENTCONFIG_H
+
+// Qserv headers
+#include "loader/ConfigBase.h"
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+/// A class for reading the configuration file for the client which consists of
+/// a collection of key-value pairs and provide access functions for those values.
+///
+class ClientConfig : public ConfigBase {
+public:
+ explicit ClientConfig(std::string const& configFileName)
+ : ClientConfig(util::ConfigStore(configFileName)) {}
+
+ ClientConfig() = delete;
+ ClientConfig(ClientConfig const&) = delete;
+ ClientConfig& operator=(ClientConfig const&) = delete;
+
+ std::string getMasterHost() const { return _masterHost->getValue(); }
+ int getMasterPortUdp() const { return _masterPortUdp->getInt(); }
+ int getDefWorkerPortUdp() const { return _defWorkerPortUdp->getInt(); }
+ std::string getDefWorkerHost() const { return _defWorkerHost->getValue(); }
+ int getClientPortUdp() const { return _clientPortUdp->getInt(); }
+ int getThreadPoolSize() const { return _threadPoolSize->getInt(); }
+ int getLoopSleepTime() const { return _loopSleepTime->getInt(); } // TODO: Maybe chrono types for times
+ int getMaxLookups() const { return _maxLookups->getInt(); }
+ int getMaxInserts() const { return _maxInserts->getInt(); }
+ int getMaxRequestSleepTime() const { return _maxRequestSleepTime->getInt(); }
+ int getIOThreads() const { return _iOThreads->getInt(); }
+
+ std::ostream& dump(std::ostream &os) const override;
+
+ std::string const header{"client"};
+private:
+ ClientConfig(util::ConfigStore const& configStore);
+
+ /// Master host name
+ ConfigElement::Ptr _masterHost{
+ ConfigElement::create(cfgList, header, "masterHost", ConfigElement::STRING, true)};
+ /// Master UDP port
+ ConfigElement::Ptr _masterPortUdp{
+ ConfigElement::create(cfgList, header, "masterPortUdp", ConfigElement::INT, true)};
+ /// UDP port for default worker. Reasonable value - 9876
+ ConfigElement::Ptr _clientPortUdp{
+ ConfigElement::create(cfgList, header, "clientPortUdp", ConfigElement::INT, true)};
+ /// Default worker host name
+ ConfigElement::Ptr _defWorkerHost{
+ ConfigElement::create(cfgList, header, "defWorkerHost", ConfigElement::STRING, true)};
+ /// Default worker UDP port. Reasonable value - 9876
+ ConfigElement::Ptr _defWorkerPortUdp{
+ ConfigElement::create(cfgList, header, "defWorkerPortUdp", ConfigElement::INT, true)};
+ /// Size of the thread pool. Reasonable value - 10
+ ConfigElement::Ptr _threadPoolSize{
+ ConfigElement::create(cfgList, header, "threadPoolSize", ConfigElement::INT, true)};
+ /// Time spent sleeping between checking elements in the DoList in micro seconds. 100000
+ ConfigElement::Ptr _loopSleepTime{
+ ConfigElement::create(cfgList, header, "loopSleepTime", ConfigElement::INT, false, "100000")};
+ /// Maximum number of lookup requests allowed in the DoList.
+ ConfigElement::Ptr _maxLookups{
+ ConfigElement::create(cfgList, header, "maxLookups", ConfigElement::INT, true)};
+ /// Maximum number of insert requests allowed in the DoList.
+ ConfigElement::Ptr _maxInserts{
+ ConfigElement::create(cfgList, header, "maxInserts", ConfigElement::INT, true)};
+ /// When reaching maxInserts or maxLookups, sleep this long before trying to add more,
+ /// in micro seconds. 100000micro = 0.1sec
+ ConfigElement::Ptr _maxRequestSleepTime{
+ ConfigElement::create(cfgList, header,
+ "maxRequestSleepTime", ConfigElement::INT, false, "100000")};
+ /// Number of IO threads the server should run.
+ ConfigElement::Ptr _iOThreads{
+ ConfigElement::create(cfgList, header, "iOThreads", ConfigElement::INT, false, "4")};
+};
+
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CLIENTCONFIG_H
diff --git a/core/modules/loader/ClientServer.cc b/core/modules/loader/ClientServer.cc
new file mode 100644
index 0000000000..89da87cbfc
--- /dev/null
+++ b/core/modules/loader/ClientServer.cc
@@ -0,0 +1,177 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 AURA/LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+
+// Class header
+#include "loader/ClientServer.h"
+
+// System headers
+#include
+
+// Qserv headers
+#include "loader/CentralClient.h"
+#include "loader/LoaderMsg.h"
+#include "proto/loader.pb.h"
+#include "proto/ProtoImporter.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.ClientServer");
+}
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+BufferUdp::Ptr ClientServer::parseMsg(BufferUdp::Ptr const& data,
+ boost::asio::ip::udp::endpoint const& senderEndpoint) {
+ LOGS(_log, LOG_LVL_DEBUG, "ClientServer::parseMsg sender " << senderEndpoint <<
+ " data length=" << data->getAvailableWriteLength());
+ BufferUdp::Ptr sendData; /// nullptr for empty response.
+ LoaderMsg inMsg;
+ inMsg.parseFromData(*data);
+ LOGS(_log, LOG_LVL_INFO, "ClientServer::parseMsg sender " << senderEndpoint <<
+ " kind=" << inMsg.msgKind->element << " data length=" << data->getAvailableWriteLength());
+ switch (inMsg.msgKind->element) {
+ case LoaderMsg::MSG_RECEIVED:
+ LOGS(_log, LOG_LVL_WARN, "ClientServer::parseMsg MSG_RECEIVED");
+ _msgRecievedHandler(inMsg, data, senderEndpoint);
+ sendData.reset(); // Never send a response back for one of these, infinite loop.
+ break;
+
+ case LoaderMsg::KEY_LOOKUP:
+ LOGS(_log, LOG_LVL_DEBUG, "KEY_LOOK");
+ _centralClient->handleKeyLookup(inMsg, data);
+ break;
+
+ case LoaderMsg::KEY_INSERT_COMPLETE:
+ LOGS(_log, LOG_LVL_DEBUG, "KEY_INSERT_COMPLETE");
+ _centralClient->handleKeyInsertComplete(inMsg, data);
+ break;
+
+ case LoaderMsg::MAST_WORKER_LIST:
+ _centralClient->getWorkerList()->workerListReceive(data);
+ break;
+ case LoaderMsg::MAST_WORKER_INFO:
+ _centralClient->workerInfoReceive(data);
+ break;
+ // following not expected by client
+ case LoaderMsg::KEY_INSERT_REQ:
+ case LoaderMsg::KEY_LOOKUP_REQ:
+ case LoaderMsg::MAST_INFO:
+ case LoaderMsg::MAST_INFO_REQ:
+ case LoaderMsg::MAST_WORKER_LIST_REQ:
+ case LoaderMsg::MAST_WORKER_INFO_REQ:
+ case LoaderMsg::MAST_WORKER_ADD_REQ:
+ // TODO add response for known but unexpected message.
+ sendData = prepareReplyToMsg(senderEndpoint, inMsg, LoaderMsg::STATUS_PARSE_ERR,
+ "unexpected Msg Kind");
+ break;
+
+ default:
+ sendData = prepareReplyToMsg(senderEndpoint, inMsg, LoaderMsg::STATUS_PARSE_ERR,
+ "unknownMsgKind");
+ }
+
+ return sendData;
+}
+
+
+BufferUdp::Ptr ClientServer::prepareReplyToMsg(boost::asio::ip::udp::endpoint const& senderEndpoint,
+ LoaderMsg const& inMsg,
+ int status, std::string const& msgTxt) {
+
+ if (status != LoaderMsg::STATUS_SUCCESS) {
+ LOGS(_log,LOG_LVL_WARN, "Error response Original from " << senderEndpoint <<
+ " msg=" << msgTxt << " inMsg=" << inMsg.getStringVal());
+ }
+
+ LoaderMsg outMsg(LoaderMsg::MSG_RECEIVED, inMsg.msgId->element, getOurHostName(), getOurPort());
+
+ // create the proto buffer
+ proto::LdrMsgReceived protoBuf;
+ protoBuf.set_originalid(inMsg.msgId->element);
+ protoBuf.set_originalkind(inMsg.msgKind->element);
+ protoBuf.set_status(LoaderMsg::STATUS_PARSE_ERR);
+ protoBuf.set_errmsg(msgTxt);
+ protoBuf.set_dataentries(0);
+
+ StringElement respBuf;
+ protoBuf.SerializeToString(&(respBuf.element));
+
+ auto sendData = std::make_shared();
+ outMsg.appendToData(*sendData);
+ respBuf.appendToData(*sendData);
+ return sendData;
+}
+
+
+void ClientServer::_msgRecievedHandler(LoaderMsg const& inMsg, BufferUdp::Ptr const& data,
+ boost::asio::ip::udp::endpoint const& senderEndpoint) {
+ bool success = true;
+ // This is only really expected for parsing errors. Most responses to
+ // requests come in as normal messages.
+ StringElement::Ptr seData =
+ std::dynamic_pointer_cast(MsgElement::retrieve(*data, "ClientServer::_msgRecievedHandler"));
+ if (seData == nullptr) {
+ success = false;
+ }
+
+ std::unique_ptr protoBuf;
+ if (success) {
+ protoBuf = seData->protoParse();
+ if (protoBuf == nullptr) success = false;
+ }
+
+ std::stringstream os;
+ int status = LoaderMsg::STATUS_PARSE_ERR;
+
+ if (success) {
+ auto originalId = protoBuf->originalid();
+ auto originalKind = protoBuf->originalkind();
+ status = protoBuf->status();
+ auto errMsg = protoBuf->errmsg();
+ os << " sender=" << senderEndpoint <<
+ " id=" << originalId << " kind=" << originalKind << " status=" << status <<
+ " msg=" << errMsg;
+ } else {
+ os << " Failed to parse MsgRecieved! sender=" << senderEndpoint;
+ }
+
+ if (status != LoaderMsg::STATUS_SUCCESS) {
+ ++_errCount;
+ LOGS(_log, LOG_LVL_WARN, "MsgRecieved Message sent by this server caused error at its target" <<
+ " errCount=" << _errCount << os.str());
+ } else {
+ // There shouldn't be many of these, unless there's a need to time things.
+ LOGS(_log, LOG_LVL_INFO, "MsgRecieved " << os.str());
+ }
+}
+
+
+}}} // namespace lsst:qserv::loader
+
diff --git a/core/modules/loader/ClientServer.h b/core/modules/loader/ClientServer.h
new file mode 100644
index 0000000000..9e6d276e44
--- /dev/null
+++ b/core/modules/loader/ClientServer.h
@@ -0,0 +1,80 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CLIENTSERVER_H
+#define LSST_QSERV_LOADER_CLIENTSERVER_H
+
+// system headers
+#include
+#include
+
+// third party headers
+#include "boost/asio.hpp"
+
+// Qserv headers
+#include "loader/LoaderMsg.h"
+#include "loader/ServerUdpBase.h"
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+class CentralClient;
+
+/// This class implements a UDP server for the client so that message replies can be
+/// sent directly to the client instead of passed back through the chain of workers
+/// that were queried when looking for the worker that could handle this client's
+/// request.
+class ClientServer : public ServerUdpBase {
+public:
+ // The base class default constructor, copy constructor, and operator= have been set to delete.
+ ClientServer(boost::asio::io_service& ioService, std::string const& host, int port,
+ CentralClient* centralClient)
+ : ServerUdpBase(ioService, host, port), _centralClient(centralClient) {}
+
+ ~ClientServer() override = default;
+
+ /// Parse enough of an incoming message so it can be passed to the proper handler.
+ BufferUdp::Ptr parseMsg(BufferUdp::Ptr const& data,
+ boost::asio::ip::udp::endpoint const& endpoint) override;
+
+ /// Build a reply to a message that was received, usually used to handle unknown or unexpected messages.
+ /// @return a pointer to a buffer with the constructed message.
+ // TODO shows up in both MasterServer and WorkerServer
+ BufferUdp::Ptr prepareReplyToMsg(boost::asio::ip::udp::endpoint const& senderEndpoint,
+ LoaderMsg const& inMsg,
+ int status, std::string const& msgTxt);
+
+private:
+ /// Construct basic replies to unknown and unexpected messages.
+ void _msgRecievedHandler(LoaderMsg const& inMsg, BufferUdp::Ptr const& data,
+ boost::asio::ip::udp::endpoint const& senderEndpoint);
+
+ CentralClient* _centralClient;
+};
+
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_CLIENTSERVER_H
diff --git a/core/modules/loader/CompositeKey.cc b/core/modules/loader/CompositeKey.cc
new file mode 100644
index 0000000000..e2a3d26731
--- /dev/null
+++ b/core/modules/loader/CompositeKey.cc
@@ -0,0 +1,66 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 AURA/LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+
+// Class header
+#include "loader/CompositeKey.h"
+
+// System headers
+#include
+
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.CompositeKey");
+}
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+CompositeKey const CompositeKey::minValue(0,"");
+
+
+void CompositeKey::dump(std::ostream& os) const {
+ os << "CKey(" << kInt << ", " << kStr << ")";
+}
+
+
+std::string CompositeKey::dump() const {
+ std::stringstream os;
+ dump(os);
+ return os.str();
+}
+
+
+std::ostream& operator<<(std::ostream& os, CompositeKey const& cKey) {
+ cKey.dump(os);
+ return os;
+}
+
+
+}}} // namespace lsst::qserv::loader
+
diff --git a/core/modules/loader/CompositeKey.h b/core/modules/loader/CompositeKey.h
new file mode 100644
index 0000000000..b2a7a60df6
--- /dev/null
+++ b/core/modules/loader/CompositeKey.h
@@ -0,0 +1,98 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_COMPOSITEKEY_H
+#define LSST_QSERV_LOADER_COMPOSITEKEY_H
+
+// system headers
+#include
+#include
+#include
+#include
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+/// A key consisting of an unsigned 64 bit integer and a std::string with support for comparisons.
+/// The integer component is compared before the string component.
+class CompositeKey {
+public:
+ CompositeKey(uint64_t ki, std::string const& ks) : kInt(ki), kStr(ks) {}
+ explicit CompositeKey(uint64_t ki) : CompositeKey(ki, "") {}
+ explicit CompositeKey(std::string const& ks) : CompositeKey(0, ks) {}
+ CompositeKey(CompositeKey const& ck) : CompositeKey(ck.kInt, ck.kStr) {}
+ CompositeKey() : CompositeKey(0, "") {}
+ ~CompositeKey() = default;
+
+ static uint64_t maxIntVal() { return std::numeric_limits::max(); }
+
+ CompositeKey& operator=(CompositeKey const& other) {
+ if (this != &other) {
+ kInt = other.kInt;
+ kStr = other.kStr;
+ }
+ return *this;
+ }
+
+ /// Smallest possible value for a CompositeKey (0,"")
+ static CompositeKey const minValue;
+
+ bool operator<(CompositeKey const& other) const {
+ return std::tie(kInt, kStr) < std::tie(other.kInt, other.kStr);
+ }
+
+ bool operator>(CompositeKey const& other) const {
+ return other < *this;
+ }
+
+ bool operator==(CompositeKey const& other) const {
+ return (kInt == other.kInt) && (kStr == other.kStr);
+ }
+
+ bool operator!=(CompositeKey const& other) const {
+ return !(*this == other);
+ }
+
+ bool operator<=(CompositeKey const& other) const {
+ return !(*this > other);
+ }
+
+ bool operator>=(CompositeKey const& other) const {
+ return !(*this < other);
+ }
+
+ void dump(std::ostream& os) const;
+ std::string dump() const ;
+
+ uint64_t kInt;
+ std::string kStr;
+};
+
+std::ostream& operator<<(std::ostream& os, CompositeKey const& cKey);
+
+}}} // namespace lsst::qserv::loader
+
+#endif // LSST_QSERV_LOADER_COMPOSITEKEY_H
diff --git a/core/modules/loader/ConfigBase.cc b/core/modules/loader/ConfigBase.cc
new file mode 100644
index 0000000000..b5ecec6628
--- /dev/null
+++ b/core/modules/loader/ConfigBase.cc
@@ -0,0 +1,188 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 AURA/LSST.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ */
+
+// Class header
+#include "loader/ConfigBase.h"
+
+// System headers
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+// Qserv headers
+#include "util/ConfigStore.h"
+#include "util/ConfigStoreError.h"
+
+// Third-party headers
+#include "boost/lexical_cast.hpp"
+
+
+namespace {
+LOG_LOGGER _log = LOG_GET("lsst.qserv.loader.Config");
+}
+
+
+namespace lsst {
+namespace qserv {
+namespace loader {
+
+
+void ConfigElement::setFromConfig(util::ConfigStore const& cfgStore) {
+ if (_required) {
+ _value = cfgStore.getRequired(getFullKey());
+ } else {
+ _value = cfgStore.get(getFullKey(), _default);
+ }
+}
+
+
+bool ConfigElement::verifyValueIsOfKind() {
+ switch (_kind) {
+ case STRING:
+ return true;
+ case INT:
+ return isInteger();
+ case FLOAT:
+ return isFloat();
+ default:
+ return false;
+ }
+}
+
+
+std::string ConfigElement::getFullKey() const {
+ if (_header.empty()) return std::string(_key);
+ return std::string(_header + "." + _key);
+}
+
+
+int ConfigElement::getInt() const {
+ if (_kind != INT) {
+ throw ConfigErr(ERR_LOC, "getInt called for non-integer " + dump());
+ }
+ return boost::lexical_cast(_value);
+}
+
+
+double ConfigElement::getDouble() const {
+ if (_kind != FLOAT) {
+ throw ConfigErr(ERR_LOC, "getDouble called for non-float " + dump());
+ }
+ return boost::lexical_cast(_value);
+}
+
+
+bool ConfigElement::isInteger() const {
+ if (_kind != INT) return false;
+ try {
+ // lexical cast is more strict than std::stoi()
+ getInt();
+ } catch (boost::bad_lexical_cast const& exc) {
+ return false;
+ }
+ return true;
+}
+
+
+bool ConfigElement::isFloat() const {
+ if (_kind != FLOAT) return false;
+ try {
+ getDouble();
+ } catch (boost::bad_lexical_cast const& exc) {
+ return false;
+ }
+ return true;
+}
+
+
+std::string ConfigElement::kindToStr(Kind kind) {
+ switch (kind) {
+ case STRING:
+ return "STRING";
+ case INT:
+ return "INT";
+ case FLOAT:
+ return "FLOAT";
+ default:
+ return "undefined Kind";
+ }
+}
+
+
+std::ostream& ConfigElement::dump(std::ostream &os) const {
+ os << "(key=" << getFullKey()
+ << " val=" << _value
+ << " req=" << _required
+ << " kind=" << kindToStr(_kind)
+ << " def=" << _default << ")";
+ return os;
+}
+
+
+std::string ConfigElement::dump() const {
+ std::ostringstream os;
+ dump(os);
+ return os.str();
+}
+
+
+std::ostream& operator<<(std::ostream &os, ConfigElement const& elem) {
+ return elem.dump(os);
+}
+
+
+void ConfigBase::setFromConfig(util::ConfigStore const& configStore) {
+ for (auto& elem:cfgList) {
+ elem->setFromConfig(configStore);
+ if (not elem->verifyValueIsOfKind()) {
+ throw util::ConfigStoreError("Could not parse " + elem->dump());
+ }
+ }
+}
+
+
+std::ostream& ConfigBase::dump(std::ostream &os) const {
+ os << "(ConfigBase: ";
+ for (auto&& elem:cfgList) {
+ os << *elem << " ";
+ }
+ os << ")";
+ return os;
+}
+
+
+std::string ConfigBase::dump() const {
+ std::stringstream os;
+ dump(os);
+ return os.str();
+}
+
+
+std::ostream& operator<<(std::ostream &os, ConfigBase const& cfg) {
+ return cfg.dump(os);
+}
+
+
+}}} // namespace lsst::qserv::loader
+
+
diff --git a/core/modules/loader/ConfigBase.h b/core/modules/loader/ConfigBase.h
new file mode 100644
index 0000000000..dac19b282e
--- /dev/null
+++ b/core/modules/loader/ConfigBase.h
@@ -0,0 +1,147 @@
+// -*- LSST-C++ -*-
+/*
+ * LSST Data Management System
+ * Copyright 2018 LSST Corporation.
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program. If not,
+ * see .
+ *
+ */
+#ifndef LSST_QSERV_LOADER_CONFIGBASE_H
+#define LSST_QSERV_LOADER_CONFIGBASE_H
+
+// system headers
+#include