Skip to content

Commit

Permalink
Merge branch 'tickets/DM-47512'
Browse files Browse the repository at this point in the history
  • Loading branch information
iagaponenko committed Nov 20, 2024
2 parents 27d2637 + 482690c commit 82715e6
Show file tree
Hide file tree
Showing 84 changed files with 186 additions and 1,231 deletions.
18 changes: 9 additions & 9 deletions admin/local/docker/compose/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ services:
<< : *worker-xrootd
command: >
entrypoint worker-xrootd
--db-uri mysql://[email protected]:3306
--db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306
--db-admin-uri mysql://root:[email protected]:3306
--vnid-config "@/usr/local/lib64/libreplica.so {{db_uri}}/qservw_worker 0 0"
--repl-instance-id qserv_proj
Expand Down Expand Up @@ -138,8 +138,8 @@ services:
<< : *worker-cmsd
command: >
entrypoint worker-cmsd
--db-uri mysql://qsmaster@worker_mariadb_0:3306
--vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
--db-uri mysql://qsmaster:CHANGEME2@worker_mariadb_0:3306
--vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
--results-dirname /qserv/data/results
--repl-instance-id qserv_proj
--repl-auth-key replauthkey
Expand Down Expand Up @@ -208,9 +208,9 @@ services:
<< : *worker-xrootd
command: >
entrypoint --log-level DEBUG worker-xrootd
--db-uri mysql://[email protected]:3306?socket={{db_socket}}
--db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306?socket={{db_socket}}
--db-admin-uri mysql://root:[email protected]:3306?socket={{db_socket}}
--vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
--vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
--repl-instance-id qserv_proj
--repl-auth-key replauthkey
--repl-admin-auth-key=repladminauthkey
Expand Down Expand Up @@ -244,8 +244,8 @@ services:
<< : *worker-cmsd
command: >
entrypoint --log-level DEBUG worker-cmsd
--db-uri mysql://qsmaster@worker_mariadb_1:3306?socket=/qserv/mariadb/run/mysqld.sock
--vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
--db-uri mysql://qsmaster:CHANGEME2@worker_mariadb_1:3306?socket=/qserv/mariadb/run/mysqld.sock
--vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
--results-dirname /qserv/data/results
--repl-instance-id qserv_proj
--repl-auth-key replauthkey
Expand Down Expand Up @@ -357,7 +357,7 @@ services:
init: true
command: >
entrypoint --log-level DEBUG proxy
--db-uri mysql://[email protected]:3306?socket={{db_socket}}
--db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306?socket={{db_socket}}
--db-admin-uri mysql://root:[email protected]:3306?socket={{db_socket}}
--targs db_socket=/qserv/mariadb/run/mysqld.sock
--xrootd-manager czar_xrootd
Expand Down Expand Up @@ -395,7 +395,7 @@ services:
init: true
command: >
entrypoint --log-level DEBUG czar-http
--db-uri mysql://qsmaster@czar_mariadb:3306/
--db-uri mysql://qsmaster:CHANGEME2@czar_mariadb:3306/
--xrootd-manager czar_xrootd
--czar-name http
--http-port 4048
Expand Down
1 change: 1 addition & 0 deletions admin/tools/docker/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ RUN cd /usr/share/java \
RUN cd /tmp \
&& git clone https://github.com/lsst/mysqlproxy \
&& cd mysqlproxy \
&& git checkout 0.8.5.lsst8 \
&& cp /usr/lib64/pkgconfig/lua-5.1.pc /usr/lib64/pkgconfig/lua5.1.pc \
&& ./configure \
&& make \
Expand Down
4 changes: 2 additions & 2 deletions admin/tools/docker/mariadb/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ FROM ubuntu:focal AS mariadb-scisql-builder

RUN apt-get update \
&& apt-get install -y curl \
&& curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.6.8" \
&& curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-11.4.4" \
&& apt-get install -y g++ make git python3 libmariadb-dev \
&& update-alternatives --install /usr/bin/python python /usr/bin/python3 0

Expand All @@ -40,7 +40,7 @@ RUN chmod a+r /tmp/scisql/build/deploy.mysql
# off-the-shelf mariadb image. This produces a generic "mariadb with scisql installed" container image.
#-------------------------------------------------------------------------------------------------------------

FROM mariadb:10.6.8 AS mariadb-scisql
FROM mariadb:11.4.4 AS mariadb-scisql
COPY --from=mariadb-scisql-builder /tmp/scisql/build/libscisql-scisql_0.3.so /usr/lib/mysql/plugin
COPY --from=mariadb-scisql-builder /tmp/scisql/build/deploy.mysql /docker-entrypoint-initdb.d/scisql.sql

Expand Down
30 changes: 30 additions & 0 deletions doc/admin/admin-upgrade.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
.. _admin-upgrade:

Instructins for upgrading Qserv to newer releases
=================================================

.. _admin-upgrade-mariadb-11-4-4:

Upgrading Qserv to MariaDB 11.4.4
---------------------------------

MariaDB 11.4.4 is a major release that introduces a number of new features and
improvements. If your instance of Qserv is still based on MariaDB 10.6.8, you
need to upgrade your data to the new version.

To upgrade your Qserv installation to MariaDB 11.4.4, follow these steps:

- Read the `MariaDB 11.4.4 release notes <https://mariadb.com/kb/en/mariadb-11-4-4-release-notes>`_ to learn about the new features and improvements.
- Read the guide `Upgrading from MariaDB 10.6 to MariaDB 10.11 <https://mariadb.com/kb/en/upgrading-from-mariadb-10-6-to-mariadb-10-11>`_ to learn how to upgrade your databases.
- Turn Qserv into the database-only mode by ensuring that all but the MariaDB services are being run.
- Log into the MariaDB containers and run the database upgrade command as shown below:

.. code-block:: bash
mariadb-upgrade -P3306 -h127.0.0.1 --protocol=tcp -uroot -p******
Where ``-P`` specifies the port number, ``-h`` specifies the host, ``-u`` specifies the user, and ``-p`` specifies the password.
Note that the port number may be different in your installation.

Experiments with the production databases at USDF (SLAC) have shown that the upgrade process is safe and reliable.
However, it is still recommended to make a backup of your data before proceeding with the upgrade.
8 changes: 8 additions & 0 deletions doc/admin/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,18 @@
Administrator's Guide
#####################

.. warning::

This version of Qserv uses MariaDB version 11.4.4. If you still have an older release based on
MariaDB 10.6.8 then you should upgrade your databases as described in the following section:

- :ref:`admin-upgrade-mariadb-11-4-4`

.. toctree::
:maxdepth: 4

k8s
row-counters
data-table-indexes
director-index
admin-upgrade
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/Filter.schema
Original file line number Diff line number Diff line change
@@ -1,39 +1,6 @@
-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64)
--
-- Host: localhost Database: case01_m
-- ------------------------------------------------------
-- Server version 5.1.61-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `Filter`
--

DROP TABLE IF EXISTS `Filter`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `Filter` (
`filterId` tinyint(4) NOT NULL,
`filterName` char(3) NOT NULL,
`photClam` float NOT NULL,
`photBW` float NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-03-28 10:31:56
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/LeapSeconds.schema
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64)
--
-- Host: localhost Database: rplante_PT1_2_u_pt12prod_im3000
-- ------------------------------------------------------
-- Server version 5.1.61-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `LeapSeconds`
--

DROP TABLE IF EXISTS `LeapSeconds`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `LeapSeconds` (
`whenJd` float NOT NULL,
`offset` float NOT NULL,
Expand All @@ -29,14 +7,3 @@ CREATE TABLE `LeapSeconds` (
`whenUtc` bigint(20) DEFAULT NULL,
`whenTai` bigint(20) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-04-16 10:56:43
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/Logs.schema
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
-- MySQL dump 10.13 Distrib 5.5.20, for Linux (x86_64)
--
-- Host: localhost Database: test
-- ------------------------------------------------------
-- Server version 5.5.20-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `Logs`
--

DROP TABLE IF EXISTS `Logs`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `Logs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`HOSTID` varchar(80) DEFAULT NULL,
Expand Down Expand Up @@ -48,14 +26,3 @@ CREATE TABLE `Logs` (
PRIMARY KEY (`id`),
KEY `a` (`RUNID`)
) ENGINE=MyISAM AUTO_INCREMENT=1001 DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-04-16 16:31:24
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/Object.schema
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64)
--
-- Host: localhost Database: case01_m
-- ------------------------------------------------------
-- Server version 5.1.61-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `Object`
--

DROP TABLE IF EXISTS `Object`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `Object` (
`objectId` bigint(20) NOT NULL,
`iauId` char(34) DEFAULT NULL,
Expand Down Expand Up @@ -250,14 +228,3 @@ CREATE TABLE `Object` (
`yFlags` int(11) DEFAULT NULL,
`varBinaryField` varbinary(100) DEFAULT NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-03-28 10:31:56
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/RefSrcMatch.schema
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64)
--
-- Host: localhost Database: tmp1
-- ------------------------------------------------------
-- Server version 5.1.61-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `RefSrcMatch`
--

DROP TABLE IF EXISTS `RefSrcMatch`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `RefSrcMatch` (
`refObjectId` bigint(20) DEFAULT NULL,
`sourceId` bigint(20) DEFAULT NULL,
Expand All @@ -34,14 +12,3 @@ CREATE TABLE `RefSrcMatch` (
KEY `sourceId` (`sourceId`),
KEY `refObjectId` (`refObjectId`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-04-16 15:20:06
33 changes: 0 additions & 33 deletions itest_src/datasets/case01/data/schema/Science_Ccd_Exposure.schema
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64)
--
-- Host: localhost Database: case01_m
-- ------------------------------------------------------
-- Server version 5.1.61-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `Science_Ccd_Exposure`
--

DROP TABLE IF EXISTS `Science_Ccd_Exposure`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `Science_Ccd_Exposure` (
`scienceCcdExposureId` bigint(20) NOT NULL,
`visit` int(11) NOT NULL,
Expand Down Expand Up @@ -67,14 +45,3 @@ CREATE TABLE `Science_Ccd_Exposure` (
`poly` binary(120) NOT NULL,
`flags` int(11) NOT NULL DEFAULT '0'
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;

/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2012-03-28 10:31:57
Loading

0 comments on commit 82715e6

Please sign in to comment.