From 247ee37c8b9fa0842ac3b140f885b16f264911f4 Mon Sep 17 00:00:00 2001 From: mammo0 Date: Wed, 14 Feb 2024 16:02:33 +0100 Subject: [PATCH] simplified docker-compose.yml -> only start one OS instance by default -> use volumes for the OS data --- .dockerignore | 1 - README.md | 4 +- docker-compose.env | 18 ++----- docker-compose.yml | 104 ++++++++++++++++++++-------------------- os_data/os01/.gitignore | 5 -- os_data/os02/.gitignore | 5 -- os_data/os03/.gitignore | 5 -- 7 files changed, 58 insertions(+), 84 deletions(-) delete mode 100644 os_data/os01/.gitignore delete mode 100644 os_data/os02/.gitignore delete mode 100644 os_data/os03/.gitignore diff --git a/.dockerignore b/.dockerignore index 78a8528..a67af5b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ # the following files and directories are not needed for building the image /.* -/es_data /pcaps /docker-compose.* /LICENSE diff --git a/README.md b/README.md index ce2d924..cd48cf4 100644 --- a/README.md +++ b/README.md @@ -57,14 +57,12 @@ You can use `docker-compose` if you want to setup OpenSearch together with Arkim sysctl -w vm.max_map_count=262144 ``` -3. Run `chmod -R 777 os_data` **OR** `chown -R 1000:1000 os_data`. This is needed for the OpenSearch containers to be able to write to that direcotry. - #### Start Simply run ```shell docker-compose up -d ``` -*It can take some time until the OpenSearch instances are fully started.* +*It can take some time until the OpenSearch instance(s) is/are fully started.* The Arkime viewer instance can be accessed via diff --git a/docker-compose.env b/docker-compose.env index 89b4eb1..6ccec86 100644 --- a/docker-compose.env +++ b/docker-compose.env @@ -7,24 +7,14 @@ OS_VERSION=2.11.1 # the names of the OS nodes -# currently 3 instances are started OS_NODE1=os01 -OS_NODE2=os02 -OS_NODE3=os03 +# uncomment to create a cluster (more nodes can be added also) +# !!! do not forget to also adjust the docker-compose.yml file !!! +# OS_NODE2=os02 -# specify how much memory should one OS instance get -# keep in mind that currently 3 instances are started! +# specify how much memory should one OS instance should get OS_JAVA_MEM=512m -# the data of OS is stored in this direcotry -# it's recommended to store it on a SSD -# please note: -# 1. this directory MUST contain 3 sub-directories with the names of the OS -# nodes above or just copy the directory to the SSD and set the path here -# 2. this direcotry and especially the sub-directories must have file -# permissions 777 set. Or they must be owned by user AND group 1000. -OS_DATA_DIR=./os_data - # Arkime settings ################# diff --git a/docker-compose.yml b/docker-compose.yml index 603fdfe..6cc1b13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,60 @@ -x-os_instance: &_os_instance - image: opensearchproject/opensearch:$OS_VERSION - ulimits: - nofile: - soft: 65535 - hard: 65535 - memlock: - soft: -1 - hard: -1 - networks: - - arkime-os - - services: os01: - <<: *_os_instance + image: opensearchproject/opensearch:$OS_VERSION + ulimits: + nofile: + soft: 65535 + hard: 65535 + memlock: + soft: -1 + hard: -1 container_name: $OS_NODE1 environment: - node.name=$OS_NODE1 - - discovery.seed_hosts=$OS_NODE2,$OS_NODE3 - # default - - cluster.name=os-arkime-cluster - - cluster.initial_master_nodes=$OS_NODE1,$OS_NODE2,$OS_NODE3 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms$OS_JAVA_MEM -Xmx$OS_JAVA_MEM" - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - volumes: - - $OS_DATA_DIR/$OS_NODE1:/usr/share/opensearch/data - os02: - <<: *_os_instance - container_name: $OS_NODE2 - environment: - - node.name=$OS_NODE2 - - discovery.seed_hosts=$OS_NODE1,$OS_NODE3 - # default - - cluster.name=os-arkime-cluster - - cluster.initial_master_nodes=$OS_NODE1,$OS_NODE2,$OS_NODE3 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms$OS_JAVA_MEM -Xmx$OS_JAVA_MEM" - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - volumes: - - $OS_DATA_DIR/$OS_NODE2:/usr/share/opensearch/data - os03: - <<: *_os_instance - container_name: $OS_NODE3 - environment: - - node.name=$OS_NODE3 - - discovery.seed_hosts=$OS_NODE1,$OS_NODE2 - # default - - cluster.name=os-arkime-cluster - - cluster.initial_master_nodes=$OS_NODE1,$OS_NODE2,$OS_NODE3 - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms$OS_JAVA_MEM -Xmx$OS_JAVA_MEM" - DISABLE_SECURITY_PLUGIN=true - DISABLE_INSTALL_DEMO_CONFIG=true + + # comment this if an OS cluster should be created + - discovery.type=single-node + # uncomment this if an OS cluster should be created + # also add other OS node containers to this compose file + # - cluster.name=os-arkime-cluster + # - cluster.initial_cluster_manager_nodes=$OS_NODE1 + # this variable must contain the names (comma-separated) of the OTHER (excluding this container) OS node containers that are part of the cluster + # - discovery.seed_hosts=$OS_NODE2 volumes: - - $OS_DATA_DIR/$OS_NODE3:/usr/share/opensearch/data + - os01_data:/usr/share/opensearch/data + networks: + - arkime-os + # uncomment to add another OS node container and create a cluster + # additional some modifications to the "environment" section of the os01 container must be made + # os02: + # image: opensearchproject/opensearch:$OS_VERSION + # ulimits: + # nofile: + # soft: 65535 + # hard: 65535 + # memlock: + # soft: -1 + # hard: -1 + # container_name: $OS_NODE2 + # environment: + # - node.name=$OS_NODE2 + # - bootstrap.memory_lock=true + # - "OPENSEARCH_JAVA_OPTS=-Xms$OS_JAVA_MEM -Xmx$OS_JAVA_MEM" + # - DISABLE_SECURITY_PLUGIN=true + # - DISABLE_INSTALL_DEMO_CONFIG=true + + # # cluster settings + # - cluster.name=os-arkime-cluster + # # this variable must contain the names (comma-separated) of the OTHER (excluding this container) OS node containers that are part of the cluster + # - discovery.seed_hosts=$OS_NODE1 + # volumes: + # - os02_data:/usr/share/opensearch/data + # networks: + # - arkime-os arkime: build: @@ -64,8 +62,8 @@ services: dockerfile: Dockerfile depends_on: - $OS_NODE1 - - $OS_NODE2 - - $OS_NODE3 + # uncomment and add all OS node container names + # - $OS_NODE2 environment: - OS_HOST=$OS_NODE1 - OS_PORT=9200 @@ -82,6 +80,10 @@ volumes: arkime_config: arkime_logs: + os01_data: + # add a volume foreach OS node container + # os02_data: + networks: arkime-os: diff --git a/os_data/os01/.gitignore b/os_data/os01/.gitignore deleted file mode 100644 index 0e13320..0000000 --- a/os_data/os01/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# everything in this folder -* - -# exclude self -!.gitignore diff --git a/os_data/os02/.gitignore b/os_data/os02/.gitignore deleted file mode 100644 index 0e13320..0000000 --- a/os_data/os02/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# everything in this folder -* - -# exclude self -!.gitignore diff --git a/os_data/os03/.gitignore b/os_data/os03/.gitignore deleted file mode 100644 index 0e13320..0000000 --- a/os_data/os03/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# everything in this folder -* - -# exclude self -!.gitignore