From 6da488d339126e661b98a4eb755c8b9f88a34b7a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 16 May 2024 13:56:30 -0400 Subject: [PATCH] Adding AL2 image to support knnlib builds (#4714) Signed-off-by: Peter Zhu --- .../build.al2.opensearch.x64.arm64.dockerfile | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile new file mode 100644 index 0000000000..df32196a1b --- /dev/null +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -0,0 +1,107 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + + +FROM amazonlinux:2 + +ARG MAVEN_DIR=/usr/local/apache-maven +ARG CONTAINER_USER=ci-runner +ARG CONTAINER_USER_HOME=/home/ci-runner + +# Ensure localedef running correct with root permission +USER 0 + +# Add normal dependencies +RUN yum clean all && \ + amazon-linux-extras install epel -y && \ + yum update -y && \ + yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz + +# Create user group +RUN groupadd -g 1000 $CONTAINER_USER && \ + useradd -u 1000 -g 1000 -d $CONTAINER_USER_HOME $CONTAINER_USER && \ + mkdir -p $CONTAINER_USER_HOME && \ + chown -R 1000:1000 $CONTAINER_USER_HOME + +# Add Python dependencies +RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies +RUN yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Notebook dependencies +RUN yum install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && yum clean all + +# Add Yarn dependencies +RUN yum groupinstall -y "Development Tools" && yum clean all && rm -rf /var/cache/yum/* + +# Tools setup +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN yum install -y go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh + +# Install higher version of maven 3.8.x +RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"' | uniq | head -n 1` && \ + mkdir -p $MAVEN_DIR && (curl -s $MAVEN_URL | tar xzf - --strip-components=1 -C $MAVEN_DIR) && \ + echo "export M2_HOME=$MAVEN_DIR" > /etc/profile.d/maven_path.sh && \ + echo "export M2=\$M2_HOME/bin" >> /etc/profile.d/maven_path.sh && \ + echo "export PATH=\$M2:\$PATH" >> /etc/profile.d/maven_path.sh && \ + ln -sfn $MAVEN_DIR/bin/mvn /usr/local/bin/mvn + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install PKG builder dependencies with rvm +RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \ + curl -sSL https://get.rvm.io | bash -s stable + +# Switch shell for rvm related commands +SHELL ["/bin/bash", "-lc"] +CMD ["/bin/bash", "-l"] + +# Install ruby / rpm / fpm related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all + +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin +ENV RVM_HOME=/usr/local/rvm/bin +ENV GEM_HOME=$CONTAINER_USER_HOME/.gem +ENV GEM_PATH=$GEM_HOME +ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH + +# Install Python binary +RUN curl https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz | tar xzvf - && \ + cd Python-3.9.7 && \ + ./configure --enable-optimizations && \ + make altinstall + +# Setup Python links +RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ + ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip && \ + ln -sfn /usr/local/bin/pip3.9 /usr/local/bin/pip && \ + ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 + +# Add k-NN Library dependencies +RUN yum repolist && yum install openblas-static lapack gcc-gfortran -y +RUN pip3 install cmake==3.23.3 +# Upgrade gcc +RUN yum install -y gcc10* && \ + ln -s `which gcc10-gcc` /usr/local/bin/gcc && \ + ln -s `which gcc10-g++` /usr/local/bin/g++ + +# Change User +USER $CONTAINER_USER +WORKDIR $CONTAINER_USER_HOME + +# Install fpm for opensearch dashboards core +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 +ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH +RUN fpm -v