From 27a21ee703aabe2c651f90da38c015fe05c6f82c Mon Sep 17 00:00:00 2001 From: Mukul Chandrakant Mahadik Date: Mon, 18 Mar 2024 16:46:23 -0700 Subject: [PATCH 01/71] Create image_build_push.yml --- .github/workflows/image_build_push.yml | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/image_build_push.yml diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml new file mode 100644 index 0000000..2303ede --- /dev/null +++ b/.github/workflows/image_build_push.yml @@ -0,0 +1,49 @@ +# This is a basic workflow to help you get started with Actions + +name: docker-image-push-admin + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ image-push-merge ] + + +# Env variable +env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: docker login + run: | # log into docker hub account + echo "Docker user name: " $DOCKER_USER + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + + - name: Get current date # get the date of the build + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')" + + #Runs a single command using the runners shell + - name: Run a one-line script + run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} + + # Runs a set of commands using the runners shell + - name: build docker image + run: | + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker images + + - name: push docker image + run: | + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From 8b6c9141c6d4627be240ae59bd5294eb6848c03a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 18 Mar 2024 16:58:08 -0700 Subject: [PATCH 02/71] Corrected Dockerfile path --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 2303ede..1c45e26 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -41,7 +41,7 @@ jobs: # Runs a set of commands using the runners shell - name: build docker image run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./docker docker images - name: push docker image From 9fa5ead7612b77554caafe6e38599681a83a601e Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 19 Mar 2024 17:09:14 -0700 Subject: [PATCH 03/71] Added Dockerfile in repo root. Need to make it available outside the docker folder as the build context was specified in the docker compose as the repo root directory and corresponding paths in the Dockerfile were relative to the root directory. Additionally, added Environment variables defined in docker compose file. The image wasn't running directly after building because these things were missing: - needed to add db container as well - ports needed to be exposed for both db and dashboard container - common network needed to be added under which both the containers had to be linked - all environment variables had to be added, especially important ones like DB_HOST --- .github/workflows/image_build_push.yml | 2 +- Dockerfile | 49 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 2303ede..f8bde38 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -41,7 +41,7 @@ jobs: # Runs a set of commands using the runners shell - name: build docker image run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} ./frontend + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . docker images - name: push docker image diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c0782bd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +FROM shankari/e-mission-server:master_2024-02-10--19-38 + +ENV DASH_DEBUG_MODE True +ENV SERVER_PORT 8050 +ENV DASH_SERVER_PORT 8050 +ENV DB_HOST db +ENV CONFIG_PATH "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" +ENV STUDY_CONFIG "stage-program" +ENV DASH_SILENCE_ROUTES_LOGGING False +ENV WEB_SERVER_HOST 0.0.0.0 +ENV SERVER_BRANCH master +ENV REACT_VERSION "18.2.0" + +# the other option is cognito +ENV AUTH_TYPE "basic" + +# copy over setup files +WORKDIR /usr/src/app/dashboard_setup +COPY requirements.txt nrel_dash_components-0.0.1.tar.gz docker/setup.sh ./ + +# install requirements.txt +WORKDIR /usr/src/app +RUN bash -c "./dashboard_setup/setup.sh" + +# copy over dashboard code +WORKDIR /usr/src/app/pages +COPY ./pages ./ +WORKDIR /usr/src/app/utils +COPY ./utils ./ +WORKDIR /usr/src/app +COPY app.py config.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ + +WORKDIR /usr/src/app/assets +COPY assets/style.css ./ +RUN mkdir qrcodes + +# copy over test data +WORKDIR /usr/src/app/data +COPY data ./ + +# open listening port, this may be overridden in docker-compose file +EXPOSE ${SERVER_PORT} +EXPOSE ${DASH_SERVER_PORT} + +# run the dashboard +WORKDIR /usr/src/app/dashboard_setup +COPY docker/start.sh ./ +WORKDIR /usr/src/app +CMD ["/bin/bash", "/usr/src/app/dashboard_setup/start.sh"] From 03454903a88aa7eb1c861f4e5465c0c5f4a62fd5 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 19 Mar 2024 17:21:33 -0700 Subject: [PATCH 04/71] Replaced config.py with config-fake.py in external Dockerfile. Encountered error while building and pushing docker image as a part of the CI. Due to config.py mentioned in COPY command in Dockerfile not being there in the repo online. Added config-fake.py in its place for now, so image is pushed to Dockerhub. But eventually, will need to handle as per decision to consolidate differences in external and internal repos. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c0782bd..2a00c5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ COPY ./pages ./ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app -COPY app.py config.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ +COPY app.py config-fake.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets COPY assets/style.css ./ From 7617a53bafd21bca744e786bc24c1d237416926f Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 19 Mar 2024 17:59:26 -0700 Subject: [PATCH 05/71] Copy config-fake.py as config.py In the external public Docker image, make the config-fake.py available as config.py by copying it over into the container as config.py Internally, if custom config.py is needed, can copy it over in the internal Dockerfile. But might not even need the config.py --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2a00c5c..708ef72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,8 @@ COPY ./pages ./ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app -COPY app.py config-fake.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ +COPY config-fake.py ./config.py +COPY app.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets COPY assets/style.css ./ From 8425f6e87c027ee71c1a221b1f9706d08de41196 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 20 Mar 2024 10:02:47 -0700 Subject: [PATCH 06/71] Removed expose port duplication Found usage of two variables referring to same port. Had exposed using both variable names. But since same port exposed, should be fine to expose just once. --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 708ef72..52e7461 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,6 @@ COPY data ./ # open listening port, this may be overridden in docker-compose file EXPOSE ${SERVER_PORT} -EXPOSE ${DASH_SERVER_PORT} # run the dashboard WORKDIR /usr/src/app/dashboard_setup From a24bd99cb934308a1e33f1f9859af41810044738 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 25 Mar 2024 15:41:43 -0700 Subject: [PATCH 07/71] Multiple changes for external repo differences 1. Changes in related files including Dockerfile for loading OpenPATH logo from local image rather than URL link. 2. Changed sed to jq in start.sh. 3. Changed dash version to latest v 2.16.1 4. Added new Dockerfile for image_build_push yml. Removed assets/ from COPY command as only contents of assets/ were being copied to root app directory. In COPY assets command, copy entire directory instead of just css file. --- Dockerfile | 4 ++-- app_sidebar_collapsible.py | 8 +++++--- assets/openpath-logo.jpg | Bin 0 -> 9970 bytes docker/Dockerfile | 4 ++-- docker/start.sh | 4 ++-- requirements.txt | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 assets/openpath-logo.jpg diff --git a/Dockerfile b/Dockerfile index 52e7461..8566da7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,10 +29,10 @@ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app COPY config-fake.py ./config.py -COPY app.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ +COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets -COPY assets/style.css ./ +COPY assets/ ./ RUN mkdir qrcodes # copy over test data diff --git a/app_sidebar_collapsible.py b/app_sidebar_collapsible.py index a3276ff..75cf9c7 100644 --- a/app_sidebar_collapsible.py +++ b/app_sidebar_collapsible.py @@ -17,6 +17,7 @@ from dash import Input, Output, dcc, html, Dash import dash_auth import logging +import base64 # Set the logging right at the top to make sure that debug # logs are displayed in dev mode # until https://github.com/plotly/dash/issues/532 is fixed @@ -29,8 +30,8 @@ import flask_talisman as flt - -OPENPATH_LOGO = "https://www.nrel.gov/transportation/assets/images/openpath-logo.jpg" +OPENPATH_LOGO = os.path.join(os.getcwd(), "assets/openpath-logo.jpg") +encoded_image = base64.b64encode(open(OPENPATH_LOGO, 'rb').read()).decode("utf-8") auth_type = os.getenv('AUTH_TYPE') @@ -59,7 +60,8 @@ [ # width: 3rem ensures the logo is the exact width of the # collapsed sidebar (accounting for padding) - html.Img(src=OPENPATH_LOGO, style={"width": "3rem"}), + # html.Img(src=OPENPATH_LOGO, style={"width": "3rem"}), + html.Img(src=f"data:image/png;base64,{encoded_image}", style={"width": "3rem"}), # Working html.H2("OpenPATH"), ], className="sidebar-header", diff --git a/assets/openpath-logo.jpg b/assets/openpath-logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1e13c576eefa039050c6412a498b35079138944b GIT binary patch literal 9970 zcma)iWmFu^^7rBrAh;7e=pw=0ZE=TPLXhANVX;7P3$DR~1^2~62p-%QcM0wm^iQ69 z@AK*X@YXrgHK(hmx@W5DH{D(RH2bsw2nHY{{iWv;85Q~YLPJ4DMnOYIMMXnJMaM+P zcqVjAOsv23Tw-HmVd4EH!hZ2LWjA)q|J4k8jdE&u@u2@x3)1qBTU1N&Kw@Z27Nf`^KaMnKCgrAbI<>JorX^hP?K zn1nttw_`#^R!dvg%-k(7p$;}a$-v03=la3DrjwM1S3u`|P;h?j+R5qjpq{m$pkkpQ zBO?9n@^AOZcqsUPHJZA_qY}_b2aFTH(K2&Q$gQdESUVvk(srAm*9ko3;bmaVdzuGe zJwJzlhlmG|0JKlZQSZmF&VMKiopW;ft$Y^CE@zz(3a<=cV?j=~y5x;s_S^*z7dc5M zi!KPe@L^M29grSDRFnH}>H_aZ1zs9X@Y|oqP)s6tEV=vEiIg~cd;?z95}S9(tn*Y6 z!!=;E1>Sc7$fmYS2Nj;fY6`*T;xu+H`F0=DEdivbd=SZ`_|zyGnNZ_<*8N8RN}2@c zRpyq74D%O}WYgHgoPtS;V4+zH3y&1gUSF6L0C0& z3C?mCcy+lH?Gp+-jt{eQQNIQE^;DX@Gq>b;`U1-${IVewRq1&gyZMv0Ji72mI_B_y*wa|kM|F4+ zXkLu`MTQfBmJEnWSrsSJZDlpb=^f^@myF#bCN`XUPXZXEbaV?$=&X|4>e&6Rc=LiO%) zUM-taXC*nVD+m^OeSB*}fqf=+9+}@vyZX^M?&C@8A@rhU+4(8@+?OmfsLq8=`OUIP z(E|+E3jsPOI7z;H+FXSo0nxN~+8+&g_C|4z2Bt6e!@0EukU1`|E zrsC3GOD=so$p$m&EajHV*B!aH?&dh-p^N(+ra$_!#hztJY3tB%JL=Upepp3!1EUso z0LzyzOS|0zM7<9;F5Vux_{cQIp0MPgYCmjQHmszOQ+4gtyzMX9fICs=9XLgMQ&E@y z{z#^9ot&_&1;WNz6`AQi1AAVkbzgJLX4;NV2_?8eF?G7i6mk}hIh_0)-Ye^OB-Nt}$|x}Qu@ukU{0cYC`zv!HwMC$(%c#l26Z5($2C+}o$OG%oroRy&|q%Ub`F zh)n)R8Ry)kZsffebl1ix>V2O9ur*+1T8}5z()Lk$E-9;%pu;1qKEU#a5ukWym2Dedw9~@-okS+i{q=h<%O8YVq^y(SNAI2{a!gCtY3$TxZ3=5Q=qHz5Y-Pgj zTiXA?g5&aBT?w_1Pbx{+@HzQ}oDo?j7}ilX*4JV(cUp|p?0*=a@}cFigjn-TA1bwk zWb0$+Rfr+3*m*Q^lt0?uap&Bqcr063DGE0vHc~rl)|&vw31v0$WZ&v7*xwCpS2Q&3 zCkri?*uau;9dijyY!eM~>rKuV9h}6)74&cTKzyiB5%) z8hG^bhHcM_NNh`H{dm1vu8p%~dF0}=`#5>t$b3#RavPZtt2b-&D#E%Jz<|2H)Nn*y zecE~c8)ft=mXX~S_M=4aIVJdn(s2Rsnp zux632=E>AkVQS7tIroubdz!31Tqyh^T(iwsDhf?sdM&JxjDfz*iO#Aq;1aXD4{tJJ zvCf=YNWZH@2c{MF%8e>QJ*mYr8byv7sZI=r=GLru&2lgKxnFl^utc z7Q>V#4r%uZlHuHA3gzY|i=sTcwtNi39)<5*vNg;&61J0J5(IGjCK967#17a$dPxwQ z7k{ZCjBwa51|6=WRg!5LGWuv`BcUoIy$&xdBlZJcua2)b$|(5N6$(j}>8iEmaZ2Se zuNdzju#5Os^F-v&q@mRKpPWL>J4U%L+aCV3vk1MRgPR$AICn#XSfAI&G!|x0TYzU+ zFj;nXpcP?Otx!!any8rRe)#6IGFF|u@S4|!py9f)yZwWq)C7{JDVP()VbAQ$;vz*l z3a?pdFr+Kn-@ghcw3%H=kukpNW=TcOBlgvFa|2IT?$TgIO9X)tcmJ9S_ZDO9z~25Y zKGopsSEt9U@ILo_&E?N+9Ws379FdW9TGa-Q-q{B|DPuElAj`FjdS3Q09K!uzD9CTZ z8>c>%g$3W1ISk9p6QTUz380Mg92y1m#4Py=`0m(04hCRg0eSET-MTS{udR#|B*H%a zZaL04zo`<$OVMOw9hLUs?irXMD!7@NW8I(MZa7g3PtO8t z^_C@hIvP3Rw1ijFzzB5?!rWlqxe5Tpe1A0*)k2XQ1A#8(OjUB!1RX;yL|3uS)n0$2 z-&EMtA2efEJR+#KO-E08KDAaTbdo&*4j*EVAUQ#LPS@gak1D^rMa3ro&&zeoCxC}K zI@1%N@k*jul)SFJ`Kqo>h*tss66NYk9_Ahr6f8!dg^MvpahgI8?DW2&FqfU&AueOd~Y=;AnhUiA30pO0#??P39QQU2Gx zS*TAVa~y7N{CqR(5_4OLTm&Dh9>Y|XBzuOKINy~e2=i&Wnbm~$bH>L)&djXVpU6xz zBdRuPVe)c>6>D8YAF1@YW;ixS*Ud#MNvd;LG@k&4y8822trjqcS7!|`5T;?HdB?7S zYzlA1m}l+v#gyGGhyN2acK@|{SO?2}(bs)L*M#`jdEw4IN~@@Kbr0-9{lB#0q71%8 zQ!1UwPaX`FN#w(P^W_$NQD+l2!U&lOHfn1UFA`AK5Qy=I-&jvAn0_ex1X6r(oo1rZC z=0iVOFB>U%GGUyYiMGDnt8hCrW>I~)-jUr#sLAP#3*}A}Nu{7xzU4db#43$X$-)W` zW=@w(w%bmRL)gbRm<5T?u*AJnw_)o#UHi>ANLI7AeT>co`e15xZ<=*(n*Xo$kB>*Z zl>2K4$s#N-&fOHALEs1Q&mq5VnQL5Lc1%#2>DUzUGut)Q<5E{#2Buc z9k%ig`u=-d`xE^83sci(DFhKcv5&t6m)3ODLalFYMkA|nh%h30djEXnlm1xP`xrOP z2wP~dzQ{O+NW6}A;5_wcmV$~fj%qHE)y zFiyXa_||Wv&h#%zjz%4FNO)nth?s{K*xWR(o1jH5VN>0XByFHp5P@c~@%S$z25yPzU4Z)@-z=__MKQgTSO$<}DtHl@dIW+Lj`s71A zZ)`mIMX9k@Us5^Ys1m(MY+2YpGEdiGFBj0lZdEN9~3 zi=#@gbm8xDZQp@A5T^tsET>VRD4;tabn2{+25JeX7u6T6tAO|kxg*?sxQ7lfA3L)` z*!azQc&+UkGr5p#-h$OZ;13}%$%Z1I{myTr1_--2FUy~8S^SSJ_4ItRo&mep^7)77 zax}%ux&WGd2JCwkz$qm6FJPx^4%kxaZ;Ly3L#Ot?9M>Z;k)86TaZ^>(3e(q`aEL{= zP+%#9MlD+BqpUPFdWSw1RMWBB$w1)UD7h~n!}VDmBB5IwRM$*QJ2Mg~Fo%d=^0uUQ zraO)asQ2o`Q=5$`N^`icXcRcyMjT7NGO<@v;!NF)rPsIF6dANJ<|xFsX>N02uqYpH zh$!vXONKCsdBZm>z=%*Ixk4OShZm3r_DR||rg|rZU?`Q3yqrbZ$h(R5!`Uam^rn7> zvfw{1XJ4NGIQZi=vwrvs^(CrPkxrsrN0S_+$*Rt#tWt$-yeEKjWr+F99hnPhnVH*k zoWgbimk8S5<^p)(B^R*5M#MQZuTCXWO_8?E>7F9-{^E{9hR+pL>^Nkn!$z|coXP$M zs|Xx!ErL<5YYAU?4N?0$&x8Xb3?%*tlzpsgUw?)T&VWIw`cK>0%PY~P+7%r(FUG7t z&(mFy(}jDsep_3}aMF>sev@jfETEc&*Ss9==`W05r3<^4vA-X$6!_Ejz z0rVX*=Q9nFn4ApLuW5CeBQ)o8aAZkh$de}K43GV5qV7fDyhE*)G(QMxeUv>zE<$j- zJUxdCUSZ-UR=PVzhw&)z4=VeH2$6;9B$Lc@WM_WGA!&tfpa0%4O3{L6)vXCD2I%3B zxfEAcxDU5pROVy$CFb~8M)2~wkiO-eDNWjS)T(;P+X)Lzj?GrqZ?G{JvgIqwp z3?MkEOg2~a!_loCjdFpWFC~mu49cpjw$Pj^$o&YPTu7hpRxVd&V8h;q9!m###hl;6 zlB=8;H!l?Tyi0%ZPeFgaXO20A;1!&>-p#%_{t`hqC}yY zE8=z8DDx&p3EGq%eL{o(uf`ehWbX*Mpa5+ICRwj6`(-g@LER;_rjKX}EHveIpY0Pa z?{W>C+!8xjJh{uLFH?Dl7@vZhIG|BXq^#aM$aZi>Oqx>q#YDDf@H6>WT{Z-<6X{w@N#l7s4wSj*|fIIt)2>Mz2)hwPKOLSzN- zc9QZYE56UawXt!jkIJ>CRiYol8-tr0QL=KsAQ}ep3KC_3&f@9|+gcmq+ZL9{l*e*@ zKAdUjz5^qJF($*Eo?~mry3%tlwBP1GK4M4`!~H51%nvH@@Tt~3 zEn7bfwJ8BdK?c&d)V`i39eTu7Pvg&fL>*AUi2Is9I}uc@c_Gbgc}XGe6I>kWp(~~6 z>LdJcIEBSeFHX_;FZ*teIOLj4MV~^nRSdcERcqhh3{ieed=$LnV~g<7V<$66I2?GL zWvuZ8U>9qdBvD`S4EoAx$w_hvY*5Q-X(6pu1c3^E<$^@s^d z%EIOJky-7UrO`>U&%3Q1Ui4eoQI*{HK4A>*h4-zp@=yv=$ikKEH&0!;KY-_z=~sj z@oQ-2_5DaoK!v%hpH$K9G0~Y_9nWGq&{;uy5%~C}urq|V`&h2+B=2`MP!WZNX%Mr= zY|vHBW~_>$f~nN)P*|+YM6@&|1pkn7Rk&*t$FDo}E1%p-SWB@%Wyi0pqW9J%U*i}b9jbji){pqRv5CYZ!L*JYOFe7gIKw z7@DA<5gaH^)R__YF4x#wj=3l4+`%51Flf?w1TEPc3&m76c*3^*X`+2fHp^Qu>>l>$ zO(zX1&UIPlqsme1lg${)QLyRMng!{sPbs+K$El%3lj8%}xW}M!0GwlDgCI^is z#>Mil&jP>8dycuVIZxd>VAaqrOc5IXr? zaP`;?)Pv9Z0%v%~g1#2-N4dc)3zNS!`8u~KT0Xp4?{2b$ce&{eBxUlADrvCDea35zqr9olT+woQU7+(nepLUBGopaZ7jHKAEkDsyfN z7|xAe%os~y1B#*j03qt+E&OS1y`7a@<(+J1_m+)+>AcLtdbk^7@mOpM*6`~nIad}} zA=y2g7*=3lv|+}7G$}0*@~yjeJB=WNYf-HOe8k$a*Z*r~QBFIBki}t<2%boQ26~tM^2`YV zh2YIs;y1y2wywx*$9f`NS0UTQC7X&dYpfV9HG!%41g)-WldYr`;DzdTE;l;9Ys!Hr zIVrSl%vpOo6Gfq!4ByHrYuij7PwK9PMMbQ48%jB^k);4iDj|K#%a=_`(!y9Jwl}() zNo5u${4Z2xLH>R|g%1=-7EY&u)O6f4FmE>+CwTYsD*#D|NGt(`h zxFLzzed{+)+kGBC)9lNv4?mRWHj&;O3JPrdxKfqLZaEZSGaBX>Jls;ur7bjr$n$mP zFOqBY+0r8xyeT2BpVC7++*Y%_dzA&SV};o| z$b6Xg(C3)Zz1_5rB||>@Vkd+y?>EiX65H9dc$rJ=$`V0PZEZaz?mc4~ml(o?E)yz{ z#UVWU%UZFprK7Y4#}Kws>U()*+TrDv`ZV?I35mrqz(0?864{ZM4*aXXMx7|Vbc!`A z@_v_G7=QOrmMEfQk~P+AJS2I+>p~R1Yg*W{kwCQXV39Mwt(AO>5Cunsy5Qi{niAuw1#W=M&%+@7ppp!%LBB4p4xf zw~=Q>ZbNgwZkrU28-g)_(YO*DzieT2fI@E80j`|+i$s95T~a{N#-<^1tkVd1=*EG? zM&HE?s+f=6Ldl1f@0?gkHn-8KkT&v1LDRLocr=V0nB0Qk0Pkf&S$J|j-9Fnx-d5ov zlfquZa6kS;rZ4y zGH!&s-_FVRJIrA;n>*^xj?c6r7c1^Lu@gVX2*c%BK_Vr~17Ce{rXV;$69Qh6_x#ydi!Ke~%Em@o&y z$?+(~BImL3;*TTA69rpqgECDclLcAro(v&81_%Q`#cZ|t4upJ>;DtQKT{B~1mO~aL z4229)aFsWT8aqkvp5V{lJ*;#DX!uA(eZ;Gw&DY4P!;!L>$C1Pl-kAGAuI)v!)?fZS z*Gz6^&EpW6J|hZ5JR*o-K{IDB+HF}o2!3>1Wv!fqt^Rr*KNOooIBfX^hnDU+HwO=F z2MHW&x$<+Z%%_)e6-;k&1q=V^48d&9SVl5xE? zd&d`xsOsuZ0P6@UVWEKA-A}{R3vz@If`U2q{p2SAuO@RtkqRtl2J_K6Pt%h3T;|gK z({p7V@$0_5fjboZ&)gRFe&townxG-DhJ>z!Ngn&8gXDw8Ly|1K&H3P+Wm?_QD05Wd zv(w5r&^+j1Jrp%pa7=8DZfv;Kn2VTv&6m$T7fSYJv-2ZMCGCLC;6n?Z0BwXXmjv30 za}G;K5mq0|nov^LknRI|sJ)q~Tb!OVWDDy5R3o1RJW8&|{@c39|KyYIBBJMWJkq#G zt`W71IjWoAz@7jOqG&<4m8*pRk*Ki|hg2IFsic5)B8vCY@X^5upS%GWukNiDAVgXH z{X$G=FcMMstY`siO164ERdD=9B!T=M#vp{X^xgAwwTH|uqzh)qVd!14810gRQ)G3zpx|t!nEA+wY~@mWcbZqNZnDpbwM0J+j%Mh3eeE!}1ulgp1v=SJ z(0gI!$m`4v@qr6MBHK0`ny_m7u-2 ztza>Q44Tjjj*Mp>GS%QWV%}txe8`&7uu(O+tDA-Oq~BS`twAaj8yP_mMHfTH7Z@~1 zW0dC2Lqse%OT`@DrEGVL-6ilIw|_U(zaF=Cm?5=u)ox?5vs0874n{Eri7AZ;4}{E- zM3Oh$byBQTwT8jpCK3}Ei61>_xOkb>1x8Qt;bCjW-{5| z05R(0JET@%bYyh2Hf;THtjr=7HBd(kYa?dXE;Cfxi>u0?sT5m}BzaY1*@G4a#=ZkC z)87Wpjp$s_i!LNk81>w0fhH0Zx0vn46$r%3d-Uf-U78jB>v6oX&ewSC?~aBi73$;evf4IMIIap+Sn)nQ=p4{b9a zOPnZ_p6kwLaG}@U^>Fac0I^QmY99i%Rg~-jFnq17QT!Zm__xu-;|E>CCqPu{gUx(k z0q4_X&G(OM7AtyRyz^Cfc1b*eq-X0i!7ck3nqb0Z#eq0boAKtWEP1RD6dHlPVXt$6 zp)iGh<0|Z1!4%vpp{72r-Yv(ayQQ1axB+ zNlXhhpc<~?3qy?EOn>1^L1dK}9J__bpn7AAp2;fXg9IuE4iVC%k_aW~29!h4gD%p# zsIZq#GKNQ(DDSlQ$X3avV_|riEvLp2wyaJPOI~v-VX>34)eLrHs}Oyaid7}1wq+km z1>m5J2~QAuxVCI#WYf=)D`|q5RDqz{5|f6)lsozfG5DxWIt04Lcx(J+{LP2OG!Jz9 zF99#VZx$ag197l_p#dr+4@qBH=7QX8_AUCW65qGl#jVJ7_hcf=@m8O*rmP{EQnUnR zPfbJ$!)B|ZotZQ`T<3rUMr{fZm!z(MX&T!2CYmCK566+on69h7xwtnj=0?8|#fA~p zE`>*dr)eA;BL4cCu_01ggf4xb{l3D}ueu5=su`Odq9|xEUy{zWgxG2*n<}Bm3X+VAn}w%5s$8{Zm8l!gH+T9Gd`=Ic#Ro%J zwY((XqXfiD2MFj}wLSp`-Wxq2st^B2dil-OXw!WAp?TZf$uDa+moVTgD1?8lD|JIM z=AZ=C$c)?+)5Q~0NJIAcxU%%%;qt09DwR<+zQVT}yrFD8{M+5XSs zLlggXEd?y}t9n+0$rE7gEuPFL#w3TDJ%Y?_olgVf64?dCn4gNZ{yq2e-{U|3u=c09 F{{ysm_00eP literal 0 HcmV?d00001 diff --git a/docker/Dockerfile b/docker/Dockerfile index 2c564f3..bafd1e0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,10 +17,10 @@ COPY ./pages ./ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app -COPY app.py config.py app_sidebar_collapsible.py assets globals.py globalsUpdater.py Procfile ./ +COPY app.py config.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets -COPY assets/style.css ./ +COPY assets/ ./ RUN mkdir qrcodes # copy over test data diff --git a/docker/start.sh b/docker/start.sh index 05b2f12..97c3fa9 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -5,9 +5,9 @@ source setup/activate.sh echo "DB host = "${DB_HOST} if [ -z ${DB_HOST} ] ; then local_host=`hostname -i` - sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf else - sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf + jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf fi # run the app diff --git a/requirements.txt b/requirements.txt index 55f3429..b0b85a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ --extra-index-url https://plotly.nrel.gov/Docs/packages # dash is required to call `build:py` -dash==2.9.3 +dash==2.16.1 gunicorn==20.1.0 plotly==5.14.1 dash-bootstrap-components==1.4.1 From bda1902b9f05f9b1fb6f0c4985161a5826f1c77e Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 26 Mar 2024 10:38:34 -0700 Subject: [PATCH 08/71] Added TODO to change image push branch Currently the branch specified - "image-push-merge" is available locally on my system. I use it to test the automated docker image push mechanism whenever any changes are merged to this branch. Once, everything looks good, need to change this to master or main as per the repo. --- .github/workflows/image_build_push.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index f8bde38..bb23982 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,6 +6,10 @@ name: docker-image-push-admin # events but only for the master branch on: push: + # Mukul: + # I've added a local test branch on my system and using it for testing image push. + # So, for testing purposes, need to checkout a branch "image-push-merge" + # TODO: Need to change to build off master or main once it looks good. branches: [ image-push-merge ] From 3064af659b64d8ecfe41484d491e9326344b3143 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 26 Mar 2024 11:17:33 -0700 Subject: [PATCH 09/71] Removed printing Docker username Had added it initially for testing purposes. Can remove now so it doesn't expose any sensitive info. --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index bb23982..f31af10 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -31,7 +31,6 @@ jobs: - uses: actions/checkout@v2 - name: docker login run: | # log into docker hub account - echo "Docker user name: " $DOCKER_USER docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - name: Get current date # get the date of the build From 9900c0a9daf6d6a263a35ec3d4aba693e0f83771 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 29 Mar 2024 01:43:20 -0700 Subject: [PATCH 10/71] Replacing CognitoConfig class with ENV variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Option 1: Can we read in these values directly from environment variables instead of reading from a class? - Option 2: Create a class that makes all environment variables available as python variables For now, I’m going ahead with reading in these variables directly. So, will need to replace all uses of CognitoConfig class with variable name directly in files that use it. Shankari mentioned in a commit whether we even need config.py? https://github.com/e-mission/op-admin-dashboard/commit/927817a95a9f7f9c4307f98cbfe2ef02584c047d Also, removed references to both config-fake.py and config.py in the Dockerfiles. --- Dockerfile | 1 - config-fake.py | 21 --------------------- docker/Dockerfile | 2 +- utils/cognito_utils.py | 12 ++++++------ utils/decode_jwt.py | 15 +++++++-------- 5 files changed, 14 insertions(+), 37 deletions(-) delete mode 100644 config-fake.py diff --git a/Dockerfile b/Dockerfile index 8566da7..1db51d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,6 @@ COPY ./pages ./ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app -COPY config-fake.py ./config.py COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets diff --git a/config-fake.py b/config-fake.py deleted file mode 100644 index bcfc85f..0000000 --- a/config-fake.py +++ /dev/null @@ -1,21 +0,0 @@ -# This is a template for the "config.py" file. Put all of your valid data in this file and then change the name of this -# file to "config.py". -import os - -# Fill these variables when using AWS Cognito authentication; otherwise, keep them empty. -# All the variables of this class must be filled in using your credential data from your AWS Cognito panel. -class CognitoConfig: - CLIENT_ID = os.getenv("COGNITO_CLIENT_ID", '') - CLIENT_SECRET = os.getenv("COGNITO_CLIENT_SECRET", '') - REDIRECT_URL = os.getenv("COGNITO_REDIRECT_URL", '') - TOKEN_ENDPOINT = os.getenv("COGNITO_TOKEN_ENDPOINT", '') - USER_POOL_ID = os.getenv("COGNITO_USER_POOL_ID", '') - REGION = os.getenv("COGNITO_REGION", '') - AUTH_URL = os.getenv("COGNITO_AUTH_URL", '') - - -# Fill this variable when using basic authentication; otherwise, keep it empty. -# This dictionary contains all the valid usernames and passwords that users can authenticate with. -VALID_USERNAME_PASSWORD_PAIRS = { - 'hello': 'world' -} diff --git a/docker/Dockerfile b/docker/Dockerfile index bafd1e0..a320e37 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,7 +17,7 @@ COPY ./pages ./ WORKDIR /usr/src/app/utils COPY ./utils ./ WORKDIR /usr/src/app -COPY app.py config.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ +COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ WORKDIR /usr/src/app/assets COPY assets/ ./ diff --git a/utils/cognito_utils.py b/utils/cognito_utils.py index 59203d4..17f2e30 100644 --- a/utils/cognito_utils.py +++ b/utils/cognito_utils.py @@ -4,16 +4,16 @@ import flask import requests import dash +import os -from config import CognitoConfig from utils import decode_jwt def get_tokens(code): - client_id = CognitoConfig.CLIENT_ID - client_secret = CognitoConfig.CLIENT_SECRET - redirect_uri = CognitoConfig.REDIRECT_URL - token_endpoint = CognitoConfig.TOKEN_ENDPOINT + client_id = os.getenv("COGNITO_CLIENT_ID", '') + client_secret = os.getenv("COGNITO_CLIENT_SECRET", '') + redirect_uri = os.getenv("COGNITO_REDIRECT_URL", '') + token_endpoint = os.getenv("COGNITO_TOKEN_ENDPOINT", '') encoded_data = base64.b64encode(f'{client_id}:{client_secret}'.encode('ascii')).decode('ascii') headers = { @@ -59,7 +59,7 @@ def get_cognito_login_page(text='Welcome to the dashboard', color='black'): dash.html.Label(text, style={ 'font-size': '15px', 'display': 'block', 'verticalAlign': 'top', 'padding': '15px', 'color': color }), - dbc.Button('Login with AWS Cognito', id='login-button', href=CognitoConfig.AUTH_URL, style={ + dbc.Button('Login with AWS Cognito', id='login-button', href=os.getenv("COGNITO_AUTH_URL", ''), style={ 'font-size': '14px', 'display': 'block', 'padding': '15px', 'verticalAlign': 'top', 'background-color': 'green', 'color': 'white' }), diff --git a/utils/decode_jwt.py b/utils/decode_jwt.py index 8f565bb..dca5c02 100644 --- a/utils/decode_jwt.py +++ b/utils/decode_jwt.py @@ -14,15 +14,14 @@ import urllib.request from jose import jwk, jwt from jose.utils import base64url_decode +import os -from config import CognitoConfig - -client_id = CognitoConfig.CLIENT_ID -client_secret = CognitoConfig.CLIENT_SECRET -redirect_uri = CognitoConfig.REDIRECT_URL -token_endpoint = CognitoConfig.TOKEN_ENDPOINT -user_pool_id = CognitoConfig.USER_POOL_ID -region = CognitoConfig.REGION +client_id = os.getenv("COGNITO_CLIENT_ID", '') +client_secret = os.getenv("COGNITO_CLIENT_SECRET", '') +redirect_uri = os.getenv("COGNITO_REDIRECT_URL", '') +token_endpoint = os.getenv("COGNITO_TOKEN_ENDPOINT", '') +user_pool_id = os.getenv("COGNITO_USER_POOL_ID", '') +region = os.getenv("COGNITO_REGION", '') keys_url = f'https://cognito-idp.{region}.amazonaws.com/{user_pool_id}/.well-known/jwks.json' # instead of re-downloading the public keys every time From 8c2038e5cade8ddef474eeaf40a4b4aec8d7ca66 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 29 Mar 2024 02:00:20 -0700 Subject: [PATCH 11/71] Replaced dictionary in config.py Added actual template value directly where the variable was being used through the config.py file. --- app_sidebar_collapsible.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app_sidebar_collapsible.py b/app_sidebar_collapsible.py index 75cf9c7..19a4d6e 100644 --- a/app_sidebar_collapsible.py +++ b/app_sidebar_collapsible.py @@ -38,7 +38,9 @@ if auth_type == 'cognito': from utils.cognito_utils import authenticate_user, get_cognito_login_page elif auth_type == 'basic': - from config import VALID_USERNAME_PASSWORD_PAIRS + VALID_USERNAME_PASSWORD_PAIRS = { + 'hello': 'world' + } app = Dash( external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.FONT_AWESOME], From 2580fd133eb85582e70769e9904aa3f07b28c3c1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 2 Apr 2024 13:37:56 -0700 Subject: [PATCH 12/71] Removing ENV variables from Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other ENV vars that were added in Docker-compose files are currently present in the Dockerfile in the external repo root. Removing them from there and can add them as per requirement. For local testing, add them when “docker run” is used using the -e flag. For usage in stage / production, can be set by cloud team in AWS Codebuild as they currently do. --- Dockerfile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1db51d6..a320e37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,6 @@ FROM shankari/e-mission-server:master_2024-02-10--19-38 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 -ENV DASH_SERVER_PORT 8050 -ENV DB_HOST db -ENV CONFIG_PATH "https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/" -ENV STUDY_CONFIG "stage-program" -ENV DASH_SILENCE_ROUTES_LOGGING False -ENV WEB_SERVER_HOST 0.0.0.0 -ENV SERVER_BRANCH master -ENV REACT_VERSION "18.2.0" - -# the other option is cognito -ENV AUTH_TYPE "basic" # copy over setup files WORKDIR /usr/src/app/dashboard_setup From ba77b5c42e409b67b580baa011a348f42eae20cb Mon Sep 17 00:00:00 2001 From: Mukul Chandrakant Mahadik Date: Fri, 5 Apr 2024 14:25:53 -0700 Subject: [PATCH 13/71] Update requirements.txt Changed pillow version to 10.3.0 Editing online manually to test if Github action still works after re-adding image_build_push.yml file. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0b85a8..c5c49ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ dash_extensions==0.1.13 #dashboard_setup/nrel_dash_components-0.0.1.tar.gz # for docker-compose pylint==2.17.2 qrcode==7.4.2 -pillow==10.0.1 +pillow==10.3.0 requests==2.31.0 python-jose==3.3.0 flask==2.2.5 From a709d492cb55d487410823532790886fd0778a91 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 11 Apr 2024 20:01:34 -0700 Subject: [PATCH 14/71] Removed sed / jq usage from start scripts Can directly set DB_HOST since we can now use environment variables. No need to use jq or sed to replace file contents and copy over files. --- docker/start.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/start.sh b/docker/start.sh index 97c3fa9..15a31e1 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -5,9 +5,8 @@ source setup/activate.sh echo "DB host = "${DB_HOST} if [ -z ${DB_HOST} ] ; then local_host=`hostname -i` - jq --arg db_host "$local_host" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf -else - jq --arg db_host "$DB_HOST" '.timeseries.url = $db_host' conf/storage/db.conf.sample > conf/storage/db.conf + export DB_HOST=$local_host + echo "Setting db host environment variable to localhost" fi # run the app From 8446e3c922d5e75a6a8698dc9b0089bb920a5154 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 12 Apr 2024 02:46:27 -0700 Subject: [PATCH 15/71] Changing base image to build from redesign server image Created a new branch image-push-merge for e-mission-server in my forked repo. Also modified image push workflow to build and push docker image to docker hub on push to image-push-merge branch. Doing this since admin-dash was failing when was building from internal repo since this was still referring to old server code. Changed Dockerfile and docker/Dockerfile.dev in public-dash and admin-dash to build from this new image mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 Redesigned server image is built from the image-push-merge branch on my personal forked repository. This branch has a workflow run set up to build the docker image and push it to Dockerhub whenever a push or merge happens to image-push-merge branch. Currently, I've been pushing to image-push and then creating a PR to merge into image-push-merge. Doing this, so admin-dash and public-dash can build from the latest redesigned server code. This is not the latest server code but the latest changes from my redesign PR. --- Dockerfile | 3 ++- docker/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a320e37..c65f4aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/Dockerfile b/docker/Dockerfile index a320e37..c65f4aa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,5 @@ -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-12--01-01 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From a9e46a69ca736f74239b044eddcf980ad047afac Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 15 Apr 2024 18:40:22 -0700 Subject: [PATCH 16/71] Bumped up base server image tag Changing to build from base server image from my personal Dockerhub repository with redesigned server code. Will need to change to build from Shankari's Dockerhub repository, once all changes are final. --- Dockerfile | 4 +++- docker/Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a320e37..4158a6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/Dockerfile b/docker/Dockerfile index a320e37..4158a6c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM shankari/e-mission-server:master_2024-02-10--19-38 +# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 586117128f1929f25539b82d94a2e612559b135b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 15 Apr 2024 20:56:54 -0700 Subject: [PATCH 17/71] Bump up base server image tag --- Dockerfile | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4158a6c..f8b9eee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4158a6c..f8b9eee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--42-27 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 237df169dda2fbee900921beeb40aeded6405ed5 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 01:35:08 -0700 Subject: [PATCH 18/71] Artifact download test - 1 Added working code from join repo to fetch docker image tags using artifact download. --- .github/fetch_runID.py | 67 ++++++++++++++++++++++ .github/workflows/image_build_push.yml | 79 ++++++++++++++++++++++---- 2 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 .github/fetch_runID.py diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py new file mode 100644 index 0000000..c40005d --- /dev/null +++ b/.github/fetch_runID.py @@ -0,0 +1,67 @@ + +import json +import logging +import requests +import sys + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + + # Just checking to see if workflows are being fetched; it works!!! + # download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows" + # logging.debug("About to fetch workflows present in e-mission-server from %s" % download_url) + # r = requests.get(download_url) + # if r.status_code != 200: + # logging.debug(f"Unable to fetch workflows, status code: {r.status_code}") + # sys.exit(1) + # else: + # workflows_json = json.loads(r.text) + # logging.debug(f"Successfully fetched workflows") + # print(workflows_json) + + # for workflow_entry in workflows_json["workflows"]: + # print("Workflow name: %s ; id: %s " % (workflow_entry["name"], workflow_entry["id"])) + + + + ''' + Workflow "docker image" uses image_build_push.yml + From above commented out code, and checked via terminal as well, + workflow id for the "docker image" can be fetched using: + https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows + https://api.github.com/repos/e-mission/e-mission-server/actions/workflows + + For MukuFlash03: id = 75506902 + For e-mission-server: id = 35580278 + ''' + download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) + r = requests.get(download_url) + if r.status_code != 200: + logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}") + sys.exit(1) + else: + workflow_runs_json = json.loads(r.text) + logging.debug(f"Successfully fetched workflow runs") + # print(workflow_runs_json) + + workflow_runs = workflow_runs_json["workflow_runs"] + if workflow_runs: + successful_runs = [run for run in workflow_runs \ + if run["status"] == "completed" and \ + run["conclusion"] == "success" and \ + run["head_branch"] == "tags-artifact" + ] + # print(successful_runs) + if successful_runs: + sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) + sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) + # print(sorted_runs) + latest_run_id = sorted_runs[0]["id"] + # print(latest_run_id) + print(f"::set-output name=run_id::{latest_run_id}") + # else: + # print("No successful runs") + # else: + # print("No workflow runs found") diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index f31af10..a159e47 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -8,9 +8,9 @@ on: push: # Mukul: # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "image-push-merge" + # So, for testing purposes, need to checkout a branch "tags-artifact" # TODO: Need to change to build off master or main once it looks good. - branches: [ image-push-merge ] + branches: [ tags-artifact ] # Env variable @@ -20,11 +20,46 @@ env: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: + + fetch_run_id: + runs-on: ubuntu-latest + + outputs: + run_id: ${{ steps.get_run_id.outputs.run_id }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Run Python script + id: run_script + run: | + echo "Fetching latest successful run ID from e-misison-server docker image workflow" + python .github/fetch_runID.py + + - name: Get Run ID + id: get_run_id + run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" + # This workflow contains a single job called "build" build: + needs: fetch_run_id + # The type of runner that the job will run on runs-on: ubuntu-latest + env: + RUN_ID: ${{needs.fetch_run_id.outputs.run_id}} + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -42,11 +77,35 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - - name: build docker image - run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . - docker images - - - name: push docker image - run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + # - name: build docker image + # run: | + # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + # docker images + + # - name: push docker image + # run: | + # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Use Run ID from previous fetch_run_id job + run: echo Run ID from previous job ${{ needs.fetch_run_id.outputs.run_id }} + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: docker-image-tag + github-token: ${{ secrets.GH_PAT_TAG }} + repository: MukuFlash03/e-mission-server + run-id: ${{ env.RUN_ID }} + + - name: Print artifact tag + run: cat tag_file.txt + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image-tag + path: tag_file.txt + overwrite: true + + - name: List artifact + run: ls -R From ab51504f7e57862162b0ae7107add907965cbba1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 30 Apr 2024 01:17:51 -0700 Subject: [PATCH 19/71] Bumped up server image tag Bumped up after fixing "url" KeyError bug in this commit in server repo: https://github.com/MukuFlash03/e-mission-server/commit/e778b3f3cf050eec33ef439a1b6763b6aaf533e0 --- Dockerfile | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f8b9eee..d3f4b99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-30--59-58 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/Dockerfile b/docker/Dockerfile index f8b9eee..d3f4b99 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +FROM mukuflash03/e-mission-server:image-push-merge_2024-04-30--59-58 ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From cd9682a7dbcf345bb1a90a9ce8c59f40b8fb8709 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 05:29:55 -0700 Subject: [PATCH 20/71] Artifact + Matrix - 1 Combining both artifact and matrix methods since both workflow is triggered by both push event and workflow_dispatch event. Workflow dispatch event receives tag via input parameter sent via server workflow. Push event does not receive any tag and the DOCKER_IMAGE_TAG_2 would have empty value since workflow triggering event and hence input parameter would be empty. So, was facing the issue of having empty timestamp being suffixed to the docker image tag in the Dockerfiles. Hence, using the logic of fetching the latest run id and then downloading the artifact uploaded by server workflow to ensure that a non-empty value is also retrieved for the timestamp. This value is stored in DOCKER_IMAGE_TAG_1 and can be used for building docker image. Now, depending on the trigger event, the appropriate docker image tag can be used in the docker build command for the --build-arg flag. Additionally, Dockerfiles now use ARG to use the tags passed from the docker build command, hence using environment variables. Docker-compose files similarly have the args config parameter set. Developers would have to set the correct server image tags manually here for the docker-compose command to pass the value to the ARG in the Dockerfile and correctly set the image tag to point to the appropriate server image. Need to mention somewhere in the ReadME.md to refer to the server image list in Dockerhub to choose the latest image tag. While pushing the image in the GitHub actions, it'll be done manually. Todo: Change branch name to tags-combo-approach in fetch_runID.py --- .github/workflows/image_build_push.yml | 97 ++++++++++++++++++-------- Dockerfile | 3 +- docker-compose-dev.yml | 2 + docker-compose-prod-nginx.yml | 2 + docker-compose-prod.yml | 2 + docker/Dockerfile | 3 +- 6 files changed, 77 insertions(+), 32 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index a159e47..b9d14fa 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -8,9 +8,15 @@ on: push: # Mukul: # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "tags-artifact" + # So, for testing purposes, need to checkout a branch "tags-combo-approach" # TODO: Need to change to build off master or main once it looks good. - branches: [ tags-artifact ] + branches: [ tags-combo-approach ] + + workflow_dispatch: + inputs: + docker_image_tag: + description: "Latest Docker image tags passed from e-mission-server repository on image build and push" + required: true # Env variable @@ -28,7 +34,7 @@ jobs: run_id: ${{ steps.get_run_id.outputs.run_id }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v2 @@ -50,20 +56,71 @@ jobs: id: get_run_id run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" + + fetch_tag: + needs: fetch_run_id + runs-on: ubuntu-latest + + env: + RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }} + + outputs: + docker_image_tag: ${{ steps.get_docker_tag.outputs.run_id }} + + steps: + - uses: actions/checkout@v4 + + - name: Use Run ID from previous fetch_run_id job + run: echo Run ID from previous job ${{ env.RUN_ID }} + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: docker-image-tag + github-token: ${{ secrets.GH_PAT_TAG }} + repository: MukuFlash03/e-mission-server + run-id: ${{ env.RUN_ID }} + + - name: Print artifact tag + id: get_docker_tag + run: | + docker_image_tag=$(cat tag_file.txt) + echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" + + + # - name: Upload Artifact + # uses: actions/upload-artifact@v4 + # with: + # name: docker-image-tag + # path: tag_file.txt + # overwrite: true + + # - name: List artifact + # run: ls -R + # This workflow contains a single job called "build" build: - needs: fetch_run_id + needs: fetch_tag # The type of runner that the job will run on runs-on: ubuntu-latest env: - RUN_ID: ${{needs.fetch_run_id.outputs.run_id}} + DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} + DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + + - name: Print input docker image tag + run: | + echo "Event name: ${{ github.event_name }}" + echo "Event name: ${{ github.event.action }}" + echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" + echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + - name: docker login run: | # log into docker hub account docker login -u $DOCKER_USER -p $DOCKER_PASSWORD @@ -79,33 +136,13 @@ jobs: # Runs a set of commands using the runners shell # - name: build docker image # run: | - # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + # if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + # docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + # else + # docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + # fi # docker images # - name: push docker image # run: | # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - - - name: Use Run ID from previous fetch_run_id job - run: echo Run ID from previous job ${{ needs.fetch_run_id.outputs.run_id }} - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: docker-image-tag - github-token: ${{ secrets.GH_PAT_TAG }} - repository: MukuFlash03/e-mission-server - run-id: ${{ env.RUN_ID }} - - - name: Print artifact tag - run: cat tag_file.txt - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: docker-image-tag - path: tag_file.txt - overwrite: true - - - name: List artifact - run: ls -R diff --git a/Dockerfile b/Dockerfile index f8b9eee..48516c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +ARG DOCKER_IMAGE_TAG +FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 75f8df2..11c0980 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,6 +5,8 @@ services: build: context: . dockerfile: docker/Dockerfile + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker-compose-prod-nginx.yml b/docker-compose-prod-nginx.yml index d4d6abf..684df78 100644 --- a/docker-compose-prod-nginx.yml +++ b/docker-compose-prod-nginx.yml @@ -6,6 +6,8 @@ services: build: context: . dockerfile: docker/Dockerfile + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 environment: DASH_DEBUG_MODE: "True" diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 8b50892..c8b6754 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -5,6 +5,8 @@ services: build: context: . dockerfile: docker/Dockerfile + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker/Dockerfile b/docker/Dockerfile index f8b9eee..48516c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,7 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_2024-04-16--49-36 +ARG DOCKER_IMAGE_TAG +FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 11dd97bf32696b5b1f0fa7c95fd2a7f5a0dec85d Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 06:10:24 -0700 Subject: [PATCH 21/71] Artifact + Matrix - 2 Was unable to see docker_image_tag in echo statement in logs. Added docker_image_tag print statement to see retrieved image tag. Ah! Was using run_id instead of docker_image_tag as the output value in outputs section in fetch_tag job. --- .github/workflows/image_build_push.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index b9d14fa..8676c8b 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -18,7 +18,6 @@ on: description: "Latest Docker image tags passed from e-mission-server repository on image build and push" required: true - # Env variable env: DOCKER_USER: ${{secrets.DOCKER_USER}} @@ -26,7 +25,6 @@ env: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - fetch_run_id: runs-on: ubuntu-latest @@ -56,7 +54,6 @@ jobs: id: get_run_id run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" - fetch_tag: needs: fetch_run_id runs-on: ubuntu-latest @@ -65,7 +62,7 @@ jobs: RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }} outputs: - docker_image_tag: ${{ steps.get_docker_tag.outputs.run_id }} + docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }} steps: - uses: actions/checkout@v4 @@ -84,10 +81,11 @@ jobs: - name: Print artifact tag id: get_docker_tag run: | + cat tag_file.txt docker_image_tag=$(cat tag_file.txt) + echo $docker_image_tag echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" - # - name: Upload Artifact # uses: actions/upload-artifact@v4 # with: @@ -112,12 +110,11 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" - echo "Event name: ${{ github.event.action }}" echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" From ed3a505bf0b0dbc638a46d4f604f187477c205ab Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 14:36:47 -0700 Subject: [PATCH 22/71] Artifact + Matrix - 3 Still not seeing the env vars echo-ed. Debugging by printing various log statements. --- .github/workflows/image_build_push.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 8676c8b..ea0f593 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -84,7 +84,11 @@ jobs: cat tag_file.txt docker_image_tag=$(cat tag_file.txt) echo $docker_image_tag + # echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" + + - name: Check artifact tag in output + run: echo "${{ steps.get_docker_tag.outputs.docker_image_tag }}" # - name: Upload Artifact # uses: actions/upload-artifact@v4 @@ -115,6 +119,7 @@ jobs: - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" + echo "Tag: ${{ needs.fetch_tag.outputs.docker_image_tag }}" echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" From d66893b001309e6821b6112397e0333a179b4b68 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 17:13:40 -0700 Subject: [PATCH 23/71] Artifact + Matrix - 4 Still not seeing the env vars echo-ed. Debugging by printing various log statements. --- .github/workflows/image_build_push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index ea0f593..94dca23 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -85,7 +85,10 @@ jobs: docker_image_tag=$(cat tag_file.txt) echo $docker_image_tag # echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" + echo "Checking set-output value..." echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" + echo "Checking key=value..." + echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT - name: Check artifact tag in output run: echo "${{ steps.get_docker_tag.outputs.docker_image_tag }}" From 39b289f28d57b3c42b59f916be29ba6a784640c7 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 17:29:33 -0700 Subject: [PATCH 24/71] Artifact + Matrix - 5 Working finally! Changed the deprecated set-output command to use {key}={value} format with echo command. Also, adding commented out build and push commands to see if build, push is successful. For this commit, not changing branch = tags-artifact in fetch_runID.py Once, server code has latest working yml file, then will come back and push another commit to change branch to tags-combo-approach. --- .github/workflows/image_build_push.yml | 31 ++++++++++---------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 94dca23..083a0d0 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -84,14 +84,7 @@ jobs: cat tag_file.txt docker_image_tag=$(cat tag_file.txt) echo $docker_image_tag - # echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" - echo "Checking set-output value..." - echo "::set-output name=docker_image_tag::$docker_image_tag" >> "$GITHUB_OUTPUT" - echo "Checking key=value..." echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT - - - name: Check artifact tag in output - run: echo "${{ steps.get_docker_tag.outputs.docker_image_tag }}" # - name: Upload Artifact # uses: actions/upload-artifact@v4 @@ -139,15 +132,15 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - # - name: build docker image - # run: | - # if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - # docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . - # else - # docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . - # fi - # docker images - - # - name: push docker image - # run: | - # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + - name: build docker image + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + else + docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + fi + docker images + + - name: push docker image + run: | + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From 340d9d0474b6f567afbd3fce21c01c890b5a332b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:04:58 -0700 Subject: [PATCH 25/71] Artifact + Matrix - 6 Removed an extra echo statement. --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 083a0d0..89134a0 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -115,7 +115,6 @@ jobs: - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" - echo "Tag: ${{ needs.fetch_tag.outputs.docker_image_tag }}" echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" From 15c9da147d879e84213124bbc6c17bba3f624f8b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:31:07 -0700 Subject: [PATCH 26/71] Artifact + Matrix - 7 Updating Dockerfiles to use ARG environment variable with latest timestamp that will be passed through: - `docker build --build-arg` command in Github actions in the workflow for automated pushing to Docker hub. - `args: ` config field in docker-compose which will need to be set manually by developers locally. Also, changing branch in fetch_runID and Dockerfiles to tags-combo-approach. --- .github/fetch_runID.py | 2 +- Dockerfile | 2 +- docker/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index c40005d..fee53c7 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -51,7 +51,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "tags-artifact" + run["head_branch"] == "tags-combo-approach" ] # print(successful_runs) if successful_runs: diff --git a/Dockerfile b/Dockerfile index 48516c6..4f344f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 ARG DOCKER_IMAGE_TAG -FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} +FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/Dockerfile b/docker/Dockerfile index 48516c6..4f344f7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 ARG DOCKER_IMAGE_TAG -FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} +FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 0c78ad3af3c36771dae67fdcf15e7548a2fb78de Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 2 May 2024 04:35:18 -0700 Subject: [PATCH 27/71] Artifact + Matrix - 8 For public-dash, admin-dash where ARGS are now being used, need to add the args under build command in the docker compose files. Gives error if arg is at the same hierarchical level as build. Also, public-dash docker-compose.yml (non-dev) version changed to have build: context, dockerfile ; unlike only build: frontend. This allows adding args under build. Similar to how currently being built in docker-compose.dev.yml. Also, args to be added under notebook-server and not dashboard since viz_scripts builds off of server image and not frontend, which is a node image. --- docker-compose-dev.yml | 4 ++-- docker-compose-prod-nginx.yml | 4 ++-- docker-compose-prod.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 11c0980..f70da0d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,8 +5,8 @@ services: build: context: . dockerfile: docker/Dockerfile - args: - DOCKER_IMAGE_TAG: '' + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker-compose-prod-nginx.yml b/docker-compose-prod-nginx.yml index 684df78..0155331 100644 --- a/docker-compose-prod-nginx.yml +++ b/docker-compose-prod-nginx.yml @@ -6,8 +6,8 @@ services: build: context: . dockerfile: docker/Dockerfile - args: - DOCKER_IMAGE_TAG: '' + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 environment: DASH_DEBUG_MODE: "True" diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index c8b6754..705b565 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -5,8 +5,8 @@ services: build: context: . dockerfile: docker/Dockerfile - args: - DOCKER_IMAGE_TAG: '' + args: + DOCKER_IMAGE_TAG: '' image: e-mission/opdash:0.0.1 ports: - "8050:8050" From d03ce01665757a97576920db600c89c18d0b3f51 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 02:47:31 -0700 Subject: [PATCH 28/71] Artifact + Matrix - 9 Adding .env file which stores only docker image timestamp for the latest dockerhub e-mission-server image already pushed. .env file overwritten in both types of trigger events - push and workflow_dispatch. Added commit and push github actions as well for pushing latest changes to the .env file made via the workflow. Lastly, docker-compose now also mentions the ENV variable name to be read from the .env file for the ARG value in the Dockerfile. No changes required in the Dockerfiles. --- .env | 1 + .github/workflows/image_build_push.yml | 18 ++++++++++++++++++ docker-compose-dev.yml | 2 +- docker-compose-prod-nginx.yml | 2 +- docker-compose-prod.yml | 2 +- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..203f891 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DOCKER_IMAGE_TAG=2024-05-02--16-40 \ No newline at end of file diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 89134a0..8a2cb02 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -118,6 +118,24 @@ jobs: echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + - name: Update .env file + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + else + echo "Push event: Restoring latest server image tag in .env" + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + fi + + - name: Add, Commit, Push changes to .env file + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add .env + git commit -m "Updated docker image tag in .env to the latest timestamp: ${{ env.DOCKER_IMAGE_TAG_2 }}" + git push origin + - name: docker login run: | # log into docker hub account docker login -u $DOCKER_USER -p $DOCKER_PASSWORD diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index f70da0d..742c8a8 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -6,7 +6,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: '' + DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker-compose-prod-nginx.yml b/docker-compose-prod-nginx.yml index 0155331..5626fce 100644 --- a/docker-compose-prod-nginx.yml +++ b/docker-compose-prod-nginx.yml @@ -7,7 +7,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: '' + DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} image: e-mission/opdash:0.0.1 environment: DASH_DEBUG_MODE: "True" diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 705b565..05a9579 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -6,7 +6,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: '' + DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} image: e-mission/opdash:0.0.1 ports: - "8050:8050" From a2eca046662a431d078e8d9a507acfb2a1d9f1c3 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 09:48:29 +0000 Subject: [PATCH 29/71] Updated docker image tag in .env to the latest timestamp: --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 203f891..881e743 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 \ No newline at end of file +DOCKER_IMAGE_TAG=2024-05-02--16-40 From 3b8f312a748c3424573b7a006fd07efaf81e5856 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 10:18:23 +0000 Subject: [PATCH 30/71] Updated docker image tag in .env to the latest timestamp: 2024-05-03--14-37 --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 881e743..03219ee 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 +DOCKER_IMAGE_TAG=2024-05-03--14-37 From 352c8ae3293cdc1adcecbd3053e804cf1f3f04e4 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 03:36:34 -0700 Subject: [PATCH 31/71] Added TODOs in github actions workflow YAML file. Reminder for things to change as per master branch of e-mission-server once changes are finalized. --- .github/fetch_runID.py | 2 ++ .github/workflows/image_build_push.yml | 3 ++- Dockerfile | 2 +- docker/Dockerfile | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index fee53c7..56ddf03 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -35,6 +35,7 @@ For e-mission-server: id = 35580278 ''' download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + # TODO: Comment the above line and Uncomment the below line representing the original emission repo # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) @@ -48,6 +49,7 @@ workflow_runs = workflow_runs_json["workflow_runs"] if workflow_runs: + # TODO: Change the head_branch name which identifies only e-mission-server runs triggered by this branch successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 8a2cb02..fa584c6 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -74,6 +74,7 @@ jobs: uses: actions/download-artifact@v4 with: name: docker-image-tag + # TODO: Create a token with basic repo permissions github-token: ${{ secrets.GH_PAT_TAG }} repository: MukuFlash03/e-mission-server run-id: ${{ env.RUN_ID }} @@ -133,7 +134,7 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" git add .env - git commit -m "Updated docker image tag in .env to the latest timestamp: ${{ env.DOCKER_IMAGE_TAG_2 }}" + git commit -m "Updated docker image tag in .env to the latest timestamp" git push origin - name: docker login diff --git a/Dockerfile b/Dockerfile index 4f344f7..003fe0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 ARG DOCKER_IMAGE_TAG FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f344f7..003fe0e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# TODO: Please change once all PR changes are final, so it reads from shankari/e-mission-server # FROM shankari/e-mission-server:master_2024-02-10--19-38 ARG DOCKER_IMAGE_TAG FROM mukuflash03/e-mission-server:tags-combo-approach_${DOCKER_IMAGE_TAG} From d98f75c8ba36a8b69c547f30e8840b6af773c7a5 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 13:07:19 -0700 Subject: [PATCH 32/71] Artifact + Matrix - 10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added another TODO. evious Push event triggers run failed Error occurred in GitHub actions git add, commit, push step. If file with no changes operated upon, it leaves an error: “nothing to commit, working tree clean Error: Process completed with exit code 1.” Need to fix. —— Quick fix is to make changes to .env file only if workflow_dispatch event is the trigger. Don’t do anything for push event. So, in case anyone modifies .env file on their own by using their own timestamp during testing, and pushes it as a part of their PR, then Shankari will have to ask them to revert the changes. Else, their custom timestamp will make it to the repo code base. Found something: https://www.reddit.com/r/github/comments/ju3ipr/commit_from_github_action_only_when_changes_exist/ It should work but there’s a drawback of using “exit 0” - it will mask all errors generated during “git commit”. This is bad and we won’t be able to see the reason why something wrong happened as the workflow would be shown as successful with a green tick. Found a solution with git diff: https://github.com/simonw/til/blob/main/github-actions/commit-if-file-changed.md $ git diff --quiet || (git add README.md && git commit -m "Updated README") However, I won’t be able to log any message saying that no changes to commit, tag not modified. Hence, will possibly use just “git diff —quiet” with an if-else block. Expected results: - Push event triggers workflow. - It writes DOCKER_IMAGE_TAG_1 fetched from last successful completed run to .env file. - It sees that there is a difference in the latest committed .env file in the dashboard repo which includes older timestamp. - Hence it runs git commit part of the script to reset to latest server timestamp. --- .env | 2 +- .github/workflows/image_build_push.yml | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 03219ee..881e743 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--14-37 +DOCKER_IMAGE_TAG=2024-05-02--16-40 diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index fa584c6..1f4e446 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -75,6 +75,7 @@ jobs: with: name: docker-image-tag # TODO: Create a token with basic repo permissions + # TODO: Change user / organization to emission instead of MukuFlash03 github-token: ${{ secrets.GH_PAT_TAG }} repository: MukuFlash03/e-mission-server run-id: ${{ env.RUN_ID }} @@ -133,9 +134,13 @@ jobs: run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add .env - git commit -m "Updated docker image tag in .env to the latest timestamp" - git push origin + if git diff --quiet; then + echo "Latest timestamp already present in .env file, no changes to commit" + else + git add .env + git commit -m "Updated docker image tag in .env file to the latest timestamp" + git push origin + fi - name: docker login run: | # log into docker hub account From f1ea34cf34f4b5a8756b9500eb16d6b2a1a923fe Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 20:08:01 +0000 Subject: [PATCH 33/71] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 881e743..03219ee 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-02--16-40 +DOCKER_IMAGE_TAG=2024-05-03--14-37 From 629831f6c1ef1deeca9c476051aa5ef3212cb830 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 May 2024 20:45:38 +0000 Subject: [PATCH 34/71] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 03219ee..8a84419 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--14-37 +DOCKER_IMAGE_TAG=2024-05-03--42-05 From 5e0de4b5f6869b83eafc94e76b915706fdafce45 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 6 May 2024 22:04:16 +0000 Subject: [PATCH 35/71] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8a84419..ac9b1b4 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--42-05 +DOCKER_IMAGE_TAG=2024-05-06--00-31 From 791b6d964ed922bdc95cd91f32fe4c347d11cefc Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Mon, 6 May 2024 17:13:39 -0600 Subject: [PATCH 36/71] Update Dockerfile Reverting the Dockerfile tag for testing so that checks can run properly --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe0736e..e659200 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ ARG DOCKER_IMAGE_TAG - -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +# Please change once all PR changes are final, so it reads from shankari/e-mission-server + # FROM shankari/e-mission-server:master_2024-02-10--19-38 + FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 4a90d2ba4b3188a0219fc54952dca2ef2da41194 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Mon, 6 May 2024 17:14:12 -0600 Subject: [PATCH 37/71] Syntax Removing a space. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e659200..e57cadf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG DOCKER_IMAGE_TAG # Please change once all PR changes are final, so it reads from shankari/e-mission-server - # FROM shankari/e-mission-server:master_2024-02-10--19-38 - FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 53dddcf52eae5a59f89a448bd60b3d5c9ab2fce7 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 6 May 2024 23:14:49 +0000 Subject: [PATCH 38/71] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index ac9b1b4..8a84419 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-06--00-31 +DOCKER_IMAGE_TAG=2024-05-03--42-05 From e56b1e1570adb0a4c328924d5706cef4c6d0eb47 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 7 May 2024 09:14:36 -0600 Subject: [PATCH 39/71] Update Dockerfile again Also editing this file so that checks pass for now --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fe0736e..2628e36 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,8 @@ ARG DOCKER_IMAGE_TAG -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +# Please change once all PR changes are final, so it reads from shankari/e-mission-server +# FROM shankari/e-mission-server:master_2024-02-10--19-38 +FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 980877f366d34cd8b36173d06871c47ab058a34e Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 7 May 2024 14:37:06 -0600 Subject: [PATCH 40/71] Finalize Dockerfiile Changing this to build from actual e-mission server in hopes of merging! --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e57cadf..fe0736e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ ARG DOCKER_IMAGE_TAG -# Please change once all PR changes are final, so it reads from shankari/e-mission-server -# FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} + +FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From f1fd42c6330d03fc5161c834067d272d8679ba5f Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 7 May 2024 14:38:10 -0600 Subject: [PATCH 41/71] Update Dockerfile in docker folder Potentially finalizing the dockerfile in the docker repo. --- docker/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2628e36..fe0736e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,6 @@ ARG DOCKER_IMAGE_TAG -# Please change once all PR changes are final, so it reads from shankari/e-mission-server -# FROM shankari/e-mission-server:master_2024-02-10--19-38 -FROM mukuflash03/e-mission-server:image-push-merge_${DOCKER_IMAGE_TAG} +FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From cb502af0edc7e0924be2a351e8e26d1a97b23979 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 7 May 2024 15:44:40 -0600 Subject: [PATCH 42/71] Update fetch_runID.py Removing prints/comments, updating head_branch and download_ url. --- .github/fetch_runID.py | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index 56ddf03..34617f8 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -7,23 +7,6 @@ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - # Just checking to see if workflows are being fetched; it works!!! - # download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows" - # logging.debug("About to fetch workflows present in e-mission-server from %s" % download_url) - # r = requests.get(download_url) - # if r.status_code != 200: - # logging.debug(f"Unable to fetch workflows, status code: {r.status_code}") - # sys.exit(1) - # else: - # workflows_json = json.loads(r.text) - # logging.debug(f"Successfully fetched workflows") - # print(workflows_json) - - # for workflow_entry in workflows_json["workflows"]: - # print("Workflow name: %s ; id: %s " % (workflow_entry["name"], workflow_entry["id"])) - - - ''' Workflow "docker image" uses image_build_push.yml From above commented out code, and checked via terminal as well, @@ -34,9 +17,8 @@ For MukuFlash03: id = 75506902 For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" - # TODO: Comment the above line and Uncomment the below line representing the original emission repo - # download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + + download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -45,25 +27,18 @@ else: workflow_runs_json = json.loads(r.text) logging.debug(f"Successfully fetched workflow runs") - # print(workflow_runs_json) workflow_runs = workflow_runs_json["workflow_runs"] if workflow_runs: - # TODO: Change the head_branch name which identifies only e-mission-server runs triggered by this branch successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "tags-combo-approach" + run["head_branch"] == "master" ] - # print(successful_runs) if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) # print(sorted_runs) latest_run_id = sorted_runs[0]["id"] - # print(latest_run_id) print(f"::set-output name=run_id::{latest_run_id}") - # else: - # print("No successful runs") - # else: - # print("No workflow runs found") + From 0d4a2a3d6aca3f2b1f0d40b7d8dbddee5b480467 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Wed, 8 May 2024 11:39:03 -0600 Subject: [PATCH 43/71] Cleanup image_build_push.yml Removing comments and changing branches so it's ready for merge. --- .github/workflows/image_build_push.yml | 36 +++----------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1f4e446..44d3485 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -1,16 +1,8 @@ -# This is a basic workflow to help you get started with Actions - name: docker-image-push-admin -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: - # Mukul: - # I've added a local test branch on my system and using it for testing image push. - # So, for testing purposes, need to checkout a branch "tags-combo-approach" - # TODO: Need to change to build off master or main once it looks good. - branches: [ tags-combo-approach ] + branches: [ master ] workflow_dispatch: inputs: @@ -18,12 +10,10 @@ on: description: "Latest Docker image tags passed from e-mission-server repository on image build and push" required: true -# Env variable env: DOCKER_USER: ${{secrets.DOCKER_USER}} DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: fetch_run_id: runs-on: ubuntu-latest @@ -75,9 +65,8 @@ jobs: with: name: docker-image-tag # TODO: Create a token with basic repo permissions - # TODO: Change user / organization to emission instead of MukuFlash03 github-token: ${{ secrets.GH_PAT_TAG }} - repository: MukuFlash03/e-mission-server + repository: e-mission/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag @@ -87,31 +76,16 @@ jobs: docker_image_tag=$(cat tag_file.txt) echo $docker_image_tag echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT - - # - name: Upload Artifact - # uses: actions/upload-artifact@v4 - # with: - # name: docker-image-tag - # path: tag_file.txt - # overwrite: true - - # - name: List artifact - # run: ls -R - - # This workflow contains a single job called "build" + build: needs: fetch_tag - - # The type of runner that the job will run on runs-on: ubuntu-latest env: DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} - - # Steps represent a sequence of tasks that will be executed as part of the job + steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - name: Print input docker image tag @@ -150,11 +124,9 @@ jobs: id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d--%M-%S')" - #Runs a single command using the runners shell - name: Run a one-line script run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} - # Runs a set of commands using the runners shell - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then From f91fcb1b4625126cc94243e6e91c6726edd6eaca Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:00:31 -0600 Subject: [PATCH 44/71] Removing docker build, updating DOCKER_IMAGE_TAG --> SERVER IMAGE TAG, removing redundant db setting --- .github/workflows/image_build_push.yml | 4 ++-- Dockerfile | 4 ++-- docker-compose-dev.yml | 2 +- docker-compose-prod-nginx.yml | 2 +- docker-compose-prod.yml | 2 +- docker/Dockerfile | 4 ++-- docker/start.sh | 6 +----- 7 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 44d3485..916433a 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -130,9 +130,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build else - docker build --build-arg DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build fi docker images diff --git a/Dockerfile b/Dockerfile index fe0736e..02dd79c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -ARG DOCKER_IMAGE_TAG +ARG SERVER_IMAGE_TAG -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 742c8a8..df44e28 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -6,7 +6,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker-compose-prod-nginx.yml b/docker-compose-prod-nginx.yml index 5626fce..55fbde6 100644 --- a/docker-compose-prod-nginx.yml +++ b/docker-compose-prod-nginx.yml @@ -7,7 +7,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} image: e-mission/opdash:0.0.1 environment: DASH_DEBUG_MODE: "True" diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 05a9579..eb284fd 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -6,7 +6,7 @@ services: context: . dockerfile: docker/Dockerfile args: - DOCKER_IMAGE_TAG: ${DOCKER_IMAGE_TAG} + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} image: e-mission/opdash:0.0.1 ports: - "8050:8050" diff --git a/docker/Dockerfile b/docker/Dockerfile index fe0736e..02dd79c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ -ARG DOCKER_IMAGE_TAG +ARG SERVER_IMAGE_TAG -FROM shankari/e-mission-server:master_${DOCKER_IMAGE_TAG} +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 diff --git a/docker/start.sh b/docker/start.sh index 15a31e1..591ea53 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -2,11 +2,7 @@ source setup/activate.sh # change the db host -echo "DB host = "${DB_HOST} -if [ -z ${DB_HOST} ] ; then - local_host=`hostname -i` - export DB_HOST=$local_host - echo "Setting db host environment variable to localhost" +echo ${DB_HOST} fi # run the app From 8cb70f57d1b4efd0080ff326f516d68262787424 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:06:08 -0600 Subject: [PATCH 45/71] Reverting for testing Need to test some changes to the workflows, so temporarily reverting these changes to work with mukul's branches. --- .github/fetch_runID.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index 34617f8..054cc2d 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -18,7 +18,7 @@ For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" + download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -33,7 +33,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "master" + run["head_branch"] == "consolidate-differences" ] if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) From 1bd8e33776380cda28a0f30b675b3260c9cb0ddb Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:21:45 -0600 Subject: [PATCH 46/71] Trying to trigger run without matrix Changing branches to try to trigger run and see if I can test docker compose --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 916433a..72b6faf 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -2,7 +2,7 @@ name: docker-image-push-admin on: push: - branches: [ master ] + branches: [ master, image-push ] workflow_dispatch: inputs: From 7c826018406c9a9c8b0844de725057c014333256 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:23:11 -0600 Subject: [PATCH 47/71] Changing repo name --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 72b6faf..9f6cf81 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -66,7 +66,7 @@ jobs: name: docker-image-tag # TODO: Create a token with basic repo permissions github-token: ${{ secrets.GH_PAT_TAG }} - repository: e-mission/e-mission-server + repository: MukuFlash03/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag From da904e55ceb14885b196ccc104a300f9bb3a5cce Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 26 May 2024 07:23:55 +0000 Subject: [PATCH 48/71] Updated docker image tag in .env file to the latest timestamp --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8a84419..c06e595 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-03--42-05 +DOCKER_IMAGE_TAG=2024-05-26--10-16 From b59fd018db4c82cd2cbd2878007985dc886a1bc1 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:26:22 -0600 Subject: [PATCH 49/71] So close to trying docker compose with actions Runner couldn't find the file (because I used the wrong filename) --- .github/workflows/image_build_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 9f6cf81..de68f18 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -130,9 +130,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose-dev.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose-dev.yml build fi docker images From 80a5e4337e795216f0a1367f884483d8ec5e47f3 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:28:54 -0600 Subject: [PATCH 50/71] Build context change --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 02dd79c..c5099bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ ARG SERVER_IMAGE_TAG -FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} - +# FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} +FROM MukuFlash03/e-mission-server:master_${SERVER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 1d1cadd5bfe5946d452ee93f5cf82380cd0a5d17 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:30:25 -0600 Subject: [PATCH 51/71] dockerhub repos are case sensitive. --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c5099bd..b7593a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ ARG SERVER_IMAGE_TAG # FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} -FROM MukuFlash03/e-mission-server:master_${SERVER_IMAGE_TAG} +FROM mukuflash03/e-mission-server:master_${SERVER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From b55174088eceaeb986884f7b2583b7cd5833e3e0 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:34:17 -0600 Subject: [PATCH 52/71] Updating tag --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b7593a4..3cfa6e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ ARG SERVER_IMAGE_TAG # FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} -FROM mukuflash03/e-mission-server:master_${SERVER_IMAGE_TAG} +FROM mukuflash03/e-mission-server:consolidate-differences_${SERVER_IMAGE_TAG} ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From f14b38a3914db909c4fc2914cf7dd13a594896a6 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:46:26 -0600 Subject: [PATCH 53/71] Rename docker tag Docker image created locally is not taking the name that I passed in for it. Hopefully, every new image created in an individual runner is named the same thing, so that this workaround is successful. --- .github/workflows/image_build_push.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index de68f18..791f45d 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -136,6 +136,10 @@ jobs: fi docker images + - name: rename docker image + run: | + docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + - name: push docker image run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} From f7f93193796b954f25168c59869dd4695ec31f2b Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:51:55 -0600 Subject: [PATCH 54/71] Add artifact Adding artifact upload for internal repo to be able to pull image tag. --- .github/workflows/image_build_push.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 791f45d..82ff4cf 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -143,3 +143,16 @@ jobs: - name: push docker image run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Create artifact text file + run: | + echo ${{ steps.date.outputs.date }} > admin_dash_tag_file.txt + echo "Created tag text file" + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: admin-dash-image-tag + path: admin_dash_tag_file.txt + overwrite: true + \ No newline at end of file From ada561382d7540fbe57e6aeb1f7761583493c40b Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 01:57:39 -0600 Subject: [PATCH 55/71] Trying to use admin dash tag Trying to do this the way I originally planned. --- .github/workflows/image_build_push.yml | 6 +++--- docker-compose-dev.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 82ff4cf..e25f74b 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -136,9 +136,9 @@ jobs: fi docker images - - name: rename docker image - run: | - docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + # - name: rename docker image + # run: | + # docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker image run: | diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index df44e28..950fc2a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,7 +7,7 @@ services: dockerfile: docker/Dockerfile args: SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} - image: e-mission/opdash:0.0.1 + image: ${ADMIN_DASH_IMAGE_TAG} ports: - "8050:8050" environment: From e061cd9269ced13ad1b55a22f4ac24cf7cfb108c Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 02:24:53 -0600 Subject: [PATCH 56/71] Reverting changes Reverting the changes I made to test docker compose --- .github/fetch_runID.py | 4 ++-- .github/workflows/image_build_push.yml | 14 +++++++------- docker-compose-dev.yml | 2 +- docker/Dockerfile | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py index 054cc2d..34617f8 100644 --- a/.github/fetch_runID.py +++ b/.github/fetch_runID.py @@ -18,7 +18,7 @@ For e-mission-server: id = 35580278 ''' - download_url = "https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows/75506902/runs" + download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) r = requests.get(download_url) if r.status_code != 200: @@ -33,7 +33,7 @@ successful_runs = [run for run in workflow_runs \ if run["status"] == "completed" and \ run["conclusion"] == "success" and \ - run["head_branch"] == "consolidate-differences" + run["head_branch"] == "master" ] if successful_runs: sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index e25f74b..6d713bb 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -2,7 +2,7 @@ name: docker-image-push-admin on: push: - branches: [ master, image-push ] + branches: [ master ] workflow_dispatch: inputs: @@ -66,7 +66,7 @@ jobs: name: docker-image-tag # TODO: Create a token with basic repo permissions github-token: ${{ secrets.GH_PAT_TAG }} - repository: MukuFlash03/e-mission-server + repository: e-mission/e-mission-server run-id: ${{ env.RUN_ID }} - name: Print artifact tag @@ -130,15 +130,15 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-dev.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 ADMIN_DASH_IMAGE_TAG=$DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-dev.yml build fi docker images - # - name: rename docker image - # run: | - # docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + - name: rename docker image + run: | + docker image tag e-mission/opdash:0.0.1 $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: push docker image run: | diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 950fc2a..df44e28 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,7 +7,7 @@ services: dockerfile: docker/Dockerfile args: SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} - image: ${ADMIN_DASH_IMAGE_TAG} + image: e-mission/opdash:0.0.1 ports: - "8050:8050" environment: diff --git a/docker/Dockerfile b/docker/Dockerfile index 3cfa6e4..02dd79c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ ARG SERVER_IMAGE_TAG -# FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} -FROM mukuflash03/e-mission-server:consolidate-differences_${SERVER_IMAGE_TAG} +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} + ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From c3849adb4fa4da01c84032aa5e85defe11ef797a Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 02:31:30 -0600 Subject: [PATCH 57/71] Remove extra dockerfile --- Dockerfile | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 02dd79c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -ARG SERVER_IMAGE_TAG - -FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} - -ENV DASH_DEBUG_MODE True -ENV SERVER_PORT 8050 - -# copy over setup files -WORKDIR /usr/src/app/dashboard_setup -COPY requirements.txt nrel_dash_components-0.0.1.tar.gz docker/setup.sh ./ - -# install requirements.txt -WORKDIR /usr/src/app -RUN bash -c "./dashboard_setup/setup.sh" - -# copy over dashboard code -WORKDIR /usr/src/app/pages -COPY ./pages ./ -WORKDIR /usr/src/app/utils -COPY ./utils ./ -WORKDIR /usr/src/app -COPY app.py app_sidebar_collapsible.py globals.py globalsUpdater.py Procfile ./ - -WORKDIR /usr/src/app/assets -COPY assets/ ./ -RUN mkdir qrcodes - -# copy over test data -WORKDIR /usr/src/app/data -COPY data ./ - -# open listening port, this may be overridden in docker-compose file -EXPOSE ${SERVER_PORT} - -# run the dashboard -WORKDIR /usr/src/app/dashboard_setup -COPY docker/start.sh ./ -WORKDIR /usr/src/app -CMD ["/bin/bash", "/usr/src/app/dashboard_setup/start.sh"] From 59fd7dc45ed436c2c7ecd39787cd9753c010f928 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:00:59 -0600 Subject: [PATCH 58/71] server image tag update --- .github/workflows/image_build_push.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 6d713bb..20c0fcc 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -98,10 +98,10 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" - echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env else echo "Push event: Restoring latest server image tag in .env" - echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env fi - name: Add, Commit, Push changes to .env file @@ -155,4 +155,4 @@ jobs: name: admin-dash-image-tag path: admin_dash_tag_file.txt overwrite: true - \ No newline at end of file + From ddf6dc35a93142c8c16a1a34d4a3e3b51fbf65b2 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:08:10 -0600 Subject: [PATCH 59/71] Update .env Adding other env vars as a template --- .env | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env b/.env index c06e595..0bf51c2 100644 --- a/.env +++ b/.env @@ -1 +1,8 @@ DOCKER_IMAGE_TAG=2024-05-26--10-16 +COGNITO_CLIENT_ID='' +COGNITO_CLIENT_SECRET='' +COGNITO_REDIRECT_URL='' +COGNITO_TOKEN_ENDPOINT='' +COGNITO_USER_POOL_ID='' +COGNITO_REGION='' +COGNITO_AUTH_URL='' From a93760165caae8c36857b5d6fc4d3dc70a5ce8cb Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Sun, 26 May 2024 03:13:21 -0600 Subject: [PATCH 60/71] Update image_build_push.yml Adding other env vars to .env. Otherwise, the file will get overwritten with the server tag only every time. --- .github/workflows/image_build_push.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 20c0fcc..8c037e7 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -99,9 +99,23 @@ jobs: if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + echo "COGNITO_CLIENT_ID=''" > .env + echo "COGNITO_CLIENT_SECRET=''" > .env + echo "COGNITO_REDIRECT_URL=''" > .env + echo "COGNITO_TOKEN_ENDPOINT=''" > .env + echo "COGNITO_USER_POOL_ID=''" > .env + echo "COGNITO_REGION=''" > .env + echo "COGNITO_AUTH_URL=''" > .env else echo "Push event: Restoring latest server image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + echo "COGNITO_CLIENT_ID=''" > .env + echo "COGNITO_CLIENT_SECRET=''" > .env + echo "COGNITO_REDIRECT_URL=''" > .env + echo "COGNITO_TOKEN_ENDPOINT=''" > .env + echo "COGNITO_USER_POOL_ID=''" > .env + echo "COGNITO_REGION=''" > .env + echo "COGNITO_AUTH_URL=''" > .env fi - name: Add, Commit, Push changes to .env file From e26216a515a7798b9d787eab70cd91089ed00cec Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 28 May 2024 02:40:30 -0600 Subject: [PATCH 61/71] Update start.sh Remove extraneous fi --- docker/start.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/start.sh b/docker/start.sh index 591ea53..8500a69 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -3,7 +3,6 @@ source setup/activate.sh # change the db host echo ${DB_HOST} -fi # run the app # python app.py From 24689c5aeb5017df2b84a21d893b2715d73c7e43 Mon Sep 17 00:00:00 2001 From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com> Date: Tue, 28 May 2024 03:01:07 -0600 Subject: [PATCH 62/71] Switching to build prod compose Because that makes sense. --- .github/workflows/image_build_push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 8c037e7..1bc2813 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -144,9 +144,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-prod.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-dev.yml build + SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-prod.yml build fi docker images From 27541a7c9386dc2ddd9c57ba11e7ed97fb29ac50 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:32:16 -0700 Subject: [PATCH 63/71] Updated username to clarify that env file is being updated --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1bc2813..a61ab9d 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -121,7 +121,7 @@ jobs: - name: Add, Commit, Push changes to .env file run: | git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.name "Github Actions bot to update .env with latest tags" if git diff --quiet; then echo "Latest timestamp already present in .env file, no changes to commit" else From 85b2384620bba5939f6074cdd4d377bf55dce423 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:38:02 -0700 Subject: [PATCH 64/71] Modified tag variable names to be more relevant These store tags differently depending on the trigger event - Push OR Workflow dispatch --- .github/workflows/image_build_push.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index a61ab9d..83c7290 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -82,8 +82,8 @@ jobs: runs-on: ubuntu-latest env: - DOCKER_IMAGE_TAG_1: ${{ needs.fetch_tag.outputs.docker_image_tag }} - DOCKER_IMAGE_TAG_2: ${{ github.event.inputs.docker_image_tag }} + DOCKER_TAG_FROM_PUSH: ${{ needs.fetch_tag.outputs.docker_image_tag }} + DOCKER_TAG_FROM_WORKFLOW_DISPATCH: ${{ github.event.inputs.docker_image_tag }} steps: - uses: actions/checkout@v4 @@ -91,14 +91,14 @@ jobs: - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" - echo "Latest docker image tag (push): ${{ env.DOCKER_IMAGE_TAG_1 }}" - echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_IMAGE_TAG_2 }}" + echo "Latest docker image tag (push): ${{ env.DOCKER_TAG_FROM_PUSH }}" + echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_TAG_FROM_WORKFLOW_DISPATCH }}" - name: Update .env file run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env echo "COGNITO_CLIENT_ID=''" > .env echo "COGNITO_CLIENT_SECRET=''" > .env echo "COGNITO_REDIRECT_URL=''" > .env @@ -108,7 +108,7 @@ jobs: echo "COGNITO_AUTH_URL=''" > .env else echo "Push event: Restoring latest server image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1" > .env + echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH" > .env echo "COGNITO_CLIENT_ID=''" > .env echo "COGNITO_CLIENT_SECRET=''" > .env echo "COGNITO_REDIRECT_URL=''" > .env @@ -144,9 +144,9 @@ jobs: - name: build docker image run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_2 docker compose -f docker-compose-prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose-prod.yml build else - SERVER_IMAGE_TAG=$DOCKER_IMAGE_TAG_1 docker compose -f docker-compose-prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose-prod.yml build fi docker images From 1e6ef8f597e2b97b045313436f1ed2dfd21c8587 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:39:13 -0700 Subject: [PATCH 65/71] Removing redundant pip install --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 83c7290..06a6835 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -31,7 +31,6 @@ jobs: - name: Install Python dependencies run: | - python -m pip install --upgrade pip pip install requests - name: Run Python script From 599d991470665c84c98d44fa49c8ea1ebd839f55 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 20:47:04 -0700 Subject: [PATCH 66/71] Certificates added in Dockerfile Similarly done with public-dashboard. See comment: https://github.com/e-mission/em-public-dashboard/pull/125#issuecomment-2101651604 --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 02dd79c..8224c25 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,8 @@ ARG SERVER_IMAGE_TAG FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} +ADD https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem /etc/ssl/certs/ + ENV DASH_DEBUG_MODE True ENV SERVER_PORT 8050 From 67a88b3e26f41a5faf8b73a2e3207af97fc1571f Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 21:49:57 -0700 Subject: [PATCH 67/71] Fixed indentation --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 06a6835..fcba524 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -145,7 +145,7 @@ jobs: if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH docker compose -f docker-compose-prod.yml build else - SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose-prod.yml build + SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH docker compose -f docker-compose-prod.yml build fi docker images From 3f66ef344a70aebf11913b6202a007b051f226c1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 22:30:32 -0700 Subject: [PATCH 68/71] Added .env.cognito.template + Removed cognito variables from .env and workflow file The .env file is meant to be used only for the docker image tag to store the latest server image tag. If other variables are stored there, the workflow will overwrite the .env file to store only the image tag whenever the CI/CD runs. Hence, separating out the Cognito variables in a separate template file. I am expecting the users would either set them directly in the system or use the docker compose prod yml file. --- .env | 9 +-------- .env.cognito.template | 7 +++++++ .github/workflows/image_build_push.yml | 14 -------------- 3 files changed, 8 insertions(+), 22 deletions(-) create mode 100644 .env.cognito.template diff --git a/.env b/.env index 0bf51c2..b1bfb9f 100644 --- a/.env +++ b/.env @@ -1,8 +1 @@ -DOCKER_IMAGE_TAG=2024-05-26--10-16 -COGNITO_CLIENT_ID='' -COGNITO_CLIENT_SECRET='' -COGNITO_REDIRECT_URL='' -COGNITO_TOKEN_ENDPOINT='' -COGNITO_USER_POOL_ID='' -COGNITO_REGION='' -COGNITO_AUTH_URL='' +DOCKER_IMAGE_TAG=2024-05-26--10-16 \ No newline at end of file diff --git a/.env.cognito.template b/.env.cognito.template new file mode 100644 index 0000000..437bd68 --- /dev/null +++ b/.env.cognito.template @@ -0,0 +1,7 @@ +COGNITO_CLIENT_ID='' +COGNITO_CLIENT_SECRET='' +COGNITO_REDIRECT_URL='' +COGNITO_TOKEN_ENDPOINT='' +COGNITO_USER_POOL_ID='' +COGNITO_REGION='' +COGNITO_AUTH_URL='' diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index fcba524..dcb4281 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -98,23 +98,9 @@ jobs: if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env - echo "COGNITO_CLIENT_ID=''" > .env - echo "COGNITO_CLIENT_SECRET=''" > .env - echo "COGNITO_REDIRECT_URL=''" > .env - echo "COGNITO_TOKEN_ENDPOINT=''" > .env - echo "COGNITO_USER_POOL_ID=''" > .env - echo "COGNITO_REGION=''" > .env - echo "COGNITO_AUTH_URL=''" > .env else echo "Push event: Restoring latest server image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH" > .env - echo "COGNITO_CLIENT_ID=''" > .env - echo "COGNITO_CLIENT_SECRET=''" > .env - echo "COGNITO_REDIRECT_URL=''" > .env - echo "COGNITO_TOKEN_ENDPOINT=''" > .env - echo "COGNITO_USER_POOL_ID=''" > .env - echo "COGNITO_REGION=''" > .env - echo "COGNITO_AUTH_URL=''" > .env fi - name: Add, Commit, Push changes to .env file From 5098e071c6cf6a601d1c29b00fead026cd170fa1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 22:46:29 -0700 Subject: [PATCH 69/71] Removing artifact method This was mainly needed for Push event but since Workflow dispatch event would be setting the latest server image tag in .env file, the push event can read from this file directly. --- .github/fetch_runID.py | 44 ---------------- .github/workflows/image_build_push.yml | 71 +++----------------------- 2 files changed, 6 insertions(+), 109 deletions(-) delete mode 100644 .github/fetch_runID.py diff --git a/.github/fetch_runID.py b/.github/fetch_runID.py deleted file mode 100644 index 34617f8..0000000 --- a/.github/fetch_runID.py +++ /dev/null @@ -1,44 +0,0 @@ - -import json -import logging -import requests -import sys - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - - ''' - Workflow "docker image" uses image_build_push.yml - From above commented out code, and checked via terminal as well, - workflow id for the "docker image" can be fetched using: - https://api.github.com/repos/MukuFlash03/e-mission-server/actions/workflows - https://api.github.com/repos/e-mission/e-mission-server/actions/workflows - - For MukuFlash03: id = 75506902 - For e-mission-server: id = 35580278 - ''' - - download_url = "https://api.github.com/repos/e-mission/e-mission-server/actions/workflows/35580278/runs" - logging.debug("About to fetch workflow runs present in docker image workflow present in e-mission-server from %s" % download_url) - r = requests.get(download_url) - if r.status_code != 200: - logging.debug(f"Unable to fetch workflow runs, status code: {r.status_code}") - sys.exit(1) - else: - workflow_runs_json = json.loads(r.text) - logging.debug(f"Successfully fetched workflow runs") - - workflow_runs = workflow_runs_json["workflow_runs"] - if workflow_runs: - successful_runs = [run for run in workflow_runs \ - if run["status"] == "completed" and \ - run["conclusion"] == "success" and \ - run["head_branch"] == "master" - ] - if successful_runs: - sorted_runs = successful_runs.sort(reverse=True, key=lambda x: x["updated_at"]) - sorted_runs = sorted(successful_runs, reverse=True, key=lambda x: x["updated_at"]) - # print(sorted_runs) - latest_run_id = sorted_runs[0]["id"] - print(f"::set-output name=run_id::{latest_run_id}") - diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index dcb4281..bf26738 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -15,78 +15,20 @@ env: DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} jobs: - fetch_run_id: - runs-on: ubuntu-latest - - outputs: - run_id: ${{ steps.get_run_id.outputs.run_id }} - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install Python dependencies - run: | - pip install requests - - - name: Run Python script - id: run_script - run: | - echo "Fetching latest successful run ID from e-misison-server docker image workflow" - python .github/fetch_runID.py - - - name: Get Run ID - id: get_run_id - run: echo "run_id=${{ steps.run_script.outputs.run_id }}" >> "$GITHUB_OUTPUT" - - fetch_tag: - needs: fetch_run_id - runs-on: ubuntu-latest - - env: - RUN_ID: ${{ needs.fetch_run_id.outputs.run_id }} - - outputs: - docker_image_tag: ${{ steps.get_docker_tag.outputs.docker_image_tag }} - - steps: - - uses: actions/checkout@v4 - - - name: Use Run ID from previous fetch_run_id job - run: echo Run ID from previous job ${{ env.RUN_ID }} - - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: docker-image-tag - # TODO: Create a token with basic repo permissions - github-token: ${{ secrets.GH_PAT_TAG }} - repository: e-mission/e-mission-server - run-id: ${{ env.RUN_ID }} - - - name: Print artifact tag - id: get_docker_tag - run: | - cat tag_file.txt - docker_image_tag=$(cat tag_file.txt) - echo $docker_image_tag - echo "docker_image_tag=$(echo $docker_image_tag)" >> $GITHUB_OUTPUT - build: - needs: fetch_tag runs-on: ubuntu-latest env: - DOCKER_TAG_FROM_PUSH: ${{ needs.fetch_tag.outputs.docker_image_tag }} DOCKER_TAG_FROM_WORKFLOW_DISPATCH: ${{ github.event.inputs.docker_image_tag }} steps: - uses: actions/checkout@v4 + - name: Set docker image tag from .env file + run: | + set -a; source .env; set +a + echo "DOCKER_TAG_FROM_PUSH=${SERVER_IMAGE_TAG}" >> $GITHUB_ENV + - name: Print input docker image tag run: | echo "Event name: ${{ github.event_name }}" @@ -99,8 +41,7 @@ jobs: echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env else - echo "Push event: Restoring latest server image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_PUSH" > .env + echo "Push event: Restoring latest server image tag from .env" fi - name: Add, Commit, Push changes to .env file From 5eff0be92d455f65904f2813b0156a6edb080de4 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Tue, 13 Aug 2024 22:58:00 -0700 Subject: [PATCH 70/71] Updating latest server image tag This is probably the first and only time we'll need to manually change the image tag. Hence forth, the CI/CD should automatically update the .env file with the latest tag whenever the workflow dispatch is triggered on new merges to server. --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b1bfb9f..9305039 100644 --- a/.env +++ b/.env @@ -1 +1 @@ -DOCKER_IMAGE_TAG=2024-05-26--10-16 \ No newline at end of file +SERVER_IMAGE_TAG=2024-08-12--15-15 \ No newline at end of file From 48f6ca92ef033a7772143538a941b960d5070908 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 14 Aug 2024 17:14:21 -0700 Subject: [PATCH 71/71] Moving .env file update and git commit to the end This will ensure that if the CI/CD pipeline fails in any prior steps such as the docker related ones, the .env file isn't updated. This is because the the docker failures can include errors image not found which can occur due to incorrect tags. --- .github/workflows/image_build_push.yml | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index bf26738..2b780f8 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -35,27 +35,6 @@ jobs: echo "Latest docker image tag (push): ${{ env.DOCKER_TAG_FROM_PUSH }}" echo "Latest docker image tag (workflow_dispatch): ${{ env.DOCKER_TAG_FROM_WORKFLOW_DISPATCH }}" - - name: Update .env file - run: | - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" - echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env - else - echo "Push event: Restoring latest server image tag from .env" - fi - - - name: Add, Commit, Push changes to .env file - run: | - git config --local user.email "action@github.com" - git config --local user.name "Github Actions bot to update .env with latest tags" - if git diff --quiet; then - echo "Latest timestamp already present in .env file, no changes to commit" - else - git add .env - git commit -m "Updated docker image tag in .env file to the latest timestamp" - git push origin - fi - - name: docker login run: | # log into docker hub account docker login -u $DOCKER_USER -p $DOCKER_PASSWORD @@ -84,6 +63,27 @@ jobs: run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + - name: Update .env file + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "Workflow_dispatch: New server image built and pushed, Updating image tag in .env" + echo "SERVER_IMAGE_TAG=$DOCKER_TAG_FROM_WORKFLOW_DISPATCH" > .env + else + echo "Push event: Restoring latest server image tag from .env" + fi + + - name: Add, Commit, Push changes to .env file + run: | + git config --local user.email "action@github.com" + git config --local user.name "Github Actions bot to update .env with latest tags" + if git diff --quiet; then + echo "Latest timestamp already present in .env file, no changes to commit" + else + git add .env + git commit -m "Updated docker image tag in .env file to the latest timestamp" + git push origin + fi + - name: Create artifact text file run: | echo ${{ steps.date.outputs.date }} > admin_dash_tag_file.txt