Merge pull request #43 from horahoradev/dev

Dev -> Master

activemq/Dockerfile

@@ -1,4 +1,6 @@
 FROM symptoma/activemq:latest
 
 
+COPY activemq.xml /opt/activemq/conf/activemq.xml
+
 CMD ["/bin/sh", "-c", "bin/activemq console -Djetty.host=0.0.0.0"]

activemq/activemq.xml

@@ -0,0 +1,133 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<!-- START SNIPPET: example -->
+<beans
+  xmlns="http://www.springframework.org/schema/beans"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+  http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
+
+    <!-- Allows us to use system properties as variables in this configuration file -->
+    <bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+        <property name="locations">
+            <value>file:${activemq.conf}/credentials.properties</value>
+        </property>
+    </bean>
+
+    <!--
+        The <broker> element is used to configure the ActiveMQ broker.
+    -->
+    <broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" schedulePeriodForDestinationPurge="10000" dataDirectory="${activemq.data}">
+
+        <destinationPolicy>
+            <policyMap>
+              <policyEntries>
+                <policyEntry queue=">" gcInactiveDestinations="true" inactiveTimoutBeforeGC="30000"/>
+                <policyEntry topic=">" gcInactiveDestinations="true" >
+                    <!-- The constantPendingMessageLimitStrategy is used to prevent
+                         slow topic consumers to block producers and affect other consumers
+                         by limiting the number of messages that are retained
+                         For more information, see:
+
+                         http://activemq.apache.org/slow-consumer-handling.html
+
+                    -->
+                  <pendingMessageLimitStrategy>
+                    <constantPendingMessageLimitStrategy limit="1000"/>
+                  </pendingMessageLimitStrategy>
+                </policyEntry>
+              </policyEntries>
+            </policyMap>
+        </destinationPolicy>
+
+
+        <!--
+            The managementContext is used to configure how ActiveMQ is exposed in
+            JMX. By default, ActiveMQ uses the MBean server that is started by
+            the JVM. For more information, see:
+
+            http://activemq.apache.org/jmx.html
+        -->
+        <managementContext>
+            <managementContext createConnector="false"/>
+        </managementContext>
+
+        <!--
+            Configure message persistence for the broker. The default persistence
+            mechanism is the KahaDB store (identified by the kahaDB tag).
+            For more information, see:
+
+            http://activemq.apache.org/persistence.html
+        -->
+        <persistenceAdapter>
+            <kahaDB directory="${activemq.data}/kahadb"/>
+        </persistenceAdapter>
+
+
+
+
+          <!--
+            The systemUsage controls the maximum amount of space the broker will
+            use before disabling caching and/or slowing down producers. For more information, see:
+            http://activemq.apache.org/producer-flow-control.html
+          -->
+          <systemUsage>
+            <systemUsage>
+                <memoryUsage>
+                    <memoryUsage percentOfJvmHeap="70" />
+                </memoryUsage>
+                <storeUsage>
+                    <storeUsage limit="100 gb"/>
+                </storeUsage>
+                <tempUsage>
+                    <tempUsage limit="50 gb"/>
+                </tempUsage>
+            </systemUsage>
+        </systemUsage>
+
+        <!--
+            The transport connectors expose ActiveMQ over a given protocol to
+            clients and other brokers. For more information, see:
+
+            http://activemq.apache.org/configuring-transports.html
+        -->
+        <transportConnectors>
+            <!-- DOS protection, limit concurrent connections to 1000 and frame size to 100MB -->
+            <transportConnector name="openwire" uri="tcp://0.0.0.0:61616?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+            <transportConnector name="amqp" uri="amqp://0.0.0.0:5672?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+            <transportConnector name="stomp" uri="stomp://0.0.0.0:61613?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+            <transportConnector name="mqtt" uri="mqtt://0.0.0.0:1883?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+            <transportConnector name="ws" uri="ws://0.0.0.0:61614?maximumConnections=1000&amp;wireFormat.maxFrameSize=104857600"/>
+        </transportConnectors>
+
+        <!-- destroy the spring context on shutdown to stop jetty -->
+        <shutdownHooks>
+            <bean xmlns="http://www.springframework.org/schema/beans" class="org.apache.activemq.hooks.SpringContextHook" />
+        </shutdownHooks>
+
+    </broker>
+
+    <!--
+        Enable web consoles, REST and Ajax APIs and demos
+        The web consoles requires by default login, you can disable this in the jetty.xml file
+
+        Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
+    -->
+    <import resource="jetty.xml"/>
+
+</beans>
+<!-- END SNIPPET: example -->

scheduler/Dockerfile

@@ -12,7 +12,7 @@ RUN cd ffmpeg-git-*-amd64-static && cp ffmpeg /usr/local/bin/ffmpeg
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-FROM golang:1.16-alpine as builder
+FROM golang:1.17-alpine as builder
 LABEL org.opencontainers.image.source=https://github.com/horahoradev/horahora
 
 WORKDIR /horahora/scheduler
@@ -35,7 +35,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build go build -o /scheduler.bin
 FROM python:3.9-alpine AS runtime
 LABEL org.opencontainers.image.source=https://github.com/horahoradev/horahora
 
-RUN apk add --update --no-cache zlib-dev musl-dev libc-dev libffi-dev gcc g++ git pwgen && git clone -b STOMP_progress --depth 5 https://github.com/horahoradev/yt-dlp.git yt-dlp
+RUN apk add --update --no-cache zlib-dev musl-dev libc-dev libffi-dev gcc g++ git pwgen && git clone -b STOMP_progress --depth 2 https://github.com/horahoradev/yt-dlp.git yt-dlp
 
 # download yt-dlp and prepare it for usage
 WORKDIR /yt-dlp

scheduler/internal/models/video_dl_request.go

@@ -103,7 +103,7 @@ func (v *VideoDLRequest) PublishVideoInprogress(dlStatus int, action string) err
 		return err
 	}
 
-	err = v.Rabbitmq.Send("/topic/state", "text/json", payload, stomp.SendOpt.Receipt)
+	err = v.Rabbitmq.Send("/topic/state", "text/json", payload, stomp.SendOpt.Receipt, stomp.SendOpt.Header("persistent", "false"), stomp.SendOpt.Header("expires", fmt.Sprintf("%d", time.Now().Local().UnixMilli()+30000)))
 	if err != nil {
 		return fmt.Errorf("Publish: %v", err)
 	}

webapp/src/ArchivalPage.js

@@ -115,7 +115,6 @@ function ArchivalPage() {
             conn != null && conn.subscribe(`/topic/state`, function(message) {
                 mutex.lock();
                 let body = JSON.parse(message.body);
-                message.ack();
                 if (body.Type == "deletion") {
                     console.log(`Got delete ${body.Video.VideoID}`);
                     setVideoInProgressDataset(videosInProg => {
@@ -129,8 +128,7 @@ function ArchivalPage() {
                     console.log(`Got insert ${body.Video.VideoID}`);
 
                     setVideoInProgressDataset(videosInProg => {
-                        if (videosInProg == null) { return videosInProg; }
-                        let dataset = JSON.parse(JSON.stringify(videosInProg))
+                        let dataset = videosInProg != null ? JSON.parse(JSON.stringify(videosInProg)) : [];
 
                         // // Does it already exist? If not, subscribe
                         let videosID = dataset.filter((item)=>item.VideoID == body.Video.VideoID);
@@ -139,7 +137,7 @@ function ArchivalPage() {
                         }
 
                         // Needed for upsert, filter it out if it's in there with a different dlStatus
-                        dataset = dataset.filter((item)=>item.VideoID != body.Video.VideoID || body.Video.DlStatus != item.DlStatus);
+                        dataset = dataset.filter((item)=>item.VideoID != body.Video.VideoID || body.Video.DlStatus == item.DlStatus);
                         body.Video.progress = 0;
                         // If it's downloading, it goes at the beginning
                         if (body.Video.DlStatus == "Downloading") {
@@ -152,9 +150,10 @@ function ArchivalPage() {
                         return dataset;
                     });
                 }
+                message.ack();
+
                 mutex.unlock();
-            }, {'prefetch-count': 100, 'ack': 'client-individual', 'id': String(Math.random() * 1000)});
-            return conn;
+            }, {'prefetch-count': 1, 'ack': 'client-individual', 'id': String(Math.random() * 1000)});
 
       }, [conn]);
 

webapp/src/VideoPage.js

@@ -110,7 +110,7 @@ function VideoAdminControls(props) {
 }
 
 function VideoView(props) {
-  let { data, id, setRating, next_video, videoComments, refreshComments} = props;
+  let { rating, data, id, setRating, next_video, videoComments, refreshComments} = props;
 
   // video_id, content (content of comment), and parent (parent comment id if a reply)
   let formik = useFormik({
@@ -131,12 +131,12 @@ function VideoView(props) {
     await refreshComments();
   };
 
-  function rate(rating) {
+  async function rate(rating) {
     if (id == 0) {
       // TODO: throw
       return;
     }
-    API.postRating(id, rating);
+    await API.postRating(id, rating);
     setRating(rating);
   }
 
@@ -150,7 +150,7 @@ function VideoView(props) {
           <span className="float-right">
             <span className="text-black">{data.Views} Views</span>
           </span>
-          <div className="inline-block relative top-5 float-right left-16 mr-2"><Rate allowHalf={true} value={data.Rating} onChange={rate}></Rate></div>
+          <div className="inline-block relative top-5 float-right left-16 mr-2"><Rate allowHalf={true} value={rating} onChange={rate}></Rate></div>
           <br />
           <span className="text-gray-600 text-xs">{data.UploadDate}</span>
         </div>
@@ -269,8 +269,8 @@ function VideoPage() {
 
     let fetchData = async () => {
       let data = await API.getVideo(id);
+      if (data) setRating(data.Rating);
       if (!ignore) setPageData(data);
-      if (!ignore) setRating(data.Rating);
 
       let userData = await API.getUserdata();
       if (!ignore) setUserData(userData);
@@ -283,7 +283,7 @@ function VideoPage() {
     return () => {
       ignore = true;
     };
-  }, [id, rating]);
+  }, [id]);
 
   if (pageData == null) return null;
 
@@ -292,7 +292,7 @@ function VideoPage() {
       <Header userData={userData} />
       <div className="flex justify-center mx-4">
         <div className=" w-screen my-6 z-0 min-w-400">
-          <VideoView data={pageData} videoComments={comments} id={id} refreshComments={refreshComments} setRating={setRating} next_video={navigate_to_next_video}/>
+          <VideoView data={pageData} videoComments={comments} id={id} refreshComments={refreshComments} setRating={setRating} rating={rating} next_video={navigate_to_next_video}/>
         </div>
         <div className="ml-4 mt-2 w-100 align-top float-right">
           <VideoList videos={pageData.RecommendedVideos} title="Recommendations" inline={true}/>