大幅优化yolo识别效率，修复部分内存泄露问题

TonyJiangWJ · Sep 14, 2024 · a4f70e5 · a4f70e5
1 parent 1ca4bc9
commit a4f70e5
Show file tree

Hide file tree

Showing 15 changed files with 209 additions and 75 deletions.
diff --git a/app/build.gradle b/app/build.gradle
@@ -64,7 +64,7 @@ android {
             applicationIdSuffix ".debug"
         }
         release {
-//            debuggable true
+            debuggable false
             shrinkResources false
             minifyEnabled false
             proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'

diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro
@@ -28,11 +28,14 @@
 -dontwarn java.lang.invoke.*
 -dontwarn **$$Lambda$*
 
+-keepnames class com.stardust.autojs.onnx.YoloV8Predictor
+
 -keep class org.mozilla.javascript.** { *; }
 -keep class com.jecelyin.editor.** { *; }
 -keep class com.stardust.automator.** { *; }
 -keep class com.stardust.autojs.** { *; }
 -keep class org.greenrobot.eventbus.** { *; }
+-keep class org.autojs.autojs.** {*;}
 -keep class * extends c
 -keepattributes *Annotation*
 # Event bus
@@ -138,4 +141,16 @@
 # Bugly
 
 -dontwarn com.tencent.bugly.**
--keep public class com.tencent.bugly.**{*;}
+-keep public class com.tencent.bugly.**{*;}
+
+# OnnxRuntime
+-dontwarn ai.onnxruntime.**
+-keep public class ai.onnxruntime.**{*;}
+
+# okHttp3
+-dontwarn okhttp3.**
+-keep public class okhttp3.**{*;}
+
+# paddleocr
+-dontwarn com.baidu.paddle.lite.ocr.**
+-keep public class com.baidu.paddle.lite.ocr.**{*;}
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
@@ -45,7 +45,9 @@
         android:requestLegacyExternalStorage="true"
         tools:replace="android:label, android:icon, android:allowBackup"
         tools:targetApi="m">
-
+        <!-- 开启内存分析 -->
+        <profileable android:shell="true" android:enabled="true"
+            tools:targetApi="q" />
         <meta-data
             android:name="android.max_aspect"
             android:value="2.1"/>

diff --git a/autojs-aar/paddleocr/src/main/cpp/ocr_db_post_process.cpp b/autojs-aar/paddleocr/src/main/cpp/ocr_db_post_process.cpp
@@ -64,6 +64,13 @@ static cv::RotatedRect unclip(float **box) {
 
   return res;
 }
+// 用于动态释放内存 避免内存泄露
+static void free_array(float **array, int rows) {
+  for (int i = 0; i < rows; ++i) {
+    delete[] array[i];
+  }
+  delete[] array;
+}
 
 static float **Mat2Vec(cv::Mat mat) {
   auto **array = new float *[mat.rows];
@@ -267,13 +274,15 @@ boxes_from_bitmap(const cv::Mat &pred, const cv::Mat &bitmap) {
     // end get_mini_box
 
     if (ssid < min_size) {
+      free_array(array, 4);
       continue;
     }
 
     float score;
     score = box_score_fast(array, pred);
     // end box_score_fast
     if (score < box_thresh) {
+      free_array(array, 4);
       continue;
     }
 
@@ -284,8 +293,11 @@ boxes_from_bitmap(const cv::Mat &pred, const cv::Mat &bitmap) {
     cv::RotatedRect clipbox = points;
     auto cliparray = get_mini_boxes(clipbox, ssid);
 
-    if (ssid < min_size + 2)
+    if (ssid < min_size + 2) {
+      free_array(array, 4);
+      free_array(cliparray, 4);
       continue;
+    }
 
     int dest_width = pred.cols;
     int dest_height = pred.rows;
@@ -301,6 +313,8 @@ boxes_from_bitmap(const cv::Mat &pred, const cv::Mat &bitmap) {
       intcliparray.emplace_back(std::move(a));
     }
     boxes.emplace_back(std::move(intcliparray));
+    free_array(array, 4);
+    free_array(cliparray, 4);
 
   } // end for
   return boxes;

diff --git a/autojs/build.gradle b/autojs/build.gradle
@@ -15,7 +15,7 @@ android {
     }
     buildTypes {
         release {
-            minifyEnabled false
+            minifyEnabled true
             proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
         }
     }

diff --git a/autojs/proguard-rules.pro b/autojs/proguard-rules.pro
@@ -23,3 +23,5 @@
 # If you keep the line number information, uncomment this to
 # hide the original source file name.
 #-renamesourcefileattribute SourceFile
+#-keep public class com.stardust.autojs.onnx.YoloV8Predictor
+#-keepnames class com.stardust.autojs.onnx.YoloV8Predictor
diff --git a/autojs/src/main/assets/modules/__$ocr__.js b/autojs/src/main/assets/modules/__$ocr__.js
@@ -31,11 +31,13 @@ module.exports = function(runtime, global) {
     let region = options.region
     if (region) {
       let r = buildRegion(region, img)
-      let o = img
       img = images.clip(img, r.x, r.y, r.width, r.height)
-      o.recycle()
     }
     let text = javaOcr.recognizeText(img, options.cpuThreadNum || 4, options.useSlim || true)
+    if (region) {
+      // 进行过区域截取，需要回收截取的图片 原始图片由外部管理
+      img.recycle()
+    }
     if (text) {
       return JSON.parse(JSON.stringify(text))
     }
@@ -47,14 +49,16 @@ module.exports = function(runtime, global) {
     let region = options.region
     if (region) {
       let r = buildRegion(region, img)
-      let o = img
       img = images.clip(img, r.x, r.y, r.width, r.height)
-      o.recycle()
     }
     let resultList = runtime.bridges.bridges.toArray(javaOcr.detect(img, options.cpuThreadNum || 4, options.useSlim || true))
     if (region && region.length > 1 && resultList && resultList.length > 0) {
       resultList.forEach(r => r.bounds.offset(region[0], region[1]))
     }
+    if (region) {
+      // 进行过区域截取，需要回收截取的图片 原始图片由外部管理
+      img.recycle()
+    }
     return resultList
   }
 

diff --git a/autojs/src/main/assets/modules/__$yolo__.js b/autojs/src/main/assets/modules/__$yolo__.js
@@ -1,5 +1,6 @@
 module.exports = function (runtime, global) {
   let yoloCreator = runtime.yolo
+  let instanceList = []
   let $yolo = function () {
     this.enabled = false
   }
@@ -29,6 +30,7 @@ module.exports = function (runtime, global) {
       }
       this.yoloInstance = yoloCreator.createNcnn(options.paramPath, options.binPath, convertToList(options.labels), options.imageSize, !!options.useGpu);
       this.enabled = this.yoloInstance.isInit();
+      instanceList.push(this.yoloInstance)
     } else {
       this.type = 'onnx';
 
@@ -39,6 +41,7 @@ module.exports = function (runtime, global) {
       }
       this.yoloInstance = yoloCreator.createOnnx(options.modelPath, convertToList(options.labels), options.imageSize);
       this.enabled = this.yoloInstance != null;
+      instanceList.push(this.yoloInstance)
     }
 
     if (this.enabled) {
@@ -60,9 +63,7 @@ module.exports = function (runtime, global) {
     }
     if (region) {
       let r = buildRegion(region, img)
-      let o = img
       img = images.clip(img, r.x, r.y, r.width, r.height)
-      o.recycle()
     }
     let resultList = util.java.toJsArray(this.yoloInstance.predictYolo(img.mat))
     if (region) {
@@ -94,6 +95,12 @@ module.exports = function (runtime, global) {
 
   return $yolo;
 
+  events.on('exit', function () {
+    if (instanceList.length > 0) {
+        instanceList.forEach(instance => instance.release())
+    }
+  })
+
   function wrapForward (yoloInstance) {
     return {
       forward: function (img, filterOption, region) {

diff --git a/autojs/src/main/assets/modules/__engines__.js b/autojs/src/main/assets/modules/__engines__.js
@@ -1,62 +1,61 @@
 
-module.exports = function(__runtime__, scope){
+module.exports = function (__runtime__, scope) {
     var rtEngines = __runtime__.engines;
-    var execArgv = null;
-
     var engines = {};
 
-    engines.execScript = function(name, script, config){
+    engines.execScript = function (name, script, config) {
         return rtEngines.execScript(name, script, fillConfig(config));
     }
 
-    engines.execScriptFile = function(path, config){
+    engines.execScriptFile = function (path, config) {
         return rtEngines.execScriptFile(path, fillConfig(config));
     }
 
-    engines.execAutoFile = function(path, config){
+    engines.execAutoFile = function (path, config) {
         return rtEngines.execAutoFile(path, fillConfig(config));
     }
 
-    engines.myEngine = function(){
+    engines.myEngine = function () {
         return rtEngines.myEngine();
     }
 
-    engines.all = function(){
+    engines.all = function () {
         return rtEngines.all();
     }
 
     engines.stopAll = rtEngines.stopAll.bind(rtEngines);
     engines.stopAllAndToast = rtEngines.stopAllAndToast.bind(rtEngines);
 
-    function fillConfig(c){
+    function fillConfig (c) {
         var config = new com.stardust.autojs.execution.ExecutionConfig();
         c = c || {};
         c.path = c.path || files.cwd();
-        if(c.path){
-           config.workingDirectory = c.path;
+        if (c.path) {
+            config.workingDirectory = c.path;
         }
         config.delay = c.delay || 0;
         config.interval = c.interval || 0;
-        config.loopTimes = (c.loopTimes === undefined)? 1 : c.loopTimes;
-        if(c.arguments){
+        config.loopTimes = (c.loopTimes === undefined) ? 1 : c.loopTimes;
+        if (c.arguments) {
             var arguments = c.arguments;
-            for(var key in arguments){
-                if(arguments.hasOwnProperty(key)){
+            for (var key in arguments) {
+                if (arguments.hasOwnProperty(key)) {
                     config.setArgument(key, arguments[key]);
                 }
             }
         }
         return config;
     }
 
-    var engine = engines.myEngine();
-    var execArgv = {};
-    var iterator = engine.getTag("execution.config").arguments.entrySet().iterator();
-    while(iterator.hasNext()){
-        var entry = iterator.next();
-        execArgv[entry.getKey()] = entry.getValue();
-    }
-    engine.execArgv = execArgv;
+    ((engine) => {
+        let execArgv = {}
+        let iterator = engine.getTag("execution.config").arguments.entrySet().iterator();
+        while (iterator.hasNext()) {
+            let entry = iterator.next();
+            execArgv[entry.getKey()] = entry.getValue();
+        }
+        engine.execArgv = execArgv
+    })(engines.myEngine())
 
     return engines;
 }
diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java b/autojs/src/main/java/com/stardust/autojs/core/image/ImageWrapper.java
@@ -108,49 +108,49 @@ public static Bitmap toBitmap(Image image) {
     }
 
     public static ImageWrapper ofImageByMat(Image image, int cvType) {
-
         long start = System.currentTimeMillis();
 
         // 获取Image的平面
         Image.Plane[] planes = image.getPlanes();
-        Log.d("ImageWrapper", "ofImageByMat: planes.length: " + planes.length);
+//        Log.d("ImageWrapper", "ofImageByMat: planes.length: " + planes.length);
         // 获取Image的宽高
         int width = image.getWidth();
         int height = image.getHeight();
+//        Log.d("ImageWrapper", "ofImageByMat: width:" + width + " height:" + height);
 
-        Log.d("ImageWrapper", "ofImageByMat: width:" + width + " height:" + height);
         Image.Plane plane = planes[0];
         // 获取平面的数据缓冲区
-        ByteBuffer buffer = planes[0].getBuffer();
-        // 创建一个byte数组来存储缓冲区的数据
-        byte[] bytes = new byte[buffer.remaining()];
-        buffer.position(0);
+        ByteBuffer buffer = plane.getBuffer();
         int pixelStride = plane.getPixelStride();
-        int rowPadding = plane.getRowStride() - pixelStride * image.getWidth();
-        // 将数据从缓冲区拷贝到byte数组
-        buffer.get(bytes);
+        int rowStride = plane.getRowStride();
+        int rowPadding = rowStride - pixelStride * width;
 
-        // 创建一个Mat对象
+        long s2 = System.currentTimeMillis();
+        // 尽量避免使用临时数组
         Mat mat = new Mat(height, width + rowPadding / pixelStride, CvType.CV_8UC4);
-        // 将byte数组拷贝到Mat对象
-        mat.put(0, 0, bytes);
+        byte[] rowData = new byte[rowStride];
+        for (int i = 0; i < height; i++) {
+            buffer.get(rowData, 0, rowStride);
+            mat.put(i, 0, rowData);
+        }
+//        Log.d("ImageWrapper", "ofImageByMat: create mat by bytes cost: " + (System.currentTimeMillis() - s2) + "ms");
         if (width != mat.width()) {
-            Log.d("ImageWrapper", "ofImageByMat: mat width is not valid: " + mat.width() + " => " + width);
+//            Log.d("ImageWrapper", "ofImageByMat: mat width is not valid: " + mat.width() + " => " + width);
             // 定义裁切区域
             Rect rect = new Rect(0, 0, width, height);
-            // 裁切图像
             Mat croppedImage = new Mat(mat, rect);
             mat.release();
             mat = croppedImage;
         }
+
         if (cvType != CvType.CV_8UC4) {
             long convertStart = System.currentTimeMillis();
             Imgproc.cvtColor(mat, mat, Imgproc.COLOR_RGBA2RGB);
-            Log.d("ImageWrapper", "ofImageByMat: convert channel: " + (System.currentTimeMillis() - convertStart) + "ms");
+//            Log.d("ImageWrapper", "ofImageByMat: convert channel: " + (System.currentTimeMillis() - convertStart) + "ms");
         }
-        Log.d("ImageWrapper", "ofImageByMat: create by mat cost: " + (System.currentTimeMillis() - start) + "ms");
-        return new ImageWrapper(mat);
 
+//        Log.d("ImageWrapper", "ofImageByMat: create by mat cost: " + (System.currentTimeMillis() - start) + "ms");
+        return new ImageWrapper(mat);
     }
 
     public int getWidth() {

diff --git a/autojs/src/main/java/com/stardust/autojs/execution/RunnableScriptExecution.java b/autojs/src/main/java/com/stardust/autojs/execution/RunnableScriptExecution.java
@@ -2,7 +2,6 @@
 
 import android.util.Log;
 
-import com.stardust.autojs.AutoJs;
 import com.stardust.autojs.ScriptEngineService;
 import com.stardust.autojs.engine.ScriptEngine;
 import com.stardust.autojs.engine.ScriptEngineManager;