diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/block.png" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/block.png" new file mode 100644 index 000000000..c0944e12d Binary files /dev/null and "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/block.png" differ diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/mario.png" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/mario.png" new file mode 100644 index 000000000..042671c69 Binary files /dev/null and "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/mario.png" differ diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/super_mario.jpg" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/super_mario.jpg" new file mode 100644 index 000000000..373502ece Binary files /dev/null and "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/super_mario.jpg" differ diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\346\211\200\346\234\211\351\227\256\345\217\267\346\226\271\345\235\227.js" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\346\211\200\346\234\211\351\227\256\345\217\267\346\226\271\345\235\227.js" new file mode 100644 index 000000000..ddea51e99 --- /dev/null +++ "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\346\211\200\346\234\211\351\227\256\345\217\267\346\226\271\345\235\227.js" @@ -0,0 +1,11 @@ + +var superMario = images.read("./super_mario.jpg"); +var block = images.read("./block.png"); + +var result = images.matchTemplate(superMario, block, { + threshold: 0.8 +}).matches; +toastLog(result); + +superMario.recycle(); +block.recycle(); \ No newline at end of file diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\227\256\345\217\267\346\226\271\345\235\227\345\271\266\347\224\273\345\207\272\344\275\215\347\275\256.js" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\227\256\345\217\267\346\226\271\345\235\227\345\271\266\347\224\273\345\207\272\344\275\215\347\275\256.js" new file mode 100644 index 000000000..8b25049be --- /dev/null +++ "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\227\256\345\217\267\346\226\271\345\235\227\345\271\266\347\224\273\345\207\272\344\275\215\347\275\256.js" @@ -0,0 +1,23 @@ + +var superMario = images.read("./super_mario.jpg"); +var block = images.read("./block.png"); +var points = images.matchTemplate(superMario, block, { + threshold: 0.8 +}).points; + +toastLog(points); + +var canvas = new Canvas(superMario); +var paint = new Paint(); +paint.setColor(colors.parseColor("#2196F3")); +points.forEach(point => { + canvas.drawRect(point.x, point.y, point.x + block.width, point.y + block.height, paint); +}); +var image = canvas.toImage(); +images.save(image, "/sdcard/tmp.png"); + +app.viewFile("/sdcard/tmp.png"); + +superMario.recycle(); +block.recycle(); +image.recycle(); diff --git "a/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\251\254\351\207\214\345\245\245.js" "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\251\254\351\207\214\345\245\245.js" new file mode 100644 index 000000000..6285cbef4 --- /dev/null +++ "b/app/src/main/assets/sample/\345\233\276\347\211\207\344\270\216\345\233\276\350\211\262\345\244\204\347\220\206/\346\211\276\345\233\276/\346\211\276\345\207\272\351\251\254\351\207\214\345\245\245.js" @@ -0,0 +1,8 @@ + +var superMario = images.read("./super_mario.jpg"); +var mario = images.read("./mario.png"); +var point = findImage(superMario, mario); +toastLog(point); + +superMario.recycle(); +mario.recycle(); \ No newline at end of file diff --git a/app/src/main/java/org/autojs/autojs/autojs/build/ApkBuilder.java b/app/src/main/java/org/autojs/autojs/autojs/build/ApkBuilder.java index 6060878a4..bd0837516 100644 --- a/app/src/main/java/org/autojs/autojs/autojs/build/ApkBuilder.java +++ b/app/src/main/java/org/autojs/autojs/autojs/build/ApkBuilder.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.concurrent.Callable; +import java.util.regex.Pattern; import pxb.android.StringItem; import pxb.android.axml.AxmlWriter; @@ -201,7 +202,6 @@ public void copyDir(String relativePath, String path) throws IOException { private void encrypt(File toDir, File file) throws IOException { FileOutputStream fos = new FileOutputStream(new File(toDir, file.getName())); - EncryptedScriptFileHeader.INSTANCE.writeHeader(fos, (short) new JavaScriptFileSource(file).getExecutionMode()); encrypt(fos, file); } diff --git a/app/src/main/java/org/autojs/autojs/storage/database/Database.java b/app/src/main/java/org/autojs/autojs/storage/database/Database.java index d42fea788..ddf72f20f 100644 --- a/app/src/main/java/org/autojs/autojs/storage/database/Database.java +++ b/app/src/main/java/org/autojs/autojs/storage/database/Database.java @@ -58,6 +58,7 @@ public Observable delete(M model) { public Observable update(M model) { return exec(() -> { ContentValues values = asContentValues(model); + values.put("id", model.getId()); int update = mWritableSQLiteDatabase.update(mTable, values, "id = ?", arg(model.getId())); if (update >= 1) { mModelChange.onNext(new ModelChange<>(model, ModelChange.UPDATE)); diff --git a/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java b/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java index b6f343e5a..4354bb7ad 100644 --- a/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java +++ b/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java @@ -21,7 +21,6 @@ public TimedTaskDatabase(Context context) { @Override protected ContentValues asContentValues(TimedTask model) { ContentValues values = new ContentValues(); - values.put("id", model.getId()); values.put("time", model.getTimeFlag()); values.put("scheduled", model.isScheduled()); values.put("delay", model.getDelay()); diff --git a/autojs/src/main/assets/modules/__images__.js b/autojs/src/main/assets/modules/__images__.js index 587e10afd..1c35e4cb8 100644 --- a/autojs/src/main/assets/modules/__images__.js +++ b/autojs/src/main/assets/modules/__images__.js @@ -1,26 +1,123 @@ module.exports = function (runtime, scope) { const ResultAdapter = require("result_adapter"); - function images(){ + + var MatchingResult = (function () { + var comparators = { + "left": (l, r) => l.point.x - r.point.x, + "top": (l, r) => l.point.y - r.point.y, + "right": (l, r) => r.point.x - l.point.x, + "bottom": (l, r) => r.point.y - l.point.y + } + function MatchingResult(list) { + if (Array.isArray(list)) { + this.matches = list; + } else { + this.matches = runtime.bridges.bridges.toArray(list); + } + this.__defineGetter__("points", () => { + if (typeof (this.__points__) == 'undefined') { + this.__points__ = this.matches.map(m => m.point); + } + return this.__points__; + }); + } + MatchingResult.prototype.first = function () { + if (this.matches.length == 0) { + return null; + } + return this.matches[0]; + } + MatchingResult.prototype.last = function () { + if (this.matches.length == 0) { + return null; + } + return this.matches[this.matches.length - 1]; + } + MatchingResult.prototype.findMax = function (cmp) { + if (this.matches.length == 0) { + return null; + } + var target = this.matches[0]; + this.matches.forEach(m => { + if (cmp(target, m) > 0) { + target = m; + } + }); + return target; + } + MatchingResult.prototype.leftmost = function () { + return this.findMax(comparators.left); + } + MatchingResult.prototype.topmost = function () { + return this.findMax(comparators.top); + } + MatchingResult.prototype.rightmost = function () { + return this.findMax(comparators.right); + } + MatchingResult.prototype.bottommost = function () { + return this.findMax(comparators.bottom); + } + MatchingResult.prototype.worst = function () { + return this.findMax((l, r) => l.similarity - r.similarity); + } + MatchingResult.prototype.best = function () { + return this.findMax((l, r) => r.similarity - l.similarity); + } + MatchingResult.prototype.sortBy = function(cmp) { + var comparatorFn = null; + if(typeof(cmp) == 'string'){ + cmp.split("-").forEach(direction => { + var buildInFn = comparators[direction]; + if(!buildInFn){ + throw new Error("unknown direction '" + direction + "' in '" + cmp +"'"); + } + (function(fn){ + if(comparatorFn == null){ + comparatorFn = fn; + }else{ + comparatorFn = (function(comparatorFn, fn){ + return function(l, r){ + var cmpValue = comparatorFn(l, r); + if(cmpValue == 0){ + return fn(l, r); + } + return cmpValue; + } + })(comparatorFn, fn); + } + })(buildInFn); + }); + }else{ + comparatorFn = cmp; + } + var clone = this.matches.slice(); + clone.sort(comparatorFn); + return new MatchingResult(clone); + } + return MatchingResult; + })(); + + function images() { } - if(android.os.Build.VERSION.SDK_INT >= 21){ + if (android.os.Build.VERSION.SDK_INT >= 21) { util.__assignFunctions__(runtime.images, images, ['captureScreen', 'read', 'copy', 'load', 'clip', 'pixel']) } - images.opencvImporter = JavaImporter( - org.opencv.core.Point, - org.opencv.core.Point3, - org.opencv.core.Rect, - org.opencv.core.Algorithm, - org.opencv.core.Scalar, - org.opencv.core.Size, - org.opencv.core.Core, - org.opencv.core.CvException, - org.opencv.core.CvType, - org.opencv.core.TermCriteria, - org.opencv.core.RotatedRect, - org.opencv.core.Range, - org.opencv.imgproc.Imgproc, - com.stardust.autojs.core.opencv + images.opencvImporter = JavaImporter( + org.opencv.core.Point, + org.opencv.core.Point3, + org.opencv.core.Rect, + org.opencv.core.Algorithm, + org.opencv.core.Scalar, + org.opencv.core.Size, + org.opencv.core.Core, + org.opencv.core.CvException, + org.opencv.core.CvType, + org.opencv.core.TermCriteria, + org.opencv.core.RotatedRect, + org.opencv.core.Range, + org.opencv.imgproc.Imgproc, + com.stardust.autojs.core.opencv ); with (images.opencvImporter) { const defaultColorThreshold = 4; @@ -56,13 +153,13 @@ module.exports = function (runtime, scope) { var colorFinder = javaImages.colorFinder; - images.requestScreenCapture = function(landscape) { + images.requestScreenCapture = function (landscape) { let ScreenCapturer = com.stardust.autojs.core.image.capture.ScreenCapturer; var orientation = ScreenCapturer.ORIENTATION_AUTO; - if(landscape === true){ + if (landscape === true) { orientation = ScreenCapturer.ORIENTATION_LANDSCAPE; } - if(landscape === false){ + if (landscape === false) { orientation = ScreenCapturer.ORIENTATION_PORTRAIT; } return ResultAdapter.wait(javaImages.requestScreenCapture(orientation)); @@ -105,7 +202,7 @@ module.exports = function (runtime, scope) { colors.blue(color) - threshold, colors.alpha(color)); ub = new Scalar(colors.red(color) + threshold, colors.green(color) + threshold, colors.blue(color) + threshold, colors.alpha(color)); - }else{ + } else { throw new TypeError('lowerBound = ' + lowerBound, + 'upperBound = ' + upperBound); } } @@ -115,7 +212,7 @@ module.exports = function (runtime, scope) { } - images.adaptiveThreshold = function(img, maxValue, adaptiveMethod, thresholdType, blockSize, C){ + images.adaptiveThreshold = function (img, maxValue, adaptiveMethod, thresholdType, blockSize, C) { initIfNeeded(); var mat = new Mat(); adaptiveMethod = Imgproc["ADAPTIVE_THRESH_" + adaptiveMethod]; @@ -168,13 +265,13 @@ module.exports = function (runtime, scope) { return images.matToImage(mat); } - images.findCircles = function(grayImg, options) { + images.findCircles = function (grayImg, options) { initIfNeeded(); options = options || {}; var mat = options.region == undefined ? grayImg.mat : new Mat(grayImg.mat, buildRegion(options.region, grayImg)); var resultMat = new Mat() var dp = options.dp == undefined ? 1 : options.dp; - var minDst = options.minDst == undefined ? grayImg.height / 8 : options.minDst; + var minDst = options.minDst == undefined ? grayImg.height / 8 : options.minDst; var param1 = options.param1 == undefined ? 100 : options.param1; var param2 = options.param2 == undefined ? 100 : options.param2; var minRadius = options.minRadius == undefined ? 0 : options.minRadius; @@ -191,14 +288,14 @@ module.exports = function (runtime, scope) { }); } } - if(options.region != undefined){ + if (options.region != undefined) { mat.release(); } resultMat.release(); return result; } - images.resize = function(img, size, interpolation) { + images.resize = function (img, size, interpolation) { initIfNeeded(); var mat = new Mat(); interpolation = Imgproc["INTER_" + (interpolation || "LINEAR")]; @@ -206,7 +303,7 @@ module.exports = function (runtime, scope) { return images.matToImage(mat); } - images.scale = function(img, fx, fy, interpolation) { + images.scale = function (img, fx, fy, interpolation) { initIfNeeded(); var mat = new Mat(); interpolation = Imgproc["INTER_" + (interpolation || "LINEAR")]; @@ -214,18 +311,18 @@ module.exports = function (runtime, scope) { return images.matToImage(mat); } - images.rotate = function(img, degree, x, y) { + images.rotate = function (img, degree, x, y) { initIfNeeded(); - if(x == undefined){ + if (x == undefined) { x = img.width / 2; } - if(y == undefined){ + if (y == undefined) { y = img.height / 2; } return javaImages.rotate(img, x, y, degree); } - images.concat = function(img1, img2, direction, rect1, rect2) { + images.concat = function (img1, img2, direction, rect1, rect2) { initIfNeeded(); direction = direction || "right"; rect1 = buildRegion(rect1, img1); @@ -314,7 +411,7 @@ module.exports = function (runtime, scope) { if (typeof (options.level) == 'number') { maxLevel = options.level; } - var weakThreshold = options.weakThreshold || 0.7; + var weakThreshold = options.weakThreshold || 0.6; if (options.region) { return javaImages.findImage(img, template, weakThreshold, threshold, buildRegion(options.region, img), maxLevel); } else { @@ -322,6 +419,27 @@ module.exports = function (runtime, scope) { } } + images.matchTemplate = function (img, template, options) { + initIfNeeded(); + options = options || {}; + var threshold = options.threshold || 0.9; + var maxLevel = -1; + if (typeof (options.level) == 'number') { + maxLevel = options.level; + } + var max = options.max || 5; + var weakThreshold = options.weakThreshold || 0.6; + var result; + if (options.region) { + result = javaImages.matchTemplate(img, template, weakThreshold, threshold, buildRegion(options.region, img), maxLevel, max); + } else { + result = javaImages.matchTemplate(img, template, weakThreshold, threshold, null, maxLevel, max); + } + return new MatchingResult(result); + } + + + images.findImageInRegion = function (img, template, x, y, width, height, threshold) { return images.findImage(img, template, { region: [x, y, width, height], @@ -364,11 +482,15 @@ module.exports = function (runtime, scope) { }; } - images.matToImage = function(img){ + images.matToImage = function (img) { initIfNeeded(); return Image.ofMat(img); } + + + + function getColorDetector(color, algorithm, threshold) { switch (algorithm) { case "rgb": @@ -395,7 +517,7 @@ module.exports = function (runtime, scope) { } function buildRegion(region, img) { - if(region == undefined){ + if (region == undefined) { region = []; } var x = region[0] === undefined ? 0 : region[0]; @@ -403,6 +525,9 @@ module.exports = function (runtime, scope) { var width = region[2] === undefined ? img.getWidth() - x : region[2]; var height = region[3] === undefined ? (img.getHeight() - y) : region[3]; var r = new org.opencv.core.Rect(x, y, width, height); + if(x < 0 || y < 0 || x + width > img.width || y + height > img.height) { + throw new Error("out of region: region = [" + [x, y, width, height] + "], image.size = [" + [img.width, img.height] + "]"); + } return r; } @@ -412,7 +537,7 @@ module.exports = function (runtime, scope) { } return color; } - + function newSize(size) { if (!Array.isArray(size)) { size = [size, size]; @@ -423,14 +548,14 @@ module.exports = function (runtime, scope) { return new Size(size[0], size[1]); } - function initIfNeeded(){ + function initIfNeeded() { javaImages.initOpenCvIfNeeded(); } - + scope.__asGlobal__(images, ['requestScreenCapture', 'captureScreen', 'findImage', 'findImageInRegion', 'findColor', 'findColorInRegion', 'findColorEquals', 'findMultiColors']); - + scope.colors = colors; - + return images; } } \ No newline at end of file diff --git a/autojs/src/main/java/com/stardust/autojs/core/graphics/ScriptCanvas.java b/autojs/src/main/java/com/stardust/autojs/core/graphics/ScriptCanvas.java index d7d79df3a..a71e6aa10 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/graphics/ScriptCanvas.java +++ b/autojs/src/main/java/com/stardust/autojs/core/graphics/ScriptCanvas.java @@ -316,6 +316,7 @@ public void drawOval(@NonNull RectF oval, @NonNull Paint paint) { mCanvas.drawOval(oval, paint); } + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) public void drawOval(float left, float top, float right, float bottom, @NonNull Paint paint) { mCanvas.drawOval(left, top, right, bottom, paint); } diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java b/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java index cbba493c7..d5dba8655 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/ColorFinder.java @@ -56,6 +56,7 @@ public Point findColor(ImageWrapper image, int color, int threshold, Rect rect) } public Point[] findAllPointsForColor(ImageWrapper image, int color, int threshold, Rect rect) { + MatOfPoint matOfPoint = findColorInner(image, color, threshold, rect); if (matOfPoint == null) { return new Point[0]; diff --git a/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java index ef9295c38..065f50a3f 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java +++ b/autojs/src/main/java/com/stardust/autojs/core/image/TemplateMatching.java @@ -1,6 +1,5 @@ package com.stardust.autojs.core.image; -import android.util.Pair; import android.util.TimingLogger; import com.stardust.autojs.core.opencv.OpenCVHelper; @@ -8,12 +7,20 @@ import org.opencv.core.Core; import org.opencv.core.CvType; + import com.stardust.autojs.core.opencv.Mat; + import org.opencv.core.Point; import org.opencv.core.Rect; +import org.opencv.core.Scalar; import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + /** * Created by Stardust on 2017/11/25. @@ -21,13 +28,35 @@ public class TemplateMatching { + public static class Match { + public final Point point; + public final double similarity; + + public Match(Point point, double similarity) { + this.point = point; + this.similarity = similarity; + } + + @Override + public String toString() { + return "Match{" + + "point=" + point + + ", similarity=" + similarity + + '}'; + } + } + private static final String LOG_TAG = "TemplateMatching"; public static final int MAX_LEVEL_AUTO = -1; public static final int MATCHING_METHOD_DEFAULT = Imgproc.TM_CCOEFF_NORMED; - public static Point fastTemplateMatching(Mat img, Mat template, float threshold) { - return fastTemplateMatching(img, template, MATCHING_METHOD_DEFAULT, 0.75f, threshold, MAX_LEVEL_AUTO); + public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel) { + List result = fastTemplateMatching(img, template, matchMethod, weakThreshold, strictThreshold, maxLevel, 1); + if (result.isEmpty()) { + return null; + } + return result.get(0).point; } /** @@ -41,7 +70,7 @@ public static Point fastTemplateMatching(Mat img, Mat template, float threshold) * @param maxLevel 图像金字塔的层数 * @return */ - public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel) { + public static List fastTemplateMatching(Mat img, Mat template, int matchMethod, float weakThreshold, float strictThreshold, int maxLevel, int limit) { TimingLogger logger = new TimingLogger(LOG_TAG, "fast_tm"); if (maxLevel == MAX_LEVEL_AUTO) { //自动选取金字塔层数 @@ -49,62 +78,64 @@ public static Point fastTemplateMatching(Mat img, Mat template, int matchMethod, logger.addSplit("selectPyramidLevel:" + maxLevel); } //保存每一轮匹配到模板图片在原图片的位置 - Point p = null; - Mat matchResult = null; - double similarity = 0; + List finalMatchResult = new ArrayList<>(); + List previousMatchResult = Collections.emptyList(); boolean isFirstMatching = true; for (int level = maxLevel; level >= 0; level--) { - //放缩图片 + // 放缩图片 + List currentMatchResult = new ArrayList<>(); Mat src = getPyramidDownAtLevel(img, level); Mat currentTemplate = getPyramidDownAtLevel(template, level); - //如果在上一轮中没有匹配到图片，则考虑是否退出匹配 - if (p == null) { - //如果不是第一次匹配，并且不满足shouldContinueMatching的条件，则直接退出匹配（返回null） + // 如果在上一轮中没有匹配到图片，则考虑是否退出匹配 + if (previousMatchResult.isEmpty()) { + // 如果不是第一次匹配，并且不满足shouldContinueMatching的条件，则直接退出匹配 if (!isFirstMatching && !shouldContinueMatching(level, maxLevel)) { break; } + Mat matchResult = matchTemplate(src, currentTemplate, matchMethod); + getBestMatched(matchResult, currentTemplate, matchMethod, weakThreshold, currentMatchResult, limit, null); OpenCVHelper.release(matchResult); - matchResult = matchTemplate(src, currentTemplate, matchMethod); - Pair bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold); - p = bestMatched.first; - similarity = bestMatched.second; } else { - //根据上一轮的匹配点，计算本次匹配的区域 - Rect r = getROI(p, src, currentTemplate); - OpenCVHelper.release(matchResult); - Mat m = new Mat(src, r); - matchResult = matchTemplate(m, currentTemplate, matchMethod); - OpenCVHelper.release(m); - Pair bestMatched = getBestMatched(matchResult, matchMethod, weakThreshold); - //不满足弱阈值，返回null - if (bestMatched.second < weakThreshold) { - // p = null; - // break; + for (Match match : previousMatchResult) { + // 根据上一轮的匹配点，计算本次匹配的区域 + Rect r = getROI(match.point, src, currentTemplate); + Mat m = new Mat(src, r); + Mat matchResult = matchTemplate(m, currentTemplate, matchMethod); + getBestMatched(matchResult, currentTemplate, matchMethod, weakThreshold, currentMatchResult, limit, r); + OpenCVHelper.release(m); + OpenCVHelper.release(matchResult); } - p = bestMatched.first; - similarity = bestMatched.second; - p.x += r.x; - p.y += r.y; } + if (src != img) OpenCVHelper.release(src); if (currentTemplate != template) OpenCVHelper.release(currentTemplate); - //满足强阈值，返回当前结果 - if (similarity >= strictThreshold) { - pyrUp(p, level); - break; + + logger.addSplit("level:" + level + ", result:" + previousMatchResult); + + // 把满足强阈值的点找出来，加到最终结果列表 + if (!currentMatchResult.isEmpty()) { + Iterator iterator = currentMatchResult.iterator(); + while (iterator.hasNext()) { + Match match = iterator.next(); + if (match.similarity >= strictThreshold) { + pyrUp(match.point, level); + finalMatchResult.add(match); + iterator.remove(); + } + } + // 如果所有结果都满足强阈值，则退出循环，返回最终结果 + if (currentMatchResult.isEmpty()) { + break; + } } - logger.addSplit("level:" + level + " point:" + p); isFirstMatching = false; + previousMatchResult = currentMatchResult; } - logger.addSplit("result:" + p); + logger.addSplit("result:" + finalMatchResult); logger.dumpToLog(); - OpenCVHelper.release(matchResult); - if (similarity < strictThreshold) { - return null; - } - return p; + return finalMatchResult; } @@ -168,7 +199,7 @@ private static int selectPyramidLevel(Mat img, Mat template) { } - public static Mat matchTemplate(Mat img, Mat temp, int match_method) { + private static Mat matchTemplate(Mat img, Mat temp, int match_method) { int result_cols = img.cols() - temp.cols() + 1; int result_rows = img.rows() - temp.rows() + 1; Mat result = new Mat(result_rows, result_cols, CvType.CV_32FC1); @@ -176,10 +207,21 @@ public static Mat matchTemplate(Mat img, Mat temp, int match_method) { return result; } - public static Pair getBestMatched(Mat tmResult, int matchMethod, float threshold) { + private static void getBestMatched(Mat tmResult, Mat template, int matchMethod, float weakThreshold, List outResult, int limit, Rect rect) { + for (int i = 0; i < limit; i++) { + Match bestMatched = getBestMatched(tmResult, matchMethod, weakThreshold, rect); + if (bestMatched == null) { + break; + } + outResult.add(bestMatched); + Core.rectangle(tmResult, bestMatched.point, + new Point(bestMatched.point.x + template.cols(), bestMatched.point.y + template.rows()), + new Scalar(0, 255, 0), -1); + } + } + + private static Match getBestMatched(Mat tmResult, int matchMethod, float weakThreshold, Rect rect) { TimingLogger logger = new TimingLogger(LOG_TAG, "best_matched_point"); - // FIXME: 2017/11/26 正交化? - // Core.normalize(tmResult, tmResult, 0, 1, Core.NORM_MINMAX, -1, new Mat()); Core.MinMaxLocResult mmr = Core.minMaxLoc(tmResult); logger.addSplit("minMaxLoc"); double value; @@ -191,9 +233,15 @@ public static Pair getBestMatched(Mat tmResult, int matchMethod, pos = mmr.maxLoc; value = mmr.maxVal; } + if (value < weakThreshold) { + return null; + } + if (rect != null) { + pos.x += rect.x; + pos.y += rect.y; + } logger.addSplit("value:" + value); - logger.dumpToLog(); - return new Pair<>(pos, value); + return new Match(pos, value); } diff --git a/autojs/src/main/java/com/stardust/autojs/core/web/InjectableWebClient.java b/autojs/src/main/java/com/stardust/autojs/core/web/InjectableWebClient.java index 5099e301e..95eee69c4 100644 --- a/autojs/src/main/java/com/stardust/autojs/core/web/InjectableWebClient.java +++ b/autojs/src/main/java/com/stardust/autojs/core/web/InjectableWebClient.java @@ -92,15 +92,12 @@ private class ScriptBridge { @JavascriptInterface public String eval(final String script) { result = null; - mWebView.post(new Runnable() { - @Override - public void run() { - Log.v(TAG, "ScriptBridge.eval: " + script); - result = mContext.evaluateString(mScriptable, script, "", 1, null); - Log.v(TAG, "ScriptBridge.eval = " + result); - synchronized (ScriptBridge.this) { - ScriptBridge.this.notify(); - } + mWebView.post(() -> { + Log.v(TAG, "ScriptBridge.eval: " + script); + result = mContext.evaluateString(mScriptable, script, "", 1, null); + Log.v(TAG, "ScriptBridge.eval = " + result); + synchronized (ScriptBridge.this) { + ScriptBridge.this.notify(); } }); synchronized (ScriptBridge.this) { diff --git a/autojs/src/main/java/com/stardust/autojs/runtime/ScriptBridges.java b/autojs/src/main/java/com/stardust/autojs/runtime/ScriptBridges.java index c567d8b0f..072d9c2ce 100644 --- a/autojs/src/main/java/com/stardust/autojs/runtime/ScriptBridges.java +++ b/autojs/src/main/java/com/stardust/autojs/runtime/ScriptBridges.java @@ -26,6 +26,10 @@ public void setBridges(Bridges bridges) { mBridges = bridges; } + public Bridges getBridges() { + return mBridges; + } + public Object callFunction(Object func, Object target, Object args) { checkBridges(); return mBridges.call(func, target, args); diff --git a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java index 711a60383..53fb17f99 100644 --- a/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java +++ b/autojs/src/main/java/com/stardust/autojs/runtime/api/Images.java @@ -43,6 +43,9 @@ import java.net.HttpURLConnection; import java.net.URL; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; /** * Created by Stardust on 2017/5/20. @@ -303,6 +306,34 @@ public Point findImage(ImageWrapper image, ImageWrapper template, float weakThre return point; } + public List matchTemplate(ImageWrapper image, ImageWrapper template, float weakThreshold, float threshold, Rect rect, int maxLevel, int limit) { + initOpenCvIfNeeded(); + if (image == null) + throw new NullPointerException("image = null"); + if (template == null) + throw new NullPointerException("template = null"); + Mat src = image.getMat(); + if (rect != null) { + src = new Mat(src, rect); + } + List result = TemplateMatching.fastTemplateMatching(src, template.getMat(), TemplateMatching.MATCHING_METHOD_DEFAULT, + weakThreshold, threshold, maxLevel, limit); + for (TemplateMatching.Match match : result) { + Point point = match.point; + if (rect != null) { + point.x += rect.x; + point.y += rect.y; + } + point.x = mScreenMetrics.scaleX((int) point.x); + point.y = mScreenMetrics.scaleX((int) point.y); + } + if (src != image.getMat()) { + OpenCVHelper.release(src); + } + Collections.sort(result, (l, r) -> Double.compare(r.similarity, l.similarity)); + return result; + } + public Mat newMat() { return new Mat(); }