diff --git a/assets/javascripts/needlediff.js b/assets/javascripts/needlediff.js index 737ba0a6da9..b6ed9431b72 100644 --- a/assets/javascripts/needlediff.js +++ b/assets/javascripts/needlediff.js @@ -77,6 +77,64 @@ function NeedleDiff(id, width, height) { }); } +/** + * Function ensuring that the text fits into the area rectangle. + * @param {Object} ctx - The Canvas context. + * @param {Object} area - The OCR area to be drawn into. + * @param {Object} txtStyle - The target text style for calibration. + * @returns {number} The calculated font size. + */ +function calcFontSize(ctx, area, txtStyle) { + const ocrLines = area.ocr_str.split('\n'); + + // 1px margin in height + let ret = Math.ceil(((area.height - ocrLines.length - 1) / ocrLines.length) - 1); + + ctx.font = txtStyle.weight + ' ' + ret + 'px ' + txtStyle.typeface; + for (line of ocrLines) { + let ocrLinesWidth = ctx.measureText(line).width; + while (ocrLinesWidth > area.width - 2) { + ret--; + ctx.font = txtStyle.weight + ' ' + ret + 'px ' + txtStyle.typeface; + ocrLinesWidth = ctx.measureText(line).width; + } + } + return ret; +} + +/** + * Method drawing the OCR text into the OCR area. + * @param {Object} needlediff - The needlediff containing the canvas context. + * @param {Object} area - The OCR area to be drawn into. + */ +function drawOcrTxt(needlediff, area) { + const txtStyle = { + alignment: 'center', + typeface: 'Arial', + color: 'rgb(64,224,208)', + weight: 'bold' + } + const fontSize = calcFontSize(needlediff.ctx, area, txtStyle); + const ocrLines = area.ocr_str.split('\n'); + const ocrTxtXPos = Math.ceil(area.xpos + area.width / 2); + + // Body line appears to be at about 1/3 of whole font height + let ocrTxtYPos = Math.floor(area.ypos + area.height / 2 + fontSize / 3 - + ((ocrLines.length - 1) * fontSize) / 2); + + needlediff.ctx.textAlign = txtStyle.alignment; + needlediff.ctx.fillStyle = txtStyle.color; + needlediff.ctx.font = txtStyle.weight + ' ' + fontSize + 'px ' + + txtStyle.typeface; + + for (line of ocrLines) { + needlediff.ctx.fillText(line, ocrTxtXPos, ocrTxtYPos); + + // 1px space between lines + ocrTxtYPos += fontSize + 1; + } +} + NeedleDiff.prototype.draw = function () { // First of all, draw the screenshot as gray background (if ready) if (!this.screenshotImg) { @@ -89,11 +147,6 @@ NeedleDiff.prototype.draw = function () { this.ctx.drawImage(this.screenshotImg, 0, 0); } - // Then, check if there is a needle to compare with - if (!this.needleImg) { - return; - } - // Calculate the pixel in which the division will be done var split = this.divide * this.width; if (split < 1) { @@ -138,7 +191,12 @@ NeedleDiff.prototype.draw = function () { if (!this.fullNeedleImg) { // draw matching part of needle image this.ctx.strokeStyle = NeedleDiff.strokecolor(a.type); - this.ctx.drawImage(this.needleImg, orig.xpos, orig.ypos, usedWith, a.height, x, a.ypos, usedWith, a.height); + if (this.needleImg) { + this.ctx.drawImage(this.needleImg, orig.xpos, orig.ypos, usedWith, a.height, x, a.ypos, usedWith, a.height); + } else if (a.ocr_str) { + drawOcrTxt(this, a); + } else return; + // draw frame of match area this.ctx.lineWidth = lineWidth; this.ctx.beginPath(); @@ -221,6 +279,7 @@ NeedleDiff.prototype.draw = function () { this.ctx.strokeStyle = 'rgb(0, 0, 0)'; this.ctx.lineWidth = 3; this.ctx.font = 'bold 14px Arial'; + this.ctx.textAlign = 'left'; var text = a['similarity'] + '%'; var textSize = this.ctx.measureText(text); var tx; @@ -326,16 +385,17 @@ function setDiffScreenshot(screenshotSrc) { $('').on('load', function () { var image = $(this).get(0); - // set screenshot resolution - window.differ = new NeedleDiff('needle_diff', image.width, image.height); - window.differ.screenshotImg = image; - setNeedle(); - // create gray version of it in off screen canvas var gray_canvas = document.createElement('canvas'); gray_canvas.width = image.width; gray_canvas.height = image.height; + // set screenshot resolution + window.differ = new NeedleDiff('needle_diff', image.width, image.height); + window.differ.screenshotImg = image; + window.differ.gray_canvas = gray_canvas; + setNeedle(); + var gray_context = gray_canvas.getContext('2d'); gray_context.drawImage(image, 0, 0); diff --git a/assets/javascripts/needleeditor.js b/assets/javascripts/needleeditor.js index ab0f04894b1..524b526bfb3 100644 --- a/assets/javascripts/needleeditor.js +++ b/assets/javascripts/needleeditor.js @@ -59,6 +59,13 @@ NeedleEditor.prototype.init = function () { return; } a.type = NeedleEditor.nexttype(a.type); + + updateAreaCtrls(a); + + // Attrs only apply to some area types and are not retained elsewhere. + delete a.refstr; + delete a.margin; + shape.fill = NeedleEditor.areacolor(a.type); editor.UpdateTextArea(); cv.redraw(); @@ -110,15 +117,14 @@ NeedleEditor.prototype.init = function () { cv.addShape(shape); editor.needle.area.push(a); editor.UpdateTextArea(); + updateAreaCtrls(a); return shape; }; - var areaSpecificButtons = $('#change-match, #change-margin, #toggle-click-coordinates'); $(cv).on('shape.selected', function () { - areaSpecificButtons.removeClass('disabled').removeAttr('disabled'); - updateToggleClickCoordinatesButton(editor.currentClickCoordinates()); + updateAreaCtrls(editor.selection().area); }); $(cv).on('shape.unselected', function () { - areaSpecificButtons.addClass('disabled').attr('disabled', 1); + updateAreaCtrls(null); }); document.getElementById('needleeditor_name').onchange = function () { @@ -160,6 +166,15 @@ NeedleEditor.prototype.AddTag = function (tag, checked) { return input; }; +/** + * Method updating the reference string and it's in it's text box and the + * corresponding attribute of it's area object. + */ +NeedleEditor.prototype.updateRefstr = function () { + this.selection().area.refstr = document.getElementById('txtarea-refstr').value; + this.UpdateTextArea(); +}; + NeedleEditor.nexttype = function (type) { if (type == 'match') { return 'exclude'; @@ -377,6 +392,7 @@ NeedleEditor.prototype.toggleClickCoordinates = function () { function loadBackground() { var needle = window.needles[$('#image_select option:selected').val()]; + needle = needle.imageurl ? needle : window.needles['screenshot']; nEditor.LoadBackground(needle.imageurl); $('#needleeditor_image').val(needle.imagename); $('#needleeditor_imagedistri').val(needle.imagedistri); @@ -440,6 +456,55 @@ function loadAreas() { } } +/** + * Method disabling a HTML element. + * @param {string} elemId - The ID of the element in the HTML document. + */ +function disableElem(elemId) { + const elem = document.getElementById(elemId); + elem.classList.add('disabled'); + elem.disabled = 'disabled'; +} + +/** + * Method enabling a HTML element. + * @param {string} elemId - The ID of the element in the HTML document. + */ +function enableElem(elemId) { + const elem = document.getElementById(elemId); + elem.classList.remove('disabled'); + elem.removeAttribute('disabled'); +} + +/** + * Method enabling/disabling controls for a specific area. + * @param {Object} area - The area which the controls should be prepared for. + */ +function updateAreaCtrls(area) { + + const ctrlElemIds = { + all: ['change-match', 'change-margin', 'toggle-click-coordinates', 'txtarea-refstr'], + match: ['change-match', 'change-margin', 'toggle-click-coordinates'], + ocr: ['change-match', 'toggle-click-coordinates', 'txtarea-refstr'], + exclude: ['toggle-click-coordinates'] + } + + if (!area || !(area.type in ctrlElemIds)) { + ctrlElemIds.all.forEach(elem => disableElem(elem)); + document.getElementById('txtarea-refstr').value = ''; + return; + } + ctrlElemIds[area.type].forEach(elem => enableElem(elem)); + const otherElemIds = ctrlElemIds.all.filter(elem => !ctrlElemIds[area.type].includes(elem)); + otherElemIds.forEach(elem => disableElem(elem)); + + if (area.type === 'ocr') { + document.getElementById('txtarea-refstr').value = area.refstr; + } else { + document.getElementById('txtarea-refstr').value = ''; + } +} + function addTag() { var input = $('#newtag'); var checkbox = nEditor.AddTag(input.val(), false); @@ -457,6 +522,10 @@ function setMatch() { nEditor.setMatch($('#match').val()); } +/** Method triggering update in refstr text box in needle editor instance. */ +function updateRefstr() { + nEditor.updateRefstr(); +} function toggleClickCoordinates() { updateToggleClickCoordinatesButton(nEditor.toggleClickCoordinates()); } diff --git a/docs/Contributing.asciidoc b/docs/Contributing.asciidoc index 21a718c954e..3599aaf46e0 100644 --- a/docs/Contributing.asciidoc +++ b/docs/Contributing.asciidoc @@ -135,7 +135,7 @@ In the case of os-autoinst, only a few http://www.cpan.org/[CPAN] modules are required. Basically `Carp::Always`, `Data::Dump`. `JSON` and `YAML`. On the other hand, several external tools are needed including http://wiki.qemu.org/Main_Page[QEMU], -https://code.google.com/p/tesseract-ocr/[Tesseract] and +https://www-e.uni-magdeburg.de/jschulen/ocr/[GOCR] and http://optipng.sourceforge.net/[OptiPNG]. Last but not least, the http://opencv.org/[OpenCV] library is the core of the openQA image matching mechanism, so it must be available on the system. diff --git a/docs/GettingStarted.asciidoc b/docs/GettingStarted.asciidoc index 5e97f34fb1e..19438bb55c1 100644 --- a/docs/GettingStarted.asciidoc +++ b/docs/GettingStarted.asciidoc @@ -193,14 +193,14 @@ information and results (if any) are kept for future reference. One of the main mechanisms for openQA to know the state of the virtual machine is checking the presence of some elements in the machine's 'screen'. -This is performed using fuzzy image matching between the screen and the so -called 'needles'. A needle specifies both the elements to search for and a +This is performed matching a reference (so called 'needle') with the screen. +A needle specifies both the elements to search for and a list of tags used to decide which needles should be used at any moment. -A needle consists of a full screenshot in PNG format and a json file with -the same name (e.g. foo.png and foo.json) containing the associated data, like -which areas inside the full screenshot are relevant or the mentioned list of -tags. +A needle consists of at least a JSON file and, optionally, a full screenshot +in PNG format with the same name (e.g. foo.png and foo.json). The JSON file +contains the associated data, like which areas inside the full screenshot are +relevant and the mentioned list of tags. [source,json] ------------------------------------------------------------------- @@ -212,7 +212,8 @@ tags. "width" : INTEGER, "height" : INTEGER, "type" : ( "match" | "ocr" | "exclude" ), - "match" : INTEGER, // 0-100. similarity percentage + "match" : INTEGER, // 0-100. similarity percentage, + "refstr": STRING }, ... ], @@ -229,11 +230,11 @@ There are three kinds of areas: with at least the specified similarity percentage. Regular areas are displayed as green boxes in the needle editor and as green or red frames in the needle view (green for matching areas, red for non-matching ones). -* *OCR areas* also define relevant parts of the screenshot. However, an OCR - algorithm is used for matching. In the needle editor OCR areas are +* *OCR areas* also define relevant parts of the screenshot. They are + converted to text in order to be matched on an OCR reference text. The + reference text is stored in the needle. In the needle editor OCR areas are displayed as orange boxes. To turn a regular area into an OCR area within - the needle editor, double click the concerning area twice. Note that such - needles are only rarely used. + the needle editor, double click the concerning area twice. * *Exclude areas* can be used to ignore parts of the reference picture. In the needle editor exclude areas are displayed as red boxes. To turn a regular area into an exclude area within the needle editor, double click diff --git a/lib/OpenQA/Schema/Result/Jobs.pm b/lib/OpenQA/Schema/Result/Jobs.pm index 7db6c63c548..5af589bef8a 100644 --- a/lib/OpenQA/Schema/Result/Jobs.pm +++ b/lib/OpenQA/Schema/Result/Jobs.pm @@ -1029,7 +1029,7 @@ sub append_log ($self, $log, $file_name) { my $path = $self->worker->get_property('WORKER_TMPDIR'); return unless -d $path; # we can't help $path .= "/$file_name"; - if (open(my $fd, '>>', $path)) { + if (open(my $fd, '>>:utf8', $path)) { print $fd $log->{data}; close($fd); } diff --git a/lib/OpenQA/Task/Needle/Save.pm b/lib/OpenQA/Task/Needle/Save.pm index 4654d977c8e..d149eba9daa 100644 --- a/lib/OpenQA/Task/Needle/Save.pm +++ b/lib/OpenQA/Task/Needle/Save.pm @@ -31,8 +31,6 @@ sub _json_validation { if (!exists $djson->{tags} || !exists $djson->{tags}[0]) { die 'no tag defined'; } - my @not_ocr_area = grep { $_->{type} ne 'ocr' } @{$djson->{area}}; - die 'Cannot create a needle with only OCR areas' if scalar(@not_ocr_area) == 0; my $areas = $djson->{area}; foreach my $area (@$areas) { @@ -41,6 +39,11 @@ sub _json_validation { die 'area without type' unless exists $area->{type}; die 'area without height' unless exists $area->{height}; die 'area without width' unless exists $area->{width}; + if ($area->{type} eq 'ocr') { + die 'OCR area without refstr' unless defined $area->{refstr}; + die 'refstr is an empty string' if ($area->{refstr} eq ''); + die 'refstr contains placeholder char §' if ($area->{refstr} =~ qr/§/); + } } return $djson; } @@ -78,21 +81,25 @@ sub _save_needle { return $minion_job->finish({error => "Failed to validate $needlename.
$error"}); } - # determine imagepath + my @match_areas = grep { $_->{type} eq 'match' } @{$json_data->{area}}; + my $imagepath; - if ($imagedir) { - $imagepath = join('/', $imagedir, $imagename); - } - elsif ($imagedistri) { - $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename); - } - else { - $imagepath = join('/', $openqa_job->result_dir(), $imagename); - } - if (!-f $imagepath) { - my $error = "Image $imagename could not be found!"; - $app->log->error("Failed to save needle: $error"); - return $minion_job->fail({error => "Failed to save $needlename.
$error"}); + if (@match_areas) { + # determine imagepath + if ($imagedir) { + $imagepath = join('/', $imagedir, $imagename); + } + elsif ($imagedistri) { + $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename); + } + else { + $imagepath = join('/', $openqa_job->result_dir(), $imagename); + } + if (!-f $imagepath) { + my $error = "Image $imagename could not be found!"; + $app->log->error("Failed to save needle: $error"); + return $minion_job->fail({error => "Failed to save $needlename.
$error"}); + } } # check whether needle directory actually exists @@ -112,17 +119,19 @@ sub _save_needle { # do not overwrite the exist needle if disallow to overwrite my $baseneedle = "$needledir/$needlename"; - if (-e "$baseneedle.png" && !$args->{overwrite}) { + if (-e "$baseneedle.json" && !$args->{overwrite}) { #my $returned_data = $self->req->params->to_hash; #$returned_data->{requires_overwrite} = 1; return $minion_job->finish({requires_overwrite => 1}); } - # copy image my $success = 1; - if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) { - $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!"); - $success = 0; + if (@match_areas) { + # copy image + if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) { + $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!"); + $success = 0; + } } if ($success) { open(my $J, ">", "$baseneedle.json") or $success = 0; @@ -138,9 +147,11 @@ sub _save_needle { # commit needle in Git repository if ($git->enabled) { + my @files_to_be_comm = ["$needlename.json"]; + push(@files_to_be_comm, "$needlename.png") if (@match_areas); my $error = $git->commit( { - add => ["$needlename.json", "$needlename.png"], + add => @files_to_be_comm, message => ($commit_message || sprintf("%s for %s", $needlename, $openqa_job->name)), }); if ($error) { diff --git a/lib/OpenQA/WebAPI/Controller/Running.pm b/lib/OpenQA/WebAPI/Controller/Running.pm index 65e057b3dce..f71c9c8b358 100644 --- a/lib/OpenQA/WebAPI/Controller/Running.pm +++ b/lib/OpenQA/WebAPI/Controller/Running.pm @@ -93,7 +93,7 @@ sub streamtext ($self, $file_name, $start_hook = undef, $close_hook = undef) { # if the open fails, continue, well check later my $log; my ($ino, $size); - if (open($log, '<', $logfile)) { + if (open($log, '<:utf8', $logfile)) { # Send the last 10KB of data from the logfile, so that # the client sees some data immediately $ino = (stat $logfile)[1]; @@ -122,7 +122,7 @@ sub streamtext ($self, $file_name, $start_hook = undef, $close_hook = undef) { TEXT_STREAMING_INTERVAL() => sub { if (!$ino) { # log file was not yet opened - return unless open($log, '<', $logfile); + return unless open($log, '<:utf8', $logfile); $ino = (stat $logfile)[1]; $size = -s $logfile; } diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index 21c11b6c067..8d5623a9b24 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -263,8 +263,9 @@ sub _new_screenshot ($self, $tags, $image_name, $matches = undef) { ypos => int $area->{y}, width => int $area->{w}, height => int $area->{h}, - type => 'match', ); + $match{type} = defined $area->{refstr} ? 'ocr' : 'match'; + $match{refstr} = $area->{refstr} if defined $area->{refstr}; if (my $click_point = $area->{click_point}) { $match{click_point} = $click_point; } @@ -286,7 +287,7 @@ sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_di my $pngfile = File::Spec->catpath('', $needles_dir, $png_fname); $needle->{needledir} = $needles_dir; - $needle->{image} = $pngfile; + $needle->{image} = $pngfile if (-f $pngfile); $needle->{json} = $file_name; $needle->{name} = $name; $needle->{distri} = $distri; @@ -314,12 +315,14 @@ sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $needle_info->{title} = $needle_name; $needle_info->{suggested_name} = ensure_timestamp_appended($needle_name); - $needle_info->{imageurl} - = $self->needle_url($distri, $needle_name . '.png', $version, $needle_info->{json})->to_string(); - $needle_info->{imagename} = basename($needle_info->{image}); - $needle_info->{imagedir} = dirname($needle_info->{image}); - $needle_info->{imagedistri} = $distri; - $needle_info->{imageversion} = $version; + if ($needle_info->{image}) { + $needle_info->{imageurl} + = $self->needle_url($distri, $needle_name . '.png', $version, $needle_info->{json})->to_string(); + $needle_info->{imagename} = basename($needle_info->{image}); + $needle_info->{imagedir} = dirname($needle_info->{image}); + $needle_info->{imagedistri} = $distri; + $needle_info->{imageversion} = $version; + } $needle_info->{tags} //= []; $needle_info->{matches} //= []; $needle_info->{properties} //= []; @@ -388,6 +391,10 @@ sub save_needle_ajax ($self) { my $job_id = $job->id; my $needledir = needledir($job->DISTRI, $job->VERSION); my $needlename = $validation->param('needlename'); + my $needle_json = $validation->param('json'); + + # The json data came from an HTML form. Might contain carriage return. + $needle_json =~ s/\r\n/\n/g; $self->gru->enqueue_and_keep_track( task_name => 'save_needle', @@ -395,7 +402,7 @@ sub save_needle_ajax ($self) { task_args => { job_id => $job_id, user_id => $self->current_user->id, - needle_json => $validation->param('json'), + needle_json => $needle_json, overwrite => $self->param('overwrite'), imagedir => $self->param('imagedir') // '', imagedistri => $validation->param('imagedistri'), @@ -449,6 +456,7 @@ sub calc_matches ($needle, $areas) { type => $area->{result}, similarity => int($area->{similarity} + 0.5), ); + $match{ocr_str} = $area->{ocr_str} if defined $area->{ocr_str}; if (my $click_point = $area->{click_point}) { $match{click_point} = $click_point; } @@ -458,6 +466,45 @@ sub calc_matches ($needle, $areas) { return; } +=head2 has_image + +Checks if a needle has a needle image by it's type of areas. + +=head3 Params + +=over + +=item * + +areas of object type ARRAY containing HASH objects containing at least: + +=over + +=item * + +type => s in {'ocr', 'match', 'exclude'} | type is the area type. + +=back + +=back + +=head3 returns + +i in {1, 0} | i is 1 if the area has an image. + +=cut + +sub has_image ($areas) { + my $ocr_area; + my $img_matching_area; + + for my $area (@$areas) { + $ocr_area = 1 if $area->{type} eq 'ocr'; + $img_matching_area = 1 if $area->{type} eq 'match'; + } + return $ocr_area ? $img_matching_area : 1; +} + sub viewimg ($self) { my $module_detail = $self->stash('module_detail'); my $job = $self->stash('job'); @@ -505,13 +552,14 @@ sub viewimg ($self) { my $info = { name => $needle, needledir => $needleinfo->{needledir}, - image => $self->needle_url($distri, $needle . '.png', $dversion, $needleinfo->{json}), areas => $needleinfo->{area}, error => $module_detail->{error}, matches => [], primary_match => 1, selected => 1, }; + $info->{image} = $self->needle_url($distri, $needle . '.png', $dversion, $needleinfo->{json}) + if has_image($needleinfo->{area}); calc_matches($info, $module_detail->{area}); $primary_match = $info; $append_needle_info->($needleinfo->{tags} => $info); @@ -527,11 +575,12 @@ sub viewimg ($self) { my $info = { name => $needlename, needledir => $needleinfo->{needledir}, - image => $self->needle_url($distri, "$needlename.png", $dversion, $needleinfo->{json}), error => $needle->{error}, areas => $needleinfo->{area}, matches => [], }; + $info->{image} = $self->needle_url($distri, "$needlename.png", $dversion, $needleinfo->{json}) + if has_image($needleinfo->{area}); calc_matches($info, $needle->{area}); $append_needle_info->($needleinfo->{tags} => $info); } diff --git a/lib/OpenQA/Worker/Job.pm b/lib/OpenQA/Worker/Job.pm index c49cb364f2b..49b40518a08 100644 --- a/lib/OpenQA/Worker/Job.pm +++ b/lib/OpenQA/Worker/Job.pm @@ -25,6 +25,7 @@ use Try::Tiny; use Scalar::Util 'looks_like_number'; use File::Map 'map_file'; use List::Util 'max'; +use Encode 'decode'; # define attributes for public properties has 'worker'; @@ -1238,7 +1239,7 @@ sub _log_snippet { sysseek($fd, $offset, Fcntl::SEEK_SET); # FIXME: handle error? if (defined sysread($fd, my $buf = '', 100000)) { $ret{offset} = $offset; - $ret{data} = $buf; + $ret{data} = decode('utf-8', $buf); } if (my $new_offset = sysseek($fd, 0, Fcntl::SEEK_CUR)) { $self->{"_$offset_name"} = $new_offset; diff --git a/templates/webapi/step/edit.html.ep b/templates/webapi/step/edit.html.ep index ad4ab1b9052..7f6dc83033d 100644 --- a/templates/webapi/step/edit.html.ep +++ b/templates/webapi/step/edit.html.ep @@ -220,6 +220,7 @@ Take image from: