Skip to content

Commit bdf3452

Browse files
s1hofmanns1hofmann
authored
Merge pull request #81 from nut-tree/feature/68/scaled_search
Closes #68
2 parents 36629e7 + 10afb84 commit bdf3452

19 files changed

+460
-178
lines changed

Diff for: ‎lib/provider/opencv/__mocks__/needle.png

8.85 KB
Loading

Diff for: ‎lib/provider/opencv/bound-value.function.spec.ts

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { lowerBound, upperBound } from "./bound-value.function";
2+
3+
describe("lowerBound function", () => {
4+
it.each([
5+
[5, 10, 1, 1],
6+
[5, 5, 10, 10],
7+
[5, 1, 10, 5],
8+
[0, 0, 0, 0]
9+
])("Input: %f, Boundary: %f, minValue: %f, Expected: %f",
10+
(input: number, boundary: number, minValue: number, expected: number) => {
11+
// WHEN
12+
const result = lowerBound(input, boundary, minValue);
13+
14+
// THEN
15+
expect(result).toBe(expected);
16+
});
17+
});
18+
19+
describe("upperBound function", () => {
20+
it.each([
21+
[5, 10, 1, 5],
22+
[5, 5, 10, 10],
23+
[5, 1, 10, 10],
24+
[5, 5, 5, 5]
25+
])("Input: %f, Boundary: %f, maxValue: %f, Expected: %f",
26+
(input: number, boundary: number, maxValue: number, expected: number) => {
27+
// WHEN
28+
const result = upperBound(input, boundary, maxValue);
29+
30+
// THEN
31+
expect(result).toBe(expected);
32+
});
33+
});

Diff for: ‎lib/provider/opencv/bound-value.function.ts

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
export function lowerBound(value: number, boundary: number, minValue: number): number {
2+
return (value <= boundary) ? minValue : value;
3+
}
4+
5+
export function upperBound(value: number, boundary: number, maxValue: number): number {
6+
return (value >= boundary) ? maxValue : value;
7+
}

Diff for: ‎lib/provider/opencv/determine-searchregion.function.spec.ts

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { mockPartial } from "sneer";
2+
import { Image } from "../../image.class";
3+
import { MatchRequest } from "../../match-request.class";
4+
import { Region } from "../../region.class";
5+
import { determineScaledSearchRegion } from "./determine-searchregion.function";
6+
7+
describe("determineSearchRegion", () => {
8+
it("should return a search region adopted to pixel density", () => {
9+
// GIVEN
10+
const imageMock = mockPartial<Image>({
11+
pixelDensity: {
12+
scaleX: 1.5,
13+
scaleY: 2.0
14+
}
15+
});
16+
const needlePath = "/path/to/needle";
17+
const inputSearchRegion = new Region(0, 0, 100, 100);
18+
const expectedSearchRegion = new Region(0, 0, 150, 200);
19+
20+
const matchRequest = new MatchRequest(
21+
imageMock,
22+
needlePath,
23+
inputSearchRegion,
24+
0.99
25+
);
26+
27+
// WHEN
28+
const result = determineScaledSearchRegion(matchRequest);
29+
30+
// THEN
31+
expect(result).toEqual(expectedSearchRegion);
32+
});
33+
34+
it.each([[0, 1], [1, 0]])("should not adjust searchregion for factor 0: scaleX: %i scaleY: %i",
35+
(scaleX: number, scaleY: number) => {
36+
// GIVEN
37+
const imageMock = mockPartial<Image>({
38+
pixelDensity: {
39+
scaleX,
40+
scaleY
41+
}
42+
});
43+
const needlePath = "/path/to/needle";
44+
const inputSearchRegion = new Region(0, 0, 100, 100);
45+
const expectedSearchRegion = new Region(0, 0, 100, 100);
46+
47+
const matchRequest = new MatchRequest(
48+
imageMock,
49+
needlePath,
50+
inputSearchRegion,
51+
0.99
52+
);
53+
54+
// WHEN
55+
const result = determineScaledSearchRegion(matchRequest);
56+
57+
// THEN
58+
expect(result).toEqual(expectedSearchRegion);
59+
});
60+
});

Diff for: ‎lib/provider/opencv/determine-searchregion.function.ts

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { MatchRequest } from "../../match-request.class";
2+
import { Region } from "../../region.class";
3+
4+
export function determineScaledSearchRegion(matchRequest: MatchRequest): Region {
5+
const searchRegion = matchRequest.searchRegion;
6+
const scaleX = matchRequest.haystack.pixelDensity.scaleX || 1.0;
7+
const scaleY = matchRequest.haystack.pixelDensity.scaleY || 1.0;
8+
searchRegion.width *= scaleX;
9+
searchRegion.height *= scaleY;
10+
return searchRegion;
11+
}

Diff for: ‎lib/provider/opencv/find-edges.function.spec.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import * as cv from "opencv4nodejs-prebuilt";
2+
import { mockPartial } from "sneer";
3+
import { findEdges } from "./find-edges.function";
4+
5+
describe("findEdges", () => {
6+
it("should convert an image to grayscale and run Canny edge detection", async () => {
7+
// GIVEN
8+
const grayImageMock = mockPartial<cv.Mat>({
9+
cannyAsync: jest.fn()
10+
});
11+
const inputImageMock = mockPartial<cv.Mat>({
12+
cvtColorAsync: jest.fn(() => Promise.resolve(grayImageMock))
13+
});
14+
15+
// WHEN
16+
await findEdges(inputImageMock);
17+
18+
// THEN
19+
expect(inputImageMock.cvtColorAsync).toBeCalledWith(cv.COLOR_BGR2GRAY);
20+
expect(grayImageMock.cannyAsync).toBeCalledWith(50, 200);
21+
});
22+
});

Diff for: ‎lib/provider/opencv/find-edges.function.ts

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import * as cv from "opencv4nodejs-prebuilt";
2+
3+
export async function findEdges(image: cv.Mat): Promise<cv.Mat> {
4+
const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY);
5+
return gray.cannyAsync(50, 200);
6+
}

Diff for: ‎lib/provider/opencv/image-processor.class.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@ import * as cv from "opencv4nodejs-prebuilt";
22
import { Image } from "../../image.class";
33
import { Region } from "../../region.class";
44

5-
const determineROI = (img: Image, roi: Region): cv.Rect => {
5+
function determineROI(img: Image, roi: Region): cv.Rect {
66
return new cv.Rect(
77
Math.min(Math.max(roi.left, 0), img.width),
88
Math.min(Math.max(roi.top, 0), img.height),
99
Math.min(roi.width, img.width - roi.left),
1010
Math.min(roi.height, img.height - roi.top));
11-
};
11+
}
1212

1313
export class ImageProcessor {
1414
/**

Diff for: ‎lib/provider/opencv/match-image.function.spec.ts

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import * as cv from "opencv4nodejs-prebuilt";
2+
import { mockPartial } from "sneer";
3+
import { matchImages } from "./match-image.function";
4+
5+
describe("matchImages", () => {
6+
it("should return minLoc position and needle size", async () => {
7+
// GIVEN
8+
const minLocX = 100;
9+
const minLocY = 1000;
10+
const matchMock = mockPartial<cv.Mat>({
11+
minMaxLocAsync: jest.fn(() => Promise.resolve({
12+
maxLoc: new cv.Point2(
13+
200,
14+
2000
15+
),
16+
maxVal: 100,
17+
minLoc: new cv.Point2(
18+
minLocX,
19+
minLocY
20+
),
21+
minVal: 0,
22+
}))
23+
});
24+
const haystackMock = mockPartial<cv.Mat>({
25+
matchTemplateAsync: jest.fn(() => Promise.resolve(matchMock))
26+
});
27+
const needleMock = mockPartial<cv.Mat>({
28+
cols: 123,
29+
rows: 456
30+
});
31+
32+
// WHEN
33+
const result = await matchImages(haystackMock, needleMock);
34+
35+
// THEN
36+
expect(result.location.left).toEqual(minLocX);
37+
expect(result.location.top).toEqual(minLocY);
38+
expect(result.location.width).toEqual(needleMock.cols);
39+
expect(result.location.height).toEqual(needleMock.rows);
40+
});
41+
});

Diff for: ‎lib/provider/opencv/match-image.function.ts

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import * as cv from "opencv4nodejs-prebuilt";
2+
import { MatchResult } from "../../match-result.class";
3+
import { Region } from "../../region.class";
4+
5+
export async function matchImages(haystack: cv.Mat, needle: cv.Mat): Promise<MatchResult> {
6+
const match = await haystack.matchTemplateAsync(
7+
needle,
8+
cv.TM_SQDIFF_NORMED,
9+
);
10+
const minMax = await match.minMaxLocAsync();
11+
return new MatchResult(
12+
1.0 - minMax.minVal,
13+
new Region(
14+
minMax.minLoc.x,
15+
minMax.minLoc.y,
16+
needle.cols,
17+
needle.rows,
18+
),
19+
);
20+
}

Diff for: ‎lib/provider/opencv/scale-image.function.spec.ts

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import * as path from "path";
2+
import { ImageProcessor } from "./image-processor.class";
3+
import { ImageReader } from "./image-reader.class";
4+
import { scaleImage } from "./scale-image.function";
5+
6+
describe("scaleImage", () => {
7+
it.each([[0.5], [1.5]])("should scale an image correctly by factor %f", async (scaleFactor) => {
8+
// GIVEN
9+
const imageLoader = new ImageReader();
10+
const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png");
11+
const inputImage = await imageLoader.load(pathToinput);
12+
const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage);
13+
const expectedWidth = Math.floor(inputMat.cols * scaleFactor);
14+
const expectedHeight = Math.floor(inputMat.rows * scaleFactor);
15+
16+
// WHEN
17+
const result = await scaleImage(inputMat, scaleFactor);
18+
19+
// THEN
20+
expect(result.rows).toBe(expectedHeight);
21+
expect(result.cols).toBe(expectedWidth);
22+
});
23+
24+
it.each([[0], [-0.25]])("should keep scale if factor <= 0: Scale %f", async (scaleFactor) => {
25+
// GIVEN
26+
const imageLoader = new ImageReader();
27+
const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png");
28+
const inputImage = await imageLoader.load(pathToinput);
29+
const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage);
30+
const expectedWidth = inputMat.cols;
31+
const expectedHeight = inputMat.rows;
32+
33+
// WHEN
34+
const result = await scaleImage(inputMat, scaleFactor);
35+
36+
// THEN
37+
expect(result.rows).toBe(expectedHeight);
38+
expect(result.cols).toBe(expectedWidth);
39+
});
40+
});

Diff for: ‎lib/provider/opencv/scale-image.function.ts

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import * as cv from "opencv4nodejs-prebuilt";
2+
import { lowerBound } from "./bound-value.function";
3+
4+
export async function scaleImage(image: cv.Mat, scaleFactor: number): Promise<cv.Mat> {
5+
const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0);
6+
const scaledRows = Math.floor(image.rows * boundScaleFactor);
7+
const scaledCols = Math.floor(image.cols * boundScaleFactor);
8+
return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA);
9+
}

Diff for: ‎lib/provider/opencv/scale-location.function.spec.ts

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { Region } from "../../region.class";
2+
import { scaleLocation } from "./scale-location.function";
3+
4+
describe("scaleLocation", () => {
5+
it("should scale location of a Region for valid scale factors", () => {
6+
// GIVEN
7+
const scaleFactor = 0.5;
8+
const inputRegion = new Region(100, 100, 10, 10);
9+
const expectedRegion = new Region(200, 200, 10, 10);
10+
11+
// WHEN
12+
const result = scaleLocation(inputRegion, scaleFactor);
13+
14+
// THEN
15+
expect(result).toEqual(expectedRegion);
16+
});
17+
18+
it("should not scale location of a Region for invalid scale factors", () => {
19+
// GIVEN
20+
const scaleFactor = 0.0;
21+
const inputRegion = new Region(100, 100, 10, 10);
22+
const expectedRegion = new Region(100, 100, 10, 10);
23+
24+
// WHEN
25+
const result = scaleLocation(inputRegion, scaleFactor);
26+
27+
// THEN
28+
expect(result).toEqual(expectedRegion);
29+
});
30+
});

Diff for: ‎lib/provider/opencv/scale-location.function.ts

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { Region } from "../../region.class";
2+
import { lowerBound } from "./bound-value.function";
3+
4+
export function scaleLocation(
5+
result: Region,
6+
scaleFactor: number,
7+
): Region {
8+
const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0);
9+
return new Region(
10+
result.left / boundScaleFactor,
11+
result.top / boundScaleFactor,
12+
result.width,
13+
result.height,
14+
);
15+
}

Diff for: ‎lib/provider/opencv/template-matching-finder.class.spec.ts

+16-12
Original file line numberDiff line numberDiff line change
@@ -10,38 +10,42 @@ describe("Template-matching finder", () => {
1010
// GIVEN
1111
const imageLoader = new ImageReader();
1212
const SUT = new TemplateMatchingFinder();
13-
const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png");
14-
const needle = await imageLoader.load(imagePath);
13+
const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png");
14+
const needlePath = path.resolve(__dirname, "./__mocks__/needle.png");
15+
const haystack = await imageLoader.load(haystackPath);
16+
const needle = await imageLoader.load(needlePath);
1517
const minConfidence = 0.99;
16-
const searchRegion = new Region(0, 0, needle.width, needle.height);
17-
const haystack = new Image(needle.width, needle.height, needle.data, 3);
18-
const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence);
18+
const searchRegion = new Region(0, 0, haystack.width, haystack.height);
19+
const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence);
20+
const expectedResult = new Region(16, 31, needle.width, needle.height);
1921

2022
// WHEN
2123
const result = await SUT.findMatch(matchRequest);
2224

2325
// THEN
2426
expect(result.confidence).toBeGreaterThanOrEqual(minConfidence);
25-
expect(result.location).toEqual(searchRegion);
27+
expect(result.location).toEqual(expectedResult);
2628
});
2729

2830
it("findMatch should return a match within a search region when present in image", async () => {
2931
// GIVEN
3032
const imageLoader = new ImageReader();
3133
const SUT = new TemplateMatchingFinder();
32-
const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png");
33-
const needle = await imageLoader.load(imagePath);
34+
const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png");
35+
const needlePath = path.resolve(__dirname, "./__mocks__/needle.png");
36+
const haystack = await imageLoader.load(haystackPath);
37+
const needle = await imageLoader.load(needlePath);
3438
const minConfidence = 0.99;
35-
const searchRegion = new Region(10, 20, 100, 100);
36-
const haystack = new Image(needle.width, needle.height, needle.data, 3);
37-
const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence);
39+
const searchRegion = new Region(10, 20, 140, 100);
40+
const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence);
41+
const expectedResult = new Region(6, 11, needle.width, needle.height);
3842

3943
// WHEN
4044
const result = await SUT.findMatch(matchRequest);
4145

4246
// THEN
4347
expect(result.confidence).toBeGreaterThanOrEqual(minConfidence);
44-
expect(result.location).toEqual(searchRegion);
48+
expect(result.location).toEqual(expectedResult);
4549
});
4650

4751
it("findMatch should throw on invalid image paths", async () => {

0 commit comments

Comments
 (0)