Skip to content

Commit db54f92

Browse files
s1hofmanns1hofmann
authored
Feature/500/color finder (#501)
* (#500) Define color finder interface * (#500) Implement color finder interface + tests * (#500) Add new query kind ColorQuery + test * (#500) Introduce color finder in provider registry * (#500) Introduce and export new query function `pixelWithColor` * (#500) Make both match-request.class.ts and match-result.class.ts generic such that they can work with different types * (#500) Provide types to now generic MatchResult * (#500) Add support for new color finder and match results which return results other than Region * (#500) Add test for the new color-finder queries in screen.class.spec.ts * (#500) Add test for the new color-finder queries in screen.class.spec.ts * (#500) Add test for the new color-finder queries in screen.class.spec.ts
1 parent 7471b61 commit db54f92

16 files changed

+657
-150
lines changed

Diff for: ‎e2e/assets/dot.png

413 Bytes
Loading

Diff for: ‎e2e/assets/dots.png

424 Bytes
Loading

Diff for: ‎index.ts

+18-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@ import { LineHelper } from "./lib/util/linehelper.class";
88
import { createWindowApi } from "./lib/window.function";
99
import providerRegistry from "./lib/provider/provider-registry.class";
1010
import { loadImageResource } from "./lib/imageResources.function";
11-
import { LineQuery, WindowQuery, WordQuery } from "./lib/query.class";
11+
import {
12+
ColorQuery,
13+
LineQuery,
14+
WindowQuery,
15+
WordQuery,
16+
} from "./lib/query.class";
17+
import { RGBA } from "./lib/rgba.class";
1218

1319
export {
1420
AssertClass,
@@ -100,6 +106,16 @@ const windowWithTitle = (title: string | RegExp): WindowQuery => {
100106
};
101107
};
102108

109+
const pixelWithColor = (color: RGBA): ColorQuery => {
110+
return {
111+
type: "color",
112+
id: `pixel-by-color-query-RGBA(${color.R},${color.G},${color.B},${color.A})`,
113+
by: {
114+
color,
115+
},
116+
};
117+
};
118+
103119
export { fetchFromUrl } from "./lib/imageResources.function";
104120

105121
export {
@@ -121,4 +137,5 @@ export {
121137
singleWord,
122138
textLine,
123139
windowWithTitle,
140+
pixelWithColor,
124141
};

Diff for: ‎lib/match-request.class.ts

+35-12
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
import { Image, isImage } from "./image.class";
22
import { Region } from "./region.class";
33
import { OptionalSearchParameters } from "./optionalsearchparameters.class";
4-
import { isLineQuery, isTextQuery, TextQuery } from "./query.class";
5-
import { RegionResultFindInput } from "./screen.class";
4+
import {
5+
ColorQuery,
6+
isColorQuery,
7+
isLineQuery,
8+
isTextQuery,
9+
TextQuery,
10+
} from "./query.class";
611
import { ProviderRegistry } from "./provider/provider-registry.class";
12+
import { PointResultFindInput, RegionResultFindInput } from "./screen.class";
713

814
export class MatchRequest<NEEDLE_TYPE, PROVIDER_DATA_TYPE> {
915
public constructor(
@@ -26,42 +32,48 @@ export function isTextMatchRequest<PROVIDER_DATA_TYPE>(
2632
return isTextQuery(matchRequest.needle);
2733
}
2834

35+
export function isColorMatchRequest<PROVIDER_DATA_TYPE>(
36+
matchRequest: any
37+
): matchRequest is MatchRequest<ColorQuery, PROVIDER_DATA_TYPE> {
38+
return isColorQuery(matchRequest.needle);
39+
}
40+
2941
export function createMatchRequest<PROVIDER_DATA_TYPE>(
3042
providerRegistry: ProviderRegistry,
31-
needle: TextQuery | Promise<TextQuery>,
43+
needle: PointResultFindInput,
3244
searchRegion: Region,
3345
minMatch: number,
3446
screenImage: Image,
3547
params?: OptionalSearchParameters<PROVIDER_DATA_TYPE>
36-
): MatchRequest<TextQuery, PROVIDER_DATA_TYPE>;
48+
): MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>;
3749
export function createMatchRequest<PROVIDER_DATA_TYPE>(
3850
providerRegistry: ProviderRegistry,
39-
needle: Image | Promise<Image>,
51+
needle: RegionResultFindInput,
4052
searchRegion: Region,
4153
minMatch: number,
4254
screenImage: Image,
4355
params?: OptionalSearchParameters<PROVIDER_DATA_TYPE>
44-
): MatchRequest<Image, PROVIDER_DATA_TYPE>;
56+
): MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>;
4557
export function createMatchRequest<PROVIDER_DATA_TYPE>(
4658
providerRegistry: ProviderRegistry,
47-
needle: RegionResultFindInput | Promise<RegionResultFindInput>,
59+
needle: RegionResultFindInput | PointResultFindInput,
4860
searchRegion: Region,
4961
minMatch: number,
5062
screenImage: Image,
5163
params?: OptionalSearchParameters<PROVIDER_DATA_TYPE>
5264
):
53-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
54-
| MatchRequest<Image, PROVIDER_DATA_TYPE>;
65+
| MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
66+
| MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>;
5567
export function createMatchRequest<PROVIDER_DATA_TYPE>(
5668
providerRegistry: ProviderRegistry,
57-
needle: RegionResultFindInput | Promise<RegionResultFindInput>,
69+
needle: RegionResultFindInput | PointResultFindInput,
5870
searchRegion: Region,
5971
minMatch: number,
6072
screenImage: Image,
6173
params?: OptionalSearchParameters<PROVIDER_DATA_TYPE>
6274
):
63-
| MatchRequest<Image, PROVIDER_DATA_TYPE>
64-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE> {
75+
| MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
76+
| MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE> {
6577
if (isImage(needle)) {
6678
providerRegistry
6779
.getLogProvider()
@@ -90,6 +102,17 @@ export function createMatchRequest<PROVIDER_DATA_TYPE>(
90102
minMatch,
91103
params?.providerData
92104
);
105+
} else if (isColorQuery(needle)) {
106+
const color = needle.by.color;
107+
providerRegistry
108+
.getLogProvider()
109+
.info(
110+
`Searching for color RGBA(${color.R},${color.G},${color.B},${
111+
color.A
112+
}) in region ${searchRegion.toString()}.`
113+
);
114+
115+
return new MatchRequest(screenImage, needle, 1, params?.providerData);
93116
}
94117
throw new Error(`Unknown input type: ${JSON.stringify(needle)}`);
95118
}

Diff for: ‎lib/match-result.class.ts

+42-37
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,68 @@
11
import { Region } from "./region.class";
2-
import { isImageMatchRequest, MatchRequest } from "./match-request.class";
2+
import {
3+
isColorMatchRequest,
4+
isImageMatchRequest,
5+
isTextMatchRequest,
6+
MatchRequest,
7+
} from "./match-request.class";
38
import { ProviderRegistry } from "./provider/provider-registry.class";
4-
import { Image } from "./image.class";
5-
import { TextQuery } from "./query.class";
9+
import { Point } from "./point.class";
10+
import { PointResultFindInput, RegionResultFindInput } from "./screen.class";
611

7-
export class MatchResult {
12+
export class MatchResult<LOCATION_TYPE> {
813
constructor(
914
public readonly confidence: number,
10-
public readonly location: Region,
15+
public readonly location: LOCATION_TYPE,
1116
public readonly error?: Error
1217
) {}
1318
}
1419

1520
export async function getMatchResults<PROVIDER_DATA_TYPE>(
1621
providerRegistry: ProviderRegistry,
17-
matchRequest: MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
18-
): Promise<MatchResult[]>;
22+
matchRequest: MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
23+
): Promise<MatchResult<Region>[]>;
1924
export async function getMatchResults<PROVIDER_DATA_TYPE>(
2025
providerRegistry: ProviderRegistry,
21-
matchRequest: MatchRequest<Image, PROVIDER_DATA_TYPE>
22-
): Promise<MatchResult[]>;
26+
matchRequest: MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>
27+
): Promise<MatchResult<Point>[]>;
2328
export async function getMatchResults<PROVIDER_DATA_TYPE>(
2429
providerRegistry: ProviderRegistry,
2530
matchRequest:
26-
| MatchRequest<Image, PROVIDER_DATA_TYPE>
27-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
28-
): Promise<MatchResult[]>;
29-
export async function getMatchResults<PROVIDER_DATA_TYPE>(
30-
providerRegistry: ProviderRegistry,
31-
matchRequest:
32-
| MatchRequest<Image, PROVIDER_DATA_TYPE>
33-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
34-
): Promise<MatchResult[]> {
35-
return isImageMatchRequest(matchRequest)
36-
? providerRegistry.getImageFinder().findMatches(matchRequest)
37-
: providerRegistry.getTextFinder().findMatches(matchRequest);
31+
| MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
32+
| MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>
33+
): Promise<MatchResult<Point | Region>[]> {
34+
if (isImageMatchRequest(matchRequest)) {
35+
return providerRegistry.getImageFinder().findMatches(matchRequest);
36+
} else if (isTextMatchRequest(matchRequest)) {
37+
return providerRegistry.getTextFinder().findMatches(matchRequest);
38+
} else if (isColorMatchRequest(matchRequest)) {
39+
return providerRegistry.getColorFinder().findMatches(matchRequest);
40+
}
41+
throw new Error(
42+
`Unknown match request type: ${JSON.stringify(matchRequest.needle)}`
43+
);
3844
}
3945

4046
export async function getMatchResult<PROVIDER_DATA_TYPE>(
4147
providerRegistry: ProviderRegistry,
42-
matchRequest: MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
43-
): Promise<MatchResult>;
48+
matchRequest: MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
49+
): Promise<MatchResult<Region>>;
4450
export async function getMatchResult<PROVIDER_DATA_TYPE>(
4551
providerRegistry: ProviderRegistry,
46-
matchRequest: MatchRequest<Image, PROVIDER_DATA_TYPE>
47-
): Promise<MatchResult>;
48-
export async function getMatchResult<PROVIDER_DATA_TYPE>(
49-
providerRegistry: ProviderRegistry,
50-
matchRequest:
51-
| MatchRequest<Image, PROVIDER_DATA_TYPE>
52-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
53-
): Promise<MatchResult>;
52+
matchRequest: MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>
53+
): Promise<MatchResult<Point>>;
5454
export async function getMatchResult<PROVIDER_DATA_TYPE>(
5555
providerRegistry: ProviderRegistry,
5656
matchRequest:
57-
| MatchRequest<Image, PROVIDER_DATA_TYPE>
58-
| MatchRequest<TextQuery, PROVIDER_DATA_TYPE>
59-
): Promise<MatchResult> {
60-
return isImageMatchRequest(matchRequest)
61-
? providerRegistry.getImageFinder().findMatch(matchRequest)
62-
: providerRegistry.getTextFinder().findMatch(matchRequest);
57+
| MatchRequest<RegionResultFindInput, PROVIDER_DATA_TYPE>
58+
| MatchRequest<PointResultFindInput, PROVIDER_DATA_TYPE>
59+
): Promise<MatchResult<Point | Region>> {
60+
if (isImageMatchRequest(matchRequest)) {
61+
return providerRegistry.getImageFinder().findMatch(matchRequest);
62+
} else if (isTextMatchRequest(matchRequest)) {
63+
return providerRegistry.getTextFinder().findMatch(matchRequest);
64+
} else if (isColorMatchRequest(matchRequest)) {
65+
return providerRegistry.getColorFinder().findMatch(matchRequest);
66+
}
67+
throw new Error("Unknown match request type");
6368
}

Diff for: ‎lib/provider/color-finder.interface.ts

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { ColorQuery } from "../query.class";
2+
import { MatchResult } from "../match-result.class";
3+
import { Point } from "../point.class";
4+
import { MatchRequest } from "../match-request.class";
5+
6+
/**
7+
* A WindowFinder should provide an abstraction layer to perform window searches
8+
*
9+
* @interface ColorFinderInterface
10+
*/
11+
export interface ColorFinderInterface {
12+
/**
13+
* findMatch should provide an abstraction to search for a color on screen
14+
*
15+
* @param {ColorQuery} query A {@link ColorQuery} containing needed data
16+
* @returns {Promise<number>} A single window handle
17+
* @memberof WindowFinderInterface
18+
*/
19+
findMatch<PROVIDER_DATA_TYPE>(
20+
query: MatchRequest<ColorQuery, PROVIDER_DATA_TYPE>
21+
): Promise<MatchResult<Point>>;
22+
23+
/**
24+
* findMatches should provide an abstraction to search for a window on screen
25+
*
26+
* @param {ColorQuery} query A {@link ColorQuery} containing needed data
27+
* @returns {Promise<number[]>} A list of window handles
28+
* @memberof WindowFinderInterface
29+
*/
30+
findMatches<PROVIDER_DATA_TYPE>(
31+
query: MatchRequest<ColorQuery, PROVIDER_DATA_TYPE>
32+
): Promise<MatchResult<Point>[]>;
33+
}

Diff for: ‎lib/provider/color/color-finder.class.spec.ts

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import ColorFinder from "./color-finder.class";
2+
import { join } from "path";
3+
import {
4+
ColorQuery,
5+
loadImage,
6+
MatchRequest,
7+
MatchResult,
8+
Point,
9+
RGBA,
10+
} from "../../../index";
11+
12+
describe("color finder", () => {
13+
describe("find", () => {
14+
it("should resolve", async () => {
15+
// GIVEN
16+
const screenImage = await loadImage(
17+
join(__dirname, `../../../e2e/assets/dot.png`)
18+
);
19+
const SUT = new ColorFinder();
20+
const colorQuery: ColorQuery = {
21+
id: "colorFinderTest",
22+
type: "color",
23+
by: {
24+
color: new RGBA(255, 0, 0, 255),
25+
},
26+
};
27+
const matchRequest = new MatchRequest(screenImage, colorQuery, 0.9);
28+
29+
const expectedResult = new MatchResult(1, new Point(60, 60));
30+
31+
// WHEN
32+
const result = await SUT.findMatch(matchRequest);
33+
34+
// THEN
35+
expect(result).toEqual(expectedResult);
36+
});
37+
});
38+
39+
describe("findAll", () => {
40+
it("should resolve", async () => {
41+
// GIVEN
42+
const screenImage = await loadImage(
43+
join(__dirname, `../../../e2e/assets/dots.png`)
44+
);
45+
const SUT = new ColorFinder();
46+
const colorQuery: ColorQuery = {
47+
id: "colorFinderTest",
48+
type: "color",
49+
by: {
50+
color: new RGBA(255, 0, 0, 255),
51+
},
52+
};
53+
const matchRequest = new MatchRequest(screenImage, colorQuery, 0.9);
54+
55+
const expectedResult = [
56+
new MatchResult(1, new Point(60, 60)),
57+
new MatchResult(1, new Point(60, 80)),
58+
new MatchResult(1, new Point(60, 100)),
59+
];
60+
61+
// WHEN
62+
const result = await SUT.findMatches(matchRequest);
63+
64+
// THEN
65+
expect(result).toEqual(expectedResult);
66+
});
67+
});
68+
});

0 commit comments

Comments
 (0)