diff --git a/lib/match-result.class.ts b/lib/match-result.class.ts index dc2e916b..42568ddd 100644 --- a/lib/match-result.class.ts +++ b/lib/match-result.class.ts @@ -4,5 +4,6 @@ export class MatchResult { constructor( public readonly confidence: number, public readonly location: Region, + public readonly error?: Error ) {} } diff --git a/lib/provider/opencv/__mocks__/fat-needle.png b/lib/provider/opencv/__mocks__/fat-needle.png new file mode 100644 index 00000000..87533253 Binary files /dev/null and b/lib/provider/opencv/__mocks__/fat-needle.png differ diff --git a/lib/provider/opencv/template-matching-finder.class.spec.ts b/lib/provider/opencv/template-matching-finder.class.spec.ts index de7e2a1d..974dc897 100644 --- a/lib/provider/opencv/template-matching-finder.class.spec.ts +++ b/lib/provider/opencv/template-matching-finder.class.spec.ts @@ -65,10 +65,7 @@ describe("Template-matching finder", () => { // THEN await expect(SUT.findMatch(matchRequest)) .rejects - .toEqual( - expect - .stringMatching(expectedRejection) - ); + .toThrowError(expectedRejection); }); it("findMatch should throw on invalid image paths", async () => { @@ -91,4 +88,23 @@ describe("Template-matching finder", () => { .rejects .toThrowError(`Failed to load ${pathToHaystack}. Reason: 'Failed to load image from '${pathToHaystack}''.`); }); + + it("findMatch should reject, if needle was way lager than the haystack", async () => { + // GIVEN + const imageLoader = new ImageReader(); + const SUT = new TemplateMatchingFinder(); + const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png"); + const needlePath = path.resolve(__dirname, "./__mocks__/fat-needle.png"); + const haystack = await imageLoader.load(haystackPath); + const minConfidence = 0.99; + const searchRegion = new Region(0, 0, haystack.width, haystack.height); + const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence); + const expectedRejection = new Error("The provided image sample is larger than the provided search region") + + // WHEN + const findMatchPromise = SUT.findMatch(matchRequest); + + // THEN + await expect(findMatchPromise).rejects.toEqual(expectedRejection) + }); }); diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts index ced5db6b..4ea34976 100755 --- a/lib/provider/opencv/template-matching-finder.class.ts +++ b/lib/provider/opencv/template-matching-finder.class.ts @@ -60,6 +60,19 @@ function isValidSearch(needle: cv.Mat, haystack: cv.Mat): boolean { return (needle.cols <= haystack.cols) && (needle.rows <= haystack.rows); } +function createResultForInvalidSearch (currentScale: number) { + return new ScaledMatchResult(0, + currentScale, + new Region( + 0, + 0, + 0, + 0 + ), + new Error("The provided image sample is larger than the provided search region") + ) +} + export default class TemplateMatchingFinder implements FinderInterface { private initialScale = [1.0]; private scaleSteps = [0.9, 0.8, 0.7, 0.6, 0.5]; @@ -94,69 +107,15 @@ export default class TemplateMatchingFinder implements FinderInterface { const matchResults = this.initialScale.map( async (currentScale) => { if (!isValidSearch(needle, haystack)) { - return new ScaledMatchResult(0, - currentScale, - new Region( - 0, - 0, - 0, - 0 - ) - ); + return createResultForInvalidSearch(currentScale); } const matchResult = await matchImages(haystack, needle); return new ScaledMatchResult(matchResult.confidence, currentScale, matchResult.location); } ); + if (matchRequest.searchMultipleScales) { - const scaledNeedleResult = this.scaleSteps.map( - async (currentScale) => { - const scaledNeedle = await scaleImage(needle, currentScale); - if (!isValidSearch(scaledNeedle, haystack)) { - return new ScaledMatchResult(0, - currentScale, - new Region( - 0, - 0, - 0, - 0 - ) - ); - } - const matchResult = await matchImages(haystack, scaledNeedle); - return new ScaledMatchResult( - matchResult.confidence, - currentScale, - matchResult.location, - ); - } - ); - const scaledHaystackResult = this.scaleSteps.map( - async (currentScale) => { - const scaledHaystack = await scaleImage(haystack, currentScale); - if (!isValidSearch(needle, scaledHaystack)) { - return new ScaledMatchResult(0, - currentScale, - new Region( - 0, - 0, - 0, - 0 - ) - ); - } - const matchResult = await matchImages(scaledHaystack, needle); - return new ScaledMatchResult( - matchResult.confidence, - currentScale, - scaleLocation( - matchResult.location, - currentScale - ) - ); - } - ); - matchResults.push(...scaledHaystackResult, ...scaledNeedleResult); + matchResults.push(...this.searchMultipleScales(needle, haystack)) } return Promise.all(matchResults).then(results => { @@ -173,24 +132,58 @@ export default class TemplateMatchingFinder implements FinderInterface { } public async findMatch(matchRequest: MatchRequest, debug: boolean = false): Promise { - return new Promise(async (resolve, reject) => { - try { - const matches = await this.findMatches(matchRequest, debug); - const potentialMatches = matches - .filter(match => match.confidence >= matchRequest.confidence); - if (potentialMatches.length === 0) { - matches.sort((a, b) => a.confidence - b.confidence); - const bestMatch = matches.pop(); - if (bestMatch) { - reject(`No match with required confidence ${matchRequest.confidence}. Best match: ${bestMatch.confidence} at ${bestMatch.location}`) - } else { - reject(`Unable to locate ${matchRequest.pathToNeedle}, no match!`); - } + + const matches = await this.findMatches(matchRequest, debug); + const potentialMatches = matches + .filter(match => match.confidence >= matchRequest.confidence); + if (potentialMatches.length === 0) { + matches.sort((a, b) => a.confidence - b.confidence); + const bestMatch = matches.pop(); + if (bestMatch) { + if(bestMatch.error) { + throw bestMatch.error + }else { + throw new Error(`No match with required confidence ${matchRequest.confidence}. Best match: ${bestMatch.confidence} at ${bestMatch.location}`) } - resolve(potentialMatches[0]); - } catch (e) { - reject(e); + } else { + throw new Error(`Unable to locate ${matchRequest.pathToNeedle}, no match!`); } - }); + } + return potentialMatches[0]; + } + + private searchMultipleScales(needle: cv.Mat, haystack: cv.Mat) { + const scaledNeedleResult = this.scaleSteps.map( + async (currentScale) => { + const scaledNeedle = await scaleImage(needle, currentScale); + if (!isValidSearch(scaledNeedle, haystack)) { + return createResultForInvalidSearch(currentScale); + } + const matchResult = await matchImages(haystack, scaledNeedle); + return new ScaledMatchResult( + matchResult.confidence, + currentScale, + matchResult.location, + ); + } + ); + const scaledHaystackResult = this.scaleSteps.map( + async (currentScale) => { + const scaledHaystack = await scaleImage(haystack, currentScale); + if (!isValidSearch(needle, scaledHaystack)) { + return createResultForInvalidSearch(currentScale); + } + const matchResult = await matchImages(scaledHaystack, needle); + return new ScaledMatchResult( + matchResult.confidence, + currentScale, + scaleLocation( + matchResult.location, + currentScale + ) + ); + } + ); + return [...scaledHaystackResult, ...scaledNeedleResult]; } } diff --git a/lib/scaled-match-result.class.ts b/lib/scaled-match-result.class.ts index 8fbb12e6..85db56ca 100644 --- a/lib/scaled-match-result.class.ts +++ b/lib/scaled-match-result.class.ts @@ -5,7 +5,8 @@ export class ScaledMatchResult extends MatchResult { constructor(public readonly confidence: number, public readonly scale: number, public readonly location: Region, + public readonly error?: Error ) { - super(confidence, location); + super(confidence, location, error); } }