Skip to content

Enhancement/169/improve large img error message #170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 14, 2020
Merged

Enhancement/169/improve large img error message #170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3cbca51
(#169) Added error message in case needle is way to large for haystack
svettwer Sep 9, 2020
819ba1a
(#169) Some function extraction
svettwer Sep 9, 2020
656332a
(#169) [Proposal] Use promise-like return type instead of wrapping a …
svettwer Sep 9, 2020
e779c9b
(#169) Applied remark from review
svettwer Sep 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/match-result.class.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ export class MatchResult {
constructor(
public readonly confidence: number,
public readonly location: Region,
public readonly error?: Error
) {}
}
Binary file added lib/provider/opencv/__mocks__/fat-needle.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
24 changes: 20 additions & 4 deletions lib/provider/opencv/template-matching-finder.class.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,7 @@ describe("Template-matching finder", () => {
// THEN
await expect(SUT.findMatch(matchRequest))
.rejects
.toEqual(
expect
.stringMatching(expectedRejection)
);
.toThrowError(expectedRejection);
});

it("findMatch should throw on invalid image paths", async () => {
Expand All @@ -91,4 +88,23 @@ describe("Template-matching finder", () => {
.rejects
.toThrowError(`Failed to load ${pathToHaystack}. Reason: 'Failed to load image from '${pathToHaystack}''.`);
});

it("findMatch should reject, if needle was way lager than the haystack", async () => {
// GIVEN
const imageLoader = new ImageReader();
const SUT = new TemplateMatchingFinder();
const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png");
const needlePath = path.resolve(__dirname, "./__mocks__/fat-needle.png");
const haystack = await imageLoader.load(haystackPath);
const minConfidence = 0.99;
const searchRegion = new Region(0, 0, haystack.width, haystack.height);
const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence);
const expectedRejection = new Error("The provided image sample is larger than the provided search region")

// WHEN
const findMatchPromise = SUT.findMatch(matchRequest);

// THEN
await expect(findMatchPromise).rejects.toEqual(expectedRejection)
});
});
141 changes: 67 additions & 74 deletions lib/provider/opencv/template-matching-finder.class.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,19 @@ function isValidSearch(needle: cv.Mat, haystack: cv.Mat): boolean {
return (needle.cols <= haystack.cols) && (needle.rows <= haystack.rows);
}

function createResultForInvalidSearch (currentScale: number) {
return new ScaledMatchResult(0,
currentScale,
new Region(
0,
0,
0,
0
),
new Error("The provided image sample is larger than the provided search region")
)
}

export default class TemplateMatchingFinder implements FinderInterface {
private initialScale = [1.0];
private scaleSteps = [0.9, 0.8, 0.7, 0.6, 0.5];
Expand Down Expand Up @@ -94,69 +107,15 @@ export default class TemplateMatchingFinder implements FinderInterface {
const matchResults = this.initialScale.map(
async (currentScale) => {
if (!isValidSearch(needle, haystack)) {
return new ScaledMatchResult(0,
currentScale,
new Region(
0,
0,
0,
0
)
);
return createResultForInvalidSearch(currentScale);
}
const matchResult = await matchImages(haystack, needle);
return new ScaledMatchResult(matchResult.confidence, currentScale, matchResult.location);
}
);

if (matchRequest.searchMultipleScales) {
const scaledNeedleResult = this.scaleSteps.map(
async (currentScale) => {
const scaledNeedle = await scaleImage(needle, currentScale);
if (!isValidSearch(scaledNeedle, haystack)) {
return new ScaledMatchResult(0,
currentScale,
new Region(
0,
0,
0,
0
)
);
}
const matchResult = await matchImages(haystack, scaledNeedle);
return new ScaledMatchResult(
matchResult.confidence,
currentScale,
matchResult.location,
);
}
);
const scaledHaystackResult = this.scaleSteps.map(
async (currentScale) => {
const scaledHaystack = await scaleImage(haystack, currentScale);
if (!isValidSearch(needle, scaledHaystack)) {
return new ScaledMatchResult(0,
currentScale,
new Region(
0,
0,
0,
0
)
);
}
const matchResult = await matchImages(scaledHaystack, needle);
return new ScaledMatchResult(
matchResult.confidence,
currentScale,
scaleLocation(
matchResult.location,
currentScale
)
);
}
);
matchResults.push(...scaledHaystackResult, ...scaledNeedleResult);
matchResults.push(...this.searchMultipleScales(needle, haystack))
}

return Promise.all(matchResults).then(results => {
Expand All @@ -173,24 +132,58 @@ export default class TemplateMatchingFinder implements FinderInterface {
}

public async findMatch(matchRequest: MatchRequest, debug: boolean = false): Promise<MatchResult> {
return new Promise<MatchResult>(async (resolve, reject) => {
try {
const matches = await this.findMatches(matchRequest, debug);
const potentialMatches = matches
.filter(match => match.confidence >= matchRequest.confidence);
if (potentialMatches.length === 0) {
matches.sort((a, b) => a.confidence - b.confidence);
const bestMatch = matches.pop();
if (bestMatch) {
reject(`No match with required confidence ${matchRequest.confidence}. Best match: ${bestMatch.confidence} at ${bestMatch.location}`)
} else {
reject(`Unable to locate ${matchRequest.pathToNeedle}, no match!`);
}

const matches = await this.findMatches(matchRequest, debug);
const potentialMatches = matches
.filter(match => match.confidence >= matchRequest.confidence);
if (potentialMatches.length === 0) {
matches.sort((a, b) => a.confidence - b.confidence);
const bestMatch = matches.pop();
if (bestMatch) {
if(bestMatch.error) {
throw bestMatch.error
}else {
throw new Error(`No match with required confidence ${matchRequest.confidence}. Best match: ${bestMatch.confidence} at ${bestMatch.location}`)
}
resolve(potentialMatches[0]);
} catch (e) {
reject(e);
} else {
throw new Error(`Unable to locate ${matchRequest.pathToNeedle}, no match!`);
}
});
}
return potentialMatches[0];
}

private searchMultipleScales(needle: cv.Mat, haystack: cv.Mat) {
const scaledNeedleResult = this.scaleSteps.map(
async (currentScale) => {
const scaledNeedle = await scaleImage(needle, currentScale);
if (!isValidSearch(scaledNeedle, haystack)) {
return createResultForInvalidSearch(currentScale);
}
const matchResult = await matchImages(haystack, scaledNeedle);
return new ScaledMatchResult(
matchResult.confidence,
currentScale,
matchResult.location,
);
}
);
const scaledHaystackResult = this.scaleSteps.map(
async (currentScale) => {
const scaledHaystack = await scaleImage(haystack, currentScale);
if (!isValidSearch(needle, scaledHaystack)) {
return createResultForInvalidSearch(currentScale);
}
const matchResult = await matchImages(scaledHaystack, needle);
return new ScaledMatchResult(
matchResult.confidence,
currentScale,
scaleLocation(
matchResult.location,
currentScale
)
);
}
);
return [...scaledHaystackResult, ...scaledNeedleResult];
}
}
3 changes: 2 additions & 1 deletion lib/scaled-match-result.class.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ export class ScaledMatchResult extends MatchResult {
constructor(public readonly confidence: number,
public readonly scale: number,
public readonly location: Region,
public readonly error?: Error
) {
super(confidence, location);
super(confidence, location, error);
}
}