From 8fd761aae5cea00873e25f180e06f5721e899aa5 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Fri, 14 Jun 2019 13:57:33 +0200 Subject: [PATCH 01/11] (#68) Updated scaled image search --- lib/provider/opencv/__mocks__/needle.png | Bin 0 -> 9061 bytes .../template-matching-finder.class.spec.ts | 28 +- .../opencv/template-matching-finder.class.ts | 360 ++++++++++-------- lib/scaled-match-result.class.ts | 11 + 4 files changed, 220 insertions(+), 179 deletions(-) create mode 100644 lib/provider/opencv/__mocks__/needle.png create mode 100644 lib/scaled-match-result.class.ts diff --git a/lib/provider/opencv/__mocks__/needle.png b/lib/provider/opencv/__mocks__/needle.png new file mode 100644 index 0000000000000000000000000000000000000000..a9b012f0095086e7f7a38b7025393f5ddfabc1d8 GIT binary patch literal 9061 zcmaiabzGE9+x8M75&{B}!m^|^OV`pJ(%rE1QcH(`giA?-NJ)beFJO zMgRc7EOyW_@H9|U6|r`Oaa!8AT0uE|VQwfn03ase>t<=~1oZ@3LG2t|#6gE`T_B)? zjW|f3UyWPMO$KW3py=-b)%I7{vG#Ye7PbLNKybu-MNkAVsHY{+7v}5&7x5Jb{oxfs z{r=s}1p@wocshxL4AeA$GOiv_AU`KRCpQR!0~GVHu@%vhmH&G%swEDx_w;lV;o|b~ z@!|C0<#hG1GI@tb`-39))`6x$l{g!a?aB_3~D;(zni~NA{q|9P-g>K2N(*An){zh{ZnfAFVeqFL3vaJ#q40~Cu`{mMJdb8Bh0}q zz`?_(!vhxK6BOYVVB-dhaQ~(0A2cfUZ7e-4|2O>a1O|%ziI|9@1KiWq!|zYmUsdK_5*6zyL!3;S=223fV@1w-<>QgtK?1qR#_p z&mQTs^NP~f%CtX82CiSa^CHO$F2C6++UD*)?5N9^er`?|99^;D2;Qr(tWkZzAnY$| znz+joFM>6f|4n>1+96GOdU;JN@a5)Rp`7=c74ZViSq%}VqVCsoU+NY0JWCoQm+lsW@AHQcW&%Q^^y=2g zj-YR8Cu%AZVc+adoQP!Jq23V>+!PJr001uW?*|R=CW{;Zpq^8bmD2Ienhvz{F+rxz zb)1=PyYPj`Gf8g9KKf)S&%hkdRKXYE4Kp;V?r41V@%{*nvB)ptJ=b4f;vtZLR~>ux zZANiDu`Cb!)-XL3@sV^7Kt#fJZ!ttJ0=#b5iu{+W(Gqx*u*ke9m(N(2TAOx@-}zq( ze&odr(ZFP+001^p3~2}_9PPTiqH^leJ;Wu%#bC48jcciTOn^E4OzaRa0KjZfIxPvp zWT-a`2b;_n8BVp8fHtDtJql;*iQIE*xQv{z=aeaVhs>4}uQiyosnYS$tP72T!Gy4? ziY?rT;xyY73g9DV?n@A#(@0216qJNpw2?ntQFfDxC?F7PZnk~KjW8HJF@i5YPSxhU z@e4SS6A{ti1{I;k1Tb2jL(evZyaIa!lF@t|FJYXO#i*{DL}-IaAtjw4_3V?o;wRy} zb2;!M&|9=3Y|^-@8|OvhQa%SA{v`bSM+z|Vt*G!j%6%# zGUI6ABY{N66YR^e_Q001hQ9qXpfw-_Q?h%cMKe59X0%GhwmNDw!Ys!S3Iy=(CRJ*~ z4G+y|mDaM*h@Q5zCvg$J*zmXtGy_sa2IOgi5A}f)JJZ2i#>UizfcB@Cam@y2Q2o6y6GjfDL%tPvCNXb1T!o<_-Hi?Ou`*gm6 zE2F^1AD5RoKNz#LK)>=5-++);26Qo$ogD{ z{9>YhgO$zsSVh2&?^m+mpBMB#v>RF(CIW3@qPc7vZP)&qX+t`b;WR*Q&o7}-1VFUi zz`7raX2BO-yIp)PARQfDZi-u;=#Bja015&-XK7FOwngtC}RP6DT zSq$#rQjx{IV=VDN(rpa>2j#i3sSWtM_N*j09*mAK_VV zK68C9M$?}W{Rb9CWyi7?$H)L@mY?VL!Mf4KZPwzFvi$B0jfnPHAapLg;puL3Qt z-5Jx7zBirKxGJAtC9n8i3^2yb@?C|Hx0+U2a8J$5!f$%!H3~ff;P4=KKU?RUwMx;j zro;ZG0!Z%I{*hu*FM0jvl-_&x6*3BS#70TWiJooc2e)9YQ&${i%O9g11i|OS*|MNa zqzgy2iwo?GW+&MMwdB8^j?aCas+hhH;yDdy=UC1r)m_sHvX8bjeq~1H`NWD3I&KVh zwoU51ot2;ic-m_YyhZ>*>HLE_x6T_waqS46xM-3vW{&n+*UmE&$3yO{Ed7mib<-F3 zkDH-P4&q?7BMMt$NAu{Y$ks;~+_Pm-2l{K_E#vrHb#p69{lraGCW{7U6T|SF*#3?W zk}}3I;dt0H>H5Wy@^U<5*y0dZxQ~LGOvglZw&R zrrRKGLZa%fk|_|GD&sfMlg4OdhYIUc)9-SWSYfp*#1nHPt1E-WytaGFc?n2c0-yI0 ztUV#GpFwGsuJRF!E=}8R`U=`AyZ~S`CcVh;geB07Ln;?rG=fCm(q zn0SNh?mlN1jqhm=B!aBIe<%o`UHw$Z3nxxy!;L8jld-lNg&;(A?-nMRNkUm`=vWzyO0oWpwqnFM)+mVB(!b2r(H7DoJ zaU5!?!>iuu*&ubq(g=y%2|4t?-t&Tckz$jhDQFyDwFyFS!`b*mwd75;7MZh`ciHcO zT*PGJS7#TdHCLDBMBPs7*=CvK`%_e1+^ziKUa&y>jUqvF*PZV`2FU~pk5u|(yh-e% zB2#(UYcfMot(W)*Q9`V$TS$bN zbK^_#`KKLhbGOp>;~W8bJy%1YdUhx4gfRhMdZmg?1Qu6j*Kx6gwq-785X!@w9t~%n zjov=d4TMCR(l&BR+5`yIOr;az&&GRHw%80f&JKFLuJlWKhV zC`nyeNj3p542D;*Dd;rRC1ZI~l0-$uOuU~bZA`o-{A{p|aAEh9ZiOKoS+Pdqj1}f9r zj@uITK^HQ2bl|WICou`zCjNsX@AuorEwQ!h+oODqd=jnji{VTkE91pQCo28BD>5gQ z;cfHQrKtt0aMAE6U2sWf8ub9Wq9mS-?%M8~!J!)~B(?+8(!py@Tumhtr1<$h6`NCL z`5q?2aGeARUZixW#<)2VPQ=GGMK2j`YnV)s9F*KfiEoyWJ9F6U)Dvyubdb1-;UX8Z zcH|dD%q@$~qbJ2QWXnzk!|h9C8#Pn}#7^_ibp7ae8mX+zz1jMBh1`*5`ng#tnpy6| z6?9Sgs*@(@uAne=GRq_0yR0psKDIV6F9FdHHA{6?NQtDjffs-vqWyDnPWEa1$< z6F<90fEk!42~{gBDyS@|-qujzoh{g$T3VrWa6~QBma?7D?NI?f+Q}DnM$aaxBJH-Z zH?z~fViv#U_Ph@OhJN>!bEJoLEYCZ8Tl+O6a4|Gmt7zAM&?s%JHM2_6+5NHziv&T> zyo%0>QZMcfXqyBCR2ruHt*jP&RoS;BI2k&uG%>DK6&$J~u}i^15b(Rl zDbYj9>9fRT)n`9%O?0(0KJ9Fewu!mqQFY#DX74$;w^{lCFlYcICl?9=YZ}u%n=OTB}bd8&nQ>5wAk2cp@23tN4y-%AZ=?EV<$dJwf?V;QW-g~G1d9;y=c0|q<+wC;6`kWj;eza)$B;+ z=m1JdS?F{>@0~EnfO9Ut90tD`+dMZXBQoVABXZB5S*Z#>qy4qqH*2Ce1SIrl!I`JGEu<+F!zc*_CYmwg?HKM6`A73>B|9^2XKlcnby zFP7OTKP6#MN@6hl_OY1aQ->9+bObgb?NZCnOP`Am!rhy)>XMS=LIM8Fwc3N*xdCe) zt7T|PNhT0ikxpJ-o}Ibqm8Jvrw@JfXVea~d(ja0hJtTXtw~Eude(IHiZ?4Yb>yfAHZ5O1ven*wFRBs4)LgJ>+67R>`%I8+F zv+pxJ%7T-e$3c}I>V>1Hl7?DvlT=~r>*~pY))o}K7#>yOwTaU#UH8&r&!5Y7Tp~3Y&mxQ9- zq#ak#Fxd|bko!Xf81C4UIR3)6{;Sl(PtpudmZj39K{Jmebzvh#f@b)b=(IPd9f3@u z9*35Y^tRJ$wqZDKyvypcf$^|ieb%0+S-!lk4kJxme`5hc5>dw<&~8>ews&?V_S_t* z=2h6(*vh13s;x~?hBD|hp=Jl45LQ{a`s+u*bc`%WT z*ZD`{CeB7CKG0z*XqU71HWdg~o26%X9`wZ|Eyf|TuFHm?w7)VNS6tL)Rb1PDnoFK4$9%j$Q$dz5ke}!P_}p#_S`M1Y zw&%zj#re<*ut=I34^(9A6cR6g=SXL+K?)6CXW6Ie9_M+NPjVa)$eFHY=WTUVc^21D zlsetF88_V^%bqi~dG&ct1+@QlaiVj5;8nKhld^KPKH^$?ou?$n^A+&-?p%X~T9Ejr zqLO9$VN8~es{-1?m5$^(mK_6Yw$xtFi0yO9kjDv9^NayvImp^0Rqz>J*i{&MXg*7n zK}Bb0XG7b`T#Ggb3|3f*s*2iZIG6D~B5u0pwNYge06KBT9=BmX;Yx@*-~EL+BPXj~ zN>_{tfXN_RReH2vddxSoL5oi)wEc{RU_hX0QAE@xz5RWSkRZ

AhIkWQj9DLFS@k zrzGO-;j=j^q7>e=z%+;Vg9b}uj&hPOvVCvP&z?Op>i$ASAS;VwZfIa)XfTzU{0ozR zOD<|GNSjt_BO*Y}74n%3LkxxpS(d6Oo5lIg z^j?%KI(c1*aL6q_nv4itvH_69*&Z?#;PJM+$qrS!4M}w9I|pI_j6}NH5O>?%*c3E(5?jKX;kb7|z0Yz~(%tEK8W~X< z7xzG?4vnSxQEMg`BJeoAkGkz=WEfGcwRlGdbC*|VQ6+zCgV)|dT+0zMwWuhkK*8uC zEzOg_ig)k0@y`G9Hs8n|u>1vPIl=b#EoYp#HZE4`z z$6)RAD!meq++<&yg&Hc+MvLRw~_2y)3o2fWdF=;b;$SvU6 z?B=YQ@Rav!!Vn@tx6vmheCR`c@1E?B#ng8`1SJl;$6AJzO#h?md$EyLEUVnvDiYMzAffUKr}WRFZ1I>x7PK&Cs^pL5rG03 zORjTgRCw6k37e=}&7jkrrBhYTKVnl=sxywC}jR6Pz2{>PgvN zXFWvUDF4oYA&EL~KY00`01Vs?gjxDU`05t!-V|-W$`YRUZ0;lbK_AWKUxUW%E@zOE z5*Ji{-2c4_=-I^8m4(2G&M&?HTq>kic$WZcw8Qsk>WX*Tizt3Rh>Dm#+<#{CPLnd2sbAZ#CXbubF!0jZMM6*@7=a^5Y|lTIZi+D{(X(4MSo)D zk;T)77$#dK#jv5y!^m(OTLT}e)4TA64{Y?s2#cSdllQQ{BtffZ3s$?tno);t0Uq(| zLd0?7MCKIzxNAtF+C9W4Ya62Ekn}>W%GftR@TC<^>3qp%I0ErOV{rY9MyEqASe8dCh-4{Pwwo9h)L12U~{XQu}jj$I}P; z)5~nM07=9sCQJ#m{5Jfu$tBQea1(~!grK-)C7Ovr z;#y2DN-V51#Y9E3ob&~-Hfq-i+-VKUBw#~2{cj5hsy4v)*Luw#@bS@J$I$-L4N{>` zE>aa4i6Wj&=Lfmznwl6m@7tFR(Nf%&QG(2&#!&xC>ANGXr6l>e)zg%!kNus17~Gs%>21u9sF6sZOXmnu%8Bf z{GxW>(1N}H_)ATtPQTx{CCqPsx4OzL0stT~;q+-{{bb`8A}~o?|7!`O3Z}$MMaxR&$Ix&KXj(vp;E>hX2NR z*)30WKXXT3!SNl+b>Sb^UL8~wrxh=>iXiu|Pj?k;hi5<*I(?y%{DbPowQKP$gXepGO2H_SZakAKF7u4Ga?2m*0B*{%R7Ry?A=|=bV)Gp zE4&0Gur$=e_ixz{ZZ|g+N@-_;34}^XI@HU|SPk!{$% z=oU;WNKc!Jt=9Nc0UDQMU$Ib000Zpt4T@q6$(HU0p^)Y1w{; zreVf5X!Th6a*|9xVWI<&r=J{*JLQhOq7zbg15WZ{HY*c`U?BWuq2=jl0wY_T|zr44CD8ItsGSf~_3!^yF+ zUQyG2qarsjH|y}b?e85lV5dY~EEQ)lg8=);zzRF zBHJWt@q#gwKPqlL2s+|Od-Xv=b2UZe+=x4gZFieCsFho^gour#O2}vDs_ECMC~2xC zq4}APEjeSD>_>LNPqx;ZPO}yfi}xOuMt!`G@zFx=$KX-HlOXXA77^^jsJqm$?N1%{ zScEbCqh|gq`(C((EOc+^xAex5yOaQ%lS}uOKE|Zx!dUIpIyi<@{MP!+*38y*GEQ*PA!eZ^sKmQPcwj^n4Xa0T#sP=WE)T;=(i~`Hj2NFm}$-2#vI?f)0wsh{KqSwUzz0#Iq zaCF31!)#Fo)QERZ^senE=iM2}zd}~k$rt%EmiIwm2z7i_dP}&&4*y~V z%U%~Jh$3uvVfUng3`;%Nz|6qJ*vRm8rg4^WsZ+&;ubBP1`_m+Y-kf~v@~_P{ZuxC- z(=$|#-nG_G-j*7kFv~2hEGd_0qq0Kca;*z>O+0O$))tQyOdERY;o_XQ`t>Y8^k=|A ztEhV6vl5>qn%vn<_~E=zYPz7F0-%xjUf+;ucWLwBXZK#SXaqukR@Ut1ynLGzLL05a zExUCp4->^PINjVsS2!JJ2T6jIlF;))BuHu~cA!q8Uz}dYH0C1^cW~d)ytJK6gP%`k zc8RlPmzbG*AtS0ETX_}EUJ{HydL4@YUajS}T?zvbz3dKSQ7@P)|8Pfe@SSa@?dU`Q Q?|;WA$*Id$Nn5=5KlEm@qyPW_ literal 0 HcmV?d00001 diff --git a/lib/provider/opencv/template-matching-finder.class.spec.ts b/lib/provider/opencv/template-matching-finder.class.spec.ts index 19739e6e..dd57d237 100644 --- a/lib/provider/opencv/template-matching-finder.class.spec.ts +++ b/lib/provider/opencv/template-matching-finder.class.spec.ts @@ -10,38 +10,42 @@ describe("Template-matching finder", () => { // GIVEN const imageLoader = new ImageReader(); const SUT = new TemplateMatchingFinder(); - const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png"); - const needle = await imageLoader.load(imagePath); + const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png"); + const needlePath = path.resolve(__dirname, "./__mocks__/needle.png"); + const haystack = await imageLoader.load(haystackPath); + const needle = await imageLoader.load(needlePath); const minConfidence = 0.99; - const searchRegion = new Region(0, 0, needle.width, needle.height); - const haystack = new Image(needle.width, needle.height, needle.data, 3); - const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence); + const searchRegion = new Region(0, 0, haystack.width, haystack.height); + const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence); + const expectedResult = new Region(16, 31, needle.width, needle.height); // WHEN const result = await SUT.findMatch(matchRequest); // THEN expect(result.confidence).toBeGreaterThanOrEqual(minConfidence); - expect(result.location).toEqual(searchRegion); + expect(result.location).toEqual(expectedResult); }); it("findMatch should return a match within a search region when present in image", async () => { // GIVEN const imageLoader = new ImageReader(); const SUT = new TemplateMatchingFinder(); - const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png"); - const needle = await imageLoader.load(imagePath); + const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png"); + const needlePath = path.resolve(__dirname, "./__mocks__/needle.png"); + const haystack = await imageLoader.load(haystackPath); + const needle = await imageLoader.load(needlePath); const minConfidence = 0.99; - const searchRegion = new Region(10, 20, 100, 100); - const haystack = new Image(needle.width, needle.height, needle.data, 3); - const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence); + const searchRegion = new Region(10, 20, 140, 100); + const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence); + const expectedResult = new Region(6, 11, needle.width, needle.height); // WHEN const result = await SUT.findMatch(matchRequest); // THEN expect(result.confidence).toBeGreaterThanOrEqual(minConfidence); - expect(result.location).toEqual(searchRegion); + expect(result.location).toEqual(expectedResult); }); it("findMatch should throw on invalid image paths", async () => { diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts index 4315938a..01b229cb 100644 --- a/lib/provider/opencv/template-matching-finder.class.ts +++ b/lib/provider/opencv/template-matching-finder.class.ts @@ -4,122 +4,131 @@ import { Image } from "../../image.class"; import { MatchRequest } from "../../match-request.class"; import { MatchResult } from "../../match-result.class"; import { Region } from "../../region.class"; +import { ScaledMatchResult } from "../../scaled-match-result.class"; import { DataSource } from "./data-source.interface"; import { FinderInterface } from "./finder.interface"; import { ImageProcessor } from "./image-processor.class"; import { ImageReader } from "./image-reader.class"; -export class TemplateMatchingFinder implements FinderInterface { - private static scaleStep = 0.5; - - private static async match(haystack: cv.Mat, needle: cv.Mat): Promise { - const match = await haystack.matchTemplateAsync( - needle, - cv.TM_SQDIFF_NORMED, - ); - const minMax = await match.minMaxLocAsync(); - return new MatchResult( - 1.0 - minMax.minVal, - new Region( - minMax.minLoc.x, - minMax.minLoc.y, - Math.min(needle.cols, haystack.cols), - Math.min(needle.rows, haystack.rows), - ), - ); - } - - private static async scale(image: cv.Mat, scaleFactor: number): Promise { - const scaledRows = Math.max(Math.floor(image.rows * scaleFactor), 1.0); - const scaledCols = Math.max(Math.floor(image.cols * scaleFactor), 1.0); - return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); +const loadNeedle = async (image: Image): Promise => { + if (image.hasAlphaChannel) { + return ImageProcessor.fromImageWithAlphaChannel(image); } - - private static async scaleAndMatchNeedle( - haystack: cv.Mat, - needle: cv.Mat, - debug: boolean = false - ): Promise { - const scaledNeedle = await TemplateMatchingFinder.scale( - needle, - TemplateMatchingFinder.scaleStep, + return ImageProcessor.fromImageWithoutAlphaChannel(image); +}; + +const loadHaystack = async (matchRequest: MatchRequest): Promise => { + const searchRegion = determineScaledSearchRegion(matchRequest); + if (matchRequest.haystack.hasAlphaChannel) { + return ImageProcessor.fromImageWithAlphaChannel( + matchRequest.haystack, + searchRegion, ); - const matchResult = await TemplateMatchingFinder.match(haystack, scaledNeedle); - if (debug) { - this.debugImage(scaledNeedle, "scaled_needle.png"); - } - return new MatchResult( - matchResult.confidence, - new Region( - matchResult.location.left, - matchResult.location.top, - needle.cols, - needle.rows, - ), + } else { + return ImageProcessor.fromImageWithoutAlphaChannel( + matchRequest.haystack, + searchRegion, ); } - - private static determineScaledSearchRegion(matchRequest: MatchRequest): Region { - const searchRegion = matchRequest.searchRegion; - searchRegion.width *= matchRequest.haystack.pixelDensity.scaleX; - searchRegion.height *= matchRequest.haystack.pixelDensity.scaleY; - return searchRegion; +}; + +const matchImages = async (haystack: cv.Mat, needle: cv.Mat): Promise => { + const match = await haystack.matchTemplateAsync( + needle, + cv.TM_SQDIFF_NORMED, + ); + const minMax = await match.minMaxLocAsync(); + return new MatchResult( + 1.0 - minMax.minVal, + new Region( + minMax.minLoc.x, + minMax.minLoc.y, + Math.min(needle.cols, haystack.cols), + Math.min(needle.rows, haystack.rows), + ), + ); +}; + +const scaleImage = async (image: cv.Mat, scaleFactor: number): Promise => { + const scaledRows = Math.max(Math.floor(image.rows * scaleFactor), 1.0); + const scaledCols = Math.max(Math.floor(image.cols * scaleFactor), 1.0); + return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); +}; + +const determineScaledSearchRegion = (matchRequest: MatchRequest): Region => { + const searchRegion = matchRequest.searchRegion; + searchRegion.width *= matchRequest.haystack.pixelDensity.scaleX; + searchRegion.height *= matchRequest.haystack.pixelDensity.scaleY; + return searchRegion; +}; + +const debugImage = (image: cv.Mat, filename: string, suffix?: string) => { + const parsedPath = path.parse(filename); + let fullFilename = parsedPath.name; + if (suffix) { + fullFilename = fullFilename + "_" + suffix; } + fullFilename += parsedPath.ext; + const fullPath = path.join(parsedPath.dir, fullFilename); + cv.imwriteAsync(fullPath, image); +}; + +// const debugResult = (image: cv.Mat, result: MatchResult, filename: string, suffix?: string) => { +// const roiRect = new cv.Rect( +// Math.min(Math.max(result.location.left, 0), image.cols), +// Math.min(Math.max(result.location.top, 0), image.rows), +// Math.min(result.location.width, image.cols - result.location.left), +// Math.min(result.location.height, image.rows - result.location.top)); +// debugImage(image.getRegion(roiRect), filename, suffix); +// }; +// +// const findEdges = async (image: cv.Mat): Promise => { +// const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY); +// return gray.cannyAsync(50, 200); +// }; + +const scaleSize = ( + result: Region, + scaleFactor: number, +): Region => { + return new Region( + result.left, + result.top, + result.width / scaleFactor, + result.height / scaleFactor, + ); +}; + +const scaleLocation = ( + result: Region, + scaleFactor: number, +): Region => { + return new Region( + result.left / scaleFactor, + result.top / scaleFactor, + result.width, + result.height, + ); +}; + +const isValidSearch = (needle: cv.Mat, haystack: cv.Mat): boolean => { + return (needle.cols <= haystack.cols) && (needle.rows <= haystack.rows); +}; - private static async scaleAndMatchHaystack( - haystack: cv.Mat, - needle: cv.Mat, - debug: boolean = false - ): Promise { - const scaledHaystack = await TemplateMatchingFinder.scale( - haystack, - TemplateMatchingFinder.scaleStep, - ); - const matchResult = await TemplateMatchingFinder.match(scaledHaystack, needle); - if (debug) { - this.debugImage(scaledHaystack, "scaled_haystack.png"); - } - return new MatchResult( - matchResult.confidence, - new Region( - matchResult.location.left / TemplateMatchingFinder.scaleStep, - matchResult.location.top / TemplateMatchingFinder.scaleStep, - needle.cols, - needle.rows, - ), - ); - } - - private static async debugImage(image: cv.Mat, filename: string, suffix?: string) { - const parsedPath = path.parse(filename); - let fullFilename = parsedPath.name; - if (suffix) { - fullFilename = fullFilename + "_" + suffix; - } - fullFilename += parsedPath.ext; - const fullPath = path.join(parsedPath.dir, fullFilename); - cv.imwriteAsync(fullPath, image); - } - - private static async debugResult(image: cv.Mat, result: MatchResult, filename: string, suffix?: string) { - const roiRect = new cv.Rect( - Math.min(Math.max(result.location.left, 0), image.cols), - Math.min(Math.max(result.location.top, 0), image.rows), - Math.min(result.location.width, image.cols - result.location.left), - Math.min(result.location.height, image.rows - result.location.top)); - this.debugImage(image.getRegion(roiRect), filename, suffix); - } +export class TemplateMatchingFinder implements FinderInterface { + private initialScale = [1.0]; + private scaleSteps = [0.9, 0.8, 0.7, 0.6, 0.5]; constructor( private source: DataSource = new ImageReader(), ) { } - public async findMatches(matchRequest: MatchRequest, debug: boolean = false): Promise { - let needle; + public async findMatches(matchRequest: MatchRequest, debug: boolean = false): Promise { + let needle: cv.Mat; try { const needleInput = await this.source.load(matchRequest.pathToNeedle); - needle = await this.loadNeedle(needleInput); + needle = await loadNeedle(needleInput); } catch (e) { throw new Error( `Failed to load ${matchRequest.pathToNeedle}. Reason: '${e}'.`, @@ -130,95 +139,112 @@ export class TemplateMatchingFinder implements FinderInterface { `Failed to load ${matchRequest.pathToNeedle}, got empty image.`, ); } - const haystack = await this.loadHaystack(matchRequest); + const haystack = await loadHaystack(matchRequest); if (debug) { - TemplateMatchingFinder.debugImage(needle, "input_needle.png"); - TemplateMatchingFinder.debugImage(haystack, "input_haystack.png"); + debugImage(needle, "input_needle.png"); + debugImage(haystack, "input_haystack.png"); } - const matchResults = []; - const unscaledResult = await TemplateMatchingFinder.match(haystack, needle); - if (debug) { - TemplateMatchingFinder.debugResult( - haystack, - unscaledResult, - matchRequest.pathToNeedle, - "unscaled_result"); - } - if ( - matchRequest.searchMultipleScales && - unscaledResult.confidence >= Math.max(matchRequest.confidence - 0.1, 0.6) - ) { - const scaledHaystack = await TemplateMatchingFinder.scaleAndMatchHaystack(haystack, needle, debug); - if (debug) { - TemplateMatchingFinder.debugResult( - haystack, - scaledHaystack, - matchRequest.pathToNeedle, - "scaled_haystack_result" - ); - } - matchResults.push(scaledHaystack); - const scaledNeedle = await TemplateMatchingFinder.scaleAndMatchNeedle(haystack, needle, debug); - if (debug) { - TemplateMatchingFinder.debugResult( - haystack, - scaledNeedle, - matchRequest.pathToNeedle, - "scaled_needle_result" - ); + const matchResults = this.initialScale.map( + async (currentScale) => { + if (!isValidSearch(needle, haystack)) { + return new ScaledMatchResult(0, + currentScale, + new Region( + 0, + 0, + 0, + 0 + ) + ); + } + const matchResult = await matchImages(haystack, needle); + return new ScaledMatchResult(matchResult.confidence, currentScale, matchResult.location); } - matchResults.push(scaledNeedle); + ); + if (matchRequest.searchMultipleScales) { + const scaledNeedleResult = this.scaleSteps.map( + async (currentScale) => { + const scaledNeedle = await scaleImage(needle, currentScale); + if (!isValidSearch(scaledNeedle, haystack)) { + return new ScaledMatchResult(0, + currentScale, + new Region( + 0, + 0, + 0, + 0 + ) + ); + } + const matchResult = await matchImages(haystack, scaledNeedle); + return new ScaledMatchResult( + matchResult.confidence, + currentScale, + scaleSize( + matchResult.location, + currentScale + ) + ); + } + ); + const scaledHaystackResult = this.scaleSteps.map( + async (currentScale) => { + const scaledHaystack = await scaleImage(haystack, currentScale); + if (!isValidSearch(needle, scaledHaystack)) { + return new ScaledMatchResult(0, + currentScale, + new Region( + 0, + 0, + 0, + 0 + ) + ); + } + const matchResult = await matchImages(scaledHaystack, needle); + return new ScaledMatchResult( + matchResult.confidence, + currentScale, + scaleLocation( + matchResult.location, + currentScale + ) + ); + } + ); + matchResults.push(...scaledHaystackResult, ...scaledNeedleResult); } - matchResults.push(unscaledResult); - - // Compensate pixel density - matchResults.forEach(matchResult => { - matchResult.location.left /= matchRequest.haystack.pixelDensity.scaleX; - matchResult.location.width /= matchRequest.haystack.pixelDensity.scaleX; - matchResult.location.top /= matchRequest.haystack.pixelDensity.scaleY; - matchResult.location.height /= matchRequest.haystack.pixelDensity.scaleY; - }); - return matchResults.sort( - (first, second) => second.confidence - first.confidence, - ); + return Promise.all(matchResults).then(results => { + results.forEach(matchResult => { + matchResult.location.left /= matchRequest.haystack.pixelDensity.scaleX; + matchResult.location.width /= matchRequest.haystack.pixelDensity.scaleX; + matchResult.location.top /= matchRequest.haystack.pixelDensity.scaleY; + matchResult.location.height /= matchRequest.haystack.pixelDensity.scaleY; + }); + return results.sort( + (first, second) => second.confidence - first.confidence, + ); + }); } public async findMatch(matchRequest: MatchRequest, debug: boolean = false): Promise { return new Promise(async (resolve, reject) => { try { const matches = await this.findMatches(matchRequest, debug); - if (matches.length === 0) { + const potentialMatches = matches + .filter(match => match.confidence >= matchRequest.confidence) + .sort((first, second) => first.scale - second.scale); + if (potentialMatches.length === 0) { reject(`Unable to locate ${matchRequest.pathToNeedle}, no match!`); } - resolve(matches[0]); + resolve(potentialMatches.pop()); } catch (e) { reject(e); } }); } - private async loadNeedle(image: Image): Promise { - if (image.hasAlphaChannel) { - return ImageProcessor.fromImageWithAlphaChannel(image); - } - return ImageProcessor.fromImageWithoutAlphaChannel(image); - } - - private async loadHaystack(matchRequest: MatchRequest): Promise { - const searchRegion = TemplateMatchingFinder.determineScaledSearchRegion(matchRequest); - if (matchRequest.haystack.hasAlphaChannel) { - return ImageProcessor.fromImageWithAlphaChannel( - matchRequest.haystack, - searchRegion, - ); - } else { - return ImageProcessor.fromImageWithoutAlphaChannel( - matchRequest.haystack, - searchRegion, - ); - } - } } diff --git a/lib/scaled-match-result.class.ts b/lib/scaled-match-result.class.ts new file mode 100644 index 00000000..8fbb12e6 --- /dev/null +++ b/lib/scaled-match-result.class.ts @@ -0,0 +1,11 @@ +import { MatchResult } from "./match-result.class"; +import { Region } from "./region.class"; + +export class ScaledMatchResult extends MatchResult { + constructor(public readonly confidence: number, + public readonly scale: number, + public readonly location: Region, + ) { + super(confidence, location); + } +} From aeac4e6f2dfae732445fc3f5559319d24c101c8c Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 16:32:17 +0200 Subject: [PATCH 02/11] (#68) Moved scaling of search regions into separate file --- .../determine-searchregion.function.spec.ts | 60 +++++++++++++++++++ .../opencv/determine-searchregion.function.ts | 11 ++++ package-lock.json | 6 ++ package.json | 1 + 4 files changed, 78 insertions(+) create mode 100644 lib/provider/opencv/determine-searchregion.function.spec.ts create mode 100644 lib/provider/opencv/determine-searchregion.function.ts diff --git a/lib/provider/opencv/determine-searchregion.function.spec.ts b/lib/provider/opencv/determine-searchregion.function.spec.ts new file mode 100644 index 00000000..9885aec8 --- /dev/null +++ b/lib/provider/opencv/determine-searchregion.function.spec.ts @@ -0,0 +1,60 @@ +import { mockPartial } from "sneer"; +import { Image } from "../../image.class"; +import { MatchRequest } from "../../match-request.class"; +import { Region } from "../../region.class"; +import { determineScaledSearchRegion } from "./determine-searchregion.function"; + +describe("determineSearchRegion", () => { + it("should return a search region adopted to pixel density", () => { + // GIVEN + const imageMock = mockPartial({ + pixelDensity: { + scaleX: 1.5, + scaleY: 2.0 + } + }); + const needlePath = "/path/to/needle"; + const inputSearchRegion = new Region(0, 0, 100, 100); + const expectedSearchRegion = new Region(0, 0, 150, 200); + + const matchRequest = new MatchRequest( + imageMock, + needlePath, + inputSearchRegion, + 0.99 + ); + + // WHEN + const result = determineScaledSearchRegion(matchRequest); + + // THEN + expect(result).toEqual(expectedSearchRegion); + }); + + it.each([[0, 1], [1, 0]])("should not adjust searchregion for factor 0: scaleX: %i scaleY: %i", + (scaleX: number, scaleY: number) => { + // GIVEN + const imageMock = mockPartial({ + pixelDensity: { + scaleX, + scaleY + } + }); + const needlePath = "/path/to/needle"; + const inputSearchRegion = new Region(0, 0, 100, 100); + const expectedSearchRegion = new Region(0, 0, 100, 100); + + const matchRequest = new MatchRequest( + imageMock, + needlePath, + inputSearchRegion, + 0.99 + ); + + // WHEN + const result = determineScaledSearchRegion(matchRequest); + + // THEN + expect(result).toEqual(expectedSearchRegion); + }); +}); diff --git a/lib/provider/opencv/determine-searchregion.function.ts b/lib/provider/opencv/determine-searchregion.function.ts new file mode 100644 index 00000000..c94ea9e2 --- /dev/null +++ b/lib/provider/opencv/determine-searchregion.function.ts @@ -0,0 +1,11 @@ +import { MatchRequest } from "../../match-request.class"; +import { Region } from "../../region.class"; + +export const determineScaledSearchRegion = (matchRequest: MatchRequest): Region => { + const searchRegion = matchRequest.searchRegion; + const scaleX = matchRequest.haystack.pixelDensity.scaleX || 1.0; + const scaleY = matchRequest.haystack.pixelDensity.scaleY || 1.0; + searchRegion.width *= scaleX; + searchRegion.height *= scaleY; + return searchRegion; +}; diff --git a/package-lock.json b/package-lock.json index 2ab373dc..1bdd87dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5109,6 +5109,12 @@ } } }, + "sneer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sneer/-/sneer-1.0.1.tgz", + "integrity": "sha512-rzS5DLX+mRbYzN+pwqcOQnz4WIDI+esS2o1yIMFHCWLTRZ348oPPOjJpm/caJiaMMP1SAUgLO4FRVrF2sc8FCA==", + "dev": true + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index e5d8e45e..4b70d4a8 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@types/clipboardy": "^1.1.0", "@types/jest": "^24.0.9", "jest": "^24.7.1", + "sneer": "^1.0.1", "ts-jest": "^24.0.2", "tslint": "^5.16.0", "typescript": "^3.4.3", From 4f3e6f49b6ec41bdbd941360c08fbf3677f440a3 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 20:53:03 +0200 Subject: [PATCH 03/11] (#68) Moved scaling of images into separate file --- .../opencv/scale-image.function.spec.ts | 40 +++++++++++++++++++ lib/provider/opencv/scale-image.function.ts | 8 ++++ 2 files changed, 48 insertions(+) create mode 100644 lib/provider/opencv/scale-image.function.spec.ts create mode 100644 lib/provider/opencv/scale-image.function.ts diff --git a/lib/provider/opencv/scale-image.function.spec.ts b/lib/provider/opencv/scale-image.function.spec.ts new file mode 100644 index 00000000..ab8de540 --- /dev/null +++ b/lib/provider/opencv/scale-image.function.spec.ts @@ -0,0 +1,40 @@ +import * as path from "path"; +import { ImageProcessor } from "./image-processor.class"; +import { ImageReader } from "./image-reader.class"; +import { scaleImage } from "./scale-image.function"; + +describe("scaleImage", () => { + it.each([[0.5], [1.5]])("should scale an image correctly by factor %f", async (scaleFactor) => { + // GIVEN + const imageLoader = new ImageReader(); + const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png"); + const inputImage = await imageLoader.load(pathToinput); + const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage); + const expectedWidth = Math.floor(inputMat.cols * scaleFactor); + const expectedHeight = Math.floor(inputMat.rows * scaleFactor); + + // WHEN + const result = await scaleImage(inputMat, scaleFactor); + + // THEN + expect(result.rows).toBe(expectedHeight); + expect(result.cols).toBe(expectedWidth); + }); + + it.each([[0], [-0.25]])("should keep scale if factor <= 0: Scale %f", async (scaleFactor) => { + // GIVEN + const imageLoader = new ImageReader(); + const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png"); + const inputImage = await imageLoader.load(pathToinput); + const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage); + const expectedWidth = inputMat.cols; + const expectedHeight = inputMat.rows; + + // WHEN + const result = await scaleImage(inputMat, scaleFactor); + + // THEN + expect(result.rows).toBe(expectedHeight); + expect(result.cols).toBe(expectedWidth); + }); +}); diff --git a/lib/provider/opencv/scale-image.function.ts b/lib/provider/opencv/scale-image.function.ts new file mode 100644 index 00000000..4b9fd0c8 --- /dev/null +++ b/lib/provider/opencv/scale-image.function.ts @@ -0,0 +1,8 @@ +import * as cv from "opencv4nodejs-prebuilt"; + +export const scaleImage = async (image: cv.Mat, scaleFactor: number): Promise => { + const minScaleFactor = (scaleFactor <= 0.0) ? 1.0 : scaleFactor; + const scaledRows = Math.floor(image.rows * minScaleFactor); + const scaledCols = Math.floor(image.cols * minScaleFactor); + return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); +}; From af02890e410a9d4b9e4d68302377bb82f841b1d8 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 21:39:12 +0200 Subject: [PATCH 04/11] (#68) Added upper and lower bound utility functions --- .../opencv/bound-value.function.spec.ts | 33 +++++++++++++++++++ lib/provider/opencv/bound-value.function.ts | 7 ++++ 2 files changed, 40 insertions(+) create mode 100644 lib/provider/opencv/bound-value.function.spec.ts create mode 100644 lib/provider/opencv/bound-value.function.ts diff --git a/lib/provider/opencv/bound-value.function.spec.ts b/lib/provider/opencv/bound-value.function.spec.ts new file mode 100644 index 00000000..fc5f9323 --- /dev/null +++ b/lib/provider/opencv/bound-value.function.spec.ts @@ -0,0 +1,33 @@ +import { lowerBound, upperBound } from "./bound-value.function"; + +describe("lowerBound function", () => { + it.each([ + [5, 10, 1, 1], + [5, 5, 10, 10], + [5, 1, 10, 5], + [0, 0, 0, 0] + ])("Input: %f, Boundary: %f, minValue: %f, Expected: %f", + (input: number, boundary: number, minValue: number, expected: number) => { + // WHEN + const result = lowerBound(input, boundary, minValue); + + // THEN + expect(result).toBe(expected); + }); +}); + +describe("upperBound function", () => { + it.each([ + [5, 10, 1, 5], + [5, 5, 10, 10], + [5, 1, 10, 10], + [5, 5, 5, 5] + ])("Input: %f, Boundary: %f, maxValue: %f, Expected: %f", + (input: number, boundary: number, maxValue: number, expected: number) => { + // WHEN + const result = upperBound(input, boundary, maxValue); + + // THEN + expect(result).toBe(expected); + }); +}); diff --git a/lib/provider/opencv/bound-value.function.ts b/lib/provider/opencv/bound-value.function.ts new file mode 100644 index 00000000..e6e6b308 --- /dev/null +++ b/lib/provider/opencv/bound-value.function.ts @@ -0,0 +1,7 @@ +export function lowerBound(value: number, boundary: number, minValue: number): number { + return (value <= boundary) ? minValue : value; +} + +export function upperBound(value: number, boundary: number, maxValue: number): number { + return (value >= boundary) ? maxValue : value; +} From 90e8dd641ccc40a0c9c07f969d3a6ba57c948a07 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 21:39:48 +0200 Subject: [PATCH 05/11] (#68) Updated scaleImage function to use lowerBound function --- lib/provider/opencv/scale-image.function.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/provider/opencv/scale-image.function.ts b/lib/provider/opencv/scale-image.function.ts index 4b9fd0c8..0f1c0007 100644 --- a/lib/provider/opencv/scale-image.function.ts +++ b/lib/provider/opencv/scale-image.function.ts @@ -1,8 +1,9 @@ import * as cv from "opencv4nodejs-prebuilt"; +import { lowerBound } from "./bound-value.function"; export const scaleImage = async (image: cv.Mat, scaleFactor: number): Promise => { - const minScaleFactor = (scaleFactor <= 0.0) ? 1.0 : scaleFactor; - const scaledRows = Math.floor(image.rows * minScaleFactor); - const scaledCols = Math.floor(image.cols * minScaleFactor); + const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0); + const scaledRows = Math.floor(image.rows * boundScaleFactor); + const scaledCols = Math.floor(image.cols * boundScaleFactor); return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); }; From 918dad5330e5027592a1af0c6a292da39d977899 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 21:40:59 +0200 Subject: [PATCH 06/11] (#68) Moved scaling of Region location into separate function --- .../opencv/scale-location.function.spec.ts | 30 +++++++++++++++++++ .../opencv/scale-location.function.ts | 15 ++++++++++ 2 files changed, 45 insertions(+) create mode 100644 lib/provider/opencv/scale-location.function.spec.ts create mode 100644 lib/provider/opencv/scale-location.function.ts diff --git a/lib/provider/opencv/scale-location.function.spec.ts b/lib/provider/opencv/scale-location.function.spec.ts new file mode 100644 index 00000000..dfdf3e8a --- /dev/null +++ b/lib/provider/opencv/scale-location.function.spec.ts @@ -0,0 +1,30 @@ +import { Region } from "../../region.class"; +import { scaleLocation } from "./scale-location.function"; + +describe("scaleLocation", () => { + it("should scale location of a Region for valid scale factors", () => { + // GIVEN + const scaleFactor = 0.5; + const inputRegion = new Region(100, 100, 10, 10); + const expectedRegion = new Region(200, 200, 10, 10); + + // WHEN + const result = scaleLocation(inputRegion, scaleFactor); + + // THEN + expect(result).toEqual(expectedRegion); + }); + + it("should not scale location of a Region for invalid scale factors", () => { + // GIVEN + const scaleFactor = 0.0; + const inputRegion = new Region(100, 100, 10, 10); + const expectedRegion = new Region(100, 100, 10, 10); + + // WHEN + const result = scaleLocation(inputRegion, scaleFactor); + + // THEN + expect(result).toEqual(expectedRegion); + }); +}); diff --git a/lib/provider/opencv/scale-location.function.ts b/lib/provider/opencv/scale-location.function.ts new file mode 100644 index 00000000..2dba04c5 --- /dev/null +++ b/lib/provider/opencv/scale-location.function.ts @@ -0,0 +1,15 @@ +import { Region } from "../../region.class"; +import { lowerBound } from "./bound-value.function"; + +export const scaleLocation = ( + result: Region, + scaleFactor: number, +): Region => { + const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0); + return new Region( + result.left / boundScaleFactor, + result.top / boundScaleFactor, + result.width, + result.height, + ); +}; From 6d1b0c5510b83083c2ea472cf1712ffb16ce5b60 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 21:57:32 +0200 Subject: [PATCH 07/11] (#68) Moved edge detection in images to separate function --- .../opencv/find-edges.function.spec.ts | 22 +++++++++++++++++++ lib/provider/opencv/find-edges.function.ts | 6 +++++ 2 files changed, 28 insertions(+) create mode 100644 lib/provider/opencv/find-edges.function.spec.ts create mode 100644 lib/provider/opencv/find-edges.function.ts diff --git a/lib/provider/opencv/find-edges.function.spec.ts b/lib/provider/opencv/find-edges.function.spec.ts new file mode 100644 index 00000000..f1c7115d --- /dev/null +++ b/lib/provider/opencv/find-edges.function.spec.ts @@ -0,0 +1,22 @@ +import * as cv from "opencv4nodejs-prebuilt"; +import { mockPartial } from "sneer"; +import { findEdges } from "./find-edges.function"; + +describe("findEdges", () => { + it("should convert an image to grayscale and run Canny edge detection", async () => { + // GIVEN + const grayImageMock = mockPartial({ + cannyAsync: jest.fn() + }); + const inputImageMock = mockPartial({ + cvtColorAsync: jest.fn(() => Promise.resolve(grayImageMock)) + }); + + // WHEN + await findEdges(inputImageMock); + + // THEN + expect(inputImageMock.cvtColorAsync).toBeCalledWith(cv.COLOR_BGR2GRAY); + expect(grayImageMock.cannyAsync).toBeCalledWith(50, 200); + }); +}); diff --git a/lib/provider/opencv/find-edges.function.ts b/lib/provider/opencv/find-edges.function.ts new file mode 100644 index 00000000..f87c047d --- /dev/null +++ b/lib/provider/opencv/find-edges.function.ts @@ -0,0 +1,6 @@ +import * as cv from "opencv4nodejs-prebuilt"; + +export const findEdges = async (image: cv.Mat): Promise => { + const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY); + return gray.cannyAsync(50, 200); +}; From d3f3e51fedb648b7204c82f65ff30bd0e56a22af Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 22:17:30 +0200 Subject: [PATCH 08/11] (#68) Moved matching of images to separate function --- .../opencv/match-image.function.spec.ts | 41 +++++++++++++++++++ lib/provider/opencv/match-image.function.ts | 20 +++++++++ 2 files changed, 61 insertions(+) create mode 100644 lib/provider/opencv/match-image.function.spec.ts create mode 100644 lib/provider/opencv/match-image.function.ts diff --git a/lib/provider/opencv/match-image.function.spec.ts b/lib/provider/opencv/match-image.function.spec.ts new file mode 100644 index 00000000..41baa7e1 --- /dev/null +++ b/lib/provider/opencv/match-image.function.spec.ts @@ -0,0 +1,41 @@ +import * as cv from "opencv4nodejs-prebuilt"; +import { mockPartial } from "sneer"; +import { matchImages } from "./match-image.function"; + +describe("matchImages", () => { + it("should return minLoc position and needle size", async () => { + // GIVEN + const minLocX = 100; + const minLocY = 1000; + const matchMock = mockPartial({ + minMaxLocAsync: jest.fn(() => Promise.resolve({ + maxLoc: new cv.Point2( + 200, + 2000 + ), + maxVal: 100, + minLoc: new cv.Point2( + minLocX, + minLocY + ), + minVal: 0, + })) + }); + const haystackMock = mockPartial({ + matchTemplateAsync: jest.fn(() => Promise.resolve(matchMock)) + }); + const needleMock = mockPartial({ + cols: 123, + rows: 456 + }); + + // WHEN + const result = await matchImages(haystackMock, needleMock); + + // THEN + expect(result.location.left).toEqual(minLocX); + expect(result.location.top).toEqual(minLocY); + expect(result.location.width).toEqual(needleMock.cols); + expect(result.location.height).toEqual(needleMock.rows); + }); +}); diff --git a/lib/provider/opencv/match-image.function.ts b/lib/provider/opencv/match-image.function.ts new file mode 100644 index 00000000..2f74a815 --- /dev/null +++ b/lib/provider/opencv/match-image.function.ts @@ -0,0 +1,20 @@ +import * as cv from "opencv4nodejs-prebuilt"; +import { MatchResult } from "../../match-result.class"; +import { Region } from "../../region.class"; + +export const matchImages = async (haystack: cv.Mat, needle: cv.Mat): Promise => { + const match = await haystack.matchTemplateAsync( + needle, + cv.TM_SQDIFF_NORMED, + ); + const minMax = await match.minMaxLocAsync(); + return new MatchResult( + 1.0 - minMax.minVal, + new Region( + minMax.minLoc.x, + minMax.minLoc.y, + needle.cols, + needle.rows, + ), + ); +}; From f135faf63c6db350b9277f196159e9fad745b3a1 Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 22:18:36 +0200 Subject: [PATCH 09/11] (#68) Updated usage of new functions and enabled edge detection --- .../opencv/template-matching-finder.class.ts | 83 +++---------------- 1 file changed, 13 insertions(+), 70 deletions(-) diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts index 01b229cb..fd280861 100644 --- a/lib/provider/opencv/template-matching-finder.class.ts +++ b/lib/provider/opencv/template-matching-finder.class.ts @@ -6,9 +6,14 @@ import { MatchResult } from "../../match-result.class"; import { Region } from "../../region.class"; import { ScaledMatchResult } from "../../scaled-match-result.class"; import { DataSource } from "./data-source.interface"; +import { determineScaledSearchRegion } from "./determine-searchregion.function"; +import { findEdges } from "./find-edges.function"; import { FinderInterface } from "./finder.interface"; import { ImageProcessor } from "./image-processor.class"; import { ImageReader } from "./image-reader.class"; +import { matchImages } from "./match-image.function"; +import { scaleImage } from "./scale-image.function"; +import { scaleLocation } from "./scale-location.function"; const loadNeedle = async (image: Image): Promise => { if (image.hasAlphaChannel) { @@ -32,36 +37,6 @@ const loadHaystack = async (matchRequest: MatchRequest): Promise => { } }; -const matchImages = async (haystack: cv.Mat, needle: cv.Mat): Promise => { - const match = await haystack.matchTemplateAsync( - needle, - cv.TM_SQDIFF_NORMED, - ); - const minMax = await match.minMaxLocAsync(); - return new MatchResult( - 1.0 - minMax.minVal, - new Region( - minMax.minLoc.x, - minMax.minLoc.y, - Math.min(needle.cols, haystack.cols), - Math.min(needle.rows, haystack.rows), - ), - ); -}; - -const scaleImage = async (image: cv.Mat, scaleFactor: number): Promise => { - const scaledRows = Math.max(Math.floor(image.rows * scaleFactor), 1.0); - const scaledCols = Math.max(Math.floor(image.cols * scaleFactor), 1.0); - return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); -}; - -const determineScaledSearchRegion = (matchRequest: MatchRequest): Region => { - const searchRegion = matchRequest.searchRegion; - searchRegion.width *= matchRequest.haystack.pixelDensity.scaleX; - searchRegion.height *= matchRequest.haystack.pixelDensity.scaleY; - return searchRegion; -}; - const debugImage = (image: cv.Mat, filename: string, suffix?: string) => { const parsedPath = path.parse(filename); let fullFilename = parsedPath.name; @@ -81,35 +56,6 @@ const debugImage = (image: cv.Mat, filename: string, suffix?: string) => { // Math.min(result.location.height, image.rows - result.location.top)); // debugImage(image.getRegion(roiRect), filename, suffix); // }; -// -// const findEdges = async (image: cv.Mat): Promise => { -// const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY); -// return gray.cannyAsync(50, 200); -// }; - -const scaleSize = ( - result: Region, - scaleFactor: number, -): Region => { - return new Region( - result.left, - result.top, - result.width / scaleFactor, - result.height / scaleFactor, - ); -}; - -const scaleLocation = ( - result: Region, - scaleFactor: number, -): Region => { - return new Region( - result.left / scaleFactor, - result.top / scaleFactor, - result.width, - result.height, - ); -}; const isValidSearch = (needle: cv.Mat, haystack: cv.Mat): boolean => { return (needle.cols <= haystack.cols) && (needle.rows <= haystack.rows); @@ -140,6 +86,8 @@ export class TemplateMatchingFinder implements FinderInterface { ); } const haystack = await loadHaystack(matchRequest); + const edgeHaystack = await findEdges(haystack); + const edgeNeedle = await findEdges(needle); if (debug) { debugImage(needle, "input_needle.png"); @@ -159,7 +107,7 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(haystack, needle); + const matchResult = await matchImages(edgeHaystack, edgeNeedle); return new ScaledMatchResult(matchResult.confidence, currentScale, matchResult.location); } ); @@ -178,14 +126,11 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(haystack, scaledNeedle); + const matchResult = await matchImages(edgeHaystack, await findEdges(scaledNeedle)); return new ScaledMatchResult( matchResult.confidence, currentScale, - scaleSize( - matchResult.location, - currentScale - ) + matchResult.location, ); } ); @@ -203,7 +148,7 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(scaledHaystack, needle); + const matchResult = await matchImages(await findEdges(scaledHaystack), edgeNeedle); return new ScaledMatchResult( matchResult.confidence, currentScale, @@ -235,16 +180,14 @@ export class TemplateMatchingFinder implements FinderInterface { try { const matches = await this.findMatches(matchRequest, debug); const potentialMatches = matches - .filter(match => match.confidence >= matchRequest.confidence) - .sort((first, second) => first.scale - second.scale); + .filter(match => match.confidence >= matchRequest.confidence); if (potentialMatches.length === 0) { reject(`Unable to locate ${matchRequest.pathToNeedle}, no match!`); } - resolve(potentialMatches.pop()); + resolve(potentialMatches[0]); } catch (e) { reject(e); } }); } - } From 6116f70aac51f65aab52ccd01f6a057b35541bad Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 23:01:21 +0200 Subject: [PATCH 10/11] (#68) Disabled edge detection --- lib/provider/opencv/template-matching-finder.class.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts index fd280861..3dd68975 100644 --- a/lib/provider/opencv/template-matching-finder.class.ts +++ b/lib/provider/opencv/template-matching-finder.class.ts @@ -7,7 +7,6 @@ import { Region } from "../../region.class"; import { ScaledMatchResult } from "../../scaled-match-result.class"; import { DataSource } from "./data-source.interface"; import { determineScaledSearchRegion } from "./determine-searchregion.function"; -import { findEdges } from "./find-edges.function"; import { FinderInterface } from "./finder.interface"; import { ImageProcessor } from "./image-processor.class"; import { ImageReader } from "./image-reader.class"; @@ -86,8 +85,6 @@ export class TemplateMatchingFinder implements FinderInterface { ); } const haystack = await loadHaystack(matchRequest); - const edgeHaystack = await findEdges(haystack); - const edgeNeedle = await findEdges(needle); if (debug) { debugImage(needle, "input_needle.png"); @@ -107,7 +104,7 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(edgeHaystack, edgeNeedle); + const matchResult = await matchImages(haystack, needle); return new ScaledMatchResult(matchResult.confidence, currentScale, matchResult.location); } ); @@ -126,7 +123,7 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(edgeHaystack, await findEdges(scaledNeedle)); + const matchResult = await matchImages(haystack, scaledNeedle); return new ScaledMatchResult( matchResult.confidence, currentScale, @@ -148,7 +145,7 @@ export class TemplateMatchingFinder implements FinderInterface { ) ); } - const matchResult = await matchImages(await findEdges(scaledHaystack), edgeNeedle); + const matchResult = await matchImages(scaledHaystack, needle); return new ScaledMatchResult( matchResult.confidence, currentScale, From 10afb8401007518a36a2269137800b21eb6e94cb Mon Sep 17 00:00:00 2001 From: Simon Hofmann Date: Sun, 16 Jun 2019 23:48:16 +0200 Subject: [PATCH 11/11] (#68) Turned a few anonymous functions into named ones --- .../opencv/determine-searchregion.function.ts | 4 ++-- lib/provider/opencv/find-edges.function.ts | 4 ++-- lib/provider/opencv/image-processor.class.ts | 4 ++-- lib/provider/opencv/match-image.function.ts | 4 ++-- lib/provider/opencv/scale-image.function.ts | 4 ++-- .../opencv/scale-location.function.ts | 6 +++--- .../opencv/template-matching-finder.class.ts | 20 +++++++++---------- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/provider/opencv/determine-searchregion.function.ts b/lib/provider/opencv/determine-searchregion.function.ts index c94ea9e2..84226b1d 100644 --- a/lib/provider/opencv/determine-searchregion.function.ts +++ b/lib/provider/opencv/determine-searchregion.function.ts @@ -1,11 +1,11 @@ import { MatchRequest } from "../../match-request.class"; import { Region } from "../../region.class"; -export const determineScaledSearchRegion = (matchRequest: MatchRequest): Region => { +export function determineScaledSearchRegion(matchRequest: MatchRequest): Region { const searchRegion = matchRequest.searchRegion; const scaleX = matchRequest.haystack.pixelDensity.scaleX || 1.0; const scaleY = matchRequest.haystack.pixelDensity.scaleY || 1.0; searchRegion.width *= scaleX; searchRegion.height *= scaleY; return searchRegion; -}; +} diff --git a/lib/provider/opencv/find-edges.function.ts b/lib/provider/opencv/find-edges.function.ts index f87c047d..66927838 100644 --- a/lib/provider/opencv/find-edges.function.ts +++ b/lib/provider/opencv/find-edges.function.ts @@ -1,6 +1,6 @@ import * as cv from "opencv4nodejs-prebuilt"; -export const findEdges = async (image: cv.Mat): Promise => { +export async function findEdges(image: cv.Mat): Promise { const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY); return gray.cannyAsync(50, 200); -}; +} diff --git a/lib/provider/opencv/image-processor.class.ts b/lib/provider/opencv/image-processor.class.ts index b9b77bb2..01eb79ac 100644 --- a/lib/provider/opencv/image-processor.class.ts +++ b/lib/provider/opencv/image-processor.class.ts @@ -2,13 +2,13 @@ import * as cv from "opencv4nodejs-prebuilt"; import { Image } from "../../image.class"; import { Region } from "../../region.class"; -const determineROI = (img: Image, roi: Region): cv.Rect => { +function determineROI(img: Image, roi: Region): cv.Rect { return new cv.Rect( Math.min(Math.max(roi.left, 0), img.width), Math.min(Math.max(roi.top, 0), img.height), Math.min(roi.width, img.width - roi.left), Math.min(roi.height, img.height - roi.top)); -}; +} export class ImageProcessor { /** diff --git a/lib/provider/opencv/match-image.function.ts b/lib/provider/opencv/match-image.function.ts index 2f74a815..c49e422a 100644 --- a/lib/provider/opencv/match-image.function.ts +++ b/lib/provider/opencv/match-image.function.ts @@ -2,7 +2,7 @@ import * as cv from "opencv4nodejs-prebuilt"; import { MatchResult } from "../../match-result.class"; import { Region } from "../../region.class"; -export const matchImages = async (haystack: cv.Mat, needle: cv.Mat): Promise => { +export async function matchImages(haystack: cv.Mat, needle: cv.Mat): Promise { const match = await haystack.matchTemplateAsync( needle, cv.TM_SQDIFF_NORMED, @@ -17,4 +17,4 @@ export const matchImages = async (haystack: cv.Mat, needle: cv.Mat): Promise => { +export async function scaleImage(image: cv.Mat, scaleFactor: number): Promise { const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0); const scaledRows = Math.floor(image.rows * boundScaleFactor); const scaledCols = Math.floor(image.cols * boundScaleFactor); return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA); -}; +} diff --git a/lib/provider/opencv/scale-location.function.ts b/lib/provider/opencv/scale-location.function.ts index 2dba04c5..814160c0 100644 --- a/lib/provider/opencv/scale-location.function.ts +++ b/lib/provider/opencv/scale-location.function.ts @@ -1,10 +1,10 @@ import { Region } from "../../region.class"; import { lowerBound } from "./bound-value.function"; -export const scaleLocation = ( +export function scaleLocation( result: Region, scaleFactor: number, -): Region => { +): Region { const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0); return new Region( result.left / boundScaleFactor, @@ -12,4 +12,4 @@ export const scaleLocation = ( result.width, result.height, ); -}; +} diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts index 3dd68975..32749737 100644 --- a/lib/provider/opencv/template-matching-finder.class.ts +++ b/lib/provider/opencv/template-matching-finder.class.ts @@ -14,14 +14,14 @@ import { matchImages } from "./match-image.function"; import { scaleImage } from "./scale-image.function"; import { scaleLocation } from "./scale-location.function"; -const loadNeedle = async (image: Image): Promise => { +async function loadNeedle(image: Image): Promise { if (image.hasAlphaChannel) { return ImageProcessor.fromImageWithAlphaChannel(image); } return ImageProcessor.fromImageWithoutAlphaChannel(image); -}; +} -const loadHaystack = async (matchRequest: MatchRequest): Promise => { +async function loadHaystack(matchRequest: MatchRequest): Promise { const searchRegion = determineScaledSearchRegion(matchRequest); if (matchRequest.haystack.hasAlphaChannel) { return ImageProcessor.fromImageWithAlphaChannel( @@ -34,9 +34,9 @@ const loadHaystack = async (matchRequest: MatchRequest): Promise => { searchRegion, ); } -}; +} -const debugImage = (image: cv.Mat, filename: string, suffix?: string) => { +function debugImage(image: cv.Mat, filename: string, suffix?: string) { const parsedPath = path.parse(filename); let fullFilename = parsedPath.name; if (suffix) { @@ -45,20 +45,20 @@ const debugImage = (image: cv.Mat, filename: string, suffix?: string) => { fullFilename += parsedPath.ext; const fullPath = path.join(parsedPath.dir, fullFilename); cv.imwriteAsync(fullPath, image); -}; +} -// const debugResult = (image: cv.Mat, result: MatchResult, filename: string, suffix?: string) => { +// function debugResult(image: cv.Mat, result: MatchResult, filename: string, suffix?: string) { // const roiRect = new cv.Rect( // Math.min(Math.max(result.location.left, 0), image.cols), // Math.min(Math.max(result.location.top, 0), image.rows), // Math.min(result.location.width, image.cols - result.location.left), // Math.min(result.location.height, image.rows - result.location.top)); // debugImage(image.getRegion(roiRect), filename, suffix); -// }; +// } -const isValidSearch = (needle: cv.Mat, haystack: cv.Mat): boolean => { +function isValidSearch(needle: cv.Mat, haystack: cv.Mat): boolean { return (needle.cols <= haystack.cols) && (needle.rows <= haystack.rows); -}; +} export class TemplateMatchingFinder implements FinderInterface { private initialScale = [1.0];