From 85da0ac5e3b13c1270d2a42a8f257a29e4067cd8 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Fri, 4 May 2018 17:23:01 -0700 Subject: [PATCH 01/10] Initial version of the speech extension. There is certainly still some stuff to figure out but the core of it is here. --- package.json | 1 + .../scratch3_speech/assets/speech-rec-end.mp3 | Bin 0 -> 5634 bytes .../assets/speech-rec-start.mp3 | Bin 0 -> 5947 bytes src/extensions/scratch3_speech/index.js | 755 ++++++++++++++++++ src/extensions/scratch3_speech/manifest.js | 4 + 5 files changed, 760 insertions(+) create mode 100644 src/extensions/scratch3_speech/assets/speech-rec-end.mp3 create mode 100644 src/extensions/scratch3_speech/assets/speech-rec-start.mp3 create mode 100644 src/extensions/scratch3_speech/index.js create mode 100644 src/extensions/scratch3_speech/manifest.js diff --git a/package.json b/package.json index 37656764..3ec63b3e 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "canvas-toBlob": "1.0.0", "copy-webpack-plugin": "4.2.1", "decode-html": "2.0.0", + "diff-match-patch": "^1.0.0", "escape-html": "1.0.3", "eslint": "^4.5.0", "eslint-config-scratch": "^5.0.0", diff --git a/src/extensions/scratch3_speech/assets/speech-rec-end.mp3 b/src/extensions/scratch3_speech/assets/speech-rec-end.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..eaf6556e6cc7f8f146486084af61ca16d6c22f4c GIT binary patch literal 5634 zcmd_uS5Q+=xB&1FdW#|SDkQW(K%|L)bdX*{6X`{W(mRSK(yR0qiaB??WfJ0!UY2T3@@Ja488~4?KD_x2>Ms2T@v}`U&hWurXM{jOo>&81V8*WM&)O}{ zxThw@rwV;0t5?!CyMyhI>hJD;Jj_wG?92E__HG;_y_LX(QcwZugA`kUbvMQ2QkWZL zgaf>T=Au787u@)!#stxbJw%PwU1&$MWW&#czyY`ridE^tpn!LDB`dGWtb}FO zrHlSzSD@Y(Ht+PrAT3Z0;ZccJH-!uU)5b79Fcn>YdWHT7bu7kw-Hk#&Ny0SrDJ*)W zHF~AGiRot6ScI-h!J>uuOm$2llq?BI z1D4u@4v~*T4>-^O7CJg>^BOcCh?|w`68RL}M~h~CUwJiM_4b?a@+S&!3j=Ytg@Wim zzD8>eNKlAqVTFXkZqU2`cw?nADr3}WaJ|OXYDi+>*>O3QpNN6IKZ5Gap?^50{c=a} zV2Lf~7xUV@-XKkP8ygH?ZjxzsQ;*`}iuFA?1L->s7X`hG{(#su+d|S(b>W^WpFVrIcZTfWkrsE(7$oB57 z)c&b8TbSbLW+ciCul-C2+oGdo`BWjItYj|{MoKw49F($t#lT_fZGO9CB-P?IvLM%m zEgGj3`-|nfm9``m&CR23rkSaP*vTw;p&s&SKp#7RrM=Q($CVnY{R|bICsePCG<_?V z1<>IMC{|Hsb)(K2kFsvtvS4E`yH2n5)ieF1Ay%G8i&xaBlwkEpnM_{A^iY9zJV?k@ToT+YKkksPrL z6nACU(#C9Hbsp-|P*FM}Dn%^H1wz=Nx-8RphnbYVidP9%9nJTSSG6#H?(TO!ViL2h z4e>OFkcf8-32R))INaVoDxJr>z?5B*Q+_|@pra=Ln@3E1`9vM6H{lAzpjo3SpV@4h z{AXY$_os#bV7>0*;+S*jNVNYCiwo1ZLA#B1E=6E3H#a;6NQKu?CEciOo>dHwwD;!E1$L#n_ zRO!s`O_``!rE&7GbDDOL|8%1^{f{wEz}1c>xR(w^FWw-^*UFfo@w7E-oDx?#No7-r zRh9$3WgD^NkRT;fI?b4AXUdmn@=TDb*$M_mPA_2H^3sSHbAT&`K~Sz@6OFMP7b)rz zm#XPnk?;21VW(3sdQ=;=-Oz8+7(yo-BPNcobrPvto-uGL^3X5Jx~wyv!ipD%*+g!2 ze2InTFn*0?9?(Jz-NYoB6K}tA(~*Qj%Zr}Bva!;6t2bqZXSD27fL|U=;!7pF-X2Q` zD((s`lsy6#Qa_~Uq}2gPmO2c}K8$~LK6_{Z;EhVF?Jl|sPzx%$5_E^c$JC{KVhvP* z=+X;_rI;@*KlC)TgX!kc8_6k~=`!>1_&5-a?QOWaPWEaaDW?GC-?A$N%%14tnVWyV zhPw5P5r;?A-i)CT+e*4sN>?uzqP$AvIh>JpU)l}1NNYv8t~0w4>AbOo2_-`wvM5T^ zd_y6Jh+H>_VWs3o+}~^H6&%F4xKjlPOR=VE7&B@8ta(VyS@P}cq_8uSq6+V}g6aU& zuPR&K+pyyBf$Rl`vgRT3mnXgYHkZO=#+Qj@{Wybil?>zPa^b?bmV75(jk>nFIKX0> z&GQ>tn3P=_c~*`8gV{*SkV`Jv#quZ0twVDK3zDFyx56f-*}Zk99NX1 z6HoV1op*fE(JJz-@K+#3A>sYUt`wEb&IN?(PkU@+zL2b?%MI~_3z={B>dwfm+kToG zyM<-uO`-$M9s!??#^lWWvMo3;aS*^~pjV2>k%Ny$u#5!MhzmCYb6^6>%dQ395Go;=ZIv7w^eG-gfW5Gg*jHdRyraWrL9C4X1j($4-M#UA%YZD-@8jB0Ql%P2AC z#KV7k&*Y`qTe}HO@BN&cIOI?iNh3Xjl}1$O zir?}M1{ed#$y1|PL&OhI@ra)Kod2123}D(7@Bl>3`46TjO&;c$Gg{P zLeuGolLC*dPi|CR&DZ2rI35$Uz9))`(F|dO?-TB?iH-TsdcC9^ zmFv;#A)c4i3={zl#mP z2;4XHH_C&C3(p^x-g1)|`V@MeLVvfP29;(zmzh`9o)MyR$`5jA;IKHs&gJ(>1-bc1 zQAhL6u+dV$gbT|=oLy6f@{zac-*+ConTjkLEuML~oaWh5{rzV!@P%!#+IsJTB+r8DhlX?w~Dkrmlq)vO3|PxGQN8=5;I~4jk0v&tTfj4qyaw?lM&b#PP~8NPp68 zQaHrc(LAv0x>5w=5iWM4KA6xJ&nDFI@(_H(C-Bzc+COH~TAcH_7sbaA|an5I~LS`u>2l zecF>9wH-nwcq6BTQj*Sc&vCGA{AvPPTCF^sgRe%3-qt!mMkJT+Ob{ppKOK0=k%)m! z#kz*TZxC`>7BYA`F+C#=OFuj%sb8DxgV23|*Yw(4mre9%y|WaBPYsN#z2a)h9+5(M z46_`Ebn6o2yyk|MJB+$vYeWb&*3ZYLa$>pxZo^1wElMK)k<{;MC)fo}4lp3t%^|g2 z_?V_E8lNl77xw~wRh|cq0S)$bx7uz3qZDhmx^gQ_MCpv<^0zWo*w(_o0aG$;CdAKP zP*3d7&`l7TRP8P*Zb>j*%bMVU*YD;sKxioObaZYt_`JEJUS>_Z7KTEw`AfdQ9W##6qN93k9Vx_7hiY35~p2^rdEZA6PEa+^gI8R z8UnEx=fP%yzHwTYFMZ;xz>K`(R*u{>MoFr3+kkxzvf6M)@Ynj)?j|;uWG9>Ty3FG| z^Y|bTHrT^d?}P5h%{J5g_1i<8Rl`8D3i-0hH&|nye&T#}X zH?Xf^`evu*Vy~RnD>eX2$$8NZ-4d^nxK_v2bul15jh5v2tcTwt;oYsJQ_~^kj|gt^ z;7s+sKCY&< zm$NXIqmaz@#WaQH;ebMJuI$aUCVqZ_m(1xLOaKijVrNzpAfi!*)l|a)iTEuTnqyd- zI%>b)G-+y*CILuvDP{;B+Ywb=jQHqc4!0L%))PBaJq*Tv8B5Q9CHp?p*ezfHI(obz z8XZrn=7#QcJ~$w|54?mmqWgBpc6u)7-{b+?}7VxY1DJkNVoo zO_A%TOQCX0|LPT*#WBTtM>rTICb<<`6FcosSDs_;YZrOf=el@$F8#d}zM zI`)hp64FT8c@~R{D~T)XB9(pxLumVI6D`9Qa?{zPmyO@rS7x}wG6!IQ4cDH)#XtA_)nP#*=^x?nk?r@GwTKl>D2jQgJG%OD&BYi1NW^-Ql^X#(4&G~j$07jYJV^WoRkPuEL`vg>kJD|cex7z*=nKSbi*2qz0_`(+S2pO{pU6Zx`-q*pHQR3Y+N8=CrYPJZoHnu9k3f=8 zyQ#UkH@hOf@A1xCUp>X>jZ^W?%GTtx(+7DyBC+<@V3xj&d*EWlC#NXCzM`Aq;RdKD zUTnl0Mm_W6GBqTH$e6(9xyN*(6aoYMVeb|TShafUmg8?n@nXQiEq;>Q6~-woEs8gt z@2EH$jbaRx3;y^AKTe`oTn|c}|20ptmbH_%&DXwD_siFKe~<)UzS=a+SS0D6r~Bgy zwC5lf@6z^F$KwIX42i3@ue-F@Zrk^Da=NCd)6>gWapjxw$*Y<;^kr{ZJ>!E$kbFTnPVm!Qgct#p#cg5V1X9xmv=q%I+rBZU{~{lx z@a=1y_TnB-Q^jYqaeeB!m&~l@MA+TpUKUw-&;ZcWDi1XtWX?f$$&cqbb9lVVi`g#@ zG-#XL^#o$x$Y?;hB5JxPE|%J_#wSpGe&j7i?LMUHeI?QJuVP5XDL8@9Pt;HE^Rc$)t(7e5<8t!=Dked5#5z zIm2(O5gp8+WOf69{pjW5;HIQoL5DyL7$Nh3qkg%7R{%p2>CQPnpcz8I==)sF*(OWkBQ>vgs z6i7fpP-%JMoBx+L^FFDovGk14q@0>m7?4650PagsNPv&l3E`P^p{vLDy0OK71 z)itJT{MTgu(Y$7S&GDMgKatl`uRXt3`_G$equ1uIZT*A2c76?i?eDwvwN-S~B}L?9 zeTh5&GbJew?iKYP8 zj?SrT!mn4HJTT-G6G{X?s9n$}GIf78*%?~#Q8W{rO;Ou%FC#)4;bi!V);qHUr#V5F zEW`zIYt;T>ZW3>F;_PZ$BySqn9ygUIEW#nv)Y8DUR85VvZioA3(ub)BpGpOhW%z&M zY1P&`XUuK>qPWiV4!X}+R}fC>?#2B&g+l4e@U!O=Ht?A6kE~AY_gx#e5mSq5ONllb7KOA(^C&9`27*OqA~L-M-YMifoUV!ipiV%m9ks zRiWIFqBsx>Nt~*?zq{UIqMqSx9j^+e;OijJMA8`K$Q4CJ5`xNH#IsY}1gF`b76Nm& zBDd``1mT8UrFT<(d((993G**%mD8Po3JNIt`AzbkEaAc;&39Pa&WOCNSz_LPn#97UNJs65maJ3Z zv-vq-6in4Y$jazUCe9(NQA0@ew6)(-hff`g#4#IaD|jfMGVR|J6nk}p3Njqauy8_x za~yVJ^~JpW1*PlHnA%DqM~|za_S0Jxn@%VDF^xqFA={iOFBUmYI%7gN&!otXPFhd- z)S!4|gw;lyD~Jr)|HwfFNtI7O5g&y=j~pQb&^o`AOKI{XBDhoe0}YjHMZ73BhEo!A ztjRyJ+J&8x@&V15`9y2#T;d}-DZbyNXIuZCphmH^oT3t6t}i+8RkLSkd$`F678F+i z^SvqF;00a&G_R5Bgb{M0k36FH&_y;=9UI!_cC(PuMjasZUZ*E{OTeEn*i5JePJiqY z`t2vw^@ueg&sEv{+tX7i#8M&x03a+8E+2y$Q4-vXMX&?%`LqH{2E{(532~mXL4*R? zA#vP*qbio7QdtfHWxAW%gn%hpCQi@A{L;!La~tqXlV1`&vpDzkuAhj|7(iEbTU_%A zAbWP2jgtJ))*V@&?UvO%w|AZ2U?a-UF#^r8vO6YiT{&L`Cz?*G|9sv5xf+5xvpsnM zJP?>>ZbwP*1kRsFx{0bJ6AOEd{))z*KN_I`&}Y0f!-|CVUOcGLZh+vSh%NcKMt6l?lEXfJkwy_lZP`AqH1zQB>Ov@yeg8ucw;H#m)RQk$0;pV78zS*oeU#$(IavZk!qR&**Ci+3RU&H)+TUW}@4SSi8NZlU(`dnZDf| zwp~}^#iQu9Y?z|hLD=>mR(=Cc}oIK4aDwjkP4F}!f7j@(ZbL0!9{qaLqnQ5ah6$xY-M z97ssaUIr$G5KP1(2*GN#ZKnlKQJ?s>2ck6@Zkxeq^jxl>GSW=~4{rVvZe3>zg910} ziWY04Bof4|<;H2IqmHTl?z1k|yO^k#HCnfgW^%0yPaNWcMB9%?0u(QnDAT>w)7wz*?%PbwNo(;pOND}JBwP0tqgi?nC8Z%ZkhK+k z6Zf3=Sqi+mRugZVNh^44E-H=PIpE3-pgAw}qwOwQf2-H|t$ANH=7tXSO>0ZIRIZn6 zhJEgDr^OK{chAjTiR}BP>4(RTi*xqYmtQlyOTO2DCGtn_c{peh7m0o(*g>kc3G0t(8Yd zLJb7&X_5r9=+FDzrDP7jYglRcleaCAAd2M(MNi>II~>EDo(n?Up}Q^8m`KKE=s?H z^6t8s!Tws|0PiOD;x$sLrQRa&S0EjXG_wKM|Au5-Y)v?0J`h1c3Kk~R=aZ+-%V~?u zH`eYq^~?SdDW6VNG1m0UVhNr^Q|+WIC7;mQjH|WmDce<7*fe$o4T_bq$Qky!-LkBR z{}JCz_-*VVrB#;Mu0#*pIU%Cn{gbK=;INdM7=zRa;UAwXGm-sX@bbBCpw!UY+UbN~nz3 zEi$M>2o9`Lz~4uxhTeb{(PZk%1g5_ zLmf!1VV5Yf5&niSx8PA^rZNjKDoPEQp5v&Z%u${YJi%=$zosjsE@-izp_z9^#;6JEnY&A*S|X)<@d;rAARY2M9b)-iJ6Ba$3_~#iPdTn$f77y zL8tx8k~2eV>MjW1x7%c#G3E^I9HbnbpPCmLmGr*XEN%LTasxpzH#B?aJHHN@{UN1} zS+{N=lgGK>@z{3w4(UCI2F)-Ah)v;@~VmuUrR?1VGDPW`^Z3`>Y^^yo}{c_O*-wiL(-v@kM`M zHTdjlxqzNFK&cqWHG~4Qs!CxNN!q7m7P&k|Co4ECj*LwIw4V~K4Jcs9z+z(($HVL$ zHN@10({2VWuDJ7Sewe1ZrMDWvcCv3g>rlPdq`*QsZSfQ%G^iHAXeN8gP#Vil(ynxn zUATE!9;XrN+xG??XgP2r#U>W54)#vt+D#?Y|gQhb^#u5)XrTWC1voN$-j~u^A zxmEmeIJ*1N&ueq%OCUEUt{tY$wAXAB?)!V{vjfv7TA}#B8r_inz5RrrwN809wfCM_ zt|(;>X0JoiV=6!vMH7Q);$ql!;BElGKKCrJbqfO2ReP72sCwqadiz1*uJkG)*nY1d zqHx#7fGqJ3AbB|-#va!axS&K9B{_A!`hHeAf2ns+0G?~f`of3*2w6e*J0}l`-nhUY zPyIrOEM|VtJHPb)h|to>Tq)abQg+blltn*HHRX2l3i^=t`ttL^L8xz5KmHBx)$fIu zXs@zmEL~vFRxlkShy+sjoVBxOZcv{iG)6yaM7!RKVJW8M7=>297vsD7K(y*5U9(^p z%Ht9?T>R|q2*9=TW5u5GlV*Q^ICVuI_#c0Di2cpub>{zq} zn+W2)ZdkXMalPkZW@gI`N&eU4z>(m$WhU6-fX`gl%MqYwBmvJNhPKA37+Qy`0qVA3 zmFNW~W#`BGn{R!4v-<8EcV080CJ8V|`u*ia{h{uBuf@U5cPou}JjUfc6;m~#qmJNw zY-+pQ_U>VU3hT0?Jv8}j>z(`8rp_FS6X=bD_j`Vp;$~khmSn7eC6;QRwPM`#Z9!Z7~wQNK3CJtyQ7jHs|{sRzXLP z*ysXj8P9U3$fujbuJo^$7EA!}-%`bK_&F#_BE#cK0^YGeK2DSCN@&l>bwL;#Nb;eT zmq<~2dn9<0-;G+@`y%bw^c)wPm?c=+`Pz$V85F>!k04HD=s-yvP2&q`PbFfDO~X*Q zh@t|F`V5?F$`<^nm%cRJEl^`?tEvUp`-Wv(Jc?dt2*v4o+~gK3*2>KZotU4g4AE-j zt(&gn377PoeTW#FldLY}WSnSxp7e2EiI+D?)~76Nk*$-Qr4G(xb z;-MuXkU;+QS%8epb=eU`f-TYvu!`iIR5b#MSwQ1{s{{wlXSDHIY?K{acAZJ%#nXfU zQX)BWNBVcJnUVvup@t{7j#?_liwi6C=>U}!rtm^WLZPyW4vfLDQy3%qdFu_D2W!R} z;iF7)83Lg%WUGI#VpuBEn0u_U;P*N2KkKrPm%=aPC*_w2Ya4gNZ7w6~se+;(blYas zIKGIlEJ`X9vYAVSuymc{$kNx!=c5Ip$5VVN%^l37P1^zEE^OoHcb}{%=(4A3ad>%l z(eV@;f0|R_{pAIFQk>|l%0ku6*~R6hZzTDF6Dk!aL;8hSey-)AuAjVv%UW`<(X%KZ z@h8=)p-~N9X-)x`ojpw-vV?lcQI^lTF)++i#Cvg4fHQ0TeLp+6n@rz+6eo$F-RRMj z&~%3aNAZ3AG#%;b>+AtWf~a5WVe`GK|1=1MX8~cQtX|9Ws`oh+?1_DjGTX@J>sv`k zveiG`9-pO(YZ!W{SE{2KNH#8`X#2om;6P+i9y-ipQlQHp0%msB{3zr_)$;0|?j125 zCH(3=JlnbU_-ZK%dKZHkc9yL=7!5FPDdVB@)Dv>8kHBBH#Aa6g{X@+U59ZqJ(JF8m%?7{aF<19?P1d0eEbd#+ff`{Mb`*4dCIJnC2Lo^wLA zzYl|F1RnpiL@e3*zneD5#K-{4&N$qfV!tqxU%yKKikNsgXpH_nr( z;GTn^@rCg6Y0U6TWlf-L?Wk^j=i#FU3(L@Y8%d`lyPl*u1Ucw2=iQOIz|<<~Z>qPr z*b>&i>t#Zg9cF9H9j7be+Jc4sqUiHc+hX!3IGXjlfhQ3=H;9t*+un~zaYnYN+`8A7p(}>etuY#pULPnohRxs-6fY4 zI={T7k>R%*|#kD95Luk(MF=PdO}RSAqUA?Go?ZR{!KM^bi20$RYzG8!Di zJ3(zwgK}&C($1C45tgD7ke~19D&^+Y@1`}1{M>qziOP;WMXf64aeaz--WyVPTe4aY zLUvjTqoeTL>+FF<()5CL{syl4k`<}cGC+cQVXQYbN%Tr@A!yCQkGx)e>Q7<7D%En- z&t^U+eUKUbz#-5C zn%-ZF!g>KdloZKl`2G(}Nff5~RWn>r7ANlS1IhR0K8|MpzR$k8P|x>LZw}s{(#zQM zL7cKlr9Qyzc@Eb}#p7RYD{lb}D-CA3bXL9TAM;zUtB%ty7}&D-N!Fv_b$~ zLT0JS#evW8>+AtW-l`~+`!fvy#1L=4i3SlfrBHzY;23QQA_+0j$W|FaOrcYkSce+1 ztSPTk!4%(t8hH~FcnqmeJL|Ulq`LBDl`n-ZpXXN8!`NHe-+G#>2$GlGQP zH`xvEYoum8kh{gKDb9Y)LZj~|ZpP&AgY3rUYA~Z-HQlkK;zbfA)w)BuxGn_;b(*m}me%Q2uw1m;(TiiUa_N L|Cf9JFT?)@FILcA literal 0 HcmV?d00001 diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js new file mode 100644 index 00000000..873c1003 --- /dev/null +++ b/src/extensions/scratch3_speech/index.js @@ -0,0 +1,755 @@ +const ArgumentType = require('../../extension-support/argument-type'); +const Cast = require('../../util/cast'); +const BlockType = require('../../extension-support/block-type'); +const log = require('../../util/log'); +const DiffMatchPatch = require('diff-match-patch'); + + +/** + * Url of icon to be displayed at the left edge of each extension block. + * TODO: Find the final Icon. Replace it with the right format. data URI? + * @type {string} + */ +const iconURI = 'https://www.gstatic.com/images/icons/material/system/1x/mic_white_24dp.png'; + +/** + * Url of icon to be displayed in the toolbox menu for the extension category. + * TODO: Find the final Icon. Replace it with the right format. data URI? + * @type {string} + */ +const menuIconURI = 'https://www.gstatic.com/images/icons/material/system/1x/mic_grey600_24dp.png'; + +/** + * The url of the speech server. + * @type {string} + */ +const serverURL = 'wss://speech.scratch.mit.edu'; + +/** + * The amount of time to wait between when we stop sending speech data to the server and when + * we expect the transcription result marked with isFinal: true to come back from the server. + * @type {int} + */ +const finalResponseTimeoutDurationMs = 3000; + +/** + * The amount of time to wait between when we stop sending speech data to the server and when + * we expect the transcription result marked with isFinal: true to come back from the server. + * Currently set to 10sec. This should not exceed the speech api limit (60sec) without redoing how + * we stream the microphone data data. + * @type {int} + */ +const listenAndWaitBlockTimeoutMs = 10000; + +/** + * The start and stop sounds, loaded as static assets. + * @type {object} + */ +let assetData = {}; +try { + assetData = require('./manifest'); +} catch (e) { + // Non-webpack environment, don't worry about assets. +} + +class Scratch3SpeechBlocks { + constructor (runtime) { + /** + * The runtime instantiating this block package. + * @type {Runtime} + */ + this.runtime = runtime; + + /** + * An array of phrases from the [when I hear] hat blocks. + * The list of phrases in the when I hear hat blocks. This list is sent + * to the speech api to seed the recognition engine and for deciding + * whether the transcription results match. + * @type {Array} + * @private + */ + this._phraseList = []; + + /** + * The most recent transcription result received from the speech API that we decided to keep. + * This is the value returned by the reporter block. + * @type {String} + * @private + */ + this._currentUtterance = null; + + /** + * Similar to _currentUtterance, but set back to '' at the beginning of listening block. + * Used to get the hat blocks to edge trigger. In order to detect someone saying + * the same thing twice in two subsequent liten and wait blocks + * and still trigger the hat, we need this to go from + * '' at the beginning of the listen block to '' at the end. + * @type {string} + * @private + */ + this._utteranceForEdgeTrigger = null; + + /** + * The list of queued `resolve` callbacks for 'Listen and Wait' blocks. + * We only listen to for one utterance at a time. We may encounter multiple + * 'Listen and wait' blocks that tell us to start listening. If one starts + * and hasn't receieved results back yet, when we encounter more, any further ones + * will all resolve when we get the next acceptable transcription result back. + * @type {!Array} + * @private + */ + this._speechPromises = []; + + /** + * The id of the timeout that will run if we start listening and don't get any + * transcription results back. e.g. because we didn't hear anything. + * @type {number} + * @private + */ + this._speechTimeoutId = null; + + /** + * The id of the timeout that will run to wait for after we're done listening but + * are still waiting for a potential isFinal:true transcription result to come back. + * @type {number} + * @private + */ + this._speechFinalResponseTimeout = null; + + /** + * The ScriptProcessorNode hooked up to the audio context. + * @type {ScriptProcessorNode} + * @private + */ + this._scriptNode = null; + + /** + * The socket used to communicate with the speech server to send microphone data + * and recieve transcription results. + * @type {WebSocket} + * @private + */ + this._socket = null; + + /** + * The AudioContext used to manage the microphone. + * @type {AudioContext} + * @private + */ + this._context = null; + + /** + * MediaStreamAudioSourceNode to handle microphone data. + * @type {MediaStreamAudioSourceNode} + * @private + */ + this._sourceNode = null; + + /** + * A Promise whose fulfillment handler receives a MediaStream object when the microphone has been obtained. + * @type {Promise} + * @private + */ + this._audioPromise = null; + + /** + * Audio buffer for sound to indicate that listending has started. + * @type {bufferSourceNode} + * @private + */ + this._startSoundBuffer = null; + + /** + * Audio buffer for sound to indicate that listending has ended. + * @type {bufferSourceNode} + * @private + */ + this._endSoundBuffer = null; + + + /** + * Diff Match Patch is used to do some fuzzy matching of the transcription results + * with what is in the hat blocks. + */ + this._dmp = new DiffMatchPatch(); + // Threshold for diff match patch to use: (0.0 = perfection, 1.0 = very loose). + this._dmp.Match_Threshold = 0.3; + + // Come back and figure out which of these I really need. + this._newSocketCallback = this._newSocketCallback.bind(this); + this._setupSocketCallback = this._setupSocketCallback.bind(this); + this._socketMessageCallback = this._socketMessageCallback.bind(this); + this._processAudioCallback = this._processAudioCallback.bind(this); + this._onTranscriptionFromServer = this._onTranscriptionFromServer.bind(this); + this._resetListening = this._resetListening.bind(this); + this._stopTranscription = this._stopTranscription.bind(this); + + + this.runtime.on('PROJECT_STOP_ALL', this._resetListening.bind(this)); + + // Load in the start and stop listening indicator sounds. + this._loadUISounds(); + } + + /** + * Load the UI sounds played when listening starts and stops. + * @private + */ + _loadUISounds () { + const startSoundBuffer = assetData['speech-rec-start.mp3'].buffer; + this._decodeSound(startSoundBuffer).then(buffer => { + this._startSoundBuffer = buffer; + }); + + const endSoundBuffer = assetData['speech-rec-end.mp3'].buffer; + this._decodeSound(endSoundBuffer).then(buffer => { + this._endSoundBuffer = buffer; + }); + } + + /** + * Decode a sound and return a promise with the audio buffer. + * @param {ArrayBuffer} soundBuffer - a buffer containing the encoded audio. + * @return {Promise} - a promise which will resolve once the sound has decoded. + * @private + */ + _decodeSound (soundBuffer) { + const context = this.runtime.audioEngine && this.runtime.audioEngine.audioContext; + + if (!context) { + return Promise.reject(new Error('No Audio Context Detected')); + } + + // Check for newer promise-based API + if (context.decodeAudioData.length === 1) { + return context.decodeAudioData(soundBuffer); + } else { // eslint-disable-line no-else-return + // Fall back to callback API + return new Promise((resolve, reject) => + context.decodeAudioData(soundBuffer, + buffer => resolve(buffer), + error => reject(error) + ) + ); + } + } + + /** + * Play the given sound. + * @param {ArrayBuffer} buffer The audio buffer to play. + * @returns {Promise} A promise that resoloves when the sound is done playing. + * @private + */ + _playSound (buffer) { + if (this.runtime.audioEngine === null) return; + const context = this.runtime.audioEngine.audioContext; + const bufferSource = context.createBufferSource(); + bufferSource.buffer = buffer; + bufferSource.connect(this.runtime.audioEngine.input); + bufferSource.start(); + return new Promise(resolve => { + bufferSource.onended = () => { + resolve(); + }; + }); + } + + /** + * Scans all the 'When I hear' hat blocks for each sprite and pulls out the text. The list + * is sent off to the speech recognition server as hints. This *only* reads the value out of + * the hat block shadow. If a block is dropped on top of the shadow, it is skipped. + * @returns {Array} list of strings from the hat blocks in the project. + * @private + */ + _scanBlocksForPhraseList () { + const words = []; + // For each each target, walk through the top level blocks and check whether + // they are speech hat/when I hear blocks. + this.runtime.targets.forEach(target => { + target.blocks._scripts.forEach(id => { + const b = target.blocks.getBlock(id); + if (b.opcode === 'speech.whenIHearHat') { + // Grab the text from the hat block's shadow. + const inputId = b.inputs.PHRASE.block; + const inputBlock = target.blocks.getBlock(inputId); + // Only grab the value from text blocks. This means we'll + // miss some. e.g. values in variables or other reporters. + if (inputBlock.opcode === 'text') { + const word = target.blocks.getBlock(inputId).fields.TEXT.value; + words.push(word); + } + } + }); + }); + return words; + } + + /** + * Resets all things related to listening. Called on Red Stop sign button. + * - suspends audio processing + * - closes socket with speech socket server + * - clears out any remaining speech blocks that are waiting. + * @private. + */ + _resetListening () { + this._stopListening(); + this._closeWebsocket(); + this._resolveSpeechPromises(); + } + + /** + * Close the connection to the socket server if it is open. + * @private + */ + _closeWebsocket () { + if (this._socket && this._socket.readyState === this._socket.OPEN) { + this._socket.close(); + } + } + + /** + * Call to suspend getting data from the microphone. + * @private + */ + _stopListening () { + // Note that this can be called before any Listen And Wait block did setup, + // so check that things exist before disconnecting them. + if (this._context) { + this._context.suspend.bind(this._context); + } + // This is called on green flag to reset things that may never have existed + // in the first place. Do a bunch of checks. + if (this._scriptNode) { + this._scriptNode.disconnect(); + } + if (this._sourceNode) { + this._sourceNode.disconnect(); + } + } + + /** + * Resolves all the speech promises we've accumulated so far and empties out the list. + * @private + */ + _resolveSpeechPromises () { + for (let i = 0; i < this._speechPromises.length; i++) { + const resFn = this._speechPromises[i]; + resFn(); + } + this._speechPromises = []; + } + + /** + * Called when we want to stop listening (e.g. when a listen block times out) + * but we still want to wait a little to see if we get any transcription results + * back before yielding the block execution. + * @private + */ + _stopTranscription () { + this._stopListening(); + if (this._socket && this._socket.readyState === this._socket.OPEN) { + this._socket.send('stopTranscription'); + } + // Give it a couple seconds to response before giving up and assuming nothing else will come back. + this._speechFinalResponseTimeout = setTimeout(this._resetListening, finalResponseTimeoutDurationMs); + } + + /** + * Decides whether to keep a given transcirption result. + * @param {number} fuzzyMatchIndex Index of the fuzzy match or -1 if there is no match. + * @param {object} result The json object representing the transcription result. + * @param {string} normalizedTranscript The transcription text used for matching (i.e. lowercased, no punctuation). + * @returns {boolean} true If a result is good enough to be kept. + * @private + */ + _shouldKeepResult (fuzzyMatchIndex, result, normalizedTranscript) { + // The threshold above which we decide transcription results are unlikely to change again. + // See https://cloud.google.com/speech-to-text/docs/basics#streaming_responses. + const stabilityThreshold = .85; + + // For responsiveness of the When I Hear hat blocks, sometimes we want to keep results that are not + // yet marked 'isFinal' by the speech api. Here are some signals we use. + + // If the result from the speech api isn't very stable and we only had a fuzzy match, we don't want to use it. + const shouldKeepFuzzyMatch = fuzzyMatchIndex !== -1 && result.stability > stabilityThreshold; + + // If the result is in the phraseList (i.e. it matches one of the 'When I Hear' blocks), we keep it. + // This might be aggressive... but so far seems to be a good thing. + const shouldKeepPhraseListMatch = this._phraseList.includes(normalizedTranscript); + + if (!result.isFinal && !shouldKeepPhraseListMatch && !shouldKeepFuzzyMatch) { + return false; + } + return true; + } + + /** + * Normalizes text a bit to facilitate matching. Lowercases, removes some punctuation and whitespace. + * @param {string} text The text to normalzie + * @returns {string} The normalized text. + * @private + */ + _normalizeText (text) { + text = Cast.toString(text).toLowerCase(); + text = text.replace(/[.?!]/g, ''); + text = text.trim(); + return text; + } + + /** + * Call into diff match patch library to compute whether there is a fuzzy match. + * @param {string} text The text to search in. + * @param {string} pattern The pattern to look for in text. + * @returns {number} The index of the match or -1 if there isn't one. + */ + _computeFuzzyMatch (text, pattern) { + // Don't bother matching if any are null. + if (!pattern || !text) { + return -1; + } + let match = -1; + try { + // Look for the text in the pattern starting at position 0. + match = this._dmp.match_main(text, pattern, 0); + } catch (e) { + // This can happen inf the text or pattern gets too long. If so just substring match. + return pattern.indexOf(text); + } + return match; + } + + /** + * Processes the results we get back from the speech server. Decides whether the results + * are good enough to keep. If they are, resolves the 'Listen and Wait' blocks promise and cleans up. + * @param {object} result The transcription result. + * @private + */ + _processTranscriptionResult (result) { + log.info(`Got result: ${JSON.stringify(result)}`); + const transcriptionResult = this._normalizeText(result.alternatives[0].transcript); + + // Waiting for an exact match is not satisfying. It makes it hard to catch + // things like homonyms or things that sound similar "let us" vs "lettuce". Using the fuzzy matching helps + // more aggressively match the phrases that are in the "When I hear" hat blocks. + const phrases = this._phraseList.join(' '); + const fuzzyMatchIndex = this._computeFuzzyMatch(phrases, transcriptionResult); + + let fuzzyMatchResult = null; + if (fuzzyMatchIndex !== -1) { + fuzzyMatchResult = transcriptionResult.substring(fuzzyMatchIndex, fuzzyMatchIndex + phrases.length); + } + + // If the result isn't good enough yet, return without saving and resolving the promises. + if (!this._shouldKeepResult(fuzzyMatchIndex, result, transcriptionResult)) { + return; + } + + // TODO: Decide whether this is the right thing. + // This sets the currentUtterance (which is returned by the reporter) to the fuzzy match if we had one. + // That means it'll often get set to a phrase from one of the 'when I hear' blocks instead of the + // full phrase that the user said. + if (fuzzyMatchResult) { + this._currentUtterance = fuzzyMatchResult; + } else { + this._currentUtterance = transcriptionResult; + } + log.info(`Keeing result: ${this._currentUtterance}`); + this._utteranceForEdgeTrigger = transcriptionResult; + + // We're done listening so resolove all the promises and reset everying so we're ready for next time. + this._resetListening(); + + // We got results so clear out the timeouts. + if (this._speechTimeoutId) { + clearTimeout(this._speechTimeoutId); + this._speechTimeoutId = null; + } + if (this._speechFinalResponseTimeout) { + clearTimeout(this._speechFinalResponseTimeout); + this._speechFinalResponseTimeout = null; + } + } + + /** + * Handle a message from the socket. It contains transcription results. + * @param {MessageEvent} e The message event containing data from speech server. + * @private + */ + _onTranscriptionFromServer (e) { + let result = null; + try { + result = JSON.parse(e.data); + } catch (ex) { + log.error(`Problem parsing json. continuing: ${ex}`); + // TODO: Question - Should we kill listening and continue? + return; + } + this._processTranscriptionResult(result); + } + + + /** + * Decide whether the pattern given matches the text. Uses fuzzy matching + * @param {string} pattern The pattern to look for. Usually this is the transcription result + * @param {string} text The text to look in. Usually this is the set of phrases from the when I hear blocks + * @returns {boolean} true if there is a fuzzy match. + * @private + */ + _speechMatches (pattern, text) { + pattern = this._normalizeText(pattern); + text = this._normalizeText(text); + const match = this._computeFuzzyMatch(text, pattern); + return match !== -1; + } + + /** + * Kick off the listening process. + * @private + */ + _startListening () { + // If we've already setup the context, we can resume instead of doing all the setup again. + if (this._context) { + this._resumeListening(); + } else { + this._initListening(); + } + // Force the block to timeout if we don't get any results back/the user didn't say anything. + this._speechTimeoutId = setTimeout(this._stopTranscription, listenAndWaitBlockTimeoutMs); + } + + /** + * Resume listening for audio and re-open the socket to send data. + * @private + */ + _resumeListening () { + this._context.resume.bind(this._context); + this._newWebsocket(); + } + + /** + * Does all setup to get microphone data and initializes the web socket. + * that data to the speech server. + * @private + */ + _initListening () { + this._initializeMicrophone(); + this._initScriptNode(); + this._newWebsocket(); + } + + /** + * Initialize the audio context and connect the microphone. + * @private + */ + _initializeMicrophone () { + this._context = new AudioContext(); + this._audioPromise = navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + channelCount: 1, + sampleRate: { + ideal: 16000 + }, + sampleSize: 16 + } + }); + + const tempContext = this._context; + this._audioPromise.then(micStream => { + const microphone = tempContext.createMediaStreamSource(micStream); + const analyser = tempContext.createAnalyser(); + microphone.connect(analyser); + }).catch(e => { + log.error(`Problem connecting to microphone: ${e}`); + }); + } + + /** + * Sets up the script processor and the web socket. + * @private + * + */ + _initScriptNode () { + // Create a node that sends raw bytes across the websocket + this._scriptNode = this._context.createScriptProcessor(4096, 1, 1); + // Need the maximum value for 16-bit signed samples, to convert from float. + this._scriptNode.addEventListener('audioprocess', this._processAudioCallback); + } + + /** + * Callback called when it is time to setup the new web socket. + * @param {Function} resolve - function to call when the web socket opens succesfully. + * @param {Function} reject - function to call if opening the web socket fails. + */ + _newSocketCallback (resolve, reject) { + this._socket = new WebSocket(serverURL); + this._socket.addEventListener('open', resolve); + this._socket.addEventListener('error', reject); + } + + /** + * Callback called once we've initially established the web socket is open and working. + * Sets up the callback for subsequent messages (i.e. transcription results) and + * connects to the script node to get data. + * @private + */ + _socketMessageCallback () { + this._socket.addEventListener('message', this._onTranscriptionFromServer); + this._startByteStream(); + } + + /** + * Sets up callback for when socket and audio are initialized. + * @private + */ + _newWebsocket () { + const websocketPromise = new Promise(this._newSocketCallback); + Promise.all([this._audioPromise, websocketPromise]).then( + this._setupSocketCallback) + .catch(e => { + log.error(`Problem with setup: ${e}`); + }); + } + + /** + * Callback to handle initial setting up of a socket. + * Currently we send a setup message (only contains sample rate) but might + * be useful to send more data so we can do quota stuff. + * @param {Array} values The + */ + _setupSocketCallback (values) { + this._micStream = values[0]; + this._socket = values[1].target; + + this._socket.addEventListener('error', e => { + log.error(`Error from web socket: ${e}`); + }); + + // Send the initial configuration message. When the server acknowledges + // it, start streaming the audio bytes to the server and listening for + // transcriptions. + this._socket.addEventListener('message', this._socketMessageCallback, {once: true}); + this._socket.send(JSON.stringify( + { + sampleRate: this._context.sampleRate, + phrases: this._phraseList + } + )); + } + + /** + * Do setup so we can start streaming mic data. + * @private + */ + _startByteStream () { + // Hook up the scriptNode to the mic + this._sourceNode = this._context.createMediaStreamSource(this._micStream); + this._sourceNode.connect(this._scriptNode); + this._scriptNode.connect(this._context.destination); + } + + /** + * Called when we have data from the microphone. Takes that data and ships + * it off to the speech server for transcription. + * @param {audioProcessingEvent} e The event with audio data in it. + * @private + */ + _processAudioCallback (e) { + if (this._socket.readyState === WebSocket.CLOSED || + this._socket.readyState === WebSocket.CLOSING) { + log.error(`Not sending data because not in ready state. State: ${this._socket.readyState}`); + // TODO: should we stop trying and reset state so it might work next time? + return; + } + const MAX_INT = Math.pow(2, 16 - 1) - 1; + const floatSamples = e.inputBuffer.getChannelData(0); + // The samples are floats in range [-1, 1]. Convert to 16-bit signed + // integer. + this._socket.send(Int16Array.from(floatSamples.map(n => n * MAX_INT))); + } + + /** + * The key to load & store a target's speech-related state. + * @type {string} + */ + static get STATE_KEY () { + return 'Scratch.speech'; + } + + /** + * @returns {object} Metadata for this extension and its blocks. + */ + getInfo () { + return { + id: 'speech', + name: 'Google Speech', + menuIconURI: menuIconURI, + blockIconURI: iconURI, + blocks: [ + { + opcode: 'listenAndWait', + text: 'Listen and Wait', + blockType: BlockType.COMMAND + }, + { + opcode: 'whenIHearHat', + text: 'When I hear [PHRASE]', + blockType: BlockType.HAT, + arguments: { + PHRASE: { + type: ArgumentType.STRING, + defaultValue: 'cat' + } + } + }, + { + opcode: 'getSpeech', + text: 'speech', + blockType: BlockType.REPORTER + } + ] + }; + } + + /** + * Start the listening process if it isn't already in progress, playing a sound to indicate + * when it starts and stops. + * @return {Promise} A promise that will resolve when listening is complete. + */ + listenAndWait () { + // TODO: Look into the timing of when to start the sound. There currently seems + // to be some lag between when the sound starts and when the socket message + // callback is received. Perhaps we should play the sound after the socket is setup. + // TODO: Question - Should we only play the sound if listening isn't already in progress? + return this._playSound(this._startSoundBuffer).then(() => { + this._phraseList = this._scanBlocksForPhraseList(); + this._utteranceForEdgeTrigger = ''; + const speechPromise = new Promise(resolve => { + const listeningInProgress = this._speechPromises.length > 0; + this._speechPromises.push(resolve); + if (!listeningInProgress) { + this._startListening(); + } + }); + return speechPromise.then(() => this._playSound(this._endSoundBuffer)); + }); + } + + /** + * An edge triggered hat block to listen for a specific phrase. + * @param {object} args - the block arguments. + * @return {boolean} true if the phrase matches what was transcribed. + */ + whenIHearHat (args) { + return this._speechMatches(args.PHRASE, this._utteranceForEdgeTrigger); + } + + /** + * Reporter for the last heard phrase/utterance. + * @return {string} The lastest thing we heard from a listen and wait block. + */ + getSpeech () { + return this._currentUtterance; + } +} +module.exports = Scratch3SpeechBlocks; diff --git a/src/extensions/scratch3_speech/manifest.js b/src/extensions/scratch3_speech/manifest.js new file mode 100644 index 00000000..e5ce54d5 --- /dev/null +++ b/src/extensions/scratch3_speech/manifest.js @@ -0,0 +1,4 @@ +module.exports = { + 'speech-rec-start.mp3': require('!buffer-loader!./assets/speech-rec-start.mp3'), + 'speech-rec-end.mp3': require('!buffer-loader!./assets/speech-rec-end.mp3') +}; From f3544238da6c071e9e3ed798902f4bd144f35532 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Fri, 4 May 2018 17:26:50 -0700 Subject: [PATCH 02/10] Pin the diff-match-patch library --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3ec63b3e..4c26bfe9 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "canvas-toBlob": "1.0.0", "copy-webpack-plugin": "4.2.1", "decode-html": "2.0.0", - "diff-match-patch": "^1.0.0", + "diff-match-patch": "1.0.0", "escape-html": "1.0.3", "eslint": "^4.5.0", "eslint-config-scratch": "^5.0.0", From 08cbabe2ea0819951827e495eafb5951e9383e9a Mon Sep 17 00:00:00 2001 From: picklesrus Date: Mon, 7 May 2018 15:21:44 -0700 Subject: [PATCH 03/10] Use arraybuffer-loader instead of buffer-loader to load the start and end sounds. The music extension switched to this a few days ago. --- src/extensions/scratch3_speech/index.js | 4 ++-- src/extensions/scratch3_speech/manifest.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index 873c1003..a1ce64f6 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -196,12 +196,12 @@ class Scratch3SpeechBlocks { * @private */ _loadUISounds () { - const startSoundBuffer = assetData['speech-rec-start.mp3'].buffer; + const startSoundBuffer = assetData['speech-rec-start.mp3']; this._decodeSound(startSoundBuffer).then(buffer => { this._startSoundBuffer = buffer; }); - const endSoundBuffer = assetData['speech-rec-end.mp3'].buffer; + const endSoundBuffer = assetData['speech-rec-end.mp3']; this._decodeSound(endSoundBuffer).then(buffer => { this._endSoundBuffer = buffer; }); diff --git a/src/extensions/scratch3_speech/manifest.js b/src/extensions/scratch3_speech/manifest.js index e5ce54d5..cb76b3ac 100644 --- a/src/extensions/scratch3_speech/manifest.js +++ b/src/extensions/scratch3_speech/manifest.js @@ -1,4 +1,4 @@ module.exports = { - 'speech-rec-start.mp3': require('!buffer-loader!./assets/speech-rec-start.mp3'), - 'speech-rec-end.mp3': require('!buffer-loader!./assets/speech-rec-end.mp3') + 'speech-rec-start.mp3': require('!arraybuffer-loader!./assets/speech-rec-start.mp3'), + 'speech-rec-end.mp3': require('!arraybuffer-loader!./assets/speech-rec-end.mp3') }; From a93fea3e63e6ba9f079a861be19e56f6461a1b0f Mon Sep 17 00:00:00 2001 From: picklesrus Date: Mon, 7 May 2018 15:23:10 -0700 Subject: [PATCH 04/10] Add speech extension to the extension manager. --- src/extension-support/extension-manager.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/extension-support/extension-manager.js b/src/extension-support/extension-manager.js index 30e0f26c..646049ea 100644 --- a/src/extension-support/extension-manager.js +++ b/src/extension-support/extension-manager.js @@ -11,12 +11,14 @@ const Scratch3PenBlocks = require('../extensions/scratch3_pen'); const Scratch3WeDo2Blocks = require('../extensions/scratch3_wedo2'); const Scratch3MusicBlocks = require('../extensions/scratch3_music'); const Scratch3VideoSensingBlocks = require('../extensions/scratch3_video_sensing'); +const Scratch3SpeechBlocks = require('../extensions/scratch3_speech'); const builtinExtensions = { pen: Scratch3PenBlocks, wedo2: Scratch3WeDo2Blocks, music: Scratch3MusicBlocks, - videoSensing: Scratch3VideoSensingBlocks + videoSensing: Scratch3VideoSensingBlocks, + speech: Scratch3SpeechBlocks }; /** From b4205372c329e256012ca3067f0d935fe71d233a Mon Sep 17 00:00:00 2001 From: picklesrus Date: Mon, 7 May 2018 16:02:17 -0700 Subject: [PATCH 05/10] Change name of opcode to speech_ from speech.Plus, remove old comment. --- src/extensions/scratch3_speech/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index a1ce64f6..117362f6 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -175,7 +175,6 @@ class Scratch3SpeechBlocks { // Threshold for diff match patch to use: (0.0 = perfection, 1.0 = very loose). this._dmp.Match_Threshold = 0.3; - // Come back and figure out which of these I really need. this._newSocketCallback = this._newSocketCallback.bind(this); this._setupSocketCallback = this._setupSocketCallback.bind(this); this._socketMessageCallback = this._socketMessageCallback.bind(this); @@ -268,7 +267,7 @@ class Scratch3SpeechBlocks { this.runtime.targets.forEach(target => { target.blocks._scripts.forEach(id => { const b = target.blocks.getBlock(id); - if (b.opcode === 'speech.whenIHearHat') { + if (b.opcode === 'speech_whenIHearHat') { // Grab the text from the hat block's shadow. const inputId = b.inputs.PHRASE.block; const inputBlock = target.blocks.getBlock(inputId); From e8ce37e77629103224ed04ad2915de8f5102ca18 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Wed, 9 May 2018 09:20:49 -0700 Subject: [PATCH 06/10] Fix a copy/paste comment error and initialize current_utterance differently --- src/extensions/scratch3_speech/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index 117362f6..a0bc7884 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -33,8 +33,8 @@ const serverURL = 'wss://speech.scratch.mit.edu'; const finalResponseTimeoutDurationMs = 3000; /** - * The amount of time to wait between when we stop sending speech data to the server and when - * we expect the transcription result marked with isFinal: true to come back from the server. + * The max amount of time the Listen And Wait block will listen for. It may listen for less time + * if we get back results that are good and think the user is done talking. * Currently set to 10sec. This should not exceed the speech api limit (60sec) without redoing how * we stream the microphone data data. * @type {int} @@ -76,7 +76,7 @@ class Scratch3SpeechBlocks { * @type {String} * @private */ - this._currentUtterance = null; + this._currentUtterance = ''; /** * Similar to _currentUtterance, but set back to '' at the beginning of listening block. From d2cff42a993079fd098fb88d8c32f893adac1d82 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Wed, 9 May 2018 11:57:14 -0700 Subject: [PATCH 07/10] Remove the event listener for processing audio at the same time that we disconnect the script node. At this point we are closing the web socket and do not want to process more of the microphone data even if it is already in flight. Also add it back when we resume listening instead of just at init time. --- src/extensions/scratch3_speech/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index a0bc7884..166c6de3 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -319,6 +319,7 @@ class Scratch3SpeechBlocks { // This is called on green flag to reset things that may never have existed // in the first place. Do a bunch of checks. if (this._scriptNode) { + this._scriptNode.removeEventListener('audioprocess', this._processAudioCallback); this._scriptNode.disconnect(); } if (this._sourceNode) { @@ -522,6 +523,7 @@ class Scratch3SpeechBlocks { */ _resumeListening () { this._context.resume.bind(this._context); + this._scriptNode.addEventListener('audioprocess', this._processAudioCallback); this._newWebsocket(); } From c8b3c4289167d3db3ad625d4fd74907d02dc7226 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Wed, 9 May 2018 15:59:42 -0700 Subject: [PATCH 08/10] Move the add event listener code to be in the same place where we call connect so there is only one place to do it. --- src/extensions/scratch3_speech/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index 166c6de3..f78b4473 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -523,7 +523,6 @@ class Scratch3SpeechBlocks { */ _resumeListening () { this._context.resume.bind(this._context); - this._scriptNode.addEventListener('audioprocess', this._processAudioCallback); this._newWebsocket(); } @@ -573,8 +572,6 @@ class Scratch3SpeechBlocks { _initScriptNode () { // Create a node that sends raw bytes across the websocket this._scriptNode = this._context.createScriptProcessor(4096, 1, 1); - // Need the maximum value for 16-bit signed samples, to convert from float. - this._scriptNode.addEventListener('audioprocess', this._processAudioCallback); } /** @@ -646,6 +643,7 @@ class Scratch3SpeechBlocks { // Hook up the scriptNode to the mic this._sourceNode = this._context.createMediaStreamSource(this._micStream); this._sourceNode.connect(this._scriptNode); + this._scriptNode.addEventListener('audioprocess', this._processAudioCallback); this._scriptNode.connect(this._context.destination); } From 3eec3170b2008b582636f52623c8186f4f8d114d Mon Sep 17 00:00:00 2001 From: picklesrus Date: Wed, 9 May 2018 17:05:55 -0700 Subject: [PATCH 09/10] Remove some code that set the utterance to the fuzzy match result --- src/extensions/scratch3_speech/index.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index f78b4473..2199edeb 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -444,15 +444,7 @@ class Scratch3SpeechBlocks { return; } - // TODO: Decide whether this is the right thing. - // This sets the currentUtterance (which is returned by the reporter) to the fuzzy match if we had one. - // That means it'll often get set to a phrase from one of the 'when I hear' blocks instead of the - // full phrase that the user said. - if (fuzzyMatchResult) { - this._currentUtterance = fuzzyMatchResult; - } else { - this._currentUtterance = transcriptionResult; - } + this._currentUtterance = transcriptionResult; log.info(`Keeing result: ${this._currentUtterance}`); this._utteranceForEdgeTrigger = transcriptionResult; From 98a675b62ebf797fe211d1208f71099c11ae43e6 Mon Sep 17 00:00:00 2001 From: picklesrus Date: Wed, 9 May 2018 17:09:00 -0700 Subject: [PATCH 10/10] Now fuzzymatchResult is unused so remove it too. --- src/extensions/scratch3_speech/index.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/extensions/scratch3_speech/index.js b/src/extensions/scratch3_speech/index.js index 2199edeb..76267a93 100644 --- a/src/extensions/scratch3_speech/index.js +++ b/src/extensions/scratch3_speech/index.js @@ -434,11 +434,6 @@ class Scratch3SpeechBlocks { const phrases = this._phraseList.join(' '); const fuzzyMatchIndex = this._computeFuzzyMatch(phrases, transcriptionResult); - let fuzzyMatchResult = null; - if (fuzzyMatchIndex !== -1) { - fuzzyMatchResult = transcriptionResult.substring(fuzzyMatchIndex, fuzzyMatchIndex + phrases.length); - } - // If the result isn't good enough yet, return without saving and resolving the promises. if (!this._shouldKeepResult(fuzzyMatchIndex, result, transcriptionResult)) { return;