From 3ca272c615e365247de47a9db409b755b384043c Mon Sep 17 00:00:00 2001 From: Leroy Hopson Date: Sun, 7 Apr 2024 00:00:00 +1300 Subject: [PATCH] feat(test): add visual regression testing Will upload screenshots on failure. --- .github/workflows/main.yml | 6 ++ .gitignore | 1 + Justfile | 2 +- plug.gd | 1 + test/visual_regression/baseline/.gdignore | 0 .../baseline/default_theme.png | Bin 0 -> 822 bytes test/visual_regression/baseline/empty.png | Bin 0 -> 368 bytes .../baseline/transparency.png | Bin 0 -> 4019 bytes test/visual_regression/screenshots/.gdignore | 0 .../test_visual_regression.gd | 96 ++++++++++++++++++ 10 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 test/visual_regression/baseline/.gdignore create mode 100644 test/visual_regression/baseline/default_theme.png create mode 100644 test/visual_regression/baseline/empty.png create mode 100644 test/visual_regression/baseline/transparency.png create mode 100644 test/visual_regression/screenshots/.gdignore create mode 100644 test/visual_regression/test_visual_regression.gd diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62e6eab..f244d7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -305,6 +305,12 @@ jobs: if grep -q 'SCRIPT_ERROR:' output.log || grep -q 'Tests none' output.log; then exit 1 fi + - name: Upload screenshots + uses: actions/upload-artifact@v4 + if: failure() + with: + name: failed-screenshots + path: test/visual_regression/screenshots merge-artifacts: name: Merge Artifacts diff --git a/.gitignore b/.gitignore index 2cb4eb2..606c401 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ mono_crash.* .gutconfig.json test/results.xml test/test_metadata.json +test/visual_regression/screenshots/ # GodotXterm-specific ignores .gdxterm diff --git a/Justfile b/Justfile index d21252f..10ace7b 100644 --- a/Justfile +++ b/Justfile @@ -22,7 +22,7 @@ test: {{godot}} --headless -s addons/gut/gut_cmdln.gd -gtest={{test_files}} -gexit test-all: - {{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gdir=res://test -gopacity=0 -gexit + {{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gdir=res://test -ginclude_subdirs=true -gopacity=0 -gexit test-rendering: {{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gtest=res://test/test_rendering.gd -gopacity=0 -gexit diff --git a/plug.gd b/plug.gd index 9cca6cb..9875fa4 100644 --- a/plug.gd +++ b/plug.gd @@ -5,3 +5,4 @@ extends "res://addons/gd-plug/plug.gd" func _plugging(): plug("bitwes/Gut", {tag = "v9.2.0"}) + plug("lihop/godot-pixelmatch", {tag = "v2.0.0", include = ["addons/pixelmatch"]}) diff --git a/test/visual_regression/baseline/.gdignore b/test/visual_regression/baseline/.gdignore new file mode 100644 index 0000000..e69de29 diff --git a/test/visual_regression/baseline/default_theme.png b/test/visual_regression/baseline/default_theme.png new file mode 100644 index 0000000000000000000000000000000000000000..e0d0e256827ec9fd7f35f784992e9dd3bd4865d3 GIT binary patch literal 822 zcmeAS@N?(olHy`uVBq!ia0vp^CxAGGg9%9bJb4iVq!^2X+?^QKos)UVz`*pu)5S5Q zV$R#!`@NI{CE7lo%y|%E7A@Iy)Fi8T*+#*j98rO9LLnziZ`@kb@kDM*$=a3!F^%CH zS`$n;n6AgSE_WXM5^v6poIy8%{X1<=id)>~@^5@q5Eje#i z|M!l3clWufii#I&;=g^Hv-R`o>UGv>w{PF8`*YR$xBmIq$jFV$1f29$v*)%=sCp)~ zY+?HP@89Kjz1w}Le*L=a>@LH;x1}PUc=7LzUX<`>t8o+CH*m3cl?@F*W#+2b%iEYR#uy?_4f8Y zEXY_p`Q(yK`TbK2uT8D3UeYsl@3hpI*tKiE{@Jr8t5`QVGW?m;=}nuiU%q_#@88<& zx!yI$;XYhzWodcy)~&4iTzTAnzMs5x?fv)b*RS8dZ_n=CyLar!NX(T?+?s13!Nb;k zeL4_WTU+a{(N$g(mYbKi@ANIZeaX-OeDUmOv+?pfduB{Xjm(R$JhAD`!-Kz1pFe;8 zoYm+2{xf?@pRLo^H~zl!-mLR?Hh=r|zopr0OGlEh5!Hn literal 0 HcmV?d00001 diff --git a/test/visual_regression/baseline/empty.png b/test/visual_regression/baseline/empty.png new file mode 100644 index 0000000000000000000000000000000000000000..fc9d98d6ce23e9b0014d552585547fb9d6109783 GIT binary patch literal 368 zcmeAS@N?(olHy`uVBq!ia0vp^CxAGGg9%9bJb4iVq!^2X+?^QKos)S9WNUf4IEGZr zd3$9eZ-apVi=&Ud-sQbJe6$R1a&ewrGCyX+uZ`dLox2vLA8xU8zuWsrlI?^;$1*8cG2oeEB2}M8<3?Nv5&_gp6 z1!j=b70vFHMbaTwoGp0s!Cw{HC5c#dcB- zf{~hX8!ar~r5LIJ^BX!qX`k>40I+$(_5QX9$=-NBafjO$-4O{XOUn$RVlutze@9PW zui?^%#}A4jwvTG`=~tWj74kRL z*4mFOdHD!V3;fu9y_VBq zOuo8cSO*<@CCCHd(UrF7Hu|_H6LcXON$TpSkmf4*gU%BT^n<%(O{?Q*t?;(8ZMQ8B z7>5#yEXD{lCUEjW1e*_LNYXp|xpyNS%GoEcOKdi_;VgI_sFzd+_d@CO*()^jn;lP_v!&sVWKH5j z$J2whlDDid;Fh#^NzzcV@nBWpYEg$ML;|S{v?4p^ zRHj*M%}IqMm`(Fr&OmA1I`^>$CE=rH@Oeyw+3Gjt$Uor{x=jLjqjMSIr`58ssy5S* z0BeZ#cFM)V5yv#CD^J27&y9zlwY%?Z^EV#xZoPT&EF!+$c9^YhfA&6%`@3+FWl;OC zF#EC483+RgEkUT!N;d-mNzue%r=A#a*Pk_=bLrt`r_9Skh9q0aOjb@i^Nr=}wV_sG z^sC%ywU0Fy)>n*Gdbn5T7rQnRBfy}7isMoAmN}dT)+Tp(S>w`M&HFTplGjqM-}XN; zgS(fUm9$L#oa+m_``z%@?N3NEJ0uV)@pupo>QE~tvLn|Wt zK|JFy+!cE(Kjq7gNue0r9kaQ;y4xECw|jg3L=a%4;Xw_MBH4MTxFl`YaLw7Wg3gbF zcl)$k^M@eKU6!ynX)l^+6iPnCU#vZomXN@?3yu{7G8*jHtgUCIgJyI5f4ci`EX}EV z!$ca4?GH*c3njD+h9>7X-*YPGv`I4pZZ-Va_{Twy4wLf=zuj$e;rP9eSN9&SWhy`c z&cO_kOW~8AXFPijr9GEjDimCbnSkf~ARs>Jk7tZ(1gNFHlICYYcTwQ)HVaPk8}(&G zUp`JT2|3-dBI>9CZaUzl=`EYenK``{SSY!tXwLyP_jkJhBPYzQpQCU?=X8II_qE4! zs_&lo()hq6P!lajzw)0WoCvLTn}3jghBV<)^B1!>7&?I02FUf>yU+uDUcv&DyI+3S zkKpD6{2i0wBdeNMh|L9OReF82)r%+5jw;u^iw2!)&*M+7S`%Hjn3mkd#!;HgY6m%c z2`-!?7Rl!aGsd4qo%{L;d`6?Ae!H87?0r%;+nk+}f!G)314dien_y!L|d4Vd!mSOfx5JZv!A`;iNCu|==G?2E4T=?d8> z%m1orVC>4F7l$sri#SArECT{6lQ?-@^7ga6^YXE8P)eQHSUkEs@;GbdYgV{NO2dg~ zcz1xh_NrH9waSNy>XfISlu`sk?}r)?=lLq;^ZL;imm5y@b0>#o(zIP{VU{%7rS_i> zMzeiaKU?+%UhUNz0Dz@;eTf99yViB-H7pOakZ$#Ak@01k7U4qwPm$>?JxR|5QY;zA zebvvnA>!k{c4e+$VreNVWj>U?0A!aT<+c|DnaR`<0iK5x)$#?rzKQhz z9Tu4)f13bZao^2X5|uzSPpo+-KXQmRlKUCEm?`^}gdK#+5^~2PNs=EhiFCA;a!$^@ z?nQUK?y$&uoGCA5tyXyO1fhAa;Kn5%P|B%(J~hCwev!^0B4^Df z4Q$4-W&^T=;;lFqkRbI!;ufPheuX|UAKwtd-0_VRE?d9H^L^Dbr6oV`QT+2!1Fw9v zRO~l0e;@P)yW1{}#EM^F%Fq3_-r#7HSr-O2tFTxrgUzItGjgRJRkE2#W1`7 zf@=ffop=5GvHm=tQw6jP!naAk=kJ?$z>?)mXXUDo(tiiGz_^C;uAEOBgkp49ZWM|n z)`flkq2KJBG-8z3BwP9Q#$A0~qj^yp6E^%HG}+9wOA#Cq%bg8Qj1*bcf>!77foS3x zG8$`X!>q|qV}{IJYuPC6MzEAA(O5DN4-LbA4POQT<>2~|Qa>`HMN$?x-oDq_^by~K z*2p%tr=$hD@UZ@*08r^|)fK`#Qy3Y3Wc95hjyFoCeYE4Jw+AvyXaI%0U3te}h2 zv0^N*ezw!8A`Aa-pPz;H}I?P1j(WA2V#55Qr?`otu4tT zD+NnIL8>R?)bs7`@<=QYC6~Jn(x}v}R;6+d&q_y>R6^94w$9?BupXJG(hhDHCj;Uc)s-bZK8=dxR_Y)(6`VC{isUQ*8bWhU;J z!J;9~(pf=SHT_udj>i0vD0O;bx*w~Z0`-sS)M&6MB%8l=(r@)uGc(`of{_Z^&z+Ng zULLhf0KIF1l~*0^m!xUAKemTc?MetH&5g`+ef{hWNM8=}DhP?i|0yq5q2tB~%f102 zO3RHfN-bLIVvS>k#(XO=5?E68Y(}skwN-q2*K3j76omPW{)dCt&YyUFP+AfG>%o_T z9D(VN32BfWwYtztK<+81*x!m&f9w&XVqP+Jtn`>{5iRFXVlBJj4x;fnQ=&w*W=bAs z^PE*7rkA5*3e&DP)t94Wq(f^0@4|>@$mp~Wt_rOkFS(o;aEZ5$Ou$JoyH4gLAEsAL zSPqoA1t$-ioDkxpCh@{HI2~-lv!)#`AAA}a9n-3|>Qc20{zT{xE^MNm)kJ=KNUzD4 z{%nktuYa6tG8wr~~jrXRd_6)#`hc`Cr%Lv|E z_1QKb2)8P$D`XZ@^>iBg5>!iTr4zxBdYFDZ{!(~ip%o?aOI=SjlFO8KZ_p3U+w+Oc zJXWZ%6@%>*Pg9(*|A9(Bjw3f}8E_VM_eYE3K$$32u{kRTSx1+ zZoa+gSy7V^9O@VQsqj-b6|n4`O643J=CzrnTH`!5;javHM6YW~j0F0hm}zy{nXq+j zEVZ#+rGNiiVATtAH!M)RVu{sBPG=1}9H960;OBAu46)#IM73<_y187J-I4Lj;J!AK zGiydbTIHGXsTL#9e|Y&$+mrd)!g1vMVRFJ-R2KF@z^|?bK7}XU2+-lI-+ZfXGJU1C zlt>BTrpVJ%+m(k6fs+LeSwD@5jY)&{V4-rhL_^Vwe4@3zL~$oq3WvtH9^Xm zRb_eQ4xtO|O&rU6DE_LFinL9cZ*A+P{%y1a-);!W!e+jItgsbC;kHdyrr;IWJKGK2 zdp#99SY5en5y*&;$#OSQwCd_diaV{SaJ(I|`QNT@I&SoE(vX3){m}SPDr-B}wvP5| zNSz>)s-RQEiEC`5)6;`tp(`La*rUXl|J1U~`zsXm{19A*6ap1l+6L6znEFjs2iA4m z)gLVeq?5L8=sIJPBKaayB4I<;FAUB{kywv{>2L9Pnh`-m8UcyFghelEprC5E9nVfC z1CxCsORyhCN1L4lN(k%9)njo>Q{sz8$*_OI)Awino{sO#UQQ^lVe8-2IJW29V&TSU z#kWSt^rkqAqI&_laecpk(4?>@IdC%^&VH8xQ33kMp6(5=g1d%nM zyM&WwU2jF&rX(~eg~CNIHVpXAR}Fq{r4Wm_>M}Evwi|E{mzB2)o@t%MHPkvV_1KLK zT<0ZDx#pWp%Wp~7b^PzX`hU&)UyJgeEA2lZ+hqJB3C&L19Y4wz4#4$|^-6V|qyGy| CZMP}_ literal 0 HcmV?d00001 diff --git a/test/visual_regression/screenshots/.gdignore b/test/visual_regression/screenshots/.gdignore new file mode 100644 index 0000000..e69de29 diff --git a/test/visual_regression/test_visual_regression.gd b/test/visual_regression/test_visual_regression.gd new file mode 100644 index 0000000..d406b6b --- /dev/null +++ b/test/visual_regression/test_visual_regression.gd @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: 2024 Leroy Hopson +# SPDX-License-Identifier: MIT + +class_name VisualRegressionTest extends RenderingTest + +const Pixelmatch = preload("res://addons/pixelmatch/pixelmatch.gd") +const MenuScene = preload("res://examples/menu/menu.tscn") + +const TERMINAL_SIZE = Vector2i(200, 100) +const UPDATE = false # Set to true when you want to update baseline images. + +var matcher = Pixelmatch.new() + + +func get_described_class(): + return Terminal + + +func before_each(): + await super.before_each() + subject.set_anchors_and_offsets_preset(Control.PRESET_TOP_LEFT) + subject.call_deferred("set_size", TERMINAL_SIZE) + await wait_for_signal(subject.size_changed, 5) + + +func assert_match(reference: String): + var image = get_viewport().get_texture().get_image() + image.crop(TERMINAL_SIZE.x, TERMINAL_SIZE.y) + var reference_path = "res://test/visual_regression/baseline/%s.png" % reference + + if UPDATE or not FileAccess.file_exists(reference_path): + image.save_png(reference_path) + + var reference_image = Image.new() + reference_image.load(reference_path) + assert(reference_image, "Could not load reference image: " + reference) + var diff_image = Image.create(TERMINAL_SIZE.x, TERMINAL_SIZE.y, false, Image.FORMAT_RGBA8) + var diff = matcher.diff(image, reference_image, diff_image, TERMINAL_SIZE.x, TERMINAL_SIZE.y) + + if diff != 0: + diff_image.save_png("res://test/visual_regression/screenshots/%s.diff.png" % reference) + image.save_png("res://test/visual_regression/screenshots/%s.png" % reference) + + assert_eq(diff, 0, "Screenshot matches baseline image") + + +class TestVisualRegression: + extends VisualRegressionTest + + func test_empty(): + await wait_frames(30) + assert_match("empty") + + func test_default_theme(): + # Print every background color. + for i in range(8): + subject.write("\u001b[4%dm " % i) # Regular. + + # Print every foreground color. + + # Print every font. + for i in range(8): + subject.write("\u001b[10%dm " % i) # Bright. + + # Print every foreground color. + + # Print every font. + subject.write("\u001b[0m") # Reset. + + # Print every foreground color. + for i in range(8): + subject.write("\u001b[3%dm█" % i) # Regular. + + # Print every font. + for i in range(8): + subject.write("\u001b[9%dm█" % i) # Bright. + + # Print every font. + subject.write("\u001b[0m") # Reset. + + # Print every font. + subject.write("L\u001b[0m") # Regular. + subject.write("\u001b[1mL\u001b[0m") # Bold. + subject.write("\u001b[3mL\u001b[0m") # Italic. + subject.write("\u001b[1m\u001b[3mL\u001b[0m") # Bold Italic. + + await wait_frames(30) + assert_match("default_theme") + + func test_transparency(): + subject.add_theme_color_override("foreground_color", Color(0, 1, 0, 0.5)) + subject.add_theme_color_override("background_color", Color(1, 0, 0, 0.5)) + subject.write("bg red, 50% transparency\r\n") + subject.write("fg green, 50% transparency") + await wait_frames(30) + assert_match("transparency")