feat(test): add visual regression testing

Will upload screenshots on failure.
This commit is contained in:
Leroy Hopson 2024-04-07 00:00:00 +13:00
parent 898d49e392
commit 3ca272c615
No known key found for this signature in database
GPG key ID: D2747312A6DB51AA
10 changed files with 105 additions and 1 deletions

View file

@ -305,6 +305,12 @@ jobs:
if grep -q 'SCRIPT_ERROR:' output.log || grep -q 'Tests none' output.log; then if grep -q 'SCRIPT_ERROR:' output.log || grep -q 'Tests none' output.log; then
exit 1 exit 1
fi fi
- name: Upload screenshots
uses: actions/upload-artifact@v4
if: failure()
with:
name: failed-screenshots
path: test/visual_regression/screenshots
merge-artifacts: merge-artifacts:
name: Merge Artifacts name: Merge Artifacts

1
.gitignore vendored
View file

@ -30,6 +30,7 @@ mono_crash.*
.gutconfig.json .gutconfig.json
test/results.xml test/results.xml
test/test_metadata.json test/test_metadata.json
test/visual_regression/screenshots/
# GodotXterm-specific ignores # GodotXterm-specific ignores
.gdxterm .gdxterm

View file

@ -22,7 +22,7 @@ test:
{{godot}} --headless -s addons/gut/gut_cmdln.gd -gtest={{test_files}} -gexit {{godot}} --headless -s addons/gut/gut_cmdln.gd -gtest={{test_files}} -gexit
test-all: test-all:
{{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gdir=res://test -gopacity=0 -gexit {{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gdir=res://test -ginclude_subdirs=true -gopacity=0 -gexit
test-rendering: test-rendering:
{{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gtest=res://test/test_rendering.gd -gopacity=0 -gexit {{godot}} --windowed --resolution 400x200 --position 0,0 -s addons/gut/gut_cmdln.gd -gtest=res://test/test_rendering.gd -gopacity=0 -gexit

View file

@ -5,3 +5,4 @@ extends "res://addons/gd-plug/plug.gd"
func _plugging(): func _plugging():
plug("bitwes/Gut", {tag = "v9.2.0"}) plug("bitwes/Gut", {tag = "v9.2.0"})
plug("lihop/godot-pixelmatch", {tag = "v2.0.0", include = ["addons/pixelmatch"]})

Binary file not shown.

After

Width:  |  Height:  |  Size: 822 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 368 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

View file

@ -0,0 +1,96 @@
# SPDX-FileCopyrightText: 2024 Leroy Hopson <godot-xterm@leroy.nix.nz>
# SPDX-License-Identifier: MIT
class_name VisualRegressionTest extends RenderingTest
const Pixelmatch = preload("res://addons/pixelmatch/pixelmatch.gd")
const MenuScene = preload("res://examples/menu/menu.tscn")
const TERMINAL_SIZE = Vector2i(200, 100)
const UPDATE = false # Set to true when you want to update baseline images.
var matcher = Pixelmatch.new()
func get_described_class():
return Terminal
func before_each():
await super.before_each()
subject.set_anchors_and_offsets_preset(Control.PRESET_TOP_LEFT)
subject.call_deferred("set_size", TERMINAL_SIZE)
await wait_for_signal(subject.size_changed, 5)
func assert_match(reference: String):
var image = get_viewport().get_texture().get_image()
image.crop(TERMINAL_SIZE.x, TERMINAL_SIZE.y)
var reference_path = "res://test/visual_regression/baseline/%s.png" % reference
if UPDATE or not FileAccess.file_exists(reference_path):
image.save_png(reference_path)
var reference_image = Image.new()
reference_image.load(reference_path)
assert(reference_image, "Could not load reference image: " + reference)
var diff_image = Image.create(TERMINAL_SIZE.x, TERMINAL_SIZE.y, false, Image.FORMAT_RGBA8)
var diff = matcher.diff(image, reference_image, diff_image, TERMINAL_SIZE.x, TERMINAL_SIZE.y)
if diff != 0:
diff_image.save_png("res://test/visual_regression/screenshots/%s.diff.png" % reference)
image.save_png("res://test/visual_regression/screenshots/%s.png" % reference)
assert_eq(diff, 0, "Screenshot matches baseline image")
class TestVisualRegression:
extends VisualRegressionTest
func test_empty():
await wait_frames(30)
assert_match("empty")
func test_default_theme():
# Print every background color.
for i in range(8):
subject.write("\u001b[4%dm " % i) # Regular.
# Print every foreground color.
# Print every font.
for i in range(8):
subject.write("\u001b[10%dm " % i) # Bright.
# Print every foreground color.
# Print every font.
subject.write("\u001b[0m") # Reset.
# Print every foreground color.
for i in range(8):
subject.write("\u001b[3%dm█" % i) # Regular.
# Print every font.
for i in range(8):
subject.write("\u001b[9%dm█" % i) # Bright.
# Print every font.
subject.write("\u001b[0m") # Reset.
# Print every font.
subject.write("L\u001b[0m") # Regular.
subject.write("\u001b[1mL\u001b[0m") # Bold.
subject.write("\u001b[3mL\u001b[0m") # Italic.
subject.write("\u001b[1m\u001b[3mL\u001b[0m") # Bold Italic.
await wait_frames(30)
assert_match("default_theme")
func test_transparency():
subject.add_theme_color_override("foreground_color", Color(0, 1, 0, 0.5))
subject.add_theme_color_override("background_color", Color(1, 0, 0, 0.5))
subject.write("bg red, 50% transparency\r\n")
subject.write("fg green, 50% transparency")
await wait_frames(30)
assert_match("transparency")