perf(term): add benchmarks

Adds benchmarks by running [alacritty/vtebench](https://github.com/alacritty/vtebench)
benchmarks in the terminal.
Uses code based on [godotengine/godot-benchmarks](https://github.com/godotengine/godot-benchmarks)
to measure average GPU and CPU time spent per frame.
Uses [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark)
for continuous integration, and publishes benchmark results to https://lihop.github.io/godot-xterm/dev/bench/.
This commit is contained in:
Leroy Hopson 2024-06-09 17:18:58 +12:00 committed by Leroy Hopson
parent 8b33818751
commit 9569c9e489
10 changed files with 326 additions and 0 deletions

124
benchmark/benchmark.gd Normal file
View file

@ -0,0 +1,124 @@
extends Control
class Results:
var render_cpu := 0.0
var render_gpu := 0.0
var vtebench := {value = 0.0, variance = 0.0}
var terminal_exit_code := -1
func _ready():
var timeout := 120
var benchmark := ""
var args = OS.get_cmdline_user_args()
for arg in args:
if arg.begins_with("--benchmark"):
benchmark = arg.split("=")[1]
if benchmark.is_empty():
_quit_with_error("No benchmark specified")
RenderingServer.viewport_set_measure_render_time(get_tree().root.get_viewport_rid(), true)
var results := Results.new()
var begin_time := Time.get_ticks_usec()
var frames_captured := 0
$Terminal.run_benchmark(benchmark)
await $Terminal.started
while terminal_exit_code == -1:
await get_tree().process_frame
if Time.get_ticks_usec() - begin_time > (timeout * 1e6):
_quit_with_error("Benchmark took longer than %ss to run" % timeout)
results.render_cpu += (
RenderingServer.viewport_get_measured_render_time_cpu(
get_tree().root.get_viewport_rid()
)
+ RenderingServer.get_frame_setup_time_cpu()
)
results.render_gpu += RenderingServer.viewport_get_measured_render_time_gpu(
get_tree().root.get_viewport_rid()
)
if terminal_exit_code != 0:
_quit_with_error("Terminal exited with error code: %d" % terminal_exit_code)
results.render_cpu /= float(max(1.0, float(frames_captured)))
results.render_gpu /= float(max(1.0, float(frames_captured)))
results.vtebench = _process_dat_results("res://benchmark/results/%s.dat" % benchmark)
var json_results = (
JSON
. stringify(
[
{
name = benchmark,
unit = "milliseconds",
value = _round(results.vtebench.value),
range = results.vtebench.range,
},
{
name = "%s - render cpu" % benchmark,
unit = "milliseconds",
value = _round(results.render_cpu),
},
{
name = "%s - render gpu" % benchmark,
unit = "milliseconds",
value = _round(results.render_gpu),
}
],
" "
)
)
var file = FileAccess.open("res://benchmark/results/%s.json" % benchmark, FileAccess.WRITE)
file.store_string(json_results)
print(json_results)
get_tree().quit(terminal_exit_code)
func _on_terminal_exited(exit_code: int):
terminal_exit_code = exit_code
func _round(val: float, sig_figs := 4) -> float:
return snapped(val, pow(10, floor(log(val) / log(10)) - sig_figs + 1))
func _process_dat_results(path: String) -> Dictionary:
var file := FileAccess.open(path, FileAccess.READ)
var samples := []
file.get_line() # Skip the first 'header' line.
while !file.eof_reached():
var line := file.get_line().strip_edges()
if line.is_valid_float():
samples.append(line.to_float())
if samples.size() < 2:
_quit_with_error("Not enough samples")
var avg: float = (samples.reduce(func(acc, n): return acc + n, 0)) / samples.size()
var std_dev := 0.0
for sample in samples:
std_dev += pow(sample - avg, 2)
std_dev /= (samples.size() - 1)
return {value = avg, range = "± %.2f" % _round(sqrt(std_dev))}
func _quit_with_error(error_msg: String):
await get_tree().process_frame
push_error(error_msg)
get_tree().quit(1)

26
benchmark/benchmark.tscn Normal file
View file

@ -0,0 +1,26 @@
[gd_scene load_steps=5 format=3 uid="uid://b2axn64mqnt8n"]
[ext_resource type="Script" path="res://benchmark/benchmark.gd" id="1_tmqb5"]
[ext_resource type="PackedScene" uid="uid://cysad55lwtnc6" path="res://examples/terminal/terminal.tscn" id="2_3raq0"]
[ext_resource type="Script" path="res://benchmark/terminal_benchmark.gd" id="3_8t8od"]
[ext_resource type="FontVariation" uid="uid://ckq73bs2fwsie" path="res://themes/fonts/regular.tres" id="3_hnrrm"]
[node name="Benchmark" type="Control"]
layout_mode = 3
anchors_preset = 15
anchor_right = 1.0
anchor_bottom = 1.0
grow_horizontal = 2
grow_vertical = 2
script = ExtResource("1_tmqb5")
[node name="Terminal" parent="." instance=ExtResource("2_3raq0")]
layout_mode = 1
focus_mode = 0
theme_override_fonts/normal_font = ExtResource("3_hnrrm")
script = ExtResource("3_8t8od")
[connection signal="exited" from="Terminal" to="." method="_on_terminal_exited"]
[connection signal="data_received" from="Terminal/PTY" to="Terminal" method="_on_pty_data_received"]
[editable path="Terminal"]

24
benchmark/editor_launch.sh Executable file
View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -e
godot=${1:-godot}
if ! command -v $godot &> /dev/null; then
echo "Error: '$godot' command not found. Please provide a valid path to the Godot executable."
exit 1
fi
results_file=benchmark/results/editor_launch.json
value=$({ time -p $godot --editor --quit; } 2>&1 | tail -n3 | head -n1 | cut -d' ' -f2)
cat <<EOF > $results_file
[
{
"name": "editor_launch",
"unit": "seconds",
"value": $value
}
]
EOF
cat $results_file

2
benchmark/results/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

View file

@ -0,0 +1,31 @@
extends "res://examples/terminal/terminal.gd"
signal started
signal exited(exit_code: int)
var vtebench_dir := ProjectSettings.globalize_path("res://benchmark/vtebench")
func _ready():
pty.connect("exited", self._on_exit)
func run_benchmark(benchmark):
pty.fork(
"cargo",
["run", "--", "-b", "benchmarks/%s" % benchmark, "--dat", "../results/%s.dat" % benchmark],
vtebench_dir,
87,
29
)
func _on_exit(exit_code, _signal):
exited.emit(exit_code)
func _on_pty_data_received(data: PackedByteArray):
# Listen for the reset sequence (\x1bc), to determine that the benchmark has started.
if data.slice(0, 2) == PackedByteArray([27, 99]):
$PTY.disconnect("data_received", _on_pty_data_received)
started.emit()

1
benchmark/vtebench Submodule

@ -0,0 +1 @@
Subproject commit c75155bfc252227c0efc101c1971df3e327c71c4