update README.md

Makefile

@@ -1,7 +1,7 @@
-VERSION=v0.1.0
+VERSION=v0.2.1
 # CC := g++
 CC := clang++
-LFLAGS := -lX11 -lXrandr -lglfw -lGL
+LFLAGS := -lX11 -lXrandr -lXtst -lglfw -lGL
 LIBS := openvr/libopenvr_api.so
 SRC := src/*.cpp
 OUT := ./sinpin_vr

README.md

@@ -1,14 +1,21 @@
 # sinpin-vr
 A SteamVR overlay for Linux+X11 that displays all your screens in VR.
 
-https://user-images.githubusercontent.com/54243225/233798783-27d1a6ae-b71d-448f-bb67-76015e539452.mp4
+https://user-images.githubusercontent.com/54243225/235313348-2ce9b2d8-8458-49f8-ba94-16e577c4f502.mp4
+
+Note: only index controllers have default bindings right now, feel free to make a PR.
 
 ## features
 - one overlay per screen
 - shows cursor position
-- global visibility toggle (default: press left touch)
+- global visibility toggle (default: long press left B)
 - reset positions (default: long press left A)
-- edit mode (default: press right touch)
+- activate cursor input (default: press touchpad)
+	- left mouse default: trigger
+	- right mouse default: A
+	- middle mouse default: not bound
+	- scrolling default: joystick up/down
+- edit mode (default: long press press right B)
 	- move screens around (default: trigger)
 	- resize screens (move with two controllers)
 	- push/pull screens (default: joystick up/down)

bindings/action_manifest.json

@@ -16,18 +16,43 @@
 			"requirement": "mandatory",
 			"type": "boolean"
 		},
-		{
-			"name": "/actions/main/in/grab",
-			"requirement": "mandatory",
-			"type": "boolean"
-		},
 		{
 			"name": "/actions/main/in/reset",
 			"requirement": "suggested",
 			"type": "boolean"
 		},
 		{
-			"name": "/actions/main/in/distance",
+			"name": "/actions/edit/in/grab",
+			"requirement": "mandatory",
+			"type": "boolean"
+		},
+		{
+			"name": "/actions/edit/in/distance",
+			"requirement": "suggested",
+			"type": "vector2"
+		},
+		{
+			"name": "/actions/cursor/in/activate_cursor",
+			"requirement": "suggested",
+			"type": "boolean"
+		},
+		{
+			"name": "/actions/cursor/in/mouse_left",
+			"requirement": "suggested",
+			"type": "boolean"
+		},
+		{
+			"name": "/actions/cursor/in/mouse_right",
+			"requirement": "suggested",
+			"type": "boolean"
+		},
+		{
+			"name": "/actions/cursor/in/mouse_middle",
+			"requirement": "optional",
+			"type": "boolean"
+		},
+		{
+			"name": "/actions/cursor/in/scroll",
 			"requirement": "suggested",
 			"type": "vector2"
 		}
@@ -36,17 +61,29 @@
 		{
 			"name": "/actions/main",
 			"usage": "leftright"
+		},
+		{
+			"name": "/actions/edit",
+			"usage": "single"
+		},
+		{
+			"name": "/actions/cursor",
+			"usage": "single"
 		}
 	],
 	"localization": [
 		{
 			"language_tag": "en_us",
-			"/actions/main": "Overlay actions",
-			"/actions/main/in/toggle_visibility": "Toggle visibility",
-			"/actions/main/in/edit_mode": "Toggle edit mode",
-			"/actions/main/in/grab": "grab panel",
-			"/actions/main/in/distance": "Move away",
-			"/actions/main/in/reset": "Reset positions"
+			"/actions/main/in/toggle_visibility": "toggle visibility",
+			"/actions/main/in/edit_mode": "toggle edit mode",
+			"/actions/main/in/reset": "reset positions",
+			"/actions/edit/in/grab": "grab panel",
+			"/actions/edit/in/distance": "push/pull overlay",
+			"/actions/cursor/in/activate_cursor": "activate cursor",
+			"/actions/cursor/in/mouse_left": "left mouse button",
+			"/actions/cursor/in/mouse_right": "right mouse button",
+			"/actions/cursor/in/mouse_middle": "middle mouse button",
+			"/actions/cursor/in/scroll": "scroll"
 		}
 	]
 }

bindings/index_controller.json

@@ -140,6 +140,122 @@
    },
    "app_key" : "system.generated.sinpin_vr",
    "bindings" : {
+      "/actions/cursor" : {
+         "sources" : [
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/mouse_right"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/a"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/mouse_right"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/a"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/mouse_left"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/mouse_left"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/activate_cursor"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/cursor/in/activate_cursor"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "position" : {
+                     "output" : "/actions/cursor/in/scroll"
+                  }
+               },
+               "mode" : "joystick",
+               "path" : "/user/hand/left/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "position" : {
+                     "output" : "/actions/cursor/in/scroll"
+                  }
+               },
+               "mode" : "joystick",
+               "path" : "/user/hand/right/input/thumbstick"
+            }
+         ]
+      },
+      "/actions/edit" : {
+         "sources" : [
+            {
+               "inputs" : {
+                  "position" : {
+                     "output" : "/actions/edit/in/distance"
+                  }
+               },
+               "mode" : "joystick",
+               "path" : "/user/hand/left/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "position" : {
+                     "output" : "/actions/edit/in/distance"
+                  }
+               },
+               "mode" : "joystick",
+               "path" : "/user/hand/right/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/edit/in/grab"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/edit/in/grab"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trigger"
+            }
+         ]
+      },
       "/actions/legacy" : {
          "haptics" : [
             {
@@ -588,60 +704,6 @@
       },
       "/actions/main" : {
          "sources" : [
-            {
-               "inputs" : {
-                  "position" : {
-                     "output" : "/actions/main/in/distance"
-                  }
-               },
-               "mode" : "joystick",
-               "path" : "/user/hand/right/input/thumbstick"
-            },
-            {
-               "inputs" : {
-                  "click" : {
-                     "output" : "/actions/main/in/grab"
-                  }
-               },
-               "mode" : "button",
-               "path" : "/user/hand/right/input/trigger"
-            },
-            {
-               "inputs" : {
-                  "click" : {
-                     "output" : "/actions/main/in/grab"
-                  }
-               },
-               "mode" : "button",
-               "path" : "/user/hand/left/input/trigger"
-            },
-            {
-               "inputs" : {
-                  "position" : {
-                     "output" : "/actions/main/in/distance"
-                  }
-               },
-               "mode" : "joystick",
-               "path" : "/user/hand/left/input/thumbstick"
-            },
-            {
-               "inputs" : {
-                  "click" : {
-                     "output" : "/actions/main/in/toggle_visibility"
-                  }
-               },
-               "mode" : "button",
-               "path" : "/user/hand/left/input/trackpad"
-            },
-            {
-               "inputs" : {
-                  "click" : {
-                     "output" : "/actions/main/in/edit_mode"
-                  }
-               },
-               "mode" : "button",
-               "path" : "/user/hand/right/input/trackpad"
-            },
             {
                "inputs" : {
                   "long" : {
@@ -650,6 +712,27 @@
                },
                "mode" : "button",
                "path" : "/user/hand/left/input/a"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/main/in/mouse_left"
+                  },
+                  "long" : {
+                     "output" : "/actions/main/in/toggle_visibility"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/b"
+            },
+            {
+               "inputs" : {
+                  "long" : {
+                     "output" : "/actions/main/in/edit_mode"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/b"
             }
          ]
       }
@@ -658,7 +741,7 @@
    "controller_type" : "knuckles",
    "description" : "yes",
    "interaction_profile" : "",
-   "name" : "Saved sinpin_vr bindings",
+   "name" : "Default sinpin_vr bindings",
    "options" : {
       "mirror_actions" : true,
       "simulated_controller_type" : "none"

src/app.cpp

@@ -1,12 +1,16 @@
 #include "app.h"
 #include "controller.h"
 #include "util.h"
+#include <X11/Xlib.h>
+#include <X11/extensions/XTest.h>
 #include <X11/extensions/Xrandr.h>
 #include <cassert>
 #include <glm/matrix.hpp>
 
 const VRMat root_start_pose = {{{1, 0, 0, 0}, {0, 1, 0, 0.8f}, {0, 0, 1, 0}}}; // 0.8m above origin
 
+const int FRAME_INTERVAL = 4; // number of update loops until the frame buffer is updated
+
 App::App()
 {
 	_tracking_origin = vr::TrackingUniverseStanding;
@@ -51,17 +55,31 @@ App::App()
 		printf("actions path: %s\n", _actions_path.c_str());
 		vr_input->SetActionManifestPath(_actions_path.c_str());
 
-		auto action_err = vr_input->GetActionHandle("/actions/main/in/grab", &_input_handles.grab);
+		auto action_err = vr_input->GetActionHandle("/actions/edit/in/grab", &_input_handles.edit.grab);
 		assert(action_err == 0);
-		action_err = vr_input->GetActionHandle("/actions/main/in/toggle_visibility", &_input_handles.toggle_hidden);
+		action_err = vr_input->GetActionHandle("/actions/main/in/toggle_visibility", &_input_handles.main.toggle_hidden);
 		assert(action_err == 0);
-		action_err = vr_input->GetActionHandle("/actions/main/in/edit_mode", &_input_handles.edit_mode);
+		action_err = vr_input->GetActionHandle("/actions/cursor/in/activate_cursor", &_input_handles.cursor.activate);
 		assert(action_err == 0);
-		action_err = vr_input->GetActionHandle("/actions/main/in/reset", &_input_handles.reset);
+		action_err = vr_input->GetActionHandle("/actions/main/in/edit_mode", &_input_handles.main.edit_mode);
 		assert(action_err == 0);
-		action_err = vr_input->GetActionHandle("/actions/main/in/distance", &_input_handles.distance);
+		action_err = vr_input->GetActionHandle("/actions/main/in/reset", &_input_handles.main.reset);
 		assert(action_err == 0);
-		action_err = vr_input->GetActionSetHandle("/actions/main", &_input_handles.set);
+		action_err = vr_input->GetActionHandle("/actions/edit/in/distance", &_input_handles.edit.distance);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionHandle("/actions/cursor/in/mouse_left", &_input_handles.cursor.mouse_left);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionHandle("/actions/cursor/in/mouse_right", &_input_handles.cursor.mouse_right);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionHandle("/actions/cursor/in/mouse_middle", &_input_handles.cursor.mouse_middle);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionHandle("/actions/cursor/in/scroll", &_input_handles.cursor.scroll);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionSetHandle("/actions/main", &_input_handles.main_set);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionSetHandle("/actions/edit", &_input_handles.edit_set);
+		assert(action_err == 0);
+		action_err = vr_input->GetActionSetHandle("/actions/cursor", &_input_handles.cursor_set);
 		assert(action_err == 0);
 	}
 }
@@ -126,9 +144,9 @@ void App::InitRootOverlay()
 	_root_overlay.SetTextureToColor(110, 30, 190);
 }
 
-void App::Update()
+void App::Update(float dtime)
 {
-	UpdateInput();
+	UpdateInput(dtime);
 	if (!_hidden)
 	{
 		_root_overlay.Update();
@@ -141,57 +159,72 @@ void App::Update()
 	_frames_since_framebuffer += 1;
 }
 
-void App::UpdateInput()
+void App::UpdateInput(float dtime)
 {
-	vr::VRActiveActionSet_t main;
-	main.ulActionSet = _input_handles.set;
-	main.ulRestrictedToDevice = 0;
-	main.nPriority = 10;
-	vr::EVRInputError err = vr_input->UpdateActionState(&main, sizeof(vr::VRActiveActionSet_t), 1);
-	if (err)
+	vr::VRActiveActionSet_t active_sets[2];
+	active_sets[0].ulActionSet = _input_handles.main_set;
+	active_sets[0].ulRestrictedToDevice = 0;
+	active_sets[0].nPriority = 10;
+	int set_count = 1;
+	if (!_hidden)
 	{
-		printf("Error: (update action state): %d\n", err);
+		set_count = 2;
+		active_sets[1].ulRestrictedToDevice = 0;
+		active_sets[1].nPriority = 10;
+		active_sets[1].ulActionSet = _input_handles.cursor_set;
+		if (_edit_mode)
+			active_sets[1].ulActionSet = _input_handles.edit_set;
 	}
+	vr::EVRInputError err = vr_input->UpdateActionState(active_sets, sizeof(vr::VRActiveActionSet_t), set_count);
+	if (err)
+		printf("Error updating action state: %d\n", err);
 
 	vr_sys->GetDeviceToAbsoluteTrackingPose(_tracking_origin, 0, _tracker_poses, MAX_TRACKERS);
 
-	if (IsInputJustPressed(_input_handles.toggle_hidden))
+	if (IsInputJustPressed(_input_handles.main.toggle_hidden))
 	{
 		_hidden = !_hidden;
 		for (auto &panel : _panels)
 		{
 			panel.SetHidden(_hidden);
 		}
-		_root_overlay.SetHidden(_hidden || !_edit_mode);
-		_controllers[0]->SetHidden(_hidden || !_edit_mode);
-		_controllers[1]->SetHidden(_hidden || !_edit_mode);
+		UpdateUIVisibility();
 	}
-	if (IsInputJustPressed(_input_handles.reset))
+	if (!_hidden)
 	{
-		_root_overlay.SetTransformWorld(&root_start_pose);
-		_root_overlay.SetWidth(0.25f);
-		for (auto &panel : _panels)
+		if (IsInputJustPressed(_input_handles.main.reset))
 		{
-			panel.ResetTransform();
+			_root_overlay.SetTransformWorld(&root_start_pose);
+			_root_overlay.SetWidth(0.25f);
+			for (auto &panel : _panels)
+			{
+				panel.ResetTransform();
+			}
+		}
+		if (IsInputJustPressed(_input_handles.main.edit_mode))
+		{
+			_edit_mode = !_edit_mode;
+			UpdateUIVisibility();
+			if (_edit_mode && _active_cursor.has_value())
+			{
+				_active_cursor.value()->_cursor_active = false;
+				_active_cursor = {};
+			}
 		}
 	}
-	if (!_hidden && IsInputJustPressed(_input_handles.edit_mode))
-	{
-		_edit_mode = !_edit_mode;
-		_root_overlay.SetHidden(_hidden || !_edit_mode);
-		_controllers[0]->SetHidden(_hidden || !_edit_mode);
-		_controllers[1]->SetHidden(_hidden || !_edit_mode);
-	}
-	if (_edit_mode)
-	{
-		_controllers[0]->Update();
-		_controllers[1]->Update();
-	}
+	_controllers[0]->Update(dtime);
+	_controllers[1]->Update(dtime);
+}
+
+void App::UpdateUIVisibility()
+{
+	bool state = _hidden || !_edit_mode;
+	_root_overlay.SetHidden(state);
 }
 
 void App::UpdateFramebuffer()
 {
-	if (_frames_since_framebuffer < 2)
+	if (_frames_since_framebuffer < FRAME_INTERVAL)
 	{
 		return;
 	}
@@ -255,7 +288,7 @@ Ray App::IntersectRay(glm::vec3 origin, glm::vec3 direction, float max_len)
 
 	for (auto &panel : _panels)
 	{
-		auto r_panel = panel.GetOverlay()->IntersectRay(origin, direction, max_len);
+		auto r_panel = panel.IntersectRay(origin, direction, max_len);
 		if (r_panel.distance < ray.distance)
 		{
 			ray = r_panel;
@@ -281,3 +314,14 @@ CursorPos App::GetCursorPosition()
 		&buttons);
 	return pos;
 }
+
+void App::SetCursor(int x, int y)
+{
+	// I don't know what the return value of XWarpPointer means, it seems to be 1 on success.
+	XWarpPointer(_xdisplay, _root_window, _root_window, 0, 0, _root_width, _root_height, x, y);
+}
+
+void App::SendMouseInput(unsigned int button, bool state)
+{
+	XTestFakeButtonEvent(_xdisplay, button, state, 0);
+}

src/app.h

@@ -18,12 +18,28 @@ struct CursorPos
 
 struct InputHandles
 {
-	vr::VRActionSetHandle_t set;
-	vr::VRActionHandle_t toggle_hidden;
-	vr::VRActionHandle_t distance;
-	vr::VRActionHandle_t grab;
-	vr::VRActionHandle_t edit_mode;
-	vr::VRActionHandle_t reset;
+	struct
+	{
+		vr::VRActionHandle_t toggle_hidden;
+		vr::VRActionHandle_t edit_mode;
+		vr::VRActionHandle_t reset;
+	} main;
+	vr::VRActionSetHandle_t main_set;
+	struct
+	{
+		vr::VRActionHandle_t activate;
+		vr::VRActionHandle_t mouse_left;
+		vr::VRActionHandle_t mouse_right;
+		vr::VRActionHandle_t mouse_middle;
+		vr::VRActionHandle_t scroll;
+	} cursor;
+	vr::VRActionSetHandle_t cursor_set;
+	struct
+	{
+		vr::VRActionHandle_t distance;
+		vr::VRActionHandle_t grab;
+	} edit;
+	vr::VRActionSetHandle_t edit_set;
 };
 
 class App
@@ -31,7 +47,7 @@ class App
   public:
 	App();
 	~App();
-	void Update();
+	void Update(float dtime);
 
 	std::vector<TrackerID> GetControllers();
 	glm::mat4 GetTrackerPose(TrackerID tracker);
@@ -41,6 +57,8 @@ class App
 	CursorPos GetCursorPosition();
 
 	Ray IntersectRay(glm::vec3 origin, glm::vec3 direction, float max_len);
+	void SetCursor(int x, int y);
+	void SendMouseInput(unsigned int button, bool state);
 
 	Display *_xdisplay;
 	Window _root_window;
@@ -69,6 +87,7 @@ class App
 	std::vector<Panel> _panels;
 	bool _hidden = false;
 	bool _edit_mode = false;
+	std::optional<Controller *> _active_cursor;
 
   private:
 	void InitX11();
@@ -77,5 +96,6 @@ class App
 	void InitRootOverlay();
 
 	void UpdateFramebuffer();
-	void UpdateInput();
+	void UpdateInput(float dtime);
+	void UpdateUIVisibility();
 };

src/controller.cpp

@@ -4,7 +4,10 @@
 #include "util.h"
 #include <string>
 
-const float width = 0.004f;
+const float LASER_WIDTH = 0.004f;
+const Color EDIT_COLOR{1, 0.6f, 1};
+const Color CURSOR_COLOR{0.3f, 1, 1};
+const float SCROLL_SPEED = 48.0f;
 
 Controller::Controller(App *app, ControllerSide side)
 {
@@ -13,7 +16,8 @@ Controller::Controller(App *app, ControllerSide side)
 	_input_handle = 0;
 	_is_connected = false;
 	_side = side;
-	_hidden = false;
+	_cursor_active = false;
+	_last_sent_scroll = 0;
 
 	std::string laser_name = "controller_laser_";
 	if (side == ControllerSide::Left)
@@ -23,7 +27,7 @@ Controller::Controller(App *app, ControllerSide side)
 
 	_laser = Overlay(app, laser_name);
 	UpdateStatus();
-	_laser.SetTextureToColor(255, 200, 255);
+	_laser.SetTextureToColor(255, 255, 255);
 	_laser.SetAlpha(0.2f);
 	_laser.SetHidden(true);
 }
@@ -48,12 +52,6 @@ bool Controller::IsConnected()
 	return _is_connected;
 }
 
-void Controller::SetHidden(bool state)
-{
-	_hidden = state;
-	_laser.SetHidden(_hidden);
-}
-
 void Controller::ReleaseOverlay()
 {
 	_grabbed_overlay = nullptr;
@@ -74,20 +72,107 @@ glm::vec3 Controller::GetLastRot()
 	return _last_rotation;
 }
 
-void Controller::Update()
+void Controller::Update(float dtime)
 {
 	UpdateStatus();
-	if (!_is_connected || _hidden)
+	if (!_is_connected)
 		return;
 
 	UpdateLaser();
 
-	float move = _app->GetInputAnalog(_app->_input_handles.distance, _input_handle).y * 0.1; // TODO use frame time
-	if (_grabbed_overlay && move != 0.0f)
+	if (_app->_edit_mode)
 	{
-		auto transform = _grabbed_overlay->GetTarget()->transform;
-		transform.m[2][3] = glm::clamp(transform.m[2][3] - move, -5.0f, -0.1f); // moving along z axis
-		_grabbed_overlay->SetTransformTracker(_device_index, &transform);
+		_laser.SetColor(EDIT_COLOR);
+		if (_last_ray.overlay != nullptr)
+		{
+			auto ray = _last_ray;
+			if (_app->IsInputJustPressed(_app->_input_handles.edit.grab, _input_handle))
+			{
+				if (ray.overlay->IsHeld())
+				{
+					ray.overlay->ControllerResize(this);
+				}
+				else
+				{
+					_grabbed_overlay = ray.overlay;
+					ray.overlay->ControllerGrab(this);
+				}
+			}
+		}
+
+		if (_grabbed_overlay != nullptr)
+		{
+			float move = _app->GetInputAnalog(_app->_input_handles.edit.distance, _input_handle).y * dtime * 8;
+			if (move != 0.0f)
+			{
+				auto transform = _grabbed_overlay->GetTarget()->transform;
+				transform.m[2][3] = glm::clamp(transform.m[2][3] - move, -5.0f, -0.1f); // moving along z axis
+				_grabbed_overlay->SetTransformTracker(_device_index, &transform);
+			}
+		}
+	}
+	else // cursor mode
+	{
+		if (_app->IsInputJustPressed(_app->_input_handles.cursor.activate, _input_handle))
+		{
+			if (!_cursor_active && _app->_active_cursor.has_value())
+			{
+				_app->_active_cursor.value()->_cursor_active = false;
+				_app->_active_cursor = this;
+			}
+			_cursor_active = !_cursor_active;
+			_app->_active_cursor = this;
+			_laser.SetColor(CURSOR_COLOR);
+		}
+		if (_cursor_active)
+		{
+			// printf("update cursor on hand %d\n", _side);
+			if (_last_ray.overlay != nullptr && _last_ray.hit_panel != nullptr)
+			{
+				auto pos = glm::vec2(_last_ray.local_pos.x, _last_ray.local_pos.y);
+				// normalize positions to +-0.5
+				pos /= _last_ray.overlay->Width();
+				pos.y *= -1;
+
+				// shift to 0-1
+				pos.x += 0.5f;
+				pos.y += 0.5f * _last_ray.overlay->Ratio();
+
+				pos *= _last_ray.hit_panel->Width();
+				_last_ray.hit_panel->SetCursor(pos.x, pos.y);
+			}
+			UpdateMouseButton(_app->_input_handles.cursor.mouse_left, 1);
+			UpdateMouseButton(_app->_input_handles.cursor.mouse_middle, 2);
+			UpdateMouseButton(_app->_input_handles.cursor.mouse_right, 3);
+			auto scroll_state = _app->GetInputAnalog(_app->_input_handles.cursor.scroll, _input_handle);
+			if (scroll_state.y != 0)
+			{
+				_last_sent_scroll += dtime * glm::abs(scroll_state.y) * SCROLL_SPEED;
+				if (_last_sent_scroll > 1)
+				{
+					_last_sent_scroll = 0;
+					if (scroll_state.y > 0)
+					{
+						_app->SendMouseInput(4, true);
+						_app->SendMouseInput(4, false);
+					}
+					else if (scroll_state.y < 0)
+					{
+						_app->SendMouseInput(5, true);
+						_app->SendMouseInput(5, false);
+					}
+				}
+			}
+		}
+	}
+}
+
+void Controller::UpdateMouseButton(vr::VRActionHandle_t binding, unsigned int button)
+{
+	auto state = _app->GetInputDigital(binding, _input_handle);
+	if (state.bChanged)
+	{
+		_app->SendMouseInput(button, state.bState);
 	}
 }
 
@@ -108,24 +193,9 @@ void Controller::UpdateLaser()
 	hmd_local_pos.z = 0;
 	auto hmd_dir = glm::normalize(hmd_local_pos);
 
-	VRMat transform = {{{width * hmd_dir.y, 0, width * hmd_dir.x, 0}, {width * -hmd_dir.x, 0, width * hmd_dir.y, 0}, {0, len, 0, len * -0.5f}}};
+	VRMat transform = {{{LASER_WIDTH * hmd_dir.y, 0, LASER_WIDTH * hmd_dir.x, 0}, {LASER_WIDTH * -hmd_dir.x, 0, LASER_WIDTH * hmd_dir.y, 0}, {0, len, 0, len * -0.5f}}};
 	_laser.SetTransformTracker(_device_index, &transform);
-
-	if (ray.overlay != nullptr)
-	{
-		if (_app->IsInputJustPressed(_app->_input_handles.grab, _input_handle))
-		{
-			if (ray.overlay->IsHeld())
-			{
-				ray.overlay->ControllerResize(this);
-			}
-			else
-			{
-				_grabbed_overlay = ray.overlay;
-				ray.overlay->ControllerGrab(this);
-			}
-		}
-	}
+	_laser.SetHidden(!_is_connected || _app->_hidden || (!_app->_edit_mode && !_cursor_active));
 }
 
 void Controller::UpdateStatus()
@@ -145,8 +215,4 @@ void Controller::UpdateStatus()
 		_device_index = _app->vr_sys->GetTrackedDeviceIndexForControllerRole(vr::TrackedControllerRole_RightHand);
 	}
 	_is_connected &= _device_index < MAX_TRACKERS;
-	if (!_is_connected)
-	{
-		_laser.SetHidden(true);
-	}
 }

src/controller.h

@@ -24,25 +24,31 @@ class Controller
 	glm::vec3 GetLastPos();
 	glm::vec3 GetLastRot();
 
-	void SetHidden(bool state);
-
 	void ReleaseOverlay();
 
-	void Update();
+	void Update(float dtime);
+
+	bool _cursor_active;
 
   private:
 	void UpdateStatus();
 	void UpdateLaser();
 
+	void UpdateMouseButton(vr::VRActionHandle_t binding, unsigned int button);
+
 	App *_app;
 	Overlay _laser;
 	ControllerSide _side;
 	TrackerID _device_index;
 	vr::VRInputValueHandle_t _input_handle;
+
 	bool _is_connected;
-	bool _hidden;
+
 	Overlay *_grabbed_overlay;
+
 	Ray _last_ray;
 	glm::vec3 _last_rotation;
 	glm::vec3 _last_pos;
+
+	float _last_sent_scroll;
 };

src/main.cpp

@@ -1,7 +1,7 @@
 #include "app.h"
 #include <signal.h>
 
-#define UPDATE_RATE 60
+#define UPDATE_RATE 120
 
 bool should_exit = false;
 
@@ -18,8 +18,8 @@ int main()
 
 	while (!should_exit)
 	{
-		app.Update();
 		usleep(1000000 / UPDATE_RATE);
+		app.Update(1.0 / UPDATE_RATE);
 	}
 	printf("\nShutting down\n");
 	return 0;

src/overlay.cpp

@@ -108,6 +108,17 @@ void Overlay::SetTextureToColor(uint8_t r, uint8_t g, uint8_t b)
 	assert(set_texture_err == 0);
 }
 
+void Overlay::SetColor(float r, float g, float b)
+{
+	auto set_color_err = _app->vr_overlay->SetOverlayColor(_id, r, g, b);
+	assert(set_color_err == 0);
+}
+
+void Overlay::SetColor(Color c)
+{
+	SetColor(c.r, c.g, c.b);
+}
+
 void Overlay::SetTransformTracker(TrackerID tracker, const VRMat *transform)
 {
 	auto original_pose = _target.transform;
@@ -169,7 +180,7 @@ Ray Overlay::IntersectRay(glm::vec3 origin, glm::vec3 direction, float max_len)
 	{
 		dist = glm::min(r * max_len, max_len);
 	}
-	return Ray{.overlay = this, .distance = dist};
+	return Ray{.overlay = this, .distance = dist, .local_pos = p, .hit_panel = nullptr};
 }
 
 glm::mat4x4 Overlay::GetTransformAbsolute()
@@ -208,10 +219,10 @@ void Overlay::Update()
 
 	if (_holding_controller != nullptr)
 	{
-		bool hold_controller_holding = _app->GetInputDigital(_app->_input_handles.grab, _holding_controller->InputHandle()).bState;
+		bool hold_controller_holding = _app->GetInputDigital(_app->_input_handles.edit.grab, _holding_controller->InputHandle()).bState;
 		if (_resize_controller != nullptr)
 		{
-			bool resize_controller_holding = _app->GetInputDigital(_app->_input_handles.grab, _resize_controller->InputHandle()).bState;
+			bool resize_controller_holding = _app->GetInputDigital(_app->_input_handles.edit.grab, _resize_controller->InputHandle()).bState;
 			if (!resize_controller_holding)
 			{
 				_resize_controller = nullptr;

src/overlay.h

@@ -43,6 +43,8 @@ class Overlay
 	void SetRatio(float ratio);
 	void SetTexture(vr::Texture_t *texture);
 	void SetTextureToColor(uint8_t r, uint8_t g, uint8_t b);
+	void SetColor(float r, float g, float b);
+	void SetColor(Color c);
 
 	glm::mat4x4 GetTransformAbsolute();
 	Target *GetTarget();

src/panel.cpp

@@ -68,6 +68,18 @@ void Panel::SetHidden(bool state)
 	_overlay.SetHidden(state);
 }
 
+Ray Panel::IntersectRay(glm::vec3 origin, glm::vec3 direction, float max_len)
+{
+	auto ray = _overlay.IntersectRay(origin, direction, max_len);
+	ray.hit_panel = this;
+	return ray;
+}
+
+void Panel::SetCursor(int x, int y)
+{
+	_app->SetCursor(x + _x, y + _y);
+}
+
 void Panel::UpdateCursor()
 {
 	auto global_pos = _app->GetCursorPosition();

src/panel.h

@@ -19,6 +19,19 @@ class Panel
 	void SetHidden(bool state);
 	void ResetTransform();
 
+	int Width()
+	{
+		return _width;
+	}
+	int Height()
+	{
+		return _height;
+	}
+
+	void SetCursor(int x, int y);
+
+	Ray IntersectRay(glm::vec3 origin, glm::vec3 direction, float max_len);
+
 	Overlay *GetOverlay();
 
   private:

src/util.h

@@ -11,10 +11,20 @@ const VRMat VRMatIdentity{{{1, 0, 0, 0}, {0, 1, 0, 0}, {0, 0, 1, 0}}};
 const int MAX_TRACKERS = vr::k_unMaxTrackedDeviceCount;
 
 class Overlay;
+class Panel;
 struct Ray
 {
 	Overlay *overlay;
 	float distance;
+	glm::vec3 local_pos;
+	Panel *hit_panel;
+};
+
+struct Color
+{
+	float r;
+	float g;
+	float b;
 };
 
 inline void PrintVec(glm::vec3 v)