diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..089d863 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build/ +.cache/ + +# Obsidian per-session UI state +AlvinDoc/.obsidian/workspace.json diff --git a/AlvinDoc/.obsidian/app.json b/AlvinDoc/.obsidian/app.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/AlvinDoc/.obsidian/app.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/AlvinDoc/.obsidian/appearance.json b/AlvinDoc/.obsidian/appearance.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/AlvinDoc/.obsidian/appearance.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/AlvinDoc/.obsidian/core-plugins.json b/AlvinDoc/.obsidian/core-plugins.json new file mode 100644 index 0000000..639b90d --- /dev/null +++ b/AlvinDoc/.obsidian/core-plugins.json @@ -0,0 +1,33 @@ +{ + "file-explorer": true, + "global-search": true, + "switcher": true, + "graph": true, + "backlink": true, + "canvas": true, + "outgoing-link": true, + "tag-pane": true, + "footnotes": false, + "properties": true, + "page-preview": true, + "daily-notes": true, + "templates": true, + "note-composer": true, + "command-palette": true, + "slash-command": false, + "editor-status": true, + "bookmarks": true, + "markdown-importer": false, + "zk-prefixer": false, + "random-note": false, + "outline": true, + "word-count": true, + "slides": false, + "audio-recorder": false, + "workspaces": false, + "file-recovery": true, + "publish": false, + "sync": true, + "bases": true, + "webviewer": false +} \ No newline at end of file diff --git a/AlvinDoc/.obsidian/graph.json b/AlvinDoc/.obsidian/graph.json new file mode 100644 index 0000000..42a46ec --- /dev/null +++ b/AlvinDoc/.obsidian/graph.json @@ -0,0 +1,22 @@ +{ + "collapse-filter": true, + "search": "", + "showTags": false, + "showAttachments": false, + "hideUnresolved": false, + "showOrphans": true, + "collapse-color-groups": true, + "colorGroups": [], + "collapse-display": true, + "showArrow": false, + "textFadeMultiplier": 0, + "nodeSizeMultiplier": 1, + "lineSizeMultiplier": 1, + "collapse-forces": true, + "centerStrength": 0.518713248970312, + "repelStrength": 10, + "linkStrength": 1, + "linkDistance": 250, + "scale": 1, + "close": true +} \ No newline at end of file diff --git a/AlvinDoc/Alvin Structure.md b/AlvinDoc/Alvin Structure.md new file mode 100644 index 0000000..cf235db --- /dev/null +++ b/AlvinDoc/Alvin Structure.md @@ -0,0 +1,498 @@ +# Alvin API Reference + +Everything public lives in namespace `alvin` (vision helpers in `alvin::vision`). +Include the umbrella header for the core library: + +```cpp +#include // Input, Capture, WindowManager, Frame, keys, types +#include // optional, needs the alvin::vision target (OpenCV) +``` + +- [[#Using Alvin in your project]] +- [[#Error handling]] +- [[#Coordinates and monitors]] +- [[#Input]] +- [[#Keys and chords]] +- [[#Capture]] +- [[#Frame]] +- [[#Windows]] +- [[#Vision (OpenCV)]] +- [[#Putting it together]] + +--- + +## Using Alvin in your project + +Either build Alvin as part of your project, or install it once and use `find_package`. +The README's "Using Alvin in your project" section has step-by-step instructions for both. + +```cmake +add_subdirectory(external/Alvin) # option 1: build from source +# find_package(Alvin 0.1 REQUIRED) # option 2: use an installed Alvin + +target_link_libraries(my_agent PRIVATE alvin::alvin) # core +target_link_libraries(my_agent PRIVATE alvin::vision) # + OpenCV helpers, if built +``` + +Alvin requires C++23 (it uses `std::expected`). + +--- + +## Error handling + +Alvin never throws. Every call that can fail returns a `Result`, which is +`std::expected`. Calls with no value return `Status` +(`Result`). + +```cpp +struct Error { + ErrorCode code; + std::string message; // human-readable detail +}; + +enum class ErrorCode { + Unknown, Unsupported, NotAvailable, PermissionDenied, Cancelled, + Timeout, InvalidArgument, NotFound, Io, Protocol, +}; + +std::string_view toString(ErrorCode code); +``` + +| Code | Typical cause | +|------|---------------| +| `Unsupported` | Backend not compiled in, or the feature isn't possible on this platform | +| `NotAvailable` | Backend compiled in but its service is missing (no Wayland display, no portal interface) | +| `PermissionDenied` | `/dev/uinput` not writable, portal request denied, Windows UIPI block | +| `Cancelled` | The user dismissed a portal dialog | +| `Timeout` | No answer from a portal dialog, or no frame arrived in time | +| `NotFound` | A window or monitor that doesn't exist | + +Usage patterns: + +```cpp +auto input = alvin::Input::create(); +if (!input) { + std::cerr << alvin::toString(input.error().code) << ": " << input.error().message << '\n'; + return 1; +} +(*input)->moveTo({100, 100}); // Status; check it or ignore it + +if (auto s = (*input)->click(); !s) // check a Status + std::cerr << s.error().message << '\n'; + +auto in = alvin::Input::create().value(); // or throw std::bad_expected_access on failure +``` + +--- + +## Coordinates and monitors + +`#include ` + +All positions are **global desktop coordinates**: + +- **Linux:** logical pixels, after monitor scaling. These are the numbers `hyprctl` reports. +- **Windows:** physical pixels. Alvin makes the process per-monitor DPI aware. + +```cpp +struct Point { int x, y; }; +struct Size { int width, height; }; + +struct Rect { + int x, y, width, height; + int right() const; // x + width + int bottom() const; // y + height + Point center() const; + bool empty() const; + bool contains(Point p) const; + Rect united(const Rect& o) const; // bounding box of both +}; + +struct Monitor { + std::string name; // "DP-2", "\\.\DISPLAY1" + std::string description; // make/model + Rect bounds; // position and size on the desktop + Size pixelSize; // actual mode, e.g. 3840x2160 + double scale; // pixelSize / logical size, e.g. 1.333 + bool primary; +}; + +Result> listMonitors(); +Result desktopBounds(); // bounding box of all monitors +``` + +```cpp +for (const auto& m : alvin::listMonitors().value()) + std::printf("%s at %d,%d (%dx%d logical, scale %.2f)\n", + m.name.c_str(), m.bounds.x, m.bounds.y, m.bounds.width, m.bounds.height, m.scale); +``` + +--- + +## Input + +`#include ` + +### Creating + +```cpp +static Result> Input::create(const InputOptions& options = {}); + +struct InputOptions { + InputBackend backend = InputBackend::Auto; + std::string restoreToken; // libei only: skip the portal dialog (see restoreToken()) + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; + +enum class InputBackend { Auto, WlrVirtual, LibEi, UInput, SendInput }; +``` + +| Backend | Platform | Notes | +|---------|----------|-------| +| `WlrVirtual` | Hyprland, sway, wlroots | No dialog. Types any Unicode text. | +| `LibEi` | GNOME, KDE (RemoteDesktop portal) | Shows a permission dialog unless given a restore token. | +| `UInput` | Any Linux session | Needs write access to `/dev/uinput`. Takes about 2.5 s to create on Hyprland. | +| `SendInput` | Windows | Can't inject into elevated windows unless Alvin runs elevated too. | + +With `Auto`, Linux tries `WlrVirtual`, then `LibEi`, then `UInput`. You can force a +backend with the environment variable `ALVIN_INPUT_BACKEND=wlr|libei|uinput` (Linux only). + +Create one `Input` and keep it for the life of your program. Creation is the +expensive part (connecting to the compositor, creating devices, portal dialogs). + +### Mouse + +| Call | Description | +|------|-------------| +| `Status moveTo(Point p)` | Move the pointer to an absolute desktop position. | +| `Status moveBy(int dx, int dy)` | Relative motion. With `uinput`, the compositor applies pointer acceleration. | +| `Status buttonDown(MouseButton b)` / `buttonUp(b)` | Press or release a button. | +| `Status click(MouseButton b = Left, int count = 1)` | Click in place. `count = 2` double-clicks. | +| `Status clickAt(Point p, MouseButton b = Left, int count = 1)` | Move, then click. | +| `Status drag(Point from, Point to, MouseButton b = Left)` | Press at `from`, move to `to` in 10 steps, release. | +| `Status scroll(int dx, int dy)` | Wheel notches. Positive `dy` scrolls **down**, positive `dx` scrolls **right**. | + +`MouseButton` is `Left`, `Right`, `Middle`, `Back` or `Forward`. + +```cpp +input->clickAt({1440, 810}); // left click +input->clickAt({1440, 810}, alvin::MouseButton::Right); +input->click(alvin::MouseButton::Left, 2); // double-click in place +input->drag({100, 100}, {600, 400}); // e.g. select or move something +input->scroll(0, 3); // 3 notches down +``` + +### Keyboard + +| Call | Description | +|------|-------------| +| `Status keyDown(Key k)` / `keyUp(Key k)` | Press or release one physical key. | +| `Status keyPress(Key k)` | Down, then up. | +| `Status chord(std::initializer_list)` | Press in order, release in reverse. | +| `Status chord(std::string_view)` | Same, from text like `"ctrl+shift+t"`. | +| `Status typeText(std::string_view utf8)` | Type text. `\n` presses Enter and `\t` presses Tab. | + +```cpp +using alvin::Key; +input->keyPress(Key::Enter); +input->chord({Key::LeftCtrl, Key::C}); +input->chord("ctrl+shift+t"); +input->chord("super+enter"); +input->typeText("Hello, wörld ✓\n"); +``` + +`chord()` always releases every key it pressed, even when an earlier call fails, +so a key is never left held down. + +**How `typeText` handles layouts:** + +- **`wlr`:** builds a keymap containing exactly the characters needed, so any Unicode text works. +- **`libei`:** uses the server's keyboard layout. If the server supports libei's text capability, it sends the text directly. +- **`uinput`:** uses the layout named in `XKB_DEFAULT_LAYOUT`/`XKB_DEFAULT_VARIANT` (default `us`). Characters that aren't on that layout fail with `Unsupported` before anything is typed. +- **`sendinput`:** sends Unicode directly, so any text works. + +### Other + +| Call | Description | +|------|-------------| +| `InputBackend backend() const` | Which backend was chosen. | +| `std::string restoreToken() const` | libei: save this and pass it in `InputOptions::restoreToken` next run to skip the dialog. Empty for other backends. | +| `void setEventDelay(std::chrono::milliseconds)` | Pause between the down and up halves of clicks, key presses and typed characters. The default is 8 ms. Raise it for applications that miss fast input. | +| `std::string_view toString(InputBackend)` | `"wlr"`, `"libei"`, `"uinput"`, `"sendinput"`, `"auto"` | + +--- + +## Keys and chords + +`#include ` + +`Key` names **physical key positions** on a US QWERTY layout, the same way evdev +and Win32 scan codes do. Use `typeText()` when you want particular characters. + +``` +A–Z, Num0–Num9, F1–F24, +Escape, Enter, Tab, Backspace, Space, +Minus, Equal, LeftBracket, RightBracket, Backslash, Semicolon, Apostrophe, Grave, Comma, Period, Slash, +CapsLock, NumLock, ScrollLock, +LeftShift, RightShift, LeftCtrl, RightCtrl, LeftAlt, RightAlt, LeftSuper, RightSuper, Menu, +Insert, Delete, Home, End, PageUp, PageDown, Left, Right, Up, Down, PrintScreen, Pause, +Kp0–Kp9, KpDecimal, KpDivide, KpMultiply, KpSubtract, KpAdd, KpEnter, +VolumeMute, VolumeDown, VolumeUp, MediaPlayPause, MediaStop, MediaNext, MediaPrevious +``` + +```cpp +std::optional keyFromName(std::string_view name); // case-insensitive +std::string_view keyName(Key key); +std::optional> parseChord(std::string_view chord); +bool isModifier(Key key); +``` + +Names accepted by `keyFromName` and in chord strings: + +| Key | Names | +|-----|-------| +| Letters and digits | `a`…`z`, `0`…`9` | +| Function keys | `f1`…`f24` | +| Modifiers | `shift`/`lshift`, `rshift`, `ctrl`/`control`/`lctrl`, `rctrl`, `alt`/`lalt`, `ralt`/`altgr`, `super`/`win`/`meta`/`cmd`/`lsuper`, `rsuper` | +| Editing | `enter`/`return`, `esc`/`escape`, `tab`, `backspace`, `space`, `insert`, `delete`/`del` | +| Navigation | `home`, `end`, `pageup`/`pgup`, `pagedown`/`pgdn`, `left`, `right`, `up`, `down` | +| Punctuation | `minus`/`-`, `equal`/`=`, `[`, `]`, `\`, `;`, `'`, `` ` ``, `,`, `.`, `/` (or their word names) | +| Keypad | `kp0`…`kp9`, `kpdecimal`, `kpdivide`, `kpmultiply`, `kpsubtract`, `kpadd`, `kpenter` | +| Media | `mute`, `volumeup`, `volumedown`, `playpause`, `stop`, `next`, `prev`/`previous` | +| Other | `capslock`, `numlock`, `scrolllock`, `printscreen`/`print`, `pause`, `menu` | + +Chords are joined with `+`, and spaces around keys are ignored. `"ctrl++"` means +Ctrl plus the `+`/`=` key. + +--- + +## Capture + +`#include ` + +### Creating + +```cpp +static Result> Capture::create(const CaptureOptions& options = {}); + +struct CaptureOptions { + CaptureBackend backend = CaptureBackend::Auto; // Auto, Portal, Gdi + CaptureSource source = CaptureSource::Monitor; // Monitor, Window, Any + int monitorIndex = 0; // Windows: which monitor (index into listMonitors()) + std::optional window; // Windows: capture this window directly + bool showCursor = false; + std::string restoreToken; // Portal: skip the picker (see below) + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; +``` + +**Linux (`Portal`):** `create()` opens the desktop's share picker. On Hyprland +that's `hyprland-share-picker`. `source` controls whether the picker offers +monitors, windows or both. When the call returns, a PipeWire stream is running +in the background. + +**Windows (`Gdi`):** there's no dialog. Set `monitorIndex`, or set `window` to +capture one window. + +### Skipping the picker next time + +```cpp +alvin::CaptureOptions opts; +opts.restoreToken = loadSavedToken(); // empty the first time +auto cap = alvin::Capture::create(opts).value(); +saveToken(cap->restoreToken()); // new token each session; always re-save it +``` + +The `alvin` CLI does this for you and stores tokens in `~/.cache/alvin/`. + +> Don't leave a picker unanswered. While it's open, xdg-desktop-portal-hyprland +> blocks every other portal request on the system, screen sharing in other +> apps included. Use `permissionTimeout` to limit how long `create()` waits. + +### Getting frames + +| Call | Description | +|------|-------------| +| `Result grab(timeout = 5s)` | The most recent frame. Blocks only until the first frame exists. | +| `Result nextFrame(timeout = 5s)` | Waits for a frame newer than the last one you received. | +| `std::string restoreToken() const` | Portal token for next time. | +| `CaptureBackend backend() const` | Which backend was chosen. | + +PipeWire streams only send a frame when the screen changes, so `grab()` can +return an image that's a few seconds old if nothing has moved. That's usually +what you want. After you act (click, type), call `nextFrame()` to see the result: + +```cpp +input->clickAt(button); +auto after = cap->nextFrame(std::chrono::seconds(2)); +if (!after && after.error().code == alvin::ErrorCode::Timeout) { + // nothing on screen changed +} +``` + +A `Capture` holds a live stream, so keep one and reuse it. Don't create one per frame. + +--- + +## Frame + +`#include ` + +A `Frame` is an owned BGRA8 image (4 bytes per pixel: B, G, R, A, with no row +padding). It also records where it sits on the desktop. + +| Member | Description | +|--------|-------------| +| `Frame(int w, int h)` | New zero-filled frame. | +| `int width()`, `int height()`, `Size size()` | Size in **pixels**. | +| `std::size_t stride()` | Bytes per row (`width * 4`). | +| `uint8_t* data()`, `row(y)`, `std::span bytes()` | Raw pixel access. | +| `Color pixel(int x, int y)` | `{r, g, b, a}`. Returns all zeros if out of bounds. | +| `Rect desktopRect()` | The area of the desktop this frame shows, in desktop coordinates. | +| `Point toDesktop(Point px)` | Frame pixel to desktop coordinate. | +| `Point fromDesktop(Point p)` | Desktop coordinate to frame pixel. | +| `Frame crop(Rect r)` | Sub-image (`r` is in frame pixels and gets clipped). `desktopRect()` is adjusted to match. | +| `int64_t timestampNs()` | Capture timestamp. | +| `Status savePpm(path)` | Save without dependencies (PPM). With vision, use `vision::saveImage` for PNG/JPG. | + +**Always convert through `toDesktop()` before clicking.** On a monitor scaled +to 1.33, a 3840×2160 frame covers 2880×1620 desktop units, so frame pixels +aren't desktop coordinates: + +```cpp +auto frame = cap->grab().value(); +alvin::Point pixel{1920, 1080}; // something found in the image +input->clickAt(frame.toDesktop(pixel)); // correct on any scale +``` + +```cpp +alvin::Color c = frame.pixel(10, 10); +if (c.r > 200 && c.g < 50 && c.b < 50) { /* red */ } +``` + +--- + +## Windows + +`#include ` + +Wayland hides other windows from normal clients, so each compositor needs its +own backend. Right now that's Hyprland (IPC socket) and Win32. + +```cpp +static Result> WindowManager::create(WindowBackend b = WindowBackend::Auto); + +struct WindowInfo { + WindowId id; // opaque handle; pass it to focus() + std::string title; + std::string appId; // Hyprland class / Win32 executable name + Rect bounds; // desktop coordinates + int64_t pid; + bool focused, visible, minimized, fullscreen; + std::string workspace; // Hyprland workspace name +}; +``` + +| Call | Description | +|------|-------------| +| `Result> list()` | All mapped top-level windows. | +| `Result> active()` | The focused window, or none. | +| `Result> find(std::string_view text)` | First window whose title or appId contains `text` (case-insensitive). | +| `Status focus(WindowId id)` | Focus it. On Hyprland this switches to its workspace. | + +```cpp +auto wm = alvin::WindowManager::create().value(); +if (auto w = wm->find("firefox").value()) { + wm->focus(w->id); + input->clickAt(w->bounds.center()); + // click 40px in from the top-left of the window + input->clickAt({w->bounds.x + 40, w->bounds.y + 40}); +} +``` + +--- + +## Vision (OpenCV) + +`#include `, link `alvin::vision`. + +| Call | Description | +|------|-------------| +| `cv::Mat view(const Frame&)` | Zero-copy `CV_8UC4` view. The Frame must outlive the Mat. | +| `cv::Mat toBgr(const Frame&)` | Deep copy as `CV_8UC3`, the layout most OpenCV code expects. | +| `Result fromMat(const cv::Mat&)` | 8-bit gray, BGR or BGRA to a Frame. | +| `Result loadImage(path)` | Load a PNG/JPG as BGR. | +| `Status saveImage(const Frame&, path)` | Save as PNG/JPG/… (by extension). | +| `Result> find(frame, needle, opts)` | Best template match at or above the threshold. | +| `Result> findAll(frame, needle, opts)` | Every non-overlapping match, best first. | + +```cpp +struct Match { Rect rect; double score; }; // rect in FRAME PIXELS, score 0..1 + +struct MatchOptions { + double threshold = 0.9; // lower it (0.7–0.85) if the target is anti-aliased or slightly different + bool grayscale = false; // faster; ignores colour + int maxResults = 16; // findAll only +}; +``` + +Take template images from Alvin's own captures, e.g. `alvin capture` followed by +a crop. That keeps the scale and colours identical, which template matching needs. + +```cpp +auto needle = alvin::vision::loadImage("save_button.png").value(); +auto frame = cap->grab().value(); +if (auto hit = alvin::vision::find(frame, needle, {.threshold = 0.85}).value()) + input->clickAt(frame.toDesktop(hit->rect.center())); + +for (const auto& m : alvin::vision::findAll(frame, needle, {.threshold = 0.8}).value()) + std::printf("match at %d,%d score %.2f\n", m.rect.x, m.rect.y, m.score); +``` + +For anything else OpenCV can do, use `vision::view(frame)` to get a `cv::Mat`. + +--- + +## Putting it together + +A small agent loop: wait for a button to appear, click it, then check that the +screen changed. + +```cpp +#include +#include +#include +#include +#include + +using namespace std::chrono_literals; + +int main() { + auto input = alvin::Input::create(); + if (!input) { std::cerr << input.error().message << '\n'; return 1; } + + auto cap = alvin::Capture::create(); // the picker appears on first run + if (!cap) { std::cerr << cap.error().message << '\n'; return 1; } + + auto button = alvin::vision::loadImage("accept.png"); + if (!button) { std::cerr << button.error().message << '\n'; return 1; } + + for (int attempt = 0; attempt < 30; ++attempt) { + auto frame = (*cap)->grab(); + if (!frame) { std::cerr << frame.error().message << '\n'; return 1; } + + auto hit = alvin::vision::find(*frame, *button, {.threshold = 0.85}); + if (hit && *hit) { + (void)(*input)->clickAt(frame->toDesktop((*hit)->rect.center())); + if ((*cap)->nextFrame(2s)) std::cout << "clicked; screen updated\n"; + return 0; + } + std::this_thread::sleep_for(1s); + } + std::cerr << "button never appeared\n"; + return 1; +} +``` + +See `examples/alvin_cli.cpp` in the repo for a complete program +that uses every part of the API. diff --git a/AlvinDoc/Input.md b/AlvinDoc/Input.md new file mode 100644 index 0000000..0a0c48f --- /dev/null +++ b/AlvinDoc/Input.md @@ -0,0 +1,5 @@ +# Linux +[[Libs#[libei](https //gitlab.freedesktop.org/libinput/libei) - linux input|libei]] +# Windows +[[Libs#SendInput Win32 API - windows input|SendInput]] + diff --git a/AlvinDoc/Libs.md b/AlvinDoc/Libs.md new file mode 100644 index 0000000..861080b --- /dev/null +++ b/AlvinDoc/Libs.md @@ -0,0 +1,10 @@ +## [libei](https://gitlab.freedesktop.org/libinput/libei) - linux input +## SendInput Win32 API - windows input + +# Dear ImGui + +# Pipewire (libpipewire) + +# xdg-desktop-portal (maybe) + +# [OpenCV](https://opencv.org/) \ No newline at end of file diff --git a/AlvinDoc/Vision.md b/AlvinDoc/Vision.md new file mode 100644 index 0000000..81dc6cd --- /dev/null +++ b/AlvinDoc/Vision.md @@ -0,0 +1,15 @@ +# Screen Capture & Computer Vision (Decision Making) +To "see" the screen and make complex logical evaluations, combine low-level frame grabbers with computer vision wrappers. + + **Linux Capture (Wayland/Hyprland):** Implement **PipeWire** or utilize the **xdg-desktop-portal (ScreenCast)** API. Unprivileged apps cannot take screenshots directly in Wayland, so you must stream frames via PipeWire. [[1](https://www.reddit.com/r/kde/comments/113wt14/wayland_input_emulation_org_kde_kwin_fake_input/), [2](https://github.com/can1357/oh-my-pi/issues/7886)] + + **Cross-Platform Vision Processing:** **[OpenCV (C++)](https://opencv.org/)** is the industry standard for processing raw frame arrays. It allows for image template matching (e.g., finding a specific button on screen), color analysis, and image thresholding. [[1](https://www.reddit.com/r/hyprland/comments/1rwxfae/i_built_a_scrolling_longscreenshot_tool_for/)] + +**Text Recognition (OCR):** Integrate **[Tesseract OCR](https://github.com/tesseract-ocr/tesseract)** if the bot needs to read on-screen text, menus, or labels to inform its choices. + + +# Cross-Platform Window Management & Coordinates +A major pitfall of Wayland automation is that clients do not know where other windows are located on the global coordinate space. [[1](https://github.com/developer239/robot-cpp)] + +- **Linux (Hyprland IPC):** Use **`hyprctl`** JSON sockets or the **`hyprland-ipc`** protocol to query window positions, bounding boxes, and active window titles. Because this mechanism is compositor-dependent, implement it cleanly under a modular interface. +- **Windows/macOS:** Use Win32 (`GetWindowRect`, `GetForegroundWindow`) and the macOS `CGWindowListCopyWindowInfo` API respectively to achieve identical window tracking capabilities on alternative platforms. \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..73454e9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,259 @@ +cmake_minimum_required(VERSION 3.24) + +project(Alvin + VERSION 0.1.0 + DESCRIPTION "Agent Leveraging Vision and Input: cross-platform input synthesis and screen capture" + LANGUAGES C CXX) # C only for wayland-scanner output + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +include(GNUInstallDirs) + +# --------------------------------------------------------------------------- +# Options. "AUTO" enables a backend when its dependencies are found. +# --------------------------------------------------------------------------- +set(ALVIN_WITH_WAYLAND AUTO CACHE STRING "Wayland virtual pointer/keyboard input + monitor listing (ON/OFF/AUTO)") +set(ALVIN_WITH_LIBEI AUTO CACHE STRING "libei input via the RemoteDesktop portal (ON/OFF/AUTO)") +set(ALVIN_WITH_PORTAL AUTO CACHE STRING "ScreenCast portal + PipeWire capture (ON/OFF/AUTO)") +set(ALVIN_WITH_UINPUT ON CACHE BOOL "/dev/uinput input backend") +set(ALVIN_WITH_VISION AUTO CACHE STRING "OpenCV-based alvin::vision library (ON/OFF/AUTO)") +option(ALVIN_BUILD_EXAMPLES "Build the alvin command-line tool" ON) +option(ALVIN_BUILD_TESTS "Build unit tests" ${PROJECT_IS_TOP_LEVEL}) + +# Resolves an AUTO/ON/OFF option against whether its dependencies were found. +function(alvin_resolve option found) + if(${option} STREQUAL "AUTO") + set(${option}_ENABLED ${found} PARENT_SCOPE) + elseif(${option}) + if(NOT ${found}) + message(FATAL_ERROR "${option}=ON but its dependencies were not found") + endif() + set(${option}_ENABLED TRUE PARENT_SCOPE) + else() + set(${option}_ENABLED FALSE PARENT_SCOPE) + endif() +endfunction() + +add_library(alvin + src/common/core.cpp + src/common/frame.cpp + src/common/json.cpp + src/common/keys.cpp +) +add_library(alvin::alvin ALIAS alvin) +target_include_directories(alvin + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src) +target_compile_features(alvin PUBLIC cxx_std_23) +if(MSVC) + target_compile_options(alvin PRIVATE /W4 /utf-8) +else() + target_compile_options(alvin PRIVATE -Wall -Wextra -Wpedantic) +endif() + +if(WIN32) + # No export macros in the headers; let CMake export every symbol from the DLL. + set_target_properties(alvin PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif() + +set(ALVIN_ENABLED_BACKENDS "") + +# Links a pkg-config dependency privately and records how AlvinConfig.cmake can +# find it again: a static libalvin carries no record of its own dependencies. +set(ALVIN_CONFIG_DEPENDENCIES "") +macro(alvin_private_pkg var) + target_link_libraries(alvin PRIVATE PkgConfig::${var}) + string(APPEND ALVIN_CONFIG_DEPENDENCIES " pkg_check_modules(${var} REQUIRED QUIET IMPORTED_TARGET ${ARGN})\n") +endmacro() + +# --------------------------------------------------------------------------- +# Linux +# --------------------------------------------------------------------------- +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_package(PkgConfig REQUIRED) + pkg_check_modules(XKBCOMMON REQUIRED IMPORTED_TARGET xkbcommon) + pkg_check_modules(WAYLAND IMPORTED_TARGET wayland-client) + pkg_check_modules(WAYLAND_PROTOCOLS wayland-protocols) + find_program(WAYLAND_SCANNER wayland-scanner) + pkg_check_modules(GIO IMPORTED_TARGET gio-2.0 gio-unix-2.0) + pkg_check_modules(PIPEWIRE IMPORTED_TARGET libpipewire-0.3) + pkg_check_modules(LIBEI IMPORTED_TARGET libei-1.0) + + set(_wayland_found FALSE) + if(WAYLAND_FOUND AND WAYLAND_PROTOCOLS_FOUND AND WAYLAND_SCANNER) + set(_wayland_found TRUE) + endif() + set(_portal_found FALSE) + if(GIO_FOUND AND PIPEWIRE_FOUND) + set(_portal_found TRUE) + endif() + set(_libei_found FALSE) + if(GIO_FOUND AND LIBEI_FOUND) + set(_libei_found TRUE) + endif() + alvin_resolve(ALVIN_WITH_WAYLAND _wayland_found) + alvin_resolve(ALVIN_WITH_PORTAL _portal_found) + alvin_resolve(ALVIN_WITH_LIBEI _libei_found) + + target_sources(alvin PRIVATE + src/linux/platform_linux.cpp + src/linux/hyprland_windows.cpp + src/linux/keycodes.cpp + src/linux/xkb.cpp) + alvin_private_pkg(XKBCOMMON xkbcommon) + + if(ALVIN_WITH_WAYLAND_ENABLED) + pkg_get_variable(WAYLAND_PROTOCOLS_DIR wayland-protocols pkgdatadir) + set(_proto_out ${CMAKE_CURRENT_BINARY_DIR}/protocols) + file(MAKE_DIRECTORY ${_proto_out}) + set(_protocols + ${CMAKE_CURRENT_SOURCE_DIR}/protocols/wlr-virtual-pointer-unstable-v1.xml + ${CMAKE_CURRENT_SOURCE_DIR}/protocols/virtual-keyboard-unstable-v1.xml + ${WAYLAND_PROTOCOLS_DIR}/unstable/xdg-output/xdg-output-unstable-v1.xml) + foreach(xml ${_protocols}) + get_filename_component(name ${xml} NAME_WE) + add_custom_command( + OUTPUT ${_proto_out}/${name}-client-protocol.h ${_proto_out}/${name}-protocol.c + COMMAND ${WAYLAND_SCANNER} client-header ${xml} ${_proto_out}/${name}-client-protocol.h + COMMAND ${WAYLAND_SCANNER} private-code ${xml} ${_proto_out}/${name}-protocol.c + DEPENDS ${xml} + VERBATIM) + list(APPEND _proto_sources ${_proto_out}/${name}-protocol.c) + list(APPEND _proto_headers ${_proto_out}/${name}-client-protocol.h) + endforeach() + # wayland-scanner emits C; build it as a small C object library. + add_library(alvin_wayland_protocols OBJECT ${_proto_sources} ${_proto_headers}) + target_include_directories(alvin_wayland_protocols PUBLIC ${_proto_out}) + target_link_libraries(alvin_wayland_protocols PUBLIC PkgConfig::WAYLAND) + set_target_properties(alvin_wayland_protocols PROPERTIES POSITION_INDEPENDENT_CODE ON) + + target_sources(alvin PRIVATE + src/linux/wayland.cpp + src/linux/wlr_input.cpp + $) + target_include_directories(alvin PRIVATE ${_proto_out}) + alvin_private_pkg(WAYLAND wayland-client) + add_dependencies(alvin alvin_wayland_protocols) + target_compile_definitions(alvin PRIVATE ALVIN_HAVE_WAYLAND) + list(APPEND ALVIN_ENABLED_BACKENDS "wlr-input" "wayland-monitors") + endif() + + if(ALVIN_WITH_PORTAL_ENABLED OR ALVIN_WITH_LIBEI_ENABLED) + target_sources(alvin PRIVATE src/linux/portal.cpp) + alvin_private_pkg(GIO gio-2.0 gio-unix-2.0) + endif() + + if(ALVIN_WITH_PORTAL_ENABLED) + target_sources(alvin PRIVATE src/linux/pipewire_capture.cpp) + alvin_private_pkg(PIPEWIRE libpipewire-0.3) + target_compile_definitions(alvin PRIVATE ALVIN_HAVE_PORTAL) + list(APPEND ALVIN_ENABLED_BACKENDS "portal-capture") + endif() + + if(ALVIN_WITH_LIBEI_ENABLED) + target_sources(alvin PRIVATE src/linux/libei_input.cpp) + alvin_private_pkg(LIBEI libei-1.0) + target_compile_definitions(alvin PRIVATE ALVIN_HAVE_LIBEI) + if(LIBEI_VERSION VERSION_GREATER_EQUAL 1.6) + target_compile_definitions(alvin PRIVATE ALVIN_LIBEI_HAS_TEXT) + endif() + list(APPEND ALVIN_ENABLED_BACKENDS "libei-input") + endif() + + if(ALVIN_WITH_UINPUT) + target_sources(alvin PRIVATE src/linux/uinput_input.cpp) + target_compile_definitions(alvin PRIVATE ALVIN_HAVE_UINPUT) + list(APPEND ALVIN_ENABLED_BACKENDS "uinput-input") + endif() + + list(APPEND ALVIN_ENABLED_BACKENDS "hyprland-windows") + +# --------------------------------------------------------------------------- +# Windows +# --------------------------------------------------------------------------- +elseif(WIN32) + target_sources(alvin PRIVATE + src/windows/platform_windows.cpp + src/windows/sendinput_input.cpp + src/windows/gdi_capture.cpp + src/windows/win32_windows.cpp) + target_compile_definitions(alvin PRIVATE UNICODE _UNICODE) + target_link_libraries(alvin PRIVATE user32 gdi32 dwmapi) + list(APPEND ALVIN_ENABLED_BACKENDS "sendinput-input" "gdi-capture" "win32-windows") +else() + message(FATAL_ERROR "Alvin does not support ${CMAKE_SYSTEM_NAME} yet") +endif() + +# --------------------------------------------------------------------------- +# Optional OpenCV integration +# --------------------------------------------------------------------------- +find_package(OpenCV QUIET COMPONENTS core imgproc imgcodecs) +alvin_resolve(ALVIN_WITH_VISION OpenCV_FOUND) +if(ALVIN_WITH_VISION_ENABLED) + add_library(alvin_vision src/vision/vision.cpp) + add_library(alvin::vision ALIAS alvin_vision) + target_link_libraries(alvin_vision PUBLIC alvin ${OpenCV_LIBS}) + set_target_properties(alvin_vision PROPERTIES EXPORT_NAME vision) + if(WIN32) + set_target_properties(alvin_vision PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) + endif() + list(APPEND ALVIN_ENABLED_BACKENDS "vision(OpenCV ${OpenCV_VERSION})") +endif() + +message(STATUS "Alvin backends: ${ALVIN_ENABLED_BACKENDS}") + +if(ALVIN_BUILD_EXAMPLES) + add_subdirectory(examples) +endif() + +if(ALVIN_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() + +# --------------------------------------------------------------------------- +# Install + find_package(Alvin) support +# --------------------------------------------------------------------------- +include(CMakePackageConfigHelpers) +set(ALVIN_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/Alvin) + +set(_alvin_install_targets alvin) +if(ALVIN_WITH_VISION_ENABLED) + list(APPEND _alvin_install_targets alvin_vision) +endif() +install(TARGETS ${_alvin_install_targets} EXPORT AlvinTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(EXPORT AlvinTargets NAMESPACE alvin:: DESTINATION ${ALVIN_CMAKE_DIR}) + +# Values baked into AlvinConfig.cmake. +get_target_property(_alvin_type alvin TYPE) +if(_alvin_type STREQUAL "STATIC_LIBRARY" AND NOT ALVIN_CONFIG_DEPENDENCIES STREQUAL "") + set(ALVIN_NEEDS_PKGCONFIG TRUE) +else() + set(ALVIN_NEEDS_PKGCONFIG FALSE) +endif() +if(ALVIN_WITH_VISION_ENABLED) + set(ALVIN_HAS_VISION TRUE) +else() + set(ALVIN_HAS_VISION FALSE) +endif() + +configure_package_config_file(cmake/AlvinConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/AlvinConfig.cmake + INSTALL_DESTINATION ${ALVIN_CMAKE_DIR}) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/AlvinConfigVersion.cmake + COMPATIBILITY SameMinorVersion) # pre-1.0: minor versions may break the API +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/AlvinConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/AlvinConfigVersion.cmake + DESTINATION ${ALVIN_CMAKE_DIR}) diff --git a/README.md b/README.md index 7bbf04f..9f4e5ad 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,254 @@ # Alvin -Agent Leveraging Vision and Input. Nice. \ No newline at end of file +Agent Leveraging Vision and Input. Nice. + +A cross-platform C++23 library that lets a program **see the screen** (capture monitors or windows), +**act on it** (synthesize mouse and keyboard input), and **find things on it** (optional OpenCV template +matching). All of this sits behind one platform-neutral API, with swappable backends for each desktop. + +```cpp +#include +#include + +auto input = alvin::Input::create().value(); +auto screen = alvin::Capture::create().value(); // portal picker appears the first time only + +auto frame = screen->grab().value(); +auto button = alvin::vision::loadImage("ok_button.png").value(); +if (auto hit = alvin::vision::find(frame, button).value()) + input->clickAt(frame.toDesktop(hit->rect.center())); + +input->chord("ctrl+s"); +input->typeText("héllo wörld ✓"); +``` + +**Full API reference: [docs/API.md](docs/API.md)** + +## Layout + +``` +include/alvin/ public API (the only headers users include) + input.hpp Input: moveTo/moveBy/click/drag/scroll/keyDown/keyUp/chord/typeText + capture.hpp Capture: grab() latest frame, nextFrame() wait for a new one + frame.hpp Frame: BGRA pixels + where they sit on the desktop + window.hpp WindowManager: list/active/find/focus top-level windows + types.hpp Point/Rect/Monitor, listMonitors(), desktopBounds() + keys.hpp Key enum (physical keys), MouseButton, chord parsing + vision.hpp optional OpenCV helpers (alvin::vision target) +src/common/ platform-neutral code + the internal platform interface +src/linux/ Linux backends +src/windows/ Windows backends +src/vision/ OpenCV integration +protocols/ vendored Wayland protocol XML +examples/ the `alvin` CLI +tests/ Catch2 unit tests +``` + +Every public type is platform-neutral; backend headers (libei, PipeWire, Wayland, Win32) never leak +into `include/`. Each platform implements four entry points in `src/common/platform.hpp` +(`createInput`, `createCapture`, `createWindowManager`, `listMonitors`), which pick a backend at runtime. + +## Backends + +| Area | Backend | Where it works | Status | +|---------|---------|----------------|--------| +| Input | `wlr` — `zwlr_virtual_pointer_v1` + `zwp_virtual_keyboard_v1` | Hyprland, sway, wlroots compositors. No dialog. | Tested on Hyprland (motion, multi-monitor, fractional scale) | +| Input | `libei` — via the RemoteDesktop portal | GNOME, KDE Plasma | Builds; untested (Hyprland's portal has no RemoteDesktop) | +| Input | `uinput` — kernel virtual devices | Any Linux session incl. X11/TTY; needs write access to `/dev/uinput` | Tested on Hyprland (absolute + relative motion) | +| Input | `sendinput` — Win32 `SendInput` | Windows | Written; not yet compiled | +| Capture | `portal` — ScreenCast portal + PipeWire (SHM buffers) | Any modern Wayland desktop | Portal handshake tested; frame path awaiting first run | +| Capture | `gdi` — BitBlt / PrintWindow | Windows | Written; not yet compiled | +| Windows | `hyprland` — IPC socket | Hyprland | Tested | +| Windows | `win32` — EnumWindows + DWM | Windows | Written; not yet compiled | + +With `InputBackend::Auto`, Linux tries `wlr` → `libei` → `uinput`. Override with +`ALVIN_INPUT_BACKEND=wlr|libei|uinput` or `InputOptions::backend`. + +### Things worth knowing + +- **Coordinates** are global desktop coordinates in logical pixels on Linux (what `hyprctl` reports) + and physical pixels on Windows (Alvin makes the process per-monitor DPI aware). A `Frame` from a + scaled monitor has more pixels than its logical size; use `frame.toDesktop()` / `fromDesktop()`. +- **Portal restore tokens**: `Capture::restoreToken()` / `Input::restoreToken()` return a token; pass + it back in the options next run to skip the picker. The CLI caches them in `~/.cache/alvin/`. +- **Typing text**: `wlr` uploads a keymap containing exactly the characters needed, so any Unicode + text works regardless of layout. `uinput` and `libei` go through the layout (`libei` uses the + server's keymap; `uinput` uses `XKB_DEFAULT_LAYOUT`, defaulting to `us`). +- **uinput** devices take ~2.5 s to be adopted by Hyprland; `Input::create()` waits for the compositor + to open them. Relative motion through uinput is subject to pointer acceleration. +- **Windows UIPI** blocks injecting input into elevated windows unless Alvin runs elevated too. + +## Building + +Arch Linux dependencies (most are already on a Hyprland system): + +```sh +sudo pacman -S --needed cmake ninja pkgconf wayland wayland-protocols libxkbcommon \ + glib2 pipewire libei opencv catch2 +``` + +```sh +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build +ctest --test-dir build +``` + +Options (`ON`/`OFF`/`AUTO`, where `AUTO` enables a backend when its dependencies are found): +`ALVIN_WITH_WAYLAND`, `ALVIN_WITH_PORTAL`, `ALVIN_WITH_LIBEI`, `ALVIN_WITH_VISION`, plus +`ALVIN_WITH_UINPUT`, `ALVIN_BUILD_EXAMPLES`, `ALVIN_BUILD_TESTS`. + +On Windows, build with a recent MSVC (C++23 `std::expected`) and CMake; OpenCV is optional. + +## Using Alvin in your project + +There are two ways to use Alvin from another project. Either way you link the same two targets: + +| Target | What it gives you | +|--------|-------------------| +| `alvin::alvin` | Core library: input, capture, windows, monitors (`#include `) | +| `alvin::vision` | OpenCV helpers (`#include `). Only exists if Alvin was built with OpenCV. | + +Your project must be built as C++23 or newer. Linking `alvin::alvin` requests that for you. + +### Option 1: build Alvin with your project (`add_subdirectory`) + +Your build compiles Alvin from source alongside your own code. There's nothing to install, and +changes to Alvin are picked up on your next build. This is the easiest choice while Alvin is still +changing. + +1. Put Alvin somewhere your project can reach, e.g. as a git submodule: + + ```sh + cd my_agent + git submodule add https://github.com//Alvin.git external/Alvin + ``` + + Or just point at your existing checkout (step 2 accepts an absolute path). + +2. In your project's `CMakeLists.txt`: + + ```cmake + cmake_minimum_required(VERSION 3.24) + project(my_agent CXX) + + # Optional: skip Alvin's own tests and CLI while building your project. + set(ALVIN_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(ALVIN_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + + add_subdirectory(external/Alvin) + # With a checkout outside your source tree, give a build directory name as well: + # add_subdirectory(/home/you/gitRepos/Alvin alvin) + + add_executable(my_agent main.cpp) + target_link_libraries(my_agent PRIVATE alvin::alvin alvin::vision) + ``` + +3. Build as normal: + + ```sh + cmake -S . -B build -G Ninja + cmake --build build + ``` + +### Option 2: install Alvin once, then `find_package(Alvin)` + +This builds and installs Alvin as a compiled library (static `.a`/`.lib` or shared `.so`/`.dll`) +with its headers. Any number of projects can then find it with `find_package`, without rebuilding +Alvin. + +1. Build and install Alvin. Pick an install prefix: `~/.local` for just you (no sudo), or + `/usr/local` for the whole system. + + ```sh + cd Alvin + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$HOME/.local \ + -DBUILD_SHARED_LIBS=ON # OFF (the default) for a static library + cmake --build build + cmake --install build # prefix with sudo for /usr/local + ``` + + This installs: + + ``` + /include/alvin/*.hpp headers + /lib/libalvin.so (or .a) library, plus libalvin_vision if built + /lib/cmake/Alvin/AlvinConfig.cmake what find_package reads + /bin/alvin the CLI + ``` + +2. In your project's `CMakeLists.txt`: + + ```cmake + cmake_minimum_required(VERSION 3.24) + project(my_agent CXX) + + find_package(Alvin 0.1 REQUIRED) # core only + # find_package(Alvin 0.1 REQUIRED COMPONENTS vision) # fail early if Alvin was built without OpenCV + + add_executable(my_agent main.cpp) + target_link_libraries(my_agent PRIVATE alvin::alvin alvin::vision) + ``` + +3. Configure your project, telling CMake where Alvin is installed. `/usr/local` and `/usr` are + searched automatically; for anything else, pass the prefix: + + ```sh + cmake -S . -B build -G Ninja -DCMAKE_PREFIX_PATH=$HOME/.local + cmake --build build + ``` + +**Notes for option 2** + +- **After updating Alvin**, re-run the `cmake --build` and `cmake --install` commands from step 1. + Projects linked against the shared library use the new version the next time they run, but only + if the API didn't change; otherwise rebuild them. Projects linked against the static library + always need rebuilding. +- **Static vs shared:** a static install bakes Alvin into your executable. Alvin's own dependencies + (Wayland, PipeWire, libei, …) must still be installed wherever your program is *built*; + `AlvinConfig.cmake` finds them for you. A shared install puts all of that in `libalvin.so`. +- **"cannot open shared object file: libalvin.so"** at runtime means the library's folder isn't on + the loader's search path. This is common with `~/.local` or `/usr/local` prefixes. Either run + `export LD_LIBRARY_PATH=$HOME/.local/lib`, install to `/usr`, or add the folder to + `/etc/ld.so.conf.d/` and run `sudo ldconfig`. +- **Versioning:** `find_package(Alvin 0.1)` accepts any 0.1.x. Before 1.0, a new minor version + (0.2) may change the API, so it won't be accepted as a match for 0.1. +- **Windows:** the same steps should work with MSVC, but they haven't been tested yet. Use + `--config Release` on the build and install commands, and put the installed `bin` folder (where + `alvin.dll` goes) on `PATH` when running. + +### Without CMake + +If your project doesn't use CMake, do an option 2 install with `-DBUILD_SHARED_LIBS=ON`, then add +`/include` to your include path and link `-lalvin` (plus `-lalvin_vision` and OpenCV if you +use vision). For example: + +```sh +g++ -std=c++23 main.cpp -I$HOME/.local/include -L$HOME/.local/lib -lalvin -o my_agent +LD_LIBRARY_PATH=$HOME/.local/lib ./my_agent +``` + +## The `alvin` CLI + +```sh +alvin monitors +alvin windows +alvin focus firefox +alvin move 1440 810 +alvin clickat 1440 810 right +alvin key ctrl+shift+t +alvin type "hello" +alvin capture shot.png # first run shows the share picker +alvin find button.png 0.85 # prints desktop coordinates of the match +alvin --input uinput move 100 100 +``` + +## Roadmap + +- Promptless Hyprland capture (`hyprland-toplevel-export-v1`, `ext-image-copy-capture-v1`) + so an agent can capture any window by id without a picker +- DMA-BUF import for zero-copy PipeWire frames +- X11 backends (XTest input, XShm capture) +- Windows: DXGI Desktop Duplication / Windows.Graphics.Capture; CI build on Windows +- macOS: CGEvent input, ScreenCaptureKit capture +- OCR (Tesseract) in the vision module diff --git a/cmake/AlvinConfig.cmake.in b/cmake/AlvinConfig.cmake.in new file mode 100644 index 0000000..86309b1 --- /dev/null +++ b/cmake/AlvinConfig.cmake.in @@ -0,0 +1,18 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +# A static libalvin doesn't record its own dependencies, so find them again. +if(@ALVIN_NEEDS_PKGCONFIG@) + find_dependency(PkgConfig) +@ALVIN_CONFIG_DEPENDENCIES@endif() + +set(Alvin_vision_FOUND @ALVIN_HAS_VISION@) +if(Alvin_vision_FOUND) + find_dependency(OpenCV COMPONENTS core imgproc imgcodecs) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/AlvinTargets.cmake") + +# find_package(Alvin REQUIRED COMPONENTS vision) fails if Alvin was built without OpenCV. +check_required_components(Alvin) diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..9e6b2b0 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,498 @@ +# Alvin API Reference + +Everything public lives in namespace `alvin` (vision helpers in `alvin::vision`). +Include the umbrella header for the core library: + +```cpp +#include // Input, Capture, WindowManager, Frame, keys, types +#include // optional, needs the alvin::vision target (OpenCV) +``` + +- [Using Alvin in your project](#using-alvin-in-your-project) +- [Error handling](#error-handling) +- [Coordinates and monitors](#coordinates-and-monitors) +- [Input](#input) +- [Keys and chords](#keys-and-chords) +- [Capture](#capture) +- [Frame](#frame) +- [Windows](#windows) +- [Vision (OpenCV)](#vision-opencv) +- [Putting it together](#putting-it-together) + +--- + +## Using Alvin in your project + +Either build Alvin as part of your project, or install it once and use `find_package`. +The README's "Using Alvin in your project" section has step-by-step instructions for both. + +```cmake +add_subdirectory(external/Alvin) # option 1: build from source +# find_package(Alvin 0.1 REQUIRED) # option 2: use an installed Alvin + +target_link_libraries(my_agent PRIVATE alvin::alvin) # core +target_link_libraries(my_agent PRIVATE alvin::vision) # + OpenCV helpers, if built +``` + +Alvin requires C++23 (it uses `std::expected`). + +--- + +## Error handling + +Alvin never throws. Every call that can fail returns a `Result`, which is +`std::expected`. Calls with no value return `Status` +(`Result`). + +```cpp +struct Error { + ErrorCode code; + std::string message; // human-readable detail +}; + +enum class ErrorCode { + Unknown, Unsupported, NotAvailable, PermissionDenied, Cancelled, + Timeout, InvalidArgument, NotFound, Io, Protocol, +}; + +std::string_view toString(ErrorCode code); +``` + +| Code | Typical cause | +|------|---------------| +| `Unsupported` | Backend not compiled in, or the feature isn't possible on this platform | +| `NotAvailable` | Backend compiled in but its service is missing (no Wayland display, no portal interface) | +| `PermissionDenied` | `/dev/uinput` not writable, portal request denied, Windows UIPI block | +| `Cancelled` | The user dismissed a portal dialog | +| `Timeout` | No answer from a portal dialog, or no frame arrived in time | +| `NotFound` | A window or monitor that doesn't exist | + +Usage patterns: + +```cpp +auto input = alvin::Input::create(); +if (!input) { + std::cerr << alvin::toString(input.error().code) << ": " << input.error().message << '\n'; + return 1; +} +(*input)->moveTo({100, 100}); // Status; check it or ignore it + +if (auto s = (*input)->click(); !s) // check a Status + std::cerr << s.error().message << '\n'; + +auto in = alvin::Input::create().value(); // or throw std::bad_expected_access on failure +``` + +--- + +## Coordinates and monitors + +`#include ` + +All positions are **global desktop coordinates**: + +- **Linux:** logical pixels, after monitor scaling. These are the numbers `hyprctl` reports. +- **Windows:** physical pixels. Alvin makes the process per-monitor DPI aware. + +```cpp +struct Point { int x, y; }; +struct Size { int width, height; }; + +struct Rect { + int x, y, width, height; + int right() const; // x + width + int bottom() const; // y + height + Point center() const; + bool empty() const; + bool contains(Point p) const; + Rect united(const Rect& o) const; // bounding box of both +}; + +struct Monitor { + std::string name; // "DP-2", "\\.\DISPLAY1" + std::string description; // make/model + Rect bounds; // position and size on the desktop + Size pixelSize; // actual mode, e.g. 3840x2160 + double scale; // pixelSize / logical size, e.g. 1.333 + bool primary; +}; + +Result> listMonitors(); +Result desktopBounds(); // bounding box of all monitors +``` + +```cpp +for (const auto& m : alvin::listMonitors().value()) + std::printf("%s at %d,%d (%dx%d logical, scale %.2f)\n", + m.name.c_str(), m.bounds.x, m.bounds.y, m.bounds.width, m.bounds.height, m.scale); +``` + +--- + +## Input + +`#include ` + +### Creating + +```cpp +static Result> Input::create(const InputOptions& options = {}); + +struct InputOptions { + InputBackend backend = InputBackend::Auto; + std::string restoreToken; // libei only: skip the portal dialog (see restoreToken()) + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; + +enum class InputBackend { Auto, WlrVirtual, LibEi, UInput, SendInput }; +``` + +| Backend | Platform | Notes | +|---------|----------|-------| +| `WlrVirtual` | Hyprland, sway, wlroots | No dialog. Types any Unicode text. | +| `LibEi` | GNOME, KDE (RemoteDesktop portal) | Shows a permission dialog unless given a restore token. | +| `UInput` | Any Linux session | Needs write access to `/dev/uinput`. Takes about 2.5 s to create on Hyprland. | +| `SendInput` | Windows | Can't inject into elevated windows unless Alvin runs elevated too. | + +With `Auto`, Linux tries `WlrVirtual`, then `LibEi`, then `UInput`. You can force a +backend with the environment variable `ALVIN_INPUT_BACKEND=wlr|libei|uinput` (Linux only). + +Create one `Input` and keep it for the life of your program. Creation is the +expensive part (connecting to the compositor, creating devices, portal dialogs). + +### Mouse + +| Call | Description | +|------|-------------| +| `Status moveTo(Point p)` | Move the pointer to an absolute desktop position. | +| `Status moveBy(int dx, int dy)` | Relative motion. With `uinput`, the compositor applies pointer acceleration. | +| `Status buttonDown(MouseButton b)` / `buttonUp(b)` | Press or release a button. | +| `Status click(MouseButton b = Left, int count = 1)` | Click in place. `count = 2` double-clicks. | +| `Status clickAt(Point p, MouseButton b = Left, int count = 1)` | Move, then click. | +| `Status drag(Point from, Point to, MouseButton b = Left)` | Press at `from`, move to `to` in 10 steps, release. | +| `Status scroll(int dx, int dy)` | Wheel notches. Positive `dy` scrolls **down**, positive `dx` scrolls **right**. | + +`MouseButton` is `Left`, `Right`, `Middle`, `Back` or `Forward`. + +```cpp +input->clickAt({1440, 810}); // left click +input->clickAt({1440, 810}, alvin::MouseButton::Right); +input->click(alvin::MouseButton::Left, 2); // double-click in place +input->drag({100, 100}, {600, 400}); // e.g. select or move something +input->scroll(0, 3); // 3 notches down +``` + +### Keyboard + +| Call | Description | +|------|-------------| +| `Status keyDown(Key k)` / `keyUp(Key k)` | Press or release one physical key. | +| `Status keyPress(Key k)` | Down, then up. | +| `Status chord(std::initializer_list)` | Press in order, release in reverse. | +| `Status chord(std::string_view)` | Same, from text like `"ctrl+shift+t"`. | +| `Status typeText(std::string_view utf8)` | Type text. `\n` presses Enter and `\t` presses Tab. | + +```cpp +using alvin::Key; +input->keyPress(Key::Enter); +input->chord({Key::LeftCtrl, Key::C}); +input->chord("ctrl+shift+t"); +input->chord("super+enter"); +input->typeText("Hello, wörld ✓\n"); +``` + +`chord()` always releases every key it pressed, even when an earlier call fails, +so a key is never left held down. + +**How `typeText` handles layouts:** + +- **`wlr`:** builds a keymap containing exactly the characters needed, so any Unicode text works. +- **`libei`:** uses the server's keyboard layout. If the server supports libei's text capability, it sends the text directly. +- **`uinput`:** uses the layout named in `XKB_DEFAULT_LAYOUT`/`XKB_DEFAULT_VARIANT` (default `us`). Characters that aren't on that layout fail with `Unsupported` before anything is typed. +- **`sendinput`:** sends Unicode directly, so any text works. + +### Other + +| Call | Description | +|------|-------------| +| `InputBackend backend() const` | Which backend was chosen. | +| `std::string restoreToken() const` | libei: save this and pass it in `InputOptions::restoreToken` next run to skip the dialog. Empty for other backends. | +| `void setEventDelay(std::chrono::milliseconds)` | Pause between the down and up halves of clicks, key presses and typed characters. The default is 8 ms. Raise it for applications that miss fast input. | +| `std::string_view toString(InputBackend)` | `"wlr"`, `"libei"`, `"uinput"`, `"sendinput"`, `"auto"` | + +--- + +## Keys and chords + +`#include ` + +`Key` names **physical key positions** on a US QWERTY layout, the same way evdev +and Win32 scan codes do. Use `typeText()` when you want particular characters. + +``` +A–Z, Num0–Num9, F1–F24, +Escape, Enter, Tab, Backspace, Space, +Minus, Equal, LeftBracket, RightBracket, Backslash, Semicolon, Apostrophe, Grave, Comma, Period, Slash, +CapsLock, NumLock, ScrollLock, +LeftShift, RightShift, LeftCtrl, RightCtrl, LeftAlt, RightAlt, LeftSuper, RightSuper, Menu, +Insert, Delete, Home, End, PageUp, PageDown, Left, Right, Up, Down, PrintScreen, Pause, +Kp0–Kp9, KpDecimal, KpDivide, KpMultiply, KpSubtract, KpAdd, KpEnter, +VolumeMute, VolumeDown, VolumeUp, MediaPlayPause, MediaStop, MediaNext, MediaPrevious +``` + +```cpp +std::optional keyFromName(std::string_view name); // case-insensitive +std::string_view keyName(Key key); +std::optional> parseChord(std::string_view chord); +bool isModifier(Key key); +``` + +Names accepted by `keyFromName` and in chord strings: + +| Key | Names | +|-----|-------| +| Letters and digits | `a`…`z`, `0`…`9` | +| Function keys | `f1`…`f24` | +| Modifiers | `shift`/`lshift`, `rshift`, `ctrl`/`control`/`lctrl`, `rctrl`, `alt`/`lalt`, `ralt`/`altgr`, `super`/`win`/`meta`/`cmd`/`lsuper`, `rsuper` | +| Editing | `enter`/`return`, `esc`/`escape`, `tab`, `backspace`, `space`, `insert`, `delete`/`del` | +| Navigation | `home`, `end`, `pageup`/`pgup`, `pagedown`/`pgdn`, `left`, `right`, `up`, `down` | +| Punctuation | `minus`/`-`, `equal`/`=`, `[`, `]`, `\`, `;`, `'`, `` ` ``, `,`, `.`, `/` (or their word names) | +| Keypad | `kp0`…`kp9`, `kpdecimal`, `kpdivide`, `kpmultiply`, `kpsubtract`, `kpadd`, `kpenter` | +| Media | `mute`, `volumeup`, `volumedown`, `playpause`, `stop`, `next`, `prev`/`previous` | +| Other | `capslock`, `numlock`, `scrolllock`, `printscreen`/`print`, `pause`, `menu` | + +Chords are joined with `+`, and spaces around keys are ignored. `"ctrl++"` means +Ctrl plus the `+`/`=` key. + +--- + +## Capture + +`#include ` + +### Creating + +```cpp +static Result> Capture::create(const CaptureOptions& options = {}); + +struct CaptureOptions { + CaptureBackend backend = CaptureBackend::Auto; // Auto, Portal, Gdi + CaptureSource source = CaptureSource::Monitor; // Monitor, Window, Any + int monitorIndex = 0; // Windows: which monitor (index into listMonitors()) + std::optional window; // Windows: capture this window directly + bool showCursor = false; + std::string restoreToken; // Portal: skip the picker (see below) + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; +``` + +**Linux (`Portal`):** `create()` opens the desktop's share picker. On Hyprland +that's `hyprland-share-picker`. `source` controls whether the picker offers +monitors, windows or both. When the call returns, a PipeWire stream is running +in the background. + +**Windows (`Gdi`):** there's no dialog. Set `monitorIndex`, or set `window` to +capture one window. + +### Skipping the picker next time + +```cpp +alvin::CaptureOptions opts; +opts.restoreToken = loadSavedToken(); // empty the first time +auto cap = alvin::Capture::create(opts).value(); +saveToken(cap->restoreToken()); // new token each session; always re-save it +``` + +The `alvin` CLI does this for you and stores tokens in `~/.cache/alvin/`. + +> Don't leave a picker unanswered. While it's open, xdg-desktop-portal-hyprland +> blocks every other portal request on the system, screen sharing in other +> apps included. Use `permissionTimeout` to limit how long `create()` waits. + +### Getting frames + +| Call | Description | +|------|-------------| +| `Result grab(timeout = 5s)` | The most recent frame. Blocks only until the first frame exists. | +| `Result nextFrame(timeout = 5s)` | Waits for a frame newer than the last one you received. | +| `std::string restoreToken() const` | Portal token for next time. | +| `CaptureBackend backend() const` | Which backend was chosen. | + +PipeWire streams only send a frame when the screen changes, so `grab()` can +return an image that's a few seconds old if nothing has moved. That's usually +what you want. After you act (click, type), call `nextFrame()` to see the result: + +```cpp +input->clickAt(button); +auto after = cap->nextFrame(std::chrono::seconds(2)); +if (!after && after.error().code == alvin::ErrorCode::Timeout) { + // nothing on screen changed +} +``` + +A `Capture` holds a live stream, so keep one and reuse it. Don't create one per frame. + +--- + +## Frame + +`#include ` + +A `Frame` is an owned BGRA8 image (4 bytes per pixel: B, G, R, A, with no row +padding). It also records where it sits on the desktop. + +| Member | Description | +|--------|-------------| +| `Frame(int w, int h)` | New zero-filled frame. | +| `int width()`, `int height()`, `Size size()` | Size in **pixels**. | +| `std::size_t stride()` | Bytes per row (`width * 4`). | +| `uint8_t* data()`, `row(y)`, `std::span bytes()` | Raw pixel access. | +| `Color pixel(int x, int y)` | `{r, g, b, a}`. Returns all zeros if out of bounds. | +| `Rect desktopRect()` | The area of the desktop this frame shows, in desktop coordinates. | +| `Point toDesktop(Point px)` | Frame pixel to desktop coordinate. | +| `Point fromDesktop(Point p)` | Desktop coordinate to frame pixel. | +| `Frame crop(Rect r)` | Sub-image (`r` is in frame pixels and gets clipped). `desktopRect()` is adjusted to match. | +| `int64_t timestampNs()` | Capture timestamp. | +| `Status savePpm(path)` | Save without dependencies (PPM). With vision, use `vision::saveImage` for PNG/JPG. | + +**Always convert through `toDesktop()` before clicking.** On a monitor scaled +to 1.33, a 3840×2160 frame covers 2880×1620 desktop units, so frame pixels +aren't desktop coordinates: + +```cpp +auto frame = cap->grab().value(); +alvin::Point pixel{1920, 1080}; // something found in the image +input->clickAt(frame.toDesktop(pixel)); // correct on any scale +``` + +```cpp +alvin::Color c = frame.pixel(10, 10); +if (c.r > 200 && c.g < 50 && c.b < 50) { /* red */ } +``` + +--- + +## Windows + +`#include ` + +Wayland hides other windows from normal clients, so each compositor needs its +own backend. Right now that's Hyprland (IPC socket) and Win32. + +```cpp +static Result> WindowManager::create(WindowBackend b = WindowBackend::Auto); + +struct WindowInfo { + WindowId id; // opaque handle; pass it to focus() + std::string title; + std::string appId; // Hyprland class / Win32 executable name + Rect bounds; // desktop coordinates + int64_t pid; + bool focused, visible, minimized, fullscreen; + std::string workspace; // Hyprland workspace name +}; +``` + +| Call | Description | +|------|-------------| +| `Result> list()` | All mapped top-level windows. | +| `Result> active()` | The focused window, or none. | +| `Result> find(std::string_view text)` | First window whose title or appId contains `text` (case-insensitive). | +| `Status focus(WindowId id)` | Focus it. On Hyprland this switches to its workspace. | + +```cpp +auto wm = alvin::WindowManager::create().value(); +if (auto w = wm->find("firefox").value()) { + wm->focus(w->id); + input->clickAt(w->bounds.center()); + // click 40px in from the top-left of the window + input->clickAt({w->bounds.x + 40, w->bounds.y + 40}); +} +``` + +--- + +## Vision (OpenCV) + +`#include `, link `alvin::vision`. + +| Call | Description | +|------|-------------| +| `cv::Mat view(const Frame&)` | Zero-copy `CV_8UC4` view. The Frame must outlive the Mat. | +| `cv::Mat toBgr(const Frame&)` | Deep copy as `CV_8UC3`, the layout most OpenCV code expects. | +| `Result fromMat(const cv::Mat&)` | 8-bit gray, BGR or BGRA to a Frame. | +| `Result loadImage(path)` | Load a PNG/JPG as BGR. | +| `Status saveImage(const Frame&, path)` | Save as PNG/JPG/… (by extension). | +| `Result> find(frame, needle, opts)` | Best template match at or above the threshold. | +| `Result> findAll(frame, needle, opts)` | Every non-overlapping match, best first. | + +```cpp +struct Match { Rect rect; double score; }; // rect in FRAME PIXELS, score 0..1 + +struct MatchOptions { + double threshold = 0.9; // lower it (0.7–0.85) if the target is anti-aliased or slightly different + bool grayscale = false; // faster; ignores colour + int maxResults = 16; // findAll only +}; +``` + +Take template images from Alvin's own captures, e.g. `alvin capture` followed by +a crop. That keeps the scale and colours identical, which template matching needs. + +```cpp +auto needle = alvin::vision::loadImage("save_button.png").value(); +auto frame = cap->grab().value(); +if (auto hit = alvin::vision::find(frame, needle, {.threshold = 0.85}).value()) + input->clickAt(frame.toDesktop(hit->rect.center())); + +for (const auto& m : alvin::vision::findAll(frame, needle, {.threshold = 0.8}).value()) + std::printf("match at %d,%d score %.2f\n", m.rect.x, m.rect.y, m.score); +``` + +For anything else OpenCV can do, use `vision::view(frame)` to get a `cv::Mat`. + +--- + +## Putting it together + +A small agent loop: wait for a button to appear, click it, then check that the +screen changed. + +```cpp +#include +#include +#include +#include +#include + +using namespace std::chrono_literals; + +int main() { + auto input = alvin::Input::create(); + if (!input) { std::cerr << input.error().message << '\n'; return 1; } + + auto cap = alvin::Capture::create(); // the picker appears on first run + if (!cap) { std::cerr << cap.error().message << '\n'; return 1; } + + auto button = alvin::vision::loadImage("accept.png"); + if (!button) { std::cerr << button.error().message << '\n'; return 1; } + + for (int attempt = 0; attempt < 30; ++attempt) { + auto frame = (*cap)->grab(); + if (!frame) { std::cerr << frame.error().message << '\n'; return 1; } + + auto hit = alvin::vision::find(*frame, *button, {.threshold = 0.85}); + if (hit && *hit) { + (void)(*input)->clickAt(frame->toDesktop((*hit)->rect.center())); + if ((*cap)->nextFrame(2s)) std::cout << "clicked; screen updated\n"; + return 0; + } + std::this_thread::sleep_for(1s); + } + std::cerr << "button never appeared\n"; + return 1; +} +``` + +See [`examples/alvin_cli.cpp`](../examples/alvin_cli.cpp) for a complete program +that uses every part of the API. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..3474bdf --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(alvin_cli alvin_cli.cpp) +set_target_properties(alvin_cli PROPERTIES OUTPUT_NAME alvin) +target_link_libraries(alvin_cli PRIVATE alvin::alvin) +if(TARGET alvin::vision) + target_link_libraries(alvin_cli PRIVATE alvin::vision) + target_compile_definitions(alvin_cli PRIVATE ALVIN_CLI_VISION) +endif() +install(TARGETS alvin_cli) diff --git a/examples/alvin_cli.cpp b/examples/alvin_cli.cpp new file mode 100644 index 0000000..a99c744 --- /dev/null +++ b/examples/alvin_cli.cpp @@ -0,0 +1,262 @@ +// `alvin` - command-line front end to the library, handy for trying backends +// and scripting simple automations. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alvin/alvin.hpp" +#ifdef ALVIN_CLI_VISION +#include "alvin/vision.hpp" +#endif + +namespace fs = std::filesystem; +using namespace alvin; + +namespace { + +void usage() { + std::cerr << + R"(usage: alvin [--input wlr|libei|uinput|sendinput] [args] + + monitors list monitors (logical desktop coordinates) + windows list top-level windows + active show the focused window + focus focus the first window whose title/class contains + + move move the pointer to desktop coordinates + moveby move the pointer relatively + click [left|right|middle] [n] click n times (default: left, 1) + clickat [button] move, then click + scroll [dx] scroll wheel notches (positive = down/right) + key press a chord, e.g. ctrl+shift+t + type type UTF-8 text + + capture [--window] [--cursor] + save one frame; the portal token is cached so + the picker only appears the first time +)" +#ifdef ALVIN_CLI_VISION + R"( find [threshold] + locate an image on screen, print desktop coords +)" +#endif + ; +} + +int report(const Error& e) { + std::cerr << "alvin: " << toString(e.code) << ": " << e.message << '\n'; + return 1; +} + +int report(const Status& s) { return s ? 0 : report(s.error()); } + +std::optional toInt(const std::string& s) { + try { + std::size_t used = 0; + int v = std::stoi(s, &used); + if (used == s.size()) return v; + } catch (...) { + } + return std::nullopt; +} + +std::optional toButton(const std::string& s) { + if (s == "left") return MouseButton::Left; + if (s == "right") return MouseButton::Right; + if (s == "middle") return MouseButton::Middle; + if (s == "back") return MouseButton::Back; + if (s == "forward") return MouseButton::Forward; + return std::nullopt; +} + +fs::path tokenFile(const char* name) { + const char* cache = std::getenv("XDG_CACHE_HOME"); + const char* home = std::getenv("HOME"); + fs::path dir = cache && *cache ? fs::path(cache) : fs::path(home ? home : ".") / ".cache"; + return dir / "alvin" / name; +} + +std::string readToken(const fs::path& p) { + std::ifstream f(p); + std::string t; + std::getline(f, t); + return t; +} + +void writeToken(const fs::path& p, const std::string& token) { + if (token.empty()) return; + std::error_code ec; + fs::create_directories(p.parent_path(), ec); + std::ofstream(p) << token << '\n'; +} + +void printWindow(const WindowInfo& w) { + std::printf("%#llx %5d,%-5d %5dx%-5d %s%s[%s] %s\n", static_cast(w.id.value), w.bounds.x, + w.bounds.y, w.bounds.width, w.bounds.height, w.focused ? "* " : " ", w.minimized ? "(min) " : "", + w.appId.c_str(), w.title.c_str()); +} + +Result> makeInput(const InputOptions& base) { + InputOptions opts = base; + fs::path tf = tokenFile("remotedesktop.token"); + opts.restoreToken = readToken(tf); + auto in = Input::create(opts); + if (in) { + writeToken(tf, (*in)->restoreToken()); + if (std::getenv("ALVIN_VERBOSE")) std::cerr << "input backend: " << toString((*in)->backend()) << '\n'; + } + return in; +} + +Result captureFrame(bool window, bool cursor) { + CaptureOptions opts; + opts.source = window ? CaptureSource::Window : CaptureSource::Monitor; + opts.showCursor = cursor; + fs::path tf = tokenFile(window ? "screencast-window.token" : "screencast-monitor.token"); + opts.restoreToken = readToken(tf); + auto cap = Capture::create(opts); + if (!cap) return std::unexpected(cap.error()); + writeToken(tf, (*cap)->restoreToken()); + return (*cap)->grab(); +} + +int run(std::span args, const InputOptions& inputOpts) { + const std::string& cmd = args[0]; + auto arg = [&](std::size_t i) -> const std::string& { + static const std::string empty; + return i < args.size() ? args[i] : empty; + }; + + if (cmd == "monitors") { + auto mons = listMonitors(); + if (!mons) return report(mons.error()); + for (const auto& m : *mons) + std::printf("%-10s %5d,%-5d %5dx%-5d pixels %dx%d scale %.3g%s %s\n", m.name.c_str(), m.bounds.x, + m.bounds.y, m.bounds.width, m.bounds.height, m.pixelSize.width, m.pixelSize.height, m.scale, + m.primary ? " (primary)" : "", m.description.c_str()); + return 0; + } + + if (cmd == "windows" || cmd == "active" || cmd == "focus") { + auto wm = WindowManager::create(); + if (!wm) return report(wm.error()); + if (cmd == "windows") { + auto list = (*wm)->list(); + if (!list) return report(list.error()); + for (const auto& w : *list) printWindow(w); + return 0; + } + if (cmd == "active") { + auto a = (*wm)->active(); + if (!a) return report(a.error()); + if (*a) printWindow(**a); + return 0; + } + auto w = (*wm)->find(arg(1)); + if (!w) return report(w.error()); + if (!*w) return report(Error{ErrorCode::NotFound, "no window matches '" + arg(1) + "'"}); + return report((*wm)->focus((*w)->id)); + } + + if (cmd == "capture") { + if (arg(1).empty()) return usage(), 2; + bool window = false, cursor = false; + for (std::size_t i = 2; i < args.size(); ++i) { + if (args[i] == "--window") window = true; + else if (args[i] == "--cursor") cursor = true; + } + auto frame = captureFrame(window, cursor); + if (!frame) return report(frame.error()); + fs::path out = arg(1); + Status saved; +#ifdef ALVIN_CLI_VISION + if (out.extension() != ".ppm") saved = vision::saveImage(*frame, out); + else +#endif + saved = frame->savePpm(out); + if (!saved) return report(saved); + Rect r = frame->desktopRect(); + std::printf("%s: %dx%d pixels, desktop rect %d,%d %dx%d\n", out.c_str(), frame->width(), frame->height(), r.x, + r.y, r.width, r.height); + return 0; + } + +#ifdef ALVIN_CLI_VISION + if (cmd == "find") { + auto tpl = vision::loadImage(arg(1)); + if (!tpl) return report(tpl.error()); + vision::MatchOptions mo; + if (!arg(2).empty()) mo.threshold = std::stod(arg(2)); + auto frame = captureFrame(false, false); + if (!frame) return report(frame.error()); + auto match = vision::find(*frame, *tpl, mo); + if (!match) return report(match.error()); + if (!*match) { + std::printf("not found\n"); + return 1; + } + Point c = frame->toDesktop((*match)->rect.center()); + std::printf("%d %d (score %.3f)\n", c.x, c.y, (*match)->score); + return 0; + } +#endif + + // Everything below needs an input backend. + auto input = makeInput(inputOpts); + if (!input) return report(input.error()); + Input& in = **input; + + if (cmd == "move" || cmd == "moveby" || cmd == "clickat") { + auto x = toInt(arg(1)), y = toInt(arg(2)); + if (!x || !y) return usage(), 2; + if (cmd == "moveby") return report(in.moveBy(*x, *y)); + if (cmd == "move") return report(in.moveTo({*x, *y})); + auto b = arg(3).empty() ? std::optional{MouseButton::Left} : toButton(arg(3)); + if (!b) return usage(), 2; + return report(in.clickAt({*x, *y}, *b)); + } + if (cmd == "click") { + auto b = arg(1).empty() ? std::optional{MouseButton::Left} : toButton(arg(1)); + auto n = arg(2).empty() ? std::optional{1} : toInt(arg(2)); + if (!b || !n) return usage(), 2; + return report(in.click(*b, *n)); + } + if (cmd == "scroll") { + auto dy = toInt(arg(1)); + auto dx = arg(2).empty() ? std::optional{0} : toInt(arg(2)); + if (!dy || !dx) return usage(), 2; + return report(in.scroll(*dx, *dy)); + } + if (cmd == "key") return report(in.chord(std::string_view(arg(1)))); + if (cmd == "type") return report(in.typeText(arg(1))); + + usage(); + return 2; +} + +} // namespace + +int main(int argc, char** argv) { + std::vector args(argv + 1, argv + argc); + InputOptions inputOpts; + if (args.size() >= 2 && args[0] == "--input") { + const std::string& b = args[1]; + if (b == "wlr") inputOpts.backend = InputBackend::WlrVirtual; + else if (b == "libei") inputOpts.backend = InputBackend::LibEi; + else if (b == "uinput") inputOpts.backend = InputBackend::UInput; + else if (b == "sendinput") inputOpts.backend = InputBackend::SendInput; + else return usage(), 2; + args.erase(args.begin(), args.begin() + 2); + } + if (args.empty() || args[0] == "-h" || args[0] == "--help") { + usage(); + return args.empty() ? 2 : 0; + } + return run(args, inputOpts); +} diff --git a/include/alvin/alvin.hpp b/include/alvin/alvin.hpp new file mode 100644 index 0000000..04701d9 --- /dev/null +++ b/include/alvin/alvin.hpp @@ -0,0 +1,14 @@ +#pragma once + +// Alvin - Agent Leveraging Vision and Input. Nice. +// +// Umbrella header for the core library. Computer-vision helpers live in the +// optional alvin::vision target (#include "alvin/vision.hpp"). + +#include "alvin/capture.hpp" +#include "alvin/error.hpp" +#include "alvin/frame.hpp" +#include "alvin/input.hpp" +#include "alvin/keys.hpp" +#include "alvin/types.hpp" +#include "alvin/window.hpp" diff --git a/include/alvin/capture.hpp b/include/alvin/capture.hpp new file mode 100644 index 0000000..8ca028b --- /dev/null +++ b/include/alvin/capture.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "alvin/frame.hpp" +#include "alvin/window.hpp" + +namespace alvin { + +enum class CaptureBackend { + Auto, + Portal, // xdg-desktop-portal ScreenCast + PipeWire (any modern Wayland desktop) + Gdi, // Win32 BitBlt / PrintWindow +}; + +std::string_view toString(CaptureBackend backend) noexcept; + +enum class CaptureSource : std::uint8_t { + Monitor = 1, + Window = 2, + Any = Monitor | Window, +}; + +struct CaptureOptions { + CaptureBackend backend = CaptureBackend::Auto; + + // What the portal picker offers. On Windows, Monitor + monitorIndex or + // Window + window select the target directly with no dialog. + CaptureSource source = CaptureSource::Monitor; + int monitorIndex = 0; + std::optional window; + + bool showCursor = false; + + // Portal only: token from Capture::restoreToken() of an earlier session. + // Lets the compositor skip the picker and reuse the previous selection. + std::string restoreToken; + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; + +// A live capture of one monitor or window. +class Capture { +public: + static Result> create(const CaptureOptions& options = {}); + + virtual ~Capture() = default; + Capture(const Capture&) = delete; + Capture& operator=(const Capture&) = delete; + + virtual CaptureBackend backend() const = 0; + + // Most recent frame. Blocks until the first frame arrives (up to timeout). + // Streaming backends only deliver frames when the screen changes, so this + // does not wait for a *new* frame; use nextFrame() for that. + virtual Result grab(std::chrono::milliseconds timeout = std::chrono::seconds(5)) = 0; + + // Waits for a frame newer than any previously returned one. + virtual Result nextFrame(std::chrono::milliseconds timeout = std::chrono::seconds(5)) = 0; + + virtual std::string restoreToken() const { return {}; } + +protected: + Capture() = default; +}; + +} // namespace alvin diff --git a/include/alvin/error.hpp b/include/alvin/error.hpp new file mode 100644 index 0000000..fbd7d86 --- /dev/null +++ b/include/alvin/error.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace alvin { + +enum class ErrorCode { + Unknown, + Unsupported, // backend or feature not available on this platform/session + NotAvailable, // backend compiled in but its service/device isn't present + PermissionDenied, // e.g. /dev/uinput not writable, portal request denied + Cancelled, // user dismissed a portal dialog + Timeout, + InvalidArgument, + NotFound, + Io, + Protocol, // unexpected reply from a compositor / portal / IPC socket +}; + +struct Error { + ErrorCode code = ErrorCode::Unknown; + std::string message; +}; + +template +using Result = std::expected; + +using Status = Result; + +inline std::unexpected fail(ErrorCode code, std::string message) { + return std::unexpected(Error{code, std::move(message)}); +} + +std::string_view toString(ErrorCode code) noexcept; + +} // namespace alvin diff --git a/include/alvin/frame.hpp b/include/alvin/frame.hpp new file mode 100644 index 0000000..f141590 --- /dev/null +++ b/include/alvin/frame.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "alvin/types.hpp" + +namespace alvin { + +// Byte order in memory. Every capture backend normalises to BGRA8, which is +// what OpenCV (CV_8UC4) and Win32 DIBs use natively. +enum class PixelFormat : std::uint8_t { + BGRA8, +}; + +struct Color { + std::uint8_t r = 0, g = 0, b = 0, a = 255; + friend bool operator==(const Color&, const Color&) = default; +}; + +class Frame { +public: + Frame() = default; + Frame(int width, int height); // zero-filled BGRA8 + + int width() const { return width_; } + int height() const { return height_; } + Size size() const { return {width_, height_}; } + std::size_t stride() const { return static_cast(width_) * 4; } + PixelFormat format() const { return PixelFormat::BGRA8; } + bool empty() const { return data_.empty(); } + + std::uint8_t* data() { return data_.data(); } + const std::uint8_t* data() const { return data_.data(); } + std::span bytes() const { return data_; } + std::uint8_t* row(int y) { return data_.data() + y * stride(); } + const std::uint8_t* row(int y) const { return data_.data() + y * stride(); } + + Color pixel(int x, int y) const; + + // Where this frame sits on the desktop, in logical pixels. Frames from a + // HiDPI monitor have more pixels than their logical size; use + // toDesktop()/fromDesktop() to convert instead of assuming 1:1. + Rect desktopRect() const { return desktopRect_; } + void setDesktopRect(Rect r) { desktopRect_ = r; } + Point toDesktop(Point framePixel) const; + Point fromDesktop(Point desktopPoint) const; + + std::int64_t timestampNs() const { return timestampNs_; } + void setTimestampNs(std::int64_t t) { timestampNs_ = t; } + + Frame crop(Rect r) const; // r in frame pixels; clipped to bounds + + // Binary PPM (P6), dependency-free. Alpha is dropped. + Status savePpm(const std::filesystem::path& path) const; + +private: + int width_ = 0; + int height_ = 0; + std::vector data_; + Rect desktopRect_; + std::int64_t timestampNs_ = 0; +}; + +} // namespace alvin diff --git a/include/alvin/input.hpp b/include/alvin/input.hpp new file mode 100644 index 0000000..98fac61 --- /dev/null +++ b/include/alvin/input.hpp @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "alvin/keys.hpp" +#include "alvin/types.hpp" + +namespace alvin { + +enum class InputBackend { + Auto, + WlrVirtual, // Wayland zwlr_virtual_pointer_v1 + zwp_virtual_keyboard_v1 (Hyprland, sway, other wlroots) + LibEi, // libei via xdg-desktop-portal RemoteDesktop (GNOME, KDE) + UInput, // /dev/uinput kernel virtual devices (any Linux session; needs write access) + SendInput, // Win32 +}; + +std::string_view toString(InputBackend backend) noexcept; + +struct InputOptions { + InputBackend backend = InputBackend::Auto; + + // LibEi only: token from a previous session (Input::restoreToken()) that + // lets the portal skip its permission dialog. + std::string restoreToken; + + // How long to wait for the user to answer a portal permission dialog. + std::chrono::milliseconds permissionTimeout = std::chrono::minutes(2); +}; + +// Synthesises mouse and keyboard events. +// +// Coordinates passed to moveTo() are global desktop coordinates in logical +// pixels, matching Monitor::bounds and WindowInfo::bounds. +class Input { +public: + // Picks a backend (or uses options.backend). With Auto, the environment + // variable ALVIN_INPUT_BACKEND=wlr|libei|uinput|sendinput overrides the choice. + static Result> create(const InputOptions& options = {}); + + virtual ~Input() = default; + Input(const Input&) = delete; + Input& operator=(const Input&) = delete; + + virtual InputBackend backend() const = 0; + + virtual Status moveTo(Point p) = 0; + virtual Status moveBy(int dx, int dy) = 0; + virtual Status buttonDown(MouseButton button) = 0; + virtual Status buttonUp(MouseButton button) = 0; + // Wheel notches; positive dy scrolls down, positive dx scrolls right. + virtual Status scroll(int dx, int dy) = 0; + + virtual Status keyDown(Key key) = 0; + virtual Status keyUp(Key key) = 0; + // Types UTF-8 text using the active keyboard layout (or a synthesized one). + virtual Status typeText(std::string_view utf8) = 0; + + // Portal-backed sessions can hand out a token to skip the dialog next time. + virtual std::string restoreToken() const { return {}; } + + // ---- Convenience helpers built on the primitives above ---- + + Status click(MouseButton button = MouseButton::Left, int count = 1); + Status clickAt(Point p, MouseButton button = MouseButton::Left, int count = 1); + Status drag(Point from, Point to, MouseButton button = MouseButton::Left); + Status keyPress(Key key); + // Presses keys in order, releases in reverse: chord({Key::LeftCtrl, Key::C}). + Status chord(std::initializer_list keys); + // Same, from a string such as "ctrl+shift+esc". + Status chord(std::string_view chord); + + // Delay inserted between the down/up halves of clicks and key presses. + // Some applications drop events that arrive with zero spacing. + void setEventDelay(std::chrono::milliseconds delay) { eventDelay_ = delay; } + std::chrono::milliseconds eventDelay() const { return eventDelay_; } + +protected: + Input() = default; + void pause() const; + +private: + std::chrono::milliseconds eventDelay_{8}; +}; + +} // namespace alvin diff --git a/include/alvin/keys.hpp b/include/alvin/keys.hpp new file mode 100644 index 0000000..05d9b5a --- /dev/null +++ b/include/alvin/keys.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +namespace alvin { + +// Platform-neutral physical key identifiers. Names follow a US QWERTY layout +// but they identify key *positions*, the same way evdev codes and Win32 +// scan codes do. Use Input::typeText() to produce characters instead. +enum class Key : std::uint16_t { + Unknown = 0, + + A, B, C, D, E, F, G, H, I, J, K, L, M, + N, O, P, Q, R, S, T, U, V, W, X, Y, Z, + + Num0, Num1, Num2, Num3, Num4, Num5, Num6, Num7, Num8, Num9, + + F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, + F13, F14, F15, F16, F17, F18, F19, F20, F21, F22, F23, F24, + + Escape, Enter, Tab, Backspace, Space, + Minus, Equal, LeftBracket, RightBracket, Backslash, + Semicolon, Apostrophe, Grave, Comma, Period, Slash, + + CapsLock, NumLock, ScrollLock, + LeftShift, RightShift, LeftCtrl, RightCtrl, LeftAlt, RightAlt, LeftSuper, RightSuper, + Menu, + + Insert, Delete, Home, End, PageUp, PageDown, + Left, Right, Up, Down, + PrintScreen, Pause, + + Kp0, Kp1, Kp2, Kp3, Kp4, Kp5, Kp6, Kp7, Kp8, Kp9, + KpDecimal, KpDivide, KpMultiply, KpSubtract, KpAdd, KpEnter, + + VolumeMute, VolumeDown, VolumeUp, + MediaPlayPause, MediaStop, MediaNext, MediaPrevious, + + Count_, +}; + +enum class MouseButton : std::uint8_t { + Left, + Right, + Middle, + Back, + Forward, +}; + +// "ctrl", "a", "f5", "enter", "super", ... (case-insensitive). +std::optional keyFromName(std::string_view name); +std::string_view keyName(Key key); + +// Parses a chord such as "ctrl+shift+t" into its keys, modifiers first as written. +std::optional> parseChord(std::string_view chord); + +bool isModifier(Key key); + +} // namespace alvin diff --git a/include/alvin/types.hpp b/include/alvin/types.hpp new file mode 100644 index 0000000..6b3ad15 --- /dev/null +++ b/include/alvin/types.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include + +#include "alvin/error.hpp" + +namespace alvin { + +struct Point { + int x = 0; + int y = 0; + friend bool operator==(const Point&, const Point&) = default; +}; + +struct Size { + int width = 0; + int height = 0; + friend bool operator==(const Size&, const Size&) = default; +}; + +// All rectangles are in global desktop coordinates (logical pixels, i.e. after +// monitor scaling) unless documented otherwise. +struct Rect { + int x = 0; + int y = 0; + int width = 0; + int height = 0; + + int right() const { return x + width; } + int bottom() const { return y + height; } + Point center() const { return {x + width / 2, y + height / 2}; } + bool empty() const { return width <= 0 || height <= 0; } + bool contains(Point p) const { return p.x >= x && p.y >= y && p.x < right() && p.y < bottom(); } + + // Smallest rect containing both. An empty rect acts as identity. + Rect united(const Rect& o) const; + + friend bool operator==(const Rect&, const Rect&) = default; +}; + +struct Monitor { + std::string name; // connector name, e.g. "DP-1" or "\\\\.\\DISPLAY1" + std::string description; + Rect bounds; // logical position/size in the global desktop + Size pixelSize; // physical mode size in pixels + double scale = 1.0; + bool primary = false; +}; + +// Enumerate monitors on the current session. +Result> listMonitors(); + +// Bounding box of all monitors, i.e. the space absolute pointer motion maps onto. +Result desktopBounds(); + +} // namespace alvin diff --git a/include/alvin/vision.hpp b/include/alvin/vision.hpp new file mode 100644 index 0000000..dc49de1 --- /dev/null +++ b/include/alvin/vision.hpp @@ -0,0 +1,40 @@ +#pragma once + +// Optional OpenCV integration (link alvin::vision). + +#include +#include +#include + +#include + +#include "alvin/error.hpp" +#include "alvin/frame.hpp" + +namespace alvin::vision { + +// Zero-copy CV_8UC4 (BGRA) view. The Frame must outlive the Mat. +cv::Mat view(const Frame& frame); +// Deep copy converted to CV_8UC3 (BGR), the layout most OpenCV code expects. +cv::Mat toBgr(const Frame& frame); + +Result fromMat(const cv::Mat& mat); +Result loadImage(const std::filesystem::path& path); // BGR +Status saveImage(const Frame& frame, const std::filesystem::path& path); // png/jpg/... + +struct Match { + Rect rect; // in frame pixels + double score = 0; // 0..1, higher is better +}; + +struct MatchOptions { + double threshold = 0.9; + bool grayscale = false; // faster, ignores colour differences + int maxResults = 16; // findAll only +}; + +// Normalised cross-correlation template matching. +Result> find(const Frame& haystack, const cv::Mat& needle, const MatchOptions& opts = {}); +Result> findAll(const Frame& haystack, const cv::Mat& needle, const MatchOptions& opts = {}); + +} // namespace alvin::vision diff --git a/include/alvin/window.hpp b/include/alvin/window.hpp new file mode 100644 index 0000000..f4bc4a4 --- /dev/null +++ b/include/alvin/window.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "alvin/types.hpp" + +namespace alvin { + +// Opaque, backend-defined handle: a Hyprland client address, an HWND, ... +struct WindowId { + std::uint64_t value = 0; + friend bool operator==(const WindowId&, const WindowId&) = default; +}; + +struct WindowInfo { + WindowId id; + std::string title; + std::string appId; // Wayland app_id / X11 class / Win32 executable name + Rect bounds; // global desktop coordinates + std::int64_t pid = 0; + bool focused = false; + bool visible = true; + bool minimized = false; + bool fullscreen = false; + std::string workspace; // compositor-specific; empty when not applicable +}; + +enum class WindowBackend { + Auto, + Hyprland, // Hyprland IPC socket + Win32, +}; + +std::string_view toString(WindowBackend backend) noexcept; + +// Querying and focusing top-level windows. Wayland deliberately hides this +// from normal clients, so each compositor needs its own implementation. +class WindowManager { +public: + static Result> create(WindowBackend backend = WindowBackend::Auto); + + virtual ~WindowManager() = default; + WindowManager(const WindowManager&) = delete; + WindowManager& operator=(const WindowManager&) = delete; + + virtual WindowBackend backend() const = 0; + virtual Result> list() = 0; + virtual Result> active() = 0; + virtual Status focus(WindowId id) = 0; + + // First window whose title or appId contains `needle` (case-insensitive). + Result> find(std::string_view needle); + +protected: + WindowManager() = default; +}; + +} // namespace alvin diff --git a/protocols/virtual-keyboard-unstable-v1.xml b/protocols/virtual-keyboard-unstable-v1.xml new file mode 100644 index 0000000..5095c91 --- /dev/null +++ b/protocols/virtual-keyboard-unstable-v1.xml @@ -0,0 +1,113 @@ + + + + Copyright © 2008-2011 Kristian Høgsberg + Copyright © 2010-2013 Intel Corporation + Copyright © 2012-2013 Collabora, Ltd. + Copyright © 2018 Purism SPC + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice (including the next + paragraph) shall be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + + The virtual keyboard provides an application with requests which emulate + the behaviour of a physical keyboard. + + This interface can be used by clients on its own to provide raw input + events, or it can accompany the input method protocol. + + + + + Provide a file descriptor to the compositor which can be + memory-mapped to provide a keyboard mapping description. + + Format carries a value from the keymap_format enumeration. + + + + + + + + + + + + + A key was pressed or released. + The time argument is a timestamp with millisecond granularity, with an + undefined base. All requests regarding a single object must share the + same clock. + + Keymap must be set before issuing this request. + + State carries a value from the key_state enumeration. + + + + + + + + + Notifies the compositor that the modifier and/or group state has + changed, and it should update state. + + The client should use wl_keyboard.modifiers event to synchronize its + internal state with seat state. + + Keymap must be set before issuing this request. + + + + + + + + + + + + + + + A virtual keyboard manager allows an application to provide keyboard + input events as if they came from a physical keyboard. + + + + + + + + + Creates a new virtual keyboard associated to a seat. + + If the compositor enables a keyboard to perform arbitrary actions, it + should present an error when an untrusted client requests a new + keyboard. + + + + + + diff --git a/protocols/wlr-virtual-pointer-unstable-v1.xml b/protocols/wlr-virtual-pointer-unstable-v1.xml new file mode 100644 index 0000000..ea243e7 --- /dev/null +++ b/protocols/wlr-virtual-pointer-unstable-v1.xml @@ -0,0 +1,152 @@ + + + + Copyright © 2019 Josef Gajdusek + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice (including the next + paragraph) shall be included in all copies or substantial portions of the + Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + + + + + This protocol allows clients to emulate a physical pointer device. The + requests are mostly mirror opposites of those specified in wl_pointer. + + + + + + + + + + The pointer has moved by a relative amount to the previous request. + + Values are in the global compositor space. + + + + + + + + + The pointer has moved in an absolute coordinate frame. + + Value of x can range from 0 to x_extent, value of y can range from 0 + to y_extent. + + + + + + + + + + + A button was pressed or released. + + + + + + + + + Scroll and other axis requests. + + + + + + + + + Indicates the set of events that logically belong together. + + + + + + Source information for scroll and other axis. + + + + + + + Stop notification for scroll and other axes. + + + + + + + + Discrete step information for scroll and other axes. + + This event allows the client to extend data normally sent using the axis + event with discrete value. + + + + + + + + + + + + + + + This object allows clients to create individual virtual pointer objects. + + + + + Creates a new virtual pointer. The optional seat is a suggestion to the + compositor. + + + + + + + + + + + + + Creates a new virtual pointer. The seat and the output arguments are + optional. If the seat argument is set, the compositor should assign the + input device to the requested seat. If the output argument is set, the + compositor should map the input device to the requested output. + + + + + + + diff --git a/src/common/core.cpp b/src/common/core.cpp new file mode 100644 index 0000000..70f528c --- /dev/null +++ b/src/common/core.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "alvin/capture.hpp" +#include "alvin/error.hpp" +#include "alvin/input.hpp" +#include "alvin/types.hpp" +#include "alvin/window.hpp" +#include "platform.hpp" + +namespace alvin { + +std::string_view toString(ErrorCode code) noexcept { + switch (code) { + case ErrorCode::Unknown: return "unknown"; + case ErrorCode::Unsupported: return "unsupported"; + case ErrorCode::NotAvailable: return "not available"; + case ErrorCode::PermissionDenied: return "permission denied"; + case ErrorCode::Cancelled: return "cancelled"; + case ErrorCode::Timeout: return "timeout"; + case ErrorCode::InvalidArgument: return "invalid argument"; + case ErrorCode::NotFound: return "not found"; + case ErrorCode::Io: return "i/o error"; + case ErrorCode::Protocol: return "protocol error"; + } + return "unknown"; +} + +std::string_view toString(InputBackend backend) noexcept { + switch (backend) { + case InputBackend::Auto: return "auto"; + case InputBackend::WlrVirtual: return "wlr"; + case InputBackend::LibEi: return "libei"; + case InputBackend::UInput: return "uinput"; + case InputBackend::SendInput: return "sendinput"; + } + return "unknown"; +} + +std::string_view toString(CaptureBackend backend) noexcept { + switch (backend) { + case CaptureBackend::Auto: return "auto"; + case CaptureBackend::Portal: return "portal"; + case CaptureBackend::Gdi: return "gdi"; + } + return "unknown"; +} + +std::string_view toString(WindowBackend backend) noexcept { + switch (backend) { + case WindowBackend::Auto: return "auto"; + case WindowBackend::Hyprland: return "hyprland"; + case WindowBackend::Win32: return "win32"; + } + return "unknown"; +} + +Rect Rect::united(const Rect& o) const { + if (empty()) return o; + if (o.empty()) return *this; + int l = std::min(x, o.x), t = std::min(y, o.y); + int r = std::max(right(), o.right()), b = std::max(bottom(), o.bottom()); + return {l, t, r - l, b - t}; +} + +Result> listMonitors() { return detail::listMonitors(); } + +Result desktopBounds() { + auto monitors = listMonitors(); + if (!monitors) return std::unexpected(monitors.error()); + Rect box; + for (const auto& m : *monitors) box = box.united(m.bounds); + if (box.empty()) return fail(ErrorCode::NotFound, "no monitors reported"); + return box; +} + +// ---- Input ---- + +Result> Input::create(const InputOptions& options) { return detail::createInput(options); } + +void Input::pause() const { + if (eventDelay_.count() > 0) std::this_thread::sleep_for(eventDelay_); +} + +Status Input::click(MouseButton button, int count) { + for (int i = 0; i < count; ++i) { + if (auto s = buttonDown(button); !s) return s; + pause(); + if (auto s = buttonUp(button); !s) return s; + if (i + 1 < count) pause(); + } + return {}; +} + +Status Input::clickAt(Point p, MouseButton button, int count) { + if (auto s = moveTo(p); !s) return s; + pause(); + return click(button, count); +} + +Status Input::drag(Point from, Point to, MouseButton button) { + if (auto s = moveTo(from); !s) return s; + pause(); + if (auto s = buttonDown(button); !s) return s; + // Intermediate steps so drag thresholds and hover handlers fire. + constexpr int steps = 10; + for (int i = 1; i <= steps; ++i) { + pause(); + Point p{from.x + (to.x - from.x) * i / steps, from.y + (to.y - from.y) * i / steps}; + if (auto s = moveTo(p); !s) { + (void)buttonUp(button); + return s; + } + } + pause(); + return buttonUp(button); +} + +Status Input::keyPress(Key key) { + if (auto s = keyDown(key); !s) return s; + pause(); + return keyUp(key); +} + +// Presses keys in order and releases them in reverse. Always releases what it +// pressed, even after a failure, so no key is left stuck down. +static Status pressChord(Input& input, std::span keys, void (*pause)(const Input&)) { + std::vector pressed; + Status result; + for (Key k : keys) { + result = input.keyDown(k); + if (!result) break; + pressed.push_back(k); + pause(input); + } + for (auto it = pressed.rbegin(); it != pressed.rend(); ++it) { + auto s = input.keyUp(*it); + if (result && !s) result = s; + } + return result; +} + +Status Input::chord(std::initializer_list keys) { + return pressChord(*this, std::span(keys.begin(), keys.size()), [](const Input& in) { in.pause(); }); +} + +Status Input::chord(std::string_view text) { + auto keys = parseChord(text); + if (!keys) return fail(ErrorCode::InvalidArgument, "cannot parse key chord '" + std::string(text) + "'"); + return pressChord(*this, *keys, [](const Input& in) { in.pause(); }); +} + +// ---- Capture / windows ---- + +Result> Capture::create(const CaptureOptions& options) { return detail::createCapture(options); } + +Result> WindowManager::create(WindowBackend backend) { + return detail::createWindowManager(backend); +} + +static bool containsNoCase(std::string_view haystack, std::string_view needle) { + auto it = std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end(), [](char a, char b) { + return std::tolower(static_cast(a)) == std::tolower(static_cast(b)); + }); + return it != haystack.end(); +} + +Result> WindowManager::find(std::string_view needle) { + auto windows = list(); + if (!windows) return std::unexpected(windows.error()); + for (auto& w : *windows) + if (containsNoCase(w.title, needle) || containsNoCase(w.appId, needle)) return std::optional{std::move(w)}; + return std::optional{}; +} + +} // namespace alvin diff --git a/src/common/frame.cpp b/src/common/frame.cpp new file mode 100644 index 0000000..8846092 --- /dev/null +++ b/src/common/frame.cpp @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "alvin/frame.hpp" + +namespace alvin { + +Frame::Frame(int width, int height) + : width_(std::max(width, 0)), + height_(std::max(height, 0)), + data_(static_cast(width_) * height_ * 4, 0), + desktopRect_{0, 0, width_, height_} {} + +Color Frame::pixel(int x, int y) const { + if (x < 0 || y < 0 || x >= width_ || y >= height_) return {}; + const std::uint8_t* p = row(y) + x * 4; + return {p[2], p[1], p[0], p[3]}; +} + +Point Frame::toDesktop(Point px) const { + if (width_ == 0 || height_ == 0) return px; + double sx = static_cast(desktopRect_.width) / width_; + double sy = static_cast(desktopRect_.height) / height_; + return {desktopRect_.x + static_cast(std::lround(px.x * sx)), + desktopRect_.y + static_cast(std::lround(px.y * sy))}; +} + +Point Frame::fromDesktop(Point p) const { + if (desktopRect_.width == 0 || desktopRect_.height == 0) return p; + double sx = static_cast(width_) / desktopRect_.width; + double sy = static_cast(height_) / desktopRect_.height; + return {static_cast(std::lround((p.x - desktopRect_.x) * sx)), + static_cast(std::lround((p.y - desktopRect_.y) * sy))}; +} + +Frame Frame::crop(Rect r) const { + int x0 = std::clamp(r.x, 0, width_), y0 = std::clamp(r.y, 0, height_); + int x1 = std::clamp(r.right(), 0, width_), y1 = std::clamp(r.bottom(), 0, height_); + Frame out(x1 - x0, y1 - y0); + for (int y = 0; y < out.height_; ++y) std::memcpy(out.row(y), row(y0 + y) + x0 * 4, out.stride()); + + Point tl = toDesktop({x0, y0}), br = toDesktop({x1, y1}); + out.desktopRect_ = {tl.x, tl.y, br.x - tl.x, br.y - tl.y}; + out.timestampNs_ = timestampNs_; + return out; +} + +Status Frame::savePpm(const std::filesystem::path& path) const { + std::ofstream f(path, std::ios::binary); + if (!f) return fail(ErrorCode::Io, "cannot open " + path.string()); + f << "P6\n" << width_ << ' ' << height_ << "\n255\n"; + std::vector line(static_cast(width_) * 3); + for (int y = 0; y < height_; ++y) { + const std::uint8_t* src = row(y); + for (int x = 0; x < width_; ++x) { + line[x * 3 + 0] = static_cast(src[x * 4 + 2]); + line[x * 3 + 1] = static_cast(src[x * 4 + 1]); + line[x * 3 + 2] = static_cast(src[x * 4 + 0]); + } + f.write(line.data(), static_cast(line.size())); + } + if (!f) return fail(ErrorCode::Io, "write failed for " + path.string()); + return {}; +} + +} // namespace alvin diff --git a/src/common/json.cpp b/src/common/json.cpp new file mode 100644 index 0000000..4fc0564 --- /dev/null +++ b/src/common/json.cpp @@ -0,0 +1,215 @@ +#include "json.hpp" + +#include +#include +#include + +namespace alvin::json { +namespace { + +const Value kNull{}; +const Array kEmptyArray{}; +const Object kEmptyObject{}; + +class Parser { +public: + explicit Parser(std::string_view s) : s_(s) {} + + std::optional document() { + auto v = value(0); + skipSpace(); + if (!v || pos_ != s_.size()) return std::nullopt; + return v; + } + +private: + static constexpr int kMaxDepth = 256; + + void skipSpace() { + while (pos_ < s_.size() && std::isspace(static_cast(s_[pos_]))) ++pos_; + } + + bool consume(char c) { + skipSpace(); + if (pos_ < s_.size() && s_[pos_] == c) { + ++pos_; + return true; + } + return false; + } + + bool literal(std::string_view word) { + if (s_.substr(pos_, word.size()) != word) return false; + pos_ += word.size(); + return true; + } + + std::optional value(int depth) { + if (depth > kMaxDepth) return std::nullopt; + skipSpace(); + if (pos_ >= s_.size()) return std::nullopt; + char c = s_[pos_]; + if (c == '{') return object(depth); + if (c == '[') return array(depth); + if (c == '"') { + auto str = string(); + if (!str) return std::nullopt; + return Value{std::move(*str)}; + } + if (literal("true")) return Value{true}; + if (literal("false")) return Value{false}; + if (literal("null")) return Value{nullptr}; + return number(); + } + + std::optional number() { + std::size_t start = pos_; + while (pos_ < s_.size() && (std::isdigit(static_cast(s_[pos_])) || s_[pos_] == '-' || + s_[pos_] == '+' || s_[pos_] == '.' || s_[pos_] == 'e' || s_[pos_] == 'E')) + ++pos_; + if (start == pos_) return std::nullopt; + std::string text(s_.substr(start, pos_ - start)); + char* end = nullptr; + double d = std::strtod(text.c_str(), &end); + if (end != text.c_str() + text.size()) return std::nullopt; + return Value{d}; + } + + static void appendUtf8(std::string& out, char32_t cp) { + if (cp < 0x80) { + out += static_cast(cp); + } else if (cp < 0x800) { + out += static_cast(0xC0 | (cp >> 6)); + out += static_cast(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + out += static_cast(0xE0 | (cp >> 12)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } else { + out += static_cast(0xF0 | (cp >> 18)); + out += static_cast(0x80 | ((cp >> 12) & 0x3F)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } + } + + std::optional hex4() { + if (pos_ + 4 > s_.size()) return std::nullopt; + unsigned v = 0; + auto r = std::from_chars(s_.data() + pos_, s_.data() + pos_ + 4, v, 16); + if (r.ptr != s_.data() + pos_ + 4) return std::nullopt; + pos_ += 4; + return v; + } + + std::optional string() { + if (!consume('"')) return std::nullopt; + std::string out; + while (pos_ < s_.size()) { + char c = s_[pos_++]; + if (c == '"') return out; + if (c != '\\') { + out += c; + continue; + } + if (pos_ >= s_.size()) return std::nullopt; + switch (char e = s_[pos_++]) { + case '"': case '\\': case '/': out += e; break; + case 'b': out += '\b'; break; + case 'f': out += '\f'; break; + case 'n': out += '\n'; break; + case 'r': out += '\r'; break; + case 't': out += '\t'; break; + case 'u': { + auto hi = hex4(); + if (!hi) return std::nullopt; + char32_t cp = *hi; + if (cp >= 0xD800 && cp < 0xDC00 && s_.substr(pos_, 2) == "\\u") { + pos_ += 2; + auto lo = hex4(); + if (!lo) return std::nullopt; + cp = 0x10000 + ((cp - 0xD800) << 10) + (*lo - 0xDC00); + } + appendUtf8(out, cp); + break; + } + default: return std::nullopt; + } + } + return std::nullopt; + } + + std::optional array(int depth) { + consume('['); + auto arr = std::make_shared(); + if (consume(']')) return Value{arr}; + do { + auto v = value(depth + 1); + if (!v) return std::nullopt; + arr->push_back(std::move(*v)); + } while (consume(',')); + if (!consume(']')) return std::nullopt; + return Value{arr}; + } + + std::optional object(int depth) { + consume('{'); + auto obj = std::make_shared(); + if (consume('}')) return Value{obj}; + do { + skipSpace(); + auto key = string(); + if (!key || !consume(':')) return std::nullopt; + auto v = value(depth + 1); + if (!v) return std::nullopt; + obj->insert_or_assign(std::move(*key), std::move(*v)); + } while (consume(',')); + if (!consume('}')) return std::nullopt; + return Value{obj}; + } + + std::string_view s_; + std::size_t pos_ = 0; +}; + +} // namespace + +bool Value::boolean(bool fallback) const { + auto* b = std::get_if(&v); + return b ? *b : fallback; +} + +double Value::number(double fallback) const { + auto* d = std::get_if(&v); + return d ? *d : fallback; +} + +std::string Value::string(std::string fallback) const { + auto* s = std::get_if(&v); + return s ? *s : fallback; +} + +const Array& Value::array() const { + auto* a = std::get_if>(&v); + return a ? **a : kEmptyArray; +} + +const Object& Value::object() const { + auto* o = std::get_if>(&v); + return o ? **o : kEmptyObject; +} + +const Value& Value::operator[](std::string_view key) const { + const auto& o = object(); + auto it = o.find(key); + return it == o.end() ? kNull : it->second; +} + +const Value& Value::operator[](std::size_t index) const { + const auto& a = array(); + return index < a.size() ? a[index] : kNull; +} + +std::optional parse(std::string_view text) { return Parser(text).document(); } + +} // namespace alvin::json diff --git a/src/common/json.hpp b/src/common/json.hpp new file mode 100644 index 0000000..c24baeb --- /dev/null +++ b/src/common/json.hpp @@ -0,0 +1,42 @@ +#pragma once + +// A deliberately small JSON reader, enough for compositor IPC replies. +// Not a general-purpose library: no serialisation, numbers are doubles. + +#include +#include +#include +#include +#include +#include +#include + +namespace alvin::json { + +struct Value; +using Array = std::vector; +using Object = std::map>; + +struct Value { + std::variant, std::shared_ptr> v; + + bool isNull() const { return std::holds_alternative(v); } + bool isObject() const { return std::holds_alternative>(v); } + bool isArray() const { return std::holds_alternative>(v); } + + // Lenient accessors: return the fallback on type mismatch. + bool boolean(bool fallback = false) const; + double number(double fallback = 0) const; + std::string string(std::string fallback = {}) const; + const Array& array() const; // empty array if not an array + const Object& object() const; // empty object if not an object + + // Member lookup; returns a null value if missing or not an object. + const Value& operator[](std::string_view key) const; + // Element lookup; returns a null value if out of range or not an array. + const Value& operator[](std::size_t index) const; +}; + +std::optional parse(std::string_view text); + +} // namespace alvin::json diff --git a/src/common/keys.cpp b/src/common/keys.cpp new file mode 100644 index 0000000..fb1b6a0 --- /dev/null +++ b/src/common/keys.cpp @@ -0,0 +1,125 @@ +#include +#include +#include +#include + +#include "alvin/keys.hpp" + +namespace alvin { +namespace { + +struct NamedKey { + std::string_view name; + Key key; +}; + +// First entry for each key is its canonical name. +constexpr NamedKey kNames[] = { + {"a", Key::A}, {"b", Key::B}, {"c", Key::C}, {"d", Key::D}, {"e", Key::E}, {"f", Key::F}, + {"g", Key::G}, {"h", Key::H}, {"i", Key::I}, {"j", Key::J}, {"k", Key::K}, {"l", Key::L}, + {"m", Key::M}, {"n", Key::N}, {"o", Key::O}, {"p", Key::P}, {"q", Key::Q}, {"r", Key::R}, + {"s", Key::S}, {"t", Key::T}, {"u", Key::U}, {"v", Key::V}, {"w", Key::W}, {"x", Key::X}, + {"y", Key::Y}, {"z", Key::Z}, + {"0", Key::Num0}, {"1", Key::Num1}, {"2", Key::Num2}, {"3", Key::Num3}, {"4", Key::Num4}, + {"5", Key::Num5}, {"6", Key::Num6}, {"7", Key::Num7}, {"8", Key::Num8}, {"9", Key::Num9}, + {"f1", Key::F1}, {"f2", Key::F2}, {"f3", Key::F3}, {"f4", Key::F4}, {"f5", Key::F5}, + {"f6", Key::F6}, {"f7", Key::F7}, {"f8", Key::F8}, {"f9", Key::F9}, {"f10", Key::F10}, + {"f11", Key::F11}, {"f12", Key::F12}, {"f13", Key::F13}, {"f14", Key::F14}, {"f15", Key::F15}, + {"f16", Key::F16}, {"f17", Key::F17}, {"f18", Key::F18}, {"f19", Key::F19}, {"f20", Key::F20}, + {"f21", Key::F21}, {"f22", Key::F22}, {"f23", Key::F23}, {"f24", Key::F24}, + {"escape", Key::Escape}, {"esc", Key::Escape}, + {"enter", Key::Enter}, {"return", Key::Enter}, + {"tab", Key::Tab}, + {"backspace", Key::Backspace}, + {"space", Key::Space}, + {"minus", Key::Minus}, {"-", Key::Minus}, + {"equal", Key::Equal}, {"=", Key::Equal}, + {"leftbracket", Key::LeftBracket}, {"[", Key::LeftBracket}, + {"rightbracket", Key::RightBracket}, {"]", Key::RightBracket}, + {"backslash", Key::Backslash}, {"\\", Key::Backslash}, + {"semicolon", Key::Semicolon}, {";", Key::Semicolon}, + {"apostrophe", Key::Apostrophe}, {"'", Key::Apostrophe}, + {"grave", Key::Grave}, {"`", Key::Grave}, + {"comma", Key::Comma}, {",", Key::Comma}, + {"period", Key::Period}, {".", Key::Period}, + {"slash", Key::Slash}, {"/", Key::Slash}, + {"capslock", Key::CapsLock}, {"numlock", Key::NumLock}, {"scrolllock", Key::ScrollLock}, + {"shift", Key::LeftShift}, {"lshift", Key::LeftShift}, {"rshift", Key::RightShift}, + {"ctrl", Key::LeftCtrl}, {"control", Key::LeftCtrl}, {"lctrl", Key::LeftCtrl}, {"rctrl", Key::RightCtrl}, + {"alt", Key::LeftAlt}, {"lalt", Key::LeftAlt}, {"ralt", Key::RightAlt}, {"altgr", Key::RightAlt}, + {"super", Key::LeftSuper}, {"win", Key::LeftSuper}, {"meta", Key::LeftSuper}, {"cmd", Key::LeftSuper}, + {"lsuper", Key::LeftSuper}, {"rsuper", Key::RightSuper}, + {"menu", Key::Menu}, + {"insert", Key::Insert}, {"delete", Key::Delete}, {"del", Key::Delete}, + {"home", Key::Home}, {"end", Key::End}, + {"pageup", Key::PageUp}, {"pgup", Key::PageUp}, {"pagedown", Key::PageDown}, {"pgdn", Key::PageDown}, + {"left", Key::Left}, {"right", Key::Right}, {"up", Key::Up}, {"down", Key::Down}, + {"printscreen", Key::PrintScreen}, {"print", Key::PrintScreen}, {"pause", Key::Pause}, + {"kp0", Key::Kp0}, {"kp1", Key::Kp1}, {"kp2", Key::Kp2}, {"kp3", Key::Kp3}, {"kp4", Key::Kp4}, + {"kp5", Key::Kp5}, {"kp6", Key::Kp6}, {"kp7", Key::Kp7}, {"kp8", Key::Kp8}, {"kp9", Key::Kp9}, + {"kpdecimal", Key::KpDecimal}, {"kpdivide", Key::KpDivide}, {"kpmultiply", Key::KpMultiply}, + {"kpsubtract", Key::KpSubtract}, {"kpadd", Key::KpAdd}, {"kpenter", Key::KpEnter}, + {"mute", Key::VolumeMute}, {"volumedown", Key::VolumeDown}, {"volumeup", Key::VolumeUp}, + {"playpause", Key::MediaPlayPause}, {"stop", Key::MediaStop}, + {"next", Key::MediaNext}, {"previous", Key::MediaPrevious}, {"prev", Key::MediaPrevious}, +}; + +std::string lower(std::string_view s) { + std::string out(s); + for (char& c : out) c = static_cast(std::tolower(static_cast(c))); + return out; +} + +} // namespace + +std::optional keyFromName(std::string_view name) { + auto n = lower(name); + for (const auto& e : kNames) + if (e.name == n) return e.key; + return std::nullopt; +} + +std::string_view keyName(Key key) { + for (const auto& e : kNames) + if (e.key == key) return e.name; + return "unknown"; +} + +std::optional> parseChord(std::string_view chord) { + // "ctrl++" means ctrl plus the '+' key, which is the Equal key position on US layouts. + bool plusKey = false; + if (chord == "+") return std::vector{Key::Equal}; + if (chord.ends_with("++")) { + plusKey = true; + chord.remove_suffix(2); + } + + std::vector keys; + while (true) { + auto end = chord.find('+'); + auto token = chord.substr(0, end); + while (!token.empty() && std::isspace(static_cast(token.front()))) token.remove_prefix(1); + while (!token.empty() && std::isspace(static_cast(token.back()))) token.remove_suffix(1); + auto key = keyFromName(token); + if (!key) return std::nullopt; + keys.push_back(*key); + if (end == std::string_view::npos) break; + chord.remove_prefix(end + 1); + } + if (plusKey) keys.push_back(Key::Equal); + return keys; +} + +bool isModifier(Key key) { + switch (key) { + case Key::LeftShift: case Key::RightShift: + case Key::LeftCtrl: case Key::RightCtrl: + case Key::LeftAlt: case Key::RightAlt: + case Key::LeftSuper: case Key::RightSuper: + return true; + default: + return false; + } +} + +} // namespace alvin diff --git a/src/common/platform.hpp b/src/common/platform.hpp new file mode 100644 index 0000000..7a8dbd2 --- /dev/null +++ b/src/common/platform.hpp @@ -0,0 +1,20 @@ +#pragma once + +// Entry points every platform directory (src/linux, src/windows, ...) implements. + +#include +#include + +#include "alvin/capture.hpp" +#include "alvin/input.hpp" +#include "alvin/types.hpp" +#include "alvin/window.hpp" + +namespace alvin::detail { + +Result> createInput(const InputOptions& options); +Result> createCapture(const CaptureOptions& options); +Result> createWindowManager(WindowBackend backend); +Result> listMonitors(); + +} // namespace alvin::detail diff --git a/src/linux/backends.hpp b/src/linux/backends.hpp new file mode 100644 index 0000000..a6052c8 --- /dev/null +++ b/src/linux/backends.hpp @@ -0,0 +1,22 @@ +#pragma once + +// Constructors for each Linux backend. Which ones exist depends on the +// ALVIN_WITH_* options chosen at configure time. + +#include + +#include "alvin/capture.hpp" +#include "alvin/input.hpp" +#include "alvin/window.hpp" + +namespace alvin::linux_impl { + +Result> createWlrInput(const InputOptions& options); +Result> createUInput(const InputOptions& options); +Result> createLibEiInput(const InputOptions& options); + +Result> createPortalCapture(const CaptureOptions& options); + +Result> createHyprlandWindows(); + +} // namespace alvin::linux_impl diff --git a/src/linux/hyprland_windows.cpp b/src/linux/hyprland_windows.cpp new file mode 100644 index 0000000..cbd7e48 --- /dev/null +++ b/src/linux/hyprland_windows.cpp @@ -0,0 +1,126 @@ +// Window queries through Hyprland's IPC socket +// ($XDG_RUNTIME_DIR/hypr/$HYPRLAND_INSTANCE_SIGNATURE/.socket.sock). + +#include +#include +#include + +#include +#include + +#include "../common/json.hpp" +#include "backends.hpp" +#include "util.hpp" + +namespace alvin::linux_impl { +namespace { + +Result socketPath() { + const char* sig = std::getenv("HYPRLAND_INSTANCE_SIGNATURE"); + if (!sig || !*sig) return fail(ErrorCode::NotAvailable, "not running under Hyprland (HYPRLAND_INSTANCE_SIGNATURE unset)"); + const char* runtime = std::getenv("XDG_RUNTIME_DIR"); + std::string path = std::string(runtime ? runtime : "/run/user/" + std::to_string(getuid())) + "/hypr/" + sig + + "/.socket.sock"; + if (path.size() >= sizeof(sockaddr_un::sun_path)) return fail(ErrorCode::InvalidArgument, "Hyprland socket path too long"); + return path; +} + +// One request per connection, as hyprctl does. +Result hyprRequest(const std::string& path, std::string_view request) { + UniqueFd fd(::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)); + if (!fd) return fail(ErrorCode::Io, errnoMessage("socket")); + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::snprintf(addr.sun_path, sizeof addr.sun_path, "%s", path.c_str()); + if (::connect(fd.get(), reinterpret_cast(&addr), sizeof addr) < 0) + return fail(ErrorCode::NotAvailable, errnoMessage("connect to Hyprland IPC")); + if (::send(fd.get(), request.data(), request.size(), MSG_NOSIGNAL) != static_cast(request.size())) + return fail(ErrorCode::Io, errnoMessage("write to Hyprland IPC")); + + std::string reply; + char buf[8192]; + while (true) { + ssize_t n = ::recv(fd.get(), buf, sizeof buf, 0); + if (n < 0) { + if (errno == EINTR) continue; + return fail(ErrorCode::Io, errnoMessage("read from Hyprland IPC")); + } + if (n == 0) break; + reply.append(buf, static_cast(n)); + } + return reply; +} + +WindowInfo toWindow(const json::Value& c, const std::string& activeAddress) { + WindowInfo w; + std::string address = c["address"].string(); + w.id.value = std::strtoull(address.c_str(), nullptr, 16); + w.title = c["title"].string(); + w.appId = c["class"].string(); + w.bounds = {static_cast(c["at"][0].number()), static_cast(c["at"][1].number()), + static_cast(c["size"][0].number()), static_cast(c["size"][1].number())}; + w.pid = static_cast(c["pid"].number()); + w.focused = !activeAddress.empty() && address == activeAddress; + w.minimized = c["hidden"].boolean(); + w.visible = c["visible"].boolean(true) && !w.minimized; + w.fullscreen = c["fullscreen"].number() != 0; + w.workspace = c["workspace"]["name"].string(); + return w; +} + +class HyprlandWindows final : public WindowManager { +public: + explicit HyprlandWindows(std::string path) : path_(std::move(path)) {} + + WindowBackend backend() const override { return WindowBackend::Hyprland; } + + Result> list() override { + auto active = query("j/activewindow"); + if (!active) return std::unexpected(active.error()); + auto clients = query("j/clients"); + if (!clients) return std::unexpected(clients.error()); + + std::string activeAddress = (*active)["address"].string(); + std::vector out; + for (const auto& c : clients->array()) + if (c["mapped"].boolean(true)) out.push_back(toWindow(c, activeAddress)); + return out; + } + + Result> active() override { + auto a = query("j/activewindow"); + if (!a) return std::unexpected(a.error()); + if (!a->isObject() || a->object().empty()) return std::optional{}; + return std::optional{toWindow(*a, (*a)["address"].string())}; + } + + Status focus(WindowId id) override { + char address[32]; + std::snprintf(address, sizeof address, "0x%llx", static_cast(id.value)); + auto reply = hyprRequest(path_, std::string("dispatch focuswindow address:") + address); + if (!reply) return std::unexpected(reply.error()); + if (*reply != "ok") return fail(ErrorCode::NotFound, "Hyprland: " + *reply); + return {}; + } + +private: + Result query(std::string_view request) { + auto reply = hyprRequest(path_, request); + if (!reply) return std::unexpected(reply.error()); + auto parsed = json::parse(*reply); + if (!parsed) return fail(ErrorCode::Protocol, "unparseable Hyprland IPC reply to " + std::string(request)); + return std::move(*parsed); + } + + std::string path_; +}; + +} // namespace + +Result> createHyprlandWindows() { + auto path = socketPath(); + if (!path) return std::unexpected(path.error()); + return std::make_unique(std::move(*path)); +} + +} // namespace alvin::linux_impl diff --git a/src/linux/keycodes.cpp b/src/linux/keycodes.cpp new file mode 100644 index 0000000..ddaeeba --- /dev/null +++ b/src/linux/keycodes.cpp @@ -0,0 +1,113 @@ +#include "keycodes.hpp" + +#include + +namespace alvin::linux_impl { + +std::uint32_t toEvdev(Key key) { + switch (key) { + case Key::A: return KEY_A; case Key::B: return KEY_B; case Key::C: return KEY_C; + case Key::D: return KEY_D; case Key::E: return KEY_E; case Key::F: return KEY_F; + case Key::G: return KEY_G; case Key::H: return KEY_H; case Key::I: return KEY_I; + case Key::J: return KEY_J; case Key::K: return KEY_K; case Key::L: return KEY_L; + case Key::M: return KEY_M; case Key::N: return KEY_N; case Key::O: return KEY_O; + case Key::P: return KEY_P; case Key::Q: return KEY_Q; case Key::R: return KEY_R; + case Key::S: return KEY_S; case Key::T: return KEY_T; case Key::U: return KEY_U; + case Key::V: return KEY_V; case Key::W: return KEY_W; case Key::X: return KEY_X; + case Key::Y: return KEY_Y; case Key::Z: return KEY_Z; + + case Key::Num0: return KEY_0; case Key::Num1: return KEY_1; case Key::Num2: return KEY_2; + case Key::Num3: return KEY_3; case Key::Num4: return KEY_4; case Key::Num5: return KEY_5; + case Key::Num6: return KEY_6; case Key::Num7: return KEY_7; case Key::Num8: return KEY_8; + case Key::Num9: return KEY_9; + + case Key::F1: return KEY_F1; case Key::F2: return KEY_F2; case Key::F3: return KEY_F3; + case Key::F4: return KEY_F4; case Key::F5: return KEY_F5; case Key::F6: return KEY_F6; + case Key::F7: return KEY_F7; case Key::F8: return KEY_F8; case Key::F9: return KEY_F9; + case Key::F10: return KEY_F10; case Key::F11: return KEY_F11; case Key::F12: return KEY_F12; + case Key::F13: return KEY_F13; case Key::F14: return KEY_F14; case Key::F15: return KEY_F15; + case Key::F16: return KEY_F16; case Key::F17: return KEY_F17; case Key::F18: return KEY_F18; + case Key::F19: return KEY_F19; case Key::F20: return KEY_F20; case Key::F21: return KEY_F21; + case Key::F22: return KEY_F22; case Key::F23: return KEY_F23; case Key::F24: return KEY_F24; + + case Key::Escape: return KEY_ESC; + case Key::Enter: return KEY_ENTER; + case Key::Tab: return KEY_TAB; + case Key::Backspace: return KEY_BACKSPACE; + case Key::Space: return KEY_SPACE; + case Key::Minus: return KEY_MINUS; + case Key::Equal: return KEY_EQUAL; + case Key::LeftBracket: return KEY_LEFTBRACE; + case Key::RightBracket: return KEY_RIGHTBRACE; + case Key::Backslash: return KEY_BACKSLASH; + case Key::Semicolon: return KEY_SEMICOLON; + case Key::Apostrophe: return KEY_APOSTROPHE; + case Key::Grave: return KEY_GRAVE; + case Key::Comma: return KEY_COMMA; + case Key::Period: return KEY_DOT; + case Key::Slash: return KEY_SLASH; + + case Key::CapsLock: return KEY_CAPSLOCK; + case Key::NumLock: return KEY_NUMLOCK; + case Key::ScrollLock: return KEY_SCROLLLOCK; + case Key::LeftShift: return KEY_LEFTSHIFT; + case Key::RightShift: return KEY_RIGHTSHIFT; + case Key::LeftCtrl: return KEY_LEFTCTRL; + case Key::RightCtrl: return KEY_RIGHTCTRL; + case Key::LeftAlt: return KEY_LEFTALT; + case Key::RightAlt: return KEY_RIGHTALT; + case Key::LeftSuper: return KEY_LEFTMETA; + case Key::RightSuper: return KEY_RIGHTMETA; + case Key::Menu: return KEY_COMPOSE; + + case Key::Insert: return KEY_INSERT; + case Key::Delete: return KEY_DELETE; + case Key::Home: return KEY_HOME; + case Key::End: return KEY_END; + case Key::PageUp: return KEY_PAGEUP; + case Key::PageDown: return KEY_PAGEDOWN; + case Key::Left: return KEY_LEFT; + case Key::Right: return KEY_RIGHT; + case Key::Up: return KEY_UP; + case Key::Down: return KEY_DOWN; + case Key::PrintScreen: return KEY_SYSRQ; + case Key::Pause: return KEY_PAUSE; + + case Key::Kp0: return KEY_KP0; case Key::Kp1: return KEY_KP1; case Key::Kp2: return KEY_KP2; + case Key::Kp3: return KEY_KP3; case Key::Kp4: return KEY_KP4; case Key::Kp5: return KEY_KP5; + case Key::Kp6: return KEY_KP6; case Key::Kp7: return KEY_KP7; case Key::Kp8: return KEY_KP8; + case Key::Kp9: return KEY_KP9; + case Key::KpDecimal: return KEY_KPDOT; + case Key::KpDivide: return KEY_KPSLASH; + case Key::KpMultiply: return KEY_KPASTERISK; + case Key::KpSubtract: return KEY_KPMINUS; + case Key::KpAdd: return KEY_KPPLUS; + case Key::KpEnter: return KEY_KPENTER; + + case Key::VolumeMute: return KEY_MUTE; + case Key::VolumeDown: return KEY_VOLUMEDOWN; + case Key::VolumeUp: return KEY_VOLUMEUP; + case Key::MediaPlayPause: return KEY_PLAYPAUSE; + case Key::MediaStop: return KEY_STOPCD; + case Key::MediaNext: return KEY_NEXTSONG; + case Key::MediaPrevious: return KEY_PREVIOUSSONG; + + case Key::Unknown: + case Key::Count_: + return 0; + } + return 0; +} + +std::uint32_t toEvdev(MouseButton button) { + switch (button) { + case MouseButton::Left: return BTN_LEFT; + case MouseButton::Right: return BTN_RIGHT; + case MouseButton::Middle: return BTN_MIDDLE; + case MouseButton::Back: return BTN_SIDE; + case MouseButton::Forward: return BTN_EXTRA; + } + return BTN_LEFT; +} + +} // namespace alvin::linux_impl diff --git a/src/linux/keycodes.hpp b/src/linux/keycodes.hpp new file mode 100644 index 0000000..ac38aba --- /dev/null +++ b/src/linux/keycodes.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +#include "alvin/keys.hpp" + +namespace alvin::linux_impl { + +// Linux evdev KEY_* code (0 if unmapped). XKB keycodes are this + 8. +std::uint32_t toEvdev(Key key); +// Linux evdev BTN_* code. +std::uint32_t toEvdev(MouseButton button); + +} // namespace alvin::linux_impl diff --git a/src/linux/libei_input.cpp b/src/linux/libei_input.cpp new file mode 100644 index 0000000..bb3515a --- /dev/null +++ b/src/linux/libei_input.cpp @@ -0,0 +1,260 @@ +// Input through libei, connected via the xdg-desktop-portal RemoteDesktop +// interface. This is the sanctioned emulation path on GNOME and KDE Plasma. +// (Hyprland's portal backend does not implement RemoteDesktop; use the wlr +// backend there.) + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "backends.hpp" +#include "keycodes.hpp" +#include "portal.hpp" +#include "xkb.hpp" + +namespace alvin::linux_impl { +namespace { + +// org.freedesktop.portal.RemoteDesktop device types. +constexpr std::uint32_t kDeviceKeyboard = 1; +constexpr std::uint32_t kDevicePointer = 2; +constexpr std::uint32_t kPersistUntilRevoked = 2; + +Result startRemoteDesktop(PortalSession& session, const InputOptions& options) { + const char* iface = PortalSession::kRemoteDesktop; + std::uint32_t version = session.uintProperty(iface, "version").value_or(1); + + GVariantBuilder opts; + g_variant_builder_init(&opts, G_VARIANT_TYPE_VARDICT); + g_variant_builder_add(&opts, "{sv}", "types", g_variant_new_uint32(kDeviceKeyboard | kDevicePointer)); + if (version >= 2) { + g_variant_builder_add(&opts, "{sv}", "persist_mode", g_variant_new_uint32(kPersistUntilRevoked)); + if (!options.restoreToken.empty()) + g_variant_builder_add(&opts, "{sv}", "restore_token", g_variant_new_string(options.restoreToken.c_str())); + } + if (auto r = session.request(iface, "SelectDevices", &opts); !r) return std::unexpected(r.error()); + + GVariantBuilder startOpts; + g_variant_builder_init(&startOpts, G_VARIANT_TYPE_VARDICT); + auto results = session.request(iface, "Start", &startOpts, ""); + if (!results) return std::unexpected(results.error()); + + std::string token; + const gchar* t = nullptr; + if (g_variant_lookup(results->get(), "restore_token", "&s", &t)) token = t; + return token; +} + +struct EiDeleter { + void operator()(ei* e) const { ei_unref(e); } +}; + +class LibEiInput final : public Input { +public: + static Result> create(const InputOptions& options) { + auto session = PortalSession::create(PortalSession::kRemoteDesktop, options.permissionTimeout); + if (!session) return std::unexpected(session.error()); + auto token = startRemoteDesktop(**session, options); + if (!token) return std::unexpected(token.error()); + auto fd = (*session)->requestFd(PortalSession::kRemoteDesktop, "ConnectToEIS"); + if (!fd) return std::unexpected(fd.error()); + + std::unique_ptr in(new LibEiInput(std::move(*session), std::move(*token))); + if (auto s = in->connect(std::move(*fd)); !s) return std::unexpected(s.error()); + return in; + } + + ~LibEiInput() override { + for (auto* d : devices_) ei_device_unref(d); + if (ei_) ei_disconnect(ei_.get()); + } + + InputBackend backend() const override { return InputBackend::LibEi; } + std::string restoreToken() const override { return restoreToken_; } + + Status moveTo(Point p) override { + return emit(EI_DEVICE_CAP_POINTER_ABSOLUTE, [&](ei_device* d) { + ei_device_pointer_motion_absolute(d, p.x, p.y); + }); + } + + Status moveBy(int dx, int dy) override { + return emit(EI_DEVICE_CAP_POINTER, [&](ei_device* d) { ei_device_pointer_motion(d, dx, dy); }); + } + + Status buttonDown(MouseButton b) override { + return emit(EI_DEVICE_CAP_BUTTON, [&](ei_device* d) { ei_device_button_button(d, toEvdev(b), true); }); + } + + Status buttonUp(MouseButton b) override { + return emit(EI_DEVICE_CAP_BUTTON, [&](ei_device* d) { ei_device_button_button(d, toEvdev(b), false); }); + } + + Status scroll(int dx, int dy) override { + // libei discrete scroll uses 120 units per wheel notch. + return emit(EI_DEVICE_CAP_SCROLL, [&](ei_device* d) { ei_device_scroll_discrete(d, dx * 120, dy * 120); }); + } + + Status keyDown(Key k) override { return rawKey(toEvdev(k), true); } + Status keyUp(Key k) override { return rawKey(toEvdev(k), false); } + + Status typeText(std::string_view utf8) override { +#ifdef ALVIN_LIBEI_HAS_TEXT + // libei >= 1.6 servers may accept text directly, bypassing layouts. + if (findDevice(EI_DEVICE_CAP_TEXT)) { + std::string copy(utf8); + return emit(EI_DEVICE_CAP_TEXT, [&](ei_device* d) { ei_device_text_utf8(d, copy.c_str()); }); + } +#endif + auto text = decodeUtf8(utf8); + if (!text) return std::unexpected(text.error()); + auto keymap = keyboardKeymap(); + if (!keymap) return std::unexpected(keymap.error()); + auto strokes = (*keymap)->strokesFor(*text); + if (!strokes) return std::unexpected(strokes.error()); + for (const auto& s : *strokes) { + for (auto m : s.modifiers) + if (auto r = rawKey(m, true); !r) return r; + auto r = rawKey(s.key, true); + if (r) r = rawKey(s.key, false); + for (auto m : s.modifiers) (void)rawKey(m, false); + if (!r) return r; + pause(); + } + return {}; + } + +private: + LibEiInput(std::unique_ptr session, std::string token) + : session_(std::move(session)), restoreToken_(std::move(token)) {} + + Status connect(UniqueFd fd) { + ei_.reset(ei_new_sender(this)); + if (!ei_) return fail(ErrorCode::Unknown, "ei_new_sender failed"); + ei_configure_name(ei_.get(), "alvin"); + if (int rc = ei_setup_backend_fd(ei_.get(), fd.release()); rc < 0) + return fail(ErrorCode::Io, "ei_setup_backend_fd failed"); + + // Wait until the server has given us pointer and keyboard devices we may emulate on. + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!(findDevice(EI_DEVICE_CAP_POINTER_ABSOLUTE) && findDevice(EI_DEVICE_CAP_KEYBOARD))) { + auto left = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + if (left.count() <= 0) { + if (findDevice(EI_DEVICE_CAP_POINTER) || findDevice(EI_DEVICE_CAP_KEYBOARD)) break; // partial is usable + return fail(ErrorCode::Timeout, "EIS server did not provide input devices"); + } + pollfd pfd{ei_get_fd(ei_.get()), POLLIN, 0}; + ::poll(&pfd, 1, static_cast(left.count())); + if (auto s = dispatch(); !s) return s; + } + return {}; + } + + Status dispatch() { + ei_dispatch(ei_.get()); + while (ei_event* ev = ei_get_event(ei_.get())) { + ei_device* dev = ei_event_get_device(ev); + switch (ei_event_get_type(ev)) { + case EI_EVENT_SEAT_ADDED: + ei_seat_bind_capabilities(ei_event_get_seat(ev), EI_DEVICE_CAP_POINTER, EI_DEVICE_CAP_POINTER_ABSOLUTE, + EI_DEVICE_CAP_BUTTON, EI_DEVICE_CAP_SCROLL, EI_DEVICE_CAP_KEYBOARD, +#ifdef ALVIN_LIBEI_HAS_TEXT + EI_DEVICE_CAP_TEXT, +#endif + nullptr); + break; + case EI_EVENT_DEVICE_ADDED: + devices_.push_back(ei_device_ref(dev)); + break; + case EI_EVENT_DEVICE_REMOVED: + std::erase_if(resumed_, [&](ei_device* d) { return d == dev; }); + std::erase_if(devices_, [&](ei_device* d) { + if (d != dev) return false; + ei_device_unref(d); + return true; + }); + break; + case EI_EVENT_DEVICE_RESUMED: + ei_device_start_emulating(dev, ++sequence_); + resumed_.push_back(dev); + break; + case EI_EVENT_DEVICE_PAUSED: + std::erase_if(resumed_, [&](ei_device* d) { return d == dev; }); + break; + case EI_EVENT_DISCONNECT: + disconnected_ = true; + break; + default: + break; + } + ei_event_unref(ev); + } + if (disconnected_) return fail(ErrorCode::NotAvailable, "the EIS server disconnected"); + return {}; + } + + ei_device* findDevice(ei_device_capability cap) const { + for (auto* d : resumed_) + if (ei_device_has_capability(d, cap)) return d; + return nullptr; + } + + template + Status emit(ei_device_capability cap, F&& send) { + if (auto s = dispatch(); !s) return s; + ei_device* d = findDevice(cap); + if (!d) return fail(ErrorCode::Unsupported, "the EIS server granted no device with the needed capability"); + send(d); + ei_device_frame(d, ei_now(ei_.get())); + return {}; + } + + Status rawKey(std::uint32_t code, bool down) { + if (code == 0) return fail(ErrorCode::InvalidArgument, "key has no Linux mapping"); + return emit(EI_DEVICE_CAP_KEYBOARD, [&](ei_device* d) { ei_device_keyboard_key(d, code, down); }); + } + + // The keymap the server attached to its keyboard device, else the XKB defaults. + Result keyboardKeymap() { + if (keymap_) return &*keymap_; + if (ei_device* d = findDevice(EI_DEVICE_CAP_KEYBOARD)) { + if (ei_keymap* km = ei_device_keyboard_get_keymap(d); km && ei_keymap_get_type(km) == EI_KEYMAP_TYPE_XKB) { + std::size_t size = ei_keymap_get_size(km); + void* map = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, ei_keymap_get_fd(km), 0); + if (map != MAP_FAILED) { + auto parsed = XkbKeymap::fromString({static_cast(map), strnlen(static_cast(map), size)}); + munmap(map, size); + if (parsed) keymap_ = std::move(*parsed); + } + } + } + if (!keymap_) { + auto fallback = XkbKeymap::fromNames(); + if (!fallback) return std::unexpected(fallback.error()); + keymap_ = std::move(*fallback); + } + return &*keymap_; + } + + std::unique_ptr session_; + std::string restoreToken_; + std::unique_ptr ei_; + std::vector devices_; // owned references + std::vector resumed_; // subset of devices_ we may emulate on + std::uint32_t sequence_ = 0; + bool disconnected_ = false; + std::optional keymap_; +}; + +} // namespace + +Result> createLibEiInput(const InputOptions& options) { return LibEiInput::create(options); } + +} // namespace alvin::linux_impl diff --git a/src/linux/pipewire_capture.cpp b/src/linux/pipewire_capture.cpp new file mode 100644 index 0000000..87d80b2 --- /dev/null +++ b/src/linux/pipewire_capture.cpp @@ -0,0 +1,347 @@ +// Screen/window capture through the xdg-desktop-portal ScreenCast interface. +// The portal asks the user what to share (unless a restore token lets it skip +// that), then hands us a PipeWire remote from which we pull video frames. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "backends.hpp" +#include "portal.hpp" + +namespace alvin::linux_impl { +namespace { + +// Portal enum values (org.freedesktop.portal.ScreenCast). +constexpr std::uint32_t kCursorHidden = 1; +constexpr std::uint32_t kCursorEmbedded = 2; +constexpr std::uint32_t kPersistUntilRevoked = 2; + +struct StreamInfo { + std::uint32_t nodeId = 0; + Rect desktopRect; // position is (0,0) when the portal doesn't report one (e.g. windows) + std::string restoreToken; +}; + +Result startScreenCast(PortalSession& session, const CaptureOptions& options) { + const char* iface = PortalSession::kScreenCast; + std::uint32_t version = session.uintProperty(iface, "version").value_or(1); + std::uint32_t available = session.uintProperty(iface, "AvailableSourceTypes").value_or(1); + std::uint32_t types = static_cast(options.source) & available; + if (types == 0) return fail(ErrorCode::Unsupported, "the portal cannot capture the requested source type"); + + GVariantBuilder opts; + g_variant_builder_init(&opts, G_VARIANT_TYPE_VARDICT); + g_variant_builder_add(&opts, "{sv}", "types", g_variant_new_uint32(types)); + g_variant_builder_add(&opts, "{sv}", "multiple", g_variant_new_boolean(FALSE)); + if (version >= 2) { + std::uint32_t modes = session.uintProperty(iface, "AvailableCursorModes").value_or(0); + std::uint32_t want = options.showCursor ? kCursorEmbedded : kCursorHidden; + if (modes & want) g_variant_builder_add(&opts, "{sv}", "cursor_mode", g_variant_new_uint32(want)); + } + if (version >= 4) { + g_variant_builder_add(&opts, "{sv}", "persist_mode", g_variant_new_uint32(kPersistUntilRevoked)); + if (!options.restoreToken.empty()) + g_variant_builder_add(&opts, "{sv}", "restore_token", g_variant_new_string(options.restoreToken.c_str())); + } + if (auto r = session.request(iface, "SelectSources", &opts); !r) return std::unexpected(r.error()); + + GVariantBuilder startOpts; + g_variant_builder_init(&startOpts, G_VARIANT_TYPE_VARDICT); + auto results = session.request(iface, "Start", &startOpts, ""); + if (!results) return std::unexpected(results.error()); + + StreamInfo info; + const gchar* token = nullptr; + if (g_variant_lookup(results->get(), "restore_token", "&s", &token)) info.restoreToken = token; + + GVariant* streamsRaw = g_variant_lookup_value(results->get(), "streams", G_VARIANT_TYPE("a(ua{sv})")); + if (!streamsRaw) return fail(ErrorCode::Protocol, "ScreenCast.Start returned no streams"); + VariantPtr streams(streamsRaw); + if (g_variant_n_children(streams.get()) == 0) return fail(ErrorCode::Protocol, "ScreenCast.Start returned no streams"); + + GVariant* props = nullptr; + g_variant_get_child(streams.get(), 0, "(u@a{sv})", &info.nodeId, &props); + VariantPtr propsPtr(props); + gint32 x = 0, y = 0, w = 0, h = 0; + g_variant_lookup(props, "position", "(ii)", &x, &y); + g_variant_lookup(props, "size", "(ii)", &w, &h); + info.desktopRect = {x, y, w, h}; + return info; +} + +// Converts one row of a PipeWire video format to BGRA. +void convertRow(spa_video_format fmt, const std::uint8_t* src, std::uint8_t* dst, int width) { + switch (fmt) { + case SPA_VIDEO_FORMAT_BGRA: + std::memcpy(dst, src, static_cast(width) * 4); + return; + case SPA_VIDEO_FORMAT_BGRx: + for (int x = 0; x < width; ++x, src += 4, dst += 4) dst[0] = src[0], dst[1] = src[1], dst[2] = src[2], dst[3] = 255; + return; + case SPA_VIDEO_FORMAT_RGBA: + case SPA_VIDEO_FORMAT_RGBx: { + bool alpha = fmt == SPA_VIDEO_FORMAT_RGBA; + for (int x = 0; x < width; ++x, src += 4, dst += 4) + dst[0] = src[2], dst[1] = src[1], dst[2] = src[0], dst[3] = alpha ? src[3] : 255; + return; + } + case SPA_VIDEO_FORMAT_ARGB: + case SPA_VIDEO_FORMAT_xRGB: { + bool alpha = fmt == SPA_VIDEO_FORMAT_ARGB; + for (int x = 0; x < width; ++x, src += 4, dst += 4) + dst[0] = src[3], dst[1] = src[2], dst[2] = src[1], dst[3] = alpha ? src[0] : 255; + return; + } + case SPA_VIDEO_FORMAT_ABGR: + case SPA_VIDEO_FORMAT_xBGR: { + bool alpha = fmt == SPA_VIDEO_FORMAT_ABGR; + for (int x = 0; x < width; ++x, src += 4, dst += 4) + dst[0] = src[1], dst[1] = src[2], dst[2] = src[3], dst[3] = alpha ? src[0] : 255; + return; + } + default: + std::memset(dst, 0, static_cast(width) * 4); + } +} + +class PipeWireGuard { +public: + PipeWireGuard() { pw_init(nullptr, nullptr); } + ~PipeWireGuard() { pw_deinit(); } +}; + +class PortalCapture final : public Capture { +public: + static Result> create(const CaptureOptions& options) { + auto session = PortalSession::create(PortalSession::kScreenCast, options.permissionTimeout); + if (!session) return std::unexpected(session.error()); + auto stream = startScreenCast(**session, options); + if (!stream) return std::unexpected(stream.error()); + auto fd = (*session)->requestFd(PortalSession::kScreenCast, "OpenPipeWireRemote"); + if (!fd) return std::unexpected(fd.error()); + + std::unique_ptr cap(new PortalCapture(std::move(*session), *stream)); + if (auto s = cap->connect(std::move(*fd)); !s) return std::unexpected(s.error()); + return cap; + } + + ~PortalCapture() override { + if (loop_) pw_thread_loop_lock(loop_); + if (stream_) pw_stream_destroy(stream_); + if (core_) pw_core_disconnect(core_); + if (loop_) { + pw_thread_loop_unlock(loop_); + pw_thread_loop_stop(loop_); + } + if (context_) pw_context_destroy(context_); + if (loop_) pw_thread_loop_destroy(loop_); + } + + CaptureBackend backend() const override { return CaptureBackend::Portal; } + + Result grab(std::chrono::milliseconds timeout) override { return wait(timeout, 0); } + + Result nextFrame(std::chrono::milliseconds timeout) override { + std::uint64_t seen; + { + std::lock_guard lock(mutex_); + seen = lastReturned_; + } + return wait(timeout, seen); + } + + std::string restoreToken() const override { return info_.restoreToken; } + +private: + PortalCapture(std::unique_ptr session, StreamInfo info) + : session_(std::move(session)), info_(std::move(info)) {} + + Result wait(std::chrono::milliseconds timeout, std::uint64_t after) { + std::unique_lock lock(mutex_); + bool ok = frameReady_.wait_for(lock, timeout, [&] { return sequence_ > after || !error_.empty(); }); + if (!error_.empty()) return fail(ErrorCode::Io, error_); + if (!ok) return fail(ErrorCode::Timeout, "no frame received from PipeWire"); + lastReturned_ = sequence_; + return latest_; + } + + Status connect(UniqueFd fd) { + loop_ = pw_thread_loop_new("alvin-capture", nullptr); + if (!loop_) return fail(ErrorCode::Unknown, "pw_thread_loop_new failed"); + context_ = pw_context_new(pw_thread_loop_get_loop(loop_), nullptr, 0); + if (!context_) return fail(ErrorCode::Unknown, "pw_context_new failed"); + if (pw_thread_loop_start(loop_) < 0) return fail(ErrorCode::Unknown, "pw_thread_loop_start failed"); + + pw_thread_loop_lock(loop_); + auto status = connectLocked(std::move(fd)); + pw_thread_loop_unlock(loop_); + return status; + } + + Status connectLocked(UniqueFd fd) { + core_ = pw_context_connect_fd(context_, fd.release(), nullptr, 0); + if (!core_) return fail(ErrorCode::Io, "cannot connect to the PipeWire remote"); + + stream_ = pw_stream_new(core_, "alvin", + pw_properties_new(PW_KEY_MEDIA_TYPE, "Video", PW_KEY_MEDIA_CATEGORY, "Capture", + PW_KEY_MEDIA_ROLE, "Screen", nullptr)); + if (!stream_) return fail(ErrorCode::Unknown, "pw_stream_new failed"); + static const pw_stream_events events = [] { + pw_stream_events e{}; + e.version = PW_VERSION_STREAM_EVENTS; + e.state_changed = &PortalCapture::onStateChanged; + e.param_changed = &PortalCapture::onParamChanged; + e.process = &PortalCapture::onProcess; + return e; + }(); + pw_stream_add_listener(stream_, &listener_, &events, this); + + // Offer only shared-memory friendly formats (no DMA-BUF modifiers), so the + // compositor hands us CPU-mappable buffers. + std::uint8_t buffer[1024]; + spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof buffer); + spa_rectangle defSize{1920, 1080}, minSize{1, 1}, maxSize{16384, 16384}; + spa_fraction defRate{0, 1}, minRate{0, 1}, maxRate{1000, 1}; + const spa_pod* params[1]; + params[0] = static_cast(spa_pod_builder_add_object( + &b, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), + SPA_FORMAT_VIDEO_format, + SPA_POD_CHOICE_ENUM_Id(9, SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_BGRA, + SPA_VIDEO_FORMAT_RGBx, SPA_VIDEO_FORMAT_RGBA, SPA_VIDEO_FORMAT_xRGB, + SPA_VIDEO_FORMAT_ARGB, SPA_VIDEO_FORMAT_xBGR, SPA_VIDEO_FORMAT_ABGR), + SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(&defSize, &minSize, &maxSize), + SPA_FORMAT_VIDEO_framerate, SPA_POD_CHOICE_RANGE_Fraction(&defRate, &minRate, &maxRate))); + + int rc = pw_stream_connect(stream_, PW_DIRECTION_INPUT, info_.nodeId, + static_cast(PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS), + params, 1); + if (rc < 0) return fail(ErrorCode::Io, "pw_stream_connect failed"); + return {}; + } + + static void onStateChanged(void* data, pw_stream_state, pw_stream_state state, const char* error) { + auto* self = static_cast(data); + std::lock_guard lock(self->mutex_); + if (state == PW_STREAM_STATE_ERROR || (state == PW_STREAM_STATE_UNCONNECTED && self->sequence_ > 0)) { + self->error_ = error ? std::string("PipeWire stream error: ") + error : "the capture stream ended"; + self->frameReady_.notify_all(); + } + } + + static void onParamChanged(void* data, std::uint32_t id, const spa_pod* param) { + auto* self = static_cast(data); + if (!param || id != SPA_PARAM_Format) return; + std::uint32_t mediaType = 0, mediaSubtype = 0; + if (spa_format_parse(param, &mediaType, &mediaSubtype) < 0 || mediaType != SPA_MEDIA_TYPE_video || + mediaSubtype != SPA_MEDIA_SUBTYPE_raw) + return; + spa_video_info_raw raw{}; + if (spa_format_video_raw_parse(param, &raw) < 0) return; + self->format_ = raw.format; + self->width_ = static_cast(raw.size.width); + self->height_ = static_cast(raw.size.height); + + std::uint8_t buffer[1024]; + spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof buffer); + const spa_pod* params[3]; + params[0] = static_cast(spa_pod_builder_add_object( + &b, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd)))); + params[1] = static_cast(spa_pod_builder_add_object( + &b, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, + SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header), + SPA_PARAM_META_size, SPA_POD_Int(sizeof(spa_meta_header)))); + params[2] = static_cast(spa_pod_builder_add_object( + &b, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, + SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoCrop), + SPA_PARAM_META_size, SPA_POD_Int(sizeof(spa_meta_region)))); + pw_stream_update_params(self->stream_, params, 3); + } + + static void onProcess(void* data) { + auto* self = static_cast(data); + pw_buffer* b = pw_stream_dequeue_buffer(self->stream_); + if (!b) return; + // Skip to the newest queued buffer; we only care about the latest image. + while (pw_buffer* next = pw_stream_dequeue_buffer(self->stream_)) { + pw_stream_queue_buffer(self->stream_, b); + b = next; + } + self->consume(b->buffer); + pw_stream_queue_buffer(self->stream_, b); + } + + void consume(spa_buffer* buf) { + if (auto* h = static_cast(spa_buffer_find_meta_data(buf, SPA_META_Header, sizeof(spa_meta_header)))) + if (h->flags & SPA_META_HEADER_FLAG_CORRUPTED) return; + const spa_data& d = buf->datas[0]; + // Empty chunks carry cursor-only updates; no new image. + if (!d.data || !d.chunk || d.chunk->size == 0 || (d.chunk->flags & SPA_CHUNK_FLAG_CORRUPTED)) return; + + Rect region{0, 0, width_, height_}; + if (auto* crop = static_cast(spa_buffer_find_meta_data(buf, SPA_META_VideoCrop, sizeof(spa_meta_region)))) + if (spa_meta_region_is_valid(crop)) + region = {crop->region.position.x, crop->region.position.y, static_cast(crop->region.size.width), + static_cast(crop->region.size.height)}; + + std::size_t stride = d.chunk->stride > 0 ? static_cast(d.chunk->stride) : static_cast(width_) * 4; + const auto* base = static_cast(d.data) + d.chunk->offset; + std::size_t needed = stride * static_cast(region.y + region.height); + if (region.empty() || needed > d.maxsize - d.chunk->offset) return; + + if (scratch_.width() != region.width || scratch_.height() != region.height) + scratch_ = Frame(region.width, region.height); + for (int y = 0; y < region.height; ++y) + convertRow(format_, base + (region.y + y) * stride + region.x * 4, scratch_.row(y), region.width); + + Rect desk = info_.desktopRect; + if (desk.empty()) desk = {0, 0, region.width, region.height}; + scratch_.setDesktopRect(desk); + scratch_.setTimestampNs(static_cast(pw_stream_get_nsec(stream_))); + + std::lock_guard lock(mutex_); + std::swap(latest_, scratch_); + ++sequence_; + frameReady_.notify_all(); + } + + PipeWireGuard pw_; + std::unique_ptr session_; // closing it ends the stream + StreamInfo info_; + + pw_thread_loop* loop_ = nullptr; + pw_context* context_ = nullptr; + pw_core* core_ = nullptr; + pw_stream* stream_ = nullptr; + spa_hook listener_{}; + + // Touched only on the PipeWire thread. + spa_video_format format_ = SPA_VIDEO_FORMAT_UNKNOWN; + int width_ = 0, height_ = 0; + Frame scratch_; + + std::mutex mutex_; + std::condition_variable frameReady_; + Frame latest_; + std::uint64_t sequence_ = 0; + std::uint64_t lastReturned_ = 0; + std::string error_; +}; + +} // namespace + +Result> createPortalCapture(const CaptureOptions& options) { + return PortalCapture::create(options); +} + +} // namespace alvin::linux_impl diff --git a/src/linux/platform_linux.cpp b/src/linux/platform_linux.cpp new file mode 100644 index 0000000..7ca15bf --- /dev/null +++ b/src/linux/platform_linux.cpp @@ -0,0 +1,107 @@ +// Backend selection for Linux. + +#include +#include + +#include "../common/platform.hpp" +#include "backends.hpp" +#include "wayland.hpp" + +namespace alvin::detail { +namespace { + +bool envSet(const char* name) { + const char* v = std::getenv(name); + return v && *v; +} + +Result inputBackendFromEnv() { + const char* v = std::getenv("ALVIN_INPUT_BACKEND"); + if (!v || !*v) return InputBackend::Auto; + std::string s = v; + if (s == "wlr") return InputBackend::WlrVirtual; + if (s == "libei") return InputBackend::LibEi; + if (s == "uinput") return InputBackend::UInput; + return fail(ErrorCode::InvalidArgument, "unknown ALVIN_INPUT_BACKEND '" + s + "'"); +} + +Result> createInputBackend(InputBackend backend, const InputOptions& options) { + switch (backend) { +#ifdef ALVIN_HAVE_WAYLAND + case InputBackend::WlrVirtual: return linux_impl::createWlrInput(options); +#endif +#ifdef ALVIN_HAVE_LIBEI + case InputBackend::LibEi: return linux_impl::createLibEiInput(options); +#endif +#ifdef ALVIN_HAVE_UINPUT + case InputBackend::UInput: return linux_impl::createUInput(options); +#endif + default: + return fail(ErrorCode::Unsupported, + "input backend '" + std::string(toString(backend)) + "' is not compiled into this build"); + } +} + +} // namespace + +Result> createInput(const InputOptions& options) { + InputBackend chosen = options.backend; + if (chosen == InputBackend::Auto) { + auto env = inputBackendFromEnv(); + if (!env) return std::unexpected(env.error()); + chosen = *env; + } + if (chosen != InputBackend::Auto) return createInputBackend(chosen, options); + + // Prefer what needs no dialog or extra privileges, then the portal, then uinput. + std::string tried; + InputBackend order[] = {InputBackend::WlrVirtual, InputBackend::LibEi, InputBackend::UInput}; + for (InputBackend b : order) { + if (b == InputBackend::WlrVirtual && !envSet("WAYLAND_DISPLAY")) continue; + auto in = createInputBackend(b, options); + if (in) return in; + // A user saying "no" to the portal is an answer, not a reason to try something else. + auto code = in.error().code; + if (b == InputBackend::LibEi && (code == ErrorCode::Cancelled || code == ErrorCode::PermissionDenied)) return in; + tried += "\n " + std::string(toString(b)) + ": " + in.error().message; + } + return fail(ErrorCode::NotAvailable, "no usable input backend:" + tried); +} + +Result> createCapture(const CaptureOptions& options) { + switch (options.backend) { + case CaptureBackend::Auto: + case CaptureBackend::Portal: +#ifdef ALVIN_HAVE_PORTAL + return linux_impl::createPortalCapture(options); +#else + return fail(ErrorCode::Unsupported, "portal capture is not compiled into this build"); +#endif + default: + return fail(ErrorCode::Unsupported, "capture backend '" + std::string(toString(options.backend)) + + "' is not available on Linux"); + } +} + +Result> createWindowManager(WindowBackend backend) { + switch (backend) { + case WindowBackend::Auto: + case WindowBackend::Hyprland: + return linux_impl::createHyprlandWindows(); + default: + return fail(ErrorCode::Unsupported, + "window backend '" + std::string(toString(backend)) + "' is not available on Linux"); + } +} + +Result> listMonitors() { +#ifdef ALVIN_HAVE_WAYLAND + auto wl = linux_impl::WaylandConnection::connect(); + if (!wl) return std::unexpected(wl.error()); + return (*wl)->monitors(); +#else + return fail(ErrorCode::Unsupported, "monitor enumeration needs Wayland support in this build"); +#endif +} + +} // namespace alvin::detail diff --git a/src/linux/portal.cpp b/src/linux/portal.cpp new file mode 100644 index 0000000..1892d05 --- /dev/null +++ b/src/linux/portal.cpp @@ -0,0 +1,232 @@ +#include "portal.hpp" + +#include + +#include +#include +#include + +namespace alvin::linux_impl { +namespace { + +constexpr const char* kBusName = "org.freedesktop.portal.Desktop"; +constexpr const char* kObjectPath = "/org/freedesktop/portal/desktop"; + +struct ErrorGuard { + GError* error = nullptr; + ~ErrorGuard() { + if (error) g_error_free(error); + } + std::string message() const { return error ? error->message : "unknown error"; } + ErrorCode code() const { + if (error && g_error_matches(error, G_DBUS_ERROR, G_DBUS_ERROR_ACCESS_DENIED)) return ErrorCode::PermissionDenied; + if (error && (g_error_matches(error, G_DBUS_ERROR, G_DBUS_ERROR_UNKNOWN_METHOD) || + g_error_matches(error, G_DBUS_ERROR, G_DBUS_ERROR_UNKNOWN_INTERFACE) || + g_error_matches(error, G_DBUS_ERROR, G_DBUS_ERROR_SERVICE_UNKNOWN))) + return ErrorCode::NotAvailable; + return ErrorCode::Protocol; + } +}; + +// Makes `ctx` the thread-default context for the current scope so that signal +// subscriptions made here are dispatched when we iterate it. +struct ContextScope { + GMainContext* ctx; + explicit ContextScope(GMainContext* c) : ctx(c) { g_main_context_push_thread_default(ctx); } + ~ContextScope() { g_main_context_pop_thread_default(ctx); } +}; + +} // namespace + +void VariantDeleter::operator()(GVariant* v) const { + if (v) g_variant_unref(v); +} + +Result> PortalSession::create(const char* iface, std::chrono::milliseconds timeout) { + std::unique_ptr s(new PortalSession); + s->timeout_ = timeout; + s->context_ = g_main_context_new(); + ContextScope scope(s->context_); + + // A private connection, so the session's lifetime is tied to this object. + ErrorGuard err; + gchar* address = g_dbus_address_get_for_bus_sync(G_BUS_TYPE_SESSION, nullptr, &err.error); + if (!address) return fail(ErrorCode::NotAvailable, "no D-Bus session bus: " + err.message()); + s->bus_ = g_dbus_connection_new_for_address_sync( + address, + static_cast(G_DBUS_CONNECTION_FLAGS_AUTHENTICATION_CLIENT | + G_DBUS_CONNECTION_FLAGS_MESSAGE_BUS_CONNECTION), + nullptr, nullptr, &err.error); + g_free(address); + if (!s->bus_) return fail(ErrorCode::NotAvailable, "cannot connect to D-Bus session bus: " + err.message()); + + // ":1.42" -> "1_42", as used in portal request object paths. + std::string sender = g_dbus_connection_get_unique_name(s->bus_) + 1; + std::replace(sender.begin(), sender.end(), '.', '_'); + s->senderPath_ = sender; + + GVariantBuilder opts; + g_variant_builder_init(&opts, G_VARIANT_TYPE_VARDICT); + std::string sessionToken = s->newToken(); + g_variant_builder_add(&opts, "{sv}", "session_handle_token", g_variant_new_string(sessionToken.c_str())); + + std::string token = s->newToken(); + g_variant_builder_add(&opts, "{sv}", "handle_token", g_variant_new_string(token.c_str())); + std::string path = s->requestPath(token); + + unsigned sub = g_dbus_connection_signal_subscribe( + s->bus_, kBusName, "org.freedesktop.portal.Request", "Response", path.c_str(), nullptr, + G_DBUS_SIGNAL_FLAGS_NO_MATCH_RULE, + &PortalSession::onResponse, + s.get(), nullptr); + + VariantPtr reply(g_dbus_connection_call_sync(s->bus_, kBusName, kObjectPath, iface, "CreateSession", + g_variant_new("(a{sv})", &opts), G_VARIANT_TYPE("(o)"), + G_DBUS_CALL_FLAGS_NONE, -1, nullptr, &err.error)); + if (!reply) { + g_dbus_connection_signal_unsubscribe(s->bus_, sub); + return fail(err.code(), std::string(iface) + ".CreateSession failed: " + err.message()); + } + auto results = s->waitForResponse(path, std::move(reply), sub); + if (!results) return std::unexpected(results.error()); + + const gchar* handle = nullptr; + if (!g_variant_lookup(results->get(), "session_handle", "&s", &handle)) + return fail(ErrorCode::Protocol, "CreateSession response lacks session_handle"); + s->sessionHandle_ = handle; + return s; +} + +PortalSession::~PortalSession() { + if (bus_) { + if (!sessionHandle_.empty()) { + VariantPtr r(g_dbus_connection_call_sync(bus_, kBusName, sessionHandle_.c_str(), + "org.freedesktop.portal.Session", "Close", nullptr, nullptr, + G_DBUS_CALL_FLAGS_NONE, 1000, nullptr, nullptr)); + } + g_dbus_connection_close_sync(bus_, nullptr, nullptr); + g_object_unref(bus_); + } + if (context_) g_main_context_unref(context_); +} + +void PortalSession::onResponse(GDBusConnection*, const char*, const char*, const char*, const char*, GVariant* params, + void* data) { + auto* self = static_cast(data); + GVariant* results = nullptr; + g_variant_get(params, "(u@a{sv})", &self->responseCode_, &results); + self->responseResults_.reset(results); + self->responded_ = true; +} + +std::string PortalSession::newToken() { + return "alvin" + std::to_string(::getpid()) + "_" + std::to_string(++tokenCounter_); +} + +std::string PortalSession::requestPath(const std::string& token) const { + return std::string(kObjectPath) + "/request/" + senderPath_ + "/" + token; +} + +Result PortalSession::uintProperty(const char* iface, const char* name) { + ErrorGuard err; + VariantPtr reply(g_dbus_connection_call_sync(bus_, kBusName, kObjectPath, "org.freedesktop.DBus.Properties", "Get", + g_variant_new("(ss)", iface, name), G_VARIANT_TYPE("(v)"), + G_DBUS_CALL_FLAGS_NONE, -1, nullptr, &err.error)); + if (!reply) return fail(err.code(), std::string("cannot read ") + iface + "." + name + ": " + err.message()); + GVariant* inner = nullptr; + g_variant_get(reply.get(), "(v)", &inner); + VariantPtr value(inner); + if (!g_variant_is_of_type(value.get(), G_VARIANT_TYPE_UINT32)) + return fail(ErrorCode::Protocol, std::string(iface) + "." + name + " is not a uint32"); + return g_variant_get_uint32(value.get()); +} + +Result PortalSession::request(const char* iface, const char* method, GVariantBuilder* options, + const char* parentWindow) { + ContextScope scope(context_); + std::string token = newToken(); + g_variant_builder_add(options, "{sv}", "handle_token", g_variant_new_string(token.c_str())); + std::string path = requestPath(token); + + responded_ = false; + unsigned sub = g_dbus_connection_signal_subscribe( + bus_, kBusName, "org.freedesktop.portal.Request", "Response", path.c_str(), nullptr, + G_DBUS_SIGNAL_FLAGS_NO_MATCH_RULE, + &PortalSession::onResponse, + this, nullptr); + + GVariant* params = parentWindow + ? g_variant_new("(osa{sv})", sessionHandle_.c_str(), parentWindow, options) + : g_variant_new("(oa{sv})", sessionHandle_.c_str(), options); + ErrorGuard err; + VariantPtr reply(g_dbus_connection_call_sync(bus_, kBusName, kObjectPath, iface, method, params, + G_VARIANT_TYPE("(o)"), G_DBUS_CALL_FLAGS_NONE, -1, nullptr, + &err.error)); + if (!reply) { + g_dbus_connection_signal_unsubscribe(bus_, sub); + return fail(err.code(), std::string(iface) + "." + method + " failed: " + err.message()); + } + return waitForResponse(path, std::move(reply), sub); +} + +Result PortalSession::waitForResponse(const std::string& path, VariantPtr callResult, unsigned sub) { + // Old portal versions ignore handle_token and return a different request path. + const gchar* actual = nullptr; + g_variant_get(callResult.get(), "(&o)", &actual); + if (path != actual) { + g_dbus_connection_signal_unsubscribe(bus_, sub); + return fail(ErrorCode::Protocol, "xdg-desktop-portal is too old (request path mismatch)"); + } + + bool timedOut = false; + GSource* timer = g_timeout_source_new(static_cast(std::max(timeout_.count(), 1))); + g_source_set_callback( + timer, + [](gpointer data) -> gboolean { + *static_cast(data) = true; + return G_SOURCE_REMOVE; + }, + &timedOut, nullptr); + g_source_attach(timer, context_); + + while (!responded_ && !timedOut) g_main_context_iteration(context_, TRUE); + + g_source_destroy(timer); + g_source_unref(timer); + g_dbus_connection_signal_unsubscribe(bus_, sub); + + if (!responded_) { + // Dismiss the dialog so it doesn't linger on screen. + VariantPtr r(g_dbus_connection_call_sync(bus_, kBusName, path.c_str(), "org.freedesktop.portal.Request", + "Close", nullptr, nullptr, G_DBUS_CALL_FLAGS_NONE, 1000, nullptr, + nullptr)); + return fail(ErrorCode::Timeout, "timed out waiting for the portal dialog"); + } + responded_ = false; + switch (responseCode_) { + case 0: return std::move(responseResults_); + case 1: return fail(ErrorCode::Cancelled, "the portal request was cancelled by the user"); + default: return fail(ErrorCode::PermissionDenied, "the portal request was denied"); + } +} + +Result PortalSession::requestFd(const char* iface, const char* method) { + ContextScope scope(context_); + GVariantBuilder opts; + g_variant_builder_init(&opts, G_VARIANT_TYPE_VARDICT); + GUnixFDList* fds = nullptr; + ErrorGuard err; + VariantPtr reply(g_dbus_connection_call_with_unix_fd_list_sync( + bus_, kBusName, kObjectPath, iface, method, g_variant_new("(oa{sv})", sessionHandle_.c_str(), &opts), + G_VARIANT_TYPE("(h)"), G_DBUS_CALL_FLAGS_NONE, -1, nullptr, &fds, nullptr, &err.error)); + if (!reply) return fail(err.code(), std::string(iface) + "." + method + " failed: " + err.message()); + + gint32 index = -1; + g_variant_get(reply.get(), "(h)", &index); + int fd = fds ? g_unix_fd_list_get(fds, index, &err.error) : -1; + if (fds) g_object_unref(fds); + if (fd < 0) return fail(ErrorCode::Protocol, std::string(method) + " returned no file descriptor"); + return UniqueFd(fd); +} + +} // namespace alvin::linux_impl diff --git a/src/linux/portal.hpp b/src/linux/portal.hpp new file mode 100644 index 0000000..bf0e824 --- /dev/null +++ b/src/linux/portal.hpp @@ -0,0 +1,73 @@ +#pragma once + +// Minimal synchronous client for xdg-desktop-portal's request/response +// pattern, built on GDBus. Only used during session setup, so blocking is fine. + +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "util.hpp" + +typedef struct _GDBusConnection GDBusConnection; +typedef struct _GMainContext GMainContext; +typedef struct _GVariant GVariant; +typedef struct _GVariantBuilder GVariantBuilder; + +namespace alvin::linux_impl { + +// Owning GVariant pointer. +struct VariantDeleter { + void operator()(GVariant* v) const; +}; +using VariantPtr = std::unique_ptr; + +class PortalSession { +public: + static constexpr const char* kScreenCast = "org.freedesktop.portal.ScreenCast"; + static constexpr const char* kRemoteDesktop = "org.freedesktop.portal.RemoteDesktop"; + + // Connects to the session bus and calls .CreateSession. + static Result> create(const char* iface, std::chrono::milliseconds timeout); + ~PortalSession(); // closes the portal session + + // Property of the portal interface, e.g. ("...ScreenCast", "AvailableSourceTypes"). + Result uintProperty(const char* iface, const char* name); + + // Calls .(session_handle, [parent_window,] options) and waits for + // the matching Request.Response signal. `options` is an a{sv} builder the + // caller filled; handle_token is added here and the builder is consumed. + // Pass parentWindow (usually "") for methods like Start that take one. + // Returns the Response's results dictionary. + Result request(const char* iface, const char* method, GVariantBuilder* options, + const char* parentWindow = nullptr); + + // Methods returning a file descriptor directly: OpenPipeWireRemote, ConnectToEIS. + Result requestFd(const char* iface, const char* method); + + const std::string& handle() const { return sessionHandle_; } + +private: + PortalSession() = default; + std::string newToken(); + std::string requestPath(const std::string& token) const; + static void onResponse(GDBusConnection*, const char* sender, const char* path, const char* iface, + const char* signal, GVariant* params, void* self); + Result waitForResponse(const std::string& path, VariantPtr callResult, unsigned subscription); + + GMainContext* context_ = nullptr; + GDBusConnection* bus_ = nullptr; + std::string senderPath_; // unique bus name, sanitised for object paths + std::string sessionHandle_; + std::chrono::milliseconds timeout_{}; + unsigned tokenCounter_ = 0; + + // Filled by the Response signal handler. + bool responded_ = false; + std::uint32_t responseCode_ = 0; + VariantPtr responseResults_; +}; + +} // namespace alvin::linux_impl diff --git a/src/linux/uinput_input.cpp b/src/linux/uinput_input.cpp new file mode 100644 index 0000000..2e3eed2 --- /dev/null +++ b/src/linux/uinput_input.cpp @@ -0,0 +1,261 @@ +// Input through kernel virtual devices (/dev/uinput). Works under any Linux +// session - Wayland, X11 or a bare TTY - because events enter the stack below +// the compositor. Requires write access to /dev/uinput (e.g. the `input` group +// or a udev rule granting it to the seat user). +// +// Caveats: the compositor applies pointer acceleration to relative motion, and +// typed text goes through whatever keyboard layout the compositor uses; set +// XKB_DEFAULT_LAYOUT (and _VARIANT) if it isn't "us". + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "alvin/types.hpp" +#include "backends.hpp" +#include "keycodes.hpp" +#include "util.hpp" +#include "xkb.hpp" + +namespace alvin::linux_impl { +namespace { + +constexpr int kAbsMax = 65535; + +Status ioctlOk(int rc, const char* what) { + if (rc < 0) return fail(ErrorCode::Io, errnoMessage(what)); + return {}; +} + +Result openUinput() { + UniqueFd fd(::open("/dev/uinput", O_WRONLY | O_NONBLOCK | O_CLOEXEC)); + if (!fd) { + auto code = (errno == EACCES || errno == EPERM) ? ErrorCode::PermissionDenied : ErrorCode::NotAvailable; + return fail(code, errnoMessage("open /dev/uinput")); + } + return fd; +} + +Status finishDevice(int fd, const char* name, bool absolute) { + uinput_setup setup{}; + setup.id.bustype = BUS_VIRTUAL; + setup.id.vendor = 0x414c; // "AL" + setup.id.product = absolute ? 0x5601 : 0x5600; + std::snprintf(setup.name, UINPUT_MAX_NAME_SIZE, "%s", name); + + if (absolute) { + for (int axis : {ABS_X, ABS_Y}) { + uinput_abs_setup abs{}; + abs.code = static_cast<__u16>(axis); + abs.absinfo.minimum = 0; + abs.absinfo.maximum = kAbsMax; + if (auto s = ioctlOk(ioctl(fd, UI_ABS_SETUP, &abs), "UI_ABS_SETUP"); !s) return s; + } + } + if (auto s = ioctlOk(ioctl(fd, UI_DEV_SETUP, &setup), "UI_DEV_SETUP"); !s) return s; + return ioctlOk(ioctl(fd, UI_DEV_CREATE), "UI_DEV_CREATE"); +} + +class UInput final : public Input { +public: + static Result> create() { + auto keymap = XkbKeymap::fromNames(); + if (!keymap) return std::unexpected(keymap.error()); + + std::unique_ptr in(new UInput(std::move(*keymap))); + if (auto s = in->init(); !s) return std::unexpected(s.error()); + return in; + } + + ~UInput() override { + for (int fd : {keyboard_.get(), pointer_.get()}) + if (fd >= 0) ioctl(fd, UI_DEV_DESTROY); + } + + InputBackend backend() const override { return InputBackend::UInput; } + + Status moveTo(Point p) override { + if (desktop_.empty()) { + auto box = desktopBounds(); + if (!box) return fail(ErrorCode::Unsupported, "absolute motion needs the monitor layout: " + box.error().message); + desktop_ = *box; + } + // The compositor maps the device's absolute range onto the whole layout. + auto scale = [](int v, int origin, int extent) { + long long rel = std::clamp(v - origin, 0, extent - 1); + return static_cast((rel * kAbsMax + extent / 2) / std::max(1, extent - 1)); + }; + emit(pointer_.get(), EV_ABS, ABS_X, scale(p.x, desktop_.x, desktop_.width)); + emit(pointer_.get(), EV_ABS, ABS_Y, scale(p.y, desktop_.y, desktop_.height)); + return syn(pointer_.get()); + } + + Status moveBy(int dx, int dy) override { + if (dx) emit(keyboard_.get(), EV_REL, REL_X, dx); + if (dy) emit(keyboard_.get(), EV_REL, REL_Y, dy); + return syn(keyboard_.get()); + } + + Status buttonDown(MouseButton b) override { + emit(pointer_.get(), EV_KEY, static_cast(toEvdev(b)), 1); + return syn(pointer_.get()); + } + + Status buttonUp(MouseButton b) override { + emit(pointer_.get(), EV_KEY, static_cast(toEvdev(b)), 0); + return syn(pointer_.get()); + } + + Status scroll(int dx, int dy) override { + // REL_WHEEL is positive for "up"; Alvin's dy is positive for "down". + if (dy) { + emit(pointer_.get(), EV_REL, REL_WHEEL, -dy); + emit(pointer_.get(), EV_REL, REL_WHEEL_HI_RES, -dy * 120); + } + if (dx) { + emit(pointer_.get(), EV_REL, REL_HWHEEL, dx); + emit(pointer_.get(), EV_REL, REL_HWHEEL_HI_RES, dx * 120); + } + return syn(pointer_.get()); + } + + Status keyDown(Key k) override { return rawKey(toEvdev(k), true); } + Status keyUp(Key k) override { return rawKey(toEvdev(k), false); } + + Status typeText(std::string_view utf8) override { + auto text = decodeUtf8(utf8); + if (!text) return std::unexpected(text.error()); + auto strokes = keymap_.strokesFor(*text); + if (!strokes) return std::unexpected(strokes.error()); + for (const auto& s : *strokes) { + for (auto m : s.modifiers) + if (auto r = rawKey(m, true); !r) return r; + auto r = rawKey(s.key, true); + if (r) r = rawKey(s.key, false); + for (auto m : s.modifiers) (void)rawKey(m, false); + if (!r) return r; + pause(); + } + return {}; + } + +private: + explicit UInput(XkbKeymap keymap) : keymap_(std::move(keymap)) {} + + Status init() { + auto kb = openUinput(); + if (!kb) return std::unexpected(kb.error()); + keyboard_ = std::move(*kb); + int fd = keyboard_.get(); + // Keyboard + relative mouse, like a wireless combo receiver. + if (auto s = ioctlOk(ioctl(fd, UI_SET_EVBIT, EV_KEY), "UI_SET_EVBIT"); !s) return s; + for (int code = KEY_ESC; code <= KEY_MICMUTE; ++code) ioctl(fd, UI_SET_KEYBIT, code); + for (int code = BTN_LEFT; code <= BTN_TASK; ++code) ioctl(fd, UI_SET_KEYBIT, code); + ioctl(fd, UI_SET_EVBIT, EV_REL); + ioctl(fd, UI_SET_RELBIT, REL_X); + ioctl(fd, UI_SET_RELBIT, REL_Y); + if (auto s = finishDevice(fd, "Alvin virtual keyboard", false); !s) return s; + + // Absolute pointer (like QEMU's USB tablet): buttons + wheel + ABS_X/Y. + auto ptr = openUinput(); + if (!ptr) return std::unexpected(ptr.error()); + pointer_ = std::move(*ptr); + fd = pointer_.get(); + ioctl(fd, UI_SET_EVBIT, EV_KEY); + for (int code = BTN_LEFT; code <= BTN_TASK; ++code) ioctl(fd, UI_SET_KEYBIT, code); + ioctl(fd, UI_SET_EVBIT, EV_REL); + for (int code : {REL_WHEEL, REL_HWHEEL, REL_WHEEL_HI_RES, REL_HWHEEL_HI_RES}) ioctl(fd, UI_SET_RELBIT, code); + ioctl(fd, UI_SET_EVBIT, EV_ABS); + ioctl(fd, UI_SET_ABSBIT, ABS_X); + ioctl(fd, UI_SET_ABSBIT, ABS_Y); + ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_POINTER); + if (auto s = finishDevice(fd, "Alvin virtual pointer", true); !s) return s; + + // Events written before the compositor (via udev and libinput) adopts the + // devices are silently lost. That can take seconds, so wait until some other + // process has opened each event node, falling back to a fixed wait. + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(4); + for (int dev : {keyboard_.get(), pointer_.get()}) waitForConsumer(dev, deadline); + // The consumer opens the node slightly before it starts reading it. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return {}; + } + + static std::filesystem::path eventNode(int uinputFd) { + namespace fs = std::filesystem; + char sysname[64] = {}; + if (ioctl(uinputFd, UI_GET_SYSNAME(sizeof sysname), sysname) < 0) return {}; + std::error_code ec; + for (const auto& entry : fs::directory_iterator(fs::path("/sys/devices/virtual/input") / sysname, ec)) + if (entry.path().filename().string().starts_with("event")) return fs::path("/dev/input") / entry.path().filename(); + return {}; + } + + // Is `node` open in any process other than ours? Only processes we may + // inspect (same user) are seen, which covers Wayland compositors. + static bool openedElsewhere(const std::filesystem::path& node) { + namespace fs = std::filesystem; + const std::string self = std::to_string(::getpid()); + std::error_code ec; + for (const auto& proc : fs::directory_iterator("/proc", ec)) { + const std::string pid = proc.path().filename().string(); + if (pid.empty() || !std::isdigit(static_cast(pid[0])) || pid == self) continue; + std::error_code fdEc; + for (const auto& fd : fs::directory_iterator(proc.path() / "fd", fdEc)) { + std::error_code linkEc; + if (fs::read_symlink(fd.path(), linkEc) == node) return true; + } + } + return false; + } + + static void waitForConsumer(int uinputFd, std::chrono::steady_clock::time_point deadline) { + auto node = eventNode(uinputFd); + if (node.empty()) { + std::this_thread::sleep_until(deadline); + return; + } + while (std::chrono::steady_clock::now() < deadline && !openedElsewhere(node)) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + Status rawKey(std::uint32_t code, bool down) { + if (code == 0) return fail(ErrorCode::InvalidArgument, "key has no Linux mapping"); + emit(keyboard_.get(), EV_KEY, static_cast(code), down ? 1 : 0); + return syn(keyboard_.get()); + } + + void emit(int fd, int type, int code, int value) { + input_event ev{}; + ev.type = static_cast<__u16>(type); + ev.code = static_cast<__u16>(code); + ev.value = value; + if (::write(fd, &ev, sizeof ev) != static_cast(sizeof ev) && writeError_.empty()) + writeError_ = errnoMessage("write /dev/uinput"); + } + + Status syn(int fd) { + emit(fd, EV_SYN, SYN_REPORT, 0); + if (!writeError_.empty()) return fail(ErrorCode::Io, std::exchange(writeError_, {})); + return {}; + } + + XkbKeymap keymap_; + UniqueFd keyboard_; + UniqueFd pointer_; + Rect desktop_; + std::string writeError_; +}; + +} // namespace + +Result> createUInput(const InputOptions&) { return UInput::create(); } + +} // namespace alvin::linux_impl diff --git a/src/linux/util.hpp b/src/linux/util.hpp new file mode 100644 index 0000000..c47afb6 --- /dev/null +++ b/src/linux/util.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "alvin/error.hpp" + +namespace alvin::linux_impl { + +class UniqueFd { +public: + UniqueFd() = default; + explicit UniqueFd(int fd) : fd_(fd) {} + UniqueFd(UniqueFd&& o) noexcept : fd_(std::exchange(o.fd_, -1)) {} + UniqueFd& operator=(UniqueFd&& o) noexcept { + if (this != &o) reset(std::exchange(o.fd_, -1)); + return *this; + } + ~UniqueFd() { reset(); } + + int get() const { return fd_; } + int release() { return std::exchange(fd_, -1); } + explicit operator bool() const { return fd_ >= 0; } + void reset(int fd = -1) { + if (fd_ >= 0) ::close(fd_); + fd_ = fd; + } + +private: + int fd_ = -1; +}; + +inline std::string errnoMessage(std::string_view what) { + return std::string(what) + ": " + std::strerror(errno); +} + +// Anonymous memfd holding `contents` plus a terminating NUL (what Wayland keymaps expect). +inline Result memfdWithContents(std::string_view contents) { + UniqueFd fd(memfd_create("alvin", MFD_CLOEXEC | MFD_ALLOW_SEALING)); + if (!fd) return fail(ErrorCode::Io, errnoMessage("memfd_create")); + std::size_t total = contents.size() + 1; + if (ftruncate(fd.get(), static_cast(total)) < 0) return fail(ErrorCode::Io, errnoMessage("ftruncate")); + void* map = mmap(nullptr, total, PROT_WRITE, MAP_SHARED, fd.get(), 0); + if (map == MAP_FAILED) return fail(ErrorCode::Io, errnoMessage("mmap")); + std::memcpy(map, contents.data(), contents.size()); + static_cast(map)[contents.size()] = '\0'; + munmap(map, total); + return fd; +} + +inline std::uint32_t monotonicMs() { + using namespace std::chrono; + return static_cast(duration_cast(steady_clock::now().time_since_epoch()).count()); +} + +inline std::uint64_t monotonicUs() { + using namespace std::chrono; + return static_cast(duration_cast(steady_clock::now().time_since_epoch()).count()); +} + +} // namespace alvin::linux_impl diff --git a/src/linux/wayland.cpp b/src/linux/wayland.cpp new file mode 100644 index 0000000..91ed2db --- /dev/null +++ b/src/linux/wayland.cpp @@ -0,0 +1,200 @@ +#include "wayland.hpp" + +#include +#include +#include + +#include + +#include "virtual-keyboard-unstable-v1-client-protocol.h" +#include "wlr-virtual-pointer-unstable-v1-client-protocol.h" +#include "xdg-output-unstable-v1-client-protocol.h" + +namespace alvin::linux_impl { +namespace { + +using Output = WaylandConnection::Output; + +// ---- wl_output ---- + +void outputGeometry(void* data, wl_output*, int32_t x, int32_t y, int32_t, int32_t, int32_t, const char*, + const char*, int32_t) { + auto* o = static_cast(data); + o->physicalX = x; + o->physicalY = y; +} + +void outputMode(void* data, wl_output*, uint32_t flags, int32_t w, int32_t h, int32_t) { + if (flags & WL_OUTPUT_MODE_CURRENT) static_cast(data)->info.pixelSize = {w, h}; +} + +void outputDone(void*, wl_output*) {} + +void outputScale(void* data, wl_output*, int32_t factor) { static_cast(data)->info.scale = factor; } + +void outputName(void* data, wl_output*, const char* name) { static_cast(data)->info.name = name; } + +void outputDescription(void* data, wl_output*, const char* desc) { + static_cast(data)->info.description = desc; +} + +const wl_output_listener kOutputListener = { + .geometry = outputGeometry, + .mode = outputMode, + .done = outputDone, + .scale = outputScale, + .name = outputName, + .description = outputDescription, +}; + +// ---- zxdg_output_v1 (logical position/size; handles fractional scaling) ---- + +void xdgLogicalPosition(void* data, zxdg_output_v1*, int32_t x, int32_t y) { + auto* o = static_cast(data); + o->info.bounds.x = x; + o->info.bounds.y = y; + o->haveLogical = true; +} + +void xdgLogicalSize(void* data, zxdg_output_v1*, int32_t w, int32_t h) { + auto* o = static_cast(data); + o->info.bounds.width = w; + o->info.bounds.height = h; +} + +void xdgDone(void*, zxdg_output_v1*) {} + +void xdgName(void* data, zxdg_output_v1*, const char* name) { + auto* o = static_cast(data); + if (o->info.name.empty()) o->info.name = name; +} + +void xdgDescription(void* data, zxdg_output_v1*, const char* desc) { + auto* o = static_cast(data); + if (o->info.description.empty()) o->info.description = desc; +} + +const zxdg_output_v1_listener kXdgOutputListener = { + .logical_position = xdgLogicalPosition, + .logical_size = xdgLogicalSize, + .done = xdgDone, + .name = xdgName, + .description = xdgDescription, +}; + +const wl_registry_listener kRegistryListener = { + .global = [](void* d, wl_registry* r, uint32_t n, const char* i, uint32_t v) { + WaylandConnection::onGlobal(d, r, n, i, v); + }, + .global_remove = [](void* d, wl_registry* r, uint32_t n) { WaylandConnection::onGlobalRemove(d, r, n); }, +}; + +} // namespace + +Result> WaylandConnection::connect() { + std::unique_ptr c(new WaylandConnection); + c->display_ = wl_display_connect(nullptr); + if (!c->display_) return fail(ErrorCode::NotAvailable, "cannot connect to Wayland display (is WAYLAND_DISPLAY set?)"); + c->registry_ = wl_display_get_registry(c->display_); + wl_registry_add_listener(c->registry_, &kRegistryListener, c.get()); + // First roundtrip: globals. Second: output/xdg-output state events. + if (auto s = c->roundtrip(); !s) return std::unexpected(s.error()); + if (c->xdgOutputManager_) + for (auto& o : c->outputs_) c->bindXdgOutput(*o); + if (auto s = c->roundtrip(); !s) return std::unexpected(s.error()); + return c; +} + +WaylandConnection::~WaylandConnection() { + for (auto& o : outputs_) { + if (o->xdgOutput) zxdg_output_v1_destroy(o->xdgOutput); + if (o->output) wl_output_release(o->output); + } + if (virtualPointerManager_) zwlr_virtual_pointer_manager_v1_destroy(virtualPointerManager_); + if (virtualKeyboardManager_) zwp_virtual_keyboard_manager_v1_destroy(virtualKeyboardManager_); + if (xdgOutputManager_) zxdg_output_manager_v1_destroy(xdgOutputManager_); + if (seat_) wl_seat_release(seat_); + if (registry_) wl_registry_destroy(registry_); + if (display_) wl_display_disconnect(display_); +} + +void WaylandConnection::onGlobal(void* data, wl_registry* registry, uint32_t name, const char* interface, + uint32_t version) { + auto* self = static_cast(data); + std::string_view iface = interface; + if (iface == wl_output_interface.name && version >= 4) { + auto o = std::make_unique(); + o->owner = self; + o->globalName = name; + o->output = static_cast(wl_registry_bind(registry, name, &wl_output_interface, 4)); + wl_output_add_listener(o->output, &kOutputListener, o.get()); + if (self->xdgOutputManager_) self->bindXdgOutput(*o); + self->outputs_.push_back(std::move(o)); + } else if (iface == wl_seat_interface.name && !self->seat_) { + self->seat_ = static_cast(wl_registry_bind(registry, name, &wl_seat_interface, std::min(version, 5u))); + } else if (iface == zxdg_output_manager_v1_interface.name) { + self->xdgOutputManager_ = static_cast( + wl_registry_bind(registry, name, &zxdg_output_manager_v1_interface, std::min(version, 3u))); + } else if (iface == zwlr_virtual_pointer_manager_v1_interface.name) { + self->virtualPointerManager_ = static_cast( + wl_registry_bind(registry, name, &zwlr_virtual_pointer_manager_v1_interface, std::min(version, 2u))); + } else if (iface == zwp_virtual_keyboard_manager_v1_interface.name) { + self->virtualKeyboardManager_ = static_cast( + wl_registry_bind(registry, name, &zwp_virtual_keyboard_manager_v1_interface, 1)); + } +} + +void WaylandConnection::onGlobalRemove(void* data, wl_registry*, uint32_t name) { + auto* self = static_cast(data); + std::erase_if(self->outputs_, [&](const std::unique_ptr& o) { + if (o->globalName != name) return false; + if (o->xdgOutput) zxdg_output_v1_destroy(o->xdgOutput); + wl_output_release(o->output); + return true; + }); +} + +void WaylandConnection::bindXdgOutput(Output& o) { + if (o.xdgOutput) return; + o.xdgOutput = zxdg_output_manager_v1_get_xdg_output(xdgOutputManager_, o.output); + zxdg_output_v1_add_listener(o.xdgOutput, &kXdgOutputListener, &o); +} + +std::vector WaylandConnection::monitors() const { + std::vector out; + for (const auto& o : outputs_) { + Monitor m = o->info; + if (!o->haveLogical) { + // No xdg-output: derive logical size from the mode and integer scale. + m.bounds = {o->physicalX, o->physicalY, m.pixelSize.width / std::max(1, static_cast(m.scale)), + m.pixelSize.height / std::max(1, static_cast(m.scale))}; + } else if (m.bounds.width > 0) { + m.scale = static_cast(m.pixelSize.width) / m.bounds.width; + } + out.push_back(std::move(m)); + } + std::sort(out.begin(), out.end(), [](const Monitor& a, const Monitor& b) { + return a.bounds.x != b.bounds.x ? a.bounds.x < b.bounds.x : a.bounds.y < b.bounds.y; + }); + // Wayland has no notion of a primary output; call the one at the origin primary. + for (auto& m : out) + if (m.bounds.x == 0 && m.bounds.y == 0) m.primary = true; + return out; +} + +Status WaylandConnection::roundtrip() { + if (wl_display_roundtrip(display_) < 0) + return fail(ErrorCode::Protocol, std::string("Wayland roundtrip failed: ") + std::strerror(errno)); + return {}; +} + +Status WaylandConnection::flush() { + while (wl_display_flush(display_) < 0) { + if (errno != EAGAIN) return fail(ErrorCode::Io, std::string("Wayland flush failed: ") + std::strerror(errno)); + // Socket buffer full: let the compositor drain it. + if (wl_display_roundtrip(display_) < 0) return fail(ErrorCode::Protocol, "Wayland roundtrip failed"); + } + return {}; +} + +} // namespace alvin::linux_impl diff --git a/src/linux/wayland.hpp b/src/linux/wayland.hpp new file mode 100644 index 0000000..0641daa --- /dev/null +++ b/src/linux/wayland.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include + +#include "alvin/error.hpp" +#include "alvin/types.hpp" + +struct wl_display; +struct wl_registry; +struct wl_seat; +struct wl_output; +struct zxdg_output_manager_v1; +struct zxdg_output_v1; +struct zwlr_virtual_pointer_manager_v1; +struct zwp_virtual_keyboard_manager_v1; + +namespace alvin::linux_impl { + +// A client connection to the Wayland compositor with the globals Alvin cares about bound. +class WaylandConnection { +public: + static Result> connect(); + ~WaylandConnection(); + + wl_display* display() const { return display_; } + wl_seat* seat() const { return seat_; } + zwlr_virtual_pointer_manager_v1* virtualPointerManager() const { return virtualPointerManager_; } + zwp_virtual_keyboard_manager_v1* virtualKeyboardManager() const { return virtualKeyboardManager_; } + + // Logical layout as reported by wl_output + xdg_output at connect time. + std::vector monitors() const; + + Status roundtrip(); + Status flush(); + + struct Output { + WaylandConnection* owner = nullptr; + std::uint32_t globalName = 0; + wl_output* output = nullptr; + zxdg_output_v1* xdgOutput = nullptr; + Monitor info; + bool haveLogical = false; + int physicalX = 0, physicalY = 0; + }; + + // Registry callbacks (public so the C listener table can reach them). + static void onGlobal(void* data, wl_registry* registry, std::uint32_t name, const char* interface, + std::uint32_t version); + static void onGlobalRemove(void* data, wl_registry* registry, std::uint32_t name); + +private: + WaylandConnection() = default; + void bindXdgOutput(Output& o); + + wl_display* display_ = nullptr; + wl_registry* registry_ = nullptr; + wl_seat* seat_ = nullptr; + zxdg_output_manager_v1* xdgOutputManager_ = nullptr; + zwlr_virtual_pointer_manager_v1* virtualPointerManager_ = nullptr; + zwp_virtual_keyboard_manager_v1* virtualKeyboardManager_ = nullptr; + std::vector> outputs_; +}; + +} // namespace alvin::linux_impl diff --git a/src/linux/wlr_input.cpp b/src/linux/wlr_input.cpp new file mode 100644 index 0000000..b1aa070 --- /dev/null +++ b/src/linux/wlr_input.cpp @@ -0,0 +1,196 @@ +// Input through the zwlr_virtual_pointer_v1 / zwp_virtual_keyboard_v1 Wayland +// protocols. Supported by Hyprland, sway and most wlroots compositors, and needs +// no permission dialog. + +#include +#include + +#include +#include +#include + +#include "backends.hpp" +#include "keycodes.hpp" +#include "util.hpp" +#include "virtual-keyboard-unstable-v1-client-protocol.h" +#include "wayland.hpp" +#include "wlr-virtual-pointer-unstable-v1-client-protocol.h" +#include "xkb.hpp" + +namespace alvin::linux_impl { +namespace { + +struct XkbStateDeleter { + void operator()(xkb_state* s) const { xkb_state_unref(s); } +}; + +class WlrInput final : public Input { +public: + static Result> create() { + auto conn = WaylandConnection::connect(); + if (!conn) return std::unexpected(conn.error()); + auto& wl = **conn; + if (!wl.seat()) return fail(ErrorCode::NotAvailable, "compositor has no wl_seat"); + if (!wl.virtualPointerManager() || !wl.virtualKeyboardManager()) + return fail(ErrorCode::NotAvailable, + "compositor lacks zwlr_virtual_pointer_manager_v1 / zwp_virtual_keyboard_manager_v1"); + + auto keymap = XkbKeymap::fromNames(); + if (!keymap) return std::unexpected(keymap.error()); + + std::unique_ptr in(new WlrInput(std::move(*conn), std::move(*keymap))); + if (auto s = in->init(); !s) return std::unexpected(s.error()); + return in; + } + + ~WlrInput() override { + if (keyboard_) zwp_virtual_keyboard_v1_destroy(keyboard_); + if (pointer_) zwlr_virtual_pointer_v1_destroy(pointer_); + if (wl_) (void)wl_->roundtrip(); + } + + InputBackend backend() const override { return InputBackend::WlrVirtual; } + + Status moveTo(Point p) override { + // Absolute motion is expressed as a fraction of the whole output layout. + Rect box = desktop_; + std::uint32_t x = static_cast(std::clamp(p.x - box.x, 0, box.width - 1)); + std::uint32_t y = static_cast(std::clamp(p.y - box.y, 0, box.height - 1)); + zwlr_virtual_pointer_v1_motion_absolute(pointer_, monotonicMs(), x, y, static_cast(box.width), + static_cast(box.height)); + zwlr_virtual_pointer_v1_frame(pointer_); + return sync(); + } + + Status moveBy(int dx, int dy) override { + zwlr_virtual_pointer_v1_motion(pointer_, monotonicMs(), wl_fixed_from_int(dx), wl_fixed_from_int(dy)); + zwlr_virtual_pointer_v1_frame(pointer_); + return sync(); + } + + Status buttonDown(MouseButton b) override { return button(b, WL_POINTER_BUTTON_STATE_PRESSED); } + Status buttonUp(MouseButton b) override { return button(b, WL_POINTER_BUTTON_STATE_RELEASED); } + + Status scroll(int dx, int dy) override { + std::uint32_t t = monotonicMs(); + zwlr_virtual_pointer_v1_axis_source(pointer_, WL_POINTER_AXIS_SOURCE_WHEEL); + // 15 is the conventional per-notch distance (matches libinput / wlroots). + if (dy) zwlr_virtual_pointer_v1_axis_discrete(pointer_, t, WL_POINTER_AXIS_VERTICAL_SCROLL, + wl_fixed_from_int(15 * dy), dy); + if (dx) zwlr_virtual_pointer_v1_axis_discrete(pointer_, t, WL_POINTER_AXIS_HORIZONTAL_SCROLL, + wl_fixed_from_int(15 * dx), dx); + zwlr_virtual_pointer_v1_frame(pointer_); + return sync(); + } + + Status keyDown(Key key) override { return this->key(key, true); } + Status keyUp(Key key) override { return this->key(key, false); } + + Status typeText(std::string_view utf8) override { + auto text = decodeUtf8(utf8); + if (!text) return std::unexpected(text.error()); + + // Upload a keymap containing exactly the characters we need, type them + // with no modifiers, then switch back. Chunked because XKB caps keycodes at 255. + std::size_t pos = 0; + while (pos < text->size()) { + std::u32string alphabet; + std::size_t end = pos; + for (; end < text->size(); ++end) { + if (alphabet.find((*text)[end]) != std::u32string::npos) continue; + if (alphabet.size() == 247) break; + alphabet.push_back((*text)[end]); + } + auto source = synthesizeKeymap(alphabet); + if (!source) return std::unexpected(source.error()); + if (auto s = uploadKeymap(*source); !s) return s; + zwp_virtual_keyboard_v1_modifiers(keyboard_, 0, 0, 0, 0); + + for (std::size_t i = pos; i < end; ++i) { + auto code = static_cast(alphabet.find((*text)[i]) + 1); + zwp_virtual_keyboard_v1_key(keyboard_, monotonicMs(), code, WL_KEYBOARD_KEY_STATE_PRESSED); + zwp_virtual_keyboard_v1_key(keyboard_, monotonicMs(), code, WL_KEYBOARD_KEY_STATE_RELEASED); + if (auto s = sync(); !s) return s; + pause(); + } + pos = end; + } + if (auto s = uploadKeymap(baseKeymapSource_); !s) return s; + sendModifiers(); + return sync(); + } + +private: + WlrInput(std::unique_ptr wl, XkbKeymap keymap) : wl_(std::move(wl)), keymap_(std::move(keymap)) {} + + Status init() { + for (const auto& m : wl_->monitors()) desktop_ = desktop_.united(m.bounds); + if (desktop_.empty()) return fail(ErrorCode::NotAvailable, "no outputs reported by the compositor"); + + pointer_ = zwlr_virtual_pointer_manager_v1_create_virtual_pointer(wl_->virtualPointerManager(), wl_->seat()); + keyboard_ = zwp_virtual_keyboard_manager_v1_create_virtual_keyboard(wl_->virtualKeyboardManager(), wl_->seat()); + + char* text = xkb_keymap_get_as_string(keymap_.get(), XKB_KEYMAP_FORMAT_TEXT_V1); + if (!text) return fail(ErrorCode::Unknown, "cannot serialise XKB keymap"); + baseKeymapSource_ = text; + std::free(text); + state_.reset(xkb_state_new(keymap_.get())); + + if (auto s = uploadKeymap(baseKeymapSource_); !s) return s; + // The compositor sends a protocol error here if we're not allowed to use these. + return wl_->roundtrip(); + } + + Status uploadKeymap(const std::string& source) { + auto fd = memfdWithContents(source); + if (!fd) return std::unexpected(fd.error()); + zwp_virtual_keyboard_v1_keymap(keyboard_, WL_KEYBOARD_KEYMAP_FORMAT_XKB_V1, fd->get(), + static_cast(source.size() + 1)); + return sync(); + } + + Status button(MouseButton b, wl_pointer_button_state state) { + zwlr_virtual_pointer_v1_button(pointer_, monotonicMs(), toEvdev(b), state); + zwlr_virtual_pointer_v1_frame(pointer_); + return sync(); + } + + Status key(Key k, bool down) { + std::uint32_t code = toEvdev(k); + if (code == 0) return fail(ErrorCode::InvalidArgument, "key has no Linux mapping"); + zwp_virtual_keyboard_v1_key(keyboard_, monotonicMs(), code, + down ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED); + // Virtual keyboards must report modifier state themselves; track it with XKB. + xkb_state_update_key(state_.get(), code + 8, down ? XKB_KEY_DOWN : XKB_KEY_UP); + sendModifiers(); + return sync(); + } + + void sendModifiers() { + zwp_virtual_keyboard_v1_modifiers(keyboard_, xkb_state_serialize_mods(state_.get(), XKB_STATE_MODS_DEPRESSED), + xkb_state_serialize_mods(state_.get(), XKB_STATE_MODS_LATCHED), + xkb_state_serialize_mods(state_.get(), XKB_STATE_MODS_LOCKED), + xkb_state_serialize_layout(state_.get(), XKB_STATE_LAYOUT_EFFECTIVE)); + } + + // Flush and wait for the compositor to process everything, so a following + // capture or IPC query observes the effect. + Status sync() { + if (auto s = wl_->flush(); !s) return s; + return wl_->roundtrip(); + } + + std::unique_ptr wl_; + XkbKeymap keymap_; + std::unique_ptr state_; + std::string baseKeymapSource_; + zwlr_virtual_pointer_v1* pointer_ = nullptr; + zwp_virtual_keyboard_v1* keyboard_ = nullptr; + Rect desktop_; +}; + +} // namespace + +Result> createWlrInput(const InputOptions&) { return WlrInput::create(); } + +} // namespace alvin::linux_impl diff --git a/src/linux/xkb.cpp b/src/linux/xkb.cpp new file mode 100644 index 0000000..5569e90 --- /dev/null +++ b/src/linux/xkb.cpp @@ -0,0 +1,150 @@ +#include "xkb.hpp" + +#include +#include +#include + +namespace alvin::linux_impl { + +void XkbKeymap::Deleter::operator()(xkb_keymap* k) const { xkb_keymap_unref(k); } + +static std::shared_ptr newContext() { + xkb_context* ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS); + if (!ctx) return nullptr; + return {ctx, [](xkb_context* c) { xkb_context_unref(c); }}; +} + +Result XkbKeymap::fromNames(const std::string& layout, const std::string& variant, + const std::string& options) { + XkbKeymap km; + km.context_ = newContext(); + if (!km.context_) return fail(ErrorCode::Unknown, "xkb_context_new failed"); + xkb_rule_names names{}; + if (!layout.empty()) names.layout = layout.c_str(); + if (!variant.empty()) names.variant = variant.c_str(); + if (!options.empty()) names.options = options.c_str(); + km.keymap_.reset(xkb_keymap_new_from_names(km.context_.get(), &names, XKB_KEYMAP_COMPILE_NO_FLAGS)); + if (!km.keymap_) return fail(ErrorCode::InvalidArgument, "cannot compile XKB keymap for layout '" + layout + "'"); + return km; +} + +Result XkbKeymap::fromString(std::string_view text) { + XkbKeymap km; + km.context_ = newContext(); + if (!km.context_) return fail(ErrorCode::Unknown, "xkb_context_new failed"); + km.keymap_.reset(xkb_keymap_new_from_buffer(km.context_.get(), text.data(), text.size(), + XKB_KEYMAP_FORMAT_TEXT_V1, XKB_KEYMAP_COMPILE_NO_FLAGS)); + if (!km.keymap_) return fail(ErrorCode::Protocol, "cannot parse XKB keymap"); + return km; +} + +std::uint32_t keysymFor(char32_t cp) { + switch (cp) { + case U'\n': case U'\r': return XKB_KEY_Return; + case U'\t': return XKB_KEY_Tab; + case U'\b': return XKB_KEY_BackSpace; + case 0x1b: return XKB_KEY_Escape; + default: return xkb_utf32_to_keysym(cp); + } +} + +std::optional XkbKeymap::strokeFor(char32_t cp) const { + xkb_keysym_t want = keysymFor(cp); + if (want == XKB_KEY_NoSymbol) return std::nullopt; + + xkb_keymap* km = keymap_.get(); + const xkb_mod_index_t shift = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_SHIFT); + // AltGr ("ISO_Level3_Shift") is bound to Mod5 in the stock XKB data. + const xkb_mod_index_t level3 = xkb_keymap_mod_get_index(km, "Mod5"); + + std::optional best; + int bestMods = 99; + const xkb_layout_index_t layout = 0; + for (xkb_keycode_t kc = xkb_keymap_min_keycode(km); kc <= xkb_keymap_max_keycode(km); ++kc) { + xkb_level_index_t levels = xkb_keymap_num_levels_for_key(km, kc, layout); + for (xkb_level_index_t level = 0; level < levels; ++level) { + const xkb_keysym_t* syms = nullptr; + int n = xkb_keymap_key_get_syms_by_level(km, kc, layout, level, &syms); + if (n != 1 || syms[0] != want) continue; + + xkb_mod_mask_t masks[16]; + std::size_t nmasks = xkb_keymap_key_get_mods_for_level(km, kc, layout, level, masks, 16); + for (std::size_t i = 0; i < nmasks; ++i) { + xkb_mod_mask_t mask = masks[i]; + Stroke s{kc - 8, {}}; + if (shift != XKB_MOD_INVALID && (mask & (1u << shift))) { + s.modifiers.push_back(KEY_LEFTSHIFT); + mask &= ~(1u << shift); + } + if (level3 != XKB_MOD_INVALID && (mask & (1u << level3))) { + s.modifiers.push_back(KEY_RIGHTALT); + mask &= ~(1u << level3); + } + if (mask != 0) continue; // needs a modifier we can't press (e.g. Lock) + int count = static_cast(s.modifiers.size()); + if (count < bestMods) { + best = std::move(s); + bestMods = count; + } + } + } + if (bestMods == 0) break; + } + return best; +} + +Result> XkbKeymap::strokesFor(std::u32string_view text) const { + std::vector out; + out.reserve(text.size()); + for (char32_t cp : text) { + auto s = strokeFor(cp); + if (!s) { + char buf[16]; + std::snprintf(buf, sizeof buf, "U+%04X", static_cast(cp)); + return fail(ErrorCode::Unsupported, std::string("character ") + buf + " is not on the active keyboard layout"); + } + out.push_back(std::move(*s)); + } + return out; +} + +Result synthesizeKeymap(std::u32string_view codepoints) { + // XKB keycodes are 8..255 and evdev keycode = xkb keycode - 8. + if (codepoints.size() > 247) return fail(ErrorCode::InvalidArgument, "too many distinct characters for one keymap"); + std::string keycodes, symbols; + for (std::size_t i = 0; i < codepoints.size(); ++i) { + char name[64]; + if (xkb_keysym_get_name(keysymFor(codepoints[i]), name, sizeof name) < 0) + return fail(ErrorCode::InvalidArgument, "no keysym for character"); + std::size_t xkbCode = i + 9; + keycodes += " = " + std::to_string(xkbCode) + ";\n"; + symbols += " key { [ " + name + " ] };\n"; + } + return "xkb_keymap {\n" + " xkb_keycodes \"alvin\" {\n minimum = 8;\n maximum = 255;\n" + keycodes + " };\n" + " xkb_types \"alvin\" { include \"complete\" };\n" + " xkb_compatibility \"alvin\" { include \"complete\" };\n" + " xkb_symbols \"alvin\" {\n" + symbols + " };\n" + "};\n"; +} + +Result decodeUtf8(std::string_view text) { + std::u32string out; + out.reserve(text.size()); + for (std::size_t i = 0; i < text.size();) { + auto c = static_cast(text[i]); + int len = c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xE ? 3 : (c >> 3) == 0x1E ? 4 : 0; + if (len == 0 || i + len > text.size()) return fail(ErrorCode::InvalidArgument, "invalid UTF-8"); + char32_t cp = len == 1 ? c : c & (0x7F >> len); + for (int k = 1; k < len; ++k) { + auto cc = static_cast(text[i + k]); + if ((cc & 0xC0) != 0x80) return fail(ErrorCode::InvalidArgument, "invalid UTF-8"); + cp = (cp << 6) | (cc & 0x3F); + } + out.push_back(cp); + i += len; + } + return out; +} + +} // namespace alvin::linux_impl diff --git a/src/linux/xkb.hpp b/src/linux/xkb.hpp new file mode 100644 index 0000000..b0a3765 --- /dev/null +++ b/src/linux/xkb.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "alvin/error.hpp" + +struct xkb_context; +struct xkb_keymap; + +namespace alvin::linux_impl { + +// One character's worth of key presses: hold `modifiers` (evdev codes), tap `key`. +struct Stroke { + std::uint32_t key = 0; + std::vector modifiers; +}; + +class XkbKeymap { +public: + // Empty strings fall back to XKB_DEFAULT_* environment variables, then "us". + static Result fromNames(const std::string& layout = {}, const std::string& variant = {}, + const std::string& options = {}); + static Result fromString(std::string_view text); + + // Which physical keys produce this code point on the first layout group. + std::optional strokeFor(char32_t codepoint) const; + + // Resolves every code point up front so we fail before typing anything. + Result> strokesFor(std::u32string_view text) const; + + xkb_keymap* get() const { return keymap_.get(); } + +private: + struct Deleter { + void operator()(xkb_keymap*) const; + }; + std::shared_ptr context_; + std::unique_ptr keymap_; +}; + +// Keymap source text where entry i of `codepoints` is produced by evdev keycode +// i + 1 with no modifiers. Used with the Wayland virtual keyboard to type any +// Unicode text regardless of the user's layout (the approach wtype uses). +Result synthesizeKeymap(std::u32string_view codepoints); + +std::uint32_t keysymFor(char32_t codepoint); + +Result decodeUtf8(std::string_view text); + +} // namespace alvin::linux_impl diff --git a/src/vision/vision.cpp b/src/vision/vision.cpp new file mode 100644 index 0000000..4113c9f --- /dev/null +++ b/src/vision/vision.cpp @@ -0,0 +1,102 @@ +#include "alvin/vision.hpp" + +#include + +#include +#include + +namespace alvin::vision { + +cv::Mat view(const Frame& frame) { + return cv::Mat(frame.height(), frame.width(), CV_8UC4, const_cast(frame.data()), frame.stride()); +} + +cv::Mat toBgr(const Frame& frame) { + cv::Mat out; + cv::cvtColor(view(frame), out, cv::COLOR_BGRA2BGR); + return out; +} + +Result fromMat(const cv::Mat& mat) { + if (mat.empty()) return fail(ErrorCode::InvalidArgument, "empty image"); + if (mat.depth() != CV_8U) return fail(ErrorCode::InvalidArgument, "only 8-bit images are supported"); + Frame frame(mat.cols, mat.rows); + cv::Mat dst = view(frame); + switch (mat.channels()) { + case 1: cv::cvtColor(mat, dst, cv::COLOR_GRAY2BGRA); break; + case 3: cv::cvtColor(mat, dst, cv::COLOR_BGR2BGRA); break; + case 4: mat.copyTo(dst); break; + default: return fail(ErrorCode::InvalidArgument, "unsupported channel count"); + } + return frame; +} + +Result loadImage(const std::filesystem::path& path) { + cv::Mat img = cv::imread(path.string(), cv::IMREAD_COLOR); + if (img.empty()) return fail(ErrorCode::Io, "cannot read image " + path.string()); + return img; +} + +Status saveImage(const Frame& frame, const std::filesystem::path& path) { + try { + if (!cv::imwrite(path.string(), view(frame))) return fail(ErrorCode::Io, "cannot write image " + path.string()); + } catch (const cv::Exception& e) { + return fail(ErrorCode::Io, e.what()); + } + return {}; +} + +namespace { + +// Brings both images to the same channel layout and returns the TM_CCOEFF_NORMED score map. +Result scoreMap(const Frame& haystack, const cv::Mat& needle, bool grayscale) { + if (needle.empty()) return fail(ErrorCode::InvalidArgument, "empty template"); + if (needle.cols > haystack.width() || needle.rows > haystack.height()) + return fail(ErrorCode::InvalidArgument, "template is larger than the frame"); + + cv::Mat hay, tpl; + if (grayscale) { + cv::cvtColor(view(haystack), hay, cv::COLOR_BGRA2GRAY); + if (needle.channels() == 1) tpl = needle; + else cv::cvtColor(needle, tpl, needle.channels() == 4 ? cv::COLOR_BGRA2GRAY : cv::COLOR_BGR2GRAY); + } else { + cv::cvtColor(view(haystack), hay, cv::COLOR_BGRA2BGR); + if (needle.channels() == 3) tpl = needle; + else cv::cvtColor(needle, tpl, needle.channels() == 4 ? cv::COLOR_BGRA2BGR : cv::COLOR_GRAY2BGR); + } + cv::Mat scores; + cv::matchTemplate(hay, tpl, scores, cv::TM_CCOEFF_NORMED); + return scores; +} + +} // namespace + +Result> find(const Frame& haystack, const cv::Mat& needle, const MatchOptions& opts) { + auto scores = scoreMap(haystack, needle, opts.grayscale); + if (!scores) return std::unexpected(scores.error()); + double best = 0; + cv::Point loc; + cv::minMaxLoc(*scores, nullptr, &best, nullptr, &loc); + if (best < opts.threshold) return std::optional{}; + return std::optional{Match{{loc.x, loc.y, needle.cols, needle.rows}, best}}; +} + +Result> findAll(const Frame& haystack, const cv::Mat& needle, const MatchOptions& opts) { + auto scores = scoreMap(haystack, needle, opts.grayscale); + if (!scores) return std::unexpected(scores.error()); + std::vector out; + // Greedy non-maximum suppression: take the best peak, blank out its neighbourhood, repeat. + while (static_cast(out.size()) < opts.maxResults) { + double best = 0; + cv::Point loc; + cv::minMaxLoc(*scores, nullptr, &best, nullptr, &loc); + if (best < opts.threshold) break; + out.push_back({{loc.x, loc.y, needle.cols, needle.rows}, best}); + cv::Rect suppress(loc.x - needle.cols / 2, loc.y - needle.rows / 2, needle.cols, needle.rows); + suppress &= cv::Rect(0, 0, scores->cols, scores->rows); + (*scores)(suppress).setTo(-1.0f); + } + return out; +} + +} // namespace alvin::vision diff --git a/src/windows/backends.hpp b/src/windows/backends.hpp new file mode 100644 index 0000000..13e13c8 --- /dev/null +++ b/src/windows/backends.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "alvin/capture.hpp" +#include "alvin/input.hpp" +#include "alvin/window.hpp" + +namespace alvin::win_impl { + +Result> createSendInput(const InputOptions& options); +Result> createGdiCapture(const CaptureOptions& options); +Result> createWin32Windows(); + +} // namespace alvin::win_impl diff --git a/src/windows/gdi_capture.cpp b/src/windows/gdi_capture.cpp new file mode 100644 index 0000000..1a6efdc --- /dev/null +++ b/src/windows/gdi_capture.cpp @@ -0,0 +1,128 @@ +// Capture with GDI: BitBlt for monitors, PrintWindow for individual windows. +// Simple and dependency-free. Hardware-accelerated or protected content may +// come back black; DXGI Desktop Duplication / Windows.Graphics.Capture are the +// planned faster alternatives. + +#include +#include + +#include "alvin/types.hpp" +#include "backends.hpp" +#include "win_util.hpp" + +namespace alvin::win_impl { +namespace { + +// Owns a screen-compatible memory DC with a top-down 32-bit DIB selected in. +class DibCanvas { +public: + DibCanvas(HDC reference, int w, int h) : width_(w), height_(h) { + dc_ = CreateCompatibleDC(reference); + BITMAPINFO bi{}; + bi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); + bi.bmiHeader.biWidth = w; + bi.bmiHeader.biHeight = -h; // top-down + bi.bmiHeader.biPlanes = 1; + bi.bmiHeader.biBitCount = 32; + bi.bmiHeader.biCompression = BI_RGB; + bitmap_ = CreateDIBSection(dc_, &bi, DIB_RGB_COLORS, &bits_, nullptr, 0); + if (bitmap_) previous_ = SelectObject(dc_, bitmap_); + } + ~DibCanvas() { + if (previous_) SelectObject(dc_, previous_); + if (bitmap_) DeleteObject(bitmap_); + if (dc_) DeleteDC(dc_); + } + DibCanvas(const DibCanvas&) = delete; + DibCanvas& operator=(const DibCanvas&) = delete; + + bool ok() const { return dc_ && bitmap_ && bits_; } + HDC dc() const { return dc_; } + + Frame toFrame(Rect desktop) const { + Frame f(width_, height_); + std::memcpy(f.data(), bits_, static_cast(width_) * height_ * 4); + // GDI leaves the alpha byte undefined; make the image opaque. + for (std::size_t i = 3; i < static_cast(width_) * height_ * 4; i += 4) f.data()[i] = 255; + f.setDesktopRect(desktop); + f.setTimestampNs(static_cast(GetTickCount64()) * 1'000'000); + return f; + } + +private: + int width_, height_; + HDC dc_ = nullptr; + HBITMAP bitmap_ = nullptr; + HGDIOBJ previous_ = nullptr; + void* bits_ = nullptr; +}; + +class GdiCapture final : public Capture { +public: + explicit GdiCapture(const CaptureOptions& o) : options_(o) {} + + CaptureBackend backend() const override { return CaptureBackend::Gdi; } + + Result grab(std::chrono::milliseconds) override { + return options_.window ? captureWindow(reinterpret_cast(options_.window->value)) : captureMonitor(); + } + + // GDI captures on demand, so every grab is a new frame. + Result nextFrame(std::chrono::milliseconds timeout) override { return grab(timeout); } + +private: + Result captureMonitor() { + auto monitors = listMonitors(); + if (!monitors) return std::unexpected(monitors.error()); + if (options_.monitorIndex < 0 || options_.monitorIndex >= static_cast(monitors->size())) + return fail(ErrorCode::NotFound, "monitor index out of range"); + Rect r = (*monitors)[options_.monitorIndex].bounds; + + HDC screen = GetDC(nullptr); + DibCanvas canvas(screen, r.width, r.height); + bool ok = canvas.ok() && BitBlt(canvas.dc(), 0, 0, r.width, r.height, screen, r.x, r.y, SRCCOPY | CAPTUREBLT); + ReleaseDC(nullptr, screen); + if (!ok) return fail(ErrorCode::Io, lastErrorMessage("BitBlt")); + if (options_.showCursor) drawCursor(canvas.dc(), r); + return canvas.toFrame(r); + } + + Result captureWindow(HWND hwnd) { + if (!IsWindow(hwnd)) return fail(ErrorCode::NotFound, "window no longer exists"); + RECT wr; + if (!GetWindowRect(hwnd, &wr)) return fail(ErrorCode::Io, lastErrorMessage("GetWindowRect")); + int w = wr.right - wr.left, h = wr.bottom - wr.top; + if (w <= 0 || h <= 0) return fail(ErrorCode::NotAvailable, "window has no visible area (minimised?)"); + + HDC screen = GetDC(nullptr); + DibCanvas canvas(screen, w, h); + ReleaseDC(nullptr, screen); + if (!canvas.ok()) return fail(ErrorCode::Io, lastErrorMessage("CreateDIBSection")); + // PW_RENDERFULLCONTENT (2) captures DirectComposition / GPU-rendered windows on Windows 8.1+. + if (!PrintWindow(hwnd, canvas.dc(), 2)) return fail(ErrorCode::Io, lastErrorMessage("PrintWindow")); + return canvas.toFrame({wr.left, wr.top, w, h}); + } + + static void drawCursor(HDC dc, Rect r) { + CURSORINFO ci{}; + ci.cbSize = sizeof ci; + if (!GetCursorInfo(&ci) || !(ci.flags & CURSOR_SHOWING)) return; + ICONINFO ii{}; + if (!GetIconInfo(ci.hCursor, &ii)) return; + DrawIconEx(dc, ci.ptScreenPos.x - r.x - static_cast(ii.xHotspot), + ci.ptScreenPos.y - r.y - static_cast(ii.yHotspot), ci.hCursor, 0, 0, 0, nullptr, DI_NORMAL); + if (ii.hbmMask) DeleteObject(ii.hbmMask); + if (ii.hbmColor) DeleteObject(ii.hbmColor); + } + + CaptureOptions options_; +}; + +} // namespace + +Result> createGdiCapture(const CaptureOptions& options) { + ensureDpiAware(); + return std::make_unique(options); +} + +} // namespace alvin::win_impl diff --git a/src/windows/platform_windows.cpp b/src/windows/platform_windows.cpp new file mode 100644 index 0000000..527949a --- /dev/null +++ b/src/windows/platform_windows.cpp @@ -0,0 +1,122 @@ +// Backend selection and shared helpers for Windows. + +#include +#include + +#include "../common/platform.hpp" +#include "backends.hpp" +#include "win_util.hpp" + +namespace alvin { +namespace win_impl { + +std::string narrow(std::wstring_view w) { + if (w.empty()) return {}; + int n = WideCharToMultiByte(CP_UTF8, 0, w.data(), static_cast(w.size()), nullptr, 0, nullptr, nullptr); + std::string out(static_cast(n), '\0'); + WideCharToMultiByte(CP_UTF8, 0, w.data(), static_cast(w.size()), out.data(), n, nullptr, nullptr); + return out; +} + +std::wstring widen(std::string_view s) { + if (s.empty()) return {}; + int n = MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), nullptr, 0); + std::wstring out(static_cast(n), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), out.data(), n); + return out; +} + +std::string lastErrorMessage(std::string_view what) { + DWORD code = GetLastError(); + wchar_t* buf = nullptr; + FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, + code, 0, reinterpret_cast(&buf), 0, nullptr); + std::string msg = std::string(what) + ": " + (buf ? narrow(buf) : "error " + std::to_string(code)); + if (buf) LocalFree(buf); + while (!msg.empty() && (msg.back() == '\n' || msg.back() == '\r')) msg.pop_back(); + return msg; +} + +void ensureDpiAware() { + static std::once_flag once; + std::call_once(once, [] { + // Fails harmlessly if the host application already chose a mode. + SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + }); +} + +} // namespace win_impl + +namespace detail { + +Result> createInput(const InputOptions& options) { + switch (options.backend) { + case InputBackend::Auto: + case InputBackend::SendInput: + return win_impl::createSendInput(options); + default: + return fail(ErrorCode::Unsupported, + "input backend '" + std::string(toString(options.backend)) + "' is not available on Windows"); + } +} + +Result> createCapture(const CaptureOptions& options) { + switch (options.backend) { + case CaptureBackend::Auto: + case CaptureBackend::Gdi: + return win_impl::createGdiCapture(options); + default: + return fail(ErrorCode::Unsupported, "capture backend '" + std::string(toString(options.backend)) + + "' is not available on Windows"); + } +} + +Result> createWindowManager(WindowBackend backend) { + switch (backend) { + case WindowBackend::Auto: + case WindowBackend::Win32: + return win_impl::createWin32Windows(); + default: + return fail(ErrorCode::Unsupported, + "window backend '" + std::string(toString(backend)) + "' is not available on Windows"); + } +} + +Result> listMonitors() { + win_impl::ensureDpiAware(); + std::vector out; + auto callback = [](HMONITOR mon, HDC, LPRECT, LPARAM data) -> BOOL { + auto* list = reinterpret_cast*>(data); + MONITORINFOEXW info{}; + info.cbSize = sizeof info; + if (!GetMonitorInfoW(mon, &info)) return TRUE; + Monitor m; + m.name = win_impl::narrow(info.szDevice); + const RECT& r = info.rcMonitor; + m.bounds = {r.left, r.top, r.right - r.left, r.bottom - r.top}; + m.pixelSize = {m.bounds.width, m.bounds.height}; + m.primary = (info.dwFlags & MONITORINFOF_PRIMARY) != 0; + UINT dpi = GetDpiForWindow(GetDesktopWindow()); + if (HMODULE shcore = LoadLibraryW(L"Shcore.dll")) { + using Fn = HRESULT(WINAPI*)(HMONITOR, int, UINT*, UINT*); + if (auto fn = reinterpret_cast(GetProcAddress(shcore, "GetDpiForMonitor"))) { + UINT dx = 0, dy = 0; + if (SUCCEEDED(fn(mon, 0 /* MDT_EFFECTIVE_DPI */, &dx, &dy))) dpi = dx; + } + FreeLibrary(shcore); + } + m.scale = dpi / 96.0; + + DISPLAY_DEVICEW dev{}; + dev.cb = sizeof dev; + if (EnumDisplayDevicesW(info.szDevice, 0, &dev, 0)) m.description = win_impl::narrow(dev.DeviceString); + list->push_back(std::move(m)); + return TRUE; + }; + if (!EnumDisplayMonitors(nullptr, nullptr, callback, reinterpret_cast(&out))) + return fail(ErrorCode::Io, win_impl::lastErrorMessage("EnumDisplayMonitors")); + return out; +} + +} // namespace detail +} // namespace alvin diff --git a/src/windows/sendinput_input.cpp b/src/windows/sendinput_input.cpp new file mode 100644 index 0000000..9ee9802 --- /dev/null +++ b/src/windows/sendinput_input.cpp @@ -0,0 +1,220 @@ +// Input through the Win32 SendInput API. +// +// Note: Windows' UIPI blocks injected input into windows of elevated (admin) +// processes unless this process is elevated too. + +#include +#include + +#include "backends.hpp" +#include "win_util.hpp" + +namespace alvin::win_impl { +namespace { + +WORD toVirtualKey(Key key) { + if (key >= Key::A && key <= Key::Z) return static_cast('A' + (static_cast(key) - static_cast(Key::A))); + if (key >= Key::Num0 && key <= Key::Num9) + return static_cast('0' + (static_cast(key) - static_cast(Key::Num0))); + if (key >= Key::F1 && key <= Key::F24) + return static_cast(VK_F1 + (static_cast(key) - static_cast(Key::F1))); + if (key >= Key::Kp0 && key <= Key::Kp9) + return static_cast(VK_NUMPAD0 + (static_cast(key) - static_cast(Key::Kp0))); + switch (key) { + case Key::Escape: return VK_ESCAPE; + case Key::Enter: return VK_RETURN; + case Key::Tab: return VK_TAB; + case Key::Backspace: return VK_BACK; + case Key::Space: return VK_SPACE; + case Key::Minus: return VK_OEM_MINUS; + case Key::Equal: return VK_OEM_PLUS; + case Key::LeftBracket: return VK_OEM_4; + case Key::RightBracket: return VK_OEM_6; + case Key::Backslash: return VK_OEM_5; + case Key::Semicolon: return VK_OEM_1; + case Key::Apostrophe: return VK_OEM_7; + case Key::Grave: return VK_OEM_3; + case Key::Comma: return VK_OEM_COMMA; + case Key::Period: return VK_OEM_PERIOD; + case Key::Slash: return VK_OEM_2; + case Key::CapsLock: return VK_CAPITAL; + case Key::NumLock: return VK_NUMLOCK; + case Key::ScrollLock: return VK_SCROLL; + case Key::LeftShift: return VK_LSHIFT; + case Key::RightShift: return VK_RSHIFT; + case Key::LeftCtrl: return VK_LCONTROL; + case Key::RightCtrl: return VK_RCONTROL; + case Key::LeftAlt: return VK_LMENU; + case Key::RightAlt: return VK_RMENU; + case Key::LeftSuper: return VK_LWIN; + case Key::RightSuper: return VK_RWIN; + case Key::Menu: return VK_APPS; + case Key::Insert: return VK_INSERT; + case Key::Delete: return VK_DELETE; + case Key::Home: return VK_HOME; + case Key::End: return VK_END; + case Key::PageUp: return VK_PRIOR; + case Key::PageDown: return VK_NEXT; + case Key::Left: return VK_LEFT; + case Key::Right: return VK_RIGHT; + case Key::Up: return VK_UP; + case Key::Down: return VK_DOWN; + case Key::PrintScreen: return VK_SNAPSHOT; + case Key::Pause: return VK_PAUSE; + case Key::KpDecimal: return VK_DECIMAL; + case Key::KpDivide: return VK_DIVIDE; + case Key::KpMultiply: return VK_MULTIPLY; + case Key::KpSubtract: return VK_SUBTRACT; + case Key::KpAdd: return VK_ADD; + case Key::KpEnter: return VK_RETURN; // distinguished by the extended flag + case Key::VolumeMute: return VK_VOLUME_MUTE; + case Key::VolumeDown: return VK_VOLUME_DOWN; + case Key::VolumeUp: return VK_VOLUME_UP; + case Key::MediaPlayPause: return VK_MEDIA_PLAY_PAUSE; + case Key::MediaStop: return VK_MEDIA_STOP; + case Key::MediaNext: return VK_MEDIA_NEXT_TRACK; + case Key::MediaPrevious: return VK_MEDIA_PREV_TRACK; + default: return 0; + } +} + +// Keys whose scan codes carry the E0 prefix. +bool isExtended(Key key) { + switch (key) { + case Key::RightCtrl: case Key::RightAlt: case Key::LeftSuper: case Key::RightSuper: case Key::Menu: + case Key::Insert: case Key::Delete: case Key::Home: case Key::End: case Key::PageUp: case Key::PageDown: + case Key::Left: case Key::Right: case Key::Up: case Key::Down: + case Key::KpDivide: case Key::KpEnter: case Key::NumLock: case Key::PrintScreen: + return true; + default: + return false; + } +} + +class SendInputBackend final : public Input { +public: + InputBackend backend() const override { return InputBackend::SendInput; } + + Status moveTo(Point p) override { + // Normalised 0..65535 across the whole virtual desktop. + int vx = GetSystemMetrics(SM_XVIRTUALSCREEN), vy = GetSystemMetrics(SM_YVIRTUALSCREEN); + int vw = GetSystemMetrics(SM_CXVIRTUALSCREEN), vh = GetSystemMetrics(SM_CYVIRTUALSCREEN); + INPUT in{}; + in.type = INPUT_MOUSE; + in.mi.dx = static_cast((static_cast(std::clamp(p.x - vx, 0, vw - 1)) * 65535) / std::max(1, vw - 1)); + in.mi.dy = static_cast((static_cast(std::clamp(p.y - vy, 0, vh - 1)) * 65535) / std::max(1, vh - 1)); + in.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK; + return send({in}); + } + + Status moveBy(int dx, int dy) override { + // Relative SendInput motion is subject to pointer acceleration; go absolute for exactness. + POINT cur; + if (!GetCursorPos(&cur)) return fail(ErrorCode::Io, lastErrorMessage("GetCursorPos")); + return moveTo({cur.x + dx, cur.y + dy}); + } + + Status buttonDown(MouseButton b) override { return button(b, true); } + Status buttonUp(MouseButton b) override { return button(b, false); } + + Status scroll(int dx, int dy) override { + std::vector events; + if (dy) { + INPUT in{}; + in.type = INPUT_MOUSE; + in.mi.dwFlags = MOUSEEVENTF_WHEEL; + in.mi.mouseData = static_cast(-dy * WHEEL_DELTA); // positive = away from user (up) + events.push_back(in); + } + if (dx) { + INPUT in{}; + in.type = INPUT_MOUSE; + in.mi.dwFlags = MOUSEEVENTF_HWHEEL; + in.mi.mouseData = static_cast(dx * WHEEL_DELTA); + events.push_back(in); + } + return send(events); + } + + Status keyDown(Key k) override { return key(k, true); } + Status keyUp(Key k) override { return key(k, false); } + + Status typeText(std::string_view utf8) override { + std::wstring text = widen(utf8); + std::size_t i = 0; + while (i < text.size()) { + wchar_t c = text[i]; + if (c == L'\r') { + ++i; + continue; + } + if (c == L'\n') { + if (auto s = keyPress(Key::Enter); !s) return s; + ++i; + continue; + } + // KEYEVENTF_UNICODE sends UTF-16 units; a surrogate pair goes as two units. + std::size_t units = (IS_HIGH_SURROGATE(c) && i + 1 < text.size()) ? 2 : 1; + std::vector events; + for (std::size_t k = 0; k < units; ++k) { + INPUT down{}; + down.type = INPUT_KEYBOARD; + down.ki.wScan = text[i + k]; + down.ki.dwFlags = KEYEVENTF_UNICODE; + INPUT up = down; + up.ki.dwFlags |= KEYEVENTF_KEYUP; + events.push_back(down); + events.push_back(up); + } + i += units; + if (auto s = send(std::move(events)); !s) return s; + pause(); + } + return {}; + } + +private: + Status button(MouseButton b, bool down) { + INPUT in{}; + in.type = INPUT_MOUSE; + switch (b) { + case MouseButton::Left: in.mi.dwFlags = down ? MOUSEEVENTF_LEFTDOWN : MOUSEEVENTF_LEFTUP; break; + case MouseButton::Right: in.mi.dwFlags = down ? MOUSEEVENTF_RIGHTDOWN : MOUSEEVENTF_RIGHTUP; break; + case MouseButton::Middle: in.mi.dwFlags = down ? MOUSEEVENTF_MIDDLEDOWN : MOUSEEVENTF_MIDDLEUP; break; + case MouseButton::Back: + case MouseButton::Forward: + in.mi.dwFlags = down ? MOUSEEVENTF_XDOWN : MOUSEEVENTF_XUP; + in.mi.mouseData = b == MouseButton::Back ? XBUTTON1 : XBUTTON2; + break; + } + return send({in}); + } + + Status key(Key k, bool down) { + WORD vk = toVirtualKey(k); + if (vk == 0) return fail(ErrorCode::InvalidArgument, "key has no Windows mapping"); + INPUT in{}; + in.type = INPUT_KEYBOARD; + in.ki.wVk = vk; + in.ki.wScan = static_cast(MapVirtualKeyW(vk, MAPVK_VK_TO_VSC)); + in.ki.dwFlags = (down ? 0 : KEYEVENTF_KEYUP) | (isExtended(k) ? KEYEVENTF_EXTENDEDKEY : 0); + return send({in}); + } + + Status send(std::vector events) { + if (events.empty()) return {}; + UINT sent = SendInput(static_cast(events.size()), events.data(), sizeof(INPUT)); + if (sent != events.size()) + return fail(ErrorCode::PermissionDenied, lastErrorMessage("SendInput (blocked by UIPI?)")); + return {}; + } +}; + +} // namespace + +Result> createSendInput(const InputOptions&) { + ensureDpiAware(); + return std::make_unique(); +} + +} // namespace alvin::win_impl diff --git a/src/windows/win32_windows.cpp b/src/windows/win32_windows.cpp new file mode 100644 index 0000000..ec65650 --- /dev/null +++ b/src/windows/win32_windows.cpp @@ -0,0 +1,118 @@ +// Top-level window enumeration and focus via Win32 + DWM. + +#include + +#include "backends.hpp" +#include "win_util.hpp" + +namespace alvin::win_impl { +namespace { + +std::string processName(DWORD pid) { + HANDLE proc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (!proc) return {}; + wchar_t path[MAX_PATH]; + DWORD size = MAX_PATH; + std::string name; + if (QueryFullProcessImageNameW(proc, 0, path, &size)) { + std::wstring_view p(path, size); + if (auto slash = p.find_last_of(L"\\/"); slash != std::wstring_view::npos) p.remove_prefix(slash + 1); + name = narrow(p); + } + CloseHandle(proc); + return name; +} + +// Windows a user would consider "open windows" (roughly what Alt+Tab shows). +bool isUserWindow(HWND hwnd) { + if (!IsWindowVisible(hwnd) || GetWindow(hwnd, GW_OWNER) != nullptr) return false; + if (GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW) return false; + DWORD cloaked = 0; // hidden UWP frames and windows on other virtual desktops + if (SUCCEEDED(DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &cloaked, sizeof cloaked)) && cloaked) return false; + return GetWindowTextLengthW(hwnd) > 0; +} + +WindowInfo describe(HWND hwnd, HWND foreground) { + WindowInfo w; + w.id.value = reinterpret_cast(hwnd); + int len = GetWindowTextLengthW(hwnd); + std::wstring title(static_cast(len) + 1, L'\0'); + GetWindowTextW(hwnd, title.data(), len + 1); + title.resize(static_cast(len)); + w.title = narrow(title); + + DWORD pid = 0; + GetWindowThreadProcessId(hwnd, &pid); + w.pid = pid; + w.appId = processName(pid); + + // Extended frame bounds exclude the invisible resize borders of Windows 10+. + RECT r{}; + if (FAILED(DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &r, sizeof r))) GetWindowRect(hwnd, &r); + w.bounds = {r.left, r.top, r.right - r.left, r.bottom - r.top}; + + w.focused = hwnd == foreground; + w.minimized = IsIconic(hwnd) != 0; + w.visible = !w.minimized; + + HMONITOR mon = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONULL); + MONITORINFO mi{}; + mi.cbSize = sizeof mi; + if (mon && GetMonitorInfoW(mon, &mi)) { + const RECT& m = mi.rcMonitor; + w.fullscreen = r.left <= m.left && r.top <= m.top && r.right >= m.right && r.bottom >= m.bottom; + } + return w; +} + +class Win32Windows final : public WindowManager { +public: + WindowBackend backend() const override { return WindowBackend::Win32; } + + Result> list() override { + struct Ctx { + HWND foreground; + std::vector out; + } ctx{GetForegroundWindow(), {}}; + EnumWindows( + [](HWND hwnd, LPARAM data) -> BOOL { + auto* c = reinterpret_cast(data); + if (isUserWindow(hwnd)) c->out.push_back(describe(hwnd, c->foreground)); + return TRUE; + }, + reinterpret_cast(&ctx)); + return std::move(ctx.out); + } + + Result> active() override { + HWND fg = GetForegroundWindow(); + if (!fg) return std::optional{}; + return std::optional{describe(fg, fg)}; + } + + Status focus(WindowId id) override { + HWND hwnd = reinterpret_cast(id.value); + if (!IsWindow(hwnd)) return fail(ErrorCode::NotFound, "no such window"); + if (IsIconic(hwnd)) ShowWindow(hwnd, SW_RESTORE); + if (SetForegroundWindow(hwnd)) return {}; + + // The foreground lock only lets the process that received the last input + // change focus. A synthetic Alt tap makes us that process. + INPUT alt[2]{}; + alt[0].type = alt[1].type = INPUT_KEYBOARD; + alt[0].ki.wVk = alt[1].ki.wVk = VK_MENU; + alt[1].ki.dwFlags = KEYEVENTF_KEYUP; + SendInput(2, alt, sizeof(INPUT)); + if (SetForegroundWindow(hwnd)) return {}; + return fail(ErrorCode::PermissionDenied, "Windows refused to change the foreground window"); + } +}; + +} // namespace + +Result> createWin32Windows() { + ensureDpiAware(); + return std::make_unique(); +} + +} // namespace alvin::win_impl diff --git a/src/windows/win_util.hpp b/src/windows/win_util.hpp new file mode 100644 index 0000000..ce02300 --- /dev/null +++ b/src/windows/win_util.hpp @@ -0,0 +1,29 @@ +#pragma once + +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include + +#include "alvin/error.hpp" + +namespace alvin::win_impl { + +std::string narrow(std::wstring_view w); +std::wstring widen(std::string_view s); + +// Message for GetLastError(), prefixed with `what`. +std::string lastErrorMessage(std::string_view what); + +// Opt the process into per-monitor DPI awareness once, so that every Win32 +// coordinate Alvin sees or produces is in physical pixels and consistent +// between capture, input and window bounds. +void ensureDpiAware(); + +} // namespace alvin::win_impl diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..a41f47f --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,22 @@ +find_package(Catch2 3 QUIET) +if(NOT Catch2_FOUND) + include(FetchContent) + FetchContent_Declare(Catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.7.1) + FetchContent_MakeAvailable(Catch2) +endif() + +add_executable(alvin_tests + test_core.cpp + test_json.cpp) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_sources(alvin_tests PRIVATE test_linux.cpp) + target_link_libraries(alvin_tests PRIVATE PkgConfig::XKBCOMMON) +endif() +# Tests exercise internal helpers too, so they see the private source tree. +target_include_directories(alvin_tests PRIVATE ${PROJECT_SOURCE_DIR}/src) +target_link_libraries(alvin_tests PRIVATE alvin::alvin Catch2::Catch2WithMain) + +include(Catch) +catch_discover_tests(alvin_tests) diff --git a/tests/test_core.cpp b/tests/test_core.cpp new file mode 100644 index 0000000..9d56823 --- /dev/null +++ b/tests/test_core.cpp @@ -0,0 +1,66 @@ +#include + +#include "alvin/frame.hpp" +#include "alvin/keys.hpp" +#include "alvin/types.hpp" + +using namespace alvin; + +TEST_CASE("key names round-trip and aliases resolve") { + CHECK(keyFromName("A") == Key::A); + CHECK(keyFromName("ctrl") == Key::LeftCtrl); + CHECK(keyFromName("Control") == Key::LeftCtrl); + CHECK(keyFromName("esc") == Key::Escape); + CHECK(keyFromName("F12") == Key::F12); + CHECK_FALSE(keyFromName("nope")); + for (int k = 1; k < static_cast(Key::Count_); ++k) { + auto key = static_cast(k); + INFO("key index " << k); + CHECK(keyFromName(keyName(key)) == key); + } +} + +TEST_CASE("chords parse in order") { + auto c = parseChord("ctrl+shift+t"); + REQUIRE(c); + CHECK(*c == std::vector{Key::LeftCtrl, Key::LeftShift, Key::T}); + + CHECK(parseChord(" super + enter ") == std::vector{Key::LeftSuper, Key::Enter}); + CHECK(parseChord("ctrl++") == std::vector{Key::LeftCtrl, Key::Equal}); + CHECK_FALSE(parseChord("ctrl+")); + CHECK_FALSE(parseChord("")); + CHECK_FALSE(parseChord("ctrl+bogus")); +} + +TEST_CASE("rect union ignores empty rects") { + Rect a{0, 0, 10, 10}, b{20, -5, 5, 5}; + CHECK(a.united(b) == Rect{0, -5, 25, 15}); + CHECK(Rect{}.united(a) == a); + CHECK(a.united(Rect{}) == a); + CHECK(a.contains({9, 9})); + CHECK_FALSE(a.contains({10, 0})); +} + +TEST_CASE("frame maps between pixels and desktop coordinates") { + // A 1.5x scaled monitor: 300x150 pixels covering 200x100 logical at (1000, 50). + Frame f(300, 150); + f.setDesktopRect({1000, 50, 200, 100}); + CHECK(f.toDesktop({0, 0}) == Point{1000, 50}); + CHECK(f.toDesktop({150, 75}) == Point{1100, 100}); + CHECK(f.fromDesktop({1100, 100}) == Point{150, 75}); + + Frame c = f.crop({150, 0, 1000, 75}); // clipped to the frame + CHECK(c.width() == 150); + CHECK(c.height() == 75); + CHECK(c.desktopRect() == Rect{1100, 50, 100, 50}); +} + +TEST_CASE("frame pixel reads BGRA") { + Frame f(2, 1); + f.data()[4] = 1; // B + f.data()[5] = 2; // G + f.data()[6] = 3; // R + f.data()[7] = 4; // A + CHECK(f.pixel(1, 0) == Color{3, 2, 1, 4}); + CHECK(f.pixel(5, 5) == Color{}); +} diff --git a/tests/test_json.cpp b/tests/test_json.cpp new file mode 100644 index 0000000..33f7fb0 --- /dev/null +++ b/tests/test_json.cpp @@ -0,0 +1,33 @@ +#include + +#include "common/json.hpp" + +using namespace alvin; + +TEST_CASE("json parses a Hyprland client entry") { + auto v = json::parse(R"([{ + "address": "0x55a4165aec60", + "at": [2903, 58], "size": [2834, 1539], + "workspace": {"id": 8, "name": "8"}, + "hidden": false, "title": "Café \"quoted\"", "pid": 3163 + }])"); + REQUIRE(v); + const auto& c = (*v)[0]; + CHECK(c["address"].string() == "0x55a4165aec60"); + CHECK(c["at"][0].number() == 2903); + CHECK(c["size"][1].number() == 1539); + CHECK(c["workspace"]["name"].string() == "8"); + CHECK(c["hidden"].boolean(true) == false); + CHECK(c["title"].string() == "Caf\xc3\xa9 \"quoted\""); + CHECK(c["missing"].isNull()); + CHECK(c["missing"]["deeper"].isNull()); +} + +TEST_CASE("json rejects malformed input") { + CHECK_FALSE(json::parse("{")); + CHECK_FALSE(json::parse("[1,]")); + CHECK_FALSE(json::parse("{\"a\" 1}")); + CHECK_FALSE(json::parse("[1] trailing")); + CHECK(json::parse(" {} ")); + CHECK(json::parse("\"\\ud83d\\ude00\"")->string() == "\xf0\x9f\x98\x80"); +} diff --git a/tests/test_linux.cpp b/tests/test_linux.cpp new file mode 100644 index 0000000..36d927f --- /dev/null +++ b/tests/test_linux.cpp @@ -0,0 +1,60 @@ +#include +#include + +#include "linux/keycodes.hpp" +#include "linux/xkb.hpp" + +using namespace alvin; +using namespace alvin::linux_impl; + +TEST_CASE("utf8 decoding") { + auto t = decodeUtf8("a\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80"); + REQUIRE(t); + CHECK(*t == U"aé€\U0001F600"); + CHECK_FALSE(decodeUtf8("\xc3")); + CHECK_FALSE(decodeUtf8("\xff")); +} + +TEST_CASE("us layout strokes") { + auto km = XkbKeymap::fromNames("us"); + REQUIRE(km); + + auto a = km->strokeFor(U'a'); + REQUIRE(a); + CHECK(a->key == KEY_A); + CHECK(a->modifiers.empty()); + + auto upper = km->strokeFor(U'A'); + REQUIRE(upper); + CHECK(upper->key == KEY_A); + CHECK(upper->modifiers == std::vector{KEY_LEFTSHIFT}); + + auto bang = km->strokeFor(U'!'); + REQUIRE(bang); + CHECK(bang->key == KEY_1); + + CHECK(km->strokeFor(U'\n')->key == KEY_ENTER); + CHECK_FALSE(km->strokeFor(U'é')); // not on plain US + CHECK_FALSE(km->strokesFor(U"oké")); +} + +TEST_CASE("synthesized keymaps compile and map codes 1..n") { + std::u32string alphabet = U"hé\U0001F600\n"; + auto source = synthesizeKeymap(alphabet); + REQUIRE(source); + auto km = XkbKeymap::fromString(*source); + REQUIRE(km); + for (std::size_t i = 0; i < alphabet.size(); ++i) { + auto s = km->strokeFor(alphabet[i]); + REQUIRE(s); + CHECK(s->key == i + 1); + CHECK(s->modifiers.empty()); + } +} + +TEST_CASE("every key maps to an evdev code") { + for (int k = 1; k < static_cast(Key::Count_); ++k) { + INFO("key index " << k); + CHECK(toEvdev(static_cast(k)) != 0); + } +}