diff --git a/README.md b/README.md index c5a0b2ffb1..05241a00c0 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ You may need to grant screen recording permissions depending on your desktop env Everything in the editor and export is the same on macOS, Windows, and Linux: zooms, backgrounds, motion blur, crop/trim/speed, blur regions, annotations, auto-captions, AI editing, projects, export, and all languages. All three now record through a native capture pipeline; the remaining differences are narrower than they used to be: - **Native recording**: macOS (ScreenCaptureKit), Windows (Windows Graphics Capture), and Linux (PipeWire via the ScreenCast portal) all record through a native pipeline for higher quality and clean window-level capture. On Linux the browser pipeline stays as an automatic fallback if the helper isn't available. -- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs read access to your mouse: a udev rule for pointer devices (recommended) or the `input` group, set up as [Mouse clicks on Wayland](https://getopenscreen.com/docs/installation#mouse-clicks-on-wayland) shows. Without that, recording is unaffected and every cursor sample is simply a move. +- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs read access to your mouse. The `.deb`, `.rpm` and `.pacman` packages install a udev rule that grants it for mice and touchpads only, never keyboards; with the AppImage, add that rule by hand as [Mouse clicks on Wayland](https://getopenscreen.com/docs/installation#mouse-clicks-on-wayland) shows (the `input` group also works, but opens keyboards too and is not recommended). Without that, recording is unaffected and every cursor sample is simply a move. - **Webcam**: every platform saves the webcam as a separate file next to the screen recording. Windows captures it natively in the recording helper; macOS and Linux record it in the app. It works as a picture-in-picture overlay on all three. - **System audio** support varies by OS: - **macOS**: works on every supported version. On macOS 14.2+ you'll be prompted to grant audio capture permission. diff --git a/build/linux/70-openscreen-pointer.rules b/build/linux/70-openscreen-pointer.rules new file mode 100644 index 0000000000..85a473c2aa --- /dev/null +++ b/build/linux/70-openscreen-pointer.rules @@ -0,0 +1,16 @@ +# OpenScreen click zooms: let the logged-in user read mice and touchpads, never keyboards. +# +# The Linux capture helper reads left-button presses from /dev/input/event* (Wayland has no +# portal for mouse buttons; see electron/native/pipewire-capture/src/input.rs). Those nodes +# are root:input. The `input` group would open every input device, keyboards included, to +# every program the user runs. `uaccess` instead has systemd-logind put an ACL for the user +# of the active local session on the matched nodes, and on nothing else. A node that reports +# any keyboard key (ID_INPUT_KEY) is never matched, even when it is also a mouse. +# +# The 70- prefix is load-bearing: the tag has to be set before systemd's 73-seat-late.rules, +# which is where uaccess is applied. The deb, rpm and pacman packages install this file to +# /usr/lib/udev/rules.d/, and the NixOS module adds it to services.udev.packages. For the +# AppImage, copy it to /etc/udev/rules.d/ by hand: website/docs/installation.md, +# "Mouse clicks on Wayland". +SUBSYSTEM=="input", KERNEL=="event*", ENV{ID_INPUT_KEY}!="1", ENV{ID_INPUT_MOUSE}=="1", TAG+="uaccess" +SUBSYSTEM=="input", KERNEL=="event*", ENV{ID_INPUT_KEY}!="1", ENV{ID_INPUT_TOUCHPAD}=="1", TAG+="uaccess" diff --git a/build/linux/after-install.tpl b/build/linux/after-install.tpl new file mode 100644 index 0000000000..8ca294cd28 --- /dev/null +++ b/build/linux/after-install.tpl @@ -0,0 +1,76 @@ +#!/bin/bash + +if type update-alternatives >/dev/null 2>&1; then + # Remove previous link if it doesn't use update-alternatives + if [ -L '/usr/bin/${executable}' -a -e '/usr/bin/${executable}' -a "`readlink '/usr/bin/${executable}'`" != '/etc/alternatives/${executable}' ]; then + rm -f '/usr/bin/${executable}' + fi + update-alternatives --install '/usr/bin/${executable}' '${executable}' '/opt/${sanitizedProductName}/${executable}' 100 || ln -sf '/opt/${sanitizedProductName}/${executable}' '/usr/bin/${executable}' +else + ln -sf '/opt/${sanitizedProductName}/${executable}' '/usr/bin/${executable}' +fi + +# Check if user namespaces are supported by the kernel and working with a quick test: +if ! { [[ -L /proc/self/ns/user ]] && unshare --user true; }; then + # Use SUID chrome-sandbox only on systems without user namespaces: + chmod 4755 '/opt/${sanitizedProductName}/chrome-sandbox' || true +else + chmod 0755 '/opt/${sanitizedProductName}/chrome-sandbox' || true +fi + +if hash update-mime-database 2>/dev/null; then + update-mime-database /usr/share/mime || true +fi + +if hash update-desktop-database 2>/dev/null; then + update-desktop-database /usr/share/applications || true +fi + +# Install apparmor profile. (Ubuntu 24+) +# First check if the version of AppArmor running on the device supports our profile. +# This is in order to keep backwards compatibility with Ubuntu 22.04 which does not support abi/4.0. +# In that case, we just skip installing the profile since the app runs fine without it on 22.04. +# +# Those apparmor_parser flags are akin to performing a dry run of loading a profile. +# https://wiki.debian.org/AppArmor/HowToUse#Dumping_profiles +# +# Unfortunately, at the moment AppArmor doesn't have a good story for backwards compatibility. +# https://askubuntu.com/questions/1517272/writing-a-backwards-compatible-apparmor-profile +if apparmor_status --enabled > /dev/null 2>&1; then + APPARMOR_PROFILE_SOURCE='/opt/${sanitizedProductName}/resources/apparmor-profile' + APPARMOR_PROFILE_TARGET='/etc/apparmor.d/${executable}' + if apparmor_parser --skip-kernel-load --debug "$APPARMOR_PROFILE_SOURCE" > /dev/null 2>&1; then + cp -f "$APPARMOR_PROFILE_SOURCE" "$APPARMOR_PROFILE_TARGET" + + # Updating the current AppArmor profile is not possible and probably not meaningful in a chroot'ed environment. + # Use cases are for example environments where images for clients are maintained. + # There, AppArmor might correctly be installed, but live updating makes no sense. + if ! { [ -x '/usr/bin/ischroot' ] && /usr/bin/ischroot; } && hash apparmor_parser 2>/dev/null; then + # Extra flags taken from dh_apparmor: + # > By using '-W -T' we ensure that any abstraction updates are also pulled in. + # https://wiki.debian.org/AppArmor/Contribute/FirstTimeProfileImport + apparmor_parser --replace --write-cache --skip-read-cache "$APPARMOR_PROFILE_TARGET" + fi + else + echo "Skipping the installation of the AppArmor profile as this version of AppArmor does not seem to support the bundled profile" + fi +fi + +# --------------------------------------------------------------------------------------- +# OpenScreen. Everything above is app-builder-lib's templates/linux/after-install.tpl, +# verbatim: naming a script in electron-builder.json5 REPLACES that template instead of +# extending it, and it is what creates /usr/bin/openscreen and sets chrome-sandbox's mode. +# scripts/linux-package-scripts.test.mjs fails when the two drift apart. +# +# The package ships /usr/lib/udev/rules.d/70-openscreen-pointer.rules (build/linux/), which +# gives the logged-in user read access to mice and touchpads for click capture. udev only +# applies a rule to an event, so a mouse that is already plugged in would wait for a replug +# or a reboot; reloading and replaying a change event on the input devices applies it now. +# +# Skipped where no udev daemon is listening (a container, a chroot, an image build): there +# is nothing to reload, and the rule applies on the first boot anyway. Never fatal. +if command -v udevadm >/dev/null 2>&1 && [ -S /run/udev/control ] \ + && ! { [ -x '/usr/bin/ischroot' ] && /usr/bin/ischroot; }; then + udevadm control --reload-rules || true + udevadm trigger --subsystem-match=input --action=change || true +fi diff --git a/build/linux/after-remove.tpl b/build/linux/after-remove.tpl new file mode 100644 index 0000000000..80eda1a31f --- /dev/null +++ b/build/linux/after-remove.tpl @@ -0,0 +1,46 @@ +#!/bin/bash + +# Delete the link to the binary +# update-alternatives --remove : 'path' must be the registered alternative binary, +# not the generic symlink — see https://man7.org/linux/man-pages/man1/update-alternatives.1.html +if type update-alternatives >/dev/null 2>&1; then + update-alternatives --remove '${executable}' '/opt/${sanitizedProductName}/${executable}' +else + rm -f '/usr/bin/${executable}' +fi + +APPARMOR_PROFILE_DEST='/etc/apparmor.d/${executable}' + +# Remove and unload apparmor profile. +if [ -f "$APPARMOR_PROFILE_DEST" ]; then + # Unload the profile from the running kernel before deleting the file so the + # policy is not left enforced until the next reboot. Mirror the chroot guard + # used in the after-install script — live AppArmor operations are not + # meaningful inside a chroot. + # https://wiki.debian.org/AppArmor/HowToUse + if apparmor_status --enabled > /dev/null 2>&1; then + if ! { [ -x '/usr/bin/ischroot' ] && /usr/bin/ischroot; } && hash apparmor_parser 2>/dev/null; then + apparmor_parser --remove "$APPARMOR_PROFILE_DEST" || true + fi + fi + rm -f "$APPARMOR_PROFILE_DEST" +fi + +# --------------------------------------------------------------------------------------- +# OpenScreen. Everything above is app-builder-lib's templates/linux/after-remove.tpl, +# verbatim: naming a script in electron-builder.json5 REPLACES that template instead of +# extending it. scripts/linux-package-scripts.test.mjs fails when the two drift apart. +# +# The package manager has already deleted 70-openscreen-pointer.rules by now. Reload so +# udev stops tagging pointer devices for uaccess, and replay a change event so the current +# devices are re-evaluated without it. An ACL logind already placed on a node can outlive +# this until the device is replugged or the machine reboots: udev has no event that +# withdraws one. +# +# This also runs on upgrade (deb postrm, rpm %postun), when the new package's copy of the +# rule is in place. Reloading then is harmless. Same guard as after-install. +if command -v udevadm >/dev/null 2>&1 && [ -S /run/udev/control ] \ + && ! { [ -x '/usr/bin/ischroot' ] && /usr/bin/ischroot; }; then + udevadm control --reload-rules || true + udevadm trigger --subsystem-match=input --action=change || true +fi diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 4eb981919c..b0e9c339a4 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -55,6 +55,10 @@ pub struct CursorTrack { /// (`follow_in_view`). Partagé entre clones, qui ont les mêmes échantillons. view_cache: std::sync::Arc>)>>>, + /// Le pointeur est déjà PEINT dans les pixels (`.pointer.json`, cf. `load`) : la + /// piste sert au suivi auto du zoom, mais rien ne la dessine, ni sprite ni impact de clic, + /// sans quoi un second curseur se poserait sur le premier (`frame_geometry::cursor_alpha`). + painted_in: bool, } type ViewKey = (u32, u32, u32); @@ -161,9 +165,15 @@ impl CursorTrack { visibility, types, view_cache: Default::default(), + painted_in: false, } } + /// Vrai pour une piste chargée depuis un `.pointer.json` : voir le champ `painted_in`. + pub fn painted_in(&self) -> bool { + self.painted_in + } + /// Visibilité enregistrée au temps `t`. Les anciens sidecars qui n'ont pas ce champ sont /// visibles par défaut, comme les lecteurs TypeScript. pub fn visible_at(&self, t: f32) -> bool { @@ -188,7 +198,28 @@ impl CursorTrack { } /// Charge la fenêtre [offset_ms, offset_ms + dur_s*1000] et la ramène à t=0. + /// + /// Un `.cursor.json` absent retombe sur `.pointer.json` : une prise dont le + /// pointeur est peint dans l'image (Linux sous Hyprland, dont le portail ne sait pas l'en + /// sortir) ne sait que où il était. Résolu ICI, pas à chaque appelant : l'aperçu, l'export + /// MP4 et le GIF passent tous le chemin du `.cursor.json` et passent tous par ici. pub fn load(path: &str, offset_ms: f64, dur_s: f64) -> Result { + match CursorTrack::load_file(path, offset_ms, dur_s) { + Err(error) if !std::path::Path::new(path).exists() => { + let Some(stem) = path.strip_suffix(".cursor.json") else { return Err(error) }; + let pointer = format!("{stem}.pointer.json"); + if !std::path::Path::new(&pointer).exists() { + return Err(error); + } + let mut track = CursorTrack::load_file(&pointer, offset_ms, dur_s)?; + track.painted_in = true; + Ok(track) + } + loaded => loaded, + } + } + + fn load_file(path: &str, offset_ms: f64, dur_s: f64) -> Result { let txt = std::fs::read_to_string(path).with_context(|| format!("lecture {path}"))?; let v: serde_json::Value = serde_json::from_str(&txt)?; let arr = v["samples"].as_array().context("samples[]")?; @@ -471,12 +502,17 @@ impl CursorTrack { /// bruts (le bounce est temporel, pas positionnel — ne doit pas suivre le lissage). pub fn smoothed(&self, factor: f32) -> CursorTrack { if self.samples.len() < 2 || factor <= 0.0 { - return CursorTrack::new_with_visibility( - self.samples.clone(), - self.clicks.clone(), - self.types.clone(), - self.visibility.clone(), - ); + // Le marqueur suit ici aussi : lissage coupé, une prise peinte dessinait sinon un + // second curseur par-dessus le sien. + return CursorTrack { + painted_in: self.painted_in, + ..CursorTrack::new_with_visibility( + self.samples.clone(), + self.clicks.clone(), + self.types.clone(), + self.visibility.clone(), + ) + }; } const STEP_S: f32 = 1.0 / 240.0; let start = self.samples[0].0; @@ -536,6 +572,7 @@ impl CursorTrack { // cliqués restent ceux de la piste brute. CursorTrack { click_points: self.click_points.clone(), + painted_in: self.painted_in, ..CursorTrack::new_with_visibility( samples, self.clicks.clone(), @@ -1151,4 +1188,36 @@ mod tests { // Et s'éteint 1.8s après le clic (4.0 + 1.8 = 5.8s) assert_eq!(track.opacity_at(5.9, true), 0.0); } + + #[test] + fn falls_back_to_the_pointer_sidecar_of_a_painted_in_take() { + let dir = std::env::temp_dir().join(format!("pointer-sidecar-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let video = dir.join("take.mp4"); + let cursor_json = format!("{}.cursor.json", video.to_str().expect("utf-8 path")); + let pointer_json = format!("{}.pointer.json", video.to_str().expect("utf-8 path")); + std::fs::write( + &pointer_json, + r#"{"samples":[{"timeMs":0,"cx":0.2,"cy":0.2},{"timeMs":1000,"cx":0.8,"cy":0.8}]}"#, + ) + .expect("write pointer sidecar"); + + let track = CursorTrack::load(&cursor_json, 0.0, 2.0).expect("falls back"); + assert!(track.painted_in()); + assert!(track.at(1.0).is_some()); + // Le lissage (slider « smoothing ») garde le marqueur : sinon le curseur réapparaît. + assert!(track.smoothed(0.5).painted_in()); + // Lissage coupé (0) : le chemin court garde le marqueur, lui aussi. + assert!(track.smoothed(0.0).painted_in()); + + // Un `.cursor.json` présent l'emporte, et se dessine. + std::fs::write(&cursor_json, r#"{"samples":[{"timeMs":0,"cx":0.5,"cy":0.5}]}"#) + .expect("write cursor sidecar"); + let drawn = CursorTrack::load(&cursor_json, 0.0, 2.0).expect("loads the drawable one"); + assert!(!drawn.painted_in()); + let _ = std::fs::remove_dir_all(&dir); + + // Ni l'un ni l'autre : l'erreur d'origine, comme avant. + assert!(CursorTrack::load(&cursor_json, 0.0, 2.0).is_err()); + } } diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index e7410ad558..38ffa2b8c9 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3777,7 +3777,8 @@ pub struct CursorPlanInput<'a> { pub t: f32, } -/// Opacité du curseur à `t`, avant placement : 0 quand il est masqué (`cursor.show`), sinon +/// Opacité du curseur à `t`, avant placement : 0 quand il est masqué (`cursor.show`) ou déjà +/// peint dans l'image (`CursorTrack::painted_in`), sinon /// l'auto-hide × le `hideCursor` des régions de zoom. Partagée par `plan_cursor` et l'impact du /// clic : le plan ne bascule que sous un pointeur qu'on voit. pub fn cursor_alpha( @@ -3787,7 +3788,7 @@ pub fn cursor_alpha( track: &crate::cursor::CursorTrack, t: f32, ) -> f32 { - if !scene.map(|s| s.cursor.show).unwrap_or(cfg.cursor) { + if !scene.map(|s| s.cursor.show).unwrap_or(cfg.cursor) || track.painted_in() { return 0.0; } let idle_alpha = track.opacity_at(t, live.cursor_auto_hide); @@ -8586,6 +8587,38 @@ mod tests { /// clickBounce au maximum du slider (5) : le creux de la pression donnerait 1 - 0.24 * 5 < 0. /// La taille ne doit jamais devenir négative ; au creux, rien n'est dessiné. + #[test] + fn plan_cursor_draws_nothing_for_a_pointer_painted_into_the_pixels() { + let cfg = crate::config::all().pop().expect("cfg"); + let scene = zoomed_golden_scene(); + let fg = full_frame_geometry(); + let dir = std::env::temp_dir().join(format!("painted-in-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let video = dir.join("take.mp4"); + std::fs::write( + dir.join("take.mp4.pointer.json"), + r#"{"samples":[{"timeMs":0,"cx":0.5,"cy":0.5},{"timeMs":500,"cx":0.5,"cy":0.5,"interactionType":"click"},{"timeMs":2000,"cx":0.5,"cy":0.5}]}"#, + ) + .expect("write pointer sidecar"); + let cursor_json = format!("{}.cursor.json", video.to_str().expect("utf-8 path")); + let track = crate::cursor::CursorTrack::load(&cursor_json, 0.0, 3.0).expect("load"); + let _ = std::fs::remove_dir_all(&dir); + + let input = |t: f32| CursorPlanInput { + render_px: [1920.0, 1080.0], + u_max: 1.0, + v_max: 1.0, + cfg: &cfg, + live: LiveParams::default(), + scene: Some(&scene), + track: &track, + t, + }; + // Ni sprite ni anneau de clic, juste après le clic compris : le pointeur est dans l'image. + assert!(plan_cursor(&fg, &input(0.6)).is_none()); + assert!(plan_cursor(&fg, &input(1.0)).is_none()); + } + #[test] fn plan_cursor_never_yields_a_negative_size_at_max_bounce() { let cfg = crate::config::all().pop().expect("cfg"); diff --git a/electron-builder.json5 b/electron-builder.json5 index 8b11d8cc77..29dc9d9b6b 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -327,7 +327,26 @@ "libasound2", "libgomp1", "xdg-desktop-portal" - ] + ], + // Click capture without manual setup: ship build/linux/70-openscreen-pointer.rules, which + // grants the logged-in user (systemd-logind `uaccess`) read access to mice and touchpads, + // never keyboards. The same three keys are repeated on `pacman` and `rpm`, because the + // `linux` block does not accept them: fpm options are per target. + // + // `fpm` is extra fpm arguments, and a `src=dest` argument maps one more file into the + // package. The source is relative to fpm's working directory, which is the project root + // whenever electron-builder runs from an npm script. The package owns the file, so the + // package manager also deletes it on uninstall. + // + // A custom `afterInstall`/`afterRemove` REPLACES electron-builder's default script instead + // of extending it, and the default is what creates /usr/bin/openscreen and sets + // chrome-sandbox's mode. So build/linux/after-*.tpl are those templates verbatim with a + // udev reload appended, and scripts/linux-package-scripts.test.mjs fails when they drift + // after an electron-builder upgrade. The reload is what lets the rule reach a mouse that + // is already plugged in, without a replug or a reboot. + "afterInstall": "build/linux/after-install.tpl", + "afterRemove": "build/linux/after-remove.tpl", + "fpm": ["build/linux/70-openscreen-pointer.rules=/usr/lib/udev/rules.d/70-openscreen-pointer.rules"] }, "pacman": { // `vulkan-swrast` est le lavapipe d'Arch ; il tire `vulkan-icd-loader` avec lui. @@ -365,7 +384,12 @@ "alsa-lib", "libgomp", "xdg-desktop-portal" - ] + ], + // The pointer udev rule and the scripts that reload udev: see `deb` above. Arch's own + // systemd pacman hook also reloads udev when a file under usr/lib/udev/rules.d changes. + "afterInstall": "build/linux/after-install.tpl", + "afterRemove": "build/linux/after-remove.tpl", + "fpm": ["build/linux/70-openscreen-pointer.rules=/usr/lib/udev/rules.d/70-openscreen-pointer.rules"] }, "rpm": { // Même règle que ci-dessus : la liste REMPLACE le défaut d'electron-builder, donc @@ -404,7 +428,11 @@ "alsa-lib", "libgomp", "xdg-desktop-portal" - ] + ], + // The pointer udev rule and the scripts that reload udev: see `deb` above. + "afterInstall": "build/linux/after-install.tpl", + "afterRemove": "build/linux/after-remove.tpl", + "fpm": ["build/linux/70-openscreen-pointer.rules=/usr/lib/udev/rules.d/70-openscreen-pointer.rules"] }, "win": { "target": [ diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index b85d616ced..e090afcf79 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -464,6 +464,8 @@ interface Window { quitApp: () => void; setTitleBarOverlay: (color: string, symbolColor: string) => void; getPlatform: () => string; + /** True in the HUD bar's and panel's windows when the compositor draws their frame. */ + isHudFramedByCompositor?: () => boolean; getSystemVersion: () => string; getAppInfo: () => Promise<{ version: string; canCheckForUpdates: boolean }>; checkForUpdates: () => Promise; @@ -522,6 +524,25 @@ interface Window { width: number; height: number; }) => void; + /** Linux: what the HUD's floating surfaces show, for the panel window to render + * (src/components/launch/hudPanel.ts). An empty state hides the panel. */ + sendHudPanelState: (state: import("../src/components/launch/hudPanel").HudPanelState) => void; + /** Linux: what the user did in the panel window, delivered to the HUD. */ + onHudPanelAction: ( + callback: (action: import("../src/components/launch/hudPanel").HudPanelAction) => void, + ) => () => void; + /** The panel window's side of {@link sendHudPanelState}. */ + onHudPanelState: ( + callback: (state: import("../src/components/launch/hudPanel").HudPanelState) => void, + ) => () => void; + /** The panel window asks for the current snapshot once it is listening for them. */ + requestHudPanelState: () => void; + /** The panel window's side of {@link onHudPanelAction}. */ + sendHudPanelAction: ( + action: import("../src/components/launch/hudPanel").HudPanelAction, + ) => void; + /** The panel window reports the size of what it shows; the window is sized to it. */ + setHudPanelSize: (width: number, height: number) => void; showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; diff --git a/electron/hudWindowArgs.ts b/electron/hudWindowArgs.ts new file mode 100644 index 0000000000..d033a6a5ec --- /dev/null +++ b/electron/hudWindowArgs.ts @@ -0,0 +1,5 @@ +/** + * Passed to the HUD bar's and panel's preload when the compositor frames their windows itself + * (Hyprland), so the renderer draws them square inside that frame. Shared by main and preload. + */ +export const HUD_FRAMED_BY_COMPOSITOR_ARG = "--hud-framed-by-compositor"; diff --git a/electron/hudWindowBounds.ts b/electron/hudWindowBounds.ts index 071c89a109..7509a2aa47 100644 --- a/electron/hudWindowBounds.ts +++ b/electron/hudWindowBounds.ts @@ -24,8 +24,12 @@ import { type EditorWindowRect, } from "./editorWindowState"; -/** The HUD's own floor, matching `minWidth`/`minHeight` at construction. */ -export const HUD_WINDOW_MIN = { width: 120, height: 80 }; +/** + * The HUD's own floor, matching `minWidth`/`minHeight` at construction. Below the bar's + * own thickness (about 54px), since on Linux the window is exactly the bar + * (computeHudLinuxWindowSize); elsewhere the window is always far larger than this. + */ +export const HUD_WINDOW_MIN = { width: 40, height: 40 }; /** * The visible stack's rect relative to the window, as measured by the renderer: diff --git a/electron/hyprland.test.ts b/electron/hyprland.test.ts new file mode 100644 index 0000000000..5f7b109ff3 --- /dev/null +++ b/electron/hyprland.test.ts @@ -0,0 +1,473 @@ +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + barPositionAtBottom, + hyprlandSocketPath, + monitorWorkArea, + panelPositionNearBar, + placeHudBarAtBottom, + placeHudPanelNearBar, + presetHudBarPlacement, + presetHudPanelPlacement, + runsUnderHyprland, + undecoratedWindowRule, + windowPlacementRule, +} from "./hyprland"; + +const workArea = { x: 0, y: 22, width: 1920, height: 1058 }; + +describe("panelPositionNearBar", () => { + it("centres the panel on the bar, just above it", () => { + const bar = { x: 672, y: 476, width: 576, height: 150 }; + expect(panelPositionNearBar(bar, { width: 382, height: 300 }, workArea, 8)).toEqual({ + x: 769, + y: 168, + }); + }); + + it("goes below the bar when there is not room above it", () => { + const bar = { x: 672, y: 40, width: 576, height: 150 }; + expect(panelPositionNearBar(bar, { width: 382, height: 300 }, workArea, 8)).toEqual({ + x: 769, + y: 198, + }); + }); + + it("stays above the bar when below has even less room", () => { + // 140 px free above, 92 below: neither fits a 300 px panel. + const bar = { x: 672, y: 170, width: 576, height: 800 }; + expect(panelPositionNearBar(bar, { width: 382, height: 300 }, workArea, 8)).toEqual({ + x: 769, + y: 22, + }); + }); + + it("stays inside the work area at a screen edge", () => { + const bar = { x: 1700, y: 900, width: 200, height: 150 }; + expect(panelPositionNearBar(bar, { width: 382, height: 300 }, workArea, 8)).toEqual({ + x: 1920 - 382, + y: 592, + }); + }); +}); + +describe("barPositionAtBottom", () => { + it("centres the bar at the bottom of the work area", () => { + expect(barPositionAtBottom({ width: 576, height: 150 }, workArea, 5)).toEqual({ + x: 672, + y: 22 + 1058 - 150 - 5, + }); + }); +}); + +describe("windowPlacementRule", () => { + it("floats the panel at its size and a monitor-relative position", () => { + expect( + windowPlacementRule( + "openscreen-hud-panel", + "OpenScreen panel", + { width: 382, height: 300 }, + 769.4, + 168, + ), + ).toBe( + 'hl.window_rule({ name = "openscreen-hud-panel", match = { title = "^(OpenScreen panel)$" }, ' + + 'float = true, size = "382 300", move = "769 168" })', + ); + }); + + it("refuses a title it would have to escape", () => { + expect( + windowPlacementRule( + "openscreen-hud-panel", + 'a "quoted" (title)', + { width: 1, height: 1 }, + 0, + 0, + ), + ).toBeNull(); + }); +}); + +describe("undecoratedWindowRule", () => { + // A transparent overlay draws its own shape; Hyprland's frame and blur go around its whole + // rectangle otherwise. + it("drops the border, the blur and the shadow of the window it matches", () => { + expect(undecoratedWindowRule("openscreen-countdown", "OpenScreen countdown")).toBe( + 'hl.window_rule({ name = "openscreen-countdown", match = { title = "^(OpenScreen countdown)$" }, ' + + "decorate = false, no_blur = true, no_shadow = true })", + ); + }); + + it("refuses a title it would have to escape", () => { + expect(undecoratedWindowRule("openscreen-countdown", "a (title)")).toBeNull(); + }); +}); + +describe("monitorWorkArea", () => { + it("is the monitor in logical px less what bars reserve", () => { + expect( + monitorWorkArea({ + id: 0, + x: 1920, + y: 0, + width: 3072, + height: 1920, + scale: 2, + reserved: [0, 22, 0, 0], + }), + ).toEqual({ x: 1920, y: 22, width: 1536, height: 938 }); + }); +}); + +describe("runsUnderHyprland", () => { + // Gates every Hyprland-only behaviour, so no other desktop is touched by it. + it("is true only on Linux with a Hyprland instance", () => { + const env = { HYPRLAND_INSTANCE_SIGNATURE: "sig" }; + expect(runsUnderHyprland(env, "linux")).toBe(true); + expect(runsUnderHyprland({}, "linux")).toBe(false); + expect(runsUnderHyprland({ XDG_CURRENT_DESKTOP: "GNOME" }, "linux")).toBe(false); + expect(runsUnderHyprland(env, "win32")).toBe(false); + }); +}); + +describe("hyprlandSocketPath", () => { + it("is null outside a Hyprland session", () => { + expect(hyprlandSocketPath({ XDG_RUNTIME_DIR: "/run/user/1000" })).toBeNull(); + }); + + it("points at the instance's request socket", () => { + expect( + hyprlandSocketPath({ XDG_RUNTIME_DIR: "/run/user/1000", HYPRLAND_INSTANCE_SIGNATURE: "abc" }), + ).toBe("/run/user/1000/hypr/abc/.socket.sock"); + }); +}); + +describe("placeHudPanelNearBar", () => { + let server: net.Server | null = null; + let runtimeDir = ""; + + afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = null; + if (runtimeDir) fs.rmSync(runtimeDir, { recursive: true, force: true }); + }); + + /** A stand-in for Hyprland's request socket: one reply per command, recorded. */ + async function fakeHyprland(replies: Record) { + runtimeDir = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-hypr-")); + fs.mkdirSync(path.join(runtimeDir, "hypr", "sig"), { recursive: true }); + const received: string[] = []; + server = net.createServer((socket) => { + socket.on("data", (chunk) => { + const command = chunk.toString("utf8"); + received.push(command); + const key = command.startsWith("dispatch") ? "dispatch" : command; + socket.end(replies[key] ?? "unknown request"); + }); + }); + const socketPath = path.join(runtimeDir, "hypr", "sig", ".socket.sock"); + await new Promise((resolve) => server?.listen(socketPath, resolve)); + return { env: { XDG_RUNTIME_DIR: runtimeDir, HYPRLAND_INSTANCE_SIGNATURE: "sig" }, received }; + } + + const clients = JSON.stringify([ + { + address: "0xbar", + pid: 42, + title: "openscreen", + at: [672, 476], + size: [576, 150], + monitor: 2, + }, + { + address: "0xpanel", + pid: 42, + title: "OpenScreen panel", + at: [769, 306], + size: [382, 300], + monitor: 2, + }, + { + address: "0xother", + pid: 7, + title: "OpenScreen panel", + at: [0, 0], + size: [10, 10], + monitor: 2, + }, + ]); + const monitors = JSON.stringify([ + { id: 2, x: 0, y: 0, width: 3840, height: 2160, scale: 2, reserved: [0, 22, 0, 0] }, + ]); + + it("moves this process's panel above its bar", async () => { + const { env, received } = await fakeHyprland({ + "j/clients": clients, + "j/monitors": monitors, + dispatch: "ok", + }); + + const placed = await placeHudPanelNearBar({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + gap: 8, + }); + + expect(placed).toBe(true); + expect(received).toContain( + 'dispatch hl.dsp.window.move({ x = 769, y = 168, window = "address:0xpanel" })', + ); + }); + + it("falls back to the classic dispatcher on a Hyprland without Lua", async () => { + const { env, received } = await fakeHyprland({ + "j/clients": clients, + "j/monitors": monitors, + }); + // The fake answers both dispatch forms alike, so make only the classic one succeed. + server?.removeAllListeners("connection"); + server?.on("connection", (socket) => { + socket.on("data", (chunk) => { + const command = chunk.toString("utf8"); + received.push(command); + if (command === "j/clients") socket.end(clients); + else if (command === "j/monitors") socket.end(monitors); + else socket.end(command.startsWith("dispatch movewindowpixel") ? "ok" : "error: lua"); + }); + }); + + const placed = await placeHudPanelNearBar({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + gap: 8, + }); + + expect(placed).toBe(true); + expect(received).toContain("dispatch movewindowpixel exact 769 168,address:0xpanel"); + }); + + it("moves nothing when the bar is not among this process's windows", async () => { + const { env, received } = await fakeHyprland({ + "j/clients": clients, + "j/monitors": monitors, + dispatch: "ok", + }); + + const placed = await placeHudPanelNearBar({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 999, height: 150 }, + gap: 8, + }); + + expect(placed).toBe(false); + expect(received.some((command) => command.startsWith("dispatch"))).toBe(false); + }); + + // Hyprland centres a new window and ignores the position it asked for; every other + // platform puts the bar at the bottom centre. + it("moves this process's bar to the bottom of its monitor", async () => { + const { env, received } = await fakeHyprland({ + "j/clients": clients, + "j/monitors": monitors, + dispatch: "ok", + }); + + const placed = await placeHudBarAtBottom({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + margin: 5, + }); + + expect(placed).toBe(true); + expect(received).toContain( + 'dispatch hl.dsp.window.move({ x = 672, y = 925, window = "address:0xbar" })', + ); + // Above a maximized or fullscreen window, which otherwise hides it. + expect(received).toContain('dispatch hl.dsp.window.pin({ window = "address:0xbar" })'); + }); + + it("does not unpin a bar that is already pinned", async () => { + const pinned = JSON.stringify( + (JSON.parse(clients) as Record[]).map((client) => ({ + ...client, + pinned: true, + })), + ); + const { env, received } = await fakeHyprland({ + "j/clients": pinned, + "j/monitors": monitors, + dispatch: "ok", + }); + + await placeHudBarAtBottom({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + margin: 5, + }); + + expect(received.some((command) => command.includes("pin"))).toBe(false); + }); + + it("moves no bar it cannot find by its size", async () => { + const { env, received } = await fakeHyprland({ + "j/clients": clients, + "j/monitors": monitors, + dispatch: "ok", + }); + + const placed = await placeHudBarAtBottom({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 999, height: 150 }, + margin: 5, + }); + + expect(placed).toBe(false); + expect(received.some((command) => command.startsWith("dispatch"))).toBe(false); + }); + + // Without the rule the panel maps mid-screen and is then seen jumping to the bar. + it("sets a rule that maps the panel next to the bar before it is shown", async () => { + // A second monitor at x 1920, with the bar on it. + const onSecondMonitor = JSON.stringify([ + { + address: "0xbar", + pid: 42, + title: "openscreen", + at: [1920 + 672, 476], + size: [576, 150], + monitor: 3, + }, + ]); + // Placed in that monitor's work area, but handed to the rule relative to the monitor. + const rule = `eval ${windowPlacementRule("openscreen-hud-panel", "OpenScreen panel", { width: 382, height: 300 }, 769, 168)}`; + const { env, received } = await fakeHyprland({ + "j/clients": onSecondMonitor, + "j/monitors": JSON.stringify([ + { id: 3, x: 1920, y: 0, width: 3840, height: 2160, scale: 2, reserved: [0, 22, 0, 0] }, + ]), + [rule]: "ok", + }); + + const preset = await presetHudPanelPlacement({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + panelSize: { width: 382, height: 300 }, + barSize: { width: 576, height: 150 }, + gap: 8, + }); + + expect(received).toContain(rule); + expect(preset).toBe(true); + }); + + // At launch the bar has not mapped yet: the rule is set for the focused monitor, where it + // opens, so it never appears mid-screen first. + it("sets a rule that maps the bar at the bottom of the focused monitor before it shows", async () => { + const rule = `eval ${windowPlacementRule("openscreen-hud-bar", "OpenScreen recording bar", { width: 528, height: 54 }, 696, 1021)}`; + const { env, received } = await fakeHyprland({ + "j/clients": "[]", + "j/monitors": JSON.stringify([ + { id: 1, x: 0, y: 0, width: 1920, height: 1080, scale: 1, reserved: [0, 0, 0, 0] }, + { + id: 2, + x: 1920, + y: 0, + width: 3840, + height: 2160, + scale: 2, + reserved: [0, 22, 0, 0], + focused: true, + }, + ]), + [rule]: "ok", + }); + + const preset = await presetHudBarPlacement({ + env, + pid: 42, + barTitle: "OpenScreen recording bar", + panelTitle: "OpenScreen panel", + barSize: { width: 528, height: 54 }, + margin: 5, + }); + + // Bottom centre of the focused monitor's work area, relative to that monitor. + expect(received).toContain(rule); + expect(preset).toBe(true); + }); + + // Pinning toggles. Two placements that overlap used to both read `pinned: false` and both + // toggle, leaving the bar unpinned and under a fullscreen window again. + it("pins once when two placements overlap", async () => { + const { env, received } = await fakeHyprland({}); + let pinned = false; + server?.removeAllListeners("connection"); + server?.on("connection", (socket) => { + socket.on("data", (chunk) => { + const command = chunk.toString("utf8"); + received.push(command); + if (command === "j/clients") { + socket.end( + JSON.stringify( + (JSON.parse(clients) as Record[]).map((client) => + client.address === "0xbar" ? { ...client, pinned } : client, + ), + ), + ); + } else if (command === "j/monitors") { + socket.end(monitors); + } else if (command.includes("pin")) { + // Slow, and the state changes only as the reply goes out. + setTimeout(() => { + pinned = !pinned; + socket.end("ok"); + }, 50); + } else { + socket.end("ok"); + } + }); + }); + const place = () => + placeHudBarAtBottom({ + env, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + margin: 5, + }); + + await Promise.all([place(), place()]); + + expect(pinned).toBe(true); + expect(received.filter((command) => command.includes("pin"))).toHaveLength(1); + }); + + it("gives up quietly outside Hyprland", async () => { + await expect( + placeHudPanelNearBar({ + env: {}, + pid: 42, + panelTitle: "OpenScreen panel", + barSize: { width: 576, height: 150 }, + gap: 8, + }), + ).resolves.toBe(false); + }); +}); diff --git a/electron/hyprland.ts b/electron/hyprland.ts new file mode 100644 index 0000000000..e843a2f6f0 --- /dev/null +++ b/electron/hyprland.ts @@ -0,0 +1,431 @@ +// Hyprland: placing the HUD bar and its panel, which Wayland gives a client no way to do. +// +// A toplevel cannot position itself on Wayland, and Hyprland centres every new floating window +// on its monitor. The bar asks for the bottom centre of the screen, where Windows and macOS put +// it, and was left in the middle instead; the panel (src/components/launch/hudPanel.ts) then +// opened on top of it, and a toast covered the very controls it was about. Hyprland's IPC socket +// can move a window, so on Hyprland the bar is moved to the bottom and the panel next to it +// after each maps. The panel, shown and hidden as often as a popover opens, is also placed +// before it maps, by a window rule, or it would be seen opening mid-screen and then jumping. Both are pinned too: `alwaysOnTop` means nothing on Wayland, and a window +// made fullscreen or maximized (Omarchy's Super+F, Super+Alt+F) otherwise covers the bar, +// which is how the user stops a recording. Best effort only: anything unexpected leaves the +// window where the compositor put it. +import net from "node:net"; +import path from "node:path"; + +type Env = Readonly>; + +export interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +interface HyprlandClient { + address: string; + pid: number; + title: string; + at: [number, number]; + size: [number, number]; + monitor: number; + pinned?: boolean; +} + +interface HyprlandMonitor { + id: number; + x: number; + y: number; + width: number; + height: number; + scale: number; + /** Logical px taken by bars and docks: left, top, right, bottom. */ + reserved: [number, number, number, number]; + /** The monitor a new window opens on. */ + focused?: boolean; +} + +const IPC_TIMEOUT_MS = 500; + +/** Whether this process runs in a Hyprland session, whatever distribution ships it. */ +export function runsUnderHyprland(env: Env, platform: string = process.platform): boolean { + return platform === "linux" && Boolean(env.HYPRLAND_INSTANCE_SIGNATURE); +} + +/** The request socket of the Hyprland instance this process runs under, or null outside one. */ +export function hyprlandSocketPath(env: Env): string | null { + const signature = env.HYPRLAND_INSTANCE_SIGNATURE; + const runtimeDir = env.XDG_RUNTIME_DIR; + if (!signature || !runtimeDir) return null; + return path.join(runtimeDir, "hypr", signature, ".socket.sock"); +} + +/** One request over Hyprland's IPC socket; null on any failure, so callers simply give up. */ +function request(socketPath: string, command: string): Promise { + return new Promise((resolve) => { + let reply = ""; + const socket = net.createConnection(socketPath); + const done = (value: string | null) => { + socket.destroy(); + resolve(value); + }; + socket.setTimeout(IPC_TIMEOUT_MS, () => done(null)); + socket.on("connect", () => socket.write(command)); + socket.on("data", (chunk) => { + reply += chunk.toString("utf8"); + }); + socket.on("end", () => done(reply)); + socket.on("error", () => done(null)); + }); +} + +async function requestJson(socketPath: string, command: string): Promise { + const reply = await request(socketPath, `j/${command}`); + if (!reply) return null; + try { + return JSON.parse(reply) as T; + } catch { + return null; + } +} + +/** A monitor's usable area in logical px: its size less the bars and docks reserved on it. */ +export function monitorWorkArea(monitor: HyprlandMonitor): Rect { + const [left, top, right, bottom] = monitor.reserved; + const width = monitor.width / monitor.scale; + const height = monitor.height / monitor.scale; + return { + x: monitor.x + left, + y: monitor.y + top, + width: width - left - right, + height: height - top - bottom, + }; +} + +/** + * Where the panel goes: centred on the bar, above it with `gap` between, or below it when there + * is room below and not above. When neither side has room it takes the roomier one, so the clamp + * into the work area covers as little of the bar as it can. + */ +export function panelPositionNearBar( + bar: Rect, + panel: { width: number; height: number }, + workArea: Rect, + gap: number, +): { x: number; y: number } { + const above = bar.y - gap - panel.height; + const below = bar.y + bar.height + gap; + const roomAbove = bar.y - gap - workArea.y; + const roomBelow = workArea.y + workArea.height - below; + let y = above; + if (panel.height > roomAbove) { + y = panel.height <= roomBelow || roomBelow > roomAbove ? below : above; + } + const x = bar.x + (bar.width - panel.width) / 2; + const clamp = (value: number, min: number, max: number) => + Math.round(Math.min(Math.max(value, min), Math.max(min, max))); + return { + x: clamp(x, workArea.x, workArea.x + workArea.width - panel.width), + y: clamp(y, workArea.y, workArea.y + workArea.height - panel.height), + }; +} + +/** Where the bar goes: centred at the bottom of the work area, `margin` above its edge. */ +export function barPositionAtBottom( + bar: { width: number; height: number }, + workArea: Rect, + margin: number, +): { x: number; y: number } { + return { + x: Math.round(workArea.x + (workArea.width - bar.width) / 2), + y: Math.round(workArea.y + workArea.height - bar.height - margin), + }; +} + +const toRect = (client: HyprlandClient): Rect => ({ + x: client.at[0], + y: client.at[1], + width: client.size[0], + height: client.size[1], +}); + +/** The bar among this process's windows: the size this process gave it, and not the panel. */ +function findBar( + clients: HyprlandClient[], + options: { pid: number; panelTitle: string; barSize: { width: number; height: number } }, +): HyprlandClient | undefined { + return clients.find( + (client) => + client.pid === options.pid && + client.title !== options.panelTitle && + client.size[0] === options.barSize.width && + client.size[1] === options.barSize.height, + ); +} + +/** Names of the window rules that place the panel and the bar as they map; replaced as they change. */ +export const PANEL_RULE = "openscreen-hud-panel"; +export const BAR_RULE = "openscreen-hud-bar"; + +/** + * The Lua that (re)defines a window rule placing the window titled `title`: floating, at its + * size, and at `x`,`y` relative to its monitor, which is how a rule's `move` is read. The size is + * in the rule because a window mapping for the first time has not had the size it was given + * applied yet: it maps small and then grows, away from where it was placed. Not pinned here: a + * window pinned by a rule as it maps stays under a fullscreen window, where one pinned after it + * maps is drawn above it, so the moves after mapping pin it. Null for a title that would need + * escaping. + */ +export function windowPlacementRule( + ruleName: string, + title: string, + size: { width: number; height: number }, + x: number, + y: number, +): string | null { + // Nothing to escape, for the regex or the Lua string it sits in. + if (!/^[\w ]+$/.test(title)) return null; + return ( + `hl.window_rule({ name = "${ruleName}", match = { title = "^(${title})$" }, ` + + `float = true, size = "${Math.round(size.width)} ${Math.round(size.height)}", ` + + `move = "${Math.round(x)} ${Math.round(y)}" })` + ); +} + +/** Name of the window rule that leaves the countdown overlay undecorated. */ +export const COUNTDOWN_RULE = "openscreen-countdown"; + +/** + * The Lua that (re)defines a rule leaving the window titled `title` undecorated: no border, no + * blur behind it, no shadow. For a transparent overlay that draws its own shape: Hyprland frames + * and blurs every window's whole rectangle, which put a bordered, blurred box around the + * countdown's circle. Null for a title that would need escaping. + */ +export function undecoratedWindowRule(ruleName: string, title: string): string | null { + if (!/^[\w ]+$/.test(title)) return null; + return ( + `hl.window_rule({ name = "${ruleName}", match = { title = "^(${title})$" }, ` + + "decorate = false, no_blur = true, no_shadow = true })" + ); +} + +/** + * Sets the rule that leaves the window titled `title` undecorated (see undecoratedWindowRule), + * before it is shown. Resolves false outside Hyprland or on a Hyprland without Lua. + */ +export async function presetUndecoratedWindow(options: { + env: Env; + ruleName: string; + title: string; +}): Promise { + const socketPath = hyprlandSocketPath(options.env); + if (!socketPath) return false; + const rule = undecoratedWindowRule(options.ruleName, options.title); + if (!rule) return false; + return (await request(socketPath, `eval ${rule}`))?.trim() === "ok"; +} + +/** + * Before the panel is shown: a window rule that maps it next to the bar, so it never appears + * where Hyprland would centre it. Resolves false when not under Hyprland, on a Hyprland + * without Lua, or when the bar cannot be found; the panel is then moved after it maps instead. + */ +export async function presetHudPanelPlacement(options: { + env: Env; + pid: number; + panelTitle: string; + panelSize: { width: number; height: number }; + barSize: { width: number; height: number }; + gap: number; +}): Promise { + const socketPath = hyprlandSocketPath(options.env); + if (!socketPath) return false; + const [clients, monitors] = await Promise.all([ + requestJson(socketPath, "clients"), + requestJson(socketPath, "monitors"), + ]); + if (!Array.isArray(clients) || !Array.isArray(monitors)) return false; + const bar = findBar(clients, options); + const monitor = bar && monitors.find((candidate) => candidate.id === bar.monitor); + if (!bar || !monitor) return false; + + const { x, y } = panelPositionNearBar( + toRect(bar), + options.panelSize, + monitorWorkArea(monitor), + options.gap, + ); + const rule = windowPlacementRule( + PANEL_RULE, + options.panelTitle, + options.panelSize, + x - monitor.x, + y - monitor.y, + ); + if (!rule) return false; + return (await request(socketPath, `eval ${rule}`))?.trim() === "ok"; +} + +/** + * Before the bar is shown, and again whenever its size changes: a window rule that maps it at the + * bottom centre, so it never appears where Hyprland would centre it — at launch, or shown again + * after being hidden. On the bar's monitor once it has one, else the focused monitor, where a + * new window opens. Resolves false when not under Hyprland or on a Hyprland without Lua; the bar + * is then moved after it maps instead. + */ +export async function presetHudBarPlacement(options: { + env: Env; + pid: number; + barTitle: string; + panelTitle: string; + barSize: { width: number; height: number }; + margin: number; +}): Promise { + const socketPath = hyprlandSocketPath(options.env); + if (!socketPath) return false; + const [clients, monitors] = await Promise.all([ + requestJson(socketPath, "clients"), + requestJson(socketPath, "monitors"), + ]); + if (!Array.isArray(clients) || !Array.isArray(monitors)) return false; + const bar = findBar(clients, options); + const monitor = + (bar && monitors.find((candidate) => candidate.id === bar.monitor)) ?? + monitors.find((candidate) => candidate.focused); + if (!monitor) return false; + + const { x, y } = barPositionAtBottom(options.barSize, monitorWorkArea(monitor), options.margin); + const rule = windowPlacementRule( + BAR_RULE, + options.barTitle, + options.barSize, + x - monitor.x, + y - monitor.y, + ); + if (!rule) return false; + return (await request(socketPath, `eval ${rule}`))?.trim() === "ok"; +} + +/** + * Moves the panel window next to the bar. Both are found among this process's clients: the + * panel by its title, the bar by the size this process gave it. Resolves false when not under + * Hyprland or when anything does not match, in which case nothing was moved. + */ +export function placeHudPanelNearBar(options: { + env: Env; + pid: number; + panelTitle: string; + barSize: { width: number; height: number }; + gap: number; +}): Promise { + return oneAtATime(async () => { + const socketPath = hyprlandSocketPath(options.env); + if (!socketPath) return false; + const [clients, monitors] = await Promise.all([ + requestJson(socketPath, "clients"), + requestJson(socketPath, "monitors"), + ]); + if (!Array.isArray(clients) || !Array.isArray(monitors)) return false; + + const panel = clients.find( + (client) => client.pid === options.pid && client.title === options.panelTitle, + ); + const bar = findBar(clients, options); + const monitor = bar && monitors.find((candidate) => candidate.id === bar.monitor); + if (!panel || !bar || !monitor) return false; + + const { x, y } = panelPositionNearBar( + toRect(bar), + { width: panel.size[0], height: panel.size[1] }, + monitorWorkArea(monitor), + options.gap, + ); + await pinClient(socketPath, panel); + if (x === panel.at[0] && y === panel.at[1]) return true; + return moveClient(socketPath, panel.address, x, y); + }); +} + +/** + * Moves the bar window to the bottom centre of the monitor it is on. Found among this + * process's clients by the size this process gave it, skipping the panel. Resolves false + * when not under Hyprland or when anything does not match, in which case nothing was moved. + */ +export function placeHudBarAtBottom(options: { + env: Env; + pid: number; + panelTitle: string; + barSize: { width: number; height: number }; + margin: number; +}): Promise { + return oneAtATime(async () => { + const socketPath = hyprlandSocketPath(options.env); + if (!socketPath) return false; + const [clients, monitors] = await Promise.all([ + requestJson(socketPath, "clients"), + requestJson(socketPath, "monitors"), + ]); + if (!Array.isArray(clients) || !Array.isArray(monitors)) return false; + + const bar = findBar(clients, options); + const monitor = bar && monitors.find((candidate) => candidate.id === bar.monitor); + if (!bar || !monitor) return false; + + const { x, y } = barPositionAtBottom( + { width: bar.size[0], height: bar.size[1] }, + monitorWorkArea(monitor), + options.margin, + ); + await pinClient(socketPath, bar); + if (x === bar.at[0] && y === bar.at[1]) return true; + return moveClient(socketPath, bar.address, x, y); + }); +} + +/** + * Pins a window: shown on every workspace and above a fullscreen or maximized one. Both + * dispatchers toggle, so an already pinned window is left alone. + */ +/** The placement in flight, which the next one waits for; see `oneAtATime`. */ +let placementInFlight: Promise = Promise.resolve(); + +/** + * Runs placements one at a time. Each decides whether to pin from the `pinned` of its own + * `clients` snapshot, and pinning toggles: two overlapping placements could both read + * `pinned: false` and both toggle, leaving the window unpinned — under a fullscreen window + * again — until the next one. A failed placement does not hold up the ones after it. + */ +function oneAtATime(run: () => Promise): Promise { + const result = placementInFlight.then(run, run); + placementInFlight = result.catch(() => undefined); + return result; +} + +async function pinClient(socketPath: string, client: HyprlandClient): Promise { + if (client.pinned) return; + const target = `address:${client.address}`; + for (const command of [ + `dispatch hl.dsp.window.pin({ window = "${target}" })`, + `dispatch pin ${target}`, + ]) { + if ((await request(socketPath, command))?.trim() === "ok") return; + } +} + +async function moveClient( + socketPath: string, + address: string, + x: number, + y: number, +): Promise { + // Hyprland 0.56 evaluates `dispatch` as Lua (hl.dsp.window.move); earlier releases only + // know the classic dispatcher. Each refuses the other's syntax, so try both in turn. + const target = `address:${address}`; + for (const command of [ + `dispatch hl.dsp.window.move({ x = ${x}, y = ${y}, window = "${target}" })`, + `dispatch movewindowpixel exact ${x} ${y},${target}`, + ]) { + if ((await request(socketPath, command))?.trim() === "ok") return true; + } + return false; +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 75c3a23df7..6fd1ea7bdb 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -72,6 +72,7 @@ import { AppSettingsStore } from "../app-settings"; import { createRunnerHost } from "../cli/runnerHost"; import { hideCountdownOverlay } from "../countdownOverlay"; import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer"; +import { runsUnderHyprland } from "../hyprland"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; import { @@ -90,8 +91,9 @@ import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { + POINTER_TELEMETRY_SUFFIX, + readCursorActivity, readCursorRecordingFile as readCursorRecordingFileFrom, - readCursorSidecar, readCursorTelemetryFile as readCursorTelemetryFileFrom, } from "../media/cursorSidecar"; import { findMediaLinksByFingerprint, registerMediaLinks } from "../media/mediaLinksRegistry"; @@ -920,6 +922,28 @@ let preparedLinuxCapture: { * nothing recording behind it. */ let preparingLinuxCaptureToken: symbol | null = null; +/** + * The source of the take Restart just discarded, to share again without the picker. + * + * Memory only, and spent by the next start whatever it is: a token replayed on a + * fresh recording would record the old source with no way to pick another. Restart + * needs it because a second picker there stalls behind the portal closing the + * discarded take's session (a 25 s timeout on Hyprland), so nothing appears. + */ +let linuxRestartRestoreToken: string | null = null; +/** + * Whether takes ask the portal for a grant Restart can replay. Only on Hyprland, the one place + * the second picker was seen to stall: everywhere else a take keeps the unpersisted grant it + * always had, leaves nothing in the portal's permission store, and Restart asks again. + */ +const LINUX_CAPTURE_PERSISTS_GRANT = runsUnderHyprland(process.env); + +/** Hands the discarded take's token to a Restart, and drops it for anything else. */ +function takeLinuxRestartRestoreToken(request: NativeLinuxRecordingRequest): string | undefined { + const token = linuxRestartRestoreToken; + linuxRestartRestoreToken = null; + return request?.restart && token ? token : undefined; +} /** * Claims the prepared session when it matches the recording about to start. @@ -1057,7 +1081,7 @@ const agentCursorTelemetryReader: CursorTelemetryReader = { probe: async ({ originalPath }) => { const approved = resolveApprovedVideoPath(originalPath); if (!approved) return false; - return (await readCursorSidecar(approved, cursorSidecarOptions())).found; + return (await readCursorActivity(approved, cursorSidecarOptions())).found; }, read: async ({ assetId, originalPath }) => { const approved = resolveApprovedVideoPath(originalPath); @@ -1068,7 +1092,7 @@ const agentCursorTelemetryReader: CursorTelemetryReader = { note: "The asset's file is outside the folders this app may read, so its cursor sidecar was not opened.", }; } - const sidecar = await readCursorSidecar(approved, cursorSidecarOptions()); + const sidecar = await readCursorActivity(approved, cursorSidecarOptions()); if (!sidecar.found) return { status: "no-sidecar", assetId }; return { status: "ok", assetId, samples: sidecar.data.samples }; }, @@ -2630,6 +2654,7 @@ export function registerIpcHandlers( return { success: false, reason: "already-recording" }; } discardPreparedLinuxCapture("superseded by a new prepare"); + linuxRestartRestoreToken = null; const token = Symbol("prepare-native-linux-recording"); preparingLinuxCaptureToken = token; @@ -2664,6 +2689,7 @@ export function registerIpcHandlers( }, maxCursorSamples: MAX_CURSOR_SAMPLES, deferStart: true, + ...(LINUX_CAPTURE_PERSISTS_GRANT ? { persistGrant: true } : {}), }); lastLinuxCaptureSession = session; @@ -2731,6 +2757,7 @@ export function registerIpcHandlers( // below, just with the picker after its countdown instead of // before. Nothing has to know which path it is on. const prepared = takePreparedLinuxSession(outputPath, request); + const restoreToken = takeLinuxRestartRestoreToken(request); const session = prepared ?? new LinuxNativeCaptureSession({ @@ -2749,12 +2776,15 @@ export function registerIpcHandlers( }, }, maxCursorSamples: MAX_CURSOR_SAMPLES, + ...(restoreToken ? { restoreToken } : {}), + ...(LINUX_CAPTURE_PERSISTS_GRANT ? { persistGrant: true } : {}), }); lastLinuxCaptureSession = session; console.info("[native-linux] starting capture", { outputPath, prepared: Boolean(prepared), + restoredSource: Boolean(restoreToken && !prepared), cursor: { mode: cursorCaptureMode }, audio: request.audio, video: request.video, @@ -2772,10 +2802,13 @@ export function registerIpcHandlers( linuxNativeCaptureSession = session; linuxNativeCaptureRecordingId = recordingId; - // The portal may lack EMBEDDED and get METADATA instead (#1357): the - // pointer is then not in the pixels, so the editor has to draw it. + // The negotiated mode, not the requested one: the portal may lack + // EMBEDDED and get METADATA instead (#1357), so the editor has to draw + // the pointer; or lack METADATA and get HIDDEN (Hyprland, positions from + // its IPC), so the editor draws it too; or EMBEDDED, so the pointer is + // already in the pixels and its samples, if any, only place zooms. linuxNativeCaptureCursorMode = - session.cursorMode === "metadata" ? "editable-overlay" : cursorCaptureMode; + session.cursorMode === "embedded" ? "system" : "editable-overlay"; // The portal's answer, not an in-app selection — on Wayland there // is none to have. This used to read `selectedSource || { name: @@ -2826,12 +2859,14 @@ export function registerIpcHandlers( } try { + linuxRestartRestoreToken = discard ? (session.restoreToken ?? null) : null; if (discard) { session.discard(); const discarded = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); await Promise.all([ fs.rm(discarded, { force: true }), fs.rm(`${discarded}.cursor.json`, { force: true }), + fs.rm(`${discarded}${POINTER_TELEMETRY_SUFFIX}`, { force: true }), ]); return { success: true, discarded: true }; } @@ -2848,11 +2883,24 @@ export function registerIpcHandlers( "utf-8", ); } + // The pointer is painted into the video, yet the helper still reported where it + // was (Hyprland's IPC, when its portal could not leave it out). Kept apart from + // `.cursor.json`, which the editor and the compositor DRAW: these samples only + // place zooms and click sounds, over the pointer already in the pixels. + const pointerTelemetry = cursorCaptureMode === "system" && result.cursor.samples.length > 0; + if (pointerTelemetry) { + await fs.writeFile( + `${result.path}${POINTER_TELEMETRY_SUFFIX}`, + JSON.stringify({ ...result.cursor, assets: [] }, null, 2), + "utf-8", + ); + } const session_: RecordingSession = { screenVideoPath: result.path, createdAt: recordingId, cursorCaptureMode, + ...(pointerTelemetry ? { pointerTelemetry: true } : {}), }; setCurrentRecordingSessionState(session_); currentProjectPath = null; @@ -3919,12 +3967,21 @@ export function registerIpcHandlers( const webcamOffsetMs = Number.isFinite(payload.webcamOffsetMs) ? payload.webcamOffsetMs : undefined; + // This session replaces the one the stopped take stored, so it carries that + // take's pointer telemetry forward: read from the file, which is what set it. + const pointerTelemetry = + cursorCaptureMode === "system" && + (await fs + .access(`${screenVideoPath}${POINTER_TELEMETRY_SUFFIX}`, fsConstants.F_OK) + .then(() => true) + .catch(() => false)); const session: RecordingSession = { screenVideoPath, webcamVideoPath, createdAt, ...(webcamOffsetMs !== undefined ? { webcamOffsetMs } : {}), ...(cursorCaptureMode ? { cursorCaptureMode } : {}), + ...(pointerTelemetry ? { pointerTelemetry: true } : {}), }; setCurrentRecordingSessionState(session); currentProjectPath = null; diff --git a/electron/media/cursorSidecar.test.ts b/electron/media/cursorSidecar.test.ts index 88c7ed0dac..7899829b62 100644 --- a/electron/media/cursorSidecar.test.ts +++ b/electron/media/cursorSidecar.test.ts @@ -13,6 +13,9 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + POINTER_TELEMETRY_SUFFIX, + readCursorActivity, + readCursorRecordingFile, readCursorRecordingFileAt, readCursorSidecar, readCursorTelemetryFile, @@ -193,4 +196,64 @@ describe("readCursorTelemetryFile", () => { "middle-click", ]); }); + + it("falls back to the pointer telemetry of a take with the pointer painted in", async () => { + const video = path.join(dir, "painted.mp4"); + await fs.writeFile( + `${video}${POINTER_TELEMETRY_SUFFIX}`, + JSON.stringify({ samples: [{ timeMs: 10, cx: 0.25, cy: 0.75, interactionType: "click" }] }), + "utf-8", + ); + + const telemetry = await readCursorTelemetryFile(video, {}); + + expect(telemetry.samples).toEqual([ + { timeMs: 10, cx: 0.25, cy: 0.75, visible: true, interactionType: "click" }, + ]); + // What draws a cursor never sees it: a second pointer would sit over the one + // already in the pixels. + expect((await readCursorRecordingFile(video, {})).samples).toEqual([]); + }); + + it("gives the agent the pointer telemetry of a painted-in take", async () => { + const video = path.join(dir, "painted-agent.mp4"); + await fs.writeFile( + `${video}${POINTER_TELEMETRY_SUFFIX}`, + JSON.stringify({ samples: [{ timeMs: 10, cx: 0.25, cy: 0.75, interactionType: "click" }] }), + "utf-8", + ); + + const activity = await readCursorActivity(video, {}); + + expect(activity.found).toBe(true); + expect(activity.source).toBe("pointer"); + expect(activity.data.samples.map((sample) => sample.cx)).toEqual([0.25]); + // The drawable lookup still finds nothing. + expect((await readCursorSidecar(video, {})).found).toBe(false); + }); + + it("reports a malformed pointer telemetry file as not found", async () => { + const video = path.join(dir, "painted-broken.mp4"); + await fs.writeFile(`${video}${POINTER_TELEMETRY_SUFFIX}`, "{ not json", "utf-8"); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const activity = await readCursorActivity(video, {}); + + expect(activity.found).toBe(false); + expect(activity.source).toBe("none"); + }); + + it("prefers the drawable sidecar when a take has both", async () => { + const video = path.join(dir, "both.mp4"); + await writeSidecar(video, { samples: [{ timeMs: 10, cx: 0.1, cy: 0.1 }] }); + await fs.writeFile( + `${video}${POINTER_TELEMETRY_SUFFIX}`, + JSON.stringify({ samples: [{ timeMs: 10, cx: 0.9, cy: 0.9 }] }), + "utf-8", + ); + + const telemetry = await readCursorTelemetryFile(video, {}); + + expect(telemetry.samples.map((sample) => sample.cx)).toEqual([0.1]); + }); }); diff --git a/electron/media/cursorSidecar.ts b/electron/media/cursorSidecar.ts index 0a588e0ae7..7e16ad5ff2 100644 --- a/electron/media/cursorSidecar.ts +++ b/electron/media/cursorSidecar.ts @@ -27,10 +27,19 @@ import { findMediaLinksByFingerprint } from "./mediaLinksRegistry"; export const CURSOR_TELEMETRY_VERSION = 2; +/** + * `