From f169c17e188e71f649eef7e04035ee005eef460b Mon Sep 17 00:00:00 2001 From: redphx <96280+redphx@users.noreply.github.com> Date: Fri, 21 Jun 2024 17:45:43 +0700 Subject: [PATCH] Add WebGL2 renderer --- LICENSE | 1 + package.json | 6 +- src/assets/css/number-stepper.styl | 6 + src/assets/css/root.styl | 5 + src/assets/css/stream-settings.styl | 5 + src/assets/css/stream-stats.styl | 1 - src/assets/css/stream.styl | 8 + .../game-pass-gallery.ts} | 0 .../mkb/definitions.ts => enums/mkb.ts} | 2 +- src/{utils => enums}/prompt-font.ts | 0 src/enums/stream-player.ts | 9 + src/enums/user-agent.ts | 10 + src/index.ts | 13 +- src/modules/controller-shortcut.ts | 4 +- src/modules/mkb/key-helper.ts | 2 +- src/modules/mkb/mkb-handler.ts | 2 +- src/modules/mkb/mkb-preset.ts | 2 +- src/modules/mkb/mkb-remapper.ts | 3 +- src/modules/patcher.ts | 16 +- src/modules/patches/expose-stream-session.js | 29 ++ src/modules/player/shaders/clarity_boost.fs | 121 ++++++++ src/modules/player/shaders/clarity_boost.vert | 5 + src/modules/player/webgl2-player.ts | 250 ++++++++++++++++ src/modules/stream-player.ts | 268 ++++++++++++++++++ src/modules/stream/stream-stats.ts | 7 +- src/modules/ui/global-settings.ts | 3 +- src/modules/ui/ui.ts | 181 ++++-------- src/types/index.d.ts | 6 +- src/types/mkb.d.ts | 2 +- src/utils/bx-exposed.ts | 3 +- src/utils/monkey-patches.ts | 61 ++-- src/utils/network.ts | 2 +- src/utils/preferences.ts | 34 ++- src/utils/preload-state.ts | 2 +- src/utils/screenshot.ts | 32 ++- src/utils/settings.ts | 2 +- src/utils/translation.ts | 24 +- src/utils/user-agent.ts | 45 ++- src/utils/utils.ts | 2 +- tsconfig.json | 1 + 40 files changed, 955 insertions(+), 220 deletions(-) rename src/{utils/gamepass-gallery.ts => enums/game-pass-gallery.ts} (100%) rename src/{modules/mkb/definitions.ts => enums/mkb.ts} (98%) rename src/{utils => enums}/prompt-font.ts (100%) create mode 100644 src/enums/stream-player.ts create mode 100644 src/enums/user-agent.ts create mode 100644 src/modules/patches/expose-stream-session.js create mode 100644 src/modules/player/shaders/clarity_boost.fs create mode 100644 src/modules/player/shaders/clarity_boost.vert create mode 100644 src/modules/player/webgl2-player.ts create mode 100644 src/modules/stream-player.ts diff --git a/LICENSE b/LICENSE index 91d8739..a71b95a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ MIT License Copyright (c) 2023 redphx +Copyright (c) 2023 Advanced Micro Devices, Inc. Copyright (c) 2020 Phosphor Icons Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/package.json b/package.json index c7cfd96..b201482 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,12 @@ "build": "build.ts" }, "devDependencies": { - "@types/bun": "^1.1.3", - "@types/node": "^20.13.0", + "@types/bun": "^1.1.5", + "@types/node": "^20.14.7", "@types/stylus": "^0.48.42", "stylus": "^0.63.0" }, "peerDependencies": { - "typescript": "^5.4.5" + "typescript": "^5.5.2" } } diff --git a/src/assets/css/number-stepper.styl b/src/assets/css/number-stepper.styl index c1c154a..e6ad59f 100644 --- a/src/assets/css/number-stepper.styl +++ b/src/assets/css/number-stepper.styl @@ -47,4 +47,10 @@ input[type=range]:disabled, button:disabled { display: none; } + + &[data-disabled=true] { + input[type=range], button { + display: none; + } + } } diff --git a/src/assets/css/root.styl b/src/assets/css/root.styl index 75bfd2d..a4e6d72 100644 --- a/src/assets/css/root.styl +++ b/src/assets/css/root.styl @@ -79,6 +79,11 @@ div[class^=HUDButton-module__hiddenContainer] ~ div:not([class^=HUDButton-module visibility: hidden !important; } +.bx-pixel { + width: 1px !important; + height: 1px !important; +} + .bx-no-margin { margin: 0 !important; } diff --git a/src/assets/css/stream-settings.styl b/src/assets/css/stream-settings.styl index 6446f73..24e40bc 100644 --- a/src/assets/css/stream-settings.styl +++ b/src/assets/css/stream-settings.styl @@ -113,6 +113,11 @@ background: transparent; text-align-last: right; border: none; + color: #fff; + } + + select option:disabled { + display: none; } } diff --git a/src/assets/css/stream-stats.styl b/src/assets/css/stream-stats.styl index aea7f0e..0f0d02f 100644 --- a/src/assets/css/stream-stats.styl +++ b/src/assets/css/stream-stats.styl @@ -16,7 +16,6 @@ margin: 0 8px 8px 0; box-shadow: 0px 0px 6px #000; border-radius: 4px; - height: 30px; } .bx-badge-name { diff --git a/src/assets/css/stream.styl b/src/assets/css/stream.styl index c2c8722..50beea1 100644 --- a/src/assets/css/stream.styl +++ b/src/assets/css/stream.styl @@ -66,6 +66,14 @@ div[data-testid=media-container] { background: #000; } +#game-stream canvas { + position: absolute; + align-self: center; + margin: auto; + left: 0; + right: 0; +} + #gamepass-dialog-root div[class^=Guide-module__guide] { .bx-button { overflow: visible; diff --git a/src/utils/gamepass-gallery.ts b/src/enums/game-pass-gallery.ts similarity index 100% rename from src/utils/gamepass-gallery.ts rename to src/enums/game-pass-gallery.ts diff --git a/src/modules/mkb/definitions.ts b/src/enums/mkb.ts similarity index 98% rename from src/modules/mkb/definitions.ts rename to src/enums/mkb.ts index 2ab7431..8140eb2 100644 --- a/src/modules/mkb/definitions.ts +++ b/src/enums/mkb.ts @@ -1,5 +1,5 @@ import type { GamepadKeyNameType } from "@/types/mkb"; -import { PrompFont } from "@/utils/prompt-font"; +import { PrompFont } from "@enums/prompt-font"; export enum GamepadKey { A = 0, diff --git a/src/utils/prompt-font.ts b/src/enums/prompt-font.ts similarity index 100% rename from src/utils/prompt-font.ts rename to src/enums/prompt-font.ts diff --git a/src/enums/stream-player.ts b/src/enums/stream-player.ts new file mode 100644 index 0000000..4379e45 --- /dev/null +++ b/src/enums/stream-player.ts @@ -0,0 +1,9 @@ +export enum StreamPlayerType { + VIDEO = 'default', + WEBGL2 = 'webgl2', +} + +export enum StreamVideoProcessing { + USM = 'usm', + CAS = 'cas', +} diff --git a/src/enums/user-agent.ts b/src/enums/user-agent.ts new file mode 100644 index 0000000..d685f84 --- /dev/null +++ b/src/enums/user-agent.ts @@ -0,0 +1,10 @@ +export enum UserAgentProfile { + WINDOWS_EDGE = 'windows-edge', + MACOS_SAFARI = 'macos-safari', + SMARTTV_GENERIC = 'smarttv-generic', + SMARTTV_TIZEN = 'smarttv-tizen', + VR_OCULUS = 'vr-oculus', + ANDROID_KIWI_V123 = 'android-kiwi-v123', + DEFAULT = 'default', + CUSTOM = 'custom', +} diff --git a/src/index.ts b/src/index.ts index ddd3851..725fed7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { StreamBadges } from "@modules/stream/stream-badges"; import { StreamStats } from "@modules/stream/stream-stats"; import { addCss } from "@utils/css"; import { Toast } from "@utils/toast"; -import { setupStreamUi, updateVideoPlayerCss } from "@modules/ui/ui"; +import { setupStreamUi, updateVideoPlayer } from "@modules/ui/ui"; import { PrefKey, getPref } from "@utils/preferences"; import { LoadingScreen } from "@modules/loading-screen"; import { MouseCursorHider } from "@modules/mkb/mouse-cursor-hider"; @@ -146,9 +146,6 @@ window.addEventListener(BxEvent.STREAM_STARTING, e => { }); window.addEventListener(BxEvent.STREAM_PLAYING, e => { - const $video = (e as any).$video as HTMLVideoElement; - STATES.currentStream.$video = $video; - STATES.isPlaying = true; injectStreamMenuButtons(); @@ -159,9 +156,10 @@ window.addEventListener(BxEvent.STREAM_PLAYING, e => { gameBar.showBar(); } + const $video = (e as any).$video as HTMLVideoElement; Screenshot.updateCanvasSize($video.videoWidth, $video.videoHeight); - updateVideoPlayerCss(); + updateVideoPlayer(); }); window.addEventListener(BxEvent.STREAM_ERROR_PAGE, e => { @@ -177,6 +175,9 @@ function unload() { EmulatedMkbHandler.getInstance().destroy(); NativeMkbHandler.getInstance().destroy(); + // Destroy StreamPlayer + STATES.currentStream.streamPlayer?.destroy(); + STATES.isPlaying = false; STATES.currentStream = {}; window.BX_EXPOSED.shouldShowSensorControls = false; @@ -187,8 +188,6 @@ function unload() { $streamSettingsDialog.classList.add('bx-gone'); } - STATES.currentStream.audioGainNode = null; - STATES.currentStream.$video = null; StreamStats.getInstance().onStoppedPlaying(); MouseCursorHider.stop(); diff --git a/src/modules/controller-shortcut.ts b/src/modules/controller-shortcut.ts index faf1920..bcd24a7 100644 --- a/src/modules/controller-shortcut.ts +++ b/src/modules/controller-shortcut.ts @@ -1,6 +1,6 @@ import { Screenshot } from "@utils/screenshot"; -import { GamepadKey } from "./mkb/definitions"; -import { PrompFont } from "@utils/prompt-font"; +import { GamepadKey } from "@enums/mkb"; +import { PrompFont } from "@enums/prompt-font"; import { CE } from "@utils/html"; import { t } from "@utils/translation"; import { EmulatedMkbHandler } from "./mkb/mkb-handler"; diff --git a/src/modules/mkb/key-helper.ts b/src/modules/mkb/key-helper.ts index 47a8161..fbde4b6 100644 --- a/src/modules/mkb/key-helper.ts +++ b/src/modules/mkb/key-helper.ts @@ -1,4 +1,4 @@ -import { MouseButtonCode, WheelCode } from "./definitions"; +import { MouseButtonCode, WheelCode } from "@enums/mkb"; export class KeyHelper { static #NON_PRINTABLE_KEYS = { diff --git a/src/modules/mkb/mkb-handler.ts b/src/modules/mkb/mkb-handler.ts index c755a90..c5cf0e6 100644 --- a/src/modules/mkb/mkb-handler.ts +++ b/src/modules/mkb/mkb-handler.ts @@ -1,5 +1,5 @@ import { MkbPreset } from "./mkb-preset"; -import { GamepadKey, MkbPresetKey, GamepadStick, MouseMapTo, WheelCode } from "./definitions"; +import { GamepadKey, MkbPresetKey, GamepadStick, MouseMapTo, WheelCode } from "@enums/mkb"; import { createButton, ButtonStyle, CE } from "@utils/html"; import { BxEvent } from "@utils/bx-event"; import { PrefKey, getPref } from "@utils/preferences"; diff --git a/src/modules/mkb/mkb-preset.ts b/src/modules/mkb/mkb-preset.ts index dfb0572..81ea4a8 100644 --- a/src/modules/mkb/mkb-preset.ts +++ b/src/modules/mkb/mkb-preset.ts @@ -1,6 +1,6 @@ import { t } from "@utils/translation"; import { SettingElementType } from "@utils/settings"; -import { GamepadKey, MouseButtonCode, MouseMapTo, MkbPresetKey } from "./definitions"; +import { GamepadKey, MouseButtonCode, MouseMapTo, MkbPresetKey } from "@enums/mkb"; import { EmulatedMkbHandler } from "./mkb-handler"; import type { MkbPresetData, MkbConvertedPresetData } from "@/types/mkb"; import type { PreferenceSettings } from "@/types/preferences"; diff --git a/src/modules/mkb/mkb-remapper.ts b/src/modules/mkb/mkb-remapper.ts index 7005a44..ee7305d 100644 --- a/src/modules/mkb/mkb-remapper.ts +++ b/src/modules/mkb/mkb-remapper.ts @@ -1,9 +1,7 @@ -import { GamepadKey } from "./definitions"; import { CE, createButton, ButtonStyle } from "@utils/html"; import { t } from "@utils/translation"; import { Dialog } from "@modules/dialog"; import { getPref, setPref, PrefKey } from "@utils/preferences"; -import { MkbPresetKey, GamepadKeyName } from "./definitions"; import { KeyHelper } from "./key-helper"; import { MkbPreset } from "./mkb-preset"; import { EmulatedMkbHandler } from "./mkb-handler"; @@ -11,6 +9,7 @@ import { LocalDb } from "@utils/local-db"; import { BxIcon } from "@utils/bx-icon"; import { SettingElement } from "@utils/settings"; import type { MkbPresetData, MkbStoredPresets } from "@/types/mkb"; +import { MkbPresetKey, GamepadKey, GamepadKeyName } from "@enums/mkb"; type MkbRemapperElements = { diff --git a/src/modules/patcher.ts b/src/modules/patcher.ts index f497f23..c0793ed 100644 --- a/src/modules/patcher.ts +++ b/src/modules/patcher.ts @@ -7,6 +7,7 @@ import { hashCode, renderString } from "@utils/utils"; import { BxEvent } from "@/utils/bx-event"; import codeControllerShortcuts from "./patches/controller-shortcuts.js" with { type: "text" }; +import codeExposeStreamSession from "./patches/expose-stream-session.js" with { type: "text" }; import codeLocalCoOpEnable from "./patches/local-co-op-enable.js" with { type: "text" }; import codeRemotePlayEnable from "./patches/remote-play-enable.js" with { type: "text" }; import codeRemotePlayKeepAlive from "./patches/remote-play-keep-alive.js" with { type: "text" }; @@ -565,20 +566,7 @@ window.dispatchEvent(new Event('${BxEvent.STREAM_EVENT_TARGET_READY}')) } const newCode = `; -window.BX_EXPOSED.streamSession = this; - -const orgSetMicrophoneState = this.setMicrophoneState.bind(this); -this.setMicrophoneState = state => { - orgSetMicrophoneState(state); - - const evt = new Event('${BxEvent.MICROPHONE_STATE_CHANGED}'); - evt.microphoneState = state; - - window.dispatchEvent(evt); -}; - -window.dispatchEvent(new Event('${BxEvent.STREAM_SESSION_READY}')) - +${codeExposeStreamSession} true` + text; str = str.replace(text, newCode); diff --git a/src/modules/patches/expose-stream-session.js b/src/modules/patches/expose-stream-session.js new file mode 100644 index 0000000..4584d41 --- /dev/null +++ b/src/modules/patches/expose-stream-session.js @@ -0,0 +1,29 @@ +window.BX_EXPOSED.streamSession = this; + +const orgSetMicrophoneState = this.setMicrophoneState.bind(this); +this.setMicrophoneState = state => { + orgSetMicrophoneState(state); + + const evt = new Event(BxEvent.MICROPHONE_STATE_CHANGED); + evt.microphoneState = state; + + window.dispatchEvent(evt); +}; + +window.dispatchEvent(new Event(BxEvent.STREAM_SESSION_READY)); + +// Patch updateDimensions() to make native touch work correctly with WebGL2 +let updateDimensionsStr = this.updateDimensions.toString(); + +// if(r){ +const renderTargetVar = updateDimensionsStr.match(/if\((\w+)\){/)[1]; + +updateDimensionsStr = updateDimensionsStr.replaceAll(renderTargetVar + '.scroll', 'scroll'); + +updateDimensionsStr = updateDimensionsStr.replace(`if(${renderTargetVar}){`, ` +if (${renderTargetVar}) { + const scrollWidth = ${renderTargetVar}.dataset.width ? parseInt(${renderTargetVar}.dataset.width) : ${renderTargetVar}.scrollWidth; + const scrollHeight = ${renderTargetVar}.dataset.height ? parseInt(${renderTargetVar}.dataset.height) : ${renderTargetVar}.scrollHeight; +`); + +eval(`this.updateDimensions = function ${updateDimensionsStr}`); diff --git a/src/modules/player/shaders/clarity_boost.fs b/src/modules/player/shaders/clarity_boost.fs new file mode 100644 index 0000000..5dc265b --- /dev/null +++ b/src/modules/player/shaders/clarity_boost.fs @@ -0,0 +1,121 @@ +const int FILTER_UNSHARP_MASKING = 1; +const int FILTER_CAS = 2; + +precision highp float; +uniform sampler2D data; +uniform vec2 iResolution; + +uniform int filterId; +uniform float sharpenFactor; +uniform float brightness; +uniform float contrast; +uniform float saturation; + +vec3 textureAt(sampler2D tex, vec2 coord) { + return texture2D(tex, coord / iResolution.xy).rgb; +} + +vec3 clarityBoost(sampler2D tex, vec2 coord) +{ + // Load a collection of samples in a 3x3 neighorhood, where e is the current pixel. + // a b c + // d e f + // g h i + vec3 a = textureAt(tex, coord + vec2(-1, 1)); + vec3 b = textureAt(tex, coord + vec2(0, 1)); + vec3 c = textureAt(tex, coord + vec2(1, 1)); + + vec3 d = textureAt(tex, coord + vec2(-1, 0)); + vec3 e = textureAt(tex, coord); + vec3 f = textureAt(tex, coord + vec2(1, 0)); + + vec3 g = textureAt(tex, coord + vec2(-1, -1)); + vec3 h = textureAt(tex, coord + vec2(0, -1)); + vec3 i = textureAt(tex, coord + vec2(1, -1)); + + if (filterId == FILTER_CAS) { + // Soft min and max. + // a b c b + // d e f * 0.5 + d e f * 0.5 + // g h i h + // These are 2.0x bigger (factored out the extra multiply). + vec3 minRgb = min(min(min(d, e), min(f, b)), h); + vec3 minRgb2 = min(min(a, c), min(g, i)); + minRgb += min(minRgb, minRgb2); + + vec3 maxRgb = max(max(max(d, e), max(f, b)), h); + vec3 maxRgb2 = max(max(a, c), max(g, i)); + maxRgb += max(maxRgb, maxRgb2); + + // Smooth minimum distance to signal limit divided by smooth max. + vec3 reciprocalMaxRgb = 1.0 / maxRgb; + vec3 amplifyRgb = clamp(min(minRgb, 2.0 - maxRgb) * reciprocalMaxRgb, 0.0, 1.0); + + // Shaping amount of sharpening. + amplifyRgb = inversesqrt(amplifyRgb); + + float contrast = 0.8; + float peak = -3.0 * contrast + 8.0; + vec3 weightRgb = -(1.0 / (amplifyRgb * peak)); + + vec3 reciprocalWeightRgb = 1.0 / (4.0 * weightRgb + 1.0); + + // 0 w 0 + // Filter shape: w 1 w + // 0 w 0 + vec3 window = (b + d) + (f + h); + vec3 outColor = clamp((window * weightRgb + e) * reciprocalWeightRgb, 0.0, 1.0); + + outColor = mix(e, outColor, sharpenFactor / 2.0); + + return outColor; + } else if (filterId == FILTER_UNSHARP_MASKING) { + vec3 gaussianBlur = (a * 1.0 + b * 2.0 + c * 1.0 + + d * 2.0 + e * 4.0 + f * 2.0 + + g * 1.0 + h * 2.0 + i * 1.0) / 16.0; + + // Return edge detection + return e + (e - gaussianBlur) * sharpenFactor / 3.0; + } + + return e; +} + +vec3 adjustBrightness(vec3 color) { + return (1.0 + brightness) * color; +} + +vec3 adjustContrast(vec3 color) { + return 0.5 + (1.0 + contrast) * (color - 0.5); +} + +vec3 adjustSaturation(vec3 color) { + const vec3 luminosityFactor = vec3(0.2126, 0.7152, 0.0722); + vec3 grayscale = vec3(dot(color, luminosityFactor)); + + return mix(grayscale, color, 1.0 + saturation); +} + +void main() { + vec3 color; + + if (sharpenFactor > 0.0) { + color = clarityBoost(data, gl_FragCoord.xy); + } else { + color = textureAt(data, gl_FragCoord.xy); + } + + if (saturation != 0.0) { + color = adjustSaturation(color); + } + + if (contrast != 0.0) { + color = adjustContrast(color); + } + + if (brightness != 0.0) { + color = adjustBrightness(color); + } + + gl_FragColor = vec4(color, 1.0); +} diff --git a/src/modules/player/shaders/clarity_boost.vert b/src/modules/player/shaders/clarity_boost.vert new file mode 100644 index 0000000..c8d8b42 --- /dev/null +++ b/src/modules/player/shaders/clarity_boost.vert @@ -0,0 +1,5 @@ +attribute vec2 position; + +void main() { + gl_Position = vec4(position, 0, 1); +} diff --git a/src/modules/player/webgl2-player.ts b/src/modules/player/webgl2-player.ts new file mode 100644 index 0000000..df009a3 --- /dev/null +++ b/src/modules/player/webgl2-player.ts @@ -0,0 +1,250 @@ +import vertClarityBoost from "./shaders/clarity_boost.vert" with { type: "text" }; +import fsClarityBoost from "./shaders/clarity_boost.fs" with { type: "text" }; +import { BxLogger } from "@/utils/bx-logger"; + + +const LOG_TAG = 'WebGL2Player'; + +export class WebGL2Player { + #$video: HTMLVideoElement; + #$canvas: HTMLCanvasElement; + + #gl: WebGL2RenderingContext | null = null; + #resources: Array = []; + #program: WebGLProgram | null = null; + + #stopped: boolean = false; + + #options = { + filterId: 1, + sharpenFactor: 0, + brightness: 0.0, + contrast: 0.0, + saturation: 0.0, + }; + + #animFrameId: number | null = null; + + constructor($video: HTMLVideoElement) { + BxLogger.info(LOG_TAG, 'Initialize'); + this.#$video = $video; + + const $canvas = document.createElement('canvas'); + $canvas.width = $video.videoWidth; + $canvas.height = $video.videoHeight; + this.#$canvas = $canvas; + + this.#setupShaders(); + this.#setupRendering(); + + $video.insertAdjacentElement('afterend', $canvas); + } + + setFilter(filterId: number, update = true) { + this.#options.filterId = filterId; + update && this.updateCanvas(); + } + + setSharpness(sharpness: number, update = true) { + this.#options.sharpenFactor = sharpness; + update && this.updateCanvas(); + } + + setBrightness(brightness: number, update = true) { + this.#options.brightness = (brightness - 100) / 100; + update && this.updateCanvas(); + } + + setContrast(contrast: number, update = true) { + this.#options.contrast = (contrast - 100) / 100; + update && this.updateCanvas(); + } + + setSaturation(saturation: number, update = true) { + this.#options.saturation = (saturation - 100) / 100; + update && this.updateCanvas(); + } + + getCanvas() { + return this.#$canvas; + } + + updateCanvas() { + const gl = this.#gl!; + const program = this.#program!; + + gl.uniform2f(gl.getUniformLocation(program, 'iResolution'), this.#$canvas.width, this.#$canvas.height); + + gl.uniform1i(gl.getUniformLocation(program, 'filterId'), this.#options.filterId); + gl.uniform1f(gl.getUniformLocation(program, 'sharpenFactor'), this.#options.sharpenFactor); + gl.uniform1f(gl.getUniformLocation(program, 'brightness'), this.#options.brightness); + gl.uniform1f(gl.getUniformLocation(program, 'contrast'), this.#options.contrast); + gl.uniform1f(gl.getUniformLocation(program, 'saturation'), this.#options.saturation); + } + + drawFrame() { + const gl = this.#gl!; + const $video = this.#$video; + + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, $video); + gl.drawArrays(gl.TRIANGLES, 0, 6); + } + + #setupRendering() { + let animate: any; + + if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) { + const $video = this.#$video; + animate = () => { + if (this.#stopped) { + return; + } + + this.drawFrame(); + this.#animFrameId = $video.requestVideoFrameCallback(animate); + } + + this.#animFrameId = $video.requestVideoFrameCallback(animate); + } else { + animate = () => { + if (this.#stopped) { + return; + } + + this.drawFrame(); + this.#animFrameId = requestAnimationFrame(animate); + } + + this.#animFrameId = requestAnimationFrame(animate); + } + } + + #setupShaders() { + const gl = this.#$canvas.getContext('webgl2', { + isBx: true, + antialias: true, + alpha: false, + powerPreference: 'high-performance', + }) as WebGL2RenderingContext; + this.#gl = gl; + + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferWidth); + + // Vertex shader: Identity map + const vShader = gl.createShader(gl.VERTEX_SHADER)!; + gl.shaderSource(vShader, vertClarityBoost); + gl.compileShader(vShader); + + const fShader = gl.createShader(gl.FRAGMENT_SHADER)!; + gl.shaderSource(fShader, fsClarityBoost); + gl.compileShader(fShader); + + // Create and link program + const program = gl.createProgram()!; + this.#program = program; + + gl.attachShader(program, vShader); + gl.attachShader(program, fShader); + gl.linkProgram(program); + gl.useProgram(program); + + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + console.error(`Link failed: ${gl.getProgramInfoLog(program)}`); + console.error(`vs info-log: ${gl.getShaderInfoLog(vShader)}`); + console.error(`fs info-log: ${gl.getShaderInfoLog(fShader)}`); + } + + this.updateCanvas(); + + // Vertices: A screen-filling quad made from two triangles + const buffer = gl.createBuffer(); + this.#resources.push(buffer); + + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ + -1, -1, + 1, -1, + -1, 1, + -1, 1, + 1, -1, + 1, 1, + ]), gl.STATIC_DRAW); + + gl.enableVertexAttribArray(0); + gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0); + + // Texture to contain the video data + const texture = gl.createTexture(); + this.#resources.push(texture); + + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + + // Bind texture to the "data" argument to the fragment shader + gl.uniform1i(gl.getUniformLocation(program, 'data'), 0); + + gl.activeTexture(gl.TEXTURE0); + // gl.bindTexture(gl.TEXTURE_2D, texture); + } + + resume() { + this.stop(); + this.#stopped = false; + BxLogger.info(LOG_TAG, 'Resume'); + + this.#$canvas.classList.remove('bx-gone'); + this.#setupRendering(); + } + + stop() { + BxLogger.info(LOG_TAG, 'Stop'); + this.#$canvas.classList.add('bx-gone'); + + this.#stopped = true; + if (this.#animFrameId) { + if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) { + this.#$video.cancelVideoFrameCallback(this.#animFrameId); + } else { + cancelAnimationFrame(this.#animFrameId); + } + + this.#animFrameId = null; + } + } + + destroy() { + BxLogger.info(LOG_TAG, 'Destroy'); + this.stop(); + + const gl = this.#gl; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + + for (const resource of this.#resources) { + if (resource instanceof WebGLProgram) { + gl.useProgram(null); + gl.deleteProgram(resource); + } else if (resource instanceof WebGLShader) { + gl.deleteShader(resource); + } else if (resource instanceof WebGLTexture) { + gl.deleteTexture(resource); + } else if (resource instanceof WebGLBuffer) { + gl.deleteBuffer(resource); + } + } + + this.#gl = null; + } + + if (this.#$canvas.isConnected) { + this.#$canvas.parentElement?.removeChild(this.#$canvas); + } + + this.#$canvas.width = 1; + this.#$canvas.height = 1; + } +} diff --git a/src/modules/stream-player.ts b/src/modules/stream-player.ts new file mode 100644 index 0000000..3efd139 --- /dev/null +++ b/src/modules/stream-player.ts @@ -0,0 +1,268 @@ +import { CE } from "@/utils/html"; +import { WebGL2Player } from "./player/webgl2-player"; +import { getPref, PrefKey } from "@/utils/preferences"; +import { Screenshot } from "@/utils/screenshot"; +import { StreamPlayerType, StreamVideoProcessing } from "@enums/stream-player"; +import { STATES } from "@/utils/global"; + +export type StreamPlayerOptions = Partial<{ + processing: string, + sharpness: number, + saturation: number, + contrast: number, + brightness: number, +}>; + +export class StreamPlayer { + #$video: HTMLVideoElement; + #playerType: StreamPlayerType = StreamPlayerType.VIDEO; + + #options: StreamPlayerOptions = {}; + + #webGL2Player: WebGL2Player | null = null; + + #$videoCss: HTMLStyleElement | null = null; + #$usmMatrix: SVGFEConvolveMatrixElement | null = null; + + constructor($video: HTMLVideoElement, type: StreamPlayerType, options: StreamPlayerOptions) { + this.#setupVideoElements(); + + this.#$video = $video; + this.#options = options || {}; + this.setPlayerType(type); + } + + #setupVideoElements() { + this.#$videoCss = document.getElementById('bx-video-css') as HTMLStyleElement; + if (this.#$videoCss) { + this.#$usmMatrix = this.#$videoCss.querySelector('#bx-filter-usm-matrix') as any; + return; + } + + const $fragment = document.createDocumentFragment(); + + this.#$videoCss = CE('style', {id: 'bx-video-css'}); + $fragment.appendChild(this.#$videoCss); + + // Setup SVG filters + const $svg = CE('svg', { + id: 'bx-video-filters', + xmlns: 'http://www.w3.org/2000/svg', + class: 'bx-gone', + }, CE('defs', {xmlns: 'http://www.w3.org/2000/svg'}, + CE('filter', { + id: 'bx-filter-usm', + xmlns: 'http://www.w3.org/2000/svg', + }, this.#$usmMatrix = CE('feConvolveMatrix', { + id: 'bx-filter-usm-matrix', + order: '3', + xmlns: 'http://www.w3.org/2000/svg', + })), + ), + ); + $fragment.appendChild($svg); + document.documentElement.appendChild($fragment); + } + + #getVideoPlayerFilterStyle() { + const filters = []; + + const sharpness = this.#options.sharpness || 0; + if (sharpness != 0) { + const level = (7 - ((sharpness / 2) - 1) * 0.5).toFixed(1); // 5, 5.5, 6, 6.5, 7 + const matrix = `0 -1 0 -1 ${level} -1 0 -1 0`; + this.#$usmMatrix?.setAttributeNS(null, 'kernelMatrix', matrix); + + filters.push(`url(#bx-filter-usm)`); + } + + const saturation = this.#options.saturation || 100; + if (saturation != 100) { + filters.push(`saturate(${saturation}%)`); + } + + const contrast = this.#options.contrast || 100; + if (contrast != 100) { + filters.push(`contrast(${contrast}%)`); + } + + const brightness = this.#options.brightness || 100; + if (brightness != 100) { + filters.push(`brightness(${brightness}%)`); + } + + return filters.join(' '); + } + + #resizePlayer() { + const PREF_RATIO = getPref(PrefKey.VIDEO_RATIO); + const $video = this.#$video; + const isNativeTouchGame = STATES.currentStream.titleInfo?.details.hasNativeTouchSupport; + + let $webGL2Canvas; + if (this.#playerType == StreamPlayerType.WEBGL2) { + $webGL2Canvas = this.#webGL2Player?.getCanvas()!; + } + + let targetWidth; + let targetHeight; + let targetObjectFit; + + if (PREF_RATIO.includes(':')) { + const tmp = PREF_RATIO.split(':'); + + // Get preferred ratio + const videoRatio = parseFloat(tmp[0]) / parseFloat(tmp[1]); + + let width = 0; + let height = 0; + + // Get parent's ratio + const parentRect = $video.parentElement!.getBoundingClientRect(); + const parentRatio = parentRect.width / parentRect.height; + + // Get target width & height + if (parentRatio > videoRatio) { + height = parentRect.height; + width = height * videoRatio; + } else { + width = parentRect.width; + height = width / videoRatio; + } + + // Prevent floating points + width = Math.ceil(Math.min(parentRect.width, width)); + height = Math.ceil(Math.min(parentRect.height, height)); + + $video.dataset.width = width.toString(); + $video.dataset.height = height.toString(); + + // Update size + targetWidth = `${width}px`; + targetHeight = `${height}px`; + targetObjectFit = PREF_RATIO === '16:9' ? 'contain' : 'fill'; + } else { + targetWidth = '100%'; + targetHeight = '100%'; + targetObjectFit = PREF_RATIO; + + $video.dataset.width = window.innerWidth.toString(); + $video.dataset.height = window.innerHeight.toString(); + } + + $video.style.width = targetWidth; + $video.style.height = targetHeight; + $video.style.objectFit = targetObjectFit; + + // $video.style.padding = padding; + + if ($webGL2Canvas) { + $webGL2Canvas.style.width = targetWidth; + $webGL2Canvas.style.height = targetHeight; + $webGL2Canvas.style.objectFit = targetObjectFit; + } + + // Update video dimensions + if (isNativeTouchGame && this.#playerType == StreamPlayerType.WEBGL2) { + window.BX_EXPOSED.streamSession.updateDimensions(); + } + } + + setPlayerType(type: StreamPlayerType, refreshPlayer: boolean = false) { + if (this.#playerType !== type) { + // Switch from Video -> WebGL2 + if (type === StreamPlayerType.WEBGL2) { + // Initialize WebGL2 player + if (!this.#webGL2Player) { + this.#webGL2Player = new WebGL2Player(this.#$video); + } else { + this.#webGL2Player.resume(); + } + + this.#$videoCss!.textContent = ''; + + this.#$video.classList.add('bx-pixel'); + } else { + // Cleanup WebGL2 Player + this.#webGL2Player?.stop(); + + this.#$video.classList.remove('bx-pixel'); + } + } + + this.#playerType = type; + refreshPlayer && this.refreshPlayer(); + } + + setOptions(options: StreamPlayerOptions, refreshPlayer: boolean = false) { + this.#options = options; + refreshPlayer && this.refreshPlayer(); + } + + updateOptions(options: StreamPlayerOptions, refreshPlayer: boolean = false) { + this.#options = Object.assign(this.#options, options); + refreshPlayer && this.refreshPlayer(); + } + + getPlayerElement(playerType?: StreamPlayerType) { + if (typeof playerType === 'undefined') { + playerType = this.#playerType; + } + + if (playerType === StreamPlayerType.WEBGL2) { + return this.#webGL2Player?.getCanvas(); + } + + return this.#$video; + } + + getWebGL2Player() { + return this.#webGL2Player; + } + + refreshPlayer() { + if (this.#playerType === StreamPlayerType.WEBGL2) { + const options = this.#options; + const webGL2Player = this.#webGL2Player!; + + if (options.processing === StreamVideoProcessing.USM) { + webGL2Player.setFilter(1); + } else { + webGL2Player.setFilter(2); + } + + Screenshot.updateCanvasFilters('none'); + + webGL2Player.setSharpness(options.sharpness || 0); + webGL2Player.setSaturation(options.saturation || 100); + webGL2Player.setContrast(options.contrast || 100); + webGL2Player.setBrightness(options.brightness || 100); + } else { + let filters = this.#getVideoPlayerFilterStyle(); + let videoCss = ''; + if (filters) { + videoCss += `filter: ${filters} !important;`; + } + + // Apply video filters to screenshots + if (getPref(PrefKey.SCREENSHOT_APPLY_FILTERS)) { + Screenshot.updateCanvasFilters(filters); + } + + let css = ''; + if (videoCss) { + css = `#game-stream video { ${videoCss} }`; + } + + this.#$videoCss!.textContent = css; + } + + this.#resizePlayer(); + } + + destroy() { + // Cleanup WebGL2 Player + this.#webGL2Player?.destroy(); + this.#webGL2Player = null; + } +} diff --git a/src/modules/stream/stream-stats.ts b/src/modules/stream/stream-stats.ts index f368efe..a17c0d3 100644 --- a/src/modules/stream/stream-stats.ts +++ b/src/modules/stream/stream-stats.ts @@ -161,7 +161,12 @@ export class StreamStats { const totalDecodeTimeDiff = stat.totalDecodeTime - lastStat.totalDecodeTime; const framesDecodedDiff = stat.framesDecoded - lastStat.framesDecoded; const currentDecodeTime = totalDecodeTimeDiff / framesDecodedDiff * 1000; - this.#$dt!.textContent = `${currentDecodeTime.toFixed(2)}ms`; + + if (isNaN(currentDecodeTime)) { + this.#$dt!.textContent = '??ms'; + } else { + this.#$dt!.textContent = `${currentDecodeTime.toFixed(2)}ms`; + } if (PREF_STATS_CONDITIONAL_FORMATTING) { grade = (currentDecodeTime > 12) ? 'bad' : (currentDecodeTime > 9) ? 'ok' : (currentDecodeTime > 6) ? 'good' : ''; diff --git a/src/modules/ui/global-settings.ts b/src/modules/ui/global-settings.ts index 231ffce..8868aca 100644 --- a/src/modules/ui/global-settings.ts +++ b/src/modules/ui/global-settings.ts @@ -2,10 +2,11 @@ import { STATES, AppInterface, SCRIPT_VERSION } from "@utils/global"; import { CE, createButton, ButtonStyle } from "@utils/html"; import { BxIcon } from "@utils/bx-icon"; import { getPreferredServerRegion } from "@utils/region"; -import { UserAgent, UserAgentProfile } from "@utils/user-agent"; +import { UserAgent } from "@utils/user-agent"; import { getPref, Preferences, PrefKey, setPref, toPrefElement } from "@utils/preferences"; import { t, Translations } from "@utils/translation"; import { PatcherCache } from "../patcher"; +import { UserAgentProfile } from "@enums/user-agent"; const SETTINGS_UI = { 'Better xCloud': { diff --git a/src/modules/ui/ui.ts b/src/modules/ui/ui.ts index 43da773..eb66a1f 100644 --- a/src/modules/ui/ui.ts +++ b/src/modules/ui/ui.ts @@ -1,10 +1,9 @@ import { AppInterface, STATES } from "@utils/global"; import { CE, createButton, ButtonStyle, createSvgIcon } from "@utils/html"; import { BxIcon } from "@utils/bx-icon"; -import { UserAgent } from "@utils/user-agent"; import { BxEvent } from "@utils/bx-event"; import { MkbRemapper } from "@modules/mkb/mkb-remapper"; -import { getPref, Preferences, PrefKey, toPrefElement } from "@utils/preferences"; +import { getPref, Preferences, PrefKey, setPref, toPrefElement } from "@utils/preferences"; import { StreamStats } from "@modules/stream/stream-stats"; import { TouchController } from "@modules/touch-controller"; import { t } from "@utils/translation"; @@ -13,6 +12,9 @@ import { Screenshot } from "@/utils/screenshot"; import { ControllerShortcut } from "../controller-shortcut"; import { SoundShortcut } from "../shortcuts/shortcut-sound"; import { NativeMkbHandler } from "../mkb/native-mkb-handler"; +import { UserAgent } from "@/utils/user-agent"; +import type { StreamPlayerOptions } from "../stream-player"; +import { StreamPlayerType, StreamVideoProcessing } from "@enums/stream-player"; export function localRedirect(path: string) { @@ -38,40 +40,7 @@ export function localRedirect(path: string) { $anchor.click(); } - -function getVideoPlayerFilterStyle() { - const filters = []; - - const clarity = getPref(PrefKey.VIDEO_CLARITY); - if (clarity != 0) { - const level = (7 - (clarity - 1) * 0.5).toFixed(1); // 5, 5.5, 6, 6.5, 7 - const matrix = `0 -1 0 -1 ${level} -1 0 -1 0`; - document.getElementById('bx-filter-clarity-matrix')!.setAttributeNS(null, 'kernelMatrix', matrix); - - filters.push(`url(#bx-filter-clarity)`); - } - - const saturation = getPref(PrefKey.VIDEO_SATURATION); - if (saturation != 100) { - filters.push(`saturate(${saturation}%)`); - } - - const contrast = getPref(PrefKey.VIDEO_CONTRAST); - if (contrast != 100) { - filters.push(`contrast(${contrast}%)`); - } - - const brightness = getPref(PrefKey.VIDEO_BRIGHTNESS); - if (brightness != 100) { - filters.push(`brightness(${brightness}%)`); - } - - return filters.join(' '); -} - function setupStreamSettingsDialog() { - const isSafari = UserAgent.isSafari(); - const SETTINGS_UI = [ { icon: BxIcon.DISPLAY, @@ -109,29 +78,38 @@ function setupStreamSettingsDialog() { help_url: 'https://better-xcloud.github.io/ingame-features/#video', items: [ { - pref: PrefKey.VIDEO_RATIO, - onChange: updateVideoPlayerCss, + pref: PrefKey.VIDEO_PLAYER_TYPE, + onChange: onChangeVideoPlayerType, }, { - pref: PrefKey.VIDEO_CLARITY, - onChange: updateVideoPlayerCss, - unsupported: isSafari, + pref: PrefKey.VIDEO_RATIO, + onChange: updateVideoPlayer, + }, + + { + pref: PrefKey.VIDEO_PROCESSING, + onChange: updateVideoPlayer, + }, + + { + pref: PrefKey.VIDEO_SHARPNESS, + onChange: updateVideoPlayer, }, { pref: PrefKey.VIDEO_SATURATION, - onChange: updateVideoPlayerCss, + onChange: updateVideoPlayer, }, { pref: PrefKey.VIDEO_CONTRAST, - onChange: updateVideoPlayerCss, + onChange: updateVideoPlayer, }, { pref: PrefKey.VIDEO_BRIGHTNESS, - onChange: updateVideoPlayerCss, + onChange: updateVideoPlayer, }, ], }, @@ -436,94 +414,51 @@ function setupStreamSettingsDialog() { } -export function updateVideoPlayerCss() { - let $elm = document.getElementById('bx-video-css'); - if (!$elm) { - const $fragment = document.createDocumentFragment(); - $elm = CE('style', {id: 'bx-video-css'}); - $fragment.appendChild($elm); +function onChangeVideoPlayerType() { + const playerType = getPref(PrefKey.VIDEO_PLAYER_TYPE); + const $videoProcessing = document.getElementById('bx_setting_video_processing') as HTMLSelectElement; + const $videoSharpness = document.getElementById('bx_setting_video_sharpness') as HTMLElement; - // Setup SVG filters - const $svg = CE('svg', { - 'id': 'bx-video-filters', - 'xmlns': 'http://www.w3.org/2000/svg', - 'class': 'bx-gone', - }, CE('defs', {'xmlns': 'http://www.w3.org/2000/svg'}, - CE('filter', {'id': 'bx-filter-clarity', 'xmlns': 'http://www.w3.org/2000/svg'}, - CE('feConvolveMatrix', {'id': 'bx-filter-clarity-matrix', 'order': '3', 'xmlns': 'http://www.w3.org/2000/svg'})) - ) - ); - $fragment.appendChild($svg); - document.documentElement.appendChild($fragment); + let isDisabled = false; + + if (playerType === StreamPlayerType.WEBGL2) { + ($videoProcessing.querySelector(`option[value=${StreamVideoProcessing.CAS}]`) as HTMLOptionElement).disabled = false; + } else { + // Only allow USM when player type is Video + $videoProcessing.value = StreamVideoProcessing.USM; + setPref(PrefKey.VIDEO_PROCESSING, StreamVideoProcessing.USM); + + ($videoProcessing.querySelector(`option[value=${StreamVideoProcessing.CAS}]`) as HTMLOptionElement).disabled = true; + + if (UserAgent.isSafari()) { + isDisabled = true; + } } - let filters = getVideoPlayerFilterStyle(); - let videoCss = ''; - if (filters) { - videoCss += `filter: ${filters} !important;`; - } + $videoProcessing.disabled = isDisabled; + $videoSharpness.dataset.disabled = isDisabled.toString(); - // Apply video filters to screenshots - if (getPref(PrefKey.SCREENSHOT_APPLY_FILTERS)) { - Screenshot.updateCanvasFilters(filters); - } - - let css = ''; - if (videoCss) { - css = ` -#game-stream video { - ${videoCss} -} -`; - } - - $elm.textContent = css; - - resizeVideoPlayer(); + updateVideoPlayer(); } -function resizeVideoPlayer() { - const $video = STATES.currentStream.$video; - if (!$video || !$video.parentElement) { + +export function updateVideoPlayer() { + const streamPlayer = STATES.currentStream.streamPlayer; + if (!streamPlayer) { return; } - const PREF_RATIO = getPref(PrefKey.VIDEO_RATIO); - if (PREF_RATIO.includes(':')) { - const tmp = PREF_RATIO.split(':'); + const options = { + processing: getPref(PrefKey.VIDEO_PROCESSING), + sharpness: getPref(PrefKey.VIDEO_SHARPNESS), + saturation: getPref(PrefKey.VIDEO_SATURATION), + contrast: getPref(PrefKey.VIDEO_CONTRAST), + brightness: getPref(PrefKey.VIDEO_BRIGHTNESS), + } satisfies StreamPlayerOptions; - // Get preferred ratio - const videoRatio = parseFloat(tmp[0]) / parseFloat(tmp[1]); - - let width = 0; - let height = 0; - - // Get parent's ratio - const parentRect = $video.parentElement.getBoundingClientRect(); - const parentRatio = parentRect.width / parentRect.height; - - // Get target width & height - if (parentRatio > videoRatio) { - height = parentRect.height; - width = height * videoRatio; - } else { - width = parentRect.width; - height = width / videoRatio; - } - - // Prevent floating points - width = Math.min(parentRect.width, Math.ceil(width)); - height = Math.min(parentRect.height, Math.ceil(height)); - - // Update size - $video.style.width = `${width}px`; - $video.style.height = `${height}px`; - $video.style.objectFit = PREF_RATIO === '16:9' ? 'contain' : 'fill'; - } else { - $video.style.width = '100%'; - $video.style.height = '100%'; - $video.style.objectFit = PREF_RATIO; - } + streamPlayer.setPlayerType(getPref(PrefKey.VIDEO_PLAYER_TYPE)); + streamPlayer.updateOptions(options); + streamPlayer.refreshPlayer(); } @@ -545,11 +480,11 @@ export function setupStreamUi() { if (!document.querySelector('.bx-stream-settings-dialog')) { preloadFonts(); - window.addEventListener('resize', updateVideoPlayerCss); + window.addEventListener('resize', updateVideoPlayer); setupStreamSettingsDialog(); Screenshot.setup(); } - updateVideoPlayerCss(); + onChangeVideoPlayerType(); } diff --git a/src/types/index.d.ts b/src/types/index.d.ts index 7be3cfa..db74503 100644 --- a/src/types/index.d.ts +++ b/src/types/index.d.ts @@ -37,7 +37,7 @@ type BxStates = { productId: string; titleInfo: XcloudTitleInfo; - $video: HTMLVideoElement | null; + streamPlayer: StreamPlayer | null; peerConnection: RTCPeerConnection; audioContext: AudioContext | null; @@ -62,6 +62,7 @@ type XcloudTitleInfo = { productId: string; supportedInputTypes: InputType[]; supportedTabs: any[]; + hasNativeTouchSupport: boolean; hasTouchSupport: boolean; hasFakeTouchSupport: boolean; hasMkbSupport: boolean; @@ -78,6 +79,9 @@ declare module '*.js'; declare module '*.svg'; declare module '*.styl'; +declare module '*.fs'; +declare module '*.vert'; + type MkbMouseMove = { movementX: number; movementY: number; diff --git a/src/types/mkb.d.ts b/src/types/mkb.d.ts index 228e969..14008af 100644 --- a/src/types/mkb.d.ts +++ b/src/types/mkb.d.ts @@ -1,4 +1,4 @@ -import { MkbPresetKey } from "@modules/mkb/definitions"; +import { MkbPresetKey } from "@enums/mkb"; type GamepadKeyNameType = {[index: string | number]: string[]}; diff --git a/src/utils/bx-exposed.ts b/src/utils/bx-exposed.ts index 5763d75..e20ad28 100644 --- a/src/utils/bx-exposed.ts +++ b/src/utils/bx-exposed.ts @@ -60,7 +60,8 @@ export const BxExposed = { } // Pre-check supported input types - titleInfo.details.hasTouchSupport = supportedInputTypes.includes(InputType.NATIVE_TOUCH) || + titleInfo.details.hasNativeTouchSupport = supportedInputTypes.includes(InputType.NATIVE_TOUCH); + titleInfo.details.hasTouchSupport = titleInfo.details.hasNativeTouchSupport || supportedInputTypes.includes(InputType.CUSTOM_TOUCH_OVERLAY) || supportedInputTypes.includes(InputType.GENERIC_TOUCH); diff --git a/src/utils/monkey-patches.ts b/src/utils/monkey-patches.ts index 4987341..2cba1d0 100644 --- a/src/utils/monkey-patches.ts +++ b/src/utils/monkey-patches.ts @@ -3,6 +3,7 @@ import { getPref, PrefKey } from "@utils/preferences"; import { STATES } from "@utils/global"; import { BxLogger } from "@utils/bx-logger"; import { patchSdpBitrate } from "./sdp"; +import { StreamPlayer, type StreamPlayerOptions } from "@/modules/stream-player"; export function patchVideoApi() { const PREF_SKIP_SPLASH_VIDEO = getPref(PrefKey.SKIP_SPLASH_VIDEO); @@ -16,12 +17,22 @@ export function patchVideoApi() { return; } + const playerOptions = { + processing: getPref(PrefKey.VIDEO_PROCESSING), + sharpness: getPref(PrefKey.VIDEO_SHARPNESS), + saturation: getPref(PrefKey.VIDEO_SATURATION), + contrast: getPref(PrefKey.VIDEO_CONTRAST), + brightness: getPref(PrefKey.VIDEO_BRIGHTNESS), + } satisfies StreamPlayerOptions; + STATES.currentStream.streamPlayer = new StreamPlayer(this, getPref(PrefKey.VIDEO_PLAYER_TYPE), playerOptions); + BxEvent.dispatch(window, BxEvent.STREAM_PLAYING, { $video: this, }); } const nativePlay = HTMLMediaElement.prototype.play; + (HTMLMediaElement.prototype as any).nativePlay = nativePlay; HTMLMediaElement.prototype.play = function() { if (this.className && this.className.startsWith('XboxSplashVideo')) { if (PREF_SKIP_SPLASH_VIDEO) { @@ -97,21 +108,23 @@ export function patchRtcPeerConnection() { return dataChannel; } - const nativeSetLocalDescription = RTCPeerConnection.prototype.setLocalDescription; - RTCPeerConnection.prototype.setLocalDescription = function(description?: RTCLocalSessionDescriptionInit): Promise { - // set maximum bitrate - try { - const maxVideoBitrate = getPref(PrefKey.BITRATE_VIDEO_MAX); - if (maxVideoBitrate > 0) { - arguments[0].sdp = patchSdpBitrate(arguments[0].sdp, Math.round(maxVideoBitrate / 1000)); + const maxVideoBitrate = getPref(PrefKey.BITRATE_VIDEO_MAX); + if (maxVideoBitrate > 0) { + const nativeSetLocalDescription = RTCPeerConnection.prototype.setLocalDescription; + RTCPeerConnection.prototype.setLocalDescription = function(description?: RTCLocalSessionDescriptionInit): Promise { + // set maximum bitrate + try { + if (description) { + arguments[0].sdp = patchSdpBitrate(arguments[0].sdp, Math.round(maxVideoBitrate / 1000)); + } + } catch (e) { + BxLogger.error('setLocalDescription', e); } - } catch (e) { - BxLogger.error('setLocalDescription', e); - } - // @ts-ignore - return nativeSetLocalDescription.apply(this, arguments); - }; + // @ts-ignore + return nativeSetLocalDescription.apply(this, arguments); + }; + } const OrgRTCPeerConnection = window.RTCPeerConnection; // @ts-ignore @@ -132,6 +145,10 @@ export function patchAudioContext() { // @ts-ignore window.AudioContext = function(options?: AudioContextOptions | undefined): AudioContext { + if (options && options.latencyHint) { + options.latencyHint = 0; + } + const ctx = new OrgAudioContext(options); BxLogger.info('patchAudioContext', ctx, options); @@ -160,7 +177,12 @@ export function patchMeControl() { }; const MSA = { - MeControl: {}, + MeControl: { + API: { + setDisplayMode: () => {}, + setMobileState: () => {}, + }, + }, }; const MeControl = {}; @@ -207,12 +229,13 @@ export function patchCanvasContext() { HTMLCanvasElement.prototype.getContext = function(contextType: string, contextAttributes?: any) { if (contextType.includes('webgl')) { contextAttributes = contextAttributes || {}; + if (!contextAttributes.isBx) { + contextAttributes.antialias = false; - contextAttributes.antialias = false; - - // Use low-power profile for touch controller - if (contextAttributes.powerPreference === 'high-performance') { - contextAttributes.powerPreference = 'low-power'; + // Use low-power profile for touch controller + if (contextAttributes.powerPreference === 'high-performance') { + contextAttributes.powerPreference = 'low-power'; + } } } diff --git a/src/utils/network.ts b/src/utils/network.ts index 05c61f7..15c14a1 100644 --- a/src/utils/network.ts +++ b/src/utils/network.ts @@ -7,7 +7,7 @@ import { StreamBadges } from "@modules/stream/stream-badges"; import { TouchController } from "@modules/touch-controller"; import { STATES } from "@utils/global"; import { getPreferredServerRegion } from "@utils/region"; -import { GamePassCloudGallery } from "./gamepass-gallery"; +import { GamePassCloudGallery } from "../enums/game-pass-gallery"; import { InputType } from "./bx-exposed"; import { FeatureGates } from "./feature-gates"; diff --git a/src/utils/preferences.ts b/src/utils/preferences.ts index 259fcde..47f5c39 100644 --- a/src/utils/preferences.ts +++ b/src/utils/preferences.ts @@ -1,10 +1,12 @@ import { CE } from "@utils/html"; import { SUPPORTED_LANGUAGES, t } from "@utils/translation"; import { SettingElement, SettingElementType } from "@utils/settings"; -import { UserAgent, UserAgentProfile } from "@utils/user-agent"; +import { UserAgent } from "@utils/user-agent"; import { StreamStat } from "@modules/stream/stream-stats"; import type { PreferenceSetting, PreferenceSettings } from "@/types/preferences"; import { AppInterface, STATES } from "@utils/global"; +import { StreamPlayerType, StreamVideoProcessing } from "@enums/stream-player"; +import { UserAgentProfile } from "@/enums/user-agent"; export enum PrefKey { LAST_UPDATE_CHECK = 'version_last_check', @@ -70,7 +72,9 @@ export enum PrefKey { UI_HOME_CONTEXT_MENU_DISABLED = 'ui_home_context_menu_disabled', - VIDEO_CLARITY = 'video_clarity', + VIDEO_PLAYER_TYPE = 'video_player_type', + VIDEO_PROCESSING = 'video_processing', + VIDEO_SHARPNESS = 'video_sharpness', VIDEO_RATIO = 'video_ratio', VIDEO_BRIGHTNESS = 'video_brightness', VIDEO_CONTRAST = 'video_contrast', @@ -570,19 +574,35 @@ export class Preferences { [UserAgentProfile.CUSTOM]: t('custom'), }, }, - [PrefKey.VIDEO_CLARITY]: { - label: t('clarity'), + [PrefKey.VIDEO_PLAYER_TYPE]: { + label: t('renderer'), + default: 'default', + options: { + [StreamPlayerType.VIDEO]: t('default'), + [StreamPlayerType.WEBGL2]: t('webgl2'), + }, + }, + [PrefKey.VIDEO_PROCESSING]: { + label: t('clarity-boost'), + default: StreamVideoProcessing.USM, + options: { + [StreamVideoProcessing.USM]: t('unsharp-masking'), + [StreamVideoProcessing.CAS]: t('amd-fidelity-cas'), + }, + }, + [PrefKey.VIDEO_SHARPNESS]: { + label: t('sharpness'), type: SettingElementType.NUMBER_STEPPER, default: 0, min: 0, - max: 5, + max: 10, params: { hideSlider: true, }, }, [PrefKey.VIDEO_RATIO]: { - label: t('ratio'), - note: t('stretch-note'), + label: t('aspect-ratio'), + note: t('aspect-ratio-note'), default: '16:9', options: { '16:9': '16:9', diff --git a/src/utils/preload-state.ts b/src/utils/preload-state.ts index 973d147..abfac50 100644 --- a/src/utils/preload-state.ts +++ b/src/utils/preload-state.ts @@ -1,7 +1,7 @@ import { STATES } from "@utils/global"; import { BxLogger } from "./bx-logger"; import { TouchController } from "@modules/touch-controller"; -import { GamePassCloudGallery } from "./gamepass-gallery"; +import { GamePassCloudGallery } from "../enums/game-pass-gallery"; import { getPref, PrefKey } from "./preferences"; import { BX_FLAGS } from "./bx-flags"; diff --git a/src/utils/screenshot.ts b/src/utils/screenshot.ts index 88a77ee..215d954 100644 --- a/src/utils/screenshot.ts +++ b/src/utils/screenshot.ts @@ -1,5 +1,7 @@ +import { StreamPlayerType } from "@enums/stream-player"; import { AppInterface, STATES } from "./global"; import { CE } from "./html"; +import { getPref, PrefKey } from "./preferences"; export class Screenshot { @@ -31,23 +33,39 @@ export class Screenshot { Screenshot.#canvasContext.filter = filters; } - private static onAnimationEnd(e: Event) { - (e.target as any).classList.remove('bx-taking-screenshot'); + static #onAnimationEnd(e: Event) { + const $target = e.target as HTMLElement; + $target.classList.remove('bx-taking-screenshot'); } static takeScreenshot(callback?: any) { const currentStream = STATES.currentStream; - const $video = currentStream.$video; + const streamPlayer = currentStream.streamPlayer; const $canvas = Screenshot.#$canvas; - if (!$video || !$canvas) { + if (!streamPlayer || !$canvas) { return; } - $video.parentElement?.addEventListener('animationend', this.onAnimationEnd); - $video.parentElement?.classList.add('bx-taking-screenshot'); + let $player; + if (getPref(PrefKey.SCREENSHOT_APPLY_FILTERS)) { + $player = streamPlayer.getPlayerElement(); + } else { + $player = streamPlayer.getPlayerElement(StreamPlayerType.VIDEO); + } + + if (!$player || !$player.isConnected) { + return; + } + + $player.parentElement!.addEventListener('animationend', this.#onAnimationEnd); + $player.parentElement!.classList.add('bx-taking-screenshot'); const canvasContext = Screenshot.#canvasContext; - canvasContext.drawImage($video, 0, 0, $canvas.width, $canvas.height); + + if ($player instanceof HTMLCanvasElement) { + streamPlayer.getWebGL2Player().drawFrame(); + } + canvasContext.drawImage($player, 0, 0, $canvas.width, $canvas.height); // Get data URL and pass to parent app if (AppInterface) { diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 8f61ced..50e6f88 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -155,7 +155,7 @@ export class SettingElement { return textContent; }; - const $wrapper = CE('div', {'class': 'bx-number-stepper'}, + const $wrapper = CE('div', {'class': 'bx-number-stepper', id: `bx_setting_${key}`}, $decBtn = CE('button', { 'data-type': 'dec', type: 'button', diff --git a/src/utils/translation.ts b/src/utils/translation.ts index 335829e..124e6f2 100644 --- a/src/utils/translation.ts +++ b/src/utils/translation.ts @@ -1,4 +1,5 @@ import { NATIVE_FETCH } from "./bx-flags"; +import { BxLogger } from "./bx-logger"; export const SUPPORTED_LANGUAGES = { 'en-US': 'English (United States)', @@ -26,8 +27,11 @@ const Texts = { "activated": "Activated", "active": "Active", "advanced": "Advanced", + "amd-fidelity-cas": "AMD FidelityFX CAS", "android-app-settings": "Android app settings", "apply": "Apply", + "aspect-ratio": "Aspect ratio", + "aspect-ratio-note": "Don't use with native touch games", "audio": "Audio", "auto": "Auto", "back-to-home": "Back to home", @@ -48,7 +52,7 @@ const Texts = { "can-stream-xbox-360-games": "Can stream Xbox 360 games", "cancel": "Cancel", "cant-stream-xbox-360-games": "Can't stream Xbox 360 games", - "clarity": "Clarity", + "clarity-boost": "Clarity boost", "clarity-boost-warning": "These settings don't work when the Clarity Boost mode is ON", "clear": "Clear", "close": "Close", @@ -152,14 +156,14 @@ const Texts = { , (e: any) => `${e.key}: Funktion an-/ausschalten`, , - , + (e: any) => `Pulsa ${e.key} para alternar esta función`, (e: any) => `Appuyez sur ${e.key} pour activer cette fonctionnalité`, (e: any) => `Premi ${e.key} per attivare questa funzionalità`, (e: any) => `${e.key} でこの機能を切替`, + (e: any) => `${e.key} 키를 눌러 이 기능을 켜고 끄세요`, + (e: any) => `Naciśnij ${e.key} aby przełączyć tę funkcję`, , - , - , - , + (e: any) => `Нажмите ${e.key} для переключения этой функции`, , (e: any) => `Etkinleştirmek için ${e.key} tuşuna basın`, (e: any) => `Натисніть ${e.key} щоб перемкнути цю функцію`, @@ -168,12 +172,12 @@ const Texts = { ], "press-to-bind": "Press a key or do a mouse click to bind...", "prompt-preset-name": "Preset's name:", - "ratio": "Ratio", "reduce-animations": "Reduce UI animations", "region": "Region", "reload-stream": "Reload stream", "remote-play": "Remote Play", "rename": "Rename", + "renderer": "Renderer", "right-click-to-unbind": "Right-click on a key to unbind it", "right-stick": "Right stick", "rocket-always-hide": "Always hide", @@ -191,6 +195,7 @@ const Texts = { "settings": "Settings", "settings-reload": "Reload page to reflect changes", "settings-reloading": "Reloading...", + "sharpness": "Sharpness", "shortcut-keys": "Shortcut keys", "show": "Show", "show-game-art": "Show game art", @@ -218,7 +223,6 @@ const Texts = { "stream-settings": "Stream settings", "stream-stats": "Stream stats", "stretch": "Stretch", - "stretch-note": "Don't use with native touch games", "support-better-xcloud": "Support Better xCloud", "swap-buttons": "Swap buttons", "take-screenshot": "Take screenshot", @@ -263,6 +267,7 @@ const Texts = { "unknown": "Unknown", "unlimited": "Unlimited", "unmuted": "Unmuted", + "unsharp-masking": "Unsharp masking", "use-mouse-absolute-position": "Use mouse's absolute position", "user-agent-profile": "User-Agent profile", "vertical-scroll-sensitivity": "Vertical scroll sensitivity", @@ -278,6 +283,7 @@ const Texts = { "volume": "Volume", "wait-time-countdown": "Countdown", "wait-time-estimated": "Estimated finish time", + "webgl2": "WebGL2", }; export class Translations { @@ -390,5 +396,9 @@ export class Translations { } export const t = Translations.get; +export const ut = (text: string): string => { + BxLogger.warning('Untranslated text', text); + return text; +} Translations.init(); diff --git a/src/utils/user-agent.ts b/src/utils/user-agent.ts index d016931..7aa4f55 100644 --- a/src/utils/user-agent.ts +++ b/src/utils/user-agent.ts @@ -1,19 +1,10 @@ +import { UserAgentProfile } from "@enums/user-agent"; + type UserAgentConfig = { profile: UserAgentProfile, custom?: string, }; -export enum UserAgentProfile { - WINDOWS_EDGE = 'windows-edge', - MACOS_SAFARI = 'macos-safari', - SMARTTV_GENERIC = 'smarttv-generic', - SMARTTV_TIZEN = 'smarttv-tizen', - VR_OCULUS = 'vr-oculus', - ANDROID_KIWI_V123 = 'android-kiwi-v123', - DEFAULT = 'default', - CUSTOM = 'custom', -} - let CHROMIUM_VERSION = '123.0.0.0'; if (!!(window as any).chrome || window.navigator.userAgent.includes('Chrome')) { // Get Chromium version in the original User-Agent value @@ -27,6 +18,10 @@ export class UserAgent { static readonly STORAGE_KEY = 'better_xcloud_user_agent'; static #config: UserAgentConfig; + static #isMobile: boolean | null = null; + static #isSafari: boolean | null = null; + static #isSafariMobile: boolean | null = null; + static #USER_AGENTS: PartialRecord = { [UserAgentProfile.WINDOWS_EDGE]: `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${CHROMIUM_VERSION} Safari/537.36 Edg/${CHROMIUM_VERSION}`, [UserAgentProfile.MACOS_SAFARI]: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5.2 Safari/605.1.1', @@ -79,20 +74,40 @@ export class UserAgent { } } - static isSafari(mobile=false): boolean { + static isSafari(): boolean { + if (this.#isSafari !== null) { + return this.#isSafari; + } + const userAgent = UserAgent.getDefault().toLowerCase(); let result = userAgent.includes('safari') && !userAgent.includes('chrom'); - if (result && mobile) { - result = userAgent.includes('mobile'); + this.#isSafari = result; + return result; + } + + static isSafariMobile(): boolean { + if (this.#isSafariMobile !== null) { + return this.#isSafariMobile; } + const userAgent = UserAgent.getDefault().toLowerCase(); + const result = this.isSafari() && userAgent.includes('mobile'); + + this.#isSafariMobile = result; return result; } static isMobile(): boolean { + if (this.#isMobile !== null) { + return this.#isMobile; + } + const userAgent = UserAgent.getDefault().toLowerCase(); - return /iphone|ipad|android/.test(userAgent); + const result = /iphone|ipad|android/.test(userAgent); + + this.#isMobile = result; + return result; } static spoof() { diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 382e2fe..35b76f5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -47,7 +47,7 @@ export function disablePwa() { } // Check if it's Safari on mobile - if (!!AppInterface || UserAgent.isSafari(true)) { + if (!!AppInterface || UserAgent.isSafariMobile()) { // Disable the PWA prompt Object.defineProperty(window.navigator, 'standalone', { value: true, diff --git a/tsconfig.json b/tsconfig.json index 880ae99..f18ca26 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,6 +11,7 @@ "paths": { "@/*": ["./*"], "@assets/*": ["./assets/*"], + "@enums/*": ["./enums/*"], "@macros/*": ["./macros/*"], "@modules/*": ["./modules/*"], "@utils/*": ["./utils/*"],