Add WebGL2 renderer

src/modules/player/shaders/clarity_boost.fs

@@ -0,0 +1,121 @@
+const int FILTER_UNSHARP_MASKING = 1;
+const int FILTER_CAS = 2;
+
+precision highp float;
+uniform sampler2D data;
+uniform vec2 iResolution;
+
+uniform int filterId;
+uniform float sharpenFactor;
+uniform float brightness;
+uniform float contrast;
+uniform float saturation;
+
+vec3 textureAt(sampler2D tex, vec2 coord) {
+    return texture2D(tex, coord / iResolution.xy).rgb;
+}
+
+vec3 clarityBoost(sampler2D tex, vec2 coord)
+{
+    // Load a collection of samples in a 3x3 neighorhood, where e is the current pixel.
+    // a b c
+    // d e f
+    // g h i
+    vec3 a = textureAt(tex, coord + vec2(-1, 1));
+    vec3 b = textureAt(tex, coord + vec2(0, 1));
+    vec3 c = textureAt(tex, coord + vec2(1, 1));
+
+    vec3 d = textureAt(tex, coord + vec2(-1, 0));
+    vec3 e = textureAt(tex, coord);
+    vec3 f = textureAt(tex, coord + vec2(1, 0));
+
+    vec3 g = textureAt(tex, coord + vec2(-1, -1));
+    vec3 h = textureAt(tex, coord + vec2(0, -1));
+    vec3 i = textureAt(tex, coord + vec2(1, -1));
+
+    if (filterId == FILTER_CAS) {
+        // Soft min and max.
+        //  a b c             b
+        //  d e f * 0.5  +  d e f * 0.5
+        //  g h i             h
+        // These are 2.0x bigger (factored out the extra multiply).
+        vec3 minRgb = min(min(min(d, e), min(f, b)), h);
+        vec3 minRgb2 = min(min(a, c), min(g, i));
+        minRgb += min(minRgb, minRgb2);
+
+        vec3 maxRgb = max(max(max(d, e), max(f, b)), h);
+        vec3 maxRgb2 = max(max(a, c), max(g, i));
+        maxRgb += max(maxRgb, maxRgb2);
+
+        // Smooth minimum distance to signal limit divided by smooth max.
+        vec3 reciprocalMaxRgb = 1.0 / maxRgb;
+        vec3 amplifyRgb = clamp(min(minRgb, 2.0 - maxRgb) * reciprocalMaxRgb, 0.0, 1.0);
+
+        // Shaping amount of sharpening.
+        amplifyRgb = inversesqrt(amplifyRgb);
+
+        float contrast = 0.8;
+        float peak = -3.0 * contrast + 8.0;
+        vec3 weightRgb = -(1.0 / (amplifyRgb * peak));
+
+        vec3 reciprocalWeightRgb = 1.0 / (4.0 * weightRgb + 1.0);
+
+        //                0 w 0
+        // Filter shape:  w 1 w
+        //                0 w 0
+        vec3 window = (b + d) + (f + h);
+        vec3 outColor = clamp((window * weightRgb + e) * reciprocalWeightRgb, 0.0, 1.0);
+
+        outColor = mix(e, outColor, sharpenFactor / 2.0);
+
+        return outColor;
+    } else if (filterId == FILTER_UNSHARP_MASKING) {
+        vec3 gaussianBlur = (a * 1.0 + b * 2.0 + c * 1.0 +
+            d * 2.0 + e * 4.0 + f * 2.0 +
+            g * 1.0 + h * 2.0 + i * 1.0) / 16.0;
+
+        // Return edge detection
+        return e + (e - gaussianBlur) * sharpenFactor / 3.0;
+    }
+
+    return e;
+}
+
+vec3 adjustBrightness(vec3 color) {
+    return (1.0 + brightness) * color;
+}
+
+vec3 adjustContrast(vec3 color) {
+    return 0.5 + (1.0 + contrast) * (color - 0.5);
+}
+
+vec3 adjustSaturation(vec3 color) {
+    const vec3 luminosityFactor = vec3(0.2126, 0.7152, 0.0722);
+    vec3 grayscale = vec3(dot(color, luminosityFactor));
+
+    return mix(grayscale, color, 1.0 + saturation);
+}
+
+void main() {
+    vec3 color;
+
+    if (sharpenFactor > 0.0) {
+        color = clarityBoost(data, gl_FragCoord.xy);
+    } else {
+        color = textureAt(data, gl_FragCoord.xy);
+    }
+
+    if (saturation != 0.0) {
+        color = adjustSaturation(color);
+    }
+
+    if (contrast != 0.0) {
+        color = adjustContrast(color);
+    }
+
+    if (brightness != 0.0) {
+        color = adjustBrightness(color);
+    }
+
+    gl_FragColor = vec4(color, 1.0);
+}

src/modules/player/shaders/clarity_boost.vert

@@ -0,0 +1,5 @@
+attribute vec2 position;
+
+void main() {
+    gl_Position = vec4(position, 0, 1);
+}

src/modules/player/webgl2-player.ts

@@ -0,0 +1,250 @@
+import vertClarityBoost from "./shaders/clarity_boost.vert" with { type: "text" };
+import fsClarityBoost from "./shaders/clarity_boost.fs" with { type: "text" };
+import { BxLogger } from "@/utils/bx-logger";
+
+
+const LOG_TAG = 'WebGL2Player';
+
+export class WebGL2Player {
+    #$video: HTMLVideoElement;
+    #$canvas: HTMLCanvasElement;
+
+    #gl: WebGL2RenderingContext | null = null;
+    #resources: Array<any> = [];
+    #program: WebGLProgram | null = null;
+
+    #stopped: boolean = false;
+
+    #options = {
+        filterId: 1,
+        sharpenFactor: 0,
+        brightness: 0.0,
+        contrast: 0.0,
+        saturation: 0.0,
+    };
+
+    #animFrameId: number | null = null;
+
+    constructor($video: HTMLVideoElement) {
+        BxLogger.info(LOG_TAG, 'Initialize');
+        this.#$video = $video;
+
+        const $canvas = document.createElement('canvas');
+        $canvas.width = $video.videoWidth;
+        $canvas.height = $video.videoHeight;
+        this.#$canvas = $canvas;
+
+        this.#setupShaders();
+        this.#setupRendering();
+
+        $video.insertAdjacentElement('afterend', $canvas);
+    }
+
+    setFilter(filterId: number, update = true) {
+        this.#options.filterId = filterId;
+        update && this.updateCanvas();
+    }
+
+    setSharpness(sharpness: number, update = true) {
+        this.#options.sharpenFactor = sharpness;
+        update && this.updateCanvas();
+    }
+
+    setBrightness(brightness: number, update = true) {
+        this.#options.brightness = (brightness - 100) / 100;
+        update && this.updateCanvas();
+    }
+
+    setContrast(contrast: number, update = true) {
+        this.#options.contrast = (contrast - 100) / 100;
+        update && this.updateCanvas();
+    }
+
+    setSaturation(saturation: number, update = true) {
+        this.#options.saturation = (saturation - 100) / 100;
+        update && this.updateCanvas();
+    }
+
+    getCanvas() {
+        return this.#$canvas;
+    }
+
+    updateCanvas() {
+        const gl = this.#gl!;
+        const program = this.#program!;
+
+        gl.uniform2f(gl.getUniformLocation(program, 'iResolution'), this.#$canvas.width, this.#$canvas.height);
+
+        gl.uniform1i(gl.getUniformLocation(program, 'filterId'), this.#options.filterId);
+        gl.uniform1f(gl.getUniformLocation(program, 'sharpenFactor'), this.#options.sharpenFactor);
+        gl.uniform1f(gl.getUniformLocation(program, 'brightness'), this.#options.brightness);
+        gl.uniform1f(gl.getUniformLocation(program, 'contrast'), this.#options.contrast);
+        gl.uniform1f(gl.getUniformLocation(program, 'saturation'), this.#options.saturation);
+    }
+
+    drawFrame() {
+        const gl = this.#gl!;
+        const $video = this.#$video;
+
+        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, $video);
+        gl.drawArrays(gl.TRIANGLES, 0, 6);
+    }
+
+    #setupRendering() {
+        let animate: any;
+
+        if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) {
+            const $video = this.#$video;
+            animate = () => {
+                if (this.#stopped) {
+                    return;
+                }
+
+                this.drawFrame();
+                this.#animFrameId = $video.requestVideoFrameCallback(animate);
+            }
+
+            this.#animFrameId = $video.requestVideoFrameCallback(animate);
+        } else {
+            animate = () => {
+                if (this.#stopped) {
+                    return;
+                }
+
+                this.drawFrame();
+                this.#animFrameId = requestAnimationFrame(animate);
+            }
+
+            this.#animFrameId = requestAnimationFrame(animate);
+        }
+    }
+
+    #setupShaders() {
+        const gl = this.#$canvas.getContext('webgl2', {
+            isBx: true,
+            antialias: true,
+            alpha: false,
+            powerPreference: 'high-performance',
+        }) as WebGL2RenderingContext;
+        this.#gl = gl;
+
+        gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferWidth);
+
+        // Vertex shader: Identity map
+        const vShader = gl.createShader(gl.VERTEX_SHADER)!;
+        gl.shaderSource(vShader, vertClarityBoost);
+        gl.compileShader(vShader);
+
+        const fShader = gl.createShader(gl.FRAGMENT_SHADER)!;
+        gl.shaderSource(fShader, fsClarityBoost);
+        gl.compileShader(fShader);
+
+        // Create and link program
+        const program = gl.createProgram()!;
+        this.#program = program;
+
+        gl.attachShader(program, vShader);
+        gl.attachShader(program, fShader);
+        gl.linkProgram(program);
+        gl.useProgram(program);
+
+        if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+            console.error(`Link failed: ${gl.getProgramInfoLog(program)}`);
+            console.error(`vs info-log: ${gl.getShaderInfoLog(vShader)}`);
+            console.error(`fs info-log: ${gl.getShaderInfoLog(fShader)}`);
+        }
+
+        this.updateCanvas();
+
+        // Vertices: A screen-filling quad made from two triangles
+        const buffer = gl.createBuffer();
+        this.#resources.push(buffer);
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
+        gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
+            -1, -1,
+            1, -1,
+            -1,  1,
+            -1,  1,
+            1, -1,
+            1,  1,
+        ]), gl.STATIC_DRAW);
+
+        gl.enableVertexAttribArray(0);
+        gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
+
+        // Texture to contain the video data
+        const texture = gl.createTexture();
+        this.#resources.push(texture);
+
+        gl.bindTexture(gl.TEXTURE_2D, texture);
+        gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+
+        // Bind texture to the "data" argument to the fragment shader
+        gl.uniform1i(gl.getUniformLocation(program, 'data'), 0);
+
+        gl.activeTexture(gl.TEXTURE0);
+        // gl.bindTexture(gl.TEXTURE_2D, texture);
+    }
+
+    resume() {
+        this.stop();
+        this.#stopped = false;
+        BxLogger.info(LOG_TAG, 'Resume');
+
+        this.#$canvas.classList.remove('bx-gone');
+        this.#setupRendering();
+    }
+
+    stop() {
+        BxLogger.info(LOG_TAG, 'Stop');
+        this.#$canvas.classList.add('bx-gone');
+
+        this.#stopped = true;
+        if (this.#animFrameId) {
+            if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) {
+                this.#$video.cancelVideoFrameCallback(this.#animFrameId);
+            } else {
+                cancelAnimationFrame(this.#animFrameId);
+            }
+
+            this.#animFrameId = null;
+        }
+    }
+
+    destroy() {
+        BxLogger.info(LOG_TAG, 'Destroy');
+        this.stop();
+
+        const gl = this.#gl;
+        if (gl) {
+            gl.getExtension('WEBGL_lose_context')?.loseContext();
+
+            for (const resource of this.#resources) {
+                if (resource instanceof WebGLProgram) {
+                    gl.useProgram(null);
+                    gl.deleteProgram(resource);
+                } else if (resource instanceof WebGLShader) {
+                    gl.deleteShader(resource);
+                } else if (resource instanceof WebGLTexture) {
+                    gl.deleteTexture(resource);
+                } else if (resource instanceof WebGLBuffer) {
+                    gl.deleteBuffer(resource);
+                }
+            }
+
+            this.#gl = null;
+        }
+
+        if (this.#$canvas.isConnected) {
+            this.#$canvas.parentElement?.removeChild(this.#$canvas);
+        }
+
+        this.#$canvas.width = 1;
+        this.#$canvas.height = 1;
+    }
+}