0到1理解web音视频从采集到传输到播放系列之《Jessibuca系列篇音视频播放》(完结)

本课程主要从

• 音视频采集
• 音视频编码
• 音视频协议封装传输
• 音视频协议解封装
• 音视频解码
• 音视频播放

关于Jessibuca

• 官网地址：jessibuca.com
• Demo: Demo
• Doc：Doc
• Github地址：Github

关于JessibucaPro

• 地址：JessibucaPro
• Demo: Demo
• AI:AI
• 插件：插件

第六章：音视频播放(完结)

音视频的播放主要是播放pcm 和 yuv数据

在web端，也可以借助mediaSource或者webcodec 来实现web端播放 H264 和H254的视频。

音频播放

主要就是播放pcm数据。

主要是借助AudioContext对象的api 来播放pcm 数据。

const audioContext = new (window.AudioContext || window.webkitAudioContext)()

在web端，pcm音频播放有三种方式，可以借助audioWorlet 和 createScriptProcessor 和createBufferSource 这三个API 来播放pcm音频。

audioWorlet

对于AudioWorklet

主要是通过audioContext.audioWorklet.addModule() 方法加载 AudioWorkletProcessor 模块。

然后在回调里面通过 new AudioWorkletNode()来新建 worklet node , 通过postMessage进行通讯。

demo如下：

audioContext.audioWorklet.addModule('./workletProcessor.js').then(()=>{

const workletProcessorNode = new AudioWorkletNode(this.audioContext, "worklet-processor", {
                // 传递到processor 的参数
});

workletProcessorNode.port.postMessage({
 // 传递消息。
})

workletProcessorNode.port.onmessage = (e)=>{
	// 监听消息。
}

})

// workletProcessor.js
 class WorkletProcessor extends AudioWorkletProcessor {
                constructor() {
                    super();
                
                    this.port.onmessage = (e) => {
                      // 监听消息
                    }
                }

                process(inputs, outputs, parameters) {
                  // 播放声音。

                    return true;
                }
            }
registerProcessor('worklet-processor', WorkletProcessor);

createScriptProcessor

对于AudioContext.createScriptProcessor()

主要是通过监听onaudioprocess回调方法，来播放声音。

demo如下

const scriptNode = audioContext.createScriptProcessor(1024,0,1);
scriptNode.onaudioprocess = (audioProcessingEvent) => {
        const outputBuffer = audioProcessingEvent.outputBuffer;
        // 播放音频数据
}

createBufferSource

对于AudioContext.createBufferSource()

const intervalTime = 1000 * 1024 / audioContext.sampleRate;

const scriptNodeInterval = setInterval(()=>{
	const audioSource = audioContext.createBufferSource();
	const outputBuffer = audioContext.createBuffer(audioInfo.channels, 1024, audioContext.sampleRate);
	// 播放
},intervalTime)

yuv视频播放

主要就是渲染yuv数据。

webgl

主要是借助webgl来渲染yuv的数据

function createYuvRender (gl, openWebglAlignment) {
    var vertexShaderScript = [
        'attribute vec4 vertexPos;',
        'attribute vec4 texturePos;',
        'varying vec2 textureCoord;',

        'void main()',
        '{',
        'gl_Position = vertexPos;',
        'textureCoord = texturePos.xy;',
        '}'
    ].join('\n');

    var fragmentShaderScript = [
        'precision highp float;',
        'varying highp vec2 textureCoord;',
        'uniform sampler2D ySampler;',
        'uniform sampler2D uSampler;',
        'uniform sampler2D vSampler;',
        'const mat4 YUV2RGB = mat4',
        '(',
        '1.1643828125, 0, 1.59602734375, -.87078515625,',
        '1.1643828125, -.39176171875, -.81296875, .52959375,',
        '1.1643828125, 2.017234375, 0, -1.081390625,',
        '0, 0, 0, 1',
        ');',

        'void main(void) {',
        'highp float y = texture2D(ySampler,  textureCoord).r;',
        'highp float u = texture2D(uSampler,  textureCoord).r;',
        'highp float v = texture2D(vSampler,  textureCoord).r;',
        'gl_FragColor = vec4(y, u, v, 1) * YUV2RGB;',
        '}'
    ].join('\n');

    if (openWebglAlignment) {
        gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
    }
    var vertexShader = gl.createShader(gl.VERTEX_SHADER);
    gl.shaderSource(vertexShader, vertexShaderScript);
    gl.compileShader(vertexShader);
    if (!gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS)) {
        console.log('Vertex shader failed to compile: ' + gl.getShaderInfoLog(vertexShader));
        gl.deleteShader(vertexShader)
    }

    var fragmentShader = gl.createShader(gl.FRAGMENT_SHADER);
    gl.shaderSource(fragmentShader, fragmentShaderScript);
    gl.compileShader(fragmentShader);
    if (!gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS)) {
        console.log('Fragment shader failed to compile: ' + gl.getShaderInfoLog(fragmentShader));
        gl.deleteShader(fragmentShader)
    }

    var program = gl.createProgram();
    gl.attachShader(program, vertexShader);
    gl.attachShader(program, fragmentShader);
    gl.linkProgram(program);
    if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
        console.log('Program failed to compile: ' + gl.getProgramInfoLog(program));
    }

    gl.useProgram(program);

    // initBuffers
    var vertexPosBuffer = gl.createBuffer();
    gl.bindBuffer(gl.ARRAY_BUFFER, vertexPosBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([1, 1, -1, 1, 1, -1, -1, -1]), gl.STATIC_DRAW);

    var vertexPosRef = gl.getAttribLocation(program, 'vertexPos');
    gl.enableVertexAttribArray(vertexPosRef);
    gl.vertexAttribPointer(vertexPosRef, 2, gl.FLOAT, false, 0, 0);

    var texturePosBuffer = gl.createBuffer();
    gl.bindBuffer(gl.ARRAY_BUFFER, texturePosBuffer);
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([1, 0, 0, 0, 1, 1, 0, 1]), gl.STATIC_DRAW);

    var texturePosRef = gl.getAttribLocation(program, 'texturePos');
    gl.enableVertexAttribArray(texturePosRef);
    gl.vertexAttribPointer(texturePosRef, 2, gl.FLOAT, false, 0, 0);

    function _initTexture(name, index) {
        var textureRef = gl.createTexture();
        gl.bindTexture(gl.TEXTURE_2D, textureRef);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
        gl.bindTexture(gl.TEXTURE_2D, null);
        gl.uniform1i(gl.getUniformLocation(program, name), index);
        return textureRef;
    }

    var yTextureRef = _initTexture('ySampler', 0);
    var uTextureRef = _initTexture('uSampler', 1);
    var vTextureRef = _initTexture('vSampler', 2);

    return {
        render: function (w, h, y, u, v) {
            gl.viewport(0, 0, w, h);
            gl.activeTexture(gl.TEXTURE0);
            gl.bindTexture(gl.TEXTURE_2D, yTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w, h, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, y);
            gl.activeTexture(gl.TEXTURE1);
            gl.bindTexture(gl.TEXTURE_2D, uTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w / 2, h / 2, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, u);
            gl.activeTexture(gl.TEXTURE2);
            gl.bindTexture(gl.TEXTURE_2D, vTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w / 2, h / 2, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, v);
            gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
        },
        renderYUV: function (w, h, data) {
            let y = data.slice(0, w * h);
            let u = data.slice(w * h, w * h * 5 / 4);
            let v = data.slice(w * h * 5 / 4, w * h * 3 / 2);
            gl.viewport(0, 0, w, h);
            gl.activeTexture(gl.TEXTURE0);
            gl.bindTexture(gl.TEXTURE_2D, yTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w, h, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, y);
            gl.activeTexture(gl.TEXTURE1);
            gl.bindTexture(gl.TEXTURE_2D, uTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w / 2, h / 2, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, u);
            gl.activeTexture(gl.TEXTURE2);
            gl.bindTexture(gl.TEXTURE_2D, vTextureRef);
            gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, w / 2, h / 2, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, v);
            gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
        },
        destroy: function () {
            try {
                gl.deleteProgram(program);

                gl.deleteBuffer(vertexPosBuffer)
                gl.deleteBuffer(texturePosBuffer);

                gl.deleteTexture(yTextureRef);
                gl.deleteTexture(uTextureRef);
                gl.deleteTexture(vTextureRef);
            } catch (e) {
                // console.error(e);
            }

        }
    }
};

video 标签

可以借助最新的VideoFrame 方法，可以将yuv数据新建成VideoFrame对象。

const videoFrame = new VideoFrame(ImageBitmap, {
    format:'I420|I420A|I422|I444|NV12|RGBA|RGBX|BGRA|BGRX'
    codedWidth:'',
    codedHeight:'',
    timestamp:'' // An integer representing the timestamp of the frame in microseconds.
})

结合 MediaStreamTrackGenerator 方法

const trackGenerator = new MediaStreamTrackGenerator({kind: 'video'});
const mediaStream = new MediaStream([trackGenerator]);
$videoElement.srcObject = mediaStream;
const vwriter = trackGenerator.writable.getWriter();

然后把videoFrame 对象添加到vwriter 里面去

vwriter.write(videoFrame);

webgpu

class WebGPURenderer {
  #canvas = null;
  #ctx = null;

  // Promise for `#start()`, WebGPU setup is asynchronous.
  #started = null;

  // WebGPU state shared between setup and drawing.
  #format = null;
  #device = null;
  #pipeline = null;
  #sampler = null;

  // Generates two triangles covering the whole canvas.
  static vertexShaderSource = `
    struct VertexOutput {
      @builtin(position) Position: vec4<f32>,
      @location(0) uv: vec2<f32>,
    }

    @vertex
    fn vert_main(@builtin(vertex_index) VertexIndex: u32) -> VertexOutput {
      var pos = array<vec2<f32>, 6>(
        vec2<f32>( 1.0,  1.0),
        vec2<f32>( 1.0, -1.0),
        vec2<f32>(-1.0, -1.0),
        vec2<f32>( 1.0,  1.0),
        vec2<f32>(-1.0, -1.0),
        vec2<f32>(-1.0,  1.0)
      );

      var uv = array<vec2<f32>, 6>(
        vec2<f32>(1.0, 0.0),
        vec2<f32>(1.0, 1.0),
        vec2<f32>(0.0, 1.0),
        vec2<f32>(1.0, 0.0),
        vec2<f32>(0.0, 1.0),
        vec2<f32>(0.0, 0.0)
      );

      var output : VertexOutput;
      output.Position = vec4<f32>(pos[VertexIndex], 0.0, 1.0);
      output.uv = uv[VertexIndex];
      return output;
    }
  `;

  // Samples the external texture using generated UVs.
  static fragmentShaderSource = `
    @group(0) @binding(1) var mySampler: sampler;
    @group(0) @binding(2) var myTexture: texture_external;
    
    @fragment
    fn frag_main(@location(0) uv : vec2<f32>) -> @location(0) vec4<f32> {
      return textureSampleBaseClampToEdge(myTexture, mySampler, uv);
    }
  `;

  constructor(canvas) {
    this.#canvas = canvas;
    this.#started = this.#start();
  }

  async #start() {
    const adapter = await navigator.gpu.requestAdapter();
    this.#device = await adapter.requestDevice();
    this.#format = navigator.gpu.getPreferredCanvasFormat();

    this.#ctx = this.#canvas.getContext("webgpu");
    this.#ctx.configure({
      device: this.#device,
      format: this.#format,
      alphaMode: "opaque",
    });

    this.#pipeline = this.#device.createRenderPipeline({
      layout: "auto",
      vertex: {
        module: this.#device.createShaderModule({code: WebGPURenderer.vertexShaderSource}),
        entryPoint: "vert_main"
      },
      fragment: {
        module: this.#device.createShaderModule({code: WebGPURenderer.fragmentShaderSource}),
        entryPoint: "frag_main",
        targets: [{format: this.#format}]
      },
      primitive: {
        topology: "triangle-list"
      }
    });
  
    // Default sampler configuration is nearset + clamp.
    this.#sampler = this.#device.createSampler({});
  }

  async draw(frame) {
    // Don't try to draw any frames until the context is configured.
    await this.#started;

    this.#canvas.width = frame.displayWidth;
    this.#canvas.height = frame.displayHeight;

    const uniformBindGroup = this.#device.createBindGroup({
      layout: this.#pipeline.getBindGroupLayout(0),
      entries: [
        {binding: 1, resource: this.#sampler},
        {binding: 2, resource: this.#device.importExternalTexture({source: frame})}
      ],
    });

    const commandEncoder = this.#device.createCommandEncoder();
    const textureView = this.#ctx.getCurrentTexture().createView();
    const renderPassDescriptor = {
      colorAttachments: [
        {
          view: textureView,
          clearValue: [1.0, 0.0, 0.0, 1.0],
          loadOp: "clear",
          storeOp: "store",
        },
      ],
    };

    const passEncoder = commandEncoder.beginRenderPass(renderPassDescriptor);
    passEncoder.setPipeline(this.#pipeline);
    passEncoder.setBindGroup(0, uniformBindGroup);
    passEncoder.draw(6, 1, 0, 0);
    passEncoder.end();
    this.#device.queue.submit([commandEncoder.finish()]);

    frame.close();
  }
};

H264/H265 视频播放

对于 H264/H265视频播放，需要借助mediaSource或者webcodec或者wasm 来实现web端播放。

mediaSource

对于MediaSource

需要将 H264/H265 视频数据，封装成 fmp4 格式的数据，然后喂给video进行播放。

mediasource demo: decode-and-render-by-mediasource

webcodec

对于webcodec

可以通过 webcodec 提供的 videoDecode api 进行编码，然后可以通过 webgl 或者 video 标签 进行播放。

webcodec demo:decode-and-render-by-webcodec

wasm

可以借助ffmpeg 和 webassembly 等技术将 c/c++ 编译成web端能跑的 wasm 文件。

wasm demo ：decode-and-render-by-wasm

小结

课程主要从视频的采集到视频的编码，再到视频的封装，再到视频的传输，再到视频的解封装和解码，最后就是视频的播放。

通过本课程，可以学习到音视频的从采集到传输到播放的一整个过程。