微信XR-FRAME使用透明视频

最近接到公司开发小程序AR系统的需求，需求不复杂但是奈何自己学识浅薄，还是遇到不少困难。今天终于解决了，记录一下。

根据需求，我们准备在AR中呈现视频内容，但是想让内容更炫酷，就需要让视频背景是透明的。考虑到视频体积，mp4是比较好的格式，但是mp4视频是没有透明通道的，所以想让mp4视频变成透明的就需要对其处理一下。

两种方法：

1.视频制作时约定透明部分纯黑。在渲染的时候直接把纯黑部分设为透明。这种方法有个弊端，就是没办法显示半透明部分。

const xrFrameSystem = wx.getXrFrameSystem();

xrFrameSystem.registerEffect('removeBlack', scene => scene.createEffect({
  name: "removeBlack",
  images: [{
    key: 'u_videoMap',
    default: 'white',
    macro: 'WX_USE_VIDEOMAP'
  }],
  defaultRenderQueue: 2000,
  passes: [{
    "renderStates": {
      cullOn: false,
      blendOn: true,
      blendSrc: xrFrameSystem.EBlendFactor.SRC_ALPHA,
      blendDst: xrFrameSystem.EBlendFactor.ONE_MINUS_SRC_ALPHA,
      cullFace: xrFrameSystem.ECullMode.BACK,
    },
    lightMode: "ForwardBase",
    useMaterialRenderStates: true,
    shaders: [0, 1]
  }],
  shaders: [
    `#version 100
      uniform highp mat4 u_view;
      uniform highp mat4 u_viewInverse;
      uniform highp mat4 u_vp;
      uniform highp mat4 u_projection;
      uniform highp mat4 u_world;

      attribute vec3 a_position;
      attribute highp vec2 a_texCoord;

      varying highp vec2 v_UV;

      void main()
      {
        v_UV = a_texCoord;
        vec4 worldPosition = u_world * vec4(a_position, 1.0);
        gl_Position = u_projection * u_view * worldPosition;
    }`, 
    `#version 100

    precision mediump float;
    precision highp int;
    varying highp vec2 v_UV;

    #ifdef WX_USE_VIDEOMAP
      uniform sampler2D u_videoMap;
    #endif

    void main()
    {
      vec4 baseColor = texture2D(u_videoMap, v_UV);

      float rgbSum = baseColor.r + baseColor.g + baseColor.b;
      // 设定阈值避免异常情况
      if (rgbSum < 0.1) { 
        gl_FragData[0] = vec4(1.0, 1.0, 1.0, 0.0);
      } else {
        gl_FragData[0] = vec4(baseColor.rgb, 1.0);
      }
    }
    `],
}));

2.制作视频时生成左右两个部分，左边是视频原内容。右边是黑白图像。使用时根据右边内容为左边提供透明度信息。

const xrFrameSystem = wx.getXrFrameSystem();

xrFrameSystem.registerEffect('videoEffetc', scene => scene.createEffect({
  name: "videoEffetc",
  images: [{
    key: 'u_videoMap',
    default: 'white',
    macro: 'WX_USE_VIDEOMAP'
  }],
  defaultRenderQueue: 2000,
  passes: [{
    "renderStates": {
      cullOn: false,
      blendOn: true,
      blendSrc: xrFrameSystem.EBlendFactor.SRC_ALPHA,
      blendDst: xrFrameSystem.EBlendFactor.ONE_MINUS_SRC_ALPHA,
      cullFace: xrFrameSystem.ECullMode.BACK,
    },
    lightMode: "ForwardBase",
    useMaterialRenderStates: true,
    shaders: [0, 1]
  }],
  shaders: [
    `#version 100
      uniform highp mat4 u_view;
      uniform highp mat4 u_viewInverse;
      uniform highp mat4 u_vp;
      uniform highp mat4 u_projection;
      uniform highp mat4 u_world;

      attribute vec3 a_position;
      attribute highp vec2 a_texCoord;

      varying highp vec2 v_UV;

      void main()
      {
        v_UV = a_texCoord;
        vec4 worldPosition = u_world * vec4(a_position, 1.0);
        gl_Position = u_projection * u_view * worldPosition;
    }`,
    `#version 100

    precision mediump float;
    precision highp int;
    varying highp vec2 v_UV;

    #ifdef WX_USE_VIDEOMAP
      uniform sampler2D u_videoMap;
    #endif
    float calculateGray(vec4 color) {
      return 0.299 * color.r + 0.587 * color.g + 0.114 * color.b;
    }
    void main()
    {
      // 调整纹理坐标，使视频的左半边映射到整个平面
      vec2 adjustedUV = vec2(v_UV.x * 0.5, v_UV.y);

      if (adjustedUV.x <= 1.0) {
        // 从调整后的纹理坐标中采样颜色
        vec4 baseColor = texture2D(u_videoMap, adjustedUV);

        // 获取右边对应位置的颜色
        vec4 rightColor = texture2D(u_videoMap, vec2(adjustedUV.x + 0.5, adjustedUV.y));

        // 计算灰度值
        float grayValue = calculateGray(rightColor);

        // 输出颜色，透明度为灰度值
        gl_FragData[0] = vec4(baseColor.rgb, grayValue);
      } else {
        // 丢弃右边一半的片段
        discard;
      }
    }
    `
  ],
}));