OpenGL实时音视频画面如何渲染RTP信息

背景

我们在一些测试时延的场景需要知道某一帧画面对应的RTP的时间戳，比如云应用的触控指令到画面变动被打包成RTP包的整一段时延，我们称为云测时延。知道这个信息有利于我们进行时延的分段分析

关键逻辑

底层WebRTC抛出VideoFrame信息的时候携带RTP的信息，比如RTP的时间戳（同一帧画面分包的场景，不同的RTP的时间戳也是相同的），交付到应用层的时候将时间戳通过OpenGL ES渲染出来

核心代码

渲染的逻辑：

public class VideoFrameDrawer {
  public static final String TAG = "VideoFrameDrawer";

  // 创建一个文字的纹理类
  private TextOverlayRenderer textOverlayRenderer;

  private volatile String overlayText = null;

  public void setOverlayText(String text) {
    this.overlayText = text;
  }

  public void clearOverlayText() {
    this.overlayText = null;
  }
  

public void drawFrame(VideoFrame frame, RendererCommon.GlDrawer drawer,
    @Nullable Matrix additionalRenderMatrix, int viewportX, int viewportY, int viewportWidth,
    int viewportHeight) {
    ...... // 此处省略了很多渲染的逻辑
    drawer.drawYuv(yuvUploader.getYuvTextures(),
        RendererCommon.convertMatrixFromAndroidGraphicsMatrix(renderMatrix), renderWidth,
        renderHeight, viewportX, viewportY, viewportWidth, viewportHeight);
  }


// 这里是更新的逻辑，在更新YUV的时候，同步将这一帧RTP的时间戳也更新出来
  if(textOverlayRenderer == null) {
    textOverlayRenderer = new TextOverlayRenderer();
  }

  if (overlayText != null) {
    textOverlayRenderer.updateText(overlayText);
    textOverlayRenderer.draw();
  }

文字纹理代码：

package org.webrtc;

import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.opengl.GLES20;
import android.opengl.GLUtils;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;

public class TextOverlayRenderer {
    private int textTextureId = -1;
    private int programId;
    private int positionHandle;
    private int texCoordHandle;
    private int textureHandle;

    // 简单的全屏顶点着色器
    private static final String VERTEX_SHADER =
            "attribute vec4 aPosition;\n" +
                    "attribute vec2 aTexCoord;\n" +
                    "varying vec2 vTexCoord;\n" +
                    "void main() {\n" +
                    "  gl_Position = aPosition;\n" +
                    "  vTexCoord = aTexCoord;\n" +
                    "}";

    // 采样2D纹理的片段着色器
    private static final String FRAGMENT_SHADER =
            "precision mediump float;\n" +
                    "uniform sampler2D uTexture;\n" +
                    "varying vec2 vTexCoord;\n" +
                    "void main() {\n" +
                    "  gl_FragColor = texture2D(uTexture, vTexCoord);\n" +
                    "}";

    // 顶点坐标 (左下角矩形)
    private static final float[] VERTICES = {
            -1.0f, -1.0f,
            -0.5f, -1.0f,
            -1.0f, -0.8f,
            -0.5f, -0.8f,
    };

    // 纹理坐标
    private static final float[] TEX_COORDS = {
            0.0f, 1.0f,
            1.0f, 1.0f,
            0.0f, 0.0f,
            1.0f, 0.0f,
    };

    private FloatBuffer vertexBuffer;
    private FloatBuffer texCoordBuffer;

    public TextOverlayRenderer() {
        vertexBuffer = ByteBuffer.allocateDirect(VERTICES.length * 4)
                .order(ByteOrder.nativeOrder()).asFloatBuffer();
        vertexBuffer.put(VERTICES).position(0);

        texCoordBuffer = ByteBuffer.allocateDirect(TEX_COORDS.length * 4)
                .order(ByteOrder.nativeOrder()).asFloatBuffer();
        texCoordBuffer.put(TEX_COORDS).position(0);

        programId = GlUtil.createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
        positionHandle = GLES20.glGetAttribLocation(programId, "aPosition");
        texCoordHandle = GLES20.glGetAttribLocation(programId, "aTexCoord");
        textureHandle = GLES20.glGetUniformLocation(programId, "uTexture");
    }

    /**
     * 生成带文字的纹理
     */
    public void updateText(String text) {
        Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
        paint.setTextSize(48);
        paint.setColor(Color.WHITE);

        int width = (int) paint.measureText(text);
        int height = 64;

        Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
        Canvas canvas = new Canvas(bitmap);
        canvas.drawText(text, 0, height - 10, paint);

        if (textTextureId == -1) {
            int[] textures = new int[1];
            GLES20.glGenTextures(1, textures, 0);
            textTextureId = textures[0];
            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textTextureId);
            GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
            GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
        }

        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textTextureId);
        GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);

        bitmap.recycle();
    }

    /**
     * 绘制文字纹理
     */
    public void draw() {
        if (textTextureId == -1) return;

        GLES20.glUseProgram(programId);

        GLES20.glEnableVertexAttribArray(positionHandle);
        GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);

        GLES20.glEnableVertexAttribArray(texCoordHandle);
        GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 0, texCoordBuffer);

        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textTextureId);
        GLES20.glUniform1i(textureHandle, 0);

        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);

        GLES20.glDisableVertexAttribArray(positionHandle);
        GLES20.glDisableVertexAttribArray(texCoordHandle);
    }
}

调用更新逻辑：

frameDrawer.setOverlayText("rtp: " +frame.getRtpTimestamp());

这里的frame是VideoFrame，安卓在渲染帧的时候会使用该类，所以在这个类上面做了拓展，增加了一个私有的rtp时间戳字段。