音视频开发之相机预览及简单滤镜

这是我参与8月更文挑战的第2天，活动详情查看：8月更文挑战

前面一篇介绍了使用camerax 和 PreviewView进行预览并获取一帧yuv图像的过程。本篇简要介绍一下使Opengles进行相机数据的预览。

Opengles在安卓中版本

Android 可通过开放图形库 (OpenGL®)（特别是 OpenGL ES API）来支持高性能 2D 和 3D 图形。OpenGL 是一种跨平台的图形 API，用于为 3D 图形处理硬件指定标准的软件接口。OpenGL ES 是 OpenGL 规范的一种形式，适用于嵌入式设备。Android 支持多版 OpenGL ES API：

• OpenGL ES 1.0 和 1.1 - 此 API 规范受 Android 1.0 及更高版本的支持。
• OpenGL ES 2.0 - 此 API 规范受 Android 2.2（API 级别 8）及更高版本的支持。
• OpenGL ES 3.0 - 此 API 规范受 Android 4.3（API 级别 18）及更高版本的支持。
• OpenGL ES 3.1 - 此 API 规范受 Android 5.0（API 级别 21）及更高版本的支持。

camerex 结合 GlSurfaceView 实现预览

//开启相机
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener(Runnable {
    bindImage(cameraProviderFuture)
}, ContextCompat.getMainExecutor(this))

//camerax关联GlSurfaceView
private fun bindImage(cameraProviderFuture: ListenableFuture<ProcessCameraProvider>) {
    cameraProvider = cameraProviderFuture.get()
    preview = Preview.Builder().build()
    try {
        val imageCapture = ImageCapture.Builder().build()
        gl_view.attachPreview(preview)
        cameraProvider?.unbindAll()
        val camera:Camera? = cameraProvider?.bindToLifecycle(this, cameraSelector,imageCapture ,preview)
    } catch (exc: Exception) {
        Log.e(TAG, "Use case binding failed", exc)
    }
}

//相机和opegles共用一个纹理
public void attachPreview(Preview preview) {
    preview.setSurfaceProvider(new Preview.SurfaceProvider() {
        @Override
        public void onSurfaceRequested(@NonNull SurfaceRequest request) {
            mResolution = request.getResolution();  
           //需要给纹理设置宽高，不然画面可能会很模糊  
           surfaceTexture.setDefaultBufferSize(mResolution.getWidth(),mResolution.getHeight());
            Surface surface = new Surface(surfaceTexture);
            request.provideSurface(surface, executor, new Consumer<SurfaceRequest.Result>() {
                @Override
                public void accept(SurfaceRequest.Result result) {
                    surface.release();
                    surfaceTexture.release();
                }
            });
        }
    });
}

使用Opengles需要设置版本和设置Render

setEGLContextClientVersion(3);
setRenderer(this);
// 设置非连续渲染,需要调用 requestRender 更新画面
setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);

在render中处理数据

    @Override
    public void onSurfaceCreated(GL10 gl, EGLConfig config) {
        int[] textureIds = new int[1];
        GLES30.glGenTextures(1, ids, 0);
        textureId = textureIds[0];
        surfaceTexture = new SurfaceTexture(textureId);
        surfaceTexture.setOnFrameAvailableListener(this);
        directDrawer = new DirectDrawer(textureId);
    }

    @Override
    public void onSurfaceChanged(GL10 gl, int width, int height) {
        GLES30.glViewport(0, 0, width, height);
    }
    
    @Override
    public void onFrameAvailable(SurfaceTexture surfaceTexture) {
        requestRender();
    }

public class DirectDrawer {
    //顶点着色器 attribute：输入矩阵变量， varying：传数据给片元着色器
    private final String vertexShaderCode =
                    "attribute vec4 vPosition;" +  //顶点矩阵
                    "attribute vec2 inputTextureCoordinate;" + //纹理矩阵
                    "varying vec2 textureCoordinate;" +
                    "void main()" +
                    "{"+
                        "gl_Position = vPosition;"+
                        "textureCoordinate = inputTextureCoordinate;" +
                    "}";
    //片元着色器，注意此处必须用samplerExternalOES 才能渲染到相机
    private final String fragmentShaderCode =
            "#extension GL_OES_EGL_image_external : require\n"+
                    "precision mediump float;" +  //声明float型精度
                    "varying vec2 textureCoordinate;\n" +
                    "uniform samplerExternalOES s_texture;\n" +
                    "void main() {" +
                        "gl_FragColor = texture2D( s_texture, textureCoordinate );\n" +
                    "}";


    private FloatBuffer vertexBuffer, textureVerticesBuffer;
    private ShortBuffer drawListBuffer;
    private final int mProgram;
    private int mPositionHandle;
    private int mTextureCoordHandle;
    //short占2个字节
    private static final int COORDS_PER_VERTEX = 2;
  
    //float占4个字节
    private final int vertexStride = COORDS_PER_VERTEX * 4; 
    
    //顶点矩阵
    static float squareCoords[] = {
            -1.0f,  1.0f,
            -1.0f, -1.0f,
            1.0f, 1.0f,
            1.0f,  -1.0f,

    };

    //纹理矩阵，注意坐标变换，按原始坐标会朝左旋转90度
    static float textureVertices[] = {
            0f,1f,
            0f,0f,
            1f,1f,
            1f,0f
    };

    private int texture;

    public DirectDrawer(int texture)
    {
        this.texture = texture;
        ByteBuffer bb = ByteBuffer.allocateDirect(squareCoords.length * 4);
        bb.order(ByteOrder.nativeOrder());
        vertexBuffer = bb.asFloatBuffer();
        vertexBuffer.put(squareCoords);
        vertexBuffer.position(0);

        // 将数据转为 FlostBuffer
        ByteBuffer dlb = ByteBuffer.allocateDirect(drawOrder.length * 2);
        dlb.order(ByteOrder.nativeOrder());
        drawListBuffer = dlb.asShortBuffer();
        drawListBuffer.put(drawOrder);
        drawListBuffer.position(0);
        ByteBuffer bb2 = ByteBuffer.allocateDirect(textureVertices.length * 4);
        bb2.order(ByteOrder.nativeOrder());
        textureVerticesBuffer = bb2.asFloatBuffer();
        textureVerticesBuffer.put(textureVertices);
        textureVerticesBuffer.position(0);
        //创建着色器
        int vertexShader    = loadShader(GLES30.GL_VERTEX_SHADER, vertexShaderCode);
        int fragmentShader  = loadShader(GLES30.GL_FRAGMENT_SHADER, fragmentShaderCode);
        
        //链接着色器
        mProgram = GLES30.glCreateProgram();             // create empty OpenGL ES Program
        GLES30.glAttachShader(mProgram, vertexShader);   // add the vertex shader to program
        GLES30.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
        GLES30.glLinkProgram(mProgram);                  // creates OpenGL ES program executables
    }
    
    //将顶点数据和纹理数据传给程式渲染纹理
    public void draw(float[] mtx){ 
        GLES30.glUseProgram(mProgram);
        GLES30.glActiveTexture(GLES30.GL_TEXTURE0);
        GLES30.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture);
        mPositionHandle = GLES30.glGetAttribLocation(mProgram, "vPosition");
        GLES30.glEnableVertexAttribArray(mPositionHandle);
        GLES30.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX, GLES30.GL_FLOAT, false, vertexStride, vertexBuffer);
        mTextureCoordHandle = GLES30.glGetAttribLocation(mProgram, "inputTextureCoordinate");
        GLES30.glEnableVertexAttribArray(mTextureCoordHandle);
        GLES30.glVertexAttribPointer(mTextureCoordHandle, COORDS_PER_VERTEX, GLES30.GL_FLOAT, false, vertexStride, textureVerticesBuffer);
        //GL_TRIANGLE_STRIP这里使用的是三角形片带，只需要4个顶点就可以绘制一个矩形
        //GL_TRIANGLES 三角形 绘制需要6个顶点
        GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP,0,4);
        GLES30.glDisableVertexAttribArray(mPositionHandle);
        GLES30.glDisableVertexAttribArray(mTextureCoordHandle);
    }

    private  int loadShader(int type, String shaderCode){
        int shader = GLES30.glCreateShader(type);
        GLES30.glShaderSource(shader, shaderCode);
        GLES30.glCompileShader(shader);
        return shader;
    }
  
}

这里使用了只做了相机预览处理，效果如下图：

下面可以做一点矩阵变换，添加滤镜效果

private final String fragmentShaderCode =
        "#extension GL_OES_EGL_image_external : require\n"+
        "precision mediump float;" +  //声明float型
        "varying vec2 textureCoordinate;\n" +
        "uniform samplerExternalOES s_texture;\n" +
        "void main() {" +
            "mediump vec4 textureColor = texture2D(s_texture, textureCoordinate);\n"+
            "float gray = textureColor.r * 0.2125 + textureColor.g * 0.7154 + textureColor.b * 0.0721;\n"+
            "gl_FragColor =  vec4(gray, gray, gray, textureColor.w);"+
        "}";

效果如下图：

ps：本人在纹理坐标的变换尝试了多种组合才将图形变正常，如果有详细的变换原理，望在评论区指出，不甚感激