本文已参与「新人创作礼」活动,一起开启掘金创作之路。
minicap
minicap 是一个用于android设备截屏的库。其代码在 minicap。
experimental
由于许多库不支持android 12设备的截屏,因此看到 Minicap 的experimental 项目。 新版本的minicap 尝试使用java 来实现截图。仓库地址:experimental。
- Main.kt main函数所在,命令行参数解析等
- simpleServer.kt 一个简单的socket服务器
- DisplayOutput.kt 抽象类,由
send
方法和一个用于存储数据的imageBuffer
- MinicapClientOutput.kt 实现了"minicap" 协议, 并提供了向客户端发送数据的方法
- ScreensshotOutput.kt 实现了
DisplayOutput
类的send
方法,其主要作用是将拿到的数据转为 bit 形式 - BaseProvider.kt 提供了
onImageAvailable, encode
等方法,分别用于监听是否有可用image数据,以及对image数据进行编码等 - SurfaceProvider.kt 获取原始image数据的主要逻辑在此
- SurfaceControl 反射android.view.SurfaceControl 类以及相关API
BaseProvider
BaseProvider
类的主要任务在于实现了 onImageAvailable, encode
等接口供 SurfaceProvider调用。
该类的逻辑如下:
abstract class BaseProvider(private val displayId: Int, private val targetSize: Size, val rotation: Int) : SimpleServer.Listener,
ImageReader.OnImageAvailableListener {
companion object {
val log = LoggerFactory.getLogger(BaseProvider::class.java.simpleName)
}
private lateinit var clientOutput: DisplayOutput
private lateinit var imageReader: ImageReader
private var previousTimeStamp: Long = 0L
private var framePeriodMs: Long = 0
private var bitmap: Bitmap? = null //is used to compress the images
var quality: Int = 100
var frameRate: Float = Float.MAX_VALUE
set(value) {
this.framePeriodMs = (1000 / value).toLong()
log.info("framePeriodMs: $framePeriodMs")
field = value
}
abstract fun screenshot(printer: PrintStream)
abstract fun getScreenSize(): Size
fun getTargetSize(): Size = if(rotation%2 != 0) Size(targetSize.height, targetSize.width) else targetSize
fun getImageReader(): ImageReader = imageReader
fun init(out: DisplayOutput) {
imageReader = ImageReader.newInstance(
getTargetSize().width,
getTargetSize().height,
PixelFormat.RGBA_8888,
2
)
clientOutput = out
}
/* code */
// 监听来自 imageReader 的数据是否可用,可用则调用encode 方法,对原始数据进行编码,生成图像
override fun onImageAvailable(reader: ImageReader) {
val image = reader.acquireLatestImage()
val currentTime = System.currentTimeMillis()
if (image != null) {
if (currentTime - previousTimeStamp > framePeriodMs) {
previousTimeStamp = currentTime
encode(image, quality, clientOutput.imageBuffer)
clientOutput.send()
} else {
log.warn("skipping frame ($currentTime/$previousTimeStamp)")
}
image.close()
} else {
log.warn("no image available")
}
}
// 将原始数据转换为 JPEG 格式的数据流
private fun encode(image: Image, q: Int, out: OutputStream) {
with(image) {
val planes: Array<Image.Plane> = planes
val buffer: ByteBuffer = planes[0].buffer
val pixelStride: Int = planes[0].pixelStride
val rowStride: Int = planes[0].rowStride
val rowPadding: Int = rowStride - pixelStride * width
// createBitmap can be resources consuming
bitmap ?: Bitmap.createBitmap(
width + rowPadding / pixelStride,
height,
Bitmap.Config.ARGB_8888
).apply {
copyPixelsFromBuffer(buffer)
}.run {
//the image need to be cropped
Bitmap.createBitmap(this, 0, 0, getTargetSize().width, getTargetSize().height)
}.apply {
compress(Bitmap.CompressFormat.JPEG, q, out)
}
}
}
}
SurfaceProvider
SurfaceProvider.kt
中实现了获取原始图像数据主要逻辑:
/**
* Provides screen images using [SurfaceControl]. This is pretty similar to the native version
* of minicap but here it is done at a higher level making things a bit easier.
*/
class SurfaceProvider(displayId: Int, targetSize: Size, orientation: Int) : BaseProvider(displayId, targetSize, orientation) {
constructor(display: Int) : this(display, currentScreenSize(), currentRotation())
/* code */
private val handler: Handler = Handler(Looper.getMainLooper())
private var display: IBinder? = null
val displayInfo: DisplayInfo = DisplayManagerGlobal.getDisplayInfo(displayId)
override fun getScreenSize(): Size = displayInfo.size
override fun screenshot(printer: PrintStream) {
init(ScreenshotOutput(printer))
// 监听 imageReader 类型的it, 若it 可用则调用 super.onImageAvailable 函数,将display 转成 JPEG格式图像
initSurface {
super.onImageAvailable(it)
exitProcess(0)
}
}
/* code */
/**
* Setup the Surface between the display and an ImageReader so that we can grab the
* screen.
*/
// 该部分截图的原理是,截取屏幕和ImageReader 中间的数据流来完成截屏
private fun initSurface(l: ImageReader.OnImageAvailableListener) {
//must be done on the main thread
// Support Android 12 (preview),and resolve black screen problem
val secure =
Build.VERSION.SDK_INT < Build.VERSION_CODES.R || Build.VERSION.SDK_INT == Build.VERSION_CODES.R && "S" != Build.VERSION.CODENAME
display = SurfaceControl.createDisplay("minicap", secure)
//initialise the surface to get the display in the ImageReader
SurfaceControl.openTransaction()
try {
SurfaceControl.setDisplaySurface(display, getImageReader().surface)
SurfaceControl.setDisplayProjection(
display,
0,
Rect(0, 0, getScreenSize().width, getScreenSize().height),
Rect(0, 0, getTargetSize().width, getTargetSize().height)
)
SurfaceControl.setDisplayLayerStack(display, displayInfo.layerStack)
} finally {
SurfaceControl.closeTransaction()
}
getImageReader().setOnImageAvailableListener(l, handler)
}
private fun initSurface() {
initSurface(this)
}
}
通过上述代码可以看出,其实主要完成截图工作API实现的是SurfaceControl
类,minicap 对于该类的定义如下:
/** * Provide access to the SurfaceControl which is not part of the Android SDK using
reflection. * This SurfaceControl relies on a jni bindings that manages the
SurfaceComposerClient that is * in use in minicap-shared library. */object SurfaceControl {
/* code */
}
简单来说,SurfaceControl
类是一个使用了JNI反射了android.view.SurfaceControl 类以及该类相关API的类。在了解了minicap使用了什么技术之后,再来看initSurface
函数的逻辑。
- 首先,
initSurface
会判断当前运行的android 机器的版本,之后调用android.view.SurfaceControl.createDisplay
,来创建一个 Display。
android 源码中对于createDisplay
的定义如下:
public static IBinder createDisplay(String name, boolean secure) {
if (name == null) {
throw new IllegalArgumentException("name must not be null");
}
// 调用了native 方法
return nativeCreateDisplay(name, secure);
}
- 在创建了一个 Display 之后,调用
android.view.SurfaceControl.openTransaction
,初始化一个 surface用来在 ImageReader中获取Display。 android 源码中对于openTransaction
的定义如下:
/** start a transaction */
public static void openTransaction() {
nativeOpenTransaction();
}
- 初始化结束后,调用
SurfaceControl.setDisplaySurface
从ImageReader中拿出一个surface 放到display中。 android 源码中对于setDisplaySurface
的定义如下:
public static void setDisplaySurface(IBinder displayToken, Surface surface) {
if (displayToken == null) {
throw new IllegalArgumentException("displayToken must not be null");
}
if (surface != null) {
synchronized (surface.mLock) {
nativeSetDisplaySurface(displayToken, surface.mNativeObject);
}
} else {
nativeSetDisplaySurface(displayToken, 0);
}
}
- 调用
SurfaceControl.setDisplayProjection
建立"投影",让拿到的display 以屏幕的大小而保存。 android 源码中对于`setDisplayProjection`` 的定义如下:
public static void setDisplayProjection(IBinder displayToken,
int orientation, Rect layerStackRect, Rect displayRect) {
if (displayToken == null) {
throw new IllegalArgumentException("displayToken must not be null");
}
if (layerStackRect == null) {
throw new IllegalArgumentException("layerStackRect must not be null");
}
if (displayRect == null) {
throw new IllegalArgumentException("displayRect must not be null");
}
nativeSetDisplayProjection(displayToken, orientation,
layerStackRect.left, layerStackRect.top, layerStackRect.right, layerStackRect.bottom,
displayRect.left, displayRect.top, displayRect.right, displayRect.bottom);
}
- 此时已经得到了屏幕大小的display,通过display 来设置 DisplayLayerStack,
SurfaceControl.setDisplayLayerStack
。 android 源码中对于setDisplayLayerStack
的定义如下:
public static void setDisplayLayerStack(IBinder displayToken, int layerStack) {
if (displayToken == null) {
throw new IllegalArgumentException("displayToken must not be null");
}
nativeSetDisplayLayerStack(displayToken, layerStack);
}
- 在获得了display的信息后,通过
getImageReader().setOnImageAvailableListener(l, handler)
将数据绑定。