Develop Reference

How to grab a picture from a Surface

I am pretty new to Android so I hope I can get some directions here.
I have a 360 camera running on Android 7.0. It includes a SDK to get access to the live stitched images. In this SDK there is a function to set a Surface where the output from the stitched images will be directed to.
This is the function provided by the SDK:
public static void SDK.setSurface(Surface inputSurface)
I want to grab an image from that surface every second.
How do I create the right kind of Surface? And how do I grab images from this Surface?
Any help is highly appreciated!

Since I found the answer I might share it here too.
I have create a class which works like a charm. This is what I used:
class Capture: ImageReader.OnImageAvailableListener{
private var mImageReader: ImageReader? = null
private var mThreadHandler: HandlerThread? = null
fun start() {
if (!init) {
if (mImageReader != null) {
mImageReader?.close()
mImageReader = null
}
if (mThreadHandler != null) {
mThreadHandler?.quitSafely()
mThreadHandler = null
}
mThreadHandler = HandlerThread("prev")
mThreadHandler?.start()
mImageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 5)
mImageReader?.setOnImageAvailableListener(this, Handler(mThreadHandler?.getLooper()))
SDK.setSurface(if (mImageReader == null) null else mImageReader?.getSurface())
init = true
}
}
override fun onImageAvailable(reader: ImageReader) {
val image = reader.acquireLatestImage()
val height = image.height
val stride = image.planes[0].rowStride / image.planes[0].pixelStride
captureCallback(
stride,
height,
image.timestamp,
image.planes[0].buffer
)
image.close()
}
fun stop() {
if (init) {
if (mImageReader != null) {
mImageReader!!.close()
mImageReader = null
}
if (mThreadHandler != null) {
mThreadHandler!!.quitSafely()
mThreadHandler = null
}
SDK.setSurface(null)
}
init = false
}
fun captureCallback(width: Int, Height: Int, timestamp: Long, data: ByteBuffer) {
// do something with data
}
}
`

Memory leak in Android Screenshot mechanism

I'm developing an Android app that uses screenshot mechanism and then sends the captured data over network in infinite loop. After some time of work the system kills my app without any exception. I made an experiment where the whole app was cut off besides the screen shooter and the problem is still here. Android profiler does not show any issues but I can see very large AbstractList$Itr and byte[] as on image:
If I increase the RAM then the app lives longer. I can't find the leak for two weaks...
Full source code of ScreenShooter class:
import android.content.Context
import android.content.Intent
import android.graphics.Bitmap
import android.graphics.PixelFormat
import android.graphics.Point
import android.hardware.display.VirtualDisplay
import android.media.ImageReader
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.os.Handler
import android.os.HandlerThread
import android.os.Process
import android.view.Display
import android.view.WindowManager
import my_module.service.network.Networking
import java.io.ByteArrayOutputStream
import java.util.concurrent.locks.ReentrantLock
class ScreenShooter(private val network: Networking, private val context: Context): ImageReader.OnImageAvailableListener {
private val handlerThread: HandlerThread
private val handler: Handler
private var imageReader: ImageReader? = null
private var virtualDisplay: VirtualDisplay? = null
private var projection: MediaProjection? = null
private val mediaProjectionManager: MediaProjectionManager
private var latestBitmap: Bitmap? = null
private val byteStream = ByteArrayOutputStream()
private val mut: ReentrantLock = ReentrantLock()
private var width: Int
private var height: Int
private val TAG = ScreenShooter::class.java.simpleName
private val DELAY = 1000L
init {
handlerThread = HandlerThread(
javaClass.simpleName,
Process.THREAD_PRIORITY_BACKGROUND
).apply { start() }
handler = Handler(handlerThread.looper)
val windowManager = (context.getSystemService(Context.WINDOW_SERVICE) as WindowManager)
val display: Display = windowManager.defaultDisplay
val size = Point()
display.getRealSize(size)
var width = size.x
var height = size.y
while (width * height > 2 shl 19) {
width = (width * 0.9).toInt()
height = (height * 0.9).toInt()
}
this.width = width
this.height = height
mediaProjectionManager = context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
}
override fun onImageAvailable(reader: ImageReader?) {
mut.lock()
val image = imageReader?.acquireLatestImage() ?: return
var buffer = image.planes[0].buffer
buffer.rewind()
latestBitmap?.copyPixelsFromBuffer(buffer)
latestBitmap?.compress(Bitmap.CompressFormat.JPEG, 80, byteStream)
network.sendScreen(byteStream.toByteArray())
buffer.clear()
buffer = null
image.close()
byteStream.flush()
byteStream.reset()
byteStream.close()
mut.unlock()
Thread.sleep(DELAY)
}
fun startCapture(resultCode: Int, resultData: Intent) {
latestBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
imageReader = if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.N_MR1) {
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2)
} else {
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 1)
}
imageReader?.setOnImageAvailableListener(this, handler)
projection = mediaProjectionManager.getMediaProjection(resultCode, resultData)
virtualDisplay = projection!!.createVirtualDisplay(
"shooter",
width,
height,
context.resources.displayMetrics.densityDpi,
VIRT_DISPLAY_FLAGS,
imageReader?.surface,
null,
handler
)
projection?.registerCallback(projectionCallback, handler)
}
fun stopCapture() {
mut.lock()
imageReader?.setOnImageAvailableListener(null, null)
imageReader?.acquireLatestImage()?.close()
imageReader = null
projection?.unregisterCallback(projectionCallback)
projection?.stop()
virtualDisplay?.release()
imageReader?.close()
latestBitmap?.recycle()
mut.unlock()
}
private val projectionCallback = object : MediaProjection.Callback() {
override fun onStop() {
virtualDisplay?.release()
}
}
}
Methods startCapture() and stopCapture() are called from my background service in mainthread. network.sendScreen(byteStream.toByteArray()) just pushes the byte array (screenshot) to okhttp websocket queue:
fun sendScreen(bytes: ByteArray) {
val timestamp = System.currentTimeMillis()
val mss = Base64
.encodeToString(bytes, Base64.DEFAULT)
.replace("\n", "")
val message = """{ "data": {"timestamp":"$timestamp", "image":"$mss"} }"""
.trimMargin()
ws?.send(message.trimIndent())
}
Also I often get messages like Background concurrent copying GC freed but I got them without screenshooter with only network part and everything is Ok. I see no errors in logcat, no leaks in profiler.
I tried to recreate Screenshooter object in my service, but the app started to crash more often:
val t = HandlerThread(javaClass.simpleName).apply { start() }
Handler(t.looper).post {
while (true) {
Thread.sleep(10 * 1000L)
Log.d(TAG, "recreating the shooter")
Handler(mainLooper).post {
shooter!!.stopCapture()
shooter = null
shooter = ScreenShooter(network!!, applicationContext)
shooter!!.startCapture(resultCode, resultData!!)
}
}
}
However I supposed that this method would drop the previous created object out of GC Root and all its memory.
I've already no forces to search the leak. Thank you in advance.

How to render Bitmap off-screen in Android using OpenGL?

I need to render a bitmap without displaying it on the screen. For that I create OpenGL context using EGL14 as described in this answer. Then I save OpenGL surface to bitmap using GLES20.glReadPixels. But for some reason it is not rendered as expected and is just transparent.
import android.graphics.Bitmap
import android.opengl.*
import android.opengl.EGL14.EGL_CONTEXT_CLIENT_VERSION
import java.nio.ByteBuffer
class Renderer {
private lateinit var display: EGLDisplay
private lateinit var surface: EGLSurface
private lateinit var eglContext: EGLContext
fun draw() {
// Just a stub that fills the bitmap with red color
GLES20.glClearColor(1f, 0f, 0f, 1f)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
}
fun saveBitmap(): Bitmap {
val width = 320
val height = 240
val mPixelBuf = ByteBuffer.allocate(width * height * 4)
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPixelBuf)
return Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
}
private fun initializeEglContext() {
display = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
if (display == EGL14.EGL_NO_DISPLAY) {
throw RuntimeException("eglGetDisplay failed ${EGL14.eglGetError()}")
}
val versions = IntArray(2)
if (!EGL14.eglInitialize(display, versions, 0, versions, 1)) {
throw RuntimeException("eglInitialize failed ${EGL14.eglGetError()}")
}
val configAttr = intArrayOf(
EGL14.EGL_COLOR_BUFFER_TYPE, EGL14.EGL_RGB_BUFFER,
EGL14.EGL_LEVEL, 0,
EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT,
EGL14.EGL_NONE
)
val configs: Array<EGLConfig?> = arrayOfNulls(1)
val numConfig = IntArray(1)
EGL14.eglChooseConfig(
display, configAttr, 0,
configs, 0, 1, numConfig, 0
)
if (numConfig[0] == 0) {
throw RuntimeException("No configs found")
}
val config: EGLConfig? = configs[0]
val surfAttr = intArrayOf(
EGL14.EGL_WIDTH, 320,
EGL14.EGL_HEIGHT, 240,
EGL14.EGL_NONE
)
surface = EGL14.eglCreatePbufferSurface(display, config, surfAttr, 0)
val contextAttrib = intArrayOf(
EGL_CONTEXT_CLIENT_VERSION, 2,
EGL14.EGL_NONE
)
eglContext = EGL14.eglCreateContext(display, config, EGL14.EGL_NO_CONTEXT, contextAttrib, 0)
EGL14.eglMakeCurrent(display, surface, surface, eglContext)
}
fun destroy() {
EGL14.eglMakeCurrent(display, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
EGL14.EGL_NO_CONTEXT)
EGL14.eglDestroySurface(display, surface)
EGL14.eglDestroyContext(display, eglContext)
EGL14.eglTerminate(display)
}
}
This is how I use it:
val renderer = Renderer()
renderer.initializeEglContext()
renderer.draw()
val bitmap = renderer.saveBitmap()
renderer.destroy()
The code runs without any errors. I checked that context is created successfully. For example GLES20.glCreateProgram works as expected and returns a valid id. The only warning I get is
W/OpenGLRenderer: Failed to choose config with EGL_SWAP_BEHAVIOR_PRESERVED, retrying without...
But I'm not sure if it affects the result in any way.
However bitmap is not filled with color and is transparent:
val color = bitmap[0, 0]
Log.d("Main", "onCreate: ${Color.valueOf(color)}")
Color(0.0, 0.0, 0.0, 0.0, sRGB IEC61966-2.1)
I guess that I'm missing something, but I can't figure out what. How to make it to actually render?

Pixel buffer must be copied to bitmap:
val mPixelBuf bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
bitmap.copyPixelsFromBuffer(mPixelBuf)
return bitmap

Camera X captures images in different rotation states

Okay, I went through different posts and find out that depending on mobile manufacturers there can be a complications such as capture images get rotated, so you have to be aware of that. What I did was:
fun rotateBitmap(bitmap: Bitmap): Bitmap? {
val matrix = Matrix()
when (getImageOrientation(bitmap)) {
ExifInterface.ORIENTATION_NORMAL -> return bitmap
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.setScale(-1f, 1f)
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.setRotate(-90f)
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.setRotate(180f)
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.setRotate(90f)
ExifInterface.ORIENTATION_FLIP_VERTICAL -> {
matrix.setRotate(180f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSPOSE -> {
matrix.setRotate(90f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSVERSE -> {
matrix.setRotate(-90f)
matrix.postScale(-1f, 1f)
}
else -> return bitmap
}
This worked. But then I noticed something really weird and that might be related with how I configured Camera X configuration.
With the same device I get differently rotated Bitmaps (well, this should not happen. If devices rotates image weirdly, it should rotate images in both modes - in ImageAnalysesUseCase and ImageCaptureUseCase).
So, why is this happening and how can I fix it?
Code implementation:
Binding camera X to life-cycle:
CameraX.bindToLifecycle(
this,
buildPreviewUseCase(),
buildImageAnalysisUseCase(),
buildImageCaptureUseCase()
)
Preview use case:
private fun buildPreviewUseCase(): Preview {
val previewConfig = PreviewConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetResolution(config.resolution)
.setTargetRotation(Surface.ROTATION_0)
.setLensFacing(config.lensFacing)
.build()
return AutoFitPreviewBuilder.build(previewConfig, cameraTextureView)
}
Capture use case:
private fun buildImageCaptureUseCase(): ImageCapture {
val captureConfig = ImageCaptureConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetRotation(Surface.ROTATION_0)
.setTargetResolution(config.resolution)
.setCaptureMode(config.captureMode)
.build()
val capture = ImageCapture(captureConfig)
manualModeTakePhotoButton.setOnClickListener {
capture.takePicture(object : ImageCapture.OnImageCapturedListener() {
override fun onCaptureSuccess(imageProxy: ImageProxy, rotationDegrees: Int) {
viewModel.onManualCameraModeAnalysis(imageProxy, rotationDegrees)
}
override fun onError(useCaseError: ImageCapture.UseCaseError?, message: String?, cause: Throwable?) {
//
}
})
}
return capture
}
Analysis use case:
private fun buildImageAnalysisUseCase(): ImageAnalysis {
val analysisConfig = ImageAnalysisConfig.Builder().apply {
val analyzerThread = HandlerThread("xAnalyzer").apply { start() }
analyzerHandler = Handler(analyzerThread.looper)
setCallbackHandler(analyzerHandler!!)
setTargetAspectRatio(config.aspectRatio)
setTargetRotation(Surface.ROTATION_0)
setTargetResolution(config.resolution)
setImageReaderMode(config.readerMode)
setImageQueueDepth(config.queueDepth)
}.build()
val analysis = ImageAnalysis(analysisConfig)
analysis.analyzer = ImageRecognitionAnalyzer(viewModel)
return analysis
}
AutoFitPreviewBuilder:
class AutoFitPreviewBuilder private constructor(config: PreviewConfig,
viewFinderRef: WeakReference<TextureView>) {
/** Public instance of preview use-case which can be used by consumers of this adapter */
val useCase: Preview
/** Internal variable used to keep track of the use-case's output rotation */
private var bufferRotation: Int = 0
/** Internal variable used to keep track of the view's rotation */
private var viewFinderRotation: Int? = null
/** Internal variable used to keep track of the use-case's output dimension */
private var bufferDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's dimension */
private var viewFinderDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's display */
private var viewFinderDisplay: Int = -1
/** Internal reference of the [DisplayManager] */
private lateinit var displayManager: DisplayManager
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) {
val viewFinder = viewFinderRef.get() ?: return
if (displayId == viewFinderDisplay) {
val display = displayManager.getDisplay(displayId)
val rotation = getDisplaySurfaceRotation(display)
updateTransform(viewFinder, rotation, bufferDimens, viewFinderDimens)
}
}
}
init {
// Make sure that the view finder reference is valid
val viewFinder = viewFinderRef.get() ?:
throw IllegalArgumentException("Invalid reference to view finder used")
// Initialize the display and rotation from texture view information
viewFinderDisplay = viewFinder.display.displayId
viewFinderRotation = getDisplaySurfaceRotation(viewFinder.display) ?: 0
// Initialize public use-case with the given config
useCase = Preview(config)
// Every time the view finder is updated, recompute layout
useCase.onPreviewOutputUpdateListener = Preview.OnPreviewOutputUpdateListener {
val viewFinder =
viewFinderRef.get() ?: return#OnPreviewOutputUpdateListener
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = viewFinder.parent as ViewGroup
parent.removeView(viewFinder)
parent.addView(viewFinder, 0)
viewFinder.surfaceTexture = it.surfaceTexture
bufferRotation = it.rotationDegrees
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, it.textureSize, viewFinderDimens)
}
// Every time the provided texture view changes, recompute layout
viewFinder.addOnLayoutChangeListener { view, left, top, right, bottom, _, _, _, _ ->
val viewFinder = view as TextureView
val newViewFinderDimens = Size(right - left, bottom - top)
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, bufferDimens, newViewFinderDimens)
}
// Every time the orientation of device changes, recompute layout
displayManager = viewFinder.context
.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)
// Remove the display listeners when the view is detached to avoid
// holding a reference to the View outside of a Fragment.
// NOTE: Even though using a weak reference should take care of this,
// we still try to avoid unnecessary calls to the listener this way.
viewFinder.addOnAttachStateChangeListener(object : View.OnAttachStateChangeListener {
override fun onViewAttachedToWindow(view: View?) {
displayManager.registerDisplayListener(displayListener, null)
}
override fun onViewDetachedFromWindow(view: View?) {
displayManager.unregisterDisplayListener(displayListener)
}
})
}
/** Helper function that fits a camera preview into the given [TextureView] */
private fun updateTransform(textureView: TextureView?, rotation: Int?, newBufferDimens: Size,
newViewFinderDimens: Size) {
// This should not happen anyway, but now the linter knows
val textureView = textureView ?: return
if (rotation == viewFinderRotation &&
Objects.equals(newBufferDimens, bufferDimens) &&
Objects.equals(newViewFinderDimens, viewFinderDimens)) {
// Nothing has changed, no need to transform output again
return
}
if (rotation == null) {
// Invalid rotation - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderRotation = rotation
}
if (newBufferDimens.width == 0 || newBufferDimens.height == 0) {
// Invalid buffer dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
bufferDimens = newBufferDimens
}
if (newViewFinderDimens.width == 0 || newViewFinderDimens.height == 0) {
// Invalid view finder dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderDimens = newViewFinderDimens
}
val matrix = Matrix()
// Compute the center of the view finder
val centerX = viewFinderDimens.width / 2f
val centerY = viewFinderDimens.height / 2f
// Correct preview output to account for display rotation
matrix.postRotate(-viewFinderRotation!!.toFloat(), centerX, centerY)
// Buffers are rotated relative to the device's 'natural' orientation: swap width and height
val bufferRatio = bufferDimens.height / bufferDimens.width.toFloat()
val scaledWidth: Int
val scaledHeight: Int
// Match longest sides together -- i.e. apply center-crop transformation
if (viewFinderDimens.width > viewFinderDimens.height) {
scaledHeight = viewFinderDimens.width
scaledWidth = Math.round(viewFinderDimens.width * bufferRatio)
} else {
scaledHeight = viewFinderDimens.height
scaledWidth = Math.round(viewFinderDimens.height * bufferRatio)
}
// Compute the relative scale value
val xScale = scaledWidth / viewFinderDimens.width.toFloat()
val yScale = scaledHeight / viewFinderDimens.height.toFloat()
// Scale input buffers to fill the view finder
matrix.preScale(xScale, yScale, centerX, centerY)
// Finally, apply transformations to our TextureView
textureView.setTransform(matrix)
}
companion object {
/** Helper function that gets the rotation of a [Display] in degrees */
fun getDisplaySurfaceRotation(display: Display?) = when(display?.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> null
}
/**
* Main entrypoint for users of this class: instantiates the adapter and returns an instance
* of [Preview] which automatically adjusts in size and rotation to compensate for
* config changes.
*/
fun build(config: PreviewConfig, viewFinder: TextureView) =
AutoFitPreviewBuilder(config, WeakReference(viewFinder)).useCase
}
}
If configuration is correct (it looks okay to me), then next idea was that maybe converting captured images objects to bitmap might be faulty. Below you can see implementation.
Capture mode uses this function:
fun imageProxyToBitmap(image: ImageProxy): Bitmap {
val buffer: ByteBuffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
Analysis mode uses this function:
fun toBitmapFromImage(image: Image?): Bitmap? {
try {
if (image == null || image.planes[0] == null || image.planes[1] == null || image.planes[2] == null) {
return null
}
val yBuffer = image.planes[0].buffer
val uBuffer = image.planes[1].buffer
val vBuffer = image.planes[2].buffer
val ySize = yBuffer.remaining()
val uSize = uBuffer.remaining()
val vSize = vBuffer.remaining()
val nv21 = ByteArray(ySize + uSize + vSize)
/* U and V are swapped */
yBuffer.get(nv21, 0, ySize)
vBuffer.get(nv21, ySize, vSize)
uBuffer.get(nv21, ySize + vSize, uSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, image.width, image.height, null)
val out = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, yuvImage.width, yuvImage.height), 50, out)
val imageBytes = out.toByteArray()
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
} catch (e: IllegalStateException) {
Log.e("IllegalStateException", "#ImageUtils.toBitmapFromImage(): Can't read the image file.")
return null
}
}
So, weirdly, on few devices toBitmapFromImage() sometimes comes up upwards, but at the same time (same device) imageProxyToBitmap() returns image in correct rotation - it has to be the image to bitmap functions fault, right?Why is this happening (because capture mode returns image normally) and how to fix this?

Inside onImageCaptureSuccess, get the rotationDegrees and rotate your bitmap by that degree to get the correct orientation.
override fun onImageCaptureSuccess(image: ImageProxy) {
val capturedImageBitmap = image.image?.toBitmap()?.rotate(image.imageInfo.rotationDegrees.toFloat())
mBinding.previewImage.setImageBitmap(capturedImageBitmap)
showPostClickViews()
mCurrentFlow = FLOW_CAMERA
}
toBitmap() and rotate() are extension functions.
fun Image.toBitmap(): Bitmap {
val buffer = planes[0].buffer
buffer.rewind()
val bytes = ByteArray(buffer.capacity())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
fun Bitmap.rotate(degrees: Float): Bitmap =
Bitmap.createBitmap(this, 0, 0, width, height, Matrix().apply { postRotate(degrees) }, true)

CameraX returns the captured image with a rotation value in the callback, which can be used to rotate the image.
https://developer.android.com/reference/androidx/camera/core/ImageCapture.OnImageCapturedListener.html#onCaptureSuccess(androidx.camera.core.ImageProxy,%20int)

For Analyzer UseCases, you have to get rotationDegree coming through analyze method of ImageAnalysis.Analyzer and work accordingly.
Hope it helps!

ARCore with Twilio how is it implemented?

I would like to implement ARCore with Twilio's video call. The documentation says this is possible but I could not figure out how to do it. Can Anyone tell me what I'm doing wrong?
This is my activity:
class MixActivity : AppCompatActivity() {
private lateinit var mArFragment: ArFragment
private lateinit var mVideoView: ArSceneView
private var mScreenVideoTrack: LocalVideoTrack? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_mix)
mArFragment = ar_fragment as ArFragment
mVideoView = mArFragment.arSceneView
mScreenVideoTrack = LocalVideoTrack.create(this, true,
ViewCapturer(mVideoView)
)
} }
This is view:
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="#+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent">
<fragment
android:id="#+id/ar_fragment"
android:name="com.google.ar.sceneform.ux.ArFragment"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
</RelativeLayout>
And VideoCapture:
internal class ViewCapturer(private val view: View) : VideoCapturer, PixelCopy.OnPixelCopyFinishedListener {
private val handler = Handler(Looper.getMainLooper())
private var videoCapturerListener: VideoCapturer.Listener? = null
private val started = AtomicBoolean(false)
private lateinit var mViewBitmap: Bitmap
private val viewCapturer = object : Runnable {
override fun run() {
val dropFrame = view.width == 0 || view.height == 0
// Only capture the view if the dimensions have been established
if (!dropFrame) {
// Draw view into bitmap backed canvas
val measuredWidth = View.MeasureSpec.makeMeasureSpec(
view.width,
View.MeasureSpec.EXACTLY
)
val measuredHeight = View.MeasureSpec.makeMeasureSpec(
view.height,
View.MeasureSpec.EXACTLY
)
view.measure(measuredWidth, measuredHeight)
view.layout(0, 0, view.measuredWidth, view.measuredHeight)
mViewBitmap = Bitmap.createBitmap(
view.width, view.height,
Bitmap.Config.ARGB_8888
)
val viewCanvas = Canvas(mViewBitmap)
view.draw(viewCanvas)
// Extract the frame from the bitmap
val bytes = mViewBitmap.byteCount
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener!!.onFrameCaptured(videoFrame)
}
}
// Schedule the next capture
if (started.get()) {
handler.postDelayed(this, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
}
/**
* Returns the list of supported formats for this view capturer. Currently, only supports
* capturing to RGBA_8888 bitmaps.
*
* #return list of supported formats.
*/
override fun getSupportedFormats(): List<VideoFormat> {
val videoFormats = ArrayList<VideoFormat>()
val videoDimensions = VideoDimensions(view.width, view.height)
val videoFormat = VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888)
videoFormats.add(videoFormat)
return videoFormats
}
/**
* Returns true because we are capturing screen content.
*/
override fun isScreencast(): Boolean {
return true
}
/**
* This will be invoked when it is time to start capturing frames.
*
* #param videoFormat the video format of the frames to be captured.
* #param listener capturer listener.
*/
override fun startCapture(videoFormat: VideoFormat, listener: VideoCapturer.Listener) {
// Store the capturer listener
this.videoCapturerListener = listener
this.started.set(true)
// Notify capturer API that the capturer has started
val capturerStarted = handler.postDelayed(
viewCapturer,
VIEW_CAPTURER_FRAMERATE_MS.toLong()
)
this.videoCapturerListener!!.onCapturerStarted(capturerStarted)
}
/**
* Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
*/
override fun stopCapture() {
this.started.set(false)
handler.removeCallbacks(viewCapturer)
}
override fun onPixelCopyFinished(i: Int) {
// Extract the frame from the bitmap
val bytes = mViewBitmap.getByteCount()
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener?.onFrameCaptured(videoFrame)
}
if (started.get()) {
handler.postDelayed(viewCapturer, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
companion object {
private val VIEW_CAPTURER_FRAMERATE_MS = 100
}
}
The ARCore part works but the Twilio part does not work.
I referred to another post that talked about it but it was incomplete:
Streaming CustomView ARcore with Twilio video