Okay, I went through different posts and find out that depending on mobile manufacturers there can be a complications such as capture images get rotated, so you have to be aware of that. What I did was:
fun rotateBitmap(bitmap: Bitmap): Bitmap? {
val matrix = Matrix()
when (getImageOrientation(bitmap)) {
ExifInterface.ORIENTATION_NORMAL -> return bitmap
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.setScale(-1f, 1f)
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.setRotate(-90f)
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.setRotate(180f)
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.setRotate(90f)
ExifInterface.ORIENTATION_FLIP_VERTICAL -> {
matrix.setRotate(180f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSPOSE -> {
matrix.setRotate(90f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSVERSE -> {
matrix.setRotate(-90f)
matrix.postScale(-1f, 1f)
}
else -> return bitmap
}
This worked. But then I noticed something really weird and that might be related with how I configured Camera X configuration.
With the same device I get differently rotated Bitmaps (well, this should not happen. If devices rotates image weirdly, it should rotate images in both modes - in ImageAnalysesUseCase and ImageCaptureUseCase).
So, why is this happening and how can I fix it?
Code implementation:
Binding camera X to life-cycle:
CameraX.bindToLifecycle(
this,
buildPreviewUseCase(),
buildImageAnalysisUseCase(),
buildImageCaptureUseCase()
)
Preview use case:
private fun buildPreviewUseCase(): Preview {
val previewConfig = PreviewConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetResolution(config.resolution)
.setTargetRotation(Surface.ROTATION_0)
.setLensFacing(config.lensFacing)
.build()
return AutoFitPreviewBuilder.build(previewConfig, cameraTextureView)
}
Capture use case:
private fun buildImageCaptureUseCase(): ImageCapture {
val captureConfig = ImageCaptureConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetRotation(Surface.ROTATION_0)
.setTargetResolution(config.resolution)
.setCaptureMode(config.captureMode)
.build()
val capture = ImageCapture(captureConfig)
manualModeTakePhotoButton.setOnClickListener {
capture.takePicture(object : ImageCapture.OnImageCapturedListener() {
override fun onCaptureSuccess(imageProxy: ImageProxy, rotationDegrees: Int) {
viewModel.onManualCameraModeAnalysis(imageProxy, rotationDegrees)
}
override fun onError(useCaseError: ImageCapture.UseCaseError?, message: String?, cause: Throwable?) {
//
}
})
}
return capture
}
Analysis use case:
private fun buildImageAnalysisUseCase(): ImageAnalysis {
val analysisConfig = ImageAnalysisConfig.Builder().apply {
val analyzerThread = HandlerThread("xAnalyzer").apply { start() }
analyzerHandler = Handler(analyzerThread.looper)
setCallbackHandler(analyzerHandler!!)
setTargetAspectRatio(config.aspectRatio)
setTargetRotation(Surface.ROTATION_0)
setTargetResolution(config.resolution)
setImageReaderMode(config.readerMode)
setImageQueueDepth(config.queueDepth)
}.build()
val analysis = ImageAnalysis(analysisConfig)
analysis.analyzer = ImageRecognitionAnalyzer(viewModel)
return analysis
}
AutoFitPreviewBuilder:
class AutoFitPreviewBuilder private constructor(config: PreviewConfig,
viewFinderRef: WeakReference<TextureView>) {
/** Public instance of preview use-case which can be used by consumers of this adapter */
val useCase: Preview
/** Internal variable used to keep track of the use-case's output rotation */
private var bufferRotation: Int = 0
/** Internal variable used to keep track of the view's rotation */
private var viewFinderRotation: Int? = null
/** Internal variable used to keep track of the use-case's output dimension */
private var bufferDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's dimension */
private var viewFinderDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's display */
private var viewFinderDisplay: Int = -1
/** Internal reference of the [DisplayManager] */
private lateinit var displayManager: DisplayManager
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) {
val viewFinder = viewFinderRef.get() ?: return
if (displayId == viewFinderDisplay) {
val display = displayManager.getDisplay(displayId)
val rotation = getDisplaySurfaceRotation(display)
updateTransform(viewFinder, rotation, bufferDimens, viewFinderDimens)
}
}
}
init {
// Make sure that the view finder reference is valid
val viewFinder = viewFinderRef.get() ?:
throw IllegalArgumentException("Invalid reference to view finder used")
// Initialize the display and rotation from texture view information
viewFinderDisplay = viewFinder.display.displayId
viewFinderRotation = getDisplaySurfaceRotation(viewFinder.display) ?: 0
// Initialize public use-case with the given config
useCase = Preview(config)
// Every time the view finder is updated, recompute layout
useCase.onPreviewOutputUpdateListener = Preview.OnPreviewOutputUpdateListener {
val viewFinder =
viewFinderRef.get() ?: return#OnPreviewOutputUpdateListener
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = viewFinder.parent as ViewGroup
parent.removeView(viewFinder)
parent.addView(viewFinder, 0)
viewFinder.surfaceTexture = it.surfaceTexture
bufferRotation = it.rotationDegrees
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, it.textureSize, viewFinderDimens)
}
// Every time the provided texture view changes, recompute layout
viewFinder.addOnLayoutChangeListener { view, left, top, right, bottom, _, _, _, _ ->
val viewFinder = view as TextureView
val newViewFinderDimens = Size(right - left, bottom - top)
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, bufferDimens, newViewFinderDimens)
}
// Every time the orientation of device changes, recompute layout
displayManager = viewFinder.context
.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)
// Remove the display listeners when the view is detached to avoid
// holding a reference to the View outside of a Fragment.
// NOTE: Even though using a weak reference should take care of this,
// we still try to avoid unnecessary calls to the listener this way.
viewFinder.addOnAttachStateChangeListener(object : View.OnAttachStateChangeListener {
override fun onViewAttachedToWindow(view: View?) {
displayManager.registerDisplayListener(displayListener, null)
}
override fun onViewDetachedFromWindow(view: View?) {
displayManager.unregisterDisplayListener(displayListener)
}
})
}
/** Helper function that fits a camera preview into the given [TextureView] */
private fun updateTransform(textureView: TextureView?, rotation: Int?, newBufferDimens: Size,
newViewFinderDimens: Size) {
// This should not happen anyway, but now the linter knows
val textureView = textureView ?: return
if (rotation == viewFinderRotation &&
Objects.equals(newBufferDimens, bufferDimens) &&
Objects.equals(newViewFinderDimens, viewFinderDimens)) {
// Nothing has changed, no need to transform output again
return
}
if (rotation == null) {
// Invalid rotation - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderRotation = rotation
}
if (newBufferDimens.width == 0 || newBufferDimens.height == 0) {
// Invalid buffer dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
bufferDimens = newBufferDimens
}
if (newViewFinderDimens.width == 0 || newViewFinderDimens.height == 0) {
// Invalid view finder dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderDimens = newViewFinderDimens
}
val matrix = Matrix()
// Compute the center of the view finder
val centerX = viewFinderDimens.width / 2f
val centerY = viewFinderDimens.height / 2f
// Correct preview output to account for display rotation
matrix.postRotate(-viewFinderRotation!!.toFloat(), centerX, centerY)
// Buffers are rotated relative to the device's 'natural' orientation: swap width and height
val bufferRatio = bufferDimens.height / bufferDimens.width.toFloat()
val scaledWidth: Int
val scaledHeight: Int
// Match longest sides together -- i.e. apply center-crop transformation
if (viewFinderDimens.width > viewFinderDimens.height) {
scaledHeight = viewFinderDimens.width
scaledWidth = Math.round(viewFinderDimens.width * bufferRatio)
} else {
scaledHeight = viewFinderDimens.height
scaledWidth = Math.round(viewFinderDimens.height * bufferRatio)
}
// Compute the relative scale value
val xScale = scaledWidth / viewFinderDimens.width.toFloat()
val yScale = scaledHeight / viewFinderDimens.height.toFloat()
// Scale input buffers to fill the view finder
matrix.preScale(xScale, yScale, centerX, centerY)
// Finally, apply transformations to our TextureView
textureView.setTransform(matrix)
}
companion object {
/** Helper function that gets the rotation of a [Display] in degrees */
fun getDisplaySurfaceRotation(display: Display?) = when(display?.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> null
}
/**
* Main entrypoint for users of this class: instantiates the adapter and returns an instance
* of [Preview] which automatically adjusts in size and rotation to compensate for
* config changes.
*/
fun build(config: PreviewConfig, viewFinder: TextureView) =
AutoFitPreviewBuilder(config, WeakReference(viewFinder)).useCase
}
}
If configuration is correct (it looks okay to me), then next idea was that maybe converting captured images objects to bitmap might be faulty. Below you can see implementation.
Capture mode uses this function:
fun imageProxyToBitmap(image: ImageProxy): Bitmap {
val buffer: ByteBuffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
Analysis mode uses this function:
fun toBitmapFromImage(image: Image?): Bitmap? {
try {
if (image == null || image.planes[0] == null || image.planes[1] == null || image.planes[2] == null) {
return null
}
val yBuffer = image.planes[0].buffer
val uBuffer = image.planes[1].buffer
val vBuffer = image.planes[2].buffer
val ySize = yBuffer.remaining()
val uSize = uBuffer.remaining()
val vSize = vBuffer.remaining()
val nv21 = ByteArray(ySize + uSize + vSize)
/* U and V are swapped */
yBuffer.get(nv21, 0, ySize)
vBuffer.get(nv21, ySize, vSize)
uBuffer.get(nv21, ySize + vSize, uSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, image.width, image.height, null)
val out = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, yuvImage.width, yuvImage.height), 50, out)
val imageBytes = out.toByteArray()
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
} catch (e: IllegalStateException) {
Log.e("IllegalStateException", "#ImageUtils.toBitmapFromImage(): Can't read the image file.")
return null
}
}
So, weirdly, on few devices toBitmapFromImage() sometimes comes up upwards, but at the same time (same device) imageProxyToBitmap() returns image in correct rotation - it has to be the image to bitmap functions fault, right?Why is this happening (because capture mode returns image normally) and how to fix this?
Inside onImageCaptureSuccess, get the rotationDegrees and rotate your bitmap by that degree to get the correct orientation.
override fun onImageCaptureSuccess(image: ImageProxy) {
val capturedImageBitmap = image.image?.toBitmap()?.rotate(image.imageInfo.rotationDegrees.toFloat())
mBinding.previewImage.setImageBitmap(capturedImageBitmap)
showPostClickViews()
mCurrentFlow = FLOW_CAMERA
}
toBitmap() and rotate() are extension functions.
fun Image.toBitmap(): Bitmap {
val buffer = planes[0].buffer
buffer.rewind()
val bytes = ByteArray(buffer.capacity())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
fun Bitmap.rotate(degrees: Float): Bitmap =
Bitmap.createBitmap(this, 0, 0, width, height, Matrix().apply { postRotate(degrees) }, true)
CameraX returns the captured image with a rotation value in the callback, which can be used to rotate the image.
https://developer.android.com/reference/androidx/camera/core/ImageCapture.OnImageCapturedListener.html#onCaptureSuccess(androidx.camera.core.ImageProxy,%20int)
For Analyzer UseCases, you have to get rotationDegree coming through analyze method of ImageAnalysis.Analyzer and work accordingly.
Hope it helps!
Related
I'm using camera2 API, and a TextureView to capture photo.
As the TextureView size changed by myself and to preventing stretching result photo, I used these functions:
private fun showPreview(camera: CameraDevice) {
cameraDevice = camera
val surfaceTexture: SurfaceTexture? = mainBinding.textureView.surfaceTexture
surfaceTexture?.setDefaultBufferSize(previewSize.width, previewSize.height)
val previewSurface = Surface(surfaceTexture)
val viewFinderSize =
Size(mainBinding.textureView.measuredWidth, mainBinding.textureView.measuredHeight)
matrix = calculateTransform(viewFinderSize, previewSize)
mainBinding.textureView.setTransform(matrix)
captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
captureRequestBuilder.addTarget(previewSurface)
cameraDevice.createCaptureSession(
listOf(previewSurface, imageReader.surface), captureStateCallback, null
)
}
private fun calculateTransform(viewFinderSize: Size, previewSize: Size): Matrix {
val matrix = Matrix()
val scaleFactors = if (viewFinderSize.height <= viewFinderSize.width) {
val previewRatio = previewSize.width / previewSize.height.toFloat()
val viewFinderRatio = viewFinderSize.width / viewFinderSize.height.toFloat()
val scaling = viewFinderRatio * previewRatio
PointF(1f, scaling)
} else {
val previewRatio = previewSize.height / previewSize.width.toFloat()
val viewFinderRatio = viewFinderSize.height / viewFinderSize.width.toFloat()
val scaling = viewFinderRatio * previewRatio
PointF(scaling, 1f)
}
matrix.preScale(scaleFactors.x, scaleFactors.y)
return matrix
}
These functions work correctly, But when I want to save result photo, bitmap data is more than just selected dimension in previewView.
I am using MediaProjectionManager for taking screenshots from the ForegroundService. I discovered that the behavior of capturing surface works differently in Android 10 and Android 11.
When I take a screenshot
fun captureBitmap(frame: CropFrames, response: (bitmap: Bitmap) -> Unit){
delayed(50) {
this.frames = frame
projection = mgr!!.getMediaProjection(resultCode, resultData!!)
val cb: MediaProjection.Callback = object : MediaProjection.Callback() {
override fun onStop() {
vdisplay!!.release() //?
response.invoke(latestBitmap!!)
}
}
vdisplay = projection?.createVirtualDisplay(
NAME,
width,
height,
App.densityDpi,
FLAGS,
imageReader.surface,
null,
null
)
projection?.registerCallback(cb, null)
}
}
onImageAvailable triggered
override fun onImageAvailable(reader: ImageReader) {
try {
val image = imageReader.acquireNextImage()
if (image != null) {
val planes = image.planes
val buffer = planes[0].buffer
val pixelStride = planes[0].pixelStride
val rowStride = planes[0].rowStride
val rowPadding = rowStride - pixelStride * width
val bitmapWidth = width + rowPadding / pixelStride
if (latestBitmap == null || latestBitmap!!.width != bitmapWidth || latestBitmap!!.height != height) {
if (latestBitmap != null) {
latestBitmap!!.recycle()
}
latestBitmap = Bitmap.createBitmap(
bitmapWidth,
height, Bitmap.Config.ARGB_8888
)
}
latestBitmap!!.copyPixelsFromBuffer(buffer)
image.close()
handler.parseFrame(frames!!, latestBitmap!!) {
stopCapture()
}
}
} catch (e: Exception) {
e.printStackTrace()
}
}
Then I release it
fun stopCapture() {
if (projection != null) {
projection!!.stop()
vdisplay!!.release()
projection = null
}
}
This flow can be triggered a lot of times per lifecycle, but each call has increased completion time of execution and it looks like short twitches of UI while taking ascreenshot (no main thread calculations). Probably I don't clear something properly? Any suggestions are appreciated. Thanks!
I'm using CameraX's Analyzer use case with the MLKit's BarcodeScanner. I would like to crop portion of the image received from the camera, before passing it to the scanner.
What I'm doing right now is I convert ImageProxy (that I recieve in the Analyzer) to a Bitmap, crop it and then pass it to the BarcodeScanner. The downside is that it's not a very fast and efficient process.
I've also noticed the warning I get in the Logcat when running this code:
ML Kit has detected that you seem to pass camera frames to the
detector as a Bitmap object. This is inefficient. Please use
YUV_420_888 format for camera2 API or NV21 format for (legacy) camera
API and directly pass down the byte array to ML Kit.
It would be nice to not to do ImageProxy conversion, but how do I crop the rectangle I want to analyze?
What I've already tried is to set a cropRect field of the Image (imageProxy.image.cropRect) class, but it doesn't seem to affect the end result.
Yes, it's true that if you use ViewPort and set viewport to yours UseCases(imageCapture or imageAnalysis as here https://developer.android.com/training/camerax/configuration) you can get only information about crop rectangle especially if you use ImageAnalysis(because if you use imageCapture, for on-disk the image is cropped before saving and it doesn't work for ImageAnalysis and if you use imageCapture without saving on disk) and here solution how I solved this problem:
First of all set view port for use cases as here: https://developer.android.com/training/camerax/configuration
Get cropped bitmap to analyze
override fun analyze(imageProxy: ImageProxy) {
val mediaImage = imageProxy.image
if (mediaImage != null && mediaImage.format == ImageFormat.YUV_420_888) {
croppedBitmap(mediaImage, imageProxy.cropRect).let { bitmap ->
requestDetectInImage(InputImage.fromBitmap(bitmap, rotation))
.addOnCompleteListener { imageProxy.close() }
}
} else {
imageProxy.close()
}
}
private fun croppedBitmap(mediaImage: Image, cropRect: Rect): Bitmap {
val yBuffer = mediaImage.planes[0].buffer // Y
val vuBuffer = mediaImage.planes[2].buffer // VU
val ySize = yBuffer.remaining()
val vuSize = vuBuffer.remaining()
val nv21 = ByteArray(ySize + vuSize)
yBuffer.get(nv21, 0, ySize)
vuBuffer.get(nv21, ySize, vuSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, mediaImage.width, mediaImage.height, null)
val outputStream = ByteArrayOutputStream()
yuvImage.compressToJpeg(cropRect, 100, outputStream)
val imageBytes = outputStream.toByteArray()
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
}
Possibly there is a loss in conversion speed, but on my devices I did not notice the difference. I set 100 quality in method compressToJpeg, but mb if set less quality it can improve speed, it need test.
upd: May 02 '21 :
I found another way without convert to jpeg and then to bitmap. This should be a faster way.
Set viewport as previous.
Convert YUV_420_888 to NV21, then crop and analyze.
override fun analyze(imageProxy: ImageProxy) {
val mediaImage = imageProxy.image
if (mediaImage != null && mediaImage.format == ImageFormat.YUV_420_888) {
croppedNV21(mediaImage, imageProxy.cropRect).let { byteArray ->
requestDetectInImage(
InputImage.fromByteArray(
byteArray,
imageProxy.cropRect.width(),
imageProxy.cropRect.height(),
rotation,
IMAGE_FORMAT_NV21,
)
)
.addOnCompleteListener { imageProxy.close() }
}
} else {
imageProxy.close()
}
}
private fun croppedNV21(mediaImage: Image, cropRect: Rect): ByteArray {
val yBuffer = mediaImage.planes[0].buffer // Y
val vuBuffer = mediaImage.planes[2].buffer // VU
val ySize = yBuffer.remaining()
val vuSize = vuBuffer.remaining()
val nv21 = ByteArray(ySize + vuSize)
yBuffer.get(nv21, 0, ySize)
vuBuffer.get(nv21, ySize, vuSize)
return cropByteArray(nv21, mediaImage.width, cropRect)
}
private fun cropByteArray(array: ByteArray, imageWidth: Int, cropRect: Rect): ByteArray {
val croppedArray = ByteArray(cropRect.width() * cropRect.height())
var i = 0
array.forEachIndexed { index, byte ->
val x = index % imageWidth
val y = index / imageWidth
if (cropRect.left <= x && x < cropRect.right && cropRect.top <= y && y < cropRect.bottom) {
croppedArray[i] = byte
i++
}
}
return croppedArray
}
First crop fun I took from here: Android: How to crop images using CameraX?
And I found also another crop fun, it seems that it is more complicated:
private fun cropByteArray(src: ByteArray, width: Int, height: Int, cropRect: Rect, ): ByteArray {
val x = cropRect.left * 2 / 2
val y = cropRect.top * 2 / 2
val w = cropRect.width() * 2 / 2
val h = cropRect.height() * 2 / 2
val yUnit = w * h
val uv = yUnit / 2
val nData = ByteArray(yUnit + uv)
val uvIndexDst = w * h - y / 2 * w
val uvIndexSrc = width * height + x
var srcPos0 = y * width
var destPos0 = 0
var uvSrcPos0 = uvIndexSrc
var uvDestPos0 = uvIndexDst
for (i in y until y + h) {
System.arraycopy(src, srcPos0 + x, nData, destPos0, w) //y memory block copy
srcPos0 += width
destPos0 += w
if (i and 1 == 0) {
System.arraycopy(src, uvSrcPos0, nData, uvDestPos0, w) //uv memory block copy
uvSrcPos0 += width
uvDestPos0 += w
}
}
return nData
}
Second crop fun I took from here:
https://www.programmersought.com/article/75461140907/
I would be glad if someone can help improve the code.
I'm still improving the way to do it. But this will work for me now
CameraX crop image before sending to analyze
<androidx.constraintlayout.widget.ConstraintLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:paddingBottom="#dimen/_40sdp">
<androidx.camera.view.PreviewView
android:id="#+id/previewView"
android:layout_width="match_parent"
android:layout_height="0dp"
app:layout_constraintDimensionRatio="1:1"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="parent" /></androidx.constraintlayout.widget.ConstraintLayout>
Cropping an image into 1:1 before passing it to analyze
override fun onCaptureSuccess(image: ImageProxy) {
super.onCaptureSuccess(image)
var bitmap: Bitmap = imageProxyToBitmap(image)
val dimension: Int = min(bitmap.width, bitmap.height)
bitmap = ThumbnailUtils.extractThumbnail(bitmap, dimension, dimension)
imageView.setImageBitmap(bitmap) //Here you can pass the crop[from the center] image to analyze
image.close()
}
**Function for converting into bitmap **
private fun imageProxyToBitmap(image: ImageProxy): Bitmap {
val buffer: ByteBuffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
You would use ImageProxy.SetCroprect to get the rect and then use CropRect to set it.
For example if you had imageProxy, you would do : ImageProxy.setCropRect(Rect) and then you would do ImageProxy.CropRect.
I have added CameraX to my ongoing development app a while ago. I know it was in alpha but I was ready to make the change when beta or final release will be available.
So I started working on it today. I have updated from
implementation 'androidx.camera:camera-core:1.0.0-alpha04'
implementation 'androidx.camera:camera-camera2:1.0.0-alpha04'
to this:
implementation 'androidx.camera:camera-core:1.0.0-beta01'
implementation 'androidx.camera:camera-camera2:1.0.0-beta01'
implementation 'androidx.camera:camera-lifecycle:1.0.0-beta01'
My Previous Working Code (alpha-04):
class ScannerX : AppCompatActivity() {
private lateinit var context: Context
var isOtpAuthCode = true
private val immersiveFlagTimeout = 500L
private val flagsFullscreen = View.SYSTEM_UI_FLAG_LOW_PROFILE or View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_LAYOUT_STABLE or
View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
private var preview: Preview? = null
private var lensFacing = CameraX.LensFacing.BACK
private var imageAnalyzer: ImageAnalysis? = null
private lateinit var analyzerThread: HandlerThread
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_scanner_x)
context = this
btnCancel.setOnClickListener {
finish()
}
analyzerThread = if (GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context) == ConnectionResult.SUCCESS) {
HandlerThread("BarcodeFirebaseAnalyzer").apply { start() }
} else {
HandlerThread("BarcodeZxingAnalyzer").apply { start() }
}
Dexter.withActivity(this)
.withPermissions(Manifest.permission.CAMERA)
.withListener(object : MultiplePermissionsListener {
override fun onPermissionsChecked(report: MultiplePermissionsReport?) {
textureView.post {
val metrics = DisplayMetrics().also { textureView.display.getRealMetrics(it) }
val screenAspectRatio = Rational(metrics.widthPixels, metrics.heightPixels)
val previewConfig = PreviewConfig.Builder().apply {
setLensFacing(lensFacing)
// We request aspect ratio but no resolution to let CameraX optimize our use cases
setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
setTargetRotation(textureView.display.rotation)
}.build()
val analyzerConfig = ImageAnalysisConfig.Builder().apply {
setLensFacing(lensFacing)
// Use a worker thread for image analysis to prevent preview glitches
setCallbackHandler(Handler(analyzerThread.looper))
// In our analysis, we care more about the latest image than analyzing *every* image
setImageReaderMode(ImageAnalysis.ImageReaderMode.ACQUIRE_LATEST_IMAGE)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
setTargetRotation(textureView.display.rotation)
}.build()
preview = AutoFitPreviewBuilder.build(previewConfig, textureView)
imageAnalyzer = ImageAnalysis(analyzerConfig).apply {
analyzer = if (GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context) == ConnectionResult.SUCCESS) {
BarcodeFirebaseAnalyzer { qrCode ->
if (isOtpAuthCode) {
if (qrCode.startsWith("otpauth")) {
toAddAuth(qrCode)
}
} else {
toAddAuth(qrCode)
}
}
} else {
BarcodeZxingAnalyzer { qrCode ->
if (isOtpAuthCode) {
if (qrCode.startsWith("otpauth")) {
toAddAuth(qrCode)
}
} else {
toAddAuth(qrCode)
}
}
}
}
// Apply declared configs to CameraX using the same lifecycle owner
CameraX.bindToLifecycle(this#ScannerX, preview, imageAnalyzer)
}
}
override fun onPermissionRationaleShouldBeShown(permissions: MutableList<PermissionRequest>?, token: PermissionToken?) {
//
}
}).check()
}
override fun onStart() {
super.onStart()
// Before setting full screen flags, we must wait a bit to let UI settle; otherwise, we may
// be trying to set app to immersive mode before it's ready and the flags do not stick
textureView.postDelayed({
textureView.systemUiVisibility = flagsFullscreen
}, immersiveFlagTimeout)
}
override fun onDestroy() {
analyzerThread.quit()
super.onDestroy()
}
private fun toAddAuth(scannedCode: String) {
if (CameraX.isBound(imageAnalyzer)) {
CameraX.unbind(imageAnalyzer)
}
val intent = Intent()
intent.putExtra("scanResult", scannedCode)
setResult(RESULT_OK, intent)
finish()
}
companion object {
private const val RESULT_OK = 666
}
}
And the code I have changed is as follows (beta-01):
class ScannerX : AppCompatActivity() {
private lateinit var context: Context
var isOtpAuthCode = true
private val immersiveFlagTimeout = 500L
private val flagsFullscreen = View.SYSTEM_UI_FLAG_LOW_PROFILE or View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_LAYOUT_STABLE or
View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
private var preview: Preview? = null
private var lensFacing = CameraSelector.DEFAULT_BACK_CAMERA
private var imageAnalyzer: ImageAnalysis? = null
private lateinit var analysisExecutor: ExecutorService
private lateinit var processCameraProvider: ListenableFuture<ProcessCameraProvider>
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_scanner_x)
context = this
btnCancel.setOnClickListener {
finish()
}
Dexter.withActivity(this)
.withPermissions(Manifest.permission.CAMERA)
.withListener(object : MultiplePermissionsListener {
override fun onPermissionsChecked(report: MultiplePermissionsReport?) {
textureView.post {
analysisExecutor = Executors.newSingleThreadExecutor()
processCameraProvider = ProcessCameraProvider.getInstance(context)
preview = Preview.Builder()
.setTargetAspectRatio(AspectRatio.RATIO_16_9)
.setTargetRotation(textureView.display.rotation)
.build()
imageAnalyzer = ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setTargetRotation(textureView.display.rotation)
.build()
if (GoogleApiAvailability.getInstance().isGooglePlayServicesAvailable(context) == ConnectionResult.SUCCESS) {
imageAnalyzer?.apply {
setAnalyzer(analysisExecutor, BarcodeFirebaseAnalyzer { qrCode ->
if (isOtpAuthCode) {
if (qrCode.startsWith("otpauth")) {
toAddAuth(qrCode)
}
} else {
toAddAuth(qrCode)
}
})
}
} else {
imageAnalyzer?.apply {
setAnalyzer(analysisExecutor, BarcodeZxingAnalyzer { qrCode ->
if (isOtpAuthCode) {
if (qrCode.startsWith("otpauth")) {
toAddAuth(qrCode)
}
} else {
toAddAuth(qrCode)
}
})
}
}
processCameraProvider.get().bindToLifecycle(this#ScannerX, lensFacing, imageAnalyzer)
}
}
override fun onPermissionRationaleShouldBeShown(permissions: MutableList<PermissionRequest>?, token: PermissionToken?) {
//
}
}).check()
}
override fun onStart() {
super.onStart()
// Before setting full screen flags, we must wait a bit to let UI settle; otherwise, we may
// be trying to set app to immersive mode before it's ready and the flags do not stick
textureView.postDelayed({
textureView.systemUiVisibility = flagsFullscreen
}, immersiveFlagTimeout)
}
override fun onDestroy() {
if (!analysisExecutor.isShutdown) {
analysisExecutor.shutdown()
}
super.onDestroy()
}
private fun toAddAuth(scannedCode: String) {
/*if (CameraX.isBound(imageAnalyzer)) {
CameraX.unbind(imageAnalyzer)
}*/
val intent = Intent()
intent.putExtra("scanResult", scannedCode)
setResult(RESULT_OK, intent)
finish()
}
companion object {
private const val RESULT_OK = 666
}
}
After I upgraded there were so many changes in library and now I cant make it work.
I also cant use Google Provided AutoFitPreview Class along with initial alpha release of this library. This was not necessary even with alpha04 since the only problem without this class was camera view little bit stretched out but scanning and analyzing worked properly.
/**
* Builder for [Preview] that takes in a [WeakReference] of the view finder and [PreviewConfig],
* then instantiates a [Preview] which automatically resizes and rotates reacting to config changes.
*/
class AutoFitPreviewBuilder private constructor(config: PreviewConfig, viewFinderRef: WeakReference<TextureView>) {
/** Public instance of preview use-case which can be used by consumers of this adapter */
val useCase: Preview
/** Internal variable used to keep track of the use case's output rotation */
private var bufferRotation: Int = 0
/** Internal variable used to keep track of the view's rotation */
private var viewFinderRotation: Int? = null
/** Internal variable used to keep track of the use-case's output dimension */
private var bufferDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's dimension */
private var viewFinderDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's display */
private var viewFinderDisplay: Int = -1
/** Internal reference of the [DisplayManager] */
private lateinit var displayManager: DisplayManager
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) {
val viewFinder = viewFinderRef.get() ?: return
if (displayId == viewFinderDisplay) {
val display = displayManager.getDisplay(displayId)
val rotation = getDisplaySurfaceRotation(display)
updateTransform(viewFinder, rotation, bufferDimens, viewFinderDimens)
}
}
}
init {
// Make sure that the view finder reference is valid
val viewFinder = viewFinderRef.get() ?:
throw IllegalArgumentException("Invalid reference to view finder used")
// Initialize the display and rotation from texture view information
viewFinderDisplay = viewFinder.display.displayId
viewFinderRotation = getDisplaySurfaceRotation(viewFinder.display) ?: 0
// Initialize public use-case with the given config
useCase = Preview(config)
// Every time the view finder is updated, recompute layout
useCase.onPreviewOutputUpdateListener = Preview.OnPreviewOutputUpdateListener {
val viewFinderI = viewFinderRef.get() ?: return#OnPreviewOutputUpdateListener
Log.d(TAG, "Preview output changed. " +
"Size: ${it.textureSize}. Rotation: ${it.rotationDegrees}")
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = viewFinderI.parent as ViewGroup
parent.removeView(viewFinderI)
parent.addView(viewFinderI, 0)
// Update internal texture
viewFinderI.surfaceTexture = it.surfaceTexture
// Apply relevant transformations
bufferRotation = it.rotationDegrees
val rotation = getDisplaySurfaceRotation(viewFinderI.display)
updateTransform(viewFinderI, rotation, it.textureSize, viewFinderDimens)
}
// Every time the provided texture view changes, recompute layout
viewFinder.addOnLayoutChangeListener { view, left, top, right, bottom, _, _, _, _ ->
val viewFinderII = view as TextureView
val newViewFinderDimens = Size(right - left, bottom - top)
Log.d(TAG, "View finder layout changed. Size: $newViewFinderDimens")
val rotation = getDisplaySurfaceRotation(viewFinderII.display)
updateTransform(viewFinderII, rotation, bufferDimens, newViewFinderDimens)
}
// Every time the orientation of device changes, recompute layout
// NOTE: This is unnecessary if we listen to display orientation changes in the camera
// fragment and call [Preview.setTargetRotation()] (like we do in this sample), which will
// trigger [Preview.OnPreviewOutputUpdateListener] with a new
// [PreviewOutput.rotationDegrees]. CameraX Preview use case will not rotate the frames for
// us, it will just tell us about the buffer rotation with respect to sensor orientation.
// In this sample, we ignore the buffer rotation and instead look at the view finder's
// rotation every time [updateTransform] is called, which gets triggered by
// [CameraFragment] display listener -- but the approach taken in this sample is not the
// only valid one.
displayManager = viewFinder.context
.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)
// Remove the display listeners when the view is detached to avoid holding a reference to
// it outside of the Fragment that owns the view.
// NOTE: Even though using a weak reference should take care of this, we still try to avoid
// unnecessary calls to the listener this way.
viewFinder.addOnAttachStateChangeListener(object : View.OnAttachStateChangeListener {
override fun onViewAttachedToWindow(view: View?) =
displayManager.registerDisplayListener(displayListener, null)
override fun onViewDetachedFromWindow(view: View?) =
displayManager.unregisterDisplayListener(displayListener)
})
}
/** Helper function that fits a camera preview into the given [TextureView] */
private fun updateTransform(textureView: TextureView?, rotation: Int?, newBufferDimens: Size, newViewFinderDimens: Size) {
// This should not happen anyway, but now the linter knows
val textureViewI = textureView ?: return
if (rotation == viewFinderRotation &&
Objects.equals(newBufferDimens, bufferDimens) &&
Objects.equals(newViewFinderDimens, viewFinderDimens)) {
// Nothing has changed, no need to transform output again
return
}
if (rotation == null) {
// Invalid rotation - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderRotation = rotation
}
if (newBufferDimens.width == 0 || newBufferDimens.height == 0) {
// Invalid buffer dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
bufferDimens = newBufferDimens
}
if (newViewFinderDimens.width == 0 || newViewFinderDimens.height == 0) {
// Invalid view finder dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderDimens = newViewFinderDimens
}
val matrix = Matrix()
Log.d(TAG, "Applying output transformation.\n" +
"View finder size: $viewFinderDimens.\n" +
"Preview output size: $bufferDimens\n" +
"View finder rotation: $viewFinderRotation\n" +
"Preview output rotation: $bufferRotation")
// Compute the center of the view finder
val centerX = viewFinderDimens.width / 2f
val centerY = viewFinderDimens.height / 2f
// Correct preview output to account for display rotation
matrix.postRotate(-viewFinderRotation!!.toFloat(), centerX, centerY)
// Buffers are rotated relative to the device's 'natural' orientation: swap width and height
val bufferRatio = bufferDimens.height / bufferDimens.width.toFloat()
val scaledWidth: Int
val scaledHeight: Int
// Match longest sides together -- i.e. apply center-crop transformation
if (viewFinderDimens.width > viewFinderDimens.height) {
scaledHeight = viewFinderDimens.width
scaledWidth = (viewFinderDimens.width * bufferRatio).roundToInt()
} else {
scaledHeight = viewFinderDimens.height
scaledWidth = (viewFinderDimens.height * bufferRatio).roundToInt()
}
// Compute the relative scale value
val xScale = scaledWidth / viewFinderDimens.width.toFloat()
val yScale = scaledHeight / viewFinderDimens.height.toFloat()
// Scale input buffers to fill the view finder
matrix.preScale(xScale, yScale, centerX, centerY)
// Finally, apply transformations to our TextureView
textureViewI.setTransform(matrix)
}
companion object {
private val TAG = AutoFitPreviewBuilder::class.java.simpleName
/** Helper function that gets the rotation of a [Display] in degrees */
fun getDisplaySurfaceRotation(display: Display?) = when(display?.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> null
}
/**
* Main entry point for users of this class: instantiates the adapter and returns an instance
* of [Preview] which automatically adjusts in size and rotation to compensate for
* config changes.
*/
fun build(config: PreviewConfig, viewFinder: TextureView) =
AutoFitPreviewBuilder(config, WeakReference(viewFinder)).useCase
}
}
Please Help
I would like to implement ARCore with Twilio's video call. The documentation says this is possible but I could not figure out how to do it. Can Anyone tell me what I'm doing wrong?
This is my activity:
class MixActivity : AppCompatActivity() {
private lateinit var mArFragment: ArFragment
private lateinit var mVideoView: ArSceneView
private var mScreenVideoTrack: LocalVideoTrack? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_mix)
mArFragment = ar_fragment as ArFragment
mVideoView = mArFragment.arSceneView
mScreenVideoTrack = LocalVideoTrack.create(this, true,
ViewCapturer(mVideoView)
)
} }
This is view:
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="#+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent">
<fragment
android:id="#+id/ar_fragment"
android:name="com.google.ar.sceneform.ux.ArFragment"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
</RelativeLayout>
And VideoCapture:
internal class ViewCapturer(private val view: View) : VideoCapturer, PixelCopy.OnPixelCopyFinishedListener {
private val handler = Handler(Looper.getMainLooper())
private var videoCapturerListener: VideoCapturer.Listener? = null
private val started = AtomicBoolean(false)
private lateinit var mViewBitmap: Bitmap
private val viewCapturer = object : Runnable {
override fun run() {
val dropFrame = view.width == 0 || view.height == 0
// Only capture the view if the dimensions have been established
if (!dropFrame) {
// Draw view into bitmap backed canvas
val measuredWidth = View.MeasureSpec.makeMeasureSpec(
view.width,
View.MeasureSpec.EXACTLY
)
val measuredHeight = View.MeasureSpec.makeMeasureSpec(
view.height,
View.MeasureSpec.EXACTLY
)
view.measure(measuredWidth, measuredHeight)
view.layout(0, 0, view.measuredWidth, view.measuredHeight)
mViewBitmap = Bitmap.createBitmap(
view.width, view.height,
Bitmap.Config.ARGB_8888
)
val viewCanvas = Canvas(mViewBitmap)
view.draw(viewCanvas)
// Extract the frame from the bitmap
val bytes = mViewBitmap.byteCount
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener!!.onFrameCaptured(videoFrame)
}
}
// Schedule the next capture
if (started.get()) {
handler.postDelayed(this, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
}
/**
* Returns the list of supported formats for this view capturer. Currently, only supports
* capturing to RGBA_8888 bitmaps.
*
* #return list of supported formats.
*/
override fun getSupportedFormats(): List<VideoFormat> {
val videoFormats = ArrayList<VideoFormat>()
val videoDimensions = VideoDimensions(view.width, view.height)
val videoFormat = VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888)
videoFormats.add(videoFormat)
return videoFormats
}
/**
* Returns true because we are capturing screen content.
*/
override fun isScreencast(): Boolean {
return true
}
/**
* This will be invoked when it is time to start capturing frames.
*
* #param videoFormat the video format of the frames to be captured.
* #param listener capturer listener.
*/
override fun startCapture(videoFormat: VideoFormat, listener: VideoCapturer.Listener) {
// Store the capturer listener
this.videoCapturerListener = listener
this.started.set(true)
// Notify capturer API that the capturer has started
val capturerStarted = handler.postDelayed(
viewCapturer,
VIEW_CAPTURER_FRAMERATE_MS.toLong()
)
this.videoCapturerListener!!.onCapturerStarted(capturerStarted)
}
/**
* Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
*/
override fun stopCapture() {
this.started.set(false)
handler.removeCallbacks(viewCapturer)
}
override fun onPixelCopyFinished(i: Int) {
// Extract the frame from the bitmap
val bytes = mViewBitmap.getByteCount()
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener?.onFrameCaptured(videoFrame)
}
if (started.get()) {
handler.postDelayed(viewCapturer, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
companion object {
private val VIEW_CAPTURER_FRAMERATE_MS = 100
}
}
The ARCore part works but the Twilio part does not work.
I referred to another post that talked about it but it was incomplete:
Streaming CustomView ARcore with Twilio video