Develop Reference

Android ML Kit library for QR code scanning: How to increase detection performance by reducing image resolution

This is my stripped down sourcecode for barcode scanning
build.gradle
dependencies {
.....
// MLKit Dependencies
implementation 'com.google.android.gms:play-services-vision:20.1.3'
implementation 'com.google.mlkit:barcode-scanning:17.0.2'
def camerax_version = "1.1.0-beta01"
implementation "androidx.camera:camera-core:${camerax_version}"
implementation "androidx.camera:camera-camera2:${camerax_version}"
implementation "androidx.camera:camera-lifecycle:${camerax_version}"
implementation "androidx.camera:camera-video:${camerax_version}"
......
}
ScanCameraFragment.kt
class ScanCameraFragment : BaseFragment() {
private lateinit var binding: FragmentScanCameraBinding
private lateinit var cameraExecutor: ExecutorService
//region Lifecycle Methods
override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?,
savedInstanceState: Bundle?): View? {
binding = FragmentScanCameraBinding.inflate(inflater, container, false)
cameraExecutor = Executors.newSingleThreadExecutor()
startCamera()
return binding.root
}
override fun onDestroyView() {
super.onDestroyView()
cameraExecutor.shutdown()
}
companion object {
fun newInstance() = ScanCameraFragment().apply {}
}
private fun startCamera() {
context?.let { context ->
val cameraProviderFuture = ProcessCameraProvider.getInstance(context)
cameraProviderFuture.addListener({
val cameraProvider = cameraProviderFuture.get()
// Preview
val preview = Preview.Builder()
.build()
.also {
it.setSurfaceProvider(binding.previewView.surfaceProvider)
}
// Image analyzer
val imageAnalyzer = ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.build()
.also {
it.setAnalyzer(cameraExecutor,
QrCodeAnalyzer(context, binding.barcodeBoxView,
binding.previewView.width.toFloat(),
binding.previewView.height.toFloat()
)
)
}
// Select back camera as a default
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
try {
// Unbind use cases before rebinding
cameraProvider.unbindAll()
// Bind use cases to camera
var camera = cameraProvider.bindToLifecycle(this, cameraSelector,
preview, imageAnalyzer)
} catch (exc: Exception) {
exc.printStackTrace()
}
}, ContextCompat.getMainExecutor(context))
}
}
}
QRCodeAnalyzer.kt
class QrCodeAnalyzer(private val context: Context,
private val barcodeBoxView: BarcodeBoxView, private val previewViewWidth: Float,
private val previewViewHeight: Float) : ImageAnalysis.Analyzer {
private var scaleX = 1f
private var scaleY = 1f
private fun translateX(x: Float) = x * scaleX
private fun translateY(y: Float) = y * scaleY
private fun adjustBoundingRect(rect: Rect) = RectF(
translateX(rect.left.toFloat()),
translateY(rect.top.toFloat()),
translateX(rect.right.toFloat()),
translateY(rect.bottom.toFloat())
)
#SuppressLint("UnsafeOptInUsageError")
override fun analyze(image: ImageProxy) {
val img = image.image
if (img != null) {
// Update scale factors
scaleX = previewViewWidth / img.height.toFloat()
scaleY = previewViewHeight / img.width.toFloat()
val inputImage = InputImage.fromMediaImage(img,
image.imageInfo.rotationDegrees)
// Process image searching for barcodes
val options = BarcodeScannerOptions.Builder()
.build()
val scanner = BarcodeScanning.getClient(options)
scanner.process(inputImage)
.addOnSuccessListener { barcodes ->
for (barcode in barcodes) {
barcode?.rawValue?.let {
if (it.trim().isNotBlank()) {
Scanner.updateBarcode(it)
barcode.boundingBox?.let { rect ->
barcodeBoxView.setRect(adjustBoundingRect(rect))
}
}
return#addOnSuccessListener
}
}
// coming here means no satisfiable barcode was found
barcodeBoxView.setRect(RectF())
}
.addOnFailureListener {
image.close()
}
.addOnFailureListener { }
}
image.close()
}
}
This code works and I am able to scan barcodes. But sometimes, the barcode detection is slow. The documentation says one way to increase performance is to limit the image resolution.
Don't capture input at the camera’s native resolution. On some
devices, capturing input at the native resolution produces extremely
large (10+ megapixels) images, which results in very poor latency with
no benefit to accuracy. Instead, only request the size from the camera
that's required for barcode detection, which is usually no more than 2
megapixels.
If scanning speed is important, you can further lower the image
capture resolution. However, bear in mind the minimum barcode size
requirements outlined above.
Unfortunately, the documentation doesn't specify how to reduce the image resolution. And some of my end users are using high end devices with powerful camera, so we assume the poor performance is because of the image size.
How can I reduce the resolution of the image to a fixed value (something like 1024 x 768) rather than the default camera resolution?

You can set it on the imageAnalyzer builder bij using
.setTargetResolution(Size)
val imageAnalysisUseCaseBuilder = ImageAnalysis.Builder()
imageAnalysisUseCaseBuilder.setTargetResolution(Size(1024, 768))
imageAnalysisUseCase = imageAnalysisUseCaseBuilder.build()
or in you case
val imageAnalyzer = ImageAnalysis.Builder()
.setTargetResolution(Size(1024, 768))
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.build()
.also {
it.setAnalyzer(cameraExecutor,
QrCodeAnalyzer(context, binding.barcodeBoxView,
binding.previewView.width.toFloat(),
binding.previewView.height.toFloat()
)
)
}

User HarmenH's answer correctly tells how to set the image resolution, so I am not repeating it here.
As it turns out, the performance issue on my end was not because of image resolution. It seems I was closing the imageProxy prematurely.
override fun analyze(image: ImageProxy) {
val img = image.image
if (img != null) {
// Update scale factors
scaleX = previewViewWidth / img.height.toFloat()
scaleY = previewViewHeight / img.width.toFloat()
val inputImage = InputImage.fromMediaImage(img,
image.imageInfo.rotationDegrees)
// Process image searching for barcodes
val options = BarcodeScannerOptions.Builder()
.build()
val scanner = BarcodeScanning.getClient(options)
scanner.process(inputImage)
.addOnSuccessListener { barcodes - >
for (barcode in barcodes) {
barcode?.rawValue?.let {
if (it.trim().isNotBlank()) {
Scanner.updateBarcode(it)
barcode.boundingBox?.let { rect - >
barcodeBoxView.setRect(adjustBoundingRect(rect))
}
}
return #addOnSuccessListener
}
}
// coming here means no satisfiable barcode was found
barcodeBoxView.setRect(RectF())
}
.addOnFailureListener {
image.close()
}
.addOnFailureListener {
//added this here.
image.close()
}
}
//Removed this because we don't close the
//imageProxy before analysis completes
//image.close()
}

How to increase image quality with android cameraX library?

I am creating an application which must implement its own camera.
I use the cameraX library provided by google.
I noticed that there is a difference between the quality of the image captured by my own application, and the image captured by the camera application installed on my phone.
although the 2 photos are captured with the same conditions (light, position...)
especially when I zoom the photo, the details of the image become more blurry to the image captured by my application
(in my own case, my phone is Google Pixel 5)
Please see these 2 photos to see the difference
Image by phone camera
Image by my app
And this is my code
/**
* Initialize CameraX, and prepare to bind the camera use cases
*/
private fun setupCamera()
{
val cameraProviderFuture : ListenableFuture<ProcessCameraProvider> = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener({
cameraProvider = cameraProviderFuture.get()
lensFacing = when
{
hasBackCamera() -> CameraSelector.LENS_FACING_BACK
hasFrontCamera() -> CameraSelector.LENS_FACING_FRONT
else -> throw IllegalStateException("Back and front camera are unavailable")
}
bindCameraUseCases()
setupCameraGestures()
}, ContextCompat.getMainExecutor(this))
}
/**
* Declare and bind preview, capture and analysis use cases.
*/
private fun bindCameraUseCases()
{
lifecycleScope.launch {
val cameraProvider : ProcessCameraProvider = cameraProvider ?: throw IllegalStateException("Camera initialization failed.")
// Try to apply extensions like HDR, NIGHT ##########################################
val extensionsManager : ExtensionsManager = ExtensionsManager.getInstanceAsync(this#ImageCaptureActivity, cameraProvider).await()
val defaultCameraSelector : CameraSelector = CameraSelector.Builder()
.requireLensFacing(lensFacing)
.build()
val finalCameraSelector : CameraSelector = if (extensionsManager.isExtensionAvailable(defaultCameraSelector, ExtensionMode.AUTO))
{
extensionsManager.getExtensionEnabledCameraSelector(defaultCameraSelector, ExtensionMode.AUTO)
}
else
{
defaultCameraSelector
}
// Get screen metrics used to setup camera for full screen resolution
val metrics : DisplayMetrics = resources.displayMetrics
val screenAspectRatio : Int = aspectRatio(metrics.widthPixels, metrics.heightPixels)
val rotation : Int = binding.cameraPreview.display.rotation
preview = Preview.Builder()
// We request aspect ratio but no resolution
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation
.setTargetRotation(rotation)
.build()
imageCapture = ImageCapture.Builder()
// We request aspect ratio but no resolution to match preview config, but letting
// CameraX optimize for whatever specific resolution best fits our use cases
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
.setTargetRotation(rotation)
.setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY)
.setJpegQuality(100)
.build()
imageAnalyzer = ImageAnalysis.Builder()
// We request aspect ratio but no resolution
.setTargetAspectRatio(screenAspectRatio)
.build()
imageAnalyzer?.setAnalyzer(cameraExecutor, LuminosityAnalyzer {})
// Must unbind the use-cases before rebinding them
cameraProvider.unbindAll()
try
{
// A variable number of use-cases can be passed here -
// camera provides access to CameraControl & CameraInfo
camera = cameraProvider.bindToLifecycle(this#ImageCaptureActivity, finalCameraSelector, preview, imageCapture, imageAnalyzer)
// Attach the viewfinder's surface provider to preview use case
preview?.setSurfaceProvider(binding.cameraPreview.surfaceProvider)
}
catch (exception : Exception)
{
exception.printStackTrace()
}
}
}
/**
* [androidx.camera.core.ImageAnalysisConfig] requires enum value of [androidx.camera.core.AspectRatio].
* Currently it has values of 4:3 & 16:9.
*
* Detecting the most suitable ratio for dimensions provided in #params by counting absolute
* of preview ratio to one of the provided values.
*
* #param width - preview width
* #param height - preview height
* #return suitable aspect ratio
*/
private fun aspectRatio(width : Int, height : Int) : Int
{
val previewRatio : Double = max(width, height).toDouble() / min(width, height)
return if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE))
{
AspectRatio.RATIO_4_3
}
else
{
AspectRatio.RATIO_16_9
}
}
fun captureImage()
{
if (!permissionsOk()) return
// Get a stable reference of the modifiable image capture use case
imageCapture?.let { imageCapture ->
// Create output file to hold the image
val photoFile : File = storageUtils.createFile(
baseFolder = getOutputPath(),
fileName = System.currentTimeMillis().toString(),
fileExtension = StorageUtils.PHOTO_EXTENSION)
// Setup image capture metadata
val metadata : Metadata = Metadata().also {
// Mirror image when using the front camera
it.isReversedHorizontal = lensFacing == CameraSelector.LENS_FACING_FRONT
it.location = locationManager.lastKnownLocation
}
// Create output options object which contains file + metadata
val outputOptions : ImageCapture.OutputFileOptions = ImageCapture.OutputFileOptions.Builder(photoFile)
.setMetadata(metadata)
.build()
imagesAdapter.addImage(photoFile)
// Setup image capture listener which is triggered after photo has been taken
imageCapture.takePicture(outputOptions, cameraExecutor, object : ImageCapture.OnImageSavedCallback
{
override fun onImageSaved(output : ImageCapture.OutputFileResults)
{
val savedUri : Uri = output.savedUri ?: return
StorageUtils.showInGallery(savedUri.path)
binding.list.post {
imagesAdapter.addImage(savedUri.toFile())
binding.list.smoothScrollToPosition(imagesAdapter.itemCount)
}
}
override fun onError(exception : ImageCaptureException)
{
exception.printStackTrace()
}
})
binding.cameraPreview.postDelayed({
binding.backgroundEffect.isVisible = true
binding.cameraPreview.postDelayed({
binding.backgroundEffect.isVisible = false
}, AppUtils.VERY_FAST_ANIMATION_MILLIS)
}, AppUtils.FAST_ANIMATION_MILLIS)
}
}
How can I improve the quality of my images? Is there any thing I should do? is there a special filter or algorithm?
i need your help please

if you took photo on Pixel probably using default cam app (GCam) - this app is fulfilled with quaility improvements backed up by some AI. tough task to comptetite with the biggest in quality... try to take a photo with some 3rd party like OpenCamera and compare this picture with one got by your app

You can use CameraX Extension feature to enable HDR & Low light.
this improves the image quality significantly.

Preview is stretched on CameraX PreviewView

i am exciting to use CameraX on my app.,
my reference from google officialy CameraX on github,
but something wrong., please help me.,
this my code to set aspect ratio
private fun aspectRatio(width: Int, height: Int): Int {
val previewRatio = max(width, height).toDouble() / min(width, height)
if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE)) {
return AspectRatio.RATIO_4_3
}
return AspectRatio.RATIO_16_9
}
And this my preview code
#SuppressLint("UnsafeExperimentalUsageError")private fun startCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
viewFinder=binding.surfaceView
cameraProviderFuture.addListener({
// Used to bind the lifecycle of cameras to the lifecycle owner
val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
val metrics = DisplayMetrics().also { viewFinder.display.getRealMetrics(it) }
Log.d(TAG, "Screen metrics: ${metrics.widthPixels} x ${metrics.heightPixels}")
val screenAspectRatio = aspectRatio(metrics.widthPixels, metrics.heightPixels)
Log.d(TAG, "Preview aspect ratio: $screenAspectRatio")
Size(metrics.widthPixels, metrics.heightPixels)
val rotation = viewFinder.display.rotation
// Preview
val preview = Preview.Builder()
.setTargetRotation(rotation)
.setTargetAspectRatio(screenAspectRatio)
.build()
imageCapture = if (flashStatus) {
ImageCapture.Builder()
.setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY)
.setTargetRotation(rotationimage)
.setTargetAspectRatio(screenAspectRatio)
.build()
} else {
ImageCapture.Builder()
.setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY)
.setTargetRotation(rotationimage)
.build()
}
// Select back camera as a default
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
// Unbind use cases before rebinding
cameraProvider.unbindAll()
viewFinder.scaleType=PreviewView.ScaleType.FILL_CENTER
try {
// Bind use cases to camera
val cam = cameraProvider.bindToLifecycle(
this, cameraSelector, preview, imageCapture)
if (!flashStatus) {
if (cam.cameraInfo.hasFlashUnit()) {
cam.cameraControl.enableTorch(true)
}
} else {
cam.cameraControl.enableTorch(false)
}
// Attach the viewfinder's surface provider to preview use case
preview.setSurfaceProvider(viewFinder.surfaceProvider)
} catch (exc: Exception) {
Log.e(TAG, "Use case binding failed", exc)
}
}, ContextCompat.getMainExecutor(this))
}
and this output preview
output preview is stretched
what is wrong on my code?
thank you for your help me folks.,
UPDATE
when my device on dark mode, the preview is normal.
preview normal on dark mode
how to fixed in normal mode??

capture overlay using PreviewView of cameraX

I'm trying to capture a picture with overlay included in image capture. I was able to set overlay to previewView using cameraView.overlay.add(binding.textView). How ever, it did not save when trying to save an image with imageCapture Only the picture was saved not the overlay. How do I save an image with overlay included using PreviewView of camera x.
Please don't mark this as duplicate. I researched a lot and most of the example online are using the old camera api which does not apply to camera x library. Any help is appreciated. Thanks in advance.
Here is my code
<FrameLayout
android:id="#+id/camera_wrapper"
android:layout_width="match_parent"
android:layout_height="0dp"
app:layout_constraintTop_toTopOf="#id/space1"
app:layout_constraintBottom_toBottomOf="#id/space">
<androidx.camera.view.PreviewView
android:id="#+id/camera_view"
android:layout_width="match_parent"
android:layout_height="match_parent" />
<TextView
android:id="#+id/text_view"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_gravity="center"
android:text="Hello world"
android:textSize="42sp"
android:textColor="#android:color/holo_green_dark"/>
</FrameLayout>
private lateinit var outputDirectory: File
private lateinit var cameraExecutor: ExecutorService
private var preview: Preview? = null
private var lensFacing: Int = CameraSelector.LENS_FACING_FRONT
private var imageCapture: ImageCapture? = null
private var camera: Camera? = null
private var cameraProvider: ProcessCameraProvider? = null
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
outputDirectory = getOutputDirectory()
cameraExecutor = Executors.newSingleThreadExecutor()
}
private fun setupCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext())
cameraProviderFuture.addListener(
Runnable {
// Used to bind the lifecycle of cameras to the lifecycle owner
cameraProvider = cameraProviderFuture.get()
// Get screen metrics used to setup camera for full screen resolution
val metrics = DisplayMetrics().also { binding.cameraView.display.getRealMetrics(it) }
Timber.d("Screen metrics: ${metrics.widthPixels} x ${metrics.heightPixels}")
val screenAspectRatio = aspectRatio(metrics.widthPixels, metrics.heightPixels)
Timber.d("Preview aspect ratio: $screenAspectRatio")
val rotation = binding.cameraView.display.rotation
// CameraProvider
val cameraProvider = cameraProvider
?: throw IllegalStateException("Camera initialization failed.")
// CameraSelector
val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build()
// add text overlay *---------*
binding.cameraView.overlay.add(binding.textView)
// Preview
preview = Preview.Builder()
// We request aspect ratio but no resolution
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation
.setTargetRotation(rotation)
.build()
// ImageCapture
imageCapture = ImageCapture.Builder()
.setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
// We request aspect ratio but no resolution to match preview config, but letting
// CameraX optimize for whatever specific resolution best fits our use cases
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
.setTargetRotation(rotation)
.build()
// Must unbind the use-cases before rebinding them
cameraProvider.unbindAll()
try {
// A variable number of use-cases can be passed here -
// camera provides access to CameraControl & CameraInfo
camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageCapture)
// Attach the viewfinder's surface provider to preview use case
preview?.setSurfaceProvider(binding.cameraView.surfaceProvider)
} catch (exc: Exception) {
Toast.makeText(requireContext(), "Something went wrong. Please try again.", Toast.LENGTH_SHORT).show()
findNavController().navigateUp()
}
},
ContextCompat.getMainExecutor(requireContext())
)
}
private fun takePhoto() {
imageCapture?.let { imageCapture ->
// Create output file to hold the image
val photoFile = createFile(outputDirectory, FILENAME, PHOTO_EXTENSION)
// Setup image capture metadata
val metadata = ImageCapture.Metadata().apply {
// Mirror image when using the front camera
isReversedHorizontal = lensFacing == CameraSelector.LENS_FACING_FRONT
}
// Create output options object which contains file + metadata
val outputOptions = ImageCapture.OutputFileOptions.Builder(photoFile)
.setMetadata(metadata)
.build()
// Setup image capture listener which is triggered after photo has been taken
imageCapture.takePicture(outputOptions, cameraExecutor, object : ImageCapture.OnImageSavedCallback {
override fun onError(exc: ImageCaptureException) {
Timber.e(exc, "Photo capture failed: ${exc.message}")
}
override fun onImageSaved(output: ImageCapture.OutputFileResults) {
val savedUri = output.savedUri ?: Uri.fromFile(photoFile)
Timber.d("Photo capture succeeded: $savedUri")
// Implicit broadcasts will be ignored for devices running API level >= 24
// so if you only target API level 24+ you can remove this statement
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
requireActivity()
.sendBroadcast(Intent(android.hardware.Camera.ACTION_NEW_PICTURE, savedUri))
}
// If the folder selected is an external media directory, this is
// unnecessary but otherwise other apps will not be able to access our
// images unless we scan them using [MediaScannerConnection]
val mimeType = MimeTypeMap.getSingleton()
.getMimeTypeFromExtension(savedUri.toFile().extension)
MediaScannerConnection.scanFile(
context,
arrayOf(savedUri.toFile().absolutePath),
arrayOf(mimeType)
) { _, uri ->
Timber.d("Image capture scanned into media store: $uri")
}
}
})
}
}

You must overlay the text over the image yourself. I would suggest to use takePicture(Executor, …) that puts the Jpeg in memory; then, overlay your text using one of the libraries (not part of Android framework, neither of Jetpack), and save the result in file.
If you can compromise on image quality, you can draw the Jpeg on Bitmap canvas, and draw your text on top.

Go with this plugin https://github.com/huangyz0918/AndroidWM
.Also, if you want to build your own this can help you to refer to.
The simple usage is like below
WatermarkText watermarkText = new WatermarkText(inputText)
.setPositionX(0.5)
.setPositionY(0.5)
.setTextColor(Color.WHITE)
.setTextFont(R.font.champagne)
.setTextShadow(0.1f, 5, 5, Color.BLUE)
.setTextAlpha(150)
.setRotation(30)
.setTextSize(20);
val bmFinal: Bitmap = WatermarkBuilder
.create(applicationContext, capturedImageBitmap)
.loadWatermarkText(watermarkText)
.watermark
.outputImage
// ##### Then save it #########
fun saveImage(bitmap: Bitmap, photoFile: File) {
val output: OutputStream = FileOutputStream(photoFile)
bitmap.compress(Bitmap.CompressFormat.JPEG, 100, output)
output.flush()
output.close()
Toast.makeText(
this#MainActivity,
"Imaged Saved at ${photoFile.absolutePath}",
Toast.LENGTH_LONG
).show()
}
val photoFile = File(
outputDirectory,
SimpleDateFormat(
FILENAME_FORMAT, Locale.US
).format(System.currentTimeMillis()) + ".jpg"
)
saveImage(bmFinal, photoFile)

May be you can use CameraSource class and put your preview/overlay inside :
val cameraSource = CameraSource.Builder(requireContext(),FakeDetector()).build()
cameraSource.start(your_preview_overlay)
And after you have an API :
takePicture(CameraSource.ShutterCallback shutter, CameraSource.PictureCallback jpeg)
Camera source (https://developers.google.com/android/reference/com/google/android/gms/vision/CameraSource) is for detection, but you can create a fake Detector ( nothing to detect).

#alexcohn's answer is the preferred one if you cannot afford to lose quality. However, if quality is not big deal then you can do this.
<FrameLayout
android:id="#+id/camera_wrapper"
android:layout_width="match_parent"
android:layout_height="0dp"
app:layout_constraintTop_toTopOf="#id/space1"
app:layout_constraintBottom_toBottomOf="#id/space">
<androidx.camera.view.PreviewView
android:id="#+id/camera_view"
android:layout_width="match_parent"
android:layout_height="match_parent" />
<ImageView
android:id="#+id/selfie"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:scaleType="centerCrop"
android:visibility="gone"
tools:visibility="visible"
tools:background="#color/gray" />
<ImageView
android:id="#+id/overlay"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:scaleType="centerCrop"
tools:src="#drawable/full_frame_gd" />
</FrameLayout>
PreviewView has a build in function that gives you bitmap of the preview
val bitmap = binding.cameraView.bitmap
binding.selfie.setImageBitmap(bitmap)
binding.selfie.visibility = View.VISIBLE
cameraExecutor.shutdown()
binding.cameraView.visibility = View.GONE
Now you have two images view one for selfie and one for overlay. You can't take screen shot of the previewView. There are some limitations to it that I'm not too sure of. but, I'm sure there might be a way around it.
From here you can just take a screen capture of the two combined image views like this
private fun captureView(view: View, window: Window, bitmapCallback: (Bitmap)->Unit) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
// Above Android O, use PixelCopy
val bitmap = Bitmap.createBitmap(view.width, view.height, Bitmap.Config.ARGB_8888)
val location = IntArray(2)
view.getLocationInWindow(location)
PixelCopy.request(
window,
Rect(
location[0],
location[1],
location[0] + view.width,
location[1] + view.height
),
bitmap,
{
if (it == PixelCopy.SUCCESS) {
bitmapCallback.invoke(bitmap)
}
},
Handler(Looper.getMainLooper()) )
} else {
val tBitmap = Bitmap.createBitmap(
view.width, view.height, Bitmap.Config.RGB_565
)
val canvas = Canvas(tBitmap)
view.draw(canvas)
canvas.setBitmap(null)
bitmapCallback.invoke(tBitmap)
}
}
and in takePhoto() function you can remove the imageCapture.takePicture logic and replace it with this.
Handler(Looper.getMainLooper()).postDelayed({
captureView(binding.cameraWrapper, requireActivity().window) {
// your new bitmap with overlay is here and you can save it to file just like any other bitmaps.
}
}, 500)

How to switch jetpack camerax preview from back to front vice versa?

When I try to switch the camera preview from BACK to FRONT my screen getting freeze and if I minimize the screen and restart the same then camera preview work perfectly.
below is the camera code.
private fun startCamera() {
CameraX.unbindAll()
val metrics = DisplayMetrics().also { viewFinder.display.getRealMetrics(it) }
val screenSize = Size(metrics.widthPixels, metrics.heightPixels)
val screenAspectRatio = Rational(metrics.widthPixels, metrics.heightPixels)
val previewConfig = PreviewConfig.Builder().apply {
setLensFacing(lensFacing)
setTargetResolution(screenSize)
setTargetAspectRatio(screenAspectRatio)
setTargetRotation(windowManager.defaultDisplay.rotation)
setTargetRotation(viewFinder.display.rotation)
}.build()
preview = Preview(previewConfig)
preview.setOnPreviewOutputUpdateListener {
viewFinder.surfaceTexture = it.surfaceTexture
updateTransform()
}
// Create configuration object for the image capture use case
val imageCaptureConfig = ImageCaptureConfig.Builder()
.apply {
setLensFacing(lensFacing)
setTargetAspectRatio(screenAspectRatio)
setTargetRotation(viewFinder.display.rotation)
setCaptureMode(ImageCapture.CaptureMode.MIN_LATENCY)
}.build()
// Build the image capture use case and attach button click listener
imageCapture = ImageCapture(imageCaptureConfig)
//for recording the video
val videoCaptureConfig = VideoCaptureConfig.Builder().apply {
setLensFacing(lensFacing)
setTargetAspectRatio(screenAspectRatio)
setTargetRotation(viewFinder.display.rotation)
}.build()
videoCapture = VideoCapture(videoCaptureConfig)
CameraX.bindToLifecycle(this, preview, imageCapture, videoCapture)
}
and the updateTransform code is
private fun updateTransform() {
val matrix = Matrix()
// Compute the center of the view finder
val centerX = viewFinder.width / 2f
val centerY = viewFinder.height / 2f
// Correct preview output to account for display rotation
val rotationDegrees = when (viewFinder.display.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> return
}
matrix.postRotate(-rotationDegrees.toFloat(), centerX, centerY)
// Finally, apply transformations to our TextureView
viewFinder.setTransform(matrix)
}
I have try to switch between the camera preview is :
lensFacing = if (lensFacing == CameraX.LensFacing.BACK) {
CameraX.LensFacing.FRONT
} else
CameraX.LensFacing.BACK
try {
CameraX.getCameraWithLensFacing(lensFacing)
CameraX.unbind(preview, imageCapture, videoCapture)
startCamera()
} catch (e: Exception) {
e.printStackTrace()
}
and after calling the above code on button click preview getting freez.

You may need to again add textureView in onUpdated callback.
Refer this,
https://android.googlesource.com/platform/frameworks/support/+/refs/heads/androidx-master-dev/camera/integration-tests/coretestapp/src/main/java/androidx/camera/integration/core/CameraXActivity.java#168
Let me know if it works.