Develop Reference

CameraX not launching on physical device and emulator

I'm currently using the cameraX sample code to record a video with audio . I didn't change anything from the samples but the camera simply won't launch. I have a swipe button in my fragment that is supposed to take me to the videoCaptureFragment but all I get are some logs and the fragments don't change. No errors or warnings , the app doesn't crash but it doesn't load the camera view.
here's my videoCaptureFragment:
class VideoCaptureFragment : Fragment() {
// UI with ViewBinding
private var _captureViewBinding: VideoCaptureFragmentBinding? = null
private val captureViewBinding get() = _captureViewBinding!!
private val captureLiveStatus = MutableLiveData<String>()
private val cameraCapabilities = mutableListOf<CameraCapability>()
private lateinit var videoCapture: VideoCapture<Recorder>
private var currentRecording: Recording? = null
private lateinit var recordingState:VideoRecordEvent
// Camera UI states and inputs
enum class UiState {
IDLE, // Not recording, all UI controls are active.
RECORDING, // Camera is recording, only display Pause/Resume & Stop button.
FINALIZED, // Recording just completes, disable all RECORDING UI controls.
RECOVERY // For future use.
}
private var cameraIndex = 0
private var qualityIndex = DEFAULT_QUALITY_IDX
private var audioEnabled = false
private val mainThreadExecutor by lazy { ContextCompat.getMainExecutor(requireContext()) }
private var enumerationDeferred:Deferred<Unit>? = null
// main cameraX capture functions
/**
* Always bind preview + video capture use case combinations in this sample
* (VideoCapture can work on its own). The function should always execute on
* the main thread.
*/
private suspend fun bindCaptureUsecase() {
val cameraProvider = ProcessCameraProvider.getInstance(requireContext()).await()
val cameraSelector = getCameraSelector(cameraIndex)
// create the user required QualitySelector (video resolution): we know this is
// supported, a valid qualitySelector will be created.
val quality = cameraCapabilities[cameraIndex].qualities[qualityIndex]
val qualitySelector = QualitySelector.from(quality)
captureViewBinding.previewView.updateLayoutParams<ConstraintLayout.LayoutParams> {
val orientation = this#RecordingInterviewFragment.resources.configuration.orientation
dimensionRatio = quality.getAspectRatioString(quality,
(orientation == Configuration.ORIENTATION_PORTRAIT))
}
val preview = Preview.Builder()
.setTargetAspectRatio(quality.getAspectRatio(quality))
.build().apply {
setSurfaceProvider(captureViewBinding.previewView.surfaceProvider)
}
// build a recorder, which can:
// - record video/audio to MediaStore(only shown here), File, ParcelFileDescriptor
// - be used create recording(s) (the recording performs recording)
val recorder = Recorder.Builder()
.setQualitySelector(qualitySelector)
.build()
videoCapture = VideoCapture.withOutput(recorder)
try {
cameraProvider.unbindAll()
cameraProvider.bindToLifecycle(
viewLifecycleOwner,
cameraSelector,
videoCapture,
preview
)
} catch (exc: Exception) {
// we are on main thread, let's reset the controls on the UI.
Log.e(TAG, "Use case binding failed", exc)
resetUIandState("bindToLifecycle failed: $exc")
}
enableUI(true)
}
/**
* Kick start the video recording
* - config Recorder to capture to MediaStoreOutput
* - register RecordEvent Listener
* - apply audio request from user
* - start recording!
* After this function, user could start/pause/resume/stop recording and application listens
* to VideoRecordEvent for the current recording status.
*/
#SuppressLint("MissingPermission")
private fun startRecording() {
// create MediaStoreOutputOptions for our recorder: resulting our recording!
val name = "CameraX-recording-" +
SimpleDateFormat(FILENAME_FORMAT, Locale.US)
.format(System.currentTimeMillis()) + ".mp4"
val contentValues = ContentValues().apply {
put(MediaStore.Video.Media.DISPLAY_NAME, name)
}
val mediaStoreOutput = MediaStoreOutputOptions.Builder(
requireActivity().contentResolver,
MediaStore.Video.Media.EXTERNAL_CONTENT_URI)
.setContentValues(contentValues)
.build()
// configure Recorder and Start recording to the mediaStoreOutput.
currentRecording = videoCapture.output
.prepareRecording(requireActivity(), mediaStoreOutput)
.apply { if (audioEnabled) withAudioEnabled() }
.start(mainThreadExecutor, captureListener)
Log.i(TAG, "Recording started")
}
/**
* CaptureEvent listener.
*/
private val captureListener = Consumer<VideoRecordEvent> { event ->
// cache the recording state
if (event !is VideoRecordEvent.Status)
recordingState = event
updateUI(event)
if (event is VideoRecordEvent.Finalize) {
// display the captured video
lifecycleScope.launch {
/*navController.navigate(
CaptureFragmentDirections.actionCaptureToVideoViewer(
event.outputResults.outputUri
)
)*/
}
}
}
/**
* Retrieve the asked camera's type(lens facing type). In this sample, only 2 types:
* idx is even number: CameraSelector.LENS_FACING_BACK
* odd number: CameraSelector.LENS_FACING_FRONT
*/
private fun getCameraSelector(idx: Int) : CameraSelector {
if (cameraCapabilities.size == 0) {
Log.i(TAG, "Error: This device does not have any camera, bailing out")
requireActivity().finish()
}
return (cameraCapabilities[idx % cameraCapabilities.size].camSelector)
}
data class CameraCapability(val camSelector: CameraSelector, val qualities:List<Quality>)
/**
* Query and cache this platform's camera capabilities, run only once.
*/
init {
enumerationDeferred = lifecycleScope.async {
whenCreated {
val provider = ProcessCameraProvider.getInstance(requireContext()).await()
provider.unbindAll()
for (camSelector in arrayOf(
CameraSelector.DEFAULT_BACK_CAMERA,
CameraSelector.DEFAULT_FRONT_CAMERA
)) {
try {
// just get the camera.cameraInfo to query capabilities
// we are not binding anything here.
if (provider.hasCamera(camSelector)) {
val camera = provider.bindToLifecycle(requireActivity(), camSelector)
QualitySelector
.getSupportedQualities(camera.cameraInfo)
.filter { quality ->
listOf(Quality.UHD, Quality.FHD, Quality.HD, Quality.SD)
.contains(quality)
}.also {
cameraCapabilities.add(CameraCapability(camSelector, it))
}
}
} catch (exc: java.lang.Exception) {
Log.e(TAG, "Camera Face $camSelector is not supported")
}
}
}
}
}
/**
* One time initialize for CameraFragment (as a part of fragment layout's creation process).
* This function performs the following:
* - initialize but disable all UI controls except the Quality selection.
* - set up the Quality selection recycler view.
* - bind use cases to a lifecycle camera, enable UI controls.
*/
private fun initCameraFragment() {
initializeUI()
Log.i("Camerx","insisde Init Camera")
viewLifecycleOwner.lifecycleScope.launch {
if (enumerationDeferred != null) {
enumerationDeferred!!.await()
enumerationDeferred = null
}
initializeQualitySectionsUI()
bindCaptureUsecase()
}
}
/**
* Initialize UI. Preview and Capture actions are configured in this function.
* Note that preview and capture are both initialized either by UI or CameraX callbacks
* (except the very 1st time upon entering to this fragment in onCreateView()
*/
#SuppressLint("ClickableViewAccessibility", "MissingPermission")
private fun initializeUI() {
captureViewBinding.cameraButton.apply {
setOnClickListener {
cameraIndex = (cameraIndex + 1) % cameraCapabilities.size
// camera device change is in effect instantly:
// - reset quality selection
// - restart preview
qualityIndex = DEFAULT_QUALITY_IDX
initializeQualitySectionsUI()
enableUI(false)
viewLifecycleOwner.lifecycleScope.launch {
bindCaptureUsecase()
}
}
isEnabled = false
}
// audioEnabled by default is disabled.
captureViewBinding.audioSelection.isChecked = audioEnabled
captureViewBinding.audioSelection.setOnClickListener {
audioEnabled = captureViewBinding.audioSelection.isChecked
}
Log.i("Camerx","inside init UI")
// React to user touching the capture button
captureViewBinding.captureButton.apply {
setOnClickListener {
if (!this#RecordingInterviewFragment::recordingState.isInitialized ||
recordingState is VideoRecordEvent.Finalize)
{
enableUI(false) // Our eventListener will turn on the Recording UI.
startRecording()
} else {
when (recordingState) {
is VideoRecordEvent.Start -> {
currentRecording?.pause()
captureViewBinding.stopButton.visibility = View.VISIBLE
}
is VideoRecordEvent.Pause -> currentRecording?.resume()
is VideoRecordEvent.Resume -> currentRecording?.pause()
else -> throw IllegalStateException("recordingState in unknown state")
}
}
}
isEnabled = false
}
captureViewBinding.stopButton.apply {
setOnClickListener {
// stopping: hide it after getting a click before we go to viewing fragment
captureViewBinding.stopButton.visibility = View.INVISIBLE
if (currentRecording == null || recordingState is VideoRecordEvent.Finalize) {
return#setOnClickListener
}
val recording = currentRecording
if (recording != null) {
recording.stop()
currentRecording = null
}
captureViewBinding.captureButton.setImageResource(R.drawable.ic_start)
}
// ensure the stop button is initialized disabled & invisible
visibility = View.INVISIBLE
isEnabled = false
}
captureLiveStatus.observe(viewLifecycleOwner) {
captureViewBinding.captureStatus.apply {
post { text = it }
}
}
captureLiveStatus.value = getString(R.string.Idle)
}
/**
* UpdateUI according to CameraX VideoRecordEvent type:
* - user starts capture.
* - this app disables all UI selections.
* - this app enables capture run-time UI (pause/resume/stop).
* - user controls recording with run-time UI, eventually tap "stop" to end.
* - this app informs CameraX recording to stop with recording.stop() (or recording.close()).
* - CameraX notify this app that the recording is indeed stopped, with the Finalize event.
* - this app starts VideoViewer fragment to view the captured result.
*/
private fun updateUI(event: VideoRecordEvent) {
val state = if (event is VideoRecordEvent.Status) recordingState.getNameString()
else event.getNameString()
when (event) {
is VideoRecordEvent.Status -> {
// placeholder: we update the UI with new status after this when() block,
// nothing needs to do here.
Log.i("Camerx","you're idk tbh")
}
is VideoRecordEvent.Start -> {
showUI(UiState.RECORDING, event.getNameString())
Log.i("Camerx","you're recording")
}
is VideoRecordEvent.Finalize-> {
showUI(UiState.FINALIZED, event.getNameString())
Log.i("Camerx","you're done, you're done")
}
is VideoRecordEvent.Pause -> {
captureViewBinding.captureButton.setImageResource(R.drawable.ic_resume)
Log.i("Camerx","you're paused")
}
is VideoRecordEvent.Resume -> {
captureViewBinding.captureButton.setImageResource(R.drawable.ic_pause)
Log.i("Camerx","you're resumed")
}
}
val stats = event.recordingStats
val size = stats.numBytesRecorded / 1000
val time = java.util.concurrent.TimeUnit.NANOSECONDS.toSeconds(stats.recordedDurationNanos)
var text = "${state}: recorded ${size}KB, in ${time}second"
if(event is VideoRecordEvent.Finalize)
text = "${text}\nFile saved to: ${event.outputResults.outputUri}"
captureLiveStatus.value = text
Log.i(TAG, "recording event: $text")
}
/**
* Enable/disable UI:
* User could select the capture parameters when recording is not in session
* Once recording is started, need to disable able UI to avoid conflict.
*/
private fun enableUI(enable: Boolean) {
Log.i("Camerx","insisde enableuia")
arrayOf(captureViewBinding.cameraButton,
captureViewBinding.captureButton,
captureViewBinding.stopButton,
captureViewBinding.audioSelection,
captureViewBinding.qualitySelection).forEach {
it.isEnabled = enable
}
// disable the camera button if no device to switch
if (cameraCapabilities.size <= 1) {
captureViewBinding.cameraButton.isEnabled = false
}
// disable the resolution list if no resolution to switch
if (cameraCapabilities[cameraIndex].qualities.size <= 1) {
captureViewBinding.qualitySelection.apply { isEnabled = false }
}
}
/**
* initialize UI for recording:
* - at recording: hide audio, qualitySelection,change camera UI; enable stop button
* - otherwise: show all except the stop button
*/
private fun showUI(state: UiState, status:String = "idle") {
Log.i("Camerx","insisde show ui")
captureViewBinding.let {
when(state) {
UiState.IDLE -> {
it.captureButton.setImageResource(R.drawable.ic_start)
it.stopButton.visibility = View.INVISIBLE
it.cameraButton.visibility= View.VISIBLE
it.audioSelection.visibility = View.VISIBLE
it.qualitySelection.visibility=View.VISIBLE
}
UiState.RECORDING -> {
it.cameraButton.visibility = View.INVISIBLE
it.audioSelection.visibility = View.INVISIBLE
it.qualitySelection.visibility = View.INVISIBLE
it.captureButton.setImageResource(R.drawable.ic_pause)
it.captureButton.isEnabled = true
it.stopButton.visibility = View.VISIBLE
it.stopButton.isEnabled = true
}
UiState.FINALIZED -> {
it.captureButton.setImageResource(R.drawable.ic_start)
it.stopButton.visibility = View.INVISIBLE
}
else -> {
val errorMsg = "Error: showUI($state) is not supported"
Log.e(TAG, errorMsg)
return
}
}
it.captureStatus.text = status
}
}
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
initCameraFragment()
}
override fun onDestroyView() {
_captureViewBinding = null
super.onDestroyView()
}
companion object {
// default Quality selection if no input from UI
const val DEFAULT_QUALITY_IDX = 0
val TAG:String = RecordingInterviewFragment::class.java.simpleName
private const val FILENAME_FORMAT = "yyyy-MM-dd-HH-mm-ss-SSS"
}
Thank you

CameraX preview not working after upgrade

I've updated my gradle plugins for CameraX kotlin and all of a sudden I've got an error that I don't know how to fix.
This is my camera fragment
package com.khumomashapa.notes.fragments
import android.annotation.SuppressLint
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.content.res.Configuration
import android.graphics.Color
import android.graphics.drawable.ColorDrawable
import android.hardware.display.DisplayManager
import android.media.MediaScannerConnection
import android.net.Uri
import android.os.Build
import android.os.Bundle
import android.util.DisplayMetrics
import android.util.Log
import android.view.KeyEvent
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.webkit.MimeTypeMap
import android.widget.ImageButton
import androidx.camera.core.AspectRatio
import androidx.camera.core.Camera
import androidx.camera.core.CameraInfoUnavailableException
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCapture.Metadata
import androidx.camera.core.ImageCaptureException
import androidx.camera.core.ImageProxy
import androidx.camera.core.Preview
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.constraintlayout.widget.ConstraintLayout
import androidx.core.content.ContextCompat
import androidx.core.net.toFile
import androidx.core.view.setPadding
import androidx.fragment.app.Fragment
import androidx.lifecycle.lifecycleScope
import androidx.localbroadcastmanager.content.LocalBroadcastManager
import androidx.navigation.Navigation
import com.bumptech.glide.Glide
import com.bumptech.glide.request.RequestOptions
import com.khumomashapa.notes.R
import com.khumomashapa.notes.activities.CameraActivity
import com.khumomashapa.notes.activities.KEY_EVENT_ACTION
import com.khumomashapa.notes.activities.KEY_EVENT_EXTRA
import com.khumomashapa.notes.utils.ANIMATION_FAST_MILLIS
import com.khumomashapa.notes.utils.ANIMATION_SLOW_MILLIS
import com.khumomashapa.notes.utils.simulateClick
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import java.io.File
import java.nio.ByteBuffer
import java.text.SimpleDateFormat
import java.util.ArrayDeque
import java.util.Locale
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min
/** Helper type alias used for analysis use case callbacks */
typealias LumaListener = (luma: Double) -> Unit
/**
* Main fragment for this app. Implements all camera operations including:
* - Viewfinder
* - Photo taking
* - Image analysis
*/
class CameraFragment : Fragment() {
private lateinit var container: ConstraintLayout
private lateinit var viewFinder: PreviewView
private lateinit var outputDirectory: File
private lateinit var broadcastManager: LocalBroadcastManager
private var displayId: Int = -1
private var lensFacing: Int = CameraSelector.LENS_FACING_BACK
private var preview: Preview? = null
private var imageCapture: ImageCapture? = null
private var imageAnalyzer: ImageAnalysis? = null
private var camera: Camera? = null
private var cameraProvider: ProcessCameraProvider? = null
private val displayManager by lazy {
requireContext().getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
}
/** Blocking camera operations are performed using this executor */
private lateinit var cameraExecutor: ExecutorService
/** Volume down button receiver used to trigger shutter */
private val volumeDownReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
when (intent.getIntExtra(KEY_EVENT_EXTRA, KeyEvent.KEYCODE_UNKNOWN)) {
// When the volume down button is pressed, simulate a shutter button click
KeyEvent.KEYCODE_VOLUME_DOWN -> {
val shutter = container
.findViewById<ImageButton>(R.id.camera_capture_button)
shutter.simulateClick()
}
}
}
}
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) = view?.let { view ->
if (displayId == this#CameraFragment.displayId) {
Log.d(TAG, "Rotation changed: ${view.display.rotation}")
imageCapture?.targetRotation = view.display.rotation
imageAnalyzer?.targetRotation = view.display.rotation
}
} ?: Unit
}
override fun onResume() {
super.onResume()
// Make sure that all permissions are still present, since the
// user could have removed them while the app was in paused state.
if (!PermissionsFragment.hasPermissions(requireContext())) {
Navigation.findNavController(requireActivity(), R.id.fragment_container).navigate(
CameraFragmentDirections.actionCameraToPermissions()
)
}
}
override fun onDestroyView() {
super.onDestroyView()
// Shut down our background executor
cameraExecutor.shutdown()
// Unregister the broadcast receivers and listeners
broadcastManager.unregisterReceiver(volumeDownReceiver)
displayManager.unregisterDisplayListener(displayListener)
}
override fun onCreateView(
inflater: LayoutInflater,
container: ViewGroup?,
savedInstanceState: Bundle?): View? =
inflater.inflate(R.layout.fragment_camera, container, false)
private fun setGalleryThumbnail(uri: Uri) {
// Reference of the view that holds the gallery thumbnail
val thumbnail = container.findViewById<ImageButton>(R.id.photo_view_button)
// Run the operations in the view's thread
thumbnail.post {
// Remove thumbnail padding
thumbnail.setPadding(resources.getDimension(R.dimen.stroke_small).toInt())
// Load thumbnail into circular button using Glide
Glide.with(thumbnail)
.load(uri)
.apply(RequestOptions.circleCropTransform())
.into(thumbnail)
}
}
#SuppressLint("MissingPermission")
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
container = view as ConstraintLayout
viewFinder = container.findViewById(R.id.view_finder)
// Initialize our background executor
cameraExecutor = Executors.newSingleThreadExecutor()
broadcastManager = LocalBroadcastManager.getInstance(view.context)
// Set up the intent filter that will receive events from our main activity
val filter = IntentFilter().apply { addAction(KEY_EVENT_ACTION) }
broadcastManager.registerReceiver(volumeDownReceiver, filter)
// Every time the orientation of device changes, update rotation for use cases
displayManager.registerDisplayListener(displayListener, null)
// Determine the output directory
outputDirectory = CameraActivity.getOutputDirectory(requireContext())
// Wait for the views to be properly laid out
viewFinder.post {
// Keep track of the display in which this view is attached
displayId = viewFinder.display.displayId
// Build UI controls
updateCameraUi()
// Set up the camera and its use cases
setUpCamera()
}
}
/**
* Inflate camera controls and update the UI manually upon config changes to avoid removing
* and re-adding the view finder from the view hierarchy; this provides a seamless rotation
* transition on devices that support it.
*
* NOTE: The flag is supported starting in Android 8 but there still is a small flash on the
* screen for devices that run Android 9 or below.
*/
override fun onConfigurationChanged(newConfig: Configuration) {
super.onConfigurationChanged(newConfig)
// Redraw the camera UI controls
updateCameraUi()
// Enable or disable switching between cameras
updateCameraSwitchButton()
}
/** Initialize CameraX, and prepare to bind the camera use cases */
private fun setUpCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext())
cameraProviderFuture.addListener(Runnable {
// CameraProvider
cameraProvider = cameraProviderFuture.get()
// Select lensFacing depending on the available cameras
lensFacing = when {
hasBackCamera() -> CameraSelector.LENS_FACING_BACK
hasFrontCamera() -> CameraSelector.LENS_FACING_FRONT
else -> throw IllegalStateException("Back and front camera are unavailable")
}
// Enable or disable switching between cameras
updateCameraSwitchButton()
// Build and bind the camera use cases
bindCameraUseCases()
}, ContextCompat.getMainExecutor(requireContext()))
}
/** Declare and bind preview, capture and analysis use cases */
private fun bindCameraUseCases() {
// Get screen metrics used to setup camera for full screen resolution
val metrics = DisplayMetrics().also { viewFinder.display.getRealMetrics(it) }
Log.d(TAG, "Screen metrics: ${metrics.widthPixels} x ${metrics.heightPixels}")
val screenAspectRatio = aspectRatio(metrics.widthPixels, metrics.heightPixels)
Log.d(TAG, "Preview aspect ratio: $screenAspectRatio")
val rotation = viewFinder.display.rotation
// CameraProvider
val cameraProvider = cameraProvider
?: throw IllegalStateException("Camera initialization failed.")
// CameraSelector
val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build()
// Preview
preview = Preview.Builder()
// We request aspect ratio but no resolution
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation
.setTargetRotation(rotation)
.build()
// ImageCapture
imageCapture = ImageCapture.Builder()
.setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
// We request aspect ratio but no resolution to match preview config, but letting
// CameraX optimize for whatever specific resolution best fits our use cases
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
.setTargetRotation(rotation)
.build()
// ImageAnalysis
imageAnalyzer = ImageAnalysis.Builder()
// We request aspect ratio but no resolution
.setTargetAspectRatio(screenAspectRatio)
// Set initial target rotation, we will have to call this again if rotation changes
// during the lifecycle of this use case
.setTargetRotation(rotation)
.build()
// The analyzer can then be assigned to the instance
.also {
it.setAnalyzer(cameraExecutor, LuminosityAnalyzer { luma ->
// Values returned from our analyzer are passed to the attached listener
// We log space_image_view analysis results here - you should do something useful
// instead!
Log.d(TAG, "Average luminosity: $luma")
})
}
// Must unbind the use-cases before rebinding them
cameraProvider.unbindAll()
try {
// A variable number of use-cases can be passed here -
// camera provides access to CameraControl & CameraInfo
camera = cameraProvider.bindToLifecycle(
this, cameraSelector, preview, imageCapture, imageAnalyzer)
// Attach the viewfinder's surface provider to preview use case
preview?.setSurfaceProvider(viewFinder.createSurfaceProvider())
} catch (exc: Exception) {
Log.e(TAG, "Use case binding failed", exc)
}
}
/**
* [androidx.camera.core.ImageAnalysisConfig] requires enum value of
* [androidx.camera.core.AspectRatio]. Currently it has values of 4:3 & 16:9.
*
* Detecting the most suitable ratio for dimensions provided in #params by counting absolute
* of preview ratio to one of the provided values.
*
* #param width - preview width
* #param height - preview height
* #return suitable aspect ratio
*/
private fun aspectRatio(width: Int, height: Int): Int {
val previewRatio = max(width, height).toDouble() / min(width, height)
if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE)) {
return AspectRatio.RATIO_4_3
}
return AspectRatio.RATIO_16_9
}
/** Method used to re-draw the camera UI controls, called every time configuration changes. */
private fun updateCameraUi() {
// Remove previous UI if any
container.findViewById<ConstraintLayout>(R.id.camera_ui_container)?.let {
container.removeView(it)
}
// Inflate a new view containing all UI for controlling the camera
val controls = View.inflate(requireContext(), R.layout.camera_ui_container, container)
// In the background, load latest photo taken (if any) for gallery thumbnail
lifecycleScope.launch(Dispatchers.IO) {
outputDirectory.listFiles { file ->
EXTENSION_WHITELIST.contains(file.extension.toUpperCase(Locale.ROOT))
}?.max()?.let {
setGalleryThumbnail(Uri.fromFile(it))
}
}
// Listener for button used to capture photo
controls.findViewById<ImageButton>(R.id.camera_capture_button).setOnClickListener {
// Get a stable reference of the modifiable space_image_view capture use case
imageCapture?.let { imageCapture ->
// Create output file to hold the space_image_view
val photoFile = createFile(outputDirectory, FILENAME, PHOTO_EXTENSION)
// Setup space_image_view capture metadata
val metadata = Metadata().apply {
// Mirror space_image_view when using the front camera
isReversedHorizontal = lensFacing == CameraSelector.LENS_FACING_FRONT
}
// Create output options object which contains file + metadata
val outputOptions = ImageCapture.OutputFileOptions.Builder(photoFile)
.setMetadata(metadata)
.build()
// Setup space_image_view capture listener which is triggered after photo has been taken
imageCapture.takePicture(
outputOptions, cameraExecutor, object : ImageCapture.OnImageSavedCallback {
override fun onError(exc: ImageCaptureException) {
Log.e(TAG, "Photo capture failed: ${exc.message}", exc)
}
override fun onImageSaved(output: ImageCapture.OutputFileResults) {
val savedUri = output.savedUri ?: Uri.fromFile(photoFile)
Log.d(TAG, "Photo capture succeeded: $savedUri")
// We can only change the foreground Drawable using API level 23+ API
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
// Update the gallery thumbnail with latest picture taken
setGalleryThumbnail(savedUri)
}
// Implicit broadcasts will be ignored for devices running API level >= 24
// so if you only target API level 24+ you can remove this statement
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
requireActivity().sendBroadcast(
Intent(android.hardware.Camera.ACTION_NEW_PICTURE, savedUri)
)
}
// If the folder selected is an external media directory, this is
// unnecessary but otherwise other apps will not be able to access our
// images unless we scan them using [MediaScannerConnection]
val mimeType = MimeTypeMap.getSingleton()
.getMimeTypeFromExtension(savedUri.toFile().extension)
MediaScannerConnection.scanFile(
context,
arrayOf(savedUri.toFile().absolutePath),
arrayOf(mimeType)
) { _, uri ->
Log.d(TAG, "Image capture scanned into media store: $uri")
}
}
})
// We can only change the foreground Drawable using API level 23+ API
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
// Display flash animation to indicate that photo was captured
container.postDelayed({
container.foreground = ColorDrawable(Color.WHITE)
container.postDelayed(
{ container.foreground = null }, ANIMATION_FAST_MILLIS)
}, ANIMATION_SLOW_MILLIS)
}
}
}
// Setup for button used to switch cameras
controls.findViewById<ImageButton>(R.id.camera_switch_button).let {
// Disable the button until the camera is set up
it.isEnabled = false
// Listener for button used to switch cameras. Only called if the button is enabled
it.setOnClickListener {
lensFacing = if (CameraSelector.LENS_FACING_FRONT == lensFacing) {
CameraSelector.LENS_FACING_BACK
} else {
CameraSelector.LENS_FACING_FRONT
}
// Re-bind use cases to update selected camera
bindCameraUseCases()
}
}
// Listener for button used to view the most recent photo
controls.findViewById<ImageButton>(R.id.photo_view_button).setOnClickListener {
// Only navigate when the gallery has photos
if (true == outputDirectory.listFiles()?.isNotEmpty()) {
Navigation.findNavController(
requireActivity(), R.id.fragment_container
).navigate(CameraFragmentDirections
.actionCameraToGallery(outputDirectory.absolutePath))
}
}
}
/** Enabled or disabled a button to switch cameras depending on the available cameras */
private fun updateCameraSwitchButton() {
val switchCamerasButton = container.findViewById<ImageButton>(R.id.camera_switch_button)
try {
switchCamerasButton.isEnabled = hasBackCamera() && hasFrontCamera()
} catch (exception: CameraInfoUnavailableException) {
switchCamerasButton.isEnabled = false
}
}
/** Returns true if the device has an available back camera. False otherwise */
private fun hasBackCamera(): Boolean {
return cameraProvider?.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA) ?: false
}
/** Returns true if the device has an available front camera. False otherwise */
private fun hasFrontCamera(): Boolean {
return cameraProvider?.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA) ?: false
}
/**
* Our custom space_image_view analysis class.
*
* <p>All we need to do is override the function `analyze` with our desired operations. Here,
* we compute the average luminosity of the space_image_view by looking at the Y plane of the YUV frame.
*/
private class LuminosityAnalyzer(listener: LumaListener? = null) : ImageAnalysis.Analyzer {
private val frameRateWindow = 8
private val frameTimestamps = ArrayDeque<Long>(5)
private val listeners = ArrayList<LumaListener>().apply { listener?.let { add(it) } }
private var lastAnalyzedTimestamp = 0L
var framesPerSecond: Double = -1.0
private set
/**
* Used to add listeners that will be called with each luma computed
*/
fun onFrameAnalyzed(listener: LumaListener) = listeners.add(listener)
/**
* Helper extension function used to extract a byte array from an space_image_view plane buffer
*/
private fun ByteBuffer.toByteArray(): ByteArray {
rewind() // Rewind the buffer to zero
val data = ByteArray(remaining())
get(data) // Copy the buffer into a byte array
return data // Return the byte array
}
/**
* Analyzes an space_image_view to produce a result.
*
* <p>The caller is responsible for ensuring this analysis method can be executed quickly
* enough to prevent stalls in the space_image_view acquisition pipeline. Otherwise, newly available
* images will not be acquired and analyzed.
*
* <p>The space_image_view passed to this method becomes invalid after this method returns. The caller
* should not store external references to this space_image_view, as these references will become
* invalid.
*
* #param image space_image_view being analyzed VERY IMPORTANT: Analyzer method implementation must
* call space_image_view.close() on received images when finished using them. Otherwise, new images
* may not be received or the camera may stall, depending on back pressure setting.
*
*/
override fun analyze(image: ImageProxy) {
// If there are no listeners attached, we don't need to perform analysis
if (listeners.isEmpty()) {
image.close()
return
}
// Keep track of frames analyzed
val currentTime = System.currentTimeMillis()
frameTimestamps.push(currentTime)
// Compute the FPS using a moving average
while (frameTimestamps.size >= frameRateWindow) frameTimestamps.removeLast()
val timestampFirst = frameTimestamps.peekFirst() ?: currentTime
val timestampLast = frameTimestamps.peekLast() ?: currentTime
framesPerSecond = 1.0 / ((timestampFirst - timestampLast) /
frameTimestamps.size.coerceAtLeast(1).toDouble()) * 1000.0
// Analysis could take an arbitrarily long amount of time
// Since we are running in a different thread, it won't stall other use cases
lastAnalyzedTimestamp = frameTimestamps.first
// Since format in ImageAnalysis is YUV, space_image_view.planes[0] contains the luminance plane
val buffer = image.planes[0].buffer
// Extract space_image_view data from callback object
val data = buffer.toByteArray()
// Convert the data into an array of pixel values ranging 0-255
val pixels = data.map { it.toInt() and 0xFF }
// Compute average luminance for the space_image_view
val luma = pixels.average()
// Call all listeners with new value
listeners.forEach { it(luma) }
image.close()
}
}
companion object {
private const val TAG = "CameraXBasic"
private const val FILENAME = "yyyy-MM-dd-HH-mm-ss-SSS"
private const val PHOTO_EXTENSION = ".jpg"
private const val RATIO_4_3_VALUE = 4.0 / 3.0
private const val RATIO_16_9_VALUE = 16.0 / 9.0
/** Helper function used to create a timestamped file */
private fun createFile(baseFolder: File, format: String, extension: String) =
File(baseFolder, SimpleDateFormat(format, Locale.US)
.format(System.currentTimeMillis()) + extension)
}
}
After I updated the gradle I got a "Unresolved reference: createSurfaceProvider" error. I've read the documentation about this, but it keeps giving me the same error and I don't know why. Does anyone know how to fix this problem?

Since Camera-View 1.0.0-alpha16, createSurfaceProvider() has been renamed to getSurfaceProvider()
Use:
preview?.setSurfaceProvider(viewFinder.surfaceProvider)

Kotlin
For viewBinding and/or dataBinding :
it.setSurfaceProvider(binding.preview.surfaceProvider)
// CameraX
def camerax_version = '1.0.0-rc01'
implementation "androidx.camera:camera-camera2:$camerax_version"
implementation "androidx.camera:camera-lifecycle:$camerax_version"
implementation 'androidx.camera:camera-view:1.0.0-alpha20'

Did you update both camera-lifecycle and camera-view in the build.gradle?
I had to do both to get past it.
Issue arrose as older code examples that use createSurfaceProvider needed to change to surfaceProvider
AND
I had to make sure in build.gradle(app)
androidx.camera:camera-lifecycle:1.0.0-beta09
androidx.camera:camera-camera2:1.0.0-beta09
androidx.camera:camera-view:1.0.0-alpha16
I had a mismatch on alpha16 not using beta09 and it did not work as expected.

Instead viewFinder.createSurfaceProvider() use viewFinder.surfaceProvider

Android CameraX, increase ImageAnalysis frame rate

I need to take as many frames as possible from the preview of the camera.
I'm doing this to start the camera using CameraX:
private fun startCamera() {
// Create configuration object for the viewfinder use case
val metrics = DisplayMetrics().also { view_finder.display.getRealMetrics(it) }
// define the screen size
val screenSize = Size(metrics.widthPixels, metrics.heightPixels)
val screenAspectRatio = Rational(metrics.widthPixels, metrics.heightPixels)
val previewConfig = PreviewConfig.Builder()
.setLensFacing(CameraX.LensFacing.BACK) // defaults to Back camera
.setTargetAspectRatio(screenAspectRatio)
.setTargetResolution(screenSize)
.build()
// Build the viewfinder use case
val preview = Preview(previewConfig)
// Every time the viewfinder is updated, recompute layout
preview.setOnPreviewOutputUpdateListener {
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = view_finder.parent as ViewGroup
parent.removeView(view_finder)
parent.addView(view_finder, 0)
view_finder.surfaceTexture = it.surfaceTexture
updateTransform()
}
val analyzerConfig = ImageAnalysisConfig.Builder().apply {
// Use a worker thread for image analysis to prevent glitches
val analyzerThread = HandlerThread("AnalysisThread").apply {
start()
}
setLensFacing(CameraX.LensFacing.BACK) // defaults to Back camera
setTargetAspectRatio(screenAspectRatio)
setMaxResolution(Size(600, 320))
setCallbackHandler(Handler(analyzerThread.looper))
setImageReaderMode(ImageAnalysis.ImageReaderMode.ACQUIRE_LATEST_IMAGE)
}.build()
val analyzerUseCase = ImageAnalysis(analyzerConfig).apply {
analyzer = context?.let { FrameCapturer(it) }
}
//====================== Image Analysis Config code End==========================
CameraX.bindToLifecycle(this, preview, analyzerUseCase)
}
My FrameCapturer class is:
class FrameCapturer(context: Context): ImageAnalysis.Analyzer {
private var mListener = context as FrameCapturerListener
override fun analyze(image: ImageProxy?, rotationDegrees: Int) {
val buffer = image?.planes?.get(0)?.buffer
// Extract image data from callback object
val data = buffer?.toByteArray()
// Convert the data into an array of pixel values
//val pixels = data?.map { it.toInt() and 0xFF }
mListener.updateFps()
}
private fun ByteBuffer.toByteArray(): ByteArray {
rewind() // Rewind the buffer to zero
val data = ByteArray(remaining())
get(data) // Copy the buffer into a byte array
return data // Return the byte array
}
interface FrameCapturerListener {
fun updateFps()
}
And my updateFps function is:
fun updateFps() {
fps += 1
if (!isCountDownStarted) {
var timer = object : CountDownTimer(1000, 1000) {
override fun onTick(millisUntilFinished: Long) {
}
override fun onFinish() {
Log.d("CameraFragment", fps.toString())
fps = 0
isCountDownStarted = false
}
}.start()
isCountDownStarted = true
}
}
I'm around 16-22 fps, even if I don't convert the image to an array of byte in the FrameCapturer class. So, it seems like the Analyzer take only 20 image per second. Is there a way to increase the fps? I need to take at least 40-60 image per second, because I need to do post-processing with machine learning for each frame, so they will probably drop to 20-30 after ML analysis.
EDIT: I discover that with the Pixel 2xl I got 60fps without any drop..with my device (Xiaomi Redmi Note 5 Pro) I got only 25 fps...Can I optimize the code in any way to increase FPS?

Camera X captures images in different rotation states

Okay, I went through different posts and find out that depending on mobile manufacturers there can be a complications such as capture images get rotated, so you have to be aware of that. What I did was:
fun rotateBitmap(bitmap: Bitmap): Bitmap? {
val matrix = Matrix()
when (getImageOrientation(bitmap)) {
ExifInterface.ORIENTATION_NORMAL -> return bitmap
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.setScale(-1f, 1f)
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.setRotate(-90f)
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.setRotate(180f)
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.setRotate(90f)
ExifInterface.ORIENTATION_FLIP_VERTICAL -> {
matrix.setRotate(180f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSPOSE -> {
matrix.setRotate(90f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSVERSE -> {
matrix.setRotate(-90f)
matrix.postScale(-1f, 1f)
}
else -> return bitmap
}
This worked. But then I noticed something really weird and that might be related with how I configured Camera X configuration.
With the same device I get differently rotated Bitmaps (well, this should not happen. If devices rotates image weirdly, it should rotate images in both modes - in ImageAnalysesUseCase and ImageCaptureUseCase).
So, why is this happening and how can I fix it?
Code implementation:
Binding camera X to life-cycle:
CameraX.bindToLifecycle(
this,
buildPreviewUseCase(),
buildImageAnalysisUseCase(),
buildImageCaptureUseCase()
)
Preview use case:
private fun buildPreviewUseCase(): Preview {
val previewConfig = PreviewConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetResolution(config.resolution)
.setTargetRotation(Surface.ROTATION_0)
.setLensFacing(config.lensFacing)
.build()
return AutoFitPreviewBuilder.build(previewConfig, cameraTextureView)
}
Capture use case:
private fun buildImageCaptureUseCase(): ImageCapture {
val captureConfig = ImageCaptureConfig.Builder()
.setTargetAspectRatio(config.aspectRatio)
.setTargetRotation(Surface.ROTATION_0)
.setTargetResolution(config.resolution)
.setCaptureMode(config.captureMode)
.build()
val capture = ImageCapture(captureConfig)
manualModeTakePhotoButton.setOnClickListener {
capture.takePicture(object : ImageCapture.OnImageCapturedListener() {
override fun onCaptureSuccess(imageProxy: ImageProxy, rotationDegrees: Int) {
viewModel.onManualCameraModeAnalysis(imageProxy, rotationDegrees)
}
override fun onError(useCaseError: ImageCapture.UseCaseError?, message: String?, cause: Throwable?) {
//
}
})
}
return capture
}
Analysis use case:
private fun buildImageAnalysisUseCase(): ImageAnalysis {
val analysisConfig = ImageAnalysisConfig.Builder().apply {
val analyzerThread = HandlerThread("xAnalyzer").apply { start() }
analyzerHandler = Handler(analyzerThread.looper)
setCallbackHandler(analyzerHandler!!)
setTargetAspectRatio(config.aspectRatio)
setTargetRotation(Surface.ROTATION_0)
setTargetResolution(config.resolution)
setImageReaderMode(config.readerMode)
setImageQueueDepth(config.queueDepth)
}.build()
val analysis = ImageAnalysis(analysisConfig)
analysis.analyzer = ImageRecognitionAnalyzer(viewModel)
return analysis
}
AutoFitPreviewBuilder:
class AutoFitPreviewBuilder private constructor(config: PreviewConfig,
viewFinderRef: WeakReference<TextureView>) {
/** Public instance of preview use-case which can be used by consumers of this adapter */
val useCase: Preview
/** Internal variable used to keep track of the use-case's output rotation */
private var bufferRotation: Int = 0
/** Internal variable used to keep track of the view's rotation */
private var viewFinderRotation: Int? = null
/** Internal variable used to keep track of the use-case's output dimension */
private var bufferDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's dimension */
private var viewFinderDimens: Size = Size(0, 0)
/** Internal variable used to keep track of the view's display */
private var viewFinderDisplay: Int = -1
/** Internal reference of the [DisplayManager] */
private lateinit var displayManager: DisplayManager
/**
* We need a display listener for orientation changes that do not trigger a configuration
* change, for example if we choose to override config change in manifest or for 180-degree
* orientation changes.
*/
private val displayListener = object : DisplayManager.DisplayListener {
override fun onDisplayAdded(displayId: Int) = Unit
override fun onDisplayRemoved(displayId: Int) = Unit
override fun onDisplayChanged(displayId: Int) {
val viewFinder = viewFinderRef.get() ?: return
if (displayId == viewFinderDisplay) {
val display = displayManager.getDisplay(displayId)
val rotation = getDisplaySurfaceRotation(display)
updateTransform(viewFinder, rotation, bufferDimens, viewFinderDimens)
}
}
}
init {
// Make sure that the view finder reference is valid
val viewFinder = viewFinderRef.get() ?:
throw IllegalArgumentException("Invalid reference to view finder used")
// Initialize the display and rotation from texture view information
viewFinderDisplay = viewFinder.display.displayId
viewFinderRotation = getDisplaySurfaceRotation(viewFinder.display) ?: 0
// Initialize public use-case with the given config
useCase = Preview(config)
// Every time the view finder is updated, recompute layout
useCase.onPreviewOutputUpdateListener = Preview.OnPreviewOutputUpdateListener {
val viewFinder =
viewFinderRef.get() ?: return#OnPreviewOutputUpdateListener
// To update the SurfaceTexture, we have to remove it and re-add it
val parent = viewFinder.parent as ViewGroup
parent.removeView(viewFinder)
parent.addView(viewFinder, 0)
viewFinder.surfaceTexture = it.surfaceTexture
bufferRotation = it.rotationDegrees
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, it.textureSize, viewFinderDimens)
}
// Every time the provided texture view changes, recompute layout
viewFinder.addOnLayoutChangeListener { view, left, top, right, bottom, _, _, _, _ ->
val viewFinder = view as TextureView
val newViewFinderDimens = Size(right - left, bottom - top)
val rotation = getDisplaySurfaceRotation(viewFinder.display)
updateTransform(viewFinder, rotation, bufferDimens, newViewFinderDimens)
}
// Every time the orientation of device changes, recompute layout
displayManager = viewFinder.context
.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)
// Remove the display listeners when the view is detached to avoid
// holding a reference to the View outside of a Fragment.
// NOTE: Even though using a weak reference should take care of this,
// we still try to avoid unnecessary calls to the listener this way.
viewFinder.addOnAttachStateChangeListener(object : View.OnAttachStateChangeListener {
override fun onViewAttachedToWindow(view: View?) {
displayManager.registerDisplayListener(displayListener, null)
}
override fun onViewDetachedFromWindow(view: View?) {
displayManager.unregisterDisplayListener(displayListener)
}
})
}
/** Helper function that fits a camera preview into the given [TextureView] */
private fun updateTransform(textureView: TextureView?, rotation: Int?, newBufferDimens: Size,
newViewFinderDimens: Size) {
// This should not happen anyway, but now the linter knows
val textureView = textureView ?: return
if (rotation == viewFinderRotation &&
Objects.equals(newBufferDimens, bufferDimens) &&
Objects.equals(newViewFinderDimens, viewFinderDimens)) {
// Nothing has changed, no need to transform output again
return
}
if (rotation == null) {
// Invalid rotation - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderRotation = rotation
}
if (newBufferDimens.width == 0 || newBufferDimens.height == 0) {
// Invalid buffer dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
bufferDimens = newBufferDimens
}
if (newViewFinderDimens.width == 0 || newViewFinderDimens.height == 0) {
// Invalid view finder dimens - wait for valid inputs before setting matrix
return
} else {
// Update internal field with new inputs
viewFinderDimens = newViewFinderDimens
}
val matrix = Matrix()
// Compute the center of the view finder
val centerX = viewFinderDimens.width / 2f
val centerY = viewFinderDimens.height / 2f
// Correct preview output to account for display rotation
matrix.postRotate(-viewFinderRotation!!.toFloat(), centerX, centerY)
// Buffers are rotated relative to the device's 'natural' orientation: swap width and height
val bufferRatio = bufferDimens.height / bufferDimens.width.toFloat()
val scaledWidth: Int
val scaledHeight: Int
// Match longest sides together -- i.e. apply center-crop transformation
if (viewFinderDimens.width > viewFinderDimens.height) {
scaledHeight = viewFinderDimens.width
scaledWidth = Math.round(viewFinderDimens.width * bufferRatio)
} else {
scaledHeight = viewFinderDimens.height
scaledWidth = Math.round(viewFinderDimens.height * bufferRatio)
}
// Compute the relative scale value
val xScale = scaledWidth / viewFinderDimens.width.toFloat()
val yScale = scaledHeight / viewFinderDimens.height.toFloat()
// Scale input buffers to fill the view finder
matrix.preScale(xScale, yScale, centerX, centerY)
// Finally, apply transformations to our TextureView
textureView.setTransform(matrix)
}
companion object {
/** Helper function that gets the rotation of a [Display] in degrees */
fun getDisplaySurfaceRotation(display: Display?) = when(display?.rotation) {
Surface.ROTATION_0 -> 0
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> null
}
/**
* Main entrypoint for users of this class: instantiates the adapter and returns an instance
* of [Preview] which automatically adjusts in size and rotation to compensate for
* config changes.
*/
fun build(config: PreviewConfig, viewFinder: TextureView) =
AutoFitPreviewBuilder(config, WeakReference(viewFinder)).useCase
}
}
If configuration is correct (it looks okay to me), then next idea was that maybe converting captured images objects to bitmap might be faulty. Below you can see implementation.
Capture mode uses this function:
fun imageProxyToBitmap(image: ImageProxy): Bitmap {
val buffer: ByteBuffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
Analysis mode uses this function:
fun toBitmapFromImage(image: Image?): Bitmap? {
try {
if (image == null || image.planes[0] == null || image.planes[1] == null || image.planes[2] == null) {
return null
}
val yBuffer = image.planes[0].buffer
val uBuffer = image.planes[1].buffer
val vBuffer = image.planes[2].buffer
val ySize = yBuffer.remaining()
val uSize = uBuffer.remaining()
val vSize = vBuffer.remaining()
val nv21 = ByteArray(ySize + uSize + vSize)
/* U and V are swapped */
yBuffer.get(nv21, 0, ySize)
vBuffer.get(nv21, ySize, vSize)
uBuffer.get(nv21, ySize + vSize, uSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, image.width, image.height, null)
val out = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, yuvImage.width, yuvImage.height), 50, out)
val imageBytes = out.toByteArray()
return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
} catch (e: IllegalStateException) {
Log.e("IllegalStateException", "#ImageUtils.toBitmapFromImage(): Can't read the image file.")
return null
}
}
So, weirdly, on few devices toBitmapFromImage() sometimes comes up upwards, but at the same time (same device) imageProxyToBitmap() returns image in correct rotation - it has to be the image to bitmap functions fault, right?Why is this happening (because capture mode returns image normally) and how to fix this?

Inside onImageCaptureSuccess, get the rotationDegrees and rotate your bitmap by that degree to get the correct orientation.
override fun onImageCaptureSuccess(image: ImageProxy) {
val capturedImageBitmap = image.image?.toBitmap()?.rotate(image.imageInfo.rotationDegrees.toFloat())
mBinding.previewImage.setImageBitmap(capturedImageBitmap)
showPostClickViews()
mCurrentFlow = FLOW_CAMERA
}
toBitmap() and rotate() are extension functions.
fun Image.toBitmap(): Bitmap {
val buffer = planes[0].buffer
buffer.rewind()
val bytes = ByteArray(buffer.capacity())
buffer.get(bytes)
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
fun Bitmap.rotate(degrees: Float): Bitmap =
Bitmap.createBitmap(this, 0, 0, width, height, Matrix().apply { postRotate(degrees) }, true)

CameraX returns the captured image with a rotation value in the callback, which can be used to rotate the image.
https://developer.android.com/reference/androidx/camera/core/ImageCapture.OnImageCapturedListener.html#onCaptureSuccess(androidx.camera.core.ImageProxy,%20int)

For Analyzer UseCases, you have to get rotationDegree coming through analyze method of ImageAnalysis.Analyzer and work accordingly.
Hope it helps!

ARCore with Twilio how is it implemented?

I would like to implement ARCore with Twilio's video call. The documentation says this is possible but I could not figure out how to do it. Can Anyone tell me what I'm doing wrong?
This is my activity:
class MixActivity : AppCompatActivity() {
private lateinit var mArFragment: ArFragment
private lateinit var mVideoView: ArSceneView
private var mScreenVideoTrack: LocalVideoTrack? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_mix)
mArFragment = ar_fragment as ArFragment
mVideoView = mArFragment.arSceneView
mScreenVideoTrack = LocalVideoTrack.create(this, true,
ViewCapturer(mVideoView)
)
} }
This is view:
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:id="#+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent">
<fragment
android:id="#+id/ar_fragment"
android:name="com.google.ar.sceneform.ux.ArFragment"
android:layout_width="match_parent"
android:layout_height="match_parent"/>
</RelativeLayout>
And VideoCapture:
internal class ViewCapturer(private val view: View) : VideoCapturer, PixelCopy.OnPixelCopyFinishedListener {
private val handler = Handler(Looper.getMainLooper())
private var videoCapturerListener: VideoCapturer.Listener? = null
private val started = AtomicBoolean(false)
private lateinit var mViewBitmap: Bitmap
private val viewCapturer = object : Runnable {
override fun run() {
val dropFrame = view.width == 0 || view.height == 0
// Only capture the view if the dimensions have been established
if (!dropFrame) {
// Draw view into bitmap backed canvas
val measuredWidth = View.MeasureSpec.makeMeasureSpec(
view.width,
View.MeasureSpec.EXACTLY
)
val measuredHeight = View.MeasureSpec.makeMeasureSpec(
view.height,
View.MeasureSpec.EXACTLY
)
view.measure(measuredWidth, measuredHeight)
view.layout(0, 0, view.measuredWidth, view.measuredHeight)
mViewBitmap = Bitmap.createBitmap(
view.width, view.height,
Bitmap.Config.ARGB_8888
)
val viewCanvas = Canvas(mViewBitmap)
view.draw(viewCanvas)
// Extract the frame from the bitmap
val bytes = mViewBitmap.byteCount
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener!!.onFrameCaptured(videoFrame)
}
}
// Schedule the next capture
if (started.get()) {
handler.postDelayed(this, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
}
/**
* Returns the list of supported formats for this view capturer. Currently, only supports
* capturing to RGBA_8888 bitmaps.
*
* #return list of supported formats.
*/
override fun getSupportedFormats(): List<VideoFormat> {
val videoFormats = ArrayList<VideoFormat>()
val videoDimensions = VideoDimensions(view.width, view.height)
val videoFormat = VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888)
videoFormats.add(videoFormat)
return videoFormats
}
/**
* Returns true because we are capturing screen content.
*/
override fun isScreencast(): Boolean {
return true
}
/**
* This will be invoked when it is time to start capturing frames.
*
* #param videoFormat the video format of the frames to be captured.
* #param listener capturer listener.
*/
override fun startCapture(videoFormat: VideoFormat, listener: VideoCapturer.Listener) {
// Store the capturer listener
this.videoCapturerListener = listener
this.started.set(true)
// Notify capturer API that the capturer has started
val capturerStarted = handler.postDelayed(
viewCapturer,
VIEW_CAPTURER_FRAMERATE_MS.toLong()
)
this.videoCapturerListener!!.onCapturerStarted(capturerStarted)
}
/**
* Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
*/
override fun stopCapture() {
this.started.set(false)
handler.removeCallbacks(viewCapturer)
}
override fun onPixelCopyFinished(i: Int) {
// Extract the frame from the bitmap
val bytes = mViewBitmap.getByteCount()
val buffer = ByteBuffer.allocate(bytes)
mViewBitmap.copyPixelsToBuffer(buffer)
val array = buffer.array()
val captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime())
// Create video frame
val dimensions = VideoDimensions(view.width, view.height)
val videoFrame = VideoFrame(
array,
dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs
)
// Notify the listener
if (started.get()) {
videoCapturerListener?.onFrameCaptured(videoFrame)
}
if (started.get()) {
handler.postDelayed(viewCapturer, VIEW_CAPTURER_FRAMERATE_MS.toLong())
}
}
companion object {
private val VIEW_CAPTURER_FRAMERATE_MS = 100
}
}
The ARCore part works but the Twilio part does not work.
I referred to another post that talked about it but it was incomplete:
Streaming CustomView ARcore with Twilio video