Develop Reference

Streaming H.264 video stream with Android MediaCodec when user must have the ability to save a frame with original size. Optimized solution?

Firstly, I know using OpenGL ES is more optimized, yet here not a choice.
So, when user would be able to save frames with original size in a H.264 live-stream, there are two scenarios, which would have better performance-wise?
Using MediaCodec in asynchronous mode, get YUV image and show that image on an ImageView. (Does it have overhead compared to second option??)
Using MediaCodec in synchronous mode, set TextureView's surface as MediaCodec input surface and whenever user wants to get screenshot, use textureView.getBitmap()
SurfaceView which cannot retrieve the (frame) bitmap after render because it's an output element so failing, no argument.
Code for option 1:
val frame = ...//ByteArray from server
mediaCodec.setCallback(object : MediaCodec.Callback() {
override fun onInputBufferAvailable(
_codec: MediaCodec,
index: Int
) {
try {
val buffer = _codec.getInputBuffer(index)
buffer?.put(frame)
mediaCodec.queueInputBuffer(
index,
0,
data.size,
0,
0
)
} catch (e: Exception) {
try {
_codec.flush()
} catch (e: Exception) {
}
}
}
override fun onOutputBufferAvailable(
_codec: MediaCodec,
index: Int,
info: MediaCodec.BufferInfo
) {
try {
val info = MediaCodec.BufferInfo()
val outputIndex = index
val image: Image? = _codec.getOutputImage(outputIndex)
if (image == null) {
return
}
val rect = image.cropRect
val yuvImage = YuvImage(
YUV_420_888toNV21(image),
NV21,
rect.width(),
rect.height(),
null
)
val stream = ByteArrayOutputStream()
yuvImage.compressToJpeg(
Rect(0, 0, rect.width(), rect.height()),
100,
stream
)
frameBitmap =
BitmapFactory.decodeByteArray(
stream.toByteArray(),
0,
stream.size()
)
imageView.setImageBitmap(frameBitmap)
_codec.stop()
stream.close()
image.close()
if (outputIndex >= 0) {
_codec.releaseOutputBuffer(outputIndex, false)
}
} catch (e: Exception) {
}
}
override fun onError(
_codec: MediaCodec,
e: MediaCodec.CodecException
) {
}
override fun onOutputFormatChanged(
_codec: MediaCodec,
format: MediaFormat
) {
}
})
try {
mediaCodec.start()
} catch (e: Exception) {
mediaCodec.flush()
}
Code for option 2:
val frame = ...//ByteArray from server
try {
val index = mediaCodec.dequeueInputBuffer(-1)
if (index >= 0) {
val buffer = mediaCodec.getInputBuffer(index)
buffer?.put(frame)
mediaCodec.queueInputBuffer(index, 0, data.size, 0, 0)
val info = MediaCodec.BufferInfo()
val outputIndex = mediaCodec.dequeueOutputBuffer(info, 0)
if (outputIndex >= 0) {
mediaCodec.releaseOutputBuffer(outputIndex, true)
)
lastRenderTime = System.currentTimeMillis()
}
} else {
}
} catch (e: Exception) {
//mediaCodec.flush()
}

PdfRenderer password required or incorrenct error

I work on application that create pdfs and can view them.
The problem is that for api 23->api 28, the application doesn't work as expected, I have a adapter with imageViews of first pdf page of last 5pdfs created by this app, and here is the problem, if I create pdfs without password, all works well, if I create 1 file with password, then all the files come like password protected files. And I get error java.lang.SecurityException: password required or incorrect password.
For android 9 and up all works ok. What can it be? please help
Here is the code:
CoroutineScope(Dispatchers.IO).launch {
try {
if (checkIfPdfIsPasswordProtected(Uri.fromFile(obj), context.contentResolver)) {
bitmapOfPdf = BitmapFactory.decodeResource(context.resources, R.drawable.camera)
} else {
bitmapOfPdf = pdfToBitmap(obj)!!
}
withContext(Dispatchers.Main) {
holder.binding.ivRecent.setImageBitmap(bitmapOfPdf)
}
} catch (e: Exception) {
Log.d("MainActiAdapter", e.message!!)
}
}
private fun checkIfPdfIsPasswordProtected(uri: Uri, contentResolver: ContentResolver): Boolean {
val parcelFileDescriptor = contentResolver.openFileDescriptor(uri, "r")
?: return false
return try {
PdfRenderer(parcelFileDescriptor)
false
} catch (securityException: SecurityException) {
Log.e("CeErroare:", securityException.toString())
true
}catch (e:Exception){
false
}
}
private fun pdfToBitmap(pdfFile: File): Bitmap? {
val renderer =
PdfRenderer(ParcelFileDescriptor.open(pdfFile, ParcelFileDescriptor.MODE_READ_ONLY))
var bitmap: Bitmap? = null
try {
val pageCount = renderer.pageCount
if (pageCount > 0) {
val page = renderer.openPage(0)
val width = 136
val height = 146
bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
page.render(bitmap, null, null, PdfRenderer.Page.RENDER_MODE_FOR_DISPLAY)
page.close()
renderer.close()
}
} catch (ex: Exception) {
ex.printStackTrace()
renderer.close()
}
return bitmap
}

I am trying to read an image from Gallery On app Starting Up that is pre set by the user Android Studio Kotlin

I am Saving the Path to an Image that is set by the user, when the app starts up from cold the image path is read from file and is(should be displayed) but it doesn't want to show. is there a simpler method to reading a set image from gallery?
This is my read File, within this Internal file I am saving a String Path to the Image
fun readDataLocally(context: Context, FileName: String): String {
var str = ""
try {
val inputStream = context.openFileInput(FileName)
if (inputStream != null) {
val inputStreamReader = InputStreamReader(inputStream)
val bufferedReader = BufferedReader(inputStreamReader)
val partialStr = StringBuilder()
var done = false
while (!done) {
val line = bufferedReader.readLine()
done = (line == null)
if (line != null) partialStr.append(line)
}
inputStream.close()
str = partialStr.toString()
}
} catch (e: FileNotFoundException) {
Log.e("Error: ", "file not found: " + e.toString())
} catch (e: IOException) {
Log.e("Error: ", "cannot read file: " + e.toString())
}
Log.d("FileRead",str)
return str
}
This reads the Image path and sets the bit map, any time I call this method it causes the Exception E.
fun readImageFromPath(context: Context, path : String) : Bitmap? {
var bitmap : Bitmap? = null
val uri = Uri.parse(path)
Log.d("path",path)
if (uri != null) {
try {
val parcelFileDescriptor = context.contentResolver.openFileDescriptor(uri, "r")
val fileDescriptor = parcelFileDescriptor?.fileDescriptor
bitmap = BitmapFactory.decodeFileDescriptor(fileDescriptor)
parcelFileDescriptor?.close()
} catch (e: Exception) {
Toast.makeText(context,"Image Read is unavailable please reload",Toast.LENGTH_SHORT).show()
}
}
Log.d("Rip", bitmap.toString())
return bitmap
}
Methods used too call Image to show on Activity Start Up
ImageView.setImageBitmap(readImageFromPath(this,readDataLocally(this, "HomeImage")))
and I tried
val bitmap = readDataLocally(this, "HomeImage")
ManagerScreen_Image.setImageBitmap(readImageFromPath(this, bitmap))
I have the Correct Manifest Permissions.

ImageReader's onImageAvailable method doesn't call and preview shows only 8 frames in slow motion and freezes (Camera2)

I noticed strange behavior on Xiaomi Redmi Note 9 Pro. I tested the application on hundreds of phones but this problem appears only on this device and only when used ImageReader with YUV_420_888 format and 176*144 preview resolution (for example with 320 * 240 or JPEG or without ImageReader as capture surface everything works well). onImageAvailable method doesn't call, preview shows only 8 frames in slow motion and freezes, app slows down. onCaptureCompleted() in CameraCurrentParamsReceiver also calls only 8 times.
I get the smallest resolution by using getMinPreviewSize (176 * 144 for this Xiaomi phone).
const val PREVIEW_IMAGE_FORMAT = ImageFormat.YUV_420_888
const val IMAGE_READER_MAX_SIMULTANEOUS_IMAGES = 4
val previewCaptureCallback = CameraCurrentParamsReceiver(this)
private fun startPreview(cameraDevice: CameraDevice, cameraProperties: CameraProperties)
{
val imageReader = ImageReader.newInstance(cameraProperties.previewSize.width,
cameraProperties.previewSize.height,
PREVIEW_IMAGE_FORMAT,
IMAGE_READER_MAX_SIMULTANEOUS_IMAGES)
this.imageReader = imageReader
bufferedImageConverter = BufferedImageConverter(cameraProperties.previewSize.width, cameraProperties.previewSize.height)
val previewSurface = previewSurface
val previewSurfaceForCamera =
if (previewSurface != null)
{
if (previewSurface.isValid)
{
previewSurface
}
else
{
Log.w(TAG, "Invalid preview surface - camera preview display is not available")
null
}
}
else
{
null
}
val captureSurfaces = listOfNotNull(imageReader.surface, previewSurfaceForCamera)
cameraDevice.createCaptureSession(
captureSurfaces,
object : CameraCaptureSession.StateCallback()
{
override fun onConfigureFailed(cameraCaptureSession: CameraCaptureSession)
{
Log.e(TAG, "onConfigureFailed() cannot configure camera")
if (isCameraOpened(cameraDevice))
{
shutDown("onConfigureFailed")
}
}
override fun onConfigured(cameraCaptureSession: CameraCaptureSession)
{
Log.d(TAG, "onConfigured()")
if (!isCameraOpened(cameraDevice))
{
cameraCaptureSession.close()
shutDown("onConfigured.isCameraOpened")
return
}
captureSession = cameraCaptureSession
try
{
val request = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
captureSurfaces.forEach { request.addTarget(it) }
CameraPreviewRequestInitializer.initializePreviewRequest(request, cameraProperties, controlParams, isControlParamsStrict)
captureRequestBuilder = request
val previewCallback = PreviewFrameHandler(this#Camera2)
this#Camera2.previewFrameHandler = previewCallback
imageReader.setOnImageAvailableListener(previewCallback, previewCallback.backgroundHandler)
cameraCaptureSession.setRepeatingRequest(request.build(), previewCaptureCallback, null)
}
catch (ex: CameraAccessException)
{
Log.e(TAG, "onConfigured() failed with exception", ex)
shutDown("onConfigured.CameraAccessException")
}
}
},
null)
}
private fun chooseCamera(manager: CameraManager): CameraProperties?
{
val cameraIdList = manager.cameraIdList
if (cameraIdList.isEmpty())
{
return null
}
for (cameraId in cameraIdList)
{
val characteristics = manager.getCameraCharacteristics(cameraId)
val facing = characteristics.get(CameraCharacteristics.LENS_FACING)
if (facing != null && facing == CameraCharacteristics.LENS_FACING_BACK)
{
val minPreviewSize = getMinPreviewSize(characteristics)
if (minPreviewSize == null)
{
Log.e(TAG, "chooseCamera() Cannot determine the preview size")
return null
}
Log.d(TAG, "chooseCamera() chosen camera id: $cameraId, preview size: $minPreviewSize")
return CameraProperties(cameraId,
minPreviewSize,
characteristics)
}
}
return null
}
private fun getMinPreviewSize(characteristics: CameraCharacteristics): Size?
{
val map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
if (map == null)
{
Log.e(TAG, "getMinPreviewSize() Map is empty")
return null
}
return map.getOutputSizes(Constants.Camera.PREVIEW_IMAGE_FORMAT)?.minBy { it.width * it.height }
}
PreviewFrameHandler and CameraCurrentParamsReceiver (previewCaptureCallback variable)
private class PreviewFrameHandler(private val parent: Camera2) : ImageReader.OnImageAvailableListener, Handler.Callback
{
val backgroundHandler: Handler
private val backgroundHandlerThread: HandlerThread = HandlerThread("Camera2.PreviewFrame.HandlerThread")
private val mainHandler: Handler = Handler(Looper.getMainLooper(), this)
/**
* Main thread.
*/
init
{
backgroundHandlerThread.start()
backgroundHandler = Handler(backgroundHandlerThread.looper)
}
fun shutDown()
{
backgroundHandlerThread.quit()
mainHandler.removeMessages(0)
}
override fun handleMessage(msg: Message?): Boolean
{
msg ?: return false
parent.cameraFrameListener.onFrame(msg.obj as RGBImage)
return true
}
/**
* Background thread.
*/
private val relativeTimestamp = RelativeTimestamp()
override fun onImageAvailable(reader: ImageReader)
{
var image: Image? = null
try
{
image = reader.acquireNextImage()
image ?: return
val rgbImage = parent.bufferedImageConverter?.convertYUV420spToRGB(image, relativeTimestamp.updateAndGetSeconds(image.timestamp))
rgbImage ?: return
mainHandler.sendMessage(mainHandler.obtainMessage(0, rgbImage))
}
catch (ex: Exception)
{
Log.e(TAG, "onImageAvailable()", ex)
}
finally
{
image?.close()
}
}
private class RelativeTimestamp
{
private var initialNanos = 0L
fun updateAndGetSeconds(currentNanos: Long): Double
{
if (initialNanos == 0L)
{
initialNanos = currentNanos
}
return nanosToSeconds(currentNanos - initialNanos)
}
}
}
/**
* Class used to read current camera params.
*/
private class CameraCurrentParamsReceiver(private val parent: Camera2) : CameraCaptureSession.CaptureCallback()
{
private var isExposureTimeExceptionLogged = false
private var isIsoExceptionLogged = false
override fun onCaptureSequenceAborted(session: CameraCaptureSession, sequenceId: Int)
{
}
override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult)
{
try
{
val exposureTimeNanos = result.get(CaptureResult.SENSOR_EXPOSURE_TIME)
if (exposureTimeNanos != null)
{
parent.currentExposureTimeNanos = exposureTimeNanos
}
}
catch (ex: IllegalArgumentException)
{
if (!isExposureTimeExceptionLogged)
{
isExposureTimeExceptionLogged = true
}
}
try
{
val iso = result.get(CaptureResult.SENSOR_SENSITIVITY)
if (iso != null)
{
parent.currentIso = iso
}
}
catch (ex: IllegalArgumentException)
{
if (!isIsoExceptionLogged)
{
Log.i(TAG, "Cannot get current SENSOR_SENSITIVITY, exception: " + ex.message)
isIsoExceptionLogged = true
}
}
}
override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure)
{
}
override fun onCaptureSequenceCompleted(session: CameraCaptureSession, sequenceId: Int, frameNumber: Long)
{
}
override fun onCaptureStarted(session: CameraCaptureSession, request: CaptureRequest, timestamp: Long, frameNumber: Long)
{
}
override fun onCaptureProgressed(session: CameraCaptureSession, request: CaptureRequest, partialResult: CaptureResult)
{
}
override fun onCaptureBufferLost(session: CameraCaptureSession, request: CaptureRequest, target: Surface, frameNumber: Long)
{
}
}
As I understand something is wrong with preview size but I cannot find correct way how to get this value and the strangest thing is that this problem appears only on this Xiaomi device. Any thoughts?

176x144 is sometimes a problematic resolution for devices. It's really only listed by camera devices because it's sometimes required for recording videos for MMS (multimedia text message) messages. These videos, frankly, look awful, but it's still frequently a requirement by cellular carriers that they work.
But on modern devices with 12 - 50 MP cameras, the camera hardware actually struggles to scale images down to 176x144 from the sensor full resolution (> 20x downscale!), so sometimes certain combinations of sizes can cause problems.
I'd generally recommend not using preview resolutions below 320x240, to minimize issues, and definitely not mix a 176x144 preview with a high-resolution still capture.

Android ContentResolver - save photo metadata

I'm creating a photo making app using Camera2 API.
I would want to set additional information to the photo (date, location), but I'm getting "Uknown URL" exception.
When i comment out contentResolver, photo is saved, but is lacking any additional information, and I need to have access to the location - I will be filtering gallery to only those which are in close proximity.
internal class ImageSaver(
private val image: Image,
private val file: File,
private val watermark: Bitmap,
private val mContext: Context
) : Runnable {
private val saveImageExecutor: Executor = Executors.newSingleThreadExecutor()
override fun run() {
val jpegByteBuffer = image.planes[0].buffer
val jpegByteArray = ByteArray(jpegByteBuffer.remaining())
jpegByteBuffer.get(jpegByteArray)
val width = image.width
val height = image.height
saveImageExecutor.execute {
val date = System.currentTimeMillis()
val location = getLocation(mContext)
val longitude = location?.longitude ?: 0.0
val latitude = location?.latitude ?: 0.0
// watermark
val options = BitmapFactory.Options()
options.inMutable = true
val original =
BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.size, options)
val overlayed = overlay(original, watermark)
val watermarkedByteArrayOS = ByteArrayOutputStream()
overlayed!!.compress(Bitmap.CompressFormat.JPEG, 100, watermarkedByteArrayOS)
val watermarkedByteArray = watermarkedByteArrayOS.toByteArray()
Log.d(TAG, "saving pic meta-data")
val values = ContentValues()
values.put(MediaStore.Images.ImageColumns.TITLE, file.name)
values.put(MediaStore.Images.ImageColumns.DISPLAY_NAME, file.name)
values.put(MediaStore.Images.ImageColumns.DATA, file.path)
values.put(MediaStore.Images.ImageColumns.DATE_TAKEN, date)
values.put(MediaStore.Images.ImageColumns.WIDTH, width)
values.put(MediaStore.Images.ImageColumns.HEIGHT, height)
values.put(MediaStore.Images.ImageColumns.LONGITUDE, longitude)
values.put(MediaStore.Images.ImageColumns.LATITUDE, latitude)
Log.d(TAG, "LON: ${values.get(MediaStore.Images.ImageColumns.LATITUDE)}")
Log.d(TAG, "LAT: ${values.get(MediaStore.Images.ImageColumns.LONGITUDE)}")
var output: FileOutputStream? = null
try {
output = FileOutputStream(file).apply {
write(watermarkedByteArray)
}
} catch (e: IOException) {
Log.e(TAG, e.toString())
} finally {
image.close()
output?.let {
try {
it.close()
} catch (e: IOException) {
Log.e(TAG, e.toString())
}
}
}
mContext.contentResolver.insert(Uri.fromFile(file), values)
}
}
Output:
java.lang.IllegalArgumentException: Unknown URL file:///storage/emulated/0/Android/data/(...)/DCIM/20200610165428492.jpg
at android.content.ContentResolver.insert(ContentResolver.java:1831)
at ...ImageSaver$run$1.run(ImageSaver.kt:86)
What should be the URI? Is there any better way to store location of a photo?