Develop Reference

Asynchronous MediaCodec not receiving BUFFER_FLAG_END_OF_STREAM in callback

I've been implementing a video encoder which takes raw RGB frame data and encodes/muxes it into a H264 video.
Initially I was using a sync implementation with a while loop based on examples found in https://bigflake.com/mediacodec/, which worked fine.
To improve performance and readability I wanted to switch over to an asynchronous implementation, however I ran into an issue:
calling signalEndOfInputStream often does not set the MediaCodec.BUFFER_FLAG_END_OF_STREAM flag on MediaCodec.BufferInfo
I'm not sure when I should be sending that signal (ideally it would be in the finalize function, however when I tried that I never received the BUFFER_FLAG_END_OF_STREAM flag at all.)
The encoder API looks as follows:
package com.app.encoder
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaFormat
import android.media.MediaMuxer
import android.os.Environment
import android.util.Log
import java.io.File
import java.io.IOException
import java.nio.ByteBuffer
import java.util.*
class VideoEncoder(private val width: Int, private val height: Int, private val frameRate: Int, bitRate: Int, private val fileName: String) : MediaCodec.Callback() {
private val format = MediaFormat.createVideoFormat(MIME_TYPE, width, height)
private var encoder = MediaCodec.createEncoderByType(MIME_TYPE)
private var surface: InputSurface
private lateinit var muxer: MediaMuxer
private var trackIndex: Int = -1
private var muxerStarted = false
private val sync = Object()
private var encoderDone = false
private val pendingBuffers: Queue<Pair<Int, MediaCodec.BufferInfo>> = LinkedList()
companion object {
const val MIME_TYPE = "video/avc"
const val IFRAME_INTERVAL = 10
const val TAG = "VideoEncoder"
}
init {
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL)
encoder.setCallback(this)
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
surface = InputSurface(encoder.createInputSurface())
encoder.start()
}
/**
* Prepares the media muxer
*/
fun init() {
val path = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES)
val file = File(path, fileName)
try {
muxer = MediaMuxer(file.path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
} catch (ioe: IOException) {
throw RuntimeException("Unable to create MediaMuxer", ioe)
}
}
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
return // Unused
}
/**
* Starts the MediaMuxer and processes the queue (if any)
*/
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
Log.d(TAG, "onOutputFormatChanged")
trackIndex = muxer.addTrack(format)
muxer.start()
muxerStarted = true
Log.d(TAG, "MediaMuxer started")
val queueIterator = pendingBuffers.iterator()
while (queueIterator.hasNext()) {
val p = queueIterator.next()
mux(p.first, p.second)
queueIterator.remove()
}
}
override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
mux(index, info)
}
/**
* Pushes encoded data into the muxer, queue's it if the muxer was not yet started
*/
private fun mux(index: Int, info: MediaCodec.BufferInfo) {
if (!muxerStarted) {
pendingBuffers.add(Pair(index, info))
return
}
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
encoder.releaseOutputBuffer(index, false)
return
}
val outputBuffer = encoder.getOutputBuffer(index)!!
if (info.size != 0) {
muxer.writeSampleData(trackIndex, outputBuffer, info)
}
encoder.releaseOutputBuffer(index, false)
// This flag is often not set after signalEndOfInputStream(), causing a timeout in finalize()
if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
synchronized(sync) {
encoderDone = true
sync.notifyAll()
}
}
}
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
// TODO
Log.d(TAG, "onError")
}
/**
* Pushes a frame into the encoder using a GLES20 texture
*/
fun addFrame(frameIndex: Int, data: ByteArray, endOfStream: Boolean) {
if (endOfStream) {
encoder.signalEndOfInputStream()
}
surface.makeCurrent()
surface.generateSurfaceFrame(width, height, ByteBuffer.wrap(data))
surface.setPresentationTime(frameIndex, frameRate)
surface.swapBuffers()
surface.releaseEGLContext()
}
/**
* Awaits for the encoder to finish
*/
fun finalize() {
// encoder.signalEndOfInputStream() <- I would prefer to send the signal here, but that does not work at all
Log.d(TAG, "Finalizing")
val waitUntil = System.currentTimeMillis() + 10000
var timedOut = false
synchronized(sync) {
while (!encoderDone) {
try {
sync.wait(1000)
} catch (_: InterruptedException) {
}
if (System.currentTimeMillis() > waitUntil) {
timedOut = true
break
}
}
}
Log.d(TAG, "Finalized")
release()
if (timedOut) {
throw RuntimeException("Timeout waiting for encoder to complete")
}
}
/**
* Releases any related objects
*/
private fun release() {
encoder.stop()
encoder.release()
surface.release()
if (muxerStarted) {
muxer.stop()
}
muxer.release()
}
}
I instantiate the encoder, call init(), addFrame() all the images and finally wait for the encoder to finish using finalize()
In the above implementation, I have a 50/50 chance that the BUFFER_FLAG_END_OF_STREAM flag is set, so I'm not sure what I'm doing wrong here

Huawei ML Kit Text to speech duration of audio

I have implemented On-device TTS from the Huawei ML Kit in my app and it works well.
Now I would like to find out the duration of the synthesized audio. I want to for example display the remaining time while the audio is playing.
I tried writing the generated audio fragments from the callback to a .pcm file
override fun onAudioAvailable(
taskId: String?,
audioFragment: MLTtsAudioFragment?,
offset: Int,
range: android.util.Pair<Int, Int>?,
bundle: Bundle?
) {
if (taskId != null) {
if (audioFragment != null) {
val fileName = "audio.pcm"
writeToFile(audioFragment.audioData, fileName, true)
}
}
}
fun writeToFile(buffer: ByteArray?, strFileName: String?, append: Boolean) {
if (speechFile == null) {
val pcmFileDir: File = view.getExternalFilesDir("/PCM")!!
speechFile = File(pcmFileDir, strFileName)
}
var raf: RandomAccessFile? = null
var out: FileOutputStream? = null
try {
if (append) {
raf = RandomAccessFile(speechFile, "rw")
speechFile?.length()?.let { raf.seek(it) }
raf.write(buffer)
} else {
out = FileOutputStream(speechFile)
out.write(buffer)
out.flush()
}
} catch (e: IOException) {
e.printStackTrace()
} finally {
try {
raf?.close()
out?.close()
} catch (e: IOException) {
e.printStackTrace()
}
}
}
and getting the duration that way, but the Android MediaPlayer.getDuration() doesn't seem to work with a .pcm file.
Is there a better way to get the duration of the audio? If not then is it possible to calculate the duration of the .pcm file somehow?

ImageReader's onImageAvailable method doesn't call and preview shows only 8 frames in slow motion and freezes (Camera2)

I noticed strange behavior on Xiaomi Redmi Note 9 Pro. I tested the application on hundreds of phones but this problem appears only on this device and only when used ImageReader with YUV_420_888 format and 176*144 preview resolution (for example with 320 * 240 or JPEG or without ImageReader as capture surface everything works well). onImageAvailable method doesn't call, preview shows only 8 frames in slow motion and freezes, app slows down. onCaptureCompleted() in CameraCurrentParamsReceiver also calls only 8 times.
I get the smallest resolution by using getMinPreviewSize (176 * 144 for this Xiaomi phone).
const val PREVIEW_IMAGE_FORMAT = ImageFormat.YUV_420_888
const val IMAGE_READER_MAX_SIMULTANEOUS_IMAGES = 4
val previewCaptureCallback = CameraCurrentParamsReceiver(this)
private fun startPreview(cameraDevice: CameraDevice, cameraProperties: CameraProperties)
{
val imageReader = ImageReader.newInstance(cameraProperties.previewSize.width,
cameraProperties.previewSize.height,
PREVIEW_IMAGE_FORMAT,
IMAGE_READER_MAX_SIMULTANEOUS_IMAGES)
this.imageReader = imageReader
bufferedImageConverter = BufferedImageConverter(cameraProperties.previewSize.width, cameraProperties.previewSize.height)
val previewSurface = previewSurface
val previewSurfaceForCamera =
if (previewSurface != null)
{
if (previewSurface.isValid)
{
previewSurface
}
else
{
Log.w(TAG, "Invalid preview surface - camera preview display is not available")
null
}
}
else
{
null
}
val captureSurfaces = listOfNotNull(imageReader.surface, previewSurfaceForCamera)
cameraDevice.createCaptureSession(
captureSurfaces,
object : CameraCaptureSession.StateCallback()
{
override fun onConfigureFailed(cameraCaptureSession: CameraCaptureSession)
{
Log.e(TAG, "onConfigureFailed() cannot configure camera")
if (isCameraOpened(cameraDevice))
{
shutDown("onConfigureFailed")
}
}
override fun onConfigured(cameraCaptureSession: CameraCaptureSession)
{
Log.d(TAG, "onConfigured()")
if (!isCameraOpened(cameraDevice))
{
cameraCaptureSession.close()
shutDown("onConfigured.isCameraOpened")
return
}
captureSession = cameraCaptureSession
try
{
val request = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
captureSurfaces.forEach { request.addTarget(it) }
CameraPreviewRequestInitializer.initializePreviewRequest(request, cameraProperties, controlParams, isControlParamsStrict)
captureRequestBuilder = request
val previewCallback = PreviewFrameHandler(this#Camera2)
this#Camera2.previewFrameHandler = previewCallback
imageReader.setOnImageAvailableListener(previewCallback, previewCallback.backgroundHandler)
cameraCaptureSession.setRepeatingRequest(request.build(), previewCaptureCallback, null)
}
catch (ex: CameraAccessException)
{
Log.e(TAG, "onConfigured() failed with exception", ex)
shutDown("onConfigured.CameraAccessException")
}
}
},
null)
}
private fun chooseCamera(manager: CameraManager): CameraProperties?
{
val cameraIdList = manager.cameraIdList
if (cameraIdList.isEmpty())
{
return null
}
for (cameraId in cameraIdList)
{
val characteristics = manager.getCameraCharacteristics(cameraId)
val facing = characteristics.get(CameraCharacteristics.LENS_FACING)
if (facing != null && facing == CameraCharacteristics.LENS_FACING_BACK)
{
val minPreviewSize = getMinPreviewSize(characteristics)
if (minPreviewSize == null)
{
Log.e(TAG, "chooseCamera() Cannot determine the preview size")
return null
}
Log.d(TAG, "chooseCamera() chosen camera id: $cameraId, preview size: $minPreviewSize")
return CameraProperties(cameraId,
minPreviewSize,
characteristics)
}
}
return null
}
private fun getMinPreviewSize(characteristics: CameraCharacteristics): Size?
{
val map = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
if (map == null)
{
Log.e(TAG, "getMinPreviewSize() Map is empty")
return null
}
return map.getOutputSizes(Constants.Camera.PREVIEW_IMAGE_FORMAT)?.minBy { it.width * it.height }
}
PreviewFrameHandler and CameraCurrentParamsReceiver (previewCaptureCallback variable)
private class PreviewFrameHandler(private val parent: Camera2) : ImageReader.OnImageAvailableListener, Handler.Callback
{
val backgroundHandler: Handler
private val backgroundHandlerThread: HandlerThread = HandlerThread("Camera2.PreviewFrame.HandlerThread")
private val mainHandler: Handler = Handler(Looper.getMainLooper(), this)
/**
* Main thread.
*/
init
{
backgroundHandlerThread.start()
backgroundHandler = Handler(backgroundHandlerThread.looper)
}
fun shutDown()
{
backgroundHandlerThread.quit()
mainHandler.removeMessages(0)
}
override fun handleMessage(msg: Message?): Boolean
{
msg ?: return false
parent.cameraFrameListener.onFrame(msg.obj as RGBImage)
return true
}
/**
* Background thread.
*/
private val relativeTimestamp = RelativeTimestamp()
override fun onImageAvailable(reader: ImageReader)
{
var image: Image? = null
try
{
image = reader.acquireNextImage()
image ?: return
val rgbImage = parent.bufferedImageConverter?.convertYUV420spToRGB(image, relativeTimestamp.updateAndGetSeconds(image.timestamp))
rgbImage ?: return
mainHandler.sendMessage(mainHandler.obtainMessage(0, rgbImage))
}
catch (ex: Exception)
{
Log.e(TAG, "onImageAvailable()", ex)
}
finally
{
image?.close()
}
}
private class RelativeTimestamp
{
private var initialNanos = 0L
fun updateAndGetSeconds(currentNanos: Long): Double
{
if (initialNanos == 0L)
{
initialNanos = currentNanos
}
return nanosToSeconds(currentNanos - initialNanos)
}
}
}
/**
* Class used to read current camera params.
*/
private class CameraCurrentParamsReceiver(private val parent: Camera2) : CameraCaptureSession.CaptureCallback()
{
private var isExposureTimeExceptionLogged = false
private var isIsoExceptionLogged = false
override fun onCaptureSequenceAborted(session: CameraCaptureSession, sequenceId: Int)
{
}
override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult)
{
try
{
val exposureTimeNanos = result.get(CaptureResult.SENSOR_EXPOSURE_TIME)
if (exposureTimeNanos != null)
{
parent.currentExposureTimeNanos = exposureTimeNanos
}
}
catch (ex: IllegalArgumentException)
{
if (!isExposureTimeExceptionLogged)
{
isExposureTimeExceptionLogged = true
}
}
try
{
val iso = result.get(CaptureResult.SENSOR_SENSITIVITY)
if (iso != null)
{
parent.currentIso = iso
}
}
catch (ex: IllegalArgumentException)
{
if (!isIsoExceptionLogged)
{
Log.i(TAG, "Cannot get current SENSOR_SENSITIVITY, exception: " + ex.message)
isIsoExceptionLogged = true
}
}
}
override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure)
{
}
override fun onCaptureSequenceCompleted(session: CameraCaptureSession, sequenceId: Int, frameNumber: Long)
{
}
override fun onCaptureStarted(session: CameraCaptureSession, request: CaptureRequest, timestamp: Long, frameNumber: Long)
{
}
override fun onCaptureProgressed(session: CameraCaptureSession, request: CaptureRequest, partialResult: CaptureResult)
{
}
override fun onCaptureBufferLost(session: CameraCaptureSession, request: CaptureRequest, target: Surface, frameNumber: Long)
{
}
}
As I understand something is wrong with preview size but I cannot find correct way how to get this value and the strangest thing is that this problem appears only on this Xiaomi device. Any thoughts?

176x144 is sometimes a problematic resolution for devices. It's really only listed by camera devices because it's sometimes required for recording videos for MMS (multimedia text message) messages. These videos, frankly, look awful, but it's still frequently a requirement by cellular carriers that they work.
But on modern devices with 12 - 50 MP cameras, the camera hardware actually struggles to scale images down to 176x144 from the sensor full resolution (> 20x downscale!), so sometimes certain combinations of sizes can cause problems.
I'd generally recommend not using preview resolutions below 320x240, to minimize issues, and definitely not mix a 176x144 preview with a high-resolution still capture.

camera2 api -> mediaCodec -> mediaMuxer -> mp4 file

I'm trying to get an mp4 file. I shoot video using the camera2 api and can save this as an avc file using MediaCodec. But I do not understand how I can redo this code, for encoding into an mp4 file using MediaMuxer. Sorry for my English, this is translated through a translator
private class EncoderCallback : MediaCodec.Callback() {
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
}
override fun onOutputBufferAvailable(
codec: MediaCodec,
index2: Int,
info: MediaCodec.BufferInfo
) {
outPutByteBuffer = mCodec!!.getOutputBuffer(index2)
val outDate = ByteArray(info.size)
outPutByteBuffer!![outDate]
try {
Log.i("EncoderCallBack", " outDate.length : " + outDate.size)
outputStream!!.write(outDate, 0, outDate.size)
} catch (e: IOException) {
e.printStackTrace()
}
mCodec!!.releaseOutputBuffer(index2, false)
}
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
Log.i("EncoderCallBack", "Error: $e")
}
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
Log.i("EncoderCallBack", "encoder output format changed: $format")
}
}
after initializing MediaCodec, I record the video:
var texture: SurfaceTexture = textureViewOver
texture.setDefaultBufferSize(320, 240)
surface = Surface(texture)
builder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
builder.addTarget(surface)
builder.addTarget(mEncoderSurface!!)
mCameraDevice.createCaptureSession(
mutableListOf(surface, mEncoderSurface),
object : CameraCaptureSession.StateCallback() {...

the muxer code is missing:
override fun onOutputBufferAvailable(
codec: MediaCodec,
index2: Int,
info: MediaCodec.BufferInfo
) {
outPutByteBuffer = mCodec!!.getOutputBuffer(index2)
mediaMuxer?.writeSampleData(trackIndex, outPutByteBuffer, info)
mCodec!!.releaseOutputBuffer(index2, false)
}

Android Exoplayer - Different URLs for different video quality and change manually

I am using exoplayer in my application. I am using it to play video from a url. What i am trying to do is that i have three different urls for high,medium and low quality of same the video, and i would like to let the user to be able to change the video quality manually.
{
"lowQualityUrl":"string url",
"mediumQualityUrl":"string url",
"highQualityUrl":"string url"
}
In JWplayer there is an option to add different sources/url for different qualities. Is there something similar that can be done in exoplayer...?
Edit : I don't want to play videos one after another. I just want to switch to a different quality of the same video, like in youtube. But instead of using a single url for the source, what i have are 3 different urls for 3 qualities(low,medium,high) of the same video.

I found a solution or rather a workaround for the issue. I am using the exoplayer inside a recylcerview. This code might need some optimization.
I got this idea from another answer which was under this question,which had this github link, but i think the author deleted it.
What i did was create a class for keeping all the urls for a particular video. Then showing a Spinner above the exoplayer, and when user selects a particular quality then i prepare the exoplayer with the new URL, and then seekto to the previously playing position. You can ignore the StringUtils methods.
VideoPlayerConfig.kt
object VideoPlayerConfig {
//Minimum Video you want to buffer while Playing
val MIN_BUFFER_DURATION = 3000
//Max Video you want to buffer during PlayBack
val MAX_BUFFER_DURATION = 5000
//Min Video you want to buffer before start Playing it
val MIN_PLAYBACK_START_BUFFER = 1500
//Min video You want to buffer when user resumes video
val MIN_PLAYBACK_RESUME_BUFFER = 5000
}
VideoQuality.kt
You can change this class according to your need. I need to store exactly 3 urls for low,medium and high quality urls. And i needed to show them in that order as well in the spinner.
class VideoQuality {
private val videoQualityUrls = HashMap<String, String>()
companion object {
val LOW = getStringResource(R.string.low)
val MEDIUM = getStringResource(R.string.medium)
val HIGH = getStringResource(R.string.high)
}
val qualityArray
get() = arrayListOf<String>().apply {
if (hasQuality(LOW)) add(LOW)
if (hasQuality(MEDIUM)) add(MEDIUM)
if (hasQuality(HIGH)) add(HIGH)
}
var defaultVideoQuality: String? = HIGH
var lowQuality: String?
set(value) {
setVideoQualityUrl(LOW, value)
}
get() = videoQualityUrls[LOW] ?: ""
var mediumQuality: String?
set(value) {
setVideoQualityUrl(MEDIUM, value)
}
get() = videoQualityUrls[MEDIUM] ?: ""
var highQuality: String?
set(value) {
setVideoQualityUrl(HIGH, value)
}
get() = videoQualityUrls[HIGH] ?: ""
private fun setVideoQualityUrl(quality: String?, url: String?) {
if (url != null && quality != null) {
videoQualityUrls[quality] = url
}
}
private fun hasQuality(quality: String?): Boolean {
if (videoQualityUrls[quality] == null) {
return false
}
return true
}
fun getVideoQualityUrl(quality: String?): String? {
return videoQualityUrls[quality]
}
}
Methods to implement for the exoplayer
private fun initializePlayer() {
if (exoPlayer == null) {
val loadControl = DefaultLoadControl.Builder()
.setBufferDurationsMs(2 * VideoPlayerConfig.MIN_BUFFER_DURATION, 2 * VideoPlayerConfig.MAX_BUFFER_DURATION, VideoPlayerConfig.MIN_PLAYBACK_START_BUFFER, VideoPlayerConfig.MIN_PLAYBACK_RESUME_BUFFER)
.createDefaultLoadControl()
//Create a default TrackSelector
val videoTrackSelectionFactory = AdaptiveTrackSelection.Factory()
val trackSelector = DefaultTrackSelector(videoTrackSelectionFactory)
exoPlayer = ExoPlayerFactory.newSimpleInstance(itemView.context, DefaultRenderersFactory(itemView.context), trackSelector, loadControl)
exoPlayer!!.addListener(PlayEventListener())
val videoQualityInfo:VideoQuality = videoListVideoDataHolderData!!.videoQualityUrls //Just an object that i created and stored in a dataHolder for this view.
val url = videoQualityInfo.getVideoQualityUrl(videoQualityInfo.defaultVideoQuality) ?: ""
preparePlayer(url)
}
}
private fun preparePlayer(url: String) {
if (url.isNotEmpty()) {
val mediaSource = buildMediaSource(StringUtils.makeHttpUrl(url))
exoPlayer?.prepare(mediaSource)
videoView.player = exoPlayer
} else {
Log.d(APPTAG, "NO DEFAULT URL")
}
}
private fun buildMediaSource(url: String): ProgressiveMediaSource {
val mUri: Uri = Uri.parse(url)
val dataSourceFactory = DefaultDataSourceFactory(
itemView.context,
Util.getUserAgent(itemView.context, getStringResource(R.string.app_name))
)
val videoSource = ProgressiveMediaSource.Factory(dataSourceFactory)
.createMediaSource(mUri)
return videoSource
}
And then in the Spinner/QualitySelector's OnItemSelectedListener
videoQualitySpinner.onItemSelectedListener = object : AdapterView.OnItemSelectedListener {
override fun onNothingSelected(parent: AdapterView<*>?) {
}
override fun onItemSelected(parent: AdapterView<*>?, view: View?, position: Int, id: Long) {
val currentTime = exoPlayer?.currentPosition
val isReadyToPlay = exoPlayer?.playWhenReady
val urlToBuild = when (videoQualityUrls.qualityArray[position]) {
VideoQuality.LOW -> videoQualityUrls.lowQuality
VideoQuality.MEDIUM -> videoQualityUrls.mediumQuality
else -> videoQualityUrls.highQuality
}
Log.d(APPTAG, "VIDEO DETAILS :::: ${currentTime} ${isReadyToPlay} ${urlToBuild}")
if (!urlToBuild.isNullOrEmpty()) {
val mediaSource = buildMediaSource(StringUtils.makeHttpUrl(urlToBuild))
exoPlayer?.prepare(mediaSource)
exoPlayer?.playWhenReady = isReadyToPlay ?: false
exoPlayer?.seekTo(currentTime ?: 0)
}
}
}