Develop Reference

How to get YUV frames from Dji SDK?

I am working on a Android app which render data form the Drone , I am able to render the raw frames on the SurfaceView after decoding it with the DjiVideoSteamDecoder class. I want the Yuv frames form the decoder class i.e DjiVideoSteamDecoder. The problem is I am not getting the continious yuv frames from the Yuv listner. As per the Docs we need to set
DJIVideoStreamDecoder.getInstance().changeSurface(null) it works for we frames and then it stops producing the YUv data .Let me past the decoding class which gives me the YUV frames.
private void initCodec() {
if (width == 0 || height == 0) {
return;
}
Log.e("OBJ","codec inside initcodec"+codec);
if (codec != null) {
releaseCodec();
}
loge("initVideoDecoder----------------------------------------------------------");
loge("initVideoDecoder video width = " + width + " height = " + height);
// create the media format
MediaFormat format = MediaFormat.createVideoFormat(VIDEO_ENCODING_FORMAT, width, height);
if (surface == null) {
Log.i(TAG,"initVideoDecoder: yuv output");
// The surface is null, which means that the yuv data is needed, so the color format should
// be set to YUV420.
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar);
} else {
Log.i(TAG,"initVideoDecoder: display");
// The surface is set, so the color format should be set to format surface.
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
}
try {
// Create the codec instance.
codec = MediaCodec.createDecoderByType(VIDEO_ENCODING_FORMAT);
Log.i(TAG, "initVideoDecoder create: " + (codec == null));
// Configure the codec. What should be noted here is that the hardware decoder would not output
// any yuv data if OnReceive surface is configured into, which mean that if you want the yuv frames, you
// should set "null" surface when calling the "configure" method of MediaCodec.
codec.configure(format, surface, null, 0);
Log.i(TAG, "initVideoDecoder configure");
// codec.configure(format, null, null, 0);
if (codec == null) {
Log.e(TAG, "Can't find video info!");
return;
}
// Start the codec
codec.start();
Log.i(TAG, "initVideoDecoder start");
// Get the input and output buffers of hardware decoder
inputBuffers = codec.getInputBuffers();
outputBuffers = codec.getOutputBuffers();
Log.i(TAG, "initVideoDecoder get buffers");
} catch (Exception e) {
Log.i(TAG, "init codec failed, do it again: "+ e.getMessage());
if (e instanceof MediaCodec.CodecException) {
MediaCodec.CodecException ce = (MediaCodec.CodecException) e;
ce.printStackTrace();
}
e.printStackTrace();
}
}
And decoder class :
private void decodeFrame() throws Exception {
DJIFrame inputFrame = frameQueue.poll();
if (inputFrame == null) {
return;
}
if (codec == null) {
if (dataHandler != null && !dataHandler.hasMessages(MSG_INIT_CODEC)) {
dataHandler.sendEmptyMessage(MSG_INIT_CODEC);
}
}
int inIndex = -1;
// Get input buffer index of the MediaCodec.
for (int i = 0; i < CODEC_DEQUEUE_INPUT_QUEUE_RETRY && inIndex < 0; i ++) {
try {
inIndex = codec.dequeueInputBuffer(0);
} catch (IllegalStateException e) {
logd(TAG, "decodeFrame: dequeue input: " + e);
codec.stop();
codec.reset();
initCodec();
e.printStackTrace();
}
}
logd(TAG, "decodeFrame: index=" + inIndex);
Log.e("OBJ","index "+inIndex);
// Decode the frame using MediaCodec
if (inIndex >= 0) {
ByteBuffer buffer = inputBuffers[inIndex];
buffer.clear();
buffer.rewind();
buffer.put(inputFrame.videoBuffer);
inputFrame.fedIntoCodecTime = System.currentTimeMillis();
long queueingDelay = inputFrame.getQueueDelay();
logd("input frame delay: " + queueingDelay);
// Feed the frame data to the decoder.
codec.queueInputBuffer(inIndex, 0, inputFrame.size, inputFrame.pts, 0);
hasIFrameInCodec = true;
// Get the output data from the decoder.
int outIndex = -1;
outIndex = codec.dequeueOutputBuffer(bufferInfo, 0);
Log.e("OBJ","Outputindex"+outIndex);
logd(TAG, "decodeFrame: outIndex: " + outIndex);
if (outIndex >= 0) {
if ( surface == null && yuvDataListener != null) {
//if (yuvDataListener != null) {
// If the surface is null, the yuv data should be get from the buffer and invoke the callback.
logd("decodeFrame: need callback");
ByteBuffer yuvDataBuf = outputBuffers[outIndex];
yuvDataBuf.position(bufferInfo.offset);
yuvDataBuf.limit(bufferInfo.size - bufferInfo.offset);
final byte[] bytes = new byte[bufferInfo.size - bufferInfo.offset];
yuvDataBuf.get(bytes);
callbackHandler.post(new Runnable() {
#Override
public void run() {
yuvDataListener.onYuvDataReceived(bytes, width, height);
}
});
}
// All the output buffer must be release no matter whether the yuv data is output or
// not, so that the codec can reuse the buffer.
codec.releaseOutputBuffer(outIndex, true);
} else if (outIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// The output buffer set is changed. So the decoder should be reinitialized and the
// output buffers should be retrieved.
long curTime = System.currentTimeMillis();
bufferChangedQueue.addLast(curTime);
if (bufferChangedQueue.size() >= 10) {
long headTime = bufferChangedQueue.pollFirst();
if (curTime - headTime < 1000) {
// reset decoder
loge("Reset decoder. Get INFO_OUTPUT_BUFFERS_CHANGED more than 10 times within OnReceive second.");
bufferChangedQueue.clear();
dataHandler.removeCallbacksAndMessages(null);
dataHandler.sendEmptyMessage(MSG_INIT_CODEC);
return;
}
}
if (outputBuffers == null) {
return;
}
outputBuffers = codec.getOutputBuffers();
} else if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
loge("format changed, color: " + codec.getOutputFormat().getInteger(MediaFormat.KEY_COLOR_FORMAT));
}
}
}

MediaMuxer: No sync frames for video track

I have been trying to modify bigflake's DecodeEditEncode to edit an mp4 file and then encode it back to mp4. I have looked around the internet and couldn't find a solution. I am setting the IFRAME interval and FPS, yet the Muxer still complains about sync frames and is unable to stop.
Could anyone with more experience point out what am I doing wrong?
Thanks a lot in advance :)
private static void encodeToMp4(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
MediaMuxer mediaMuxer = null;
int inputChunk = 0;
int outputCount = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "edit loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer buffer = decoderInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
if (sampleSize < 0) {
inputChunk++;
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
buffer.clear();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, extractor.getSampleTime(), 0);
extractor.advance();
inputChunk++;
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
try {
mediaMuxer = new MediaMuxer(outputFile.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
e.printStackTrace();
}
MediaFormat mediaFormat = extractor.getTrackFormat(0);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 20);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 10);
int trackIndex = mediaMuxer.addTrack(mediaFormat);
mediaMuxer.start();
while (decoderOutputAvailable || encoderOutputAvailable) {
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
encoderOutputAvailable = false;
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
} else { // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
}
// Write the data to the output "file".
if (info.size != 0) {
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
byte[] data = new byte[encodedData.remaining()];
encodedData.get(data);
mediaMuxer.writeSampleData(trackIndex, encodedData, info);
// outputData.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs);
outputCount++;
if (VERBOSE) Log.d(TAG, "encoder output " + info.size + " bytes");
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
decoderOutputAvailable = false;
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
//decoderOutputBuffers = decoder.GetOutputBuffers();
if (VERBOSE) Log.d(TAG, "decoder output buffers changed (we don't care)");
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender) {
// This waits for the image and renders it after it arrives.
if (VERBOSE) Log.d(TAG, "awaiting frame");
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "swapBuffers");
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
// forward decoder EOS to encoder
if (VERBOSE) Log.d(TAG, "signaling input EOS");
if (WORK_AROUND_BUGS) {
// Bail early, possibly dropping a frame.
return;
} else {
encoder.signalEndOfInputStream();
}
}
}
}
}
}
if (inputChunk != outputCount) {
throw new RuntimeException("frame lost: " + inputChunk + " in, " +
outputCount + " out");
}
mediaMuxer.stop();
mediaMuxer.release();
listener.onFinished();
}

Fixed all issues by using the following bigflake test insteadExtractDecodeEditEncodeMuxTest.java. For everyone that is trying to achieve something similar, this sample is life saver. :)

Recoding one H.264 video to another using opengl surfaces is very slow on my android

I'm developing function of translating one video into another with additional effects for each frame. I decided to use opengl-es for applying effects on each frame. My input and output videos are in MP4 using H.264 codec.
I use MediaCodec API (android api 18+) for decoding H.264 into the opengl texture, then draw on the surface using this texture with my shader.
I thought that using MediaCodec with H.264 will do hardware decoding on android and it will be fast. But appeared that it is not.
Recoding small 432x240 15 seconds video consumed 28 seconds of total time!
Please, take a look at my code + profile information and share some advice, critics if I'm doing something wrong.
My code:
private void editVideoFile()
{
if (VERBOSE)
{
Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight);
}
MediaCodec decoder = null;
MediaCodec encoder = null;
InputSurface inputSurface = null;
OutputSurface outputSurface = null;
try
{
File inputFile = new File(FILES_DIR, INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead())
{
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = inVideoTrackIndex = selectTrack(extractor);
if (trackIndex < 0)
{
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat inputFormat = extractor.getTrackFormat(trackIndex);
mWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
mHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
if (VERBOSE)
{
Log.d(TAG, "Video size is " + mWidth + "x" + mHeight);
}
// Create an encoder format that matches the input format. (Might be able to just
// re-use the format used to generate the video, since we want it to be the same.)
MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_BIT_RATE, BIT_RATE));
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_FRAME_RATE, FRAME_RATE));
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL,
getFormatValue(inputFormat,MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL));
try
{
encoder = MediaCodec.createEncoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
// Output filename. Ideally this would use Context.getFilesDir() rather than a
// hard-coded output directory.
String outputPath = new File(OUTPUT_DIR,
"transformed-" + mWidth + "x" + mHeight + ".mp4").toString();
Log.d(TAG, "output file is " + outputPath);
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
//
// We're not actually interested in multiplexing audio. We just want to convert
// the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
try
{
mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
catch (IOException ioe)
{
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
mTrackIndex = -1;
mMuxerStarted = false;
// OutputSurface uses the EGL context created by InputSurface.
try
{
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
decoder.configure(inputFormat, outputSurface.getSurface(), null, 0);
decoder.start();
editVideoData(decoder, outputSurface, inputSurface, encoder);
}
catch (Exception ex)
{
Log.e(TAG, "Error processing", ex);
throw new RuntimeException(ex);
}
finally
{
if (VERBOSE)
{
Log.d(TAG, "shutting down encoder, decoder");
}
if (outputSurface != null)
{
outputSurface.release();
}
if (inputSurface != null)
{
inputSurface.release();
}
if (encoder != null)
{
encoder.stop();
encoder.release();
}
if (decoder != null)
{
decoder.stop();
decoder.release();
}
if (mMuxer != null)
{
mMuxer.stop();
mMuxer.release();
mMuxer = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor)
{
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++)
{
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/"))
{
if (VERBOSE)
{
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Edits a stream of video data.
*/
private void editVideoData(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder)
{
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone)
{
if (VERBOSE)
{
Log.d(TAG, "edit loop");
}
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0)
{
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE)
{
Log.d(TAG, "sent input EOS");
}
}
else
{
if (extractor.getSampleTrackIndex() != inVideoTrackIndex)
{
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + inVideoTrackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE)
{
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
}
else
{
if (VERBOSE)
{
Log.d(TAG, "input buffer not available");
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
while (decoderOutputAvailable || encoderOutputAvailable)
{
// Start by draining any pending output from the encoder. It's important to
// do this before we try to stuff any more data in.
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from encoder available");
}
encoderOutputAvailable = false;
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
encoderOutputBuffers = encoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "encoder output buffers changed");
}
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
if (mMuxerStarted)
{
throw new RuntimeException("format changed twice");
}
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + newFormat);
// now that we have the Magic Goodies, start the muxer
mTrackIndex = mMuxer.addTrack(newFormat);
mMuxer.start();
mMuxerStarted = true;
}
else if (encoderStatus < 0)
{
throw new RuntimeException("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
}
else
{ // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null");
}
if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
{
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE)
{
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
}
info.size = 0;
}
// Write the data to the output "file".
if (info.size != 0)
{
if (!mMuxerStarted)
{
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
mMuxer.writeSampleData(mTrackIndex, encodedData, info);
if (VERBOSE)
{
Log.d(TAG, "sent " + info.size + " bytes to muxer");
}
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != MediaCodec.INFO_TRY_AGAIN_LATER)
{
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone)
{
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from decoder available");
}
decoderOutputAvailable = false;
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
//decoderOutputBuffers = decoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "decoder output buffers changed (we don't care)");
}
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE)
{
Log.d(TAG, "decoder output format changed: " + newFormat);
}
}
else if (decoderStatus < 0)
{
throw new RuntimeException("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
}
else
{ // decoderStatus >= 0
if (VERBOSE)
{
Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
}
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
// This waits for the image and renders it after it arrives.
if (VERBOSE)
{
Log.d(TAG, "awaiting frame");
}
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE)
{
Log.d(TAG, "swapBuffers");
}
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
{
// forward decoder EOS to encoder
if (VERBOSE)
{
Log.d(TAG, "signaling input EOS");
}
if (WORK_AROUND_BUGS)
{
// Bail early, possibly dropping a frame.
return;
}
else
{
encoder.signalEndOfInputStream();
}
}
}
}
}
}
}
And profile information:
Tested on Samsung Galaxy Note3 Intl (Qualcom)

Your issue probably is in how you synchronously wait for events on one single thread, with a nonzero timeout.
You could probably get better throuhput if you lower the timeout. Most of the hardware codecs work with a bit of latency; you can have a good total throughput, but don't expect to have a result (a frame encoded or decoded) immediately.
Ideally, you would use a zero timeout to check all inputs/outputs of both encoder and decoder, and in case there's no free buffers on either points, wait with a nonzero timeout on e.g. encoder output or decoder output.
If you can target Android 5.0, with asynchronous mode in MediaCodec, it's much easier to get this done right. See e.g. https://github.com/mstorsjo/android-decodeencodetest for an example on how to do this. See also https://stackoverflow.com/a/35885471/3115956 for a longer discussion on this issue.
You can also have a look at some similar questions.

Problems with audio and video Muxing

I'm modifying an audio encoder example for write audio and video both. I'm trying to get video raw data from device display by MediaProjection and get audio raw data from Microphone by AudioRecord. Then I will send them to MediaCodec(I will set two Codec instance).
After that I will send the video data and audio data to MediaMux to get a mp4 file. I have any problems:
In result I get a file with perfect video track but terrible audio track. Audio track have plays with intervals at the beginning of track and without pauses but very fast at the end.
When I play video with android MXPlayer all ok, but when I play with PC players (Windows Media or Media Player Classic) audio plays like on MXPlayer but video don't plays - shows only first frame.
Sample of result video:
Part of code:
/**
* Method run of EncoderTask
*/
public void run() {
if (mIsInitialized) {
switch (type) {
case ENCODE_AUDIO_FRAME:
if (!mStopReceived) {
_offerAudioEncoder(mAudioData, presentationTimeNs);
mDrainHandler.postDelayed(new Runnable() {
#Override
public void run() {
mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODER_VIDEO_FRAME));
}
}, DELAY_MILLIS); // 10 milliseconds
}
break;
case ENCODER_VIDEO_FRAME:
if (!mStopReceived) {
encoder._offerVideoEncoder();
mDrainHandler.postDelayed(new Runnable() {
#Override
public void run() {
mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODE_AUDIO_FRAME));
}
}, DELAY_MILLIS); // 10 milliseconds
}
break;
case FINALIZE_ENCODER:
finalizeEncoder();
break;
}
// prevent multiple execution of same task
mIsInitialized = false;
mEncodingServiceQueueLength -= 1;
} else {
Log.e(TAG, "run() called but EncoderTask not initialized");
}
}
public void _offerVideoEncoder() {
Log.d(TAG, "Offer video");
if (mStopReceived) {
closeVideoEncoder();
Log.d(TAG, "Offer video - stop");
} else {
drainEncoder(mVideoEncoder, mVideoBufferInfo, mVideoTrackIndex, false, "video");
Log.d(TAG, "Offer video - drain");
if (mStopReceived) {
closeVideoEncoder();
Log.d(TAG, "Offer video - stop");
}
}
}
public void processAudioFrame() {
long audioPresentationTimeNs = System.nanoTime();
byte[] thisBuffer;
if (mDataBuffer.isEmpty()) {
thisBuffer = new byte[mSamplesPerFrame];
} else {
thisBuffer = mDataBuffer.poll();
}
mReadResult = mAudioRecorder.read(thisBuffer, 0, mSamplesPerFrame);
if (VERBOSE) Log.i(TAG, "FillBuffer real: " + String.valueOf(mBufferWriteIndex)
+ " - " + String.valueOf(mBufferWriteIndex + mReadResult - 1));
if (mReadResult != AudioRecord.ERROR_BAD_VALUE && mReadResult != AudioRecord.ERROR_INVALID_OPERATION) {
mBufferWriteIndex = mBufferWriteIndex + mReadResult - 1;
mTotalFramesWritten++;
if (mAudioEncoder != null) {
mAudioEncoder.offerEncoder(thisBuffer, audioPresentationTimeNs);
}
if (!mIsRecording && mAudioRecorder != null) {
mAudioRecorder.setRecordPositionUpdateListener(null);
mAudioRecorder.release();
mAudioRecorder = null;
Log.i(TAG, "stopped");
}
} else {
Log.e(TAG, "Read error");
}
}
private void _offerAudioEncoder(byte[] input, long presentationTimeNs) {
if (audioBytesReceived == 0) {
mAudioStartTime = presentationTimeNs;
}
mTotalInputAudioFrameCount++;
audioBytesReceived += input.length;
if (mEosSentToAudioEncoder && mStopReceived || input == null) {
logStatistics();
if (mEosReceived) {
Log.d(TAG, "EOS received in offerAudioEncoder");
closeAudioEncoder();
mEosSentToAudioEncoder = true;
if (!mStopReceived) {
prepareAudioEncoder();
} else {
mEncodingService.shutdown();
}
}
return;
}
// transfer previously encoded data to muxer
drainEncoder(mAudioEncoder, mAudioBufferInfo, mAudioTrackIndex, false, "audio");
sendFrameToEncoder(input, presentationTimeNs);// send current frame data to encoder
}
private void sendFrameToEncoder(byte[] input, long presentationTimeNs) {
try {
ByteBuffer[] inputBuffers = mAudioEncoder.getInputBuffers();
int inputBufferIndex = mAudioEncoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(input);
if (mAudioSoftwarePoller != null) {
mAudioSoftwarePoller.recycleInputBuffer(input);
}
long presentationTimeUs = (presentationTimeNs - mAudioStartTime) / 1000; // in microseconds
if (mEosReceived) {
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
closeAudioEncoder();
mEosSentToAudioEncoder = true;
if (mStopReceived) {
mEncodingService.shutdown();
}
} else {
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, 0);
}
}
} catch (Throwable t) {
t.printStackTrace();
}
}
private void drainEncoder(MediaCodec encoder, MediaCodec.BufferInfo bufferInfo, TrackIndex trackIndex, boolean endOfStream, String type) {
final int TIMEOUT_USEC = 100;
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (!endOfStream) {
if (VERBOSE) Log.d(TAG, "INFO_TRY_AGAIN_LATER " + type + " out of while");
break; // out of while
} else {
if (VERBOSE) Log.d(TAG, "no " + type + " output available, spinning to await EOS");
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
if (VERBOSE) Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED " + type);
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// should happen before receiving buffers, and should only happen once
if (mMuxerStarted) {
throw new RuntimeException("format changed after muxer start");
}
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + newFormat + ".");
// now that we have the Magic Goodies, start the muxer
synchronized (mMuxer) {
trackIndex.index = mMuxer.addTrack(newFormat);
numTracksAdded++;
Log.d(TAG, "Added " + type + " track index: " + trackIndex.index);
if (numTracksAdded == TOTAL_NUM_TRACKS) {
mMuxer.start();
mMuxerStarted = true;
Log.d(TAG, numTracksAdded + " tracks added. Muxer started");
break;
}
}
} else if (encoderStatus < 0) {
Log.w(TAG, "unexpected result from " + type + " encoder.dequeueOutputBuffer: " +
encoderStatus);
} else {
if (encodedData == null) {
if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " encodedData == null");
throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
" was null");
}
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE) Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
if (!mMuxerStarted) {
if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " Muxer not started");
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
synchronized (mMuxer) {
mMuxer.writeSampleData(trackIndex.index, encodedData, bufferInfo);
}
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (!endOfStream) {
Log.w(TAG, "reached end of stream unexpectedly");
} else {
if (VERBOSE) Log.d(TAG, "end of stream reached");
}
break;
}
}
}
long endTime = System.nanoTime();
}

I think in drainEncoder function you missed
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
before
if (encodedData == null)

MediaCodec hasn't any available input buffer

I am using MediaCodec API for encoding video and audio into mp4 file. Data encoded in separate threads. Sometimes on some devices audio encoder stops to return any available input buffer and as result MediaMuxer crashes when trying to stop it. Here is my code:
configuring media codec:
public static final String MIME_TYPE_AUDIO = "audio/mp4a-latm";
public static final int SAMPLE_RATE = 44100;
public static final int CHANNEL_COUNT = 1;
public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
public static final int BIT_RATE_AUDIO = 128000;
public static final int SAMPLES_PER_FRAME = 1024 * 2;
public static final int FRAMES_PER_BUFFER = 24;
public static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
public static final int AUDIO_SOURCE = MediaRecorder.AudioSource.MIC;
public static final int MAX_INPUT_SIZE = 16384 * 4;
public static final int MAX_SAMPLE_SIZE = 256 * 1024;
private AudioRecord audioRecord;
private ByteBuffer[] inputBuffers;
private ByteBuffer inputBuffer;
private MediaExtractor mediaExtractor;
private boolean audioSended = false;
private boolean completed = false;
private int sampleCount;
private int iBufferSize;
public AudioEncoderCore(MovieMuxer muxer) throws IOException {
this.muxer = muxer;
bufferInfo = new MediaCodec.BufferInfo();
MediaFormat mediaFormat = null;
mediaFormat = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, SAMPLE_RATE, CHANNEL_COUNT);
mediaFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
mediaFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_INPUT_SIZE);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE_AUDIO);
encoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
encoder.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
iBufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER;
// Ensure buffer is adequately sized for the AudioRecord
// object to initialize
int iMinBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
if (iBufferSize < iMinBufferSize)
iBufferSize = ((iMinBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
audioRecord = new AudioRecord(AUDIO_SOURCE, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, iBufferSize);
audioRecord.startRecording();
}
sending data to encoder:
public void sendDataFromMic(boolean endOfStream) {
if (endOfStream)
Log.d(TAG, "sendDataFromMic end of stream");
long audioPresentationTimeNs;
byte[] mTempBuffer = new byte[SAMPLES_PER_FRAME];
audioPresentationTimeNs = System.nanoTime();
int iReadResult = audioRecord.read(mTempBuffer, 0, mTempBuffer.length);
if (iReadResult == AudioRecord.ERROR_BAD_VALUE || iReadResult == AudioRecord.ERROR_INVALID_OPERATION || iReadResult == 0) {
Log.e(TAG, "audio buffer read error");
} else {
// send current frame data to encoder
try {
if (inputBuffers == null)
inputBuffers = encoder.getInputBuffers();
//Sometimes can't get any available input buffer
int inputBufferIndex = encoder.dequeueInputBuffer(100000);
Log.d(TAG, "inputBufferIndex = " + inputBufferIndex);
if (inputBufferIndex >= 0) {
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(mTempBuffer, 0, iReadResult);
Log.d(TAG, "sending frame to audio encoder " + iReadResult + " bytes");
encoder.queueInputBuffer(inputBufferIndex, 0, iReadResult, audioPresentationTimeNs / 1000,
endOfStream ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
}
} catch (Throwable t) {
Log.e(TAG, "sendFrameToAudioEncoder exception");
t.printStackTrace();
}
}
}
drain encoder:
public void drainEncoder() {
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_NSECS);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no output available, spinning to await EOS");
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
encoderOutputBuffers = encoder.getOutputBuffers();
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder format changed: " + newFormat);
trackIndex = muxer.addTrack(newFormat);
} else if (muxer.isMuxerStarted()) {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
throw new RuntimeException("encoded buffer " + encoderStatus + " was null");
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
Log.d(TAG, "sent " + bufferInfo.size + " bytes to muxer, ts=" +
bufferInfo.presentationTimeUs + " track index=" + trackIndex);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of stream reached");
completed = true;
break; // out of while
}
}
}
}
Bug is stable reproduced on HTC One, Galaxy S3, but all works fine on Huawei Honor 3C.

After source code investigations I found solution. When I dequeue output buffer, the muxer may be not started yet, in that case buffer not released. So here's working code for drain encoder:
public void drainEncoder() {
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_NSECS);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no output available, spinning to await EOS");
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
encoderOutputBuffers = encoder.getOutputBuffers();
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder format changed: " + newFormat);
trackIndex = muxer.addTrack(newFormat);
} else if (muxer.isMuxerStarted()) {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
throw new RuntimeException("encoded buffer " + encoderStatus + " was null");
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
Log.d(TAG, "sent " + bufferInfo.size + " bytes to muxer, ts=" +
bufferInfo.presentationTimeUs + " track index=" + trackIndex);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of stream reached");
completed = true;
break; // out of while
}
}
else{
//Muxer not ready, release buffer
encoder.releaseOutputBuffer(encoderStatus, false);
Log.d(TAG, "muxer not ready, skip data");
}
}
}