I'm modifying an audio encoder example for write audio and video both. I'm trying to get video raw data from device display by MediaProjection and get audio raw data from Microphone by AudioRecord. Then I will send them to MediaCodec(I will set two Codec instance).
After that I will send the video data and audio data to MediaMux to get a mp4 file. I have any problems:
In result I get a file with perfect video track but terrible audio track. Audio track have plays with intervals at the beginning of track and without pauses but very fast at the end.
When I play video with android MXPlayer all ok, but when I play with PC players (Windows Media or Media Player Classic) audio plays like on MXPlayer but video don't plays - shows only first frame.
Sample of result video:
Part of code:
/**
* Method run of EncoderTask
*/
public void run() {
if (mIsInitialized) {
switch (type) {
case ENCODE_AUDIO_FRAME:
if (!mStopReceived) {
_offerAudioEncoder(mAudioData, presentationTimeNs);
mDrainHandler.postDelayed(new Runnable() {
#Override
public void run() {
mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODER_VIDEO_FRAME));
}
}, DELAY_MILLIS); // 10 milliseconds
}
break;
case ENCODER_VIDEO_FRAME:
if (!mStopReceived) {
encoder._offerVideoEncoder();
mDrainHandler.postDelayed(new Runnable() {
#Override
public void run() {
mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODE_AUDIO_FRAME));
}
}, DELAY_MILLIS); // 10 milliseconds
}
break;
case FINALIZE_ENCODER:
finalizeEncoder();
break;
}
// prevent multiple execution of same task
mIsInitialized = false;
mEncodingServiceQueueLength -= 1;
} else {
Log.e(TAG, "run() called but EncoderTask not initialized");
}
}
public void _offerVideoEncoder() {
Log.d(TAG, "Offer video");
if (mStopReceived) {
closeVideoEncoder();
Log.d(TAG, "Offer video - stop");
} else {
drainEncoder(mVideoEncoder, mVideoBufferInfo, mVideoTrackIndex, false, "video");
Log.d(TAG, "Offer video - drain");
if (mStopReceived) {
closeVideoEncoder();
Log.d(TAG, "Offer video - stop");
}
}
}
public void processAudioFrame() {
long audioPresentationTimeNs = System.nanoTime();
byte[] thisBuffer;
if (mDataBuffer.isEmpty()) {
thisBuffer = new byte[mSamplesPerFrame];
} else {
thisBuffer = mDataBuffer.poll();
}
mReadResult = mAudioRecorder.read(thisBuffer, 0, mSamplesPerFrame);
if (VERBOSE) Log.i(TAG, "FillBuffer real: " + String.valueOf(mBufferWriteIndex)
+ " - " + String.valueOf(mBufferWriteIndex + mReadResult - 1));
if (mReadResult != AudioRecord.ERROR_BAD_VALUE && mReadResult != AudioRecord.ERROR_INVALID_OPERATION) {
mBufferWriteIndex = mBufferWriteIndex + mReadResult - 1;
mTotalFramesWritten++;
if (mAudioEncoder != null) {
mAudioEncoder.offerEncoder(thisBuffer, audioPresentationTimeNs);
}
if (!mIsRecording && mAudioRecorder != null) {
mAudioRecorder.setRecordPositionUpdateListener(null);
mAudioRecorder.release();
mAudioRecorder = null;
Log.i(TAG, "stopped");
}
} else {
Log.e(TAG, "Read error");
}
}
private void _offerAudioEncoder(byte[] input, long presentationTimeNs) {
if (audioBytesReceived == 0) {
mAudioStartTime = presentationTimeNs;
}
mTotalInputAudioFrameCount++;
audioBytesReceived += input.length;
if (mEosSentToAudioEncoder && mStopReceived || input == null) {
logStatistics();
if (mEosReceived) {
Log.d(TAG, "EOS received in offerAudioEncoder");
closeAudioEncoder();
mEosSentToAudioEncoder = true;
if (!mStopReceived) {
prepareAudioEncoder();
} else {
mEncodingService.shutdown();
}
}
return;
}
// transfer previously encoded data to muxer
drainEncoder(mAudioEncoder, mAudioBufferInfo, mAudioTrackIndex, false, "audio");
sendFrameToEncoder(input, presentationTimeNs);// send current frame data to encoder
}
private void sendFrameToEncoder(byte[] input, long presentationTimeNs) {
try {
ByteBuffer[] inputBuffers = mAudioEncoder.getInputBuffers();
int inputBufferIndex = mAudioEncoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(input);
if (mAudioSoftwarePoller != null) {
mAudioSoftwarePoller.recycleInputBuffer(input);
}
long presentationTimeUs = (presentationTimeNs - mAudioStartTime) / 1000; // in microseconds
if (mEosReceived) {
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
closeAudioEncoder();
mEosSentToAudioEncoder = true;
if (mStopReceived) {
mEncodingService.shutdown();
}
} else {
mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, 0);
}
}
} catch (Throwable t) {
t.printStackTrace();
}
}
private void drainEncoder(MediaCodec encoder, MediaCodec.BufferInfo bufferInfo, TrackIndex trackIndex, boolean endOfStream, String type) {
final int TIMEOUT_USEC = 100;
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (!endOfStream) {
if (VERBOSE) Log.d(TAG, "INFO_TRY_AGAIN_LATER " + type + " out of while");
break; // out of while
} else {
if (VERBOSE) Log.d(TAG, "no " + type + " output available, spinning to await EOS");
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
if (VERBOSE) Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED " + type);
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// should happen before receiving buffers, and should only happen once
if (mMuxerStarted) {
throw new RuntimeException("format changed after muxer start");
}
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + newFormat + ".");
// now that we have the Magic Goodies, start the muxer
synchronized (mMuxer) {
trackIndex.index = mMuxer.addTrack(newFormat);
numTracksAdded++;
Log.d(TAG, "Added " + type + " track index: " + trackIndex.index);
if (numTracksAdded == TOTAL_NUM_TRACKS) {
mMuxer.start();
mMuxerStarted = true;
Log.d(TAG, numTracksAdded + " tracks added. Muxer started");
break;
}
}
} else if (encoderStatus < 0) {
Log.w(TAG, "unexpected result from " + type + " encoder.dequeueOutputBuffer: " +
encoderStatus);
} else {
if (encodedData == null) {
if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " encodedData == null");
throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
" was null");
}
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE) Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
if (!mMuxerStarted) {
if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " Muxer not started");
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
synchronized (mMuxer) {
mMuxer.writeSampleData(trackIndex.index, encodedData, bufferInfo);
}
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (!endOfStream) {
Log.w(TAG, "reached end of stream unexpectedly");
} else {
if (VERBOSE) Log.d(TAG, "end of stream reached");
}
break;
}
}
}
long endTime = System.nanoTime();
}
I think in drainEncoder function you missed
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
before
if (encodedData == null)
Related
The whole process is: get video data from Camera, encode and decode it and display it on Surfaceview.
Video encode: I use the MediaCodec Surface to get data.See google/grafika--https://github.com/google/grafika
Video decode: reference EncodeDecodeTest-- https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/EncodeDecodeTest.java
But there is data, it is not displayed on the surfaceview, I suspect that the problem is decoded.
This is my video codec code：
/**
* Drains all pending output from the decoder, and adds it to the circular buffer.
* <p>
*/
public void drainEncoder() {
final int TIMEOUT_USEC = 0; // no timeout -- check for buffers, bail if none
ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
while (true) {
int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = mEncoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// Should happen before receiving buffers, and should only happen once.
// The MediaFormat contains the csd-0 and csd-1 keys, which we'll need
// for MediaMuxer. It's unclear what else MediaMuxer might want, so
// rather than extract the codec-specific data and reconstruct a new
// MediaFormat later, we just grab it here and keep it around.
mEncodedFormat = mEncoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + mEncodedFormat);
} else if (encoderStatus < 0) {
Log.w(TAG, "unexpected result from encoder.dequeueOutputBuffer: " +
encoderStatus);
// let's ignore it
} else {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
" was null");
}
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out when we got the
// INFO_OUTPUT_FORMAT_CHANGED status. The MediaMuxer won't accept
// a single big blob -- it wants separate csd-0/csd-1 chunks --
// so simply saving this off won't work.
if (VERBOSE) Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
mBufferInfo.size = 0;
}
if (mBufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(mBufferInfo.offset);
encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
mEncBuffer.add(encodedData, mBufferInfo.flags,
mBufferInfo.presentationTimeUs);
if (VERBOSE) {
Log.d(TAG, "sent " + mBufferInfo.size + " bytes to muxer, ts=" +
mBufferInfo.presentationTimeUs);
}
}
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// Codec config info. Only expected on first packet. One way to
// handle this is to manually stuff the data into the MediaFormat
// and pass that to configure(). We do that here to exercise the API.
assertFalse(decoderConfigured);
try {
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
} catch (IOException e) {
e.printStackTrace();
}
MediaFormat format =
MediaFormat.createVideoFormat(MIME_TYPE, 80, 80);
format.setByteBuffer("csd-0", encodedData);
decoder.configure(format, mSurfaceView.getHolder().getSurface(),
null, 0);
decoder.start();
decoderInputBuffers = decoder.getInputBuffers();
decoderOutputBuffers = decoder.getOutputBuffers();
decoderConfigured = true;
if (VERBOSE)
Log.d(TAG, "decoder configured (" + mBufferInfo.size + "bytes)");
} else {
// Get a decoder input buffer, blocking until it's available.
assertTrue(decoderConfigured);
int inputBufIndex = decoder.dequeueInputBuffer(-1);
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
inputBuf.clear();
inputBuf.put(encodedData);
decoder.queueInputBuffer(inputBufIndex, 0, mBufferInfo.size,
mBufferInfo.presentationTimeUs, mBufferInfo.flags);
// encoderDone = (mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
// if (VERBOSE) Log.d(TAG, "passed" + mBufferInfo.size + "bytes to decoder"
// + (encoderDone ? "(EOS)" : ""));
}
mEncoder.releaseOutputBuffer(encoderStatus, false);
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.w(TAG, "reached end of stream unexpectedly");
break; // out of while
}
}
}
// Check for output from the decoder. We want to do this on every loop to avoid
// the possibility of stalling the pipeline. We use a short timeout to avoid
// burning CPU if the decoder is hard at work but the next frame isn't quite ready.
//
// If we're decoding to a Surface, we'll get notified here as usual but the
// ByteBuffer references will be null. The data is sent to Surface instead.
if (decoderConfigured) {
int decoderStatus = decoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// The storage associated with the direct ByteBuffer may already be unmapped,
// so attempting to access data through the old output buffer array could
// lead to a native crash.
if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
decoderOutputBuffers = decoder.getOutputBuffers();
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// this happens before the first frame is returned
decoderOutputFormat = decoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "decoder output format changed:" +
decoderOutputFormat);
} else if (decoderStatus < 0) {
fail("unexpected result from deocder.dequeueOutputBuffer:" + decoderStatus);
} else { // decoderStatus >= 0
if (!toSurface) {
ByteBuffer outputFrame = decoderOutputBuffers[decoderStatus];
outputFrame.position(mBufferInfo.offset);
outputFrame.limit(mBufferInfo.offset + mBufferInfo.size);
rawSize += mBufferInfo.size;
if (mBufferInfo.size == 0) {
if (VERBOSE) Log.d(TAG, "got empty frame");
} else {
// if (VERBOSE) Log.d(TAG, "decoded, checking frame" + checkIndex);
// assertEquals("Wrong time stamp", computePresentationTime(checkIndex),
// mBufferInfo.presentationTimeUs);
// if (!checkFrame(checkIndex++, decoderOutputFormat, outputFrame)) {
// badFrames++;
// }
}
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
decoder.releaseOutputBuffer(decoderStatus, false /*render*/);
} else {
if (VERBOSE) Log.d(TAG, "surface decoder given buffer" + decoderStatus +
"(size=" + mBufferInfo.size + ")");
rawSize += mBufferInfo.size;
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
boolean doRender = (mBufferInfo.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't guarantee
// that the texture will be available before the call returns, so we
// need to wait for the onFrameAvailable callback to fire.
decoder.releaseOutputBuffer(decoderStatus, doRender);
// if (doRender) {
// if (VERBOSE) Log.d(TAG, "awaiting frame" + checkIndex);
// assertEquals("Wrong time stamp", computePresentationTime(checkIndex),
// info.presentationTimeUs);
// outputSurface.awaitNewImage();
// outputSurface.drawImage();
// if (!checkSurfaceFrame(checkIndex++)) {
// badFrames++;
// }
}
}
}
}
supplement：This value is always -1 at the time of decoding, which should be passed to the decoder without data.
int decoderStatus = decoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
problem solved
1.Configure the decoder's csd-0 csd-1
2.The data that was previously passed to the decoder is incorrect.
Below is the new code：
public void drainEncoder() {
final int TIMEOUT_USEC = 0; // no timeout -- check for buffers, bail if none
int pos = 0;
ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
while (true) {
int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = mEncoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// Should happen before receiving buffers, and should only happen once.
// The MediaFormat contains the csd-0 and csd-1 keys, which we'll need
// for MediaMuxer. It's unclear what else MediaMuxer might want, so
// rather than extract the codec-specific data and reconstruct a new
// MediaFormat later, we just grab it here and keep it around.
mEncodedFormat = mEncoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + mEncodedFormat);
} else if (encoderStatus < 0) {
Log.w(TAG, "unexpected result from encoder.dequeueOutputBuffer: " +
encoderStatus);
// let's ignore it
} else { // >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
byte[] outData = new byte[mBufferInfo.size];
encodedData.get(outData);
if (arrInfo != null) {
System.arraycopy(outData, 0, arrOutput, pos, outData.length);
pos += outData.length;
} else {
ByteBuffer spsPpsBuffer = ByteBuffer.wrap(outData);
if (spsPpsBuffer.getInt() == 0x00000001) {
arrInfo = new byte[outData.length];
System.arraycopy(outData, 0, arrInfo, 0, outData.length);
findSpsAndPps(arrInfo);
}
}
if (encodedData == null) {
throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
" was null");
}
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, 120, 120);
format.setByteBuffer("csd-0", ByteBuffer.wrap(arrSps));
format.setByteBuffer("csd-1", ByteBuffer.wrap(arrPps));
decoder.configure(format, mSurfaceView.getHolder().getSurface(), null, 0);
decoder.start();
decoderConfigured = true;
} else {
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
int inputBufferIndex = decoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = decoderInputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(arrOutput, 0, pos);
decoder.queueInputBuffer(inputBufferIndex, 0, pos, System.currentTimeMillis(), 0);
}
}
mEncoder.releaseOutputBuffer(encoderStatus, false);
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.w(TAG, "reached end of stream unexpectedly");
break; // out of while
}
}
if (decoderConfigured) {
int decoderStatus = decoder.dequeueOutputBuffer(mBufferInfo, 10000);
Log.i("decoderStatus", "decoderStatus =====" + decoderStatus);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no output from decoder available");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
throw new RuntimeException(
"unexpected result from decoder.dequeueOutputBuffer: " +
decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus +
" (size=" + mBufferInfo.size + ")");
boolean doRender = (mBufferInfo.size != 0);
decoder.releaseOutputBuffer(decoderStatus, doRender);
// decoder.flush(); // reset decoder state
}
}
}
}
I have been trying to modify bigflake's DecodeEditEncode to edit an mp4 file and then encode it back to mp4. I have looked around the internet and couldn't find a solution. I am setting the IFRAME interval and FPS, yet the Muxer still complains about sync frames and is unable to stop.
Could anyone with more experience point out what am I doing wrong?
Thanks a lot in advance :)
private static void encodeToMp4(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
MediaMuxer mediaMuxer = null;
int inputChunk = 0;
int outputCount = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "edit loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer buffer = decoderInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
if (sampleSize < 0) {
inputChunk++;
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
buffer.clear();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, extractor.getSampleTime(), 0);
extractor.advance();
inputChunk++;
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
try {
mediaMuxer = new MediaMuxer(outputFile.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
e.printStackTrace();
}
MediaFormat mediaFormat = extractor.getTrackFormat(0);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 20);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 10);
int trackIndex = mediaMuxer.addTrack(mediaFormat);
mediaMuxer.start();
while (decoderOutputAvailable || encoderOutputAvailable) {
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
encoderOutputAvailable = false;
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
} else { // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
}
// Write the data to the output "file".
if (info.size != 0) {
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
byte[] data = new byte[encodedData.remaining()];
encodedData.get(data);
mediaMuxer.writeSampleData(trackIndex, encodedData, info);
// outputData.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs);
outputCount++;
if (VERBOSE) Log.d(TAG, "encoder output " + info.size + " bytes");
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
decoderOutputAvailable = false;
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
//decoderOutputBuffers = decoder.GetOutputBuffers();
if (VERBOSE) Log.d(TAG, "decoder output buffers changed (we don't care)");
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender) {
// This waits for the image and renders it after it arrives.
if (VERBOSE) Log.d(TAG, "awaiting frame");
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "swapBuffers");
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
// forward decoder EOS to encoder
if (VERBOSE) Log.d(TAG, "signaling input EOS");
if (WORK_AROUND_BUGS) {
// Bail early, possibly dropping a frame.
return;
} else {
encoder.signalEndOfInputStream();
}
}
}
}
}
}
if (inputChunk != outputCount) {
throw new RuntimeException("frame lost: " + inputChunk + " in, " +
outputCount + " out");
}
mediaMuxer.stop();
mediaMuxer.release();
listener.onFinished();
}
Fixed all issues by using the following bigflake test insteadExtractDecodeEditEncodeMuxTest.java. For everyone that is trying to achieve something similar, this sample is life saver. :)
I wanted to run ExtractDecodeEditEncodeMuxTest from https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/ExtractDecodeEditEncodeMuxTest.java.
I have edited the code so that it can take input from sdcard and output to sdcard to simplify the code. But main while loop break after the line outputSurface.awaitNewImage();. And decoding encoding stopped.
private String mInputFile = Environment.getExternalStorageDirectory().getAbsolutePath()+"/dingdong.mp4";
private String mOutputFile = Environment.getExternalStorageDirectory().getAbsolutePath()+"/compressed_output.mp4";
private String mOutputVideoMimeType;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
try {
extractDecodeEditEncodeMux();
} catch (Exception e) {
Log.e(TAG,e.getMessage(),e);
}
}
private void extractDecodeEditEncodeMux() throws Exception {
// Exception that may be thrown during release.
Exception exception = null;
MediaCodecInfo videoCodecInfo = selectCodec(OUTPUT_VIDEO_MIME_TYPE);
if (videoCodecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_VIDEO_MIME_TYPE);
return;
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
MediaCodecInfo audioCodecInfo = selectCodec(OUTPUT_AUDIO_MIME_TYPE);
if (audioCodecInfo == null) {
// Don't fail CTS if they don't have an AAC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE);
return;
}
if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName());
MediaExtractor videoExtractor = null;
MediaExtractor audioExtractor = null;
OutputSurface outputSurface = null;
MediaCodec videoDecoder = null;
MediaCodec audioDecoder = null;
MediaCodec videoEncoder = null;
MediaCodec audioEncoder = null;
MediaMuxer muxer = null;
InputSurface inputSurface = null;
try {
if (mCopyVideo) {
videoExtractor = createExtractor();
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
assertTrue("missing video track in test video", videoInputTrack != -1);
MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack);
// We avoid the device-specific limitations on width and height by using values
// that are multiples of 16, which all tested devices seem to be able to handle.
MediaFormat outputVideoFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, mOutputWidth, mOutputHeight);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
outputVideoFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT);
outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputVideoFormat.setInteger(
MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>();
videoEncoder = createVideoEncoder(
videoCodecInfo, outputVideoFormat, inputSurfaceReference);
inputSurface = new InputSurface(inputSurfaceReference.get());
inputSurface.makeCurrent();
// Create a MediaCodec for the decoder, based on the extractor's format.
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
videoDecoder = createVideoDecoder(inputFormat, outputSurface.getSurface());
}
if (mCopyAudio) {
audioExtractor = createExtractor();
int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor);
assertTrue("missing audio track in test video", audioInputTrack != -1);
MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack);
MediaFormat outputAudioFormat = MediaFormat.createAudioFormat(OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, OUTPUT_AUDIO_CHANNEL_COUNT);
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
audioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat);
// Create a MediaCodec for the decoder, based on the extractor's format.
audioDecoder = createAudioDecoder(inputFormat);
}
// Creates a muxer but do not start or add tracks just yet.
muxer = createMuxer();
doExtractDecodeEditEncodeMux(videoExtractor, audioExtractor, videoDecoder, videoEncoder, audioDecoder, audioEncoder, muxer, inputSurface, outputSurface);
} finally {
if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer");
// Try to release everything we acquired, even if one of the releases fails, in which
// case we save the first exception we got and re-throw at the end (unless something
// other exception has already been thrown). This guarantees the first exception thrown
// is reported as the cause of the error, everything is (attempted) to be released, and
// all other exceptions appear in the logs.
try {
if (videoExtractor != null) {
videoExtractor.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoExtractor", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioExtractor != null) {
audioExtractor.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioExtractor", e);
if (exception == null) {
exception = e;
}
}
try {
if (videoDecoder != null) {
videoDecoder.stop();
videoDecoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (outputSurface != null) {
outputSurface.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing outputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (videoEncoder != null) {
videoEncoder.stop();
videoEncoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoEncoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioDecoder != null) {
audioDecoder.stop();
audioDecoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioEncoder != null) {
audioEncoder.stop();
audioEncoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioEncoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (muxer != null) {
muxer.stop();
muxer.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing muxer", e);
if (exception == null) {
exception = e;
}
}
try {
if (inputSurface != null) {
inputSurface.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing inputSurface", e);
if (exception == null) {
exception = e;
}
}
}
if (exception != null) {
throw exception;
}
}
/**
* Creates an extractor that reads its frames from {#link #mSourceResId}.
*/
private MediaExtractor createExtractor() throws IOException {
MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(mInputFile);
return extractor;
}
/**
* Creates a decoder for the given format, which outputs to the given surface.
*
* #param inputFormat the format of the stream to decode
* #param surface into which to decode the frames
*/
private MediaCodec createVideoDecoder(MediaFormat inputFormat, Surface surface) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, surface, null, 0);
decoder.start();
return decoder;
}
private MediaCodec createVideoEncoder(
MediaCodecInfo codecInfo,
MediaFormat format,
AtomicReference<Surface> surfaceReference) throws IOException {
MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
// Must be called before start() is.
surfaceReference.set(encoder.createInputSurface());
encoder.start();
return encoder;
}
private MediaCodec createAudioDecoder(MediaFormat inputFormat) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private MediaCodec createAudioEncoder(MediaCodecInfo codecInfo, MediaFormat format) throws IOException {
MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
return encoder;
}
private MediaMuxer createMuxer() throws IOException {
return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
MediaFormat inputFormat = extractor.getTrackFormat(index);
mOutputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
mOutputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
return index;
}
}
return -1;
}
private int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isAudioFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private void doExtractDecodeEditEncodeMux(MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface) {
ByteBuffer[] videoDecoderInputBuffers = null;
ByteBuffer[] videoDecoderOutputBuffers = null;
ByteBuffer[] videoEncoderOutputBuffers = null;
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null;
if (mCopyVideo) {
videoDecoderInputBuffers = videoDecoder.getInputBuffers();
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
ByteBuffer[] audioDecoderInputBuffers = null;
ByteBuffer[] audioDecoderOutputBuffers = null;
ByteBuffer[] audioEncoderInputBuffers = null;
ByteBuffer[] audioEncoderOutputBuffers = null;
MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null;
if (mCopyAudio) {
audioDecoderInputBuffers = audioDecoder.getInputBuffers();
audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
audioEncoderInputBuffers = audioEncoder.getInputBuffers();
audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
// We will get these from the decoders when notified of a format change.
MediaFormat decoderOutputVideoFormat = null;
MediaFormat decoderOutputAudioFormat = null;
// We will get these from the encoders when notified of a format change.
MediaFormat encoderOutputVideoFormat = null;
MediaFormat encoderOutputAudioFormat = null;
// We will determine these once we have the output format.
int outputVideoTrack = -1;
int outputAudioTrack = -1;
// Whether things are done on the video side.
boolean videoExtractorDone = false;
boolean videoDecoderDone = false;
boolean videoEncoderDone = false;
// Whether things are done on the audio side.
boolean audioExtractorDone = false;
boolean audioDecoderDone = false;
boolean audioEncoderDone = false;
// The audio decoder output buffer to process, -1 if none.
int pendingAudioDecoderOutputBufferIndex = -1;
boolean muxing = false;
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
int videoEncodedFrameCount = 0;
int audioExtractedFrameCount = 0;
int audioDecodedFrameCount = 0;
int audioEncodedFrameCount = 0;
while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) {
//1: Extract video from file and feed to decoder.
// Do not extract video if we have determined the output format but we are not yet
// ready to mux the frames.
while (mCopyVideo && !videoExtractorDone && (encoderOutputVideoFormat == null || muxing)) {
int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
break;
}
if (VERBOSE)Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
if (VERBOSE)Log.d(TAG, "video extractor: returned buffer of size " + size +" for time "+presentationTime);
if (size >= 0) videoDecoder.queueInputBuffer(decoderInputBufferIndex, 0, size, presentationTime,videoExtractor.getSampleFlags());
videoExtractorDone = !videoExtractor.advance();
if (videoExtractorDone) {
if (VERBOSE) Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(decoderInputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
//3: Poll output frames from the video decoder and feed the encoder.
while (mCopyVideo && !videoDecoderDone && (encoderOutputVideoFormat == null || muxing)) {
int decoderOutputBufferIndex = videoDecoder.dequeueOutputBuffer(videoDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
decoderOutputVideoFormat = videoDecoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "video decoder: output format changed: " + decoderOutputVideoFormat);
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned output buffer: " + decoderOutputBufferIndex);
Log.d(TAG, "video decoder: returned buffer of size " + videoDecoderOutputBufferInfo.size);
}
ByteBuffer decoderOutputBuffer = videoDecoderOutputBuffers[decoderOutputBufferIndex];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)!= 0) {
if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE)Log.d(TAG, "video decoder: returned buffer for time " + videoDecoderOutputBufferInfo.presentationTimeUs);
boolean render = videoDecoderOutputBufferInfo.size != 0;
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
if (render) {
if (VERBOSE) Log.d(TAG, "output surface: await new image");
outputSurface.awaitNewImage();
// Edit the frame and send it to the encoder.
if (VERBOSE) Log.d(TAG, "output surface: draw image");
outputSurface.drawImage();
inputSurface.setPresentationTime(videoDecoderOutputBufferInfo.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
inputSurface.swapBuffers();
if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
}
if ((videoDecoderOutputBufferInfo.flags
& MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: EOS");
videoDecoderDone = true;
videoEncoder.signalEndOfInputStream();
}
videoDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
//6: Poll frames from the video encoder and send them to the muxer.
while (mCopyVideo && !videoEncoderDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer(videoEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
if (outputVideoTrack >= 0) {
fail("video encoder changed its output format again?");
}
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
break;
}
assertTrue("should have added track before processing output", muxing);
if (VERBOSE) {
Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "video encoder: returned buffer of size " + videoEncoderOutputBufferInfo.size);
}
ByteBuffer encoderOutputBuffer = videoEncoderOutputBuffers[encoderOutputBufferIndex];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
// Simply ignore codec config buffers.
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) Log.d(TAG, "video encoder: returned buffer for time " + videoEncoderOutputBufferInfo.presentationTimeUs);
if (videoEncoderOutputBufferInfo.size != 0) {muxer.writeSampleData(outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
if (VERBOSE) Log.d(TAG, "video encoder: EOS");
videoEncoderDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
videoEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
if (!muxing
&& (!mCopyAudio || encoderOutputAudioFormat != null)
&& (!mCopyVideo || encoderOutputVideoFormat != null)) {
if (mCopyVideo) {
Log.d(TAG, "muxer: adding video track.");
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
}
if (mCopyAudio) {
Log.d(TAG, "muxer: adding audio track.");
outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat);
}
Log.d(TAG, "muxer: starting");
muxer.start();
muxing = true;
}
}
// Basic sanity checks.
if (mCopyVideo) {
assertEquals("encoded and decoded video frame counts should match",
videoDecodedFrameCount, videoEncodedFrameCount);
assertTrue("decoded frame count should be less than extracted frame count",
videoDecodedFrameCount <= videoExtractedFrameCount);
}
if (mCopyAudio) {
assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex);
}
// TODO: Check the generated output file.
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
private static boolean isAudioFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("audio/");
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
/**
* Returns the first codec capable of encoding the specified MIME type, or null if no match was
* found.
*/
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
I have found solution. I should not call extractDecodeEditEncodeMux() directly inside onCreate() method. I need to create separate thread and call extractDecodeEditEncodeMux() from that thread.
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
Thread myThread = new Thread(new Runnable() {
#Override
public void run() {
try {
extractDecodeEditEncodeMux();
} catch (Exception e) {
e.printStackTrace();
}
}
});
myThread.start();
}
I'm developing function of translating one video into another with additional effects for each frame. I decided to use opengl-es for applying effects on each frame. My input and output videos are in MP4 using H.264 codec.
I use MediaCodec API (android api 18+) for decoding H.264 into the opengl texture, then draw on the surface using this texture with my shader.
I thought that using MediaCodec with H.264 will do hardware decoding on android and it will be fast. But appeared that it is not.
Recoding small 432x240 15 seconds video consumed 28 seconds of total time!
Please, take a look at my code + profile information and share some advice, critics if I'm doing something wrong.
My code:
private void editVideoFile()
{
if (VERBOSE)
{
Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight);
}
MediaCodec decoder = null;
MediaCodec encoder = null;
InputSurface inputSurface = null;
OutputSurface outputSurface = null;
try
{
File inputFile = new File(FILES_DIR, INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead())
{
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = inVideoTrackIndex = selectTrack(extractor);
if (trackIndex < 0)
{
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat inputFormat = extractor.getTrackFormat(trackIndex);
mWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
mHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
if (VERBOSE)
{
Log.d(TAG, "Video size is " + mWidth + "x" + mHeight);
}
// Create an encoder format that matches the input format. (Might be able to just
// re-use the format used to generate the video, since we want it to be the same.)
MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_BIT_RATE, BIT_RATE));
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_FRAME_RATE, FRAME_RATE));
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL,
getFormatValue(inputFormat,MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL));
try
{
encoder = MediaCodec.createEncoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
// Output filename. Ideally this would use Context.getFilesDir() rather than a
// hard-coded output directory.
String outputPath = new File(OUTPUT_DIR,
"transformed-" + mWidth + "x" + mHeight + ".mp4").toString();
Log.d(TAG, "output file is " + outputPath);
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
//
// We're not actually interested in multiplexing audio. We just want to convert
// the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
try
{
mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
catch (IOException ioe)
{
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
mTrackIndex = -1;
mMuxerStarted = false;
// OutputSurface uses the EGL context created by InputSurface.
try
{
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
decoder.configure(inputFormat, outputSurface.getSurface(), null, 0);
decoder.start();
editVideoData(decoder, outputSurface, inputSurface, encoder);
}
catch (Exception ex)
{
Log.e(TAG, "Error processing", ex);
throw new RuntimeException(ex);
}
finally
{
if (VERBOSE)
{
Log.d(TAG, "shutting down encoder, decoder");
}
if (outputSurface != null)
{
outputSurface.release();
}
if (inputSurface != null)
{
inputSurface.release();
}
if (encoder != null)
{
encoder.stop();
encoder.release();
}
if (decoder != null)
{
decoder.stop();
decoder.release();
}
if (mMuxer != null)
{
mMuxer.stop();
mMuxer.release();
mMuxer = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor)
{
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++)
{
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/"))
{
if (VERBOSE)
{
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Edits a stream of video data.
*/
private void editVideoData(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder)
{
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone)
{
if (VERBOSE)
{
Log.d(TAG, "edit loop");
}
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0)
{
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE)
{
Log.d(TAG, "sent input EOS");
}
}
else
{
if (extractor.getSampleTrackIndex() != inVideoTrackIndex)
{
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + inVideoTrackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE)
{
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
}
else
{
if (VERBOSE)
{
Log.d(TAG, "input buffer not available");
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
while (decoderOutputAvailable || encoderOutputAvailable)
{
// Start by draining any pending output from the encoder. It's important to
// do this before we try to stuff any more data in.
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from encoder available");
}
encoderOutputAvailable = false;
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
encoderOutputBuffers = encoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "encoder output buffers changed");
}
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
if (mMuxerStarted)
{
throw new RuntimeException("format changed twice");
}
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + newFormat);
// now that we have the Magic Goodies, start the muxer
mTrackIndex = mMuxer.addTrack(newFormat);
mMuxer.start();
mMuxerStarted = true;
}
else if (encoderStatus < 0)
{
throw new RuntimeException("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
}
else
{ // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null");
}
if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
{
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE)
{
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
}
info.size = 0;
}
// Write the data to the output "file".
if (info.size != 0)
{
if (!mMuxerStarted)
{
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
mMuxer.writeSampleData(mTrackIndex, encodedData, info);
if (VERBOSE)
{
Log.d(TAG, "sent " + info.size + " bytes to muxer");
}
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != MediaCodec.INFO_TRY_AGAIN_LATER)
{
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone)
{
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from decoder available");
}
decoderOutputAvailable = false;
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
//decoderOutputBuffers = decoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "decoder output buffers changed (we don't care)");
}
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE)
{
Log.d(TAG, "decoder output format changed: " + newFormat);
}
}
else if (decoderStatus < 0)
{
throw new RuntimeException("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
}
else
{ // decoderStatus >= 0
if (VERBOSE)
{
Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
}
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
// This waits for the image and renders it after it arrives.
if (VERBOSE)
{
Log.d(TAG, "awaiting frame");
}
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE)
{
Log.d(TAG, "swapBuffers");
}
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
{
// forward decoder EOS to encoder
if (VERBOSE)
{
Log.d(TAG, "signaling input EOS");
}
if (WORK_AROUND_BUGS)
{
// Bail early, possibly dropping a frame.
return;
}
else
{
encoder.signalEndOfInputStream();
}
}
}
}
}
}
}
And profile information:
Tested on Samsung Galaxy Note3 Intl (Qualcom)
Your issue probably is in how you synchronously wait for events on one single thread, with a nonzero timeout.
You could probably get better throuhput if you lower the timeout. Most of the hardware codecs work with a bit of latency; you can have a good total throughput, but don't expect to have a result (a frame encoded or decoded) immediately.
Ideally, you would use a zero timeout to check all inputs/outputs of both encoder and decoder, and in case there's no free buffers on either points, wait with a nonzero timeout on e.g. encoder output or decoder output.
If you can target Android 5.0, with asynchronous mode in MediaCodec, it's much easier to get this done right. See e.g. https://github.com/mstorsjo/android-decodeencodetest for an example on how to do this. See also https://stackoverflow.com/a/35885471/3115956 for a longer discussion on this issue.
You can also have a look at some similar questions.
I am using MediaCodec API for encoding video and audio into mp4 file. Data encoded in separate threads. Sometimes on some devices audio encoder stops to return any available input buffer and as result MediaMuxer crashes when trying to stop it. Here is my code:
configuring media codec:
public static final String MIME_TYPE_AUDIO = "audio/mp4a-latm";
public static final int SAMPLE_RATE = 44100;
public static final int CHANNEL_COUNT = 1;
public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
public static final int BIT_RATE_AUDIO = 128000;
public static final int SAMPLES_PER_FRAME = 1024 * 2;
public static final int FRAMES_PER_BUFFER = 24;
public static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
public static final int AUDIO_SOURCE = MediaRecorder.AudioSource.MIC;
public static final int MAX_INPUT_SIZE = 16384 * 4;
public static final int MAX_SAMPLE_SIZE = 256 * 1024;
private AudioRecord audioRecord;
private ByteBuffer[] inputBuffers;
private ByteBuffer inputBuffer;
private MediaExtractor mediaExtractor;
private boolean audioSended = false;
private boolean completed = false;
private int sampleCount;
private int iBufferSize;
public AudioEncoderCore(MovieMuxer muxer) throws IOException {
this.muxer = muxer;
bufferInfo = new MediaCodec.BufferInfo();
MediaFormat mediaFormat = null;
mediaFormat = MediaFormat.createAudioFormat(MIME_TYPE_AUDIO, SAMPLE_RATE, CHANNEL_COUNT);
mediaFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
mediaFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_INPUT_SIZE);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE_AUDIO);
encoder = MediaCodec.createEncoderByType(MIME_TYPE_AUDIO);
encoder.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
iBufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER;
// Ensure buffer is adequately sized for the AudioRecord
// object to initialize
int iMinBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT);
if (iBufferSize < iMinBufferSize)
iBufferSize = ((iMinBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
audioRecord = new AudioRecord(AUDIO_SOURCE, SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT, iBufferSize);
audioRecord.startRecording();
}
sending data to encoder:
public void sendDataFromMic(boolean endOfStream) {
if (endOfStream)
Log.d(TAG, "sendDataFromMic end of stream");
long audioPresentationTimeNs;
byte[] mTempBuffer = new byte[SAMPLES_PER_FRAME];
audioPresentationTimeNs = System.nanoTime();
int iReadResult = audioRecord.read(mTempBuffer, 0, mTempBuffer.length);
if (iReadResult == AudioRecord.ERROR_BAD_VALUE || iReadResult == AudioRecord.ERROR_INVALID_OPERATION || iReadResult == 0) {
Log.e(TAG, "audio buffer read error");
} else {
// send current frame data to encoder
try {
if (inputBuffers == null)
inputBuffers = encoder.getInputBuffers();
//Sometimes can't get any available input buffer
int inputBufferIndex = encoder.dequeueInputBuffer(100000);
Log.d(TAG, "inputBufferIndex = " + inputBufferIndex);
if (inputBufferIndex >= 0) {
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(mTempBuffer, 0, iReadResult);
Log.d(TAG, "sending frame to audio encoder " + iReadResult + " bytes");
encoder.queueInputBuffer(inputBufferIndex, 0, iReadResult, audioPresentationTimeNs / 1000,
endOfStream ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
}
} catch (Throwable t) {
Log.e(TAG, "sendFrameToAudioEncoder exception");
t.printStackTrace();
}
}
}
drain encoder:
public void drainEncoder() {
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_NSECS);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no output available, spinning to await EOS");
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
encoderOutputBuffers = encoder.getOutputBuffers();
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder format changed: " + newFormat);
trackIndex = muxer.addTrack(newFormat);
} else if (muxer.isMuxerStarted()) {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
throw new RuntimeException("encoded buffer " + encoderStatus + " was null");
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
Log.d(TAG, "sent " + bufferInfo.size + " bytes to muxer, ts=" +
bufferInfo.presentationTimeUs + " track index=" + trackIndex);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of stream reached");
completed = true;
break; // out of while
}
}
}
}
Bug is stable reproduced on HTC One, Galaxy S3, but all works fine on Huawei Honor 3C.
After source code investigations I found solution. When I dequeue output buffer, the muxer may be not started yet, in that case buffer not released. So here's working code for drain encoder:
public void drainEncoder() {
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
while (true) {
int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_NSECS);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no output available, spinning to await EOS");
break;
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
encoderOutputBuffers = encoder.getOutputBuffers();
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder format changed: " + newFormat);
trackIndex = muxer.addTrack(newFormat);
} else if (muxer.isMuxerStarted()) {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
throw new RuntimeException("encoded buffer " + encoderStatus + " was null");
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
bufferInfo.size = 0;
}
if (bufferInfo.size != 0) {
encodedData.position(bufferInfo.offset);
encodedData.limit(bufferInfo.offset + bufferInfo.size);
muxer.writeSampleData(trackIndex, encodedData, bufferInfo);
Log.d(TAG, "sent " + bufferInfo.size + " bytes to muxer, ts=" +
bufferInfo.presentationTimeUs + " track index=" + trackIndex);
}
encoder.releaseOutputBuffer(encoderStatus, false);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of stream reached");
completed = true;
break; // out of while
}
}
else{
//Muxer not ready, release buffer
encoder.releaseOutputBuffer(encoderStatus, false);
Log.d(TAG, "muxer not ready, skip data");
}
}
}