Develop Reference

I am trying to get code to work that decodes and displays a raw H264 tcp/ip video stream. Can someone diagnose what is wrong?

My goal is to play a raw H264 stream being fed through a tcp/ip port on an Android device (a Samsung S10) using the MediaCodec class. It isn't working, I just see a blank screen.
I have a few guesses to the issue: 1) Do I need to group together NAL units? Right now I feed each unit individually to the MediaCodec. 2) Do I need to make changes to the server? are there variants of H264 that the MediaCodec class cannot handle?
I was able to port FFmpeg to Android studio and got this solution to work. However, it is slow as it uses a software codec. I decided to use MediaCodec to try and use the hardware codec. The code below shows my effort. The codec is initialized in asynchronous mode. I have a separate thread to read and queue up the NAL frames from the tcp socket. Frames are stored in a buffer and if the buffer overflows, then some frames will be discarded. The onInputBufferAvailable codec feeds one NAL unit at a time to the MediaCodec class.
public void initializePlaybackCodec()
{
mWidth = 1536;
mHeight = 864;
MediaFormat decoderFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
try {
codec = MediaCodec.createDecoderByType(MIME_TYPE);
} catch (IOException e) {
Log.e(TAG, "CODEC INIT: Failed to initialize media codec", e);
Toast.makeText(this, "Failed to initialize media codec",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"HERE CODEC INITIALIZED");
final int videoQueueSize = 10;
final Semaphore mutex = new Semaphore(1);
final Semaphore queueData = new Semaphore(0);
final ArrayBlockingQueue<ByteBuffer> queue = new ArrayBlockingQueue<ByteBuffer>(videoQueueSize);
codec.setCallback(new MediaCodec.Callback() {
long reference_epoch = System.currentTimeMillis();
long current_epoch = reference_epoch;
byte[] buffer = new byte[blockSize];
int nextStart = 0;
#Override
public void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
current_epoch = System.currentTimeMillis();
ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId);
android.media.MediaCodecInfo info = codec.getCodecInfo();
//Log.i(TAG,"CODEC CALLBACK: info "+info.getName()+" Encoder: "+info.isEncoder()+" ");
//String[] types = info.getSupportedTypes();
//for (int j = 0; j < types.length; j++) {
// Log.i(TAG,"CODEC CALLBACK: supportedType "+types[j]);
//}
// Read data from the Queue
ByteBuffer b = null;
Log.i(TAG,"CODEC CALLBACK: input");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: queueData acquire interrupted");
codec.stop();
finish();
return;
}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: mutex acquire interrupted");
codec.stop();
finish();
return;
}
try {
b = queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: take interrupted");
codec.stop();
finish();
return;
}
byte[] bb = b.array();
//Log.i(TAG,"CODEC CALLBACK: Contents being sent "+bb[4]/32+" "+bb[4]%32+" "+bb.length);
Log.i(TAG,"CODEC CALLBACK: Contents being sent "+Integer.toHexString(bb[0])+" "+Integer.toHexString(bb[1])+" "+Integer.toHexString(bb[2])+" "+Integer.toHexString(bb[3])+" "+Integer.toHexString(bb[4])+" ");
int ref_idc = bb[4]/32;
int unit_type = bb[4]%32;
//for (int i = 0; i < bb.length && i < 5; ++i) {
// Log.i(TAG, "CODEC CALLBACK: bb["+i+"]="+bb[i]);
//}
mutex.release();
// fill inputBuffer with valid data
//Log.i(TAG,"CODEC CALLBACK: put "+b.remaining()+" "+b.capacity());
inputBuffer.clear();
//Log.i(TAG,"CODEC CALLBACK: before put "+inputBuffer.remaining()+" "+b.position());
b.position(0);
inputBuffer.put(b);
//Log.i(TAG,"CODEC CALLBACK: after put "+inputBuffer.remaining());
//Log.i(TAG,"CODEC CALLBACK: queue "+(current_epoch-reference_epoch)*1000+" "+inputBuffer.capacity()+" "+inputBuffer.remaining());
codec.queueInputBuffer(inputBufferId,0, b.remaining(), (current_epoch-reference_epoch)*1000, 0);
}
#Override
public void onOutputBufferAvailable(MediaCodec mc, int outputBufferId,
MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
// bufferFormat is equivalent to mOutputFormat
// outputBuffer is ready to be processed or rendered.
Log.i(TAG,"CODEC CALLBACK: output");
codec.releaseOutputBuffer(outputBufferId, true);
Log.i(TAG,"CODEC CALLBACK: output done");
}
#Override
public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
// Subsequent data will conform to new format.
// Can ignore if using getOutputFormat(outputBufferId)
//mOutputFormat = format; // option B
Log.i(TAG,"CODEC CALLBACK: output format changed");
}
#Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG,"CODEC CALLBACK: Media Codec Error");
}
});
codec.configure(decoderFormat, m_surface.getHolder().getSurface(), null, 0);
Thread thread = new Thread(){
public void run(){
Socket socket;
InputStream input;
try {
socket = new Socket(mServerAddr, Integer.parseInt(mServerPort));
input = socket.getInputStream();
} catch (IOException e) {
Log.e(TAG, "RLOOP: Failed to open video socket", e);
Toast.makeText(ARActivity.this, "Failed to open video socket",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE SOCKET OPENED");
System.out.println("Socket opened");
byte[] buffer = new byte[blockSize];
java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.allocate(blockSize);
int nextStart = 0;
while (true) {
int size = 1;
try {
size = input.read(buffer,nextStart,blockSize-nextStart);
Log.i(TAG,"RLOOP: Read from video stream "+size+" bytes start="+nextStart);
Log.i(TAG, "RLOOP: First bytes "+buffer[nextStart]+" "+buffer[nextStart+1]+" "+
buffer[nextStart+2]+" "+buffer[nextStart+3]+" "+buffer[nextStart+4]);
if (size==0) {
Log.e(TAG, "RLOOP: Video stream finished");
Toast.makeText(ARActivity.this, "Video stream finished",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
int endPos = 2;
while (endPos > 0) {
endPos = -1;
int zeroCount = 0;
for (int i = nextStart; (i < size+nextStart && endPos < 1); ++i) {
//Log.i(TAG,"Zero count pos "+i+" "+zeroCount);
if (buffer[i]==0) {
++zeroCount;
} else if (buffer[i]==1 && zeroCount > 1) {
if (zeroCount > 3) {
zeroCount = 3;
}
endPos = i-zeroCount;
Log.i(TAG,"RLOOP: Found marker at pos "+(i-zeroCount));
zeroCount = 0;
} else {
zeroCount = 0;
}
}
Log.i(TAG,"RLOOP: State nextStart="+nextStart+" endPos="+endPos+" size="+size);
if (endPos < 0) {
if (size + nextStart == blockSize) {
Log.e(TAG, "RLOOP: Error reading video stream2");
//Toast.makeText(ARActivity.this, "Error reading video stream2",
// Toast.LENGTH_LONG).show();
//finish();
endPos = blockSize;
nextStart = 0;
Log.i(TAG, "RLOOP: BLOCK OVERFLOW " + endPos);
} else {
nextStart = size + nextStart;
}
} else if (endPos==0) {
Log.i(TAG, "RLOOP: BLOCK NOT COMPLETE " + endPos);
//nextStart = size+nextStart;
} else {
Log.i(TAG, "RLOOP: PROCESSING BLOCK " + endPos);
//Log.i(TAG,"BUFFER REMAINING "+byteBuffer.remaining());
//Log.i(TAG,"BUFFER POSITION "+byteBuffer.position());
//System.arraycopy(buffer, 4, buffer, 0, size + nextStart - 4);
//nextStart = nextStart - 4;
//if (nextStart < 0) {
// size = size + nextStart;
// nextStart = 0;
//}
//endPos = endPos-4;
byteBuffer = java.nio.ByteBuffer.allocate(endPos+3);
byteBuffer.put(buffer, 0, endPos);
//byteBuffer = java.nio.ByteBuffer.wrap(buffer, 0, endPos);
//byteBuffer.put(buffer,0, endPos);
Log.i(TAG, "RLOOP: BUFFER REMAINING2 " + byteBuffer.remaining());
Log.i(TAG, "RLOOP: BUFFER POSITION2 " + byteBuffer.position());
Log.i(TAG, "RLOOP: First send bytes " + buffer[0] + " " + buffer[1] + " " +
buffer[2] + " " + buffer[3] + " " + buffer[4]);
//byte[] bb = byteBuffer.array();
Log.i(TAG,"RLOOP: Contents being sent");
//for (int i = 0; i < bb.length && i < 10; ++i) {
// Log.i(TAG, "RLOOP: bb["+i+"]="+bb[i]);
//}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: Mutex interrupted");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE1");
if (queue.size() == videoQueueSize) {
try {
queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue.take interrupted 2");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE2");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queueData.acquire() interrupted 2");
codec.stop();
finish();
return;
}
}
Log.i(TAG,"RLOOP: HERE3");
try {
queue.put(byteBuffer);
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue put interrupted");
codec.stop();
finish();
return;
}
queueData.release();
mutex.release();
if (endPos < size+nextStart) {
System.arraycopy(buffer, endPos, buffer, 0, size + nextStart - endPos);
nextStart = nextStart - endPos;
if (nextStart < 0) {
size = size + nextStart;
nextStart = 0;
}
}
}
}
nextStart = nextStart + size;
} catch (IOException e) {
Log.e(TAG, "RLOOP: Error reading from video stream");
Toast.makeText(ARActivity.this, "Error reading from video stream",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
}
}
};
thread.start();
codec.start();
return;
}
My expected result is to see a video on the android device. My actual result is that the onOutputBufferAvailable function is never called.
I am including a sample debugging output to show some of the NAL units being sent to the MediaCodec class.
2019-06-19 12:22:38.229 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.249 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.251 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.266 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.268 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.281 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.282 3325-3539/com.example.unrealar I/MediaCodec: setCodecState state : 0
2019-06-19 12:22:38.282 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input

i don't see you configuring the Codec. With that i mean sending SPS and PPS with flag BUFFER_FLAG_CODEC_CONFIG.
Such data [CSD] must be marked using the flag BUFFER_FLAG_CODEC_CONFIG in a call to queueInputBuffer
it's documented here.
There are many ways of transferring H264. The most common ones (at least to me i guess) are:
At the beginning of the stream and every time the encoding parameters change.
With every NALU. Every NALU carries it's own set of CSD. You only need to reconfigure if the values change.
SPS and PPS before each key frame and PPS before other slices. It is called AnnexB
As FFMPEG was able to decode the stream, i'd guess that these values are part of the stream.
So i guess you need to parse your H264 stream to determine the SPS and PPS and send a buffer with these values and the BUFFER_FLAG_CODEC_CONFIG to the decoder.
Or if you decide buffer some frames at the begining, before you start decoding, you could also put these values inside your MediaFormat as "csd-0" (SPS) and "csd-1" (PPS)
SPS start with the NALU sequence 0x00 0x00 0x00 0x01 0x67.
PPS start with the NALU sequence 0x00 0x00 0x00 0x01 0x68.

Why android mediaCodec encode low resolution video slower than high resolution

When I use Android media-codec to encode video to h264 format, I found that encode one frame of 1280 * 720 YUV420P raw image need about 10ms but one frame of 640 * 360 YUV420P raw image need about 100ms.
I have tied on serval mobile with MTK or Qualcomm video encoder, all device perform like that.
Can anyone tell me how to fix my following code to speed up or tell the reason why lower resolution cost more time.
private boolean openH264Encoder(int width, int height, int bitrate) throws TranscodeNativeException {
try {
mediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
Log.w(TAG, "codec name " + mediaCodec.getName());
}
for (int i = 0; i < MediaCodecList.getCodecCount(); i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (codecInfo.isEncoder()) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
if (mediaCodec.getName().equalsIgnoreCase(codecInfo.getName())) {
MediaCodecInfo.CodecCapabilities cc = codecInfo.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC);
for (int c : cc.colorFormats) {
Log.i(TAG, String.format("color format 0x%x", c));
if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar == c) {
videoColorFormat = c;
Log.i(TAG, String.format("final color format 0x%x", videoColorFormat));
break;
}
else if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == c) {
videoColorFormat = c;
Log.i(TAG, String.format("find color format 0x%x", videoColorFormat));
}
}
break;
}
}
else {
return false;
}
}
}
if (videoColorFormat == 0) {
Log.e(TAG, "can't find supported color format");
return false;
}
Log.i(TAG, String.format("use color format 0x%x", videoColorFormat));
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, videoColorFormat);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 25);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 25);
mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR);
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
encoderOutputBuffers = mediaCodec.getOutputBuffers();
} catch (Exception e) {
Log.w(TAG, "open h264 encoder failed");
throw new TranscodeNativeException("open h264 encoder failed");
}
return true;
}
public byte[] encodeH264(byte[] data, long ms) {
if (mediaCodec == null)
return null;
if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == videoColorFormat) {
byte[] newData = new byte[data.length];
System.arraycopy(data, 0, newData, 0, data.length * 2 / 3);
for (int i = 0; i < data.length / 6; i++) {
newData[data.length * 2 / 3 + i * 2] = data[data.length * 2 / 3 + i];
newData[data.length * 2 / 3 + i * 2 + 1] = data[data.length * 5 / 6 + i];
}
data = newData;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
try {
ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(20 * 1000);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(data);
mediaCodec.queueInputBuffer(inputBufferIndex, 0, data.length, ms * 1000, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
while (true) {
int encoderStatus = mediaCodec.dequeueOutputBuffer(info, -1);
Log.i(TAG, "encoder status " + encoderStatus);
switch (encoderStatus) {
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.i(TAG, "why tell me try again later?");
return null;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: {
MediaFormat encformat = mediaCodec.getOutputFormat();
Log.i(TAG, "output format changed");
return new byte[0];
}
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
encoderOutputBuffers = mediaCodec.getOutputBuffers();
Log.i(TAG, "output buffer changed");
continue;
default: {
ByteBuffer encoderOutputBuffer = encoderOutputBuffers[encoderStatus];
if (encoderOutputBuffer == null) {
Log.w(TAG, "output buffer is null");
return null;
}
byte[] outData = new byte[info.size];
encoderOutputBuffer.get(outData);
ByteBuffer byteBuffer = ByteBuffer.wrap(outData);
if (spsppsBuffer == null && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
//save sps and pps
if (byteBuffer.getInt() == 0x00000001) {
spsppsBuffer = outData;
}
String d = "";
for (byte i : outData)
d += String.format("0x%02x ", i);
Log.i(TAG, "got sps pps " + d);
mediaCodec.releaseOutputBuffer(encoderStatus, false);
continue;
}
else {
//key frame
if ((outData[4] & 0x1f) == 5) {
byte[] buffer = new byte[outData.length + spsppsBuffer.length];
System.arraycopy(spsppsBuffer, 0, buffer, 0, spsppsBuffer.length);
System.arraycopy(outData, 0, buffer, spsppsBuffer.length, outData.length);
Log.i(TAG, "got key frame");
mediaCodec.releaseOutputBuffer(encoderStatus, false);
return buffer;
}
else {
Log.i(TAG, "got non key frame");
mediaCodec.releaseOutputBuffer(encoderStatus, false);
return outData;
}
}
}
}
}
}
else {
Log.w(TAG, "dequeue input buffer failed, skip one frame");
return new byte[0];
}
}
catch (Exception e) {
e.printStackTrace();
Log.w(TAG, "encode h264 failed");
return null;
}
}

Android MediaCodec : ExtractDecodeEditEncodeMuxTest exit after line :output surface: await new image

I wanted to run ExtractDecodeEditEncodeMuxTest from https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/ExtractDecodeEditEncodeMuxTest.java.
I have edited the code so that it can take input from sdcard and output to sdcard to simplify the code. But main while loop break after the line outputSurface.awaitNewImage();. And decoding encoding stopped.
private String mInputFile = Environment.getExternalStorageDirectory().getAbsolutePath()+"/dingdong.mp4";
private String mOutputFile = Environment.getExternalStorageDirectory().getAbsolutePath()+"/compressed_output.mp4";
private String mOutputVideoMimeType;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
try {
extractDecodeEditEncodeMux();
} catch (Exception e) {
Log.e(TAG,e.getMessage(),e);
}
}
private void extractDecodeEditEncodeMux() throws Exception {
// Exception that may be thrown during release.
Exception exception = null;
MediaCodecInfo videoCodecInfo = selectCodec(OUTPUT_VIDEO_MIME_TYPE);
if (videoCodecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_VIDEO_MIME_TYPE);
return;
}
if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
MediaCodecInfo audioCodecInfo = selectCodec(OUTPUT_AUDIO_MIME_TYPE);
if (audioCodecInfo == null) {
// Don't fail CTS if they don't have an AAC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + OUTPUT_AUDIO_MIME_TYPE);
return;
}
if (VERBOSE) Log.d(TAG, "audio found codec: " + audioCodecInfo.getName());
MediaExtractor videoExtractor = null;
MediaExtractor audioExtractor = null;
OutputSurface outputSurface = null;
MediaCodec videoDecoder = null;
MediaCodec audioDecoder = null;
MediaCodec videoEncoder = null;
MediaCodec audioEncoder = null;
MediaMuxer muxer = null;
InputSurface inputSurface = null;
try {
if (mCopyVideo) {
videoExtractor = createExtractor();
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
assertTrue("missing video track in test video", videoInputTrack != -1);
MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack);
// We avoid the device-specific limitations on width and height by using values
// that are multiples of 16, which all tested devices seem to be able to handle.
MediaFormat outputVideoFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, mOutputWidth, mOutputHeight);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
outputVideoFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT);
outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputVideoFormat.setInteger(
MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>();
videoEncoder = createVideoEncoder(
videoCodecInfo, outputVideoFormat, inputSurfaceReference);
inputSurface = new InputSurface(inputSurfaceReference.get());
inputSurface.makeCurrent();
// Create a MediaCodec for the decoder, based on the extractor's format.
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
videoDecoder = createVideoDecoder(inputFormat, outputSurface.getSurface());
}
if (mCopyAudio) {
audioExtractor = createExtractor();
int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor);
assertTrue("missing audio track in test video", audioInputTrack != -1);
MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack);
MediaFormat outputAudioFormat = MediaFormat.createAudioFormat(OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, OUTPUT_AUDIO_CHANNEL_COUNT);
outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE);
outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties. Request a Surface to use for input.
audioEncoder = createAudioEncoder(audioCodecInfo, outputAudioFormat);
// Create a MediaCodec for the decoder, based on the extractor's format.
audioDecoder = createAudioDecoder(inputFormat);
}
// Creates a muxer but do not start or add tracks just yet.
muxer = createMuxer();
doExtractDecodeEditEncodeMux(videoExtractor, audioExtractor, videoDecoder, videoEncoder, audioDecoder, audioEncoder, muxer, inputSurface, outputSurface);
} finally {
if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer");
// Try to release everything we acquired, even if one of the releases fails, in which
// case we save the first exception we got and re-throw at the end (unless something
// other exception has already been thrown). This guarantees the first exception thrown
// is reported as the cause of the error, everything is (attempted) to be released, and
// all other exceptions appear in the logs.
try {
if (videoExtractor != null) {
videoExtractor.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoExtractor", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioExtractor != null) {
audioExtractor.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioExtractor", e);
if (exception == null) {
exception = e;
}
}
try {
if (videoDecoder != null) {
videoDecoder.stop();
videoDecoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (outputSurface != null) {
outputSurface.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing outputSurface", e);
if (exception == null) {
exception = e;
}
}
try {
if (videoEncoder != null) {
videoEncoder.stop();
videoEncoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing videoEncoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioDecoder != null) {
audioDecoder.stop();
audioDecoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioDecoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (audioEncoder != null) {
audioEncoder.stop();
audioEncoder.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing audioEncoder", e);
if (exception == null) {
exception = e;
}
}
try {
if (muxer != null) {
muxer.stop();
muxer.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing muxer", e);
if (exception == null) {
exception = e;
}
}
try {
if (inputSurface != null) {
inputSurface.release();
}
} catch(Exception e) {
Log.e(TAG, "error while releasing inputSurface", e);
if (exception == null) {
exception = e;
}
}
}
if (exception != null) {
throw exception;
}
}
/**
* Creates an extractor that reads its frames from {#link #mSourceResId}.
*/
private MediaExtractor createExtractor() throws IOException {
MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(mInputFile);
return extractor;
}
/**
* Creates a decoder for the given format, which outputs to the given surface.
*
* #param inputFormat the format of the stream to decode
* #param surface into which to decode the frames
*/
private MediaCodec createVideoDecoder(MediaFormat inputFormat, Surface surface) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, surface, null, 0);
decoder.start();
return decoder;
}
private MediaCodec createVideoEncoder(
MediaCodecInfo codecInfo,
MediaFormat format,
AtomicReference<Surface> surfaceReference) throws IOException {
MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
// Must be called before start() is.
surfaceReference.set(encoder.createInputSurface());
encoder.start();
return encoder;
}
private MediaCodec createAudioDecoder(MediaFormat inputFormat) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private MediaCodec createAudioEncoder(MediaCodecInfo codecInfo, MediaFormat format) throws IOException {
MediaCodec encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
return encoder;
}
private MediaMuxer createMuxer() throws IOException {
return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
MediaFormat inputFormat = extractor.getTrackFormat(index);
mOutputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
mOutputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
return index;
}
}
return -1;
}
private int getAndSelectAudioTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
if (VERBOSE) {
Log.d(TAG, "format for track " + index + " is "
+ getMimeTypeFor(extractor.getTrackFormat(index)));
}
if (isAudioFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private void doExtractDecodeEditEncodeMux(MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface) {
ByteBuffer[] videoDecoderInputBuffers = null;
ByteBuffer[] videoDecoderOutputBuffers = null;
ByteBuffer[] videoEncoderOutputBuffers = null;
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null;
if (mCopyVideo) {
videoDecoderInputBuffers = videoDecoder.getInputBuffers();
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
ByteBuffer[] audioDecoderInputBuffers = null;
ByteBuffer[] audioDecoderOutputBuffers = null;
ByteBuffer[] audioEncoderInputBuffers = null;
ByteBuffer[] audioEncoderOutputBuffers = null;
MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null;
MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null;
if (mCopyAudio) {
audioDecoderInputBuffers = audioDecoder.getInputBuffers();
audioDecoderOutputBuffers = audioDecoder.getOutputBuffers();
audioEncoderInputBuffers = audioEncoder.getInputBuffers();
audioEncoderOutputBuffers = audioEncoder.getOutputBuffers();
audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
}
// We will get these from the decoders when notified of a format change.
MediaFormat decoderOutputVideoFormat = null;
MediaFormat decoderOutputAudioFormat = null;
// We will get these from the encoders when notified of a format change.
MediaFormat encoderOutputVideoFormat = null;
MediaFormat encoderOutputAudioFormat = null;
// We will determine these once we have the output format.
int outputVideoTrack = -1;
int outputAudioTrack = -1;
// Whether things are done on the video side.
boolean videoExtractorDone = false;
boolean videoDecoderDone = false;
boolean videoEncoderDone = false;
// Whether things are done on the audio side.
boolean audioExtractorDone = false;
boolean audioDecoderDone = false;
boolean audioEncoderDone = false;
// The audio decoder output buffer to process, -1 if none.
int pendingAudioDecoderOutputBufferIndex = -1;
boolean muxing = false;
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
int videoEncodedFrameCount = 0;
int audioExtractedFrameCount = 0;
int audioDecodedFrameCount = 0;
int audioEncodedFrameCount = 0;
while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) {
//1: Extract video from file and feed to decoder.
// Do not extract video if we have determined the output format but we are not yet
// ready to mux the frames.
while (mCopyVideo && !videoExtractorDone && (encoderOutputVideoFormat == null || muxing)) {
int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder input buffer");
break;
}
if (VERBOSE)Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex);
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
if (VERBOSE)Log.d(TAG, "video extractor: returned buffer of size " + size +" for time "+presentationTime);
if (size >= 0) videoDecoder.queueInputBuffer(decoderInputBufferIndex, 0, size, presentationTime,videoExtractor.getSampleFlags());
videoExtractorDone = !videoExtractor.advance();
if (videoExtractorDone) {
if (VERBOSE) Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(decoderInputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
// We extracted a frame, let's try something else next.
break;
}
//3: Poll output frames from the video decoder and feed the encoder.
while (mCopyVideo && !videoDecoderDone && (encoderOutputVideoFormat == null || muxing)) {
int decoderOutputBufferIndex = videoDecoder.dequeueOutputBuffer(videoDecoderOutputBufferInfo, TIMEOUT_USEC);
if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
decoderOutputVideoFormat = videoDecoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "video decoder: output format changed: " + decoderOutputVideoFormat);
break;
}
if (VERBOSE) {
Log.d(TAG, "video decoder: returned output buffer: " + decoderOutputBufferIndex);
Log.d(TAG, "video decoder: returned buffer of size " + videoDecoderOutputBufferInfo.size);
}
ByteBuffer decoderOutputBuffer = videoDecoderOutputBuffers[decoderOutputBufferIndex];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)!= 0) {
if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false);
break;
}
if (VERBOSE)Log.d(TAG, "video decoder: returned buffer for time " + videoDecoderOutputBufferInfo.presentationTimeUs);
boolean render = videoDecoderOutputBufferInfo.size != 0;
videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render);
if (render) {
if (VERBOSE) Log.d(TAG, "output surface: await new image");
outputSurface.awaitNewImage();
// Edit the frame and send it to the encoder.
if (VERBOSE) Log.d(TAG, "output surface: draw image");
outputSurface.drawImage();
inputSurface.setPresentationTime(videoDecoderOutputBufferInfo.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "input surface: swap buffers");
inputSurface.swapBuffers();
if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame");
}
if ((videoDecoderOutputBufferInfo.flags
& MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "video decoder: EOS");
videoDecoderDone = true;
videoEncoder.signalEndOfInputStream();
}
videoDecodedFrameCount++;
// We extracted a pending frame, let's try something else next.
break;
}
//6: Poll frames from the video encoder and send them to the muxer.
while (mCopyVideo && !videoEncoderDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer(videoEncoderOutputBufferInfo, TIMEOUT_USEC);
if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
if (VERBOSE) Log.d(TAG, "no video encoder output buffer");
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
break;
}
if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
if (VERBOSE) Log.d(TAG, "video encoder: output format changed");
if (outputVideoTrack >= 0) {
fail("video encoder changed its output format again?");
}
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
break;
}
assertTrue("should have added track before processing output", muxing);
if (VERBOSE) {
Log.d(TAG, "video encoder: returned output buffer: " + encoderOutputBufferIndex);
Log.d(TAG, "video encoder: returned buffer of size " + videoEncoderOutputBufferInfo.size);
}
ByteBuffer encoderOutputBuffer = videoEncoderOutputBuffers[encoderOutputBufferIndex];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer");
// Simply ignore codec config buffers.
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
break;
}
if (VERBOSE) Log.d(TAG, "video encoder: returned buffer for time " + videoEncoderOutputBufferInfo.presentationTimeUs);
if (videoEncoderOutputBufferInfo.size != 0) {muxer.writeSampleData(outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
if (VERBOSE) Log.d(TAG, "video encoder: EOS");
videoEncoderDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false);
videoEncodedFrameCount++;
// We enqueued an encoded frame, let's try something else next.
break;
}
if (!muxing
&& (!mCopyAudio || encoderOutputAudioFormat != null)
&& (!mCopyVideo || encoderOutputVideoFormat != null)) {
if (mCopyVideo) {
Log.d(TAG, "muxer: adding video track.");
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
}
if (mCopyAudio) {
Log.d(TAG, "muxer: adding audio track.");
outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat);
}
Log.d(TAG, "muxer: starting");
muxer.start();
muxing = true;
}
}
// Basic sanity checks.
if (mCopyVideo) {
assertEquals("encoded and decoded video frame counts should match",
videoDecodedFrameCount, videoEncodedFrameCount);
assertTrue("decoded frame count should be less than extracted frame count",
videoDecodedFrameCount <= videoExtractedFrameCount);
}
if (mCopyAudio) {
assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex);
}
// TODO: Check the generated output file.
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
private static boolean isAudioFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("audio/");
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
/**
* Returns the first codec capable of encoding the specified MIME type, or null if no match was
* found.
*/
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}

I have found solution. I should not call extractDecodeEditEncodeMux() directly inside onCreate() method. I need to create separate thread and call extractDecodeEditEncodeMux() from that thread.
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
Thread myThread = new Thread(new Runnable() {
#Override
public void run() {
try {
extractDecodeEditEncodeMux();
} catch (Exception e) {
e.printStackTrace();
}
}
});
myThread.start();
}

Recoding one H.264 video to another using opengl surfaces is very slow on my android

I'm developing function of translating one video into another with additional effects for each frame. I decided to use opengl-es for applying effects on each frame. My input and output videos are in MP4 using H.264 codec.
I use MediaCodec API (android api 18+) for decoding H.264 into the opengl texture, then draw on the surface using this texture with my shader.
I thought that using MediaCodec with H.264 will do hardware decoding on android and it will be fast. But appeared that it is not.
Recoding small 432x240 15 seconds video consumed 28 seconds of total time!
Please, take a look at my code + profile information and share some advice, critics if I'm doing something wrong.
My code:
private void editVideoFile()
{
if (VERBOSE)
{
Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight);
}
MediaCodec decoder = null;
MediaCodec encoder = null;
InputSurface inputSurface = null;
OutputSurface outputSurface = null;
try
{
File inputFile = new File(FILES_DIR, INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead())
{
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = inVideoTrackIndex = selectTrack(extractor);
if (trackIndex < 0)
{
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat inputFormat = extractor.getTrackFormat(trackIndex);
mWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
mHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
if (VERBOSE)
{
Log.d(TAG, "Video size is " + mWidth + "x" + mHeight);
}
// Create an encoder format that matches the input format. (Might be able to just
// re-use the format used to generate the video, since we want it to be the same.)
MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_BIT_RATE, BIT_RATE));
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE,
getFormatValue(inputFormat, MediaFormat.KEY_FRAME_RATE, FRAME_RATE));
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL,
getFormatValue(inputFormat,MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL));
try
{
encoder = MediaCodec.createEncoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
// Output filename. Ideally this would use Context.getFilesDir() rather than a
// hard-coded output directory.
String outputPath = new File(OUTPUT_DIR,
"transformed-" + mWidth + "x" + mHeight + ".mp4").toString();
Log.d(TAG, "output file is " + outputPath);
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
//
// We're not actually interested in multiplexing audio. We just want to convert
// the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
try
{
mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
catch (IOException ioe)
{
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
mTrackIndex = -1;
mMuxerStarted = false;
// OutputSurface uses the EGL context created by InputSurface.
try
{
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
}
catch (IOException iex)
{
throw new RuntimeException(iex);
}
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
decoder.configure(inputFormat, outputSurface.getSurface(), null, 0);
decoder.start();
editVideoData(decoder, outputSurface, inputSurface, encoder);
}
catch (Exception ex)
{
Log.e(TAG, "Error processing", ex);
throw new RuntimeException(ex);
}
finally
{
if (VERBOSE)
{
Log.d(TAG, "shutting down encoder, decoder");
}
if (outputSurface != null)
{
outputSurface.release();
}
if (inputSurface != null)
{
inputSurface.release();
}
if (encoder != null)
{
encoder.stop();
encoder.release();
}
if (decoder != null)
{
decoder.stop();
decoder.release();
}
if (mMuxer != null)
{
mMuxer.stop();
mMuxer.release();
mMuxer = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor)
{
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++)
{
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/"))
{
if (VERBOSE)
{
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Edits a stream of video data.
*/
private void editVideoData(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder)
{
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone)
{
if (VERBOSE)
{
Log.d(TAG, "edit loop");
}
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0)
{
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE)
{
Log.d(TAG, "sent input EOS");
}
}
else
{
if (extractor.getSampleTrackIndex() != inVideoTrackIndex)
{
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + inVideoTrackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE)
{
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
}
else
{
if (VERBOSE)
{
Log.d(TAG, "input buffer not available");
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
while (decoderOutputAvailable || encoderOutputAvailable)
{
// Start by draining any pending output from the encoder. It's important to
// do this before we try to stuff any more data in.
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from encoder available");
}
encoderOutputAvailable = false;
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
encoderOutputBuffers = encoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "encoder output buffers changed");
}
}
else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
if (mMuxerStarted)
{
throw new RuntimeException("format changed twice");
}
MediaFormat newFormat = encoder.getOutputFormat();
Log.d(TAG, "encoder output format changed: " + newFormat);
// now that we have the Magic Goodies, start the muxer
mTrackIndex = mMuxer.addTrack(newFormat);
mMuxer.start();
mMuxerStarted = true;
}
else if (encoderStatus < 0)
{
throw new RuntimeException("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
}
else
{ // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null");
}
if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
{
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (VERBOSE)
{
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
}
info.size = 0;
}
// Write the data to the output "file".
if (info.size != 0)
{
if (!mMuxerStarted)
{
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
mMuxer.writeSampleData(mTrackIndex, encodedData, info);
if (VERBOSE)
{
Log.d(TAG, "sent " + info.size + " bytes to muxer");
}
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != MediaCodec.INFO_TRY_AGAIN_LATER)
{
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone)
{
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
{
// no output available yet
if (VERBOSE)
{
Log.d(TAG, "no output from decoder available");
}
decoderOutputAvailable = false;
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
{
//decoderOutputBuffers = decoder.getOutputBuffers();
if (VERBOSE)
{
Log.d(TAG, "decoder output buffers changed (we don't care)");
}
}
else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
{
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE)
{
Log.d(TAG, "decoder output format changed: " + newFormat);
}
}
else if (decoderStatus < 0)
{
throw new RuntimeException("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
}
else
{ // decoderStatus >= 0
if (VERBOSE)
{
Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
}
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
// This waits for the image and renders it after it arrives.
if (VERBOSE)
{
Log.d(TAG, "awaiting frame");
}
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE)
{
Log.d(TAG, "swapBuffers");
}
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
{
// forward decoder EOS to encoder
if (VERBOSE)
{
Log.d(TAG, "signaling input EOS");
}
if (WORK_AROUND_BUGS)
{
// Bail early, possibly dropping a frame.
return;
}
else
{
encoder.signalEndOfInputStream();
}
}
}
}
}
}
}
And profile information:
Tested on Samsung Galaxy Note3 Intl (Qualcom)

Your issue probably is in how you synchronously wait for events on one single thread, with a nonzero timeout.
You could probably get better throuhput if you lower the timeout. Most of the hardware codecs work with a bit of latency; you can have a good total throughput, but don't expect to have a result (a frame encoded or decoded) immediately.
Ideally, you would use a zero timeout to check all inputs/outputs of both encoder and decoder, and in case there's no free buffers on either points, wait with a nonzero timeout on e.g. encoder output or decoder output.
If you can target Android 5.0, with asynchronous mode in MediaCodec, it's much easier to get this done right. See e.g. https://github.com/mstorsjo/android-decodeencodetest for an example on how to do this. See also https://stackoverflow.com/a/35885471/3115956 for a longer discussion on this issue.
You can also have a look at some similar questions.

Decoding a h.264 stream with MediaCodec

For my Quadcopter project I'm triying to display a raw H.264 stream on my android phone. The input stream comes from a TCP connection.
I did some researche in how decoders work and created the following decoder thread.
The SPS and PPS variables are retreived from the data in an other function, this should be working since I copied it from a working code.
private class PlayerThread extends Thread {
private MediaCodec decoder;
private Surface surface;
public PlayerThread(Surface surface) {
this.surface = surface;
}
#Override
public void run()
{
while(SPS == null || PPS == null || SPS.length == 0 || PPS.length == 0)
{
try
{
Log.e("EncodeDecode", "DECODER_THREAD:: sps,pps not ready yet");
sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
Log.d("EncodeDecode", "DECODER_THREAD:: sps,pps READY");
decoder = MediaCodec.createDecoderByType("video/avc");
MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", 640, 480);
mediaFormat.setByteBuffer("csd-0", ByteBuffer.wrap(SPS));
mediaFormat.setByteBuffer("csd-1", ByteBuffer.wrap(PPS));
decoder.configure(mediaFormat, surface /* surface */, null /* crypto */, 0 /* flags */);
if (decoder == null)
{
Log.e("DecodeActivity", "DECODER_THREAD:: Can't find video info!");
return;
}
decoder.start();
Log.d("EncodeDecode", "DECODER_THREAD:: decoder.start() called");
ByteBuffer[] inputBuffers = decoder.getInputBuffers();
ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
int i = 0;
while(!Thread.interrupted())
{
Frame currentFrame = null;
try
{
Log.d("EncodeDecode", "DECODER_THREAD:: calling queue.take(), if there is no frame in the queue it will wait");
currentFrame = queue.take();
}
catch (InterruptedException e)
{
Log.e("EncodeDecode","DECODER_THREAD:: interrupted while PlayerThread was waiting for the next frame");
e.printStackTrace();
}
if(currentFrame == null)
Log.e("EncodeDecode","DECODER_THREAD:: null frame dequeued");
else
Log.d("EncodeDecode","DECODER_THREAD:: " + currentFrame.id + " no frame dequeued");
if(currentFrame != null && currentFrame.frameData != null && currentFrame.frameData.length != 0)
{
Log.d("EncodeDecode", "DECODER_THREAD:: decoding frame no: " + i + " , dataLength = " + currentFrame.frameData.length);
int inIndex = 0;
while ((inIndex = decoder.dequeueInputBuffer(1)) < 0)
;
if (inIndex >= 0)
{
Log.d("EncodeDecode", "DECODER_THREAD:: sample size: " + currentFrame.frameData.length);
ByteBuffer buffer = inputBuffers[inIndex];
buffer.clear();
buffer.put(currentFrame.frameData);
decoder.queueInputBuffer(inIndex, 0, currentFrame.frameData.length, 33, 0);
BufferInfo info = new BufferInfo();
int outIndex = decoder.dequeueOutputBuffer(info, 100000);
switch (outIndex)
{
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.e("EncodeDecode", "DECODER_THREAD:: INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = decoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.e("EncodeDecode", "DECODER_THREAD:: New format " + decoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.e("EncodeDecode", "DECODER_THREAD:: dequeueOutputBuffer timed out!");
break;
default:
Log.d("EncodeDecode", "DECODER_THREAD:: decoded SUCCESSFULLY!!!");
ByteBuffer outbuffer = outputBuffers[outIndex];
decoder.releaseOutputBuffer(outIndex, true);
break;
}
i++;
}
}
}
decoder.stop();
decoder.release();
}
}
This produces the next error: (i can't post pictures yet :(, so here is the link )
Error_Log
I tested the data stream on a laptop with VLC and it shows the video, so I guess that part is working.
Also the application is creating a surface and it is showing that some data is received. But when it is trying to put the outputbuffer to the surface the application exits whit the error.
I'm new to decoding and this is my first android application, so I dont't realy know were to look. So I'm hoping someone can point me in the right direction.
If you are intrested, my entire code is in this link:
H264_video.java