Develop Reference

video steganography using mediacodec

I need to create videos with data hidden in them. i managed to extract video frames using mediacodec decoder as NV21 buffer and save them, then i create mp4 file from frames using mediacodec encoder.
the class below is responsible for saving frame files if we are in encode process or check the value if we want to extract data from stego-video.
public class ExtractMpegFramesBufferDecoder {
private static final String TAG = "ExtractMpegFramesDec";
private static final boolean VERBOSE = true; // lots of logging
// where to find files (note: requires WRITE_EXTERNAL_STORAGE permission)
private File STORE_FRAME_DIRECTORY;
private String INPUT_FILE;
private int frameRate; // stop extracting after this many
private int saveWidth;
private int saveHeight;
private int decodeCount;
private Handler _progressBarHandler;
private int duration;
//
private int MAX_FRAMES;
private boolean fromDecode;
//
public ExtractMpegFramesBufferDecoder(File storeFrameDirectory, String inputVideoPath, int frameRate
, int saveWidth, int saveHeight
, double duration, int rotation
, Handler _progressBarHandler) {
this.STORE_FRAME_DIRECTORY = storeFrameDirectory;
this.INPUT_FILE = inputVideoPath;
this.frameRate = frameRate;
this.saveWidth = saveWidth;
this.saveHeight = saveHeight;
this._progressBarHandler = _progressBarHandler;
this.duration = (int) duration;
}
/**
* Tests extraction from an MP4 to a series of PNG files.
* <p>
* We scale the video to 640x480 for the PNG just to demonstrate that we can scale the
* video with the GPU. If the input video has a different aspect ratio, we could preserve
* it by adjusting the GL viewport to get letterboxing or pillarboxing, but generally if
* you're extracting frames you don't want black bars.
*/
public void extractMpegFrames(int maxFrame, boolean fromDecode) throws IOException {
MediaCodec decoder = null;
MediaExtractor extractor = null;
MAX_FRAMES = maxFrame;
this.fromDecode = fromDecode;
try {
File inputFile = new File(INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead()) {
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = selectTrack(extractor);
if (trackIndex < 0) {
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat format = extractor.getTrackFormat(trackIndex);
if (VERBOSE) {
Log.d(TAG, "Video size is " + format.getInteger(MediaFormat.KEY_WIDTH) + "x" +
format.getInteger(MediaFormat.KEY_HEIGHT));
}
// Create a MediaCodec decoder, and configure it with the MediaFormat from the
// extractor. It's very important to use the format from the extractor because
// it contains a copy of the CSD-0/CSD-1 codec-specific data chunks.
String mime = format.getString(MediaFormat.KEY_MIME);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(format, null, null, 0);
decoder.start();
doExtract(extractor, trackIndex, decoder);
} finally {
if (decoder != null) {
decoder.stop();
decoder.release();
decoder = null;
}
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor) {
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
if (VERBOSE) {
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Work loop.
*/
public void doExtract(MediaExtractor extractor, int trackIndex, MediaCodec decoder) throws IOException {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
decodeCount = 0;
long frameSaveTime = 0;
boolean outputDone = false;
boolean inputDone = false;
ByteBuffer[] decoderOutputBuffers = decoder.getOutputBuffers();
MediaFormat decoderOutputFormat = null;
long rawSize = 0;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0) {
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE) Log.d(TAG, "sent input EOS");
} else {
if (extractor.getSampleTrackIndex() != trackIndex) {
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + trackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE) {
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
} else {
if (VERBOSE) Log.d(TAG, "input buffer not available");
}
}
if (!outputDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not important for us, since we're using Surface
if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
decoderOutputBuffers = decoder.getOutputBuffers();
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = decoder.getOutputFormat();
decoderOutputFormat = newFormat;
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus +
" (size=" + info.size + ")");
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
ByteBuffer outputFrame = decoderOutputBuffers[decoderStatus];
outputFrame.position(info.offset);
outputFrame.limit(info.offset + info.size);
rawSize += info.size;
if (info.size == 0) {
if (VERBOSE) Log.d(TAG, "got empty frame");
} else {
// if it's decode then check the altered value
// else save the frames
if (fromDecode) {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
int size = saveWidth * saveHeight;
int offset = size;
int[] pixels = new int[size];
int u, v, y1, y2, y3, y4;
int uvIndex = 0;
if (decodeCount == 1) {
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < size; i += 2, k += 2) {
y1 = data[i] & 0xff;
y2 = data[i + 1] & 0xff;
y3 = data[saveWidth + i] & 0xff;
y4 = data[saveWidth + i + 1] & 0xff;
u = data[offset + k] & 0xff;
v = data[offset + k + 1] & 0xff;
// getting size
if (uvIndex == 0) {
int specialByte1P1 = u & 15;
int specialByte1P2 = v & 15;
int specialCharacter1 = (specialByte1P1 << 4) | specialByte1P2;
if (specialCharacter1 != 17) {
throw new IllegalArgumentException("value has changed");
}
}
uvIndex++;
if (i != 0 && (i + 2) % saveWidth == 0)
i += saveWidth;
}
}
} else {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
try {
File outputFile = new File(STORE_FRAME_DIRECTORY,
String.format(Locale.US, "frame_%d.frame", decodeCount));
FileOutputStream stream = new FileOutputStream(outputFile.getAbsoluteFile());
stream.write(data);
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
decodeCount++;
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
decoder.releaseOutputBuffer(decoderStatus, false);
}
}
}
int numSaved = (frameRate < decodeCount) ? frameRate : decodeCount;
Log.d(TAG, "Saving " + numSaved + " frames took " +
(frameSaveTime / numSaved / 1000) + " us per frame");
}
public int getDecodeCount() {
return decodeCount;
}
}
in class below i encode frames, alter one u v value of (frame 1), store digit number 17 in lsb of first u and v and build mp4 using mediacodec encoder.
public class YUVFrameBufferToVideoEncoder {
private static final String TAG = BitmapToVideoEncoder.class.getSimpleName();
private static final int ERROR_IN_PROCESS = 0;
private IBitmapToVideoEncoderCallback mCallback;
private File mOutputFile;
private Queue<File> mEncodeQueue = new ConcurrentLinkedQueue();
private MediaCodec mediaCodec;
private MediaMuxer mediaMuxer;
private Object mFrameSync = new Object();
private CountDownLatch mNewFrameLatch;
private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static int mWidth;
private static int mHeight;
private static int BIT_RATE;
private static int FRAME_RATE; // Frames per second
private int frameCount;
private Handler _progressBarHandler;
private Handler _processHandler;
private static final int I_FRAME_INTERVAL = 1;
private int mGenerateIndex = 0;
private int mTrackIndex;
private boolean mNoMoreFrames = false;
private boolean mAbort = false;
//
private byte[] dataToHide;
public interface IBitmapToVideoEncoderCallback {
void onEncodingComplete(File outputFile);
}
public YUVFrameBufferToVideoEncoder(IBitmapToVideoEncoderCallback callback) {
mCallback = callback;
}
public boolean isEncodingStarted() {
return (mediaCodec != null) && (mediaMuxer != null) && !mNoMoreFrames && !mAbort;
}
public int getActiveBitmaps() {
return mEncodeQueue.size();
}
public boolean startEncoding(int width, int height, int fps, int bitrate, int frameCount
, byte[] dataToHide, Handler _progressBarHandler, Handler _processHandler
, File outputFile) {
mWidth = width;
mHeight = height;
FRAME_RATE = fps;
BIT_RATE = bitrate;
this.frameCount = frameCount;
this._progressBarHandler = _progressBarHandler;
this._processHandler = _processHandler;
mOutputFile = outputFile;
this.dataToHide = dataToHide;
String outputFileString;
try {
outputFileString = outputFile.getCanonicalPath();
} catch (IOException e) {
Log.e(TAG, "Unable to get path for " + outputFile);
ErrorManager.getInstance().addErrorMessage("Unable to get path for " + outputFile);
return false;
}
MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
if (codecInfo == null) {
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
ErrorManager.getInstance().addErrorMessage("Unable to find an appropriate codec for " + MIME_TYPE);
return false;
}
Log.d(TAG, "found codec: " + codecInfo.getName());
int colorFormat;
try {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
} catch (Exception e) {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
}
try {
mediaCodec = MediaCodec.createByCodecName(codecInfo.getName());
} catch (IOException e) {
Log.e(TAG, "Unable to create MediaCodec " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("Unable to create MediaCodec " + e.getMessage());
return false;
}
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL);
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
try {
mediaMuxer = new MediaMuxer(outputFileString, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
Log.e(TAG, "MediaMuxer creation failed. " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("MediaMuxer creation failed. " + e.getMessage());
return false;
}
Log.d(TAG, "Initialization complete. Starting encoder...");
Completable.fromAction(this::encode)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe();
return true;
}
public void stopEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to stop encoding since it never started");
return;
}
Log.d(TAG, "Stopping encoding");
mNoMoreFrames = true;
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void abortEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to abort encoding since it never started");
return;
}
Log.d(TAG, "Aborting encoding");
mNoMoreFrames = true;
mAbort = true;
mEncodeQueue = new ConcurrentLinkedQueue(); // Drop all frames
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void queueFrame(File frame) {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to queue frame. Encoding not started");
return;
}
Log.d(TAG, "Queueing frame");
mEncodeQueue.add(frame);
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
private void encode() {
Log.d(TAG, "Encoder started");
while (true) {
if (mNoMoreFrames && (mEncodeQueue.size() == 0)) break;
File frame = mEncodeQueue.poll();
if (frame == null) {
synchronized (mFrameSync) {
mNewFrameLatch = new CountDownLatch(1);
}
try {
mNewFrameLatch.await();
} catch (InterruptedException e) {
}
frame = mEncodeQueue.poll();
}
if (frame == null) continue;
int size = (int) frame.length();
byte[] bytesNV21 = new byte[size];
try {
BufferedInputStream buf = new BufferedInputStream(new FileInputStream(frame));
buf.read(bytesNV21, 0, bytesNV21.length);
buf.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
int offsetSize = mWidth * mHeight;
int byteNV21Offset = offsetSize;
int u, v, y1, y2, y3, y4;
//
int dataToHideLength = 0;
if (dataToHide != null)
dataToHideLength = dataToHide.length;
boolean isLastIndexInserted1 = false;
boolean isLastIndexInserted2 = false;
boolean isLastIndexInserted3 = false;
int uvIndex = 0;
int frameByteCapacity = ((mWidth * mHeight) / 4) / 20;
Log.e(TAG, "encode: dataToHideLength: " + dataToHideLength);
Log.e(TAG, "encode: frameByteCapacity: " + dataToHideLength);
//
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < offsetSize; i += 2, k += 2) {
y1 = bytesNV21[i] & 0xff;
y2 = bytesNV21[i + 1] & 0xff;
y3 = bytesNV21[mWidth + i] & 0xff;
y4 = bytesNV21[mWidth + i + 1] & 0xff;
u = bytesNV21[byteNV21Offset + k] & 0xff;
v = bytesNV21[byteNV21Offset + k + 1] & 0xff;
// frame 1
// altering u and v for test
if (mGenerateIndex == 1) {
int Unew = u & 240;
int Vnew = v & 240;
if (uvIndex == 0) {
// used in start and end of stego bytes
int specialByte1Integer = 17;
int specialByte1P1 = specialByte1Integer & 240;
int specialByte1P2 = specialByte1Integer & 15;
// shift p1 right 4 position
specialByte1P1 = specialByte1P1 >> 4;
u = Unew | specialByte1P1;
v = Vnew | specialByte1P2;
}
bytesNV21[byteNV21Offset + k] = (byte) u;
bytesNV21[byteNV21Offset + k + 1] = (byte) v;
}
uvIndex++;
if (i != 0 && (i + 2) % mWidth == 0)
i += mWidth;
}
long TIMEOUT_USEC = 500000;
int inputBufIndex = mediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
long ptsUsec = computePresentationTime(mGenerateIndex, FRAME_RATE);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuffer = mediaCodec.getInputBuffers()[inputBufIndex];
inputBuffer.clear();
inputBuffer.put(bytesNV21);
mediaCodec.queueInputBuffer(inputBufIndex, 0, bytesNV21.length, ptsUsec, 0);
mGenerateIndex++;
int percentComplete = 70 + (int) ((((double) mGenerateIndex) / (frameCount)) * 30);
if (_progressBarHandler != null) {
_progressBarHandler.sendMessage(_progressBarHandler.obtainMessage(percentComplete));
}
Log.w("creatingVideo: ", "is:" + percentComplete);
}
MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo();
int encoderStatus = mediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
Log.e(TAG, "No output from encoder available");
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// not expected for an encoder
MediaFormat newFormat = mediaCodec.getOutputFormat();
mTrackIndex = mediaMuxer.addTrack(newFormat);
mediaMuxer.start();
} else if (encoderStatus < 0) {
Log.e(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
} else if (mBufferInfo.size != 0) {
ByteBuffer encodedData = mediaCodec.getOutputBuffers()[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
} else {
encodedData.position(mBufferInfo.offset);
encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
mediaMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
mediaCodec.releaseOutputBuffer(encoderStatus, false);
}
}
}
release();
if (mAbort) {
mOutputFile.delete();
} else {
mCallback.onEncodingComplete(mOutputFile);
}
}
private void release() {
try {
if (mediaCodec != null) {
mediaCodec.stop();
mediaCodec.release();
mediaCodec = null;
Log.d(TAG, "RELEASE CODEC");
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
mediaMuxer = null;
Log.d(TAG, "RELEASE MUXER");
}
} catch (Exception ignored) {
ErrorManager.getInstance().addErrorMessage("unsupported video file");
Message res = _processHandler.obtainMessage(ERROR_IN_PROCESS);
_processHandler.sendMessage(res);
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo,
String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo
.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private long computePresentationTime(long frameIndex, int framerate) {
return 132 + frameIndex * 1000000 / framerate;
}}
output video is created successfully without any problem, but mediacodec has changed the altered test value and i cannot retrieve it.
here is my question, is this a right approach for doing video steganography in android? if it is not the right way can you please make a suggestion?

Steganography comes with a prerequisite - lossless encoding.
None of the codecs available on Android support lossless video encoding, as of now.
So I'm afraid your LSBs would never remain the same post encoding/decoding.
Suggestion: If you don't have a lot many frames, I would suggest you use a lossless format. You may encode your frames into a sequence of PNG images.

Why android mediaCodec encode low resolution video slower than high resolution

When I use Android media-codec to encode video to h264 format, I found that encode one frame of 1280 * 720 YUV420P raw image need about 10ms but one frame of 640 * 360 YUV420P raw image need about 100ms.
I have tied on serval mobile with MTK or Qualcomm video encoder, all device perform like that.
Can anyone tell me how to fix my following code to speed up or tell the reason why lower resolution cost more time.
private boolean openH264Encoder(int width, int height, int bitrate) throws TranscodeNativeException {
try {
mediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
Log.w(TAG, "codec name " + mediaCodec.getName());
}
for (int i = 0; i < MediaCodecList.getCodecCount(); i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (codecInfo.isEncoder()) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
if (mediaCodec.getName().equalsIgnoreCase(codecInfo.getName())) {
MediaCodecInfo.CodecCapabilities cc = codecInfo.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC);
for (int c : cc.colorFormats) {
Log.i(TAG, String.format("color format 0x%x", c));
if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar == c) {
videoColorFormat = c;
Log.i(TAG, String.format("final color format 0x%x", videoColorFormat));
break;
}
else if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == c) {
videoColorFormat = c;
Log.i(TAG, String.format("find color format 0x%x", videoColorFormat));
}
}
break;
}
}
else {
return false;
}
}
}
if (videoColorFormat == 0) {
Log.e(TAG, "can't find supported color format");
return false;
}
Log.i(TAG, String.format("use color format 0x%x", videoColorFormat));
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, videoColorFormat);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 25);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 25);
mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR);
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
encoderOutputBuffers = mediaCodec.getOutputBuffers();
} catch (Exception e) {
Log.w(TAG, "open h264 encoder failed");
throw new TranscodeNativeException("open h264 encoder failed");
}
return true;
}
public byte[] encodeH264(byte[] data, long ms) {
if (mediaCodec == null)
return null;
if (MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar == videoColorFormat) {
byte[] newData = new byte[data.length];
System.arraycopy(data, 0, newData, 0, data.length * 2 / 3);
for (int i = 0; i < data.length / 6; i++) {
newData[data.length * 2 / 3 + i * 2] = data[data.length * 2 / 3 + i];
newData[data.length * 2 / 3 + i * 2 + 1] = data[data.length * 5 / 6 + i];
}
data = newData;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
try {
ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(20 * 1000);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(data);
mediaCodec.queueInputBuffer(inputBufferIndex, 0, data.length, ms * 1000, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
while (true) {
int encoderStatus = mediaCodec.dequeueOutputBuffer(info, -1);
Log.i(TAG, "encoder status " + encoderStatus);
switch (encoderStatus) {
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.i(TAG, "why tell me try again later?");
return null;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: {
MediaFormat encformat = mediaCodec.getOutputFormat();
Log.i(TAG, "output format changed");
return new byte[0];
}
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
encoderOutputBuffers = mediaCodec.getOutputBuffers();
Log.i(TAG, "output buffer changed");
continue;
default: {
ByteBuffer encoderOutputBuffer = encoderOutputBuffers[encoderStatus];
if (encoderOutputBuffer == null) {
Log.w(TAG, "output buffer is null");
return null;
}
byte[] outData = new byte[info.size];
encoderOutputBuffer.get(outData);
ByteBuffer byteBuffer = ByteBuffer.wrap(outData);
if (spsppsBuffer == null && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
//save sps and pps
if (byteBuffer.getInt() == 0x00000001) {
spsppsBuffer = outData;
}
String d = "";
for (byte i : outData)
d += String.format("0x%02x ", i);
Log.i(TAG, "got sps pps " + d);
mediaCodec.releaseOutputBuffer(encoderStatus, false);
continue;
}
else {
//key frame
if ((outData[4] & 0x1f) == 5) {
byte[] buffer = new byte[outData.length + spsppsBuffer.length];
System.arraycopy(spsppsBuffer, 0, buffer, 0, spsppsBuffer.length);
System.arraycopy(outData, 0, buffer, spsppsBuffer.length, outData.length);
Log.i(TAG, "got key frame");
mediaCodec.releaseOutputBuffer(encoderStatus, false);
return buffer;
}
else {
Log.i(TAG, "got non key frame");
mediaCodec.releaseOutputBuffer(encoderStatus, false);
return outData;
}
}
}
}
}
}
else {
Log.w(TAG, "dequeue input buffer failed, skip one frame");
return new byte[0];
}
}
catch (Exception e) {
e.printStackTrace();
Log.w(TAG, "encode h264 failed");
return null;
}
}

MediaMuxer: No sync frames for video track

I have been trying to modify bigflake's DecodeEditEncode to edit an mp4 file and then encode it back to mp4. I have looked around the internet and couldn't find a solution. I am setting the IFRAME interval and FPS, yet the Muxer still complains about sync frames and is unable to stop.
Could anyone with more experience point out what am I doing wrong?
Thanks a lot in advance :)
private static void encodeToMp4(MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
MediaMuxer mediaMuxer = null;
int inputChunk = 0;
int outputCount = 0;
boolean outputDone = false;
boolean inputDone = false;
boolean decoderDone = false;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "edit loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer buffer = decoderInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
if (sampleSize < 0) {
inputChunk++;
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
buffer.clear();
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, extractor.getSampleTime(), 0);
extractor.advance();
inputChunk++;
}
}
}
// Assume output is available. Loop until both assumptions are false.
boolean decoderOutputAvailable = !decoderDone;
boolean encoderOutputAvailable = true;
try {
mediaMuxer = new MediaMuxer(outputFile.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
e.printStackTrace();
}
MediaFormat mediaFormat = extractor.getTrackFormat(0);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 20);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 10);
int trackIndex = mediaMuxer.addTrack(mediaFormat);
mediaMuxer.start();
while (decoderOutputAvailable || encoderOutputAvailable) {
int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
encoderOutputAvailable = false;
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
encoderOutputBuffers = encoder.getOutputBuffers();
} else if (encoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = encoder.getOutputFormat();
} else { // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
}
// Write the data to the output "file".
if (info.size != 0) {
encodedData.position(info.offset);
encodedData.limit(info.offset + info.size);
byte[] data = new byte[encodedData.remaining()];
encodedData.get(data);
mediaMuxer.writeSampleData(trackIndex, encodedData, info);
// outputData.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs);
outputCount++;
if (VERBOSE) Log.d(TAG, "encoder output " + info.size + " bytes");
}
outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
encoder.releaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int) MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
decoderOutputAvailable = false;
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
//decoderOutputBuffers = decoder.GetOutputBuffers();
if (VERBOSE) Log.d(TAG, "decoder output buffers changed (we don't care)");
} else if (decoderStatus == (int) MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// expected before first buffer of data
MediaFormat newFormat = decoder.getOutputFormat();
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer "
+ decoderStatus + " (size=" + info.size + ")");
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(decoderStatus, doRender);
if (doRender) {
// This waits for the image and renders it after it arrives.
if (VERBOSE) Log.d(TAG, "awaiting frame");
outputSurface.awaitNewImage();
outputSurface.drawImage();
// Send it to the encoder.
inputSurface.setPresentationTime(info.presentationTimeUs * 1000);
if (VERBOSE) Log.d(TAG, "swapBuffers");
inputSurface.swapBuffers();
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
// forward decoder EOS to encoder
if (VERBOSE) Log.d(TAG, "signaling input EOS");
if (WORK_AROUND_BUGS) {
// Bail early, possibly dropping a frame.
return;
} else {
encoder.signalEndOfInputStream();
}
}
}
}
}
}
if (inputChunk != outputCount) {
throw new RuntimeException("frame lost: " + inputChunk + " in, " +
outputCount + " out");
}
mediaMuxer.stop();
mediaMuxer.release();
listener.onFinished();
}

Fixed all issues by using the following bigflake test insteadExtractDecodeEditEncodeMuxTest.java. For everyone that is trying to achieve something similar, this sample is life saver. :)

How can I rearrange the video frames in Android?

I'm developing a video editing app, but I encounter some problems.I want to rearrange the video frames. And I have got an idea.First video decoding,
and the frame decoded data stored in a list, rearrange and generate new video.
I used the MediaCodec class in Android Framework to complete codec work.But I'm not familiar with it.According to my ideas, I wrote the following code:
public class VideoProcesser {
private static final String TAG = "VideoProcesser";
private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory();
private static final String OUTPUT_VIDEO_MIME_TYPE = "video/avc";
private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps
private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps
private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames
private Context mContext;
public VideoProcesser(Context mContext) {
this.mContext = mContext;
}
public void startProcessing() {
VideoChunks longChunks = generateVideoChunks(new IntegerPair(0, 180));
VideoChunks shortChunks = generateVideoChunks(new IntegerPair(30, 60));
VideoChunks dstChunks = new VideoChunks();
dstChunks.addChunks(longChunks);
dstChunks.addChunks(shortChunks);
saveChunksToFile(dstChunks, new File(OUTPUT_FILENAME_DIR, "sample_processed.mp4"));
}
private void saveChunksToFile(VideoChunks inputData, File file) {
MediaMuxer muxer = null;
MediaCodec videoEncoder = null;
try {
MediaFormat outputFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, 1280, 720);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
MediaCodecInfo videoEncoderInfo = selectCodec(getMimeTypeFor(outputFormat));
int colorFormat = selectColorFormat(videoEncoderInfo, OUTPUT_VIDEO_MIME_TYPE);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
videoEncoder = MediaCodec.createByCodecName(videoEncoderInfo.getName());
videoEncoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
videoEncoder.start();
muxer = new MediaMuxer(file.getAbsolutePath(),
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
doSaveVideoChunks(inputData, videoEncoder, muxer);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down encoder, muxer");
if (videoEncoder != null) {
videoEncoder.stop();
videoEncoder.release();
}
if (muxer != null) {
muxer.stop();
muxer.release();
}
}
}
private void doSaveVideoChunks(VideoChunks inputData, MediaCodec videoEncoder,
MediaMuxer muxer) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoEncoderInputBuffers = videoEncoder.getInputBuffers();
ByteBuffer[] videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
int inputChunk = 0;
int videoEncodedFrameCount = 0;
boolean inputDone = false;
boolean videoEncodeDone = false;
boolean muxing = false;
MediaFormat encoderOutputVideoFormat = null;
int outputVideoTrack = -1;
while (!videoEncodeDone) {
Log.d(TAG, String.format("save loop: inputChunk:%s encoded:%s", inputChunk,
videoEncodedFrameCount));
if (!inputDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderInputStatus = videoEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder input buffer");
continue;
}
long time = computePresentationTime(inputChunk);
if (inputChunk == inputData.getNumChunks()) {
// End of stream -- send empty frame with EOS flag set.
videoEncoder.queueInputBuffer(encoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.d(TAG, "input data EOS");
} else {
ByteBuffer encoderInputBuffer = videoEncoderInputBuffers[encoderInputStatus];
encoderInputBuffer.clear();
inputData.getChunkData(inputChunk, encoderInputBuffer);
videoEncoder.queueInputBuffer(encoderInputStatus, 0,
encoderInputBuffer.position(), time,
inputData.getChunkFlags(inputChunk));
inputChunk++;
}
}
if (!videoEncodeDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputStatus =
videoEncoder.dequeueOutputBuffer(videoEncoderOutputBufferInfo,
TIMEOUT_USEC);
if (encoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder output buffer");
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
Log.d(TAG, "video encoder: output media format changed"
+ encoderOutputVideoFormat);
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
muxing = true;
muxer.start();
} else if (encoderOutputStatus < 0) {
//ignore it
} else {
ByteBuffer encoderOutputBuffer = videoEncoderOutputBuffers[encoderOutputStatus];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video encoder: codec config buffer");
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
continue;
// Simply ignore codec config buffers.
}
if (videoEncoderOutputBufferInfo.size != 0) {
muxer.writeSampleData(outputVideoTrack, encoderOutputBuffer,
videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video encoder: EOS");
videoEncodeDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
videoEncodedFrameCount++;
}
}
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo, String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
Log.e(TAG,
"couldn't find a good color format for " + codecInfo.getName() + " / " + mimeType);
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for this test
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private static long computePresentationTime(int frameIndex) {
return 132 + frameIndex * 1000000 / OUTPUT_VIDEO_FRAME_RATE;
}
private VideoChunks generateVideoChunks(IntegerPair segment) {
VideoChunks outputData = new VideoChunks();
MediaExtractor videoExtractor = null;
MediaCodec videoDecoder = null;
try {
videoExtractor = createVideoExtractor(R.raw.sample);
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
MediaFormat videoInputFormat = videoExtractor.getTrackFormat(videoInputTrack);
videoDecoder = createVideoDecoder(videoInputFormat);
doGenerateVideoChunks(segment, videoExtractor, videoDecoder, outputData);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down extractor, decoder");
if (videoExtractor != null) {
videoExtractor.release();
}
if (videoDecoder != null) {
videoDecoder.stop();
videoDecoder.release();
}
}
return outputData;
}
private void doGenerateVideoChunks(IntegerPair segment, MediaExtractor videoExtractor,
MediaCodec videoDecoder, VideoChunks outputData) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoDecoderInputBuffers = videoDecoder.getInputBuffers();
ByteBuffer[] videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
boolean videoExtracted = false;
boolean videoDecoded = false;
while (true) {
Log.d(TAG, String.format("loop: extracted:%s decoded:%s", videoExtractedFrameCount,
videoDecodedFrameCount));
while (!videoExtracted) {
int decoderInputStatus = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder input buffer");
break;
}
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputStatus];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
int flags = videoExtractor.getSampleFlags();
if (size >= 0) {
videoDecoder.queueInputBuffer(decoderInputStatus, 0, size, presentationTime,
flags);
}
videoExtracted = !videoExtractor.advance();
if (videoExtracted) {
Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(decoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
break;
}
while (!videoDecoded) {
int decoderOutputStatus =
videoDecoder.dequeueOutputBuffer(videoDecoderOutputBufferInfo,
TIMEOUT_USEC);
if (decoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = videoDecoder.getOutputFormat();
Log.d(TAG, "video decoder: output format changed: " + newFormat);
break;
}
ByteBuffer decoderOutputBuffer = videoDecoderOutputBuffers[decoderOutputStatus];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputStatus, false);
break;
}
boolean render = videoDecoderOutputBufferInfo.size != 0;
if (render) {
if (segment.contains(videoDecodedFrameCount)) {
outputData.addChunk(decoderOutputBuffer, videoDecoderOutputBufferInfo.flags,
videoDecoderOutputBufferInfo.presentationTimeUs);
}
}
videoDecoder.releaseOutputBuffer(decoderOutputStatus, render);
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video decoder: EOS");
videoDecoded = true;
}
videoDecodedFrameCount++;
break;
}
if (videoDecoded) {
break;
}
}
}
private MediaCodec createVideoDecoder(MediaFormat inputFormat) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private MediaExtractor createVideoExtractor(#RawRes int resId) throws IOException {
MediaExtractor extractor = new MediaExtractor();
AssetFileDescriptor srcFd = mContext.getResources().openRawResourceFd(resId);
extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(),
srcFd.getLength());
return extractor;
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
Log.d(TAG, "format for track " + index + " is " + getMimeTypeFor(
extractor.getTrackFormat(index)));
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
}
However, But I got a video which is loss the original video data.Can you help me find what's wrong with the above code. Or have any good ideas.
This is my full code.
edit1: I found the problem.The size of the output video not correctly is the key to the problem.So I keep the same size with the original video, everything is ok.At Last, Don't forget the orientation of the video， otherwise your output video may be a bit strange.

Encode wav to AAC on Android

You can use MediaRecorder to record a stream directly to AAC but there doesn't seem to be a way to encode an existing PCM/WAV file to AAC. The ability to encode to AAC exists natively in Android and I'd like to use that. Is there no way to do it with a pre-existing audio file?

Look at this beautiful (and perfectly working) example:
Mp4ParserSample
Look at the final part of the class (rows 335-442), the convert Runnable object just does the job! You have to shape that code to your needs, adjust the input and the output file paths and the conversion parameters (sampling rate, bit rate, etc).
public static final String AUDIO_RECORDING_FILE_NAME = "audio_Capturing-190814-034638.422.wav"; // Input PCM file
public static final String COMPRESSED_AUDIO_FILE_NAME = "convertedmp4.m4a"; // Output MP4/M4A file
public static final String COMPRESSED_AUDIO_FILE_MIME_TYPE = "audio/mp4a-latm";
public static final int COMPRESSED_AUDIO_FILE_BIT_RATE = 64000; // 64kbps
public static final int SAMPLING_RATE = 48000;
public static final int BUFFER_SIZE = 48000;
public static final int CODEC_TIMEOUT_IN_MS = 5000;
String LOGTAG = "CONVERT AUDIO";
Runnable convert = new Runnable() {
#TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR2)
#Override
public void run() {
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_BACKGROUND);
try {
String filePath = Environment.getExternalStorageDirectory().getPath() + "/" + AUDIO_RECORDING_FILE_NAME;
File inputFile = new File(filePath);
FileInputStream fis = new FileInputStream(inputFile);
File outputFile = new File(Environment.getExternalStorageDirectory().getAbsolutePath() + "/" + COMPRESSED_AUDIO_FILE_NAME);
if (outputFile.exists()) outputFile.delete();
MediaMuxer mux = new MediaMuxer(outputFile.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
MediaFormat outputFormat = MediaFormat.createAudioFormat(COMPRESSED_AUDIO_FILE_MIME_TYPE,SAMPLING_RATE, 1);
outputFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, COMPRESSED_AUDIO_FILE_BIT_RATE);
outputFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16384);
MediaCodec codec = MediaCodec.createEncoderByType(COMPRESSED_AUDIO_FILE_MIME_TYPE);
codec.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers(); // Note: Array of buffers
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
MediaCodec.BufferInfo outBuffInfo = new MediaCodec.BufferInfo();
byte[] tempBuffer = new byte[BUFFER_SIZE];
boolean hasMoreData = true;
double presentationTimeUs = 0;
int audioTrackIdx = 0;
int totalBytesRead = 0;
int percentComplete = 0;
do {
int inputBufIndex = 0;
while (inputBufIndex != -1 && hasMoreData) {
inputBufIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_IN_MS);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
dstBuf.clear();
int bytesRead = fis.read(tempBuffer, 0, dstBuf.limit());
Log.e("bytesRead","Readed "+bytesRead);
if (bytesRead == -1) { // -1 implies EOS
hasMoreData = false;
codec.queueInputBuffer(inputBufIndex, 0, 0, (long) presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
totalBytesRead += bytesRead;
dstBuf.put(tempBuffer, 0, bytesRead);
codec.queueInputBuffer(inputBufIndex, 0, bytesRead, (long) presentationTimeUs, 0);
presentationTimeUs = 1000000l * (totalBytesRead / 2) / SAMPLING_RATE;
}
}
}
// Drain audio
int outputBufIndex = 0;
while (outputBufIndex != MediaCodec.INFO_TRY_AGAIN_LATER) {
outputBufIndex = codec.dequeueOutputBuffer(outBuffInfo, CODEC_TIMEOUT_IN_MS);
if (outputBufIndex >= 0) {
ByteBuffer encodedData = codecOutputBuffers[outputBufIndex];
encodedData.position(outBuffInfo.offset);
encodedData.limit(outBuffInfo.offset + outBuffInfo.size);
if ((outBuffInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0 && outBuffInfo.size != 0) {
codec.releaseOutputBuffer(outputBufIndex, false);
}else{
mux.writeSampleData(audioTrackIdx, codecOutputBuffers[outputBufIndex], outBuffInfo);
codec.releaseOutputBuffer(outputBufIndex, false);
}
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
outputFormat = codec.getOutputFormat();
Log.v(LOGTAG, "Output format changed - " + outputFormat);
audioTrackIdx = mux.addTrack(outputFormat);
mux.start();
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.e(LOGTAG, "Output buffers changed during encode!");
} else if (outputBufIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// NO OP
} else {
Log.e(LOGTAG, "Unknown return code from dequeueOutputBuffer - " + outputBufIndex);
}
}
percentComplete = (int) Math.round(((float) totalBytesRead / (float) inputFile.length()) * 100.0);
Log.v(LOGTAG, "Conversion % - " + percentComplete);
} while (outBuffInfo.flags != MediaCodec.BUFFER_FLAG_END_OF_STREAM);
fis.close();
mux.stop();
mux.release();
Log.v(LOGTAG, "Compression done ...");
} catch (FileNotFoundException e) {
Log.e(LOGTAG, "File not found!", e);
} catch (IOException e) {
Log.e(LOGTAG, "IO exception!", e);
}
//mStop = false;
// Notify UI thread...
}
};

you can get your hands dirty with the native code and use the IOMX C++ interface to decoders in the framework. But this is build sensitive and will not work on other phones and android flavours.
Another option is port an opensource aac encoder like ffmpeg and write an app over it over jni. Will atleast work with phones with same architecture (arm-9, cortex a8..).
JB has a MediaCodec just to fulfill your wishes. But the problem would be the install base for devices with JB will be lean for some more time.
http://developer.android.com/about/versions/android-4.1.html#Multimedia

I think you can use this library.
https://github.com/timsu/android-aac-enc
http://betaful.com/post/82668810035/encoding-aac-audio-in-android