Related
I need to create videos with data hidden in them. i managed to extract video frames using mediacodec decoder as NV21 buffer and save them, then i create mp4 file from frames using mediacodec encoder.
the class below is responsible for saving frame files if we are in encode process or check the value if we want to extract data from stego-video.
public class ExtractMpegFramesBufferDecoder {
private static final String TAG = "ExtractMpegFramesDec";
private static final boolean VERBOSE = true; // lots of logging
// where to find files (note: requires WRITE_EXTERNAL_STORAGE permission)
private File STORE_FRAME_DIRECTORY;
private String INPUT_FILE;
private int frameRate; // stop extracting after this many
private int saveWidth;
private int saveHeight;
private int decodeCount;
private Handler _progressBarHandler;
private int duration;
//
private int MAX_FRAMES;
private boolean fromDecode;
//
public ExtractMpegFramesBufferDecoder(File storeFrameDirectory, String inputVideoPath, int frameRate
, int saveWidth, int saveHeight
, double duration, int rotation
, Handler _progressBarHandler) {
this.STORE_FRAME_DIRECTORY = storeFrameDirectory;
this.INPUT_FILE = inputVideoPath;
this.frameRate = frameRate;
this.saveWidth = saveWidth;
this.saveHeight = saveHeight;
this._progressBarHandler = _progressBarHandler;
this.duration = (int) duration;
}
/**
* Tests extraction from an MP4 to a series of PNG files.
* <p>
* We scale the video to 640x480 for the PNG just to demonstrate that we can scale the
* video with the GPU. If the input video has a different aspect ratio, we could preserve
* it by adjusting the GL viewport to get letterboxing or pillarboxing, but generally if
* you're extracting frames you don't want black bars.
*/
public void extractMpegFrames(int maxFrame, boolean fromDecode) throws IOException {
MediaCodec decoder = null;
MediaExtractor extractor = null;
MAX_FRAMES = maxFrame;
this.fromDecode = fromDecode;
try {
File inputFile = new File(INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead()) {
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = selectTrack(extractor);
if (trackIndex < 0) {
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat format = extractor.getTrackFormat(trackIndex);
if (VERBOSE) {
Log.d(TAG, "Video size is " + format.getInteger(MediaFormat.KEY_WIDTH) + "x" +
format.getInteger(MediaFormat.KEY_HEIGHT));
}
// Create a MediaCodec decoder, and configure it with the MediaFormat from the
// extractor. It's very important to use the format from the extractor because
// it contains a copy of the CSD-0/CSD-1 codec-specific data chunks.
String mime = format.getString(MediaFormat.KEY_MIME);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(format, null, null, 0);
decoder.start();
doExtract(extractor, trackIndex, decoder);
} finally {
if (decoder != null) {
decoder.stop();
decoder.release();
decoder = null;
}
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor) {
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
if (VERBOSE) {
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Work loop.
*/
public void doExtract(MediaExtractor extractor, int trackIndex, MediaCodec decoder) throws IOException {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
decodeCount = 0;
long frameSaveTime = 0;
boolean outputDone = false;
boolean inputDone = false;
ByteBuffer[] decoderOutputBuffers = decoder.getOutputBuffers();
MediaFormat decoderOutputFormat = null;
long rawSize = 0;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0) {
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE) Log.d(TAG, "sent input EOS");
} else {
if (extractor.getSampleTrackIndex() != trackIndex) {
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + trackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE) {
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
} else {
if (VERBOSE) Log.d(TAG, "input buffer not available");
}
}
if (!outputDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not important for us, since we're using Surface
if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
decoderOutputBuffers = decoder.getOutputBuffers();
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = decoder.getOutputFormat();
decoderOutputFormat = newFormat;
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus +
" (size=" + info.size + ")");
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
ByteBuffer outputFrame = decoderOutputBuffers[decoderStatus];
outputFrame.position(info.offset);
outputFrame.limit(info.offset + info.size);
rawSize += info.size;
if (info.size == 0) {
if (VERBOSE) Log.d(TAG, "got empty frame");
} else {
// if it's decode then check the altered value
// else save the frames
if (fromDecode) {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
int size = saveWidth * saveHeight;
int offset = size;
int[] pixels = new int[size];
int u, v, y1, y2, y3, y4;
int uvIndex = 0;
if (decodeCount == 1) {
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < size; i += 2, k += 2) {
y1 = data[i] & 0xff;
y2 = data[i + 1] & 0xff;
y3 = data[saveWidth + i] & 0xff;
y4 = data[saveWidth + i + 1] & 0xff;
u = data[offset + k] & 0xff;
v = data[offset + k + 1] & 0xff;
// getting size
if (uvIndex == 0) {
int specialByte1P1 = u & 15;
int specialByte1P2 = v & 15;
int specialCharacter1 = (specialByte1P1 << 4) | specialByte1P2;
if (specialCharacter1 != 17) {
throw new IllegalArgumentException("value has changed");
}
}
uvIndex++;
if (i != 0 && (i + 2) % saveWidth == 0)
i += saveWidth;
}
}
} else {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
try {
File outputFile = new File(STORE_FRAME_DIRECTORY,
String.format(Locale.US, "frame_%d.frame", decodeCount));
FileOutputStream stream = new FileOutputStream(outputFile.getAbsoluteFile());
stream.write(data);
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
decodeCount++;
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
decoder.releaseOutputBuffer(decoderStatus, false);
}
}
}
int numSaved = (frameRate < decodeCount) ? frameRate : decodeCount;
Log.d(TAG, "Saving " + numSaved + " frames took " +
(frameSaveTime / numSaved / 1000) + " us per frame");
}
public int getDecodeCount() {
return decodeCount;
}
}
in class below i encode frames, alter one u v value of (frame 1), store digit number 17 in lsb of first u and v and build mp4 using mediacodec encoder.
public class YUVFrameBufferToVideoEncoder {
private static final String TAG = BitmapToVideoEncoder.class.getSimpleName();
private static final int ERROR_IN_PROCESS = 0;
private IBitmapToVideoEncoderCallback mCallback;
private File mOutputFile;
private Queue<File> mEncodeQueue = new ConcurrentLinkedQueue();
private MediaCodec mediaCodec;
private MediaMuxer mediaMuxer;
private Object mFrameSync = new Object();
private CountDownLatch mNewFrameLatch;
private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static int mWidth;
private static int mHeight;
private static int BIT_RATE;
private static int FRAME_RATE; // Frames per second
private int frameCount;
private Handler _progressBarHandler;
private Handler _processHandler;
private static final int I_FRAME_INTERVAL = 1;
private int mGenerateIndex = 0;
private int mTrackIndex;
private boolean mNoMoreFrames = false;
private boolean mAbort = false;
//
private byte[] dataToHide;
public interface IBitmapToVideoEncoderCallback {
void onEncodingComplete(File outputFile);
}
public YUVFrameBufferToVideoEncoder(IBitmapToVideoEncoderCallback callback) {
mCallback = callback;
}
public boolean isEncodingStarted() {
return (mediaCodec != null) && (mediaMuxer != null) && !mNoMoreFrames && !mAbort;
}
public int getActiveBitmaps() {
return mEncodeQueue.size();
}
public boolean startEncoding(int width, int height, int fps, int bitrate, int frameCount
, byte[] dataToHide, Handler _progressBarHandler, Handler _processHandler
, File outputFile) {
mWidth = width;
mHeight = height;
FRAME_RATE = fps;
BIT_RATE = bitrate;
this.frameCount = frameCount;
this._progressBarHandler = _progressBarHandler;
this._processHandler = _processHandler;
mOutputFile = outputFile;
this.dataToHide = dataToHide;
String outputFileString;
try {
outputFileString = outputFile.getCanonicalPath();
} catch (IOException e) {
Log.e(TAG, "Unable to get path for " + outputFile);
ErrorManager.getInstance().addErrorMessage("Unable to get path for " + outputFile);
return false;
}
MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
if (codecInfo == null) {
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
ErrorManager.getInstance().addErrorMessage("Unable to find an appropriate codec for " + MIME_TYPE);
return false;
}
Log.d(TAG, "found codec: " + codecInfo.getName());
int colorFormat;
try {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
} catch (Exception e) {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
}
try {
mediaCodec = MediaCodec.createByCodecName(codecInfo.getName());
} catch (IOException e) {
Log.e(TAG, "Unable to create MediaCodec " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("Unable to create MediaCodec " + e.getMessage());
return false;
}
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL);
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
try {
mediaMuxer = new MediaMuxer(outputFileString, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
Log.e(TAG, "MediaMuxer creation failed. " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("MediaMuxer creation failed. " + e.getMessage());
return false;
}
Log.d(TAG, "Initialization complete. Starting encoder...");
Completable.fromAction(this::encode)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe();
return true;
}
public void stopEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to stop encoding since it never started");
return;
}
Log.d(TAG, "Stopping encoding");
mNoMoreFrames = true;
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void abortEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to abort encoding since it never started");
return;
}
Log.d(TAG, "Aborting encoding");
mNoMoreFrames = true;
mAbort = true;
mEncodeQueue = new ConcurrentLinkedQueue(); // Drop all frames
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void queueFrame(File frame) {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to queue frame. Encoding not started");
return;
}
Log.d(TAG, "Queueing frame");
mEncodeQueue.add(frame);
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
private void encode() {
Log.d(TAG, "Encoder started");
while (true) {
if (mNoMoreFrames && (mEncodeQueue.size() == 0)) break;
File frame = mEncodeQueue.poll();
if (frame == null) {
synchronized (mFrameSync) {
mNewFrameLatch = new CountDownLatch(1);
}
try {
mNewFrameLatch.await();
} catch (InterruptedException e) {
}
frame = mEncodeQueue.poll();
}
if (frame == null) continue;
int size = (int) frame.length();
byte[] bytesNV21 = new byte[size];
try {
BufferedInputStream buf = new BufferedInputStream(new FileInputStream(frame));
buf.read(bytesNV21, 0, bytesNV21.length);
buf.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
int offsetSize = mWidth * mHeight;
int byteNV21Offset = offsetSize;
int u, v, y1, y2, y3, y4;
//
int dataToHideLength = 0;
if (dataToHide != null)
dataToHideLength = dataToHide.length;
boolean isLastIndexInserted1 = false;
boolean isLastIndexInserted2 = false;
boolean isLastIndexInserted3 = false;
int uvIndex = 0;
int frameByteCapacity = ((mWidth * mHeight) / 4) / 20;
Log.e(TAG, "encode: dataToHideLength: " + dataToHideLength);
Log.e(TAG, "encode: frameByteCapacity: " + dataToHideLength);
//
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < offsetSize; i += 2, k += 2) {
y1 = bytesNV21[i] & 0xff;
y2 = bytesNV21[i + 1] & 0xff;
y3 = bytesNV21[mWidth + i] & 0xff;
y4 = bytesNV21[mWidth + i + 1] & 0xff;
u = bytesNV21[byteNV21Offset + k] & 0xff;
v = bytesNV21[byteNV21Offset + k + 1] & 0xff;
// frame 1
// altering u and v for test
if (mGenerateIndex == 1) {
int Unew = u & 240;
int Vnew = v & 240;
if (uvIndex == 0) {
// used in start and end of stego bytes
int specialByte1Integer = 17;
int specialByte1P1 = specialByte1Integer & 240;
int specialByte1P2 = specialByte1Integer & 15;
// shift p1 right 4 position
specialByte1P1 = specialByte1P1 >> 4;
u = Unew | specialByte1P1;
v = Vnew | specialByte1P2;
}
bytesNV21[byteNV21Offset + k] = (byte) u;
bytesNV21[byteNV21Offset + k + 1] = (byte) v;
}
uvIndex++;
if (i != 0 && (i + 2) % mWidth == 0)
i += mWidth;
}
long TIMEOUT_USEC = 500000;
int inputBufIndex = mediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
long ptsUsec = computePresentationTime(mGenerateIndex, FRAME_RATE);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuffer = mediaCodec.getInputBuffers()[inputBufIndex];
inputBuffer.clear();
inputBuffer.put(bytesNV21);
mediaCodec.queueInputBuffer(inputBufIndex, 0, bytesNV21.length, ptsUsec, 0);
mGenerateIndex++;
int percentComplete = 70 + (int) ((((double) mGenerateIndex) / (frameCount)) * 30);
if (_progressBarHandler != null) {
_progressBarHandler.sendMessage(_progressBarHandler.obtainMessage(percentComplete));
}
Log.w("creatingVideo: ", "is:" + percentComplete);
}
MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo();
int encoderStatus = mediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
Log.e(TAG, "No output from encoder available");
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// not expected for an encoder
MediaFormat newFormat = mediaCodec.getOutputFormat();
mTrackIndex = mediaMuxer.addTrack(newFormat);
mediaMuxer.start();
} else if (encoderStatus < 0) {
Log.e(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
} else if (mBufferInfo.size != 0) {
ByteBuffer encodedData = mediaCodec.getOutputBuffers()[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
} else {
encodedData.position(mBufferInfo.offset);
encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
mediaMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
mediaCodec.releaseOutputBuffer(encoderStatus, false);
}
}
}
release();
if (mAbort) {
mOutputFile.delete();
} else {
mCallback.onEncodingComplete(mOutputFile);
}
}
private void release() {
try {
if (mediaCodec != null) {
mediaCodec.stop();
mediaCodec.release();
mediaCodec = null;
Log.d(TAG, "RELEASE CODEC");
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
mediaMuxer = null;
Log.d(TAG, "RELEASE MUXER");
}
} catch (Exception ignored) {
ErrorManager.getInstance().addErrorMessage("unsupported video file");
Message res = _processHandler.obtainMessage(ERROR_IN_PROCESS);
_processHandler.sendMessage(res);
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo,
String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo
.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private long computePresentationTime(long frameIndex, int framerate) {
return 132 + frameIndex * 1000000 / framerate;
}}
output video is created successfully without any problem, but mediacodec has changed the altered test value and i cannot retrieve it.
here is my question, is this a right approach for doing video steganography in android? if it is not the right way can you please make a suggestion?
Steganography comes with a prerequisite - lossless encoding.
None of the codecs available on Android support lossless video encoding, as of now.
So I'm afraid your LSBs would never remain the same post encoding/decoding.
Suggestion: If you don't have a lot many frames, I would suggest you use a lossless format. You may encode your frames into a sequence of PNG images.
I am using two MediaCodec to decode video and audio streams. My final goal is to generate a new video file (.mp4 or .wmv file) after adjusting or changing audio.
Right now, i am just testing how to decode video and audio from a .wmv video file while playback.
The problem that I am having now is that the video file is playing by using MediaPlayer while decoding video and audio streams, but there is no sound.
Does anyone have any idea what is causing this?
Here is what I have tried so far.
DecodingActivity.java (SurfaceView in activity_decoding.xml)
public class DecodingActivity extends AppCompatActivity implements SurfaceHolder.Callback, MediaPlayer.OnPreparedListener{
private static final String TAG = "DecodingActivity";
private static final String FILE_PATH = Environment.getExternalStorageDirectory() + "/test.mp4";
private SurfaceView mSurfaceView;
private MediaPlayer mMediaPlayer;
private SurfaceHolder mSurfaceHolder;
private VideoDecoderThread mVideoDecoder;
private AudioDecoderThread mAudioDecoder;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
requestWindowFeature(Window.FEATURE_NO_TITLE);
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
WindowManager.LayoutParams.FLAG_FULLSCREEN);
setContentView(R.layout.activity_decoding);
mSurfaceView = (SurfaceView)findViewById(R.id.surface_view);
mSurfaceHolder = mSurfaceView.getHolder();
mSurfaceHolder.addCallback(DecodingActivity.this);
mVideoDecoder = new VideoDecoderThread();
mAudioDecoder = new AudioDecoderThread();
}
#Override
public void surfaceCreated(SurfaceHolder holder) {
mMediaPlayer = new MediaPlayer();
mMediaPlayer.setDisplay(mSurfaceHolder);
try {
mMediaPlayer.setDataSource(FILE_PATH);
//mMediaPlayer.prepare(); // this causes IllegalException Error
mMediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
mMediaPlayer.setOnPreparedListener(DecodingActivity.this);
} catch (IOException e) {
e.printStackTrace();
}
}
#Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
Log.d(TAG,"mAudioDecoder: "+mAudioDecoder);
Log.d(TAG,"mVideoDecoder: "+mVideoDecoder);
if (mVideoDecoder != null && mAudioDecoder != null) {
if (mVideoDecoder.init(holder.getSurface(), FILE_PATH)) {
mVideoDecoder.start();
mAudioDecoder.startPlay(FILE_PATH);
} else {
mVideoDecoder = null;
mAudioDecoder = null;
}
}
}
#Override
public void surfaceDestroyed(SurfaceHolder holder) {
if (mVideoDecoder != null) {
mVideoDecoder.close();
}
}
#Override
public void onPrepared(MediaPlayer mp) {
mMediaPlayer.start();
}
#Override
protected void onPause() {
super.onPause();
releaseMediaPlayer();
}
#Override
protected void onStop() {
super.onStop();
mAudioDecoder.stop();
}
#Override
protected void onDestroy() {
super.onDestroy();
releaseMediaPlayer();
}
private void releaseMediaPlayer() {
if (mMediaPlayer != null) {
mMediaPlayer.release();
mMediaPlayer = null;
}
}
}
AudioDecoderThread.java
public class AudioDecoderThread {
private static final String TAG = "AudioDecoder";
private static final int TIMEOUT_US = 1000;
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
private int mSampleRate = 0;
/**
*
* #param
*/
public void startPlay(String path) {
eosReceived = false;
mExtractor = new MediaExtractor();
try {
mExtractor.setDataSource(path);
} catch (IOException e) {
e.printStackTrace();
}
int channel = 0;
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
MediaFormat format = mExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("audio/")) {
mExtractor.selectTrack(i);
Log.d(TAG, "format : " + format);
//{aac-profile=2, mime=audio/mp4a-latm, channel-count=2, max-input-size=1572864, isDMCMMExtractor=1, durationUs=35526530, csd-0=java.nio.ByteArrayBuffer[position=0,limit=2,capacity=2], encoder-delay=2112, encoder-padding=588, sample-rate=44100}
ByteBuffer csd = format.getByteBuffer("csd-0"); //csd-0=java.nio.ByteArrayBuffer[position=0,limit=2,capacity=2]
for (int k = 0; k < csd.capacity(); ++k) {
Log.e(TAG, "csd : " + csd.array()[k]);
}
mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
channel = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
Log.d(TAG, "mSampleRate : " + mSampleRate); // 44100
Log.d(TAG, "channel : " + channel); // 2
break;
}
}
MediaFormat format = makeAACCodecSpecificData(MediaCodecInfo.CodecProfileLevel.AACObjectLC, mSampleRate, channel);
Log.d(TAG, "format[MediaFormat]: " + format);
if (format == null)
return;
try {
mDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm");
} catch (IOException e) {
e.printStackTrace();
}
mDecoder.configure(format, null, null, 0);
if (mDecoder == null) {
Log.e("DecodeActivity", "Can't find video info!");
return;
}
mDecoder.start();
new Thread(AACDecoderAndPlayRunnable).start();
}
/**
* The code profile, Sample rate, channel Count is used to
* produce the AAC Codec SpecificData.
* Android 4.4.2/frameworks/av/media/libstagefright/avc_utils.cpp refer
* to the portion of the code written.
*
* MPEG-4 Audio refer : http://wiki.multimedia.cx/index.php?title=MPEG-4_Audio#Audio_Specific_Config
*
* #param audioProfile is MPEG-4 Audio Object Types
* #param sampleRate
* #param channelConfig
* #return MediaFormat
*/
private MediaFormat makeAACCodecSpecificData(int audioProfile, int sampleRate, int channelConfig) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channelConfig);
int samplingFreq[] = {
96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050,
16000, 12000, 11025, 8000
};
int sampleIndex = -1;
for (int i = 0; i < samplingFreq.length; ++i) {
if (samplingFreq[i] == sampleRate) {
Log.d(TAG, "kSamplingFreq " + samplingFreq[i] + " i : " + i); // kSamplingFreq 44100 i : 4
sampleIndex = i;
}
}
if (sampleIndex == -1) {
return null;
}
ByteBuffer csd = ByteBuffer.allocate(2);
csd.put((byte) ((audioProfile << 3) | (sampleIndex >> 1)));
csd.position(1);
csd.put((byte) ((byte) ((sampleIndex << 7) & 0x80) | (channelConfig << 3)));
csd.flip();
format.setByteBuffer("csd-0", csd); // add csd-0
for (int k = 0; k < csd.capacity(); ++k) {
Log.e(TAG, "csd : " + csd.array()[k]);
}
return format;
}
Runnable AACDecoderAndPlayRunnable = new Runnable() {
#Override
public void run() {
AACDecoderAndPlay();
}
};
/**
* After decoding AAC, Play using Audio Track.
*/
public void AACDecoderAndPlay() {
ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mDecoder.getOutputBuffers();
Log.d(TAG, "inputBuffers: "+inputBuffers);
Log.d(TAG, "outputBuffers: "+outputBuffers);
BufferInfo info = new BufferInfo();
int buffsize = AudioTrack.getMinBufferSize(mSampleRate, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT);
Log.d(TAG, "buffsize: "+buffsize); // 14176
AudioTrack audioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, mSampleRate,
AudioFormat.CHANNEL_OUT_STEREO,
AudioFormat.ENCODING_PCM_16BIT,
buffsize,
AudioTrack.MODE_STREAM);
//Log.d(TAG, "audioTrack: "+audioTrack); //android.media.AudioTrack#4be0ac7
audioTrack.play();
Log.d(TAG, "------------------------------------------------: ");
while (!eosReceived) {
int inIndex = mDecoder.dequeueInputBuffer(TIMEOUT_US); // 버퍼 index
Log.d(TAG, "inIndex: "+inIndex);
if (inIndex > -1) {
ByteBuffer buffer = inputBuffers[inIndex];
int sampleSize = mExtractor.readSampleData(buffer, 0);
Log.d(TAG, "buffer: "+buffer);
Log.d(TAG, "sampleSize: "+sampleSize);
if (sampleSize < 0) {
// We shouldn't stop the playback at this point, just pass the EOS
// flag to mDecoder, we will get it again from the
// dequeueOutputBuffer
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mDecoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mDecoder.queueInputBuffer(inIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
mExtractor.advance();
}
int outIndex = mDecoder.dequeueOutputBuffer(info, TIMEOUT_US);
Log.d(TAG, "outIndex: "+outIndex);
switch (outIndex) { // outputBufferIndex
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = mDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
MediaFormat format = mDecoder.getOutputFormat();
Log.d(TAG, "New format " + format);
audioTrack.setPlaybackRate(format.getInteger(MediaFormat.KEY_SAMPLE_RATE));
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(TAG, "dequeueOutputBuffer timed out!");
break;
default:
ByteBuffer outBuffer = outputBuffers[outIndex];
Log.v(TAG, "We can't use this buffer but render it due to the API limit, " + outBuffer);
final byte[] chunk = new byte[info.size];
outBuffer.get(chunk); // Read the buffer all at once
outBuffer.clear(); // ** MUST DO!!! OTHERWISE THE NEXT TIME YOU GET THIS SAME BUFFER BAD THINGS WILL HAPPEN
audioTrack.write(chunk, info.offset, info.offset + info.size); // AudioTrack write data
mDecoder.releaseOutputBuffer(outIndex, false);
break;
}
// All decoded frames have been rendered, we can stop playing now
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
}
}
mDecoder.stop();
mDecoder.release();
mDecoder = null;
mExtractor.release();
mExtractor = null;
audioTrack.stop();
audioTrack.release();
audioTrack = null;
}
public void stop() {
eosReceived = true;
}
}
VideoDecoderThread.java
public class VideoDecoderThread extends Thread {
private static final String VIDEO = "video/";
private static final String TAG = "VideoDecoder";
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
public boolean init(Surface surface, String filePath) {
eosReceived = false;
try {
mExtractor = new MediaExtractor();
mExtractor.setDataSource(filePath);
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
MediaFormat format = mExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "mime : " + mime); // video/avc
if (mime.startsWith(VIDEO)) {
mExtractor.selectTrack(i);
mDecoder = MediaCodec.createDecoderByType(mime);
try {
Log.d(TAG, "format : " + format);
mDecoder.configure(format, surface, null, 0 /* Decoder */);
} catch (IllegalStateException e) {
Log.e(TAG, "codec '" + mime + "' failed configuration. " + e);
return false;
}
mDecoder.start();
break;
}
}
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
#Override
public void run() {
BufferInfo info = new BufferInfo();
ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
mDecoder.getOutputBuffers();
//Log.d(TAG, "mDecoder.getInputBuffers(); : " + mDecoder.getInputBuffers());
//Log.d(TAG, "mDecoder.getOutputBuffers(); : " + mDecoder.getOutputBuffers());
boolean isInput = true;
boolean first = false;
long startWhen = 0;
while (!eosReceived) {
if (isInput) {
int inputIndex = mDecoder.dequeueInputBuffer(10000);
if (inputIndex >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[inputIndex];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
mDecoder.queueInputBuffer(inputIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mDecoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isInput = false;
}
}
}
int outIndex = mDecoder.dequeueOutputBuffer(info, 10000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
mDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " + mDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
// Log.d(TAG, "INFO_TRY_AGAIN_LATER");
break;
default:
if (!first) {
startWhen = System.currentTimeMillis();
first = true;
}
try {
long sleepTime = (info.presentationTimeUs / 1000) - (System.currentTimeMillis() - startWhen);
Log.d(TAG, "info.presentationTimeUs : " + (info.presentationTimeUs / 1000) + " playTime: " + (System.currentTimeMillis() - startWhen) + " sleepTime : " + sleepTime);
if (sleepTime > 0)
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mDecoder.releaseOutputBuffer(outIndex, true /* Surface init */);
break;
}
// All decoded frames have been rendered, we can stop playing now
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
}
mDecoder.stop();
mDecoder.release();
mExtractor.release();
}
public void close() {
eosReceived = true;
}
}
I'm developing a video editing app, but I encounter some problems.I want to rearrange the video frames. And I have got an idea.First video decoding,
and the frame decoded data stored in a list, rearrange and generate new video.
I used the MediaCodec class in Android Framework to complete codec work.But I'm not familiar with it.According to my ideas, I wrote the following code:
public class VideoProcesser {
private static final String TAG = "VideoProcesser";
private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory();
private static final String OUTPUT_VIDEO_MIME_TYPE = "video/avc";
private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps
private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps
private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames
private Context mContext;
public VideoProcesser(Context mContext) {
this.mContext = mContext;
}
public void startProcessing() {
VideoChunks longChunks = generateVideoChunks(new IntegerPair(0, 180));
VideoChunks shortChunks = generateVideoChunks(new IntegerPair(30, 60));
VideoChunks dstChunks = new VideoChunks();
dstChunks.addChunks(longChunks);
dstChunks.addChunks(shortChunks);
saveChunksToFile(dstChunks, new File(OUTPUT_FILENAME_DIR, "sample_processed.mp4"));
}
private void saveChunksToFile(VideoChunks inputData, File file) {
MediaMuxer muxer = null;
MediaCodec videoEncoder = null;
try {
MediaFormat outputFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, 1280, 720);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
MediaCodecInfo videoEncoderInfo = selectCodec(getMimeTypeFor(outputFormat));
int colorFormat = selectColorFormat(videoEncoderInfo, OUTPUT_VIDEO_MIME_TYPE);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
videoEncoder = MediaCodec.createByCodecName(videoEncoderInfo.getName());
videoEncoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
videoEncoder.start();
muxer = new MediaMuxer(file.getAbsolutePath(),
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
doSaveVideoChunks(inputData, videoEncoder, muxer);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down encoder, muxer");
if (videoEncoder != null) {
videoEncoder.stop();
videoEncoder.release();
}
if (muxer != null) {
muxer.stop();
muxer.release();
}
}
}
private void doSaveVideoChunks(VideoChunks inputData, MediaCodec videoEncoder,
MediaMuxer muxer) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoEncoderInputBuffers = videoEncoder.getInputBuffers();
ByteBuffer[] videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
int inputChunk = 0;
int videoEncodedFrameCount = 0;
boolean inputDone = false;
boolean videoEncodeDone = false;
boolean muxing = false;
MediaFormat encoderOutputVideoFormat = null;
int outputVideoTrack = -1;
while (!videoEncodeDone) {
Log.d(TAG, String.format("save loop: inputChunk:%s encoded:%s", inputChunk,
videoEncodedFrameCount));
if (!inputDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderInputStatus = videoEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder input buffer");
continue;
}
long time = computePresentationTime(inputChunk);
if (inputChunk == inputData.getNumChunks()) {
// End of stream -- send empty frame with EOS flag set.
videoEncoder.queueInputBuffer(encoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.d(TAG, "input data EOS");
} else {
ByteBuffer encoderInputBuffer = videoEncoderInputBuffers[encoderInputStatus];
encoderInputBuffer.clear();
inputData.getChunkData(inputChunk, encoderInputBuffer);
videoEncoder.queueInputBuffer(encoderInputStatus, 0,
encoderInputBuffer.position(), time,
inputData.getChunkFlags(inputChunk));
inputChunk++;
}
}
if (!videoEncodeDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputStatus =
videoEncoder.dequeueOutputBuffer(videoEncoderOutputBufferInfo,
TIMEOUT_USEC);
if (encoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder output buffer");
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
Log.d(TAG, "video encoder: output media format changed"
+ encoderOutputVideoFormat);
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
muxing = true;
muxer.start();
} else if (encoderOutputStatus < 0) {
//ignore it
} else {
ByteBuffer encoderOutputBuffer = videoEncoderOutputBuffers[encoderOutputStatus];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video encoder: codec config buffer");
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
continue;
// Simply ignore codec config buffers.
}
if (videoEncoderOutputBufferInfo.size != 0) {
muxer.writeSampleData(outputVideoTrack, encoderOutputBuffer,
videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video encoder: EOS");
videoEncodeDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
videoEncodedFrameCount++;
}
}
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo, String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
Log.e(TAG,
"couldn't find a good color format for " + codecInfo.getName() + " / " + mimeType);
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for this test
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private static long computePresentationTime(int frameIndex) {
return 132 + frameIndex * 1000000 / OUTPUT_VIDEO_FRAME_RATE;
}
private VideoChunks generateVideoChunks(IntegerPair segment) {
VideoChunks outputData = new VideoChunks();
MediaExtractor videoExtractor = null;
MediaCodec videoDecoder = null;
try {
videoExtractor = createVideoExtractor(R.raw.sample);
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
MediaFormat videoInputFormat = videoExtractor.getTrackFormat(videoInputTrack);
videoDecoder = createVideoDecoder(videoInputFormat);
doGenerateVideoChunks(segment, videoExtractor, videoDecoder, outputData);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down extractor, decoder");
if (videoExtractor != null) {
videoExtractor.release();
}
if (videoDecoder != null) {
videoDecoder.stop();
videoDecoder.release();
}
}
return outputData;
}
private void doGenerateVideoChunks(IntegerPair segment, MediaExtractor videoExtractor,
MediaCodec videoDecoder, VideoChunks outputData) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoDecoderInputBuffers = videoDecoder.getInputBuffers();
ByteBuffer[] videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
boolean videoExtracted = false;
boolean videoDecoded = false;
while (true) {
Log.d(TAG, String.format("loop: extracted:%s decoded:%s", videoExtractedFrameCount,
videoDecodedFrameCount));
while (!videoExtracted) {
int decoderInputStatus = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder input buffer");
break;
}
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputStatus];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
int flags = videoExtractor.getSampleFlags();
if (size >= 0) {
videoDecoder.queueInputBuffer(decoderInputStatus, 0, size, presentationTime,
flags);
}
videoExtracted = !videoExtractor.advance();
if (videoExtracted) {
Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(decoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
break;
}
while (!videoDecoded) {
int decoderOutputStatus =
videoDecoder.dequeueOutputBuffer(videoDecoderOutputBufferInfo,
TIMEOUT_USEC);
if (decoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = videoDecoder.getOutputFormat();
Log.d(TAG, "video decoder: output format changed: " + newFormat);
break;
}
ByteBuffer decoderOutputBuffer = videoDecoderOutputBuffers[decoderOutputStatus];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputStatus, false);
break;
}
boolean render = videoDecoderOutputBufferInfo.size != 0;
if (render) {
if (segment.contains(videoDecodedFrameCount)) {
outputData.addChunk(decoderOutputBuffer, videoDecoderOutputBufferInfo.flags,
videoDecoderOutputBufferInfo.presentationTimeUs);
}
}
videoDecoder.releaseOutputBuffer(decoderOutputStatus, render);
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video decoder: EOS");
videoDecoded = true;
}
videoDecodedFrameCount++;
break;
}
if (videoDecoded) {
break;
}
}
}
private MediaCodec createVideoDecoder(MediaFormat inputFormat) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private MediaExtractor createVideoExtractor(#RawRes int resId) throws IOException {
MediaExtractor extractor = new MediaExtractor();
AssetFileDescriptor srcFd = mContext.getResources().openRawResourceFd(resId);
extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(),
srcFd.getLength());
return extractor;
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
Log.d(TAG, "format for track " + index + " is " + getMimeTypeFor(
extractor.getTrackFormat(index)));
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
}
However, But I got a video which is loss the original video data.Can you help me find what's wrong with the above code. Or have any good ideas.
This is my full code.
edit1: I found the problem.The size of the output video not correctly is the key to the problem.So I keep the same size with the original video, everything is ok.At Last, Don't forget the orientation of the video， otherwise your output video may be a bit strange.
I have the following function which takes a WAV (PCM) file and encodes it to an AAC-encoded MP4 file using Android's MediaCode and MediaMuxer classes. This is audio only. The function runs successfully and outputs a .mp4 of reasonable that is recognized as AAC-encoded. But it doesn't play on Android, web or iOS players, and crashes Audacity. Is there something I am missing? Code is shown below.
public void encode(final String from, final String to, final Callback callback) {
new Thread(new Runnable() {
#Override
public void run() {
try {
extractor.setDataSource(from);
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; ++i) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "Track " + i + " mime-type: " + mime);
if (true) {
extractor.selectTrack(i);
}
}
MediaCodec codec = MediaCodec.createEncoderByType("audio/mp4a-latm");
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_BIT_RATE, 128 * 1024);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 2);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
final MediaMuxer muxer = new MediaMuxer(to, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
final ByteBuffer byteBuffer = ByteBuffer.allocate(65536);
codec.setCallback(new MediaCodec.Callback() {
#Override
public void onInputBufferAvailable(MediaCodec codec, int bufferIndex) {
ByteBuffer inputBuffer = codec.getInputBuffer(bufferIndex);
if (isEndOfStream) {
return;
}
int sampleCapacity = inputBuffer.capacity() / 8;
if (numAvailable == 0) {
numAvailable = extractor.readSampleData(byteBuffer, 0);
if (numAvailable <= 0) {
codec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEndOfStream = true;
return;
}
long presentationTimeUs = extractor.getSampleTime();
extractor.advance();
}
if (numAvailable < sampleCapacity) {
codec.queueInputBuffer(bufferIndex, 0, numAvailable * 8, 0, 0);
numAvailable = 0;
} else {
codec.queueInputBuffer(bufferIndex, 0, sampleCapacity * 8, 0, 0);
numAvailable -= sampleCapacity;
}
}
#Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(index);
muxer.writeSampleData(audioTrackIndex,outputBuffer,info);
codec.releaseOutputBuffer(index, true);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of encoding!");
codec.stop();
codec.release();
extractor.release();
extractor = null;
muxer.stop();
muxer.release();
callback.run(true);
}
}
#Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG, "codec error", e);
}
#Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
audioTrackIndex = muxer.addTrack(format);
muxer.start();
}
});
codec.start();
} catch (IOException e) {
Log.e(TAG,"Unable to encode",e);
callback.run(false);
}
}
}).run();
You need to:
Add timestamp information correctly since media muxer need to use it to TAG packet time information.
Add logic to copy data buffer from extractor data buffer (PCM) to mediacodec input buffer, only refer to buffer index will only encode a random data buffer without initial.
Add code to apply input source property such as channels and sample rate to mediacodec. Not sure if you are intent to encode with different channels and sample rate!
Example code as below:
MediaExtractor extractor = null;
int numAvailable = 0;
boolean isEndOfStream = false;
int audioTrackIndex = 0;
long totalen = 0;
int channels = 0;
int sampleRate = 0;
public void encode(final String from, final String to) {
new Thread(new Runnable() {
#Override
public void run() {
try {
extractor = new MediaExtractor();
extractor.setDataSource(from);
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; ++i) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "Track " + i + " mime-type: " + mime);
if (true) {
extractor.selectTrack(i);
channels = extractor.getTrackFormat(i).getInteger(MediaFormat.KEY_CHANNEL_COUNT);
sampleRate = extractor.getTrackFormat(i).getInteger(MediaFormat.KEY_SAMPLE_RATE);
Log.e(TAG,"sampleRate:" + sampleRate + " channels:" + channels);
}
}
String mimeType = "audio/mp4a-latm";
MediaCodec codec = MediaCodec.createEncoderByType(mimeType);
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, mimeType);
format.setInteger(MediaFormat.KEY_BIT_RATE, 128 * 1024);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channels);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
final MediaMuxer muxer = new MediaMuxer(to, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
final ByteBuffer byteBuffer = ByteBuffer.allocate(65536);
codec.setCallback(new MediaCodec.Callback() {
#Override
public void onInputBufferAvailable(MediaCodec codec, int bufferIndex) {
ByteBuffer inputBuffer = codec.getInputBuffer(bufferIndex);
inputBuffer.clear();
if (isEndOfStream) {
return;
}
int sampleCapacity = inputBuffer.capacity();
if (numAvailable == 0) {
numAvailable = extractor.readSampleData(byteBuffer, 0);
if (numAvailable <= 0) {
codec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEndOfStream = true;
return;
}
extractor.advance();
}
long timestampUs = 1000000l * totalen / (2 * channels * sampleRate);
if (numAvailable < sampleCapacity) {
byte[] byteArray = new byte[numAvailable];
byteBuffer.get(byteArray);
inputBuffer.put(byteArray, 0, (int)numAvailable);
totalen += numAvailable;
codec.queueInputBuffer(bufferIndex, 0, numAvailable, timestampUs, 0);
numAvailable = 0;
} else {
byte[] byteArray = new byte[sampleCapacity];
byteBuffer.get(byteArray);
inputBuffer.put(byteArray, 0, (int)sampleCapacity);
totalen += sampleCapacity;
codec.queueInputBuffer(bufferIndex, 0, sampleCapacity, timestampUs, 0);
numAvailable -= sampleCapacity;
}
}
#Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(index);
muxer.writeSampleData(audioTrackIndex,outputBuffer,info);
codec.releaseOutputBuffer(index, true);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of encoding!");
codec.stop();
codec.release();
extractor.release();
extractor = null;
muxer.stop();
muxer.release();
//callback.run(true);
}
}
#Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG, "codec error", e);
}
#Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
audioTrackIndex = muxer.addTrack(format);
muxer.start();
}
});
codec.start();
} catch (IOException e) {
Log.e(TAG,"Unable to encode",e);
//callback.run(false);
}
}
}).run();
}
BTW,Why you need to divide 8 with the buffer length? And what's the Callback class? Please share the import module! I almost can not pass build with the callback parameter so comment it out!
You seem to be encoding your AAC into LATM format which isn't very popular. Maybe that's the reason players won't play it. Try using some other media type, audio/mp4 or audio/3gpp.
See AAC container formats.
I try to extract video from mp4 and mix with others audios,here is the video part:
MediaFormat videoFormat = null;
MediaMuxer mp4Muxer = createMediaMuxer();
MediaExtractor mp4Extractor = createMediaExtractor();
for (int i = 0; i < mp4Extractor.getTrackCount(); i++) {
MediaFormat format = mp4Extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
mp4Extractor.selectTrack(i);
videoFormat = format;
break;
}
}
int videoTrackIndex = mp4Muxer.addTrack(videoFormat);
mp4Muxer.start();
boolean videoMuxDone = false;
ByteBuffer videoSampleBuffer = ByteBuffer.allocateDirect(176 * 144);
BufferInfo videoBufferInfo = new BufferInfo();
int sampleSize;
while (!videoMuxDone) {
videoSampleBuffer.clear();
sampleSize = mp4Extractor.readSampleData(videoSampleBuffer, 0);
if (sampleSize < 0) {
videoMuxDone = true;
} else {
videoBufferInfo.presentationTimeUs = mp4Extractor.getSampleTime();
videoBufferInfo.flags = mp4Extractor.getSampleFlags();
videoBufferInfo.size = sampleSize;
mp4Muxer.writeSampleData(videoTrackIndex, videoSampleBuffer,videoBufferInfo);
mp4Extractor.advance();
}
}
mp4Extractor.release();
mp4Muxer.stop();
mp4Muxer.release();
creations:
private MediaExtractor createMediaExtractor() {
MediaExtractor extractor = new MediaExtractor();
try {
extractor.setDataSource(mp4VideoFile.getAbsolutePath());
} catch (IOException e) {
e.printStackTrace();
}
return extractor;
}
private MediaMuxer createMediaMuxer() {
MediaMuxer midiaMuxer = null;
try {
midiaMuxer = new MediaMuxer(OUTPUT_MP4,
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
e.printStackTrace();
}
return midiaMuxer;
}
the videoformat:
{max-input-size=6561, durationUs=3331377, height=144, mime=video/mp4v-es, csd-0=java.nio.ByteArrayBuffer[position=0,limit=30,capacity=30], width=176}
when it runs to line : mp4Muxer.stop(), exceptions happens:
MPEG4Writer Missing codec specific data
java.lang.IllegalStateException: Failed to stop the muxer
at android.media.MediaMuxer.nativeStop(Native Method)
at android.media.MediaMuxer.stop(MediaMuxer.java:226)
at com.darcye.media.Mp4Muxer.startMix(Mp4Muxer.java:155)
I have read the Extract Decode Encode Mux Audio,but still cant find the reason,please help me.