In my Android App I need a solution when the user can trim a video from internal storage. I am trying to achieve this without using any third-party library. I was referencing this Google's Gallery App source code here. But I am getting the following error :
timestampUs 66733 < lastTimestampUs 133467 for Video track
There are few SO questions which talk about this timestamp issue, but I am really not being to figure out as I am new to using MediaCodecs, MediaMuxers etc. Would be great if any can hep
Here's my code :
// Set up MediaExtractor to read from the source.
MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(srcPath);
int trackCount = extractor.getTrackCount();
System.out.println("tracl" + trackCount);
// Set up MediaMuxer for the destination.
MediaMuxer muxer;
muxer = new MediaMuxer(dstPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
// Set up the tracks and retrieve the max buffer size for selected
// tracks.
HashMap<Integer, Integer> indexMap = new HashMap<>(trackCount);
int bufferSize = -1;
for (int i = 0; i < trackCount; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
boolean selectCurrentTrack = false;
if (mime.startsWith("audio/") && useAudio) {
selectCurrentTrack = true;
} else if (mime.startsWith("video/") && useVideo) {
selectCurrentTrack = true;
}
if (selectCurrentTrack) {
extractor.selectTrack(i);
int dstIndex = muxer.addTrack(format);
System.out.println(format);
System.out.println("dstIndex" + dstIndex);
indexMap.put(i, dstIndex);
if (format.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) {
int newSize = format.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
bufferSize = newSize > bufferSize ? newSize : bufferSize;
}
}
}
System.out.println(indexMap);
if (bufferSize < 0) {
bufferSize = DEFAULT_BUFFER_SIZE;
}
// Set up the orientation and starting time for extractor.
MediaMetadataRetriever retrieverSrc = new MediaMetadataRetriever();
retrieverSrc.setDataSource(srcPath);
String degreesString = retrieverSrc.extractMetadata(
MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION);
if (degreesString != null) {
int degrees = Integer.parseInt(degreesString);
if (degrees >= 0) {
muxer.setOrientationHint(degrees);
}
}
if (startMs > 0) {
extractor.seekTo(startMs * 1000, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
}
// System.out.println(extractor.);
// Copy the samples from MediaExtractor to MediaMuxer. We will loop
// for copying each sample and stop when we get to the end of the source
// file or exceed the end time of the trimming.
int offset = 0;
int trackIndex = -1;
ByteBuffer dstBuf = ByteBuffer.allocate(bufferSize);
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
try {
muxer.start();
while (true) {
System.out.println("copying");
bufferInfo.offset = offset;
bufferInfo.size = extractor.readSampleData(dstBuf, offset);
if (bufferInfo.size < 0 ) {
// InstabugSDKLogger.d(TAG, "Saw input EOS.");
System.out.println("Saw input EOS.");
bufferInfo.size = 0;
break;
} else {
/**
* The presentation timestamp in microseconds for the buffer.
* This is derived from the presentation timestamp passed in
* with the corresponding input buffer. This should be ignored for
* a 0-sized buffer.
*/
/**
* Returns the current sample's presentation time in microseconds.
* or -1 if no more samples are available.
*/
bufferInfo.presentationTimeUs = extractor.getSampleTime();
//1 sec = 1000000 micco sec
if (endMs > 0 && bufferInfo.presentationTimeUs > (endMs * 1000)) {
break;
} else {
bufferInfo.flags = extractor.getSampleFlags();
trackIndex = extractor.getSampleTrackIndex();
muxer.writeSampleData(indexMap.get(trackIndex), dstBuf, bufferInfo);
//System.out.println(muxer);
extractor.advance();
}
}
}
muxer.stop();
File file = new File(srcPath);
file.delete();
} catch (IllegalStateException e) {
System.out.println("The source video file is malformed");
} finally {
muxer.release();
}
I need to create videos with data hidden in them. i managed to extract video frames using mediacodec decoder as NV21 buffer and save them, then i create mp4 file from frames using mediacodec encoder.
the class below is responsible for saving frame files if we are in encode process or check the value if we want to extract data from stego-video.
public class ExtractMpegFramesBufferDecoder {
private static final String TAG = "ExtractMpegFramesDec";
private static final boolean VERBOSE = true; // lots of logging
// where to find files (note: requires WRITE_EXTERNAL_STORAGE permission)
private File STORE_FRAME_DIRECTORY;
private String INPUT_FILE;
private int frameRate; // stop extracting after this many
private int saveWidth;
private int saveHeight;
private int decodeCount;
private Handler _progressBarHandler;
private int duration;
//
private int MAX_FRAMES;
private boolean fromDecode;
//
public ExtractMpegFramesBufferDecoder(File storeFrameDirectory, String inputVideoPath, int frameRate
, int saveWidth, int saveHeight
, double duration, int rotation
, Handler _progressBarHandler) {
this.STORE_FRAME_DIRECTORY = storeFrameDirectory;
this.INPUT_FILE = inputVideoPath;
this.frameRate = frameRate;
this.saveWidth = saveWidth;
this.saveHeight = saveHeight;
this._progressBarHandler = _progressBarHandler;
this.duration = (int) duration;
}
/**
* Tests extraction from an MP4 to a series of PNG files.
* <p>
* We scale the video to 640x480 for the PNG just to demonstrate that we can scale the
* video with the GPU. If the input video has a different aspect ratio, we could preserve
* it by adjusting the GL viewport to get letterboxing or pillarboxing, but generally if
* you're extracting frames you don't want black bars.
*/
public void extractMpegFrames(int maxFrame, boolean fromDecode) throws IOException {
MediaCodec decoder = null;
MediaExtractor extractor = null;
MAX_FRAMES = maxFrame;
this.fromDecode = fromDecode;
try {
File inputFile = new File(INPUT_FILE); // must be an absolute path
// The MediaExtractor error messages aren't very useful. Check to see if the input
// file exists so we can throw a better one if it's not there.
if (!inputFile.canRead()) {
throw new FileNotFoundException("Unable to read " + inputFile);
}
extractor = new MediaExtractor();
extractor.setDataSource(inputFile.toString());
int trackIndex = selectTrack(extractor);
if (trackIndex < 0) {
throw new RuntimeException("No video track found in " + inputFile);
}
extractor.selectTrack(trackIndex);
MediaFormat format = extractor.getTrackFormat(trackIndex);
if (VERBOSE) {
Log.d(TAG, "Video size is " + format.getInteger(MediaFormat.KEY_WIDTH) + "x" +
format.getInteger(MediaFormat.KEY_HEIGHT));
}
// Create a MediaCodec decoder, and configure it with the MediaFormat from the
// extractor. It's very important to use the format from the extractor because
// it contains a copy of the CSD-0/CSD-1 codec-specific data chunks.
String mime = format.getString(MediaFormat.KEY_MIME);
decoder = MediaCodec.createDecoderByType(mime);
decoder.configure(format, null, null, 0);
decoder.start();
doExtract(extractor, trackIndex, decoder);
} finally {
if (decoder != null) {
decoder.stop();
decoder.release();
decoder = null;
}
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private int selectTrack(MediaExtractor extractor) {
// Select the first video track we find, ignore the rest.
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
if (VERBOSE) {
Log.d(TAG, "Extractor selected track " + i + " (" + mime + "): " + format);
}
return i;
}
}
return -1;
}
/**
* Work loop.
*/
public void doExtract(MediaExtractor extractor, int trackIndex, MediaCodec decoder) throws IOException {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
decodeCount = 0;
long frameSaveTime = 0;
boolean outputDone = false;
boolean inputDone = false;
ByteBuffer[] decoderOutputBuffers = decoder.getOutputBuffers();
MediaFormat decoderOutputFormat = null;
long rawSize = 0;
while (!outputDone) {
if (VERBOSE) Log.d(TAG, "loop");
// Feed more data to the decoder.
if (!inputDone) {
int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0) {
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.readSampleData(inputBuf, 0);
if (chunkSize < 0) {
// End of stream -- send empty frame with EOS flag set.
decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
if (VERBOSE) Log.d(TAG, "sent input EOS");
} else {
if (extractor.getSampleTrackIndex() != trackIndex) {
Log.w(TAG, "WEIRD: got sample from track " +
extractor.getSampleTrackIndex() + ", expected " + trackIndex);
}
long presentationTimeUs = extractor.getSampleTime();
decoder.queueInputBuffer(inputBufIndex, 0, chunkSize,
presentationTimeUs, 0 /*flags*/);
if (VERBOSE) {
Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
chunkSize);
}
inputChunk++;
extractor.advance();
}
} else {
if (VERBOSE) Log.d(TAG, "input buffer not available");
}
}
if (!outputDone) {
int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (VERBOSE) Log.d(TAG, "no output from decoder available");
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not important for us, since we're using Surface
if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
decoderOutputBuffers = decoder.getOutputBuffers();
} else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = decoder.getOutputFormat();
decoderOutputFormat = newFormat;
if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
} else if (decoderStatus < 0) {
Log.e(TAG, "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
} else { // decoderStatus >= 0
if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus +
" (size=" + info.size + ")");
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
ByteBuffer outputFrame = decoderOutputBuffers[decoderStatus];
outputFrame.position(info.offset);
outputFrame.limit(info.offset + info.size);
rawSize += info.size;
if (info.size == 0) {
if (VERBOSE) Log.d(TAG, "got empty frame");
} else {
// if it's decode then check the altered value
// else save the frames
if (fromDecode) {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
int size = saveWidth * saveHeight;
int offset = size;
int[] pixels = new int[size];
int u, v, y1, y2, y3, y4;
int uvIndex = 0;
if (decodeCount == 1) {
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < size; i += 2, k += 2) {
y1 = data[i] & 0xff;
y2 = data[i + 1] & 0xff;
y3 = data[saveWidth + i] & 0xff;
y4 = data[saveWidth + i + 1] & 0xff;
u = data[offset + k] & 0xff;
v = data[offset + k + 1] & 0xff;
// getting size
if (uvIndex == 0) {
int specialByte1P1 = u & 15;
int specialByte1P2 = v & 15;
int specialCharacter1 = (specialByte1P1 << 4) | specialByte1P2;
if (specialCharacter1 != 17) {
throw new IllegalArgumentException("value has changed");
}
}
uvIndex++;
if (i != 0 && (i + 2) % saveWidth == 0)
i += saveWidth;
}
}
} else {
outputFrame.rewind();
byte[] data = new byte[outputFrame.remaining()];
outputFrame.get(data);
try {
File outputFile = new File(STORE_FRAME_DIRECTORY,
String.format(Locale.US, "frame_%d.frame", decodeCount));
FileOutputStream stream = new FileOutputStream(outputFile.getAbsoluteFile());
stream.write(data);
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
decodeCount++;
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
decoder.releaseOutputBuffer(decoderStatus, false);
}
}
}
int numSaved = (frameRate < decodeCount) ? frameRate : decodeCount;
Log.d(TAG, "Saving " + numSaved + " frames took " +
(frameSaveTime / numSaved / 1000) + " us per frame");
}
public int getDecodeCount() {
return decodeCount;
}
}
in class below i encode frames, alter one u v value of (frame 1), store digit number 17 in lsb of first u and v and build mp4 using mediacodec encoder.
public class YUVFrameBufferToVideoEncoder {
private static final String TAG = BitmapToVideoEncoder.class.getSimpleName();
private static final int ERROR_IN_PROCESS = 0;
private IBitmapToVideoEncoderCallback mCallback;
private File mOutputFile;
private Queue<File> mEncodeQueue = new ConcurrentLinkedQueue();
private MediaCodec mediaCodec;
private MediaMuxer mediaMuxer;
private Object mFrameSync = new Object();
private CountDownLatch mNewFrameLatch;
private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static int mWidth;
private static int mHeight;
private static int BIT_RATE;
private static int FRAME_RATE; // Frames per second
private int frameCount;
private Handler _progressBarHandler;
private Handler _processHandler;
private static final int I_FRAME_INTERVAL = 1;
private int mGenerateIndex = 0;
private int mTrackIndex;
private boolean mNoMoreFrames = false;
private boolean mAbort = false;
//
private byte[] dataToHide;
public interface IBitmapToVideoEncoderCallback {
void onEncodingComplete(File outputFile);
}
public YUVFrameBufferToVideoEncoder(IBitmapToVideoEncoderCallback callback) {
mCallback = callback;
}
public boolean isEncodingStarted() {
return (mediaCodec != null) && (mediaMuxer != null) && !mNoMoreFrames && !mAbort;
}
public int getActiveBitmaps() {
return mEncodeQueue.size();
}
public boolean startEncoding(int width, int height, int fps, int bitrate, int frameCount
, byte[] dataToHide, Handler _progressBarHandler, Handler _processHandler
, File outputFile) {
mWidth = width;
mHeight = height;
FRAME_RATE = fps;
BIT_RATE = bitrate;
this.frameCount = frameCount;
this._progressBarHandler = _progressBarHandler;
this._processHandler = _processHandler;
mOutputFile = outputFile;
this.dataToHide = dataToHide;
String outputFileString;
try {
outputFileString = outputFile.getCanonicalPath();
} catch (IOException e) {
Log.e(TAG, "Unable to get path for " + outputFile);
ErrorManager.getInstance().addErrorMessage("Unable to get path for " + outputFile);
return false;
}
MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
if (codecInfo == null) {
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
ErrorManager.getInstance().addErrorMessage("Unable to find an appropriate codec for " + MIME_TYPE);
return false;
}
Log.d(TAG, "found codec: " + codecInfo.getName());
int colorFormat;
try {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
} catch (Exception e) {
colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
}
try {
mediaCodec = MediaCodec.createByCodecName(codecInfo.getName());
} catch (IOException e) {
Log.e(TAG, "Unable to create MediaCodec " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("Unable to create MediaCodec " + e.getMessage());
return false;
}
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL);
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mediaCodec.start();
try {
mediaMuxer = new MediaMuxer(outputFileString, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
Log.e(TAG, "MediaMuxer creation failed. " + e.getMessage());
ErrorManager.getInstance().addErrorMessage("MediaMuxer creation failed. " + e.getMessage());
return false;
}
Log.d(TAG, "Initialization complete. Starting encoder...");
Completable.fromAction(this::encode)
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe();
return true;
}
public void stopEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to stop encoding since it never started");
return;
}
Log.d(TAG, "Stopping encoding");
mNoMoreFrames = true;
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void abortEncoding() {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to abort encoding since it never started");
return;
}
Log.d(TAG, "Aborting encoding");
mNoMoreFrames = true;
mAbort = true;
mEncodeQueue = new ConcurrentLinkedQueue(); // Drop all frames
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
public void queueFrame(File frame) {
if (mediaCodec == null || mediaMuxer == null) {
Log.d(TAG, "Failed to queue frame. Encoding not started");
return;
}
Log.d(TAG, "Queueing frame");
mEncodeQueue.add(frame);
synchronized (mFrameSync) {
if ((mNewFrameLatch != null) && (mNewFrameLatch.getCount() > 0)) {
mNewFrameLatch.countDown();
}
}
}
private void encode() {
Log.d(TAG, "Encoder started");
while (true) {
if (mNoMoreFrames && (mEncodeQueue.size() == 0)) break;
File frame = mEncodeQueue.poll();
if (frame == null) {
synchronized (mFrameSync) {
mNewFrameLatch = new CountDownLatch(1);
}
try {
mNewFrameLatch.await();
} catch (InterruptedException e) {
}
frame = mEncodeQueue.poll();
}
if (frame == null) continue;
int size = (int) frame.length();
byte[] bytesNV21 = new byte[size];
try {
BufferedInputStream buf = new BufferedInputStream(new FileInputStream(frame));
buf.read(bytesNV21, 0, bytesNV21.length);
buf.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
int offsetSize = mWidth * mHeight;
int byteNV21Offset = offsetSize;
int u, v, y1, y2, y3, y4;
//
int dataToHideLength = 0;
if (dataToHide != null)
dataToHideLength = dataToHide.length;
boolean isLastIndexInserted1 = false;
boolean isLastIndexInserted2 = false;
boolean isLastIndexInserted3 = false;
int uvIndex = 0;
int frameByteCapacity = ((mWidth * mHeight) / 4) / 20;
Log.e(TAG, "encode: dataToHideLength: " + dataToHideLength);
Log.e(TAG, "encode: frameByteCapacity: " + dataToHideLength);
//
// i percorre os Y and the final pixels
// k percorre os pixles U e V
for (int i = 0, k = 0; i < offsetSize; i += 2, k += 2) {
y1 = bytesNV21[i] & 0xff;
y2 = bytesNV21[i + 1] & 0xff;
y3 = bytesNV21[mWidth + i] & 0xff;
y4 = bytesNV21[mWidth + i + 1] & 0xff;
u = bytesNV21[byteNV21Offset + k] & 0xff;
v = bytesNV21[byteNV21Offset + k + 1] & 0xff;
// frame 1
// altering u and v for test
if (mGenerateIndex == 1) {
int Unew = u & 240;
int Vnew = v & 240;
if (uvIndex == 0) {
// used in start and end of stego bytes
int specialByte1Integer = 17;
int specialByte1P1 = specialByte1Integer & 240;
int specialByte1P2 = specialByte1Integer & 15;
// shift p1 right 4 position
specialByte1P1 = specialByte1P1 >> 4;
u = Unew | specialByte1P1;
v = Vnew | specialByte1P2;
}
bytesNV21[byteNV21Offset + k] = (byte) u;
bytesNV21[byteNV21Offset + k + 1] = (byte) v;
}
uvIndex++;
if (i != 0 && (i + 2) % mWidth == 0)
i += mWidth;
}
long TIMEOUT_USEC = 500000;
int inputBufIndex = mediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
long ptsUsec = computePresentationTime(mGenerateIndex, FRAME_RATE);
if (inputBufIndex >= 0) {
final ByteBuffer inputBuffer = mediaCodec.getInputBuffers()[inputBufIndex];
inputBuffer.clear();
inputBuffer.put(bytesNV21);
mediaCodec.queueInputBuffer(inputBufIndex, 0, bytesNV21.length, ptsUsec, 0);
mGenerateIndex++;
int percentComplete = 70 + (int) ((((double) mGenerateIndex) / (frameCount)) * 30);
if (_progressBarHandler != null) {
_progressBarHandler.sendMessage(_progressBarHandler.obtainMessage(percentComplete));
}
Log.w("creatingVideo: ", "is:" + percentComplete);
}
MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo();
int encoderStatus = mediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
Log.e(TAG, "No output from encoder available");
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// not expected for an encoder
MediaFormat newFormat = mediaCodec.getOutputFormat();
mTrackIndex = mediaMuxer.addTrack(newFormat);
mediaMuxer.start();
} else if (encoderStatus < 0) {
Log.e(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
} else if (mBufferInfo.size != 0) {
ByteBuffer encodedData = mediaCodec.getOutputBuffers()[encoderStatus];
if (encodedData == null) {
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
} else {
encodedData.position(mBufferInfo.offset);
encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
mediaMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
mediaCodec.releaseOutputBuffer(encoderStatus, false);
}
}
}
release();
if (mAbort) {
mOutputFile.delete();
} else {
mCallback.onEncodingComplete(mOutputFile);
}
}
private void release() {
try {
if (mediaCodec != null) {
mediaCodec.stop();
mediaCodec.release();
mediaCodec = null;
Log.d(TAG, "RELEASE CODEC");
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
mediaMuxer = null;
Log.d(TAG, "RELEASE MUXER");
}
} catch (Exception ignored) {
ErrorManager.getInstance().addErrorMessage("unsupported video file");
Message res = _processHandler.obtainMessage(ERROR_IN_PROCESS);
_processHandler.sendMessage(res);
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo,
String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo
.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private long computePresentationTime(long frameIndex, int framerate) {
return 132 + frameIndex * 1000000 / framerate;
}}
output video is created successfully without any problem, but mediacodec has changed the altered test value and i cannot retrieve it.
here is my question, is this a right approach for doing video steganography in android? if it is not the right way can you please make a suggestion?
Steganography comes with a prerequisite - lossless encoding.
None of the codecs available on Android support lossless video encoding, as of now.
So I'm afraid your LSBs would never remain the same post encoding/decoding.
Suggestion: If you don't have a lot many frames, I would suggest you use a lossless format. You may encode your frames into a sequence of PNG images.
I'm facing a problem with media muxer. Why does my app crash? and what is the solution?
java.lang.IllegalStateException: Failed to stop the muxer
at android.media.MediaMuxer.nativeStop(Native Method)
at android.media.MediaMuxer.stop(MediaMuxer.java:245)
at com.app.filter.helper.config.VideoInfoUtils.combineVideo(VideoInfoUtils.java:159)
at com.app.filter.helper.filter.video.VideoFilterView.handleSaveCompleted(VideoFilterView.java:74)
at com.app.filter.helper.filter.video.VideoFilterView$ViewHandler.handleMessage(VideoFilterView.java:167)
at android.os.Handler.dispatchMessage(Handler.java:102)
at android.os.Looper.loop(Looper.java:154)
and my java code is
public class VideoInfoUtils {
public static MediaFormat getVideoInfo(String path) {
MediaExtractor audioExtractor = new MediaExtractor();
try {
audioExtractor.setDataSource(path);
} catch (IOException e) {
e.printStackTrace();
}
int selectTrack = selectTrack(audioExtractor);
if (selectTrack < 0) {
throw new RuntimeException("No video track found in " + path);
}
MediaFormat format = audioExtractor.getTrackFormat(selectTrack);
return format;
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private static int selectTrack(MediaExtractor mMediaExtractor) {
// Select the first video track we find, ignore the rest.
int numTracks = mMediaExtractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = mMediaExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
return i;
}
}
return -1;
}
#SuppressLint("NewApi")
public static void combineVideo(String videoPath, String newVideo, String output) throws IllegalArgumentException {
try {
MediaExtractor videoExtractor = new MediaExtractor();
videoExtractor.setDataSource(newVideo);
MediaFormat videoFormat = null;
int videoTrackIndex = -1;
int videoTrackCount = videoExtractor.getTrackCount();
for (int i = 0; i < videoTrackCount; i++) {
videoFormat = videoExtractor.getTrackFormat(i);
String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("video/")) {
videoTrackIndex = i;
break;
}
}
MediaExtractor audioExtractor = new MediaExtractor();
audioExtractor.setDataSource(videoPath);
MediaFormat audioFormat = null;
int audioTrackIndex = -1;
int audioTrackCount = audioExtractor.getTrackCount();
for (int i = 0; i < audioTrackCount; i++) {
audioFormat = audioExtractor.getTrackFormat(i);
String mimeType = audioFormat.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("audio/")) {
audioTrackIndex = i;
break;
}
}
videoExtractor.selectTrack(videoTrackIndex);
audioExtractor.selectTrack(audioTrackIndex);
MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
MediaMuxer mediaMuxer;
int writeVideoTrackIndex;
int writeAudioTrackIndex;
try {
mediaMuxer = new MediaMuxer(output, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
writeVideoTrackIndex = mediaMuxer.addTrack(videoFormat);
writeAudioTrackIndex = mediaMuxer.addTrack(audioFormat);
mediaMuxer.start();
} catch (IOException ioe) {
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024);
long videoStampTime = 0;
{
videoExtractor.readSampleData(byteBuffer, 0);
if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
videoExtractor.advance();
}
videoExtractor.readSampleData(byteBuffer, 0);
long secondTime = videoExtractor.getSampleTime();
videoExtractor.advance();
long thirdTime = videoExtractor.getSampleTime();
videoStampTime = Math.abs(thirdTime - secondTime);
}
videoExtractor.unselectTrack(videoTrackIndex);
videoExtractor.selectTrack(videoTrackIndex);
long audioStampTime = 0;
//获取帧之间的间隔时间
{
audioExtractor.readSampleData(byteBuffer, 0);
if (audioExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
audioExtractor.advance();
}
audioExtractor.readSampleData(byteBuffer, 0);
long secondTime = audioExtractor.getSampleTime();
audioExtractor.advance();
audioExtractor.readSampleData(byteBuffer, 0);
long thirdTime = audioExtractor.getSampleTime();
audioStampTime = Math.abs(thirdTime - secondTime);
Log.e("time", audioStampTime + "");
}
audioExtractor.unselectTrack(audioTrackIndex);
audioExtractor.selectTrack(audioTrackIndex);
while (true) {
int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer, 0);
if (readVideoSampleSize < 0) {
break;
}
videoBufferInfo.size = readVideoSampleSize;
videoBufferInfo.presentationTimeUs += videoStampTime;
videoBufferInfo.offset = 0;
videoBufferInfo.flags = videoExtractor.getSampleFlags();
mediaMuxer.writeSampleData(writeVideoTrackIndex, byteBuffer, videoBufferInfo);
videoExtractor.advance();
}
while (true) {
int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer, 0);
if (readAudioSampleSize < 0) {
break;
}
audioBufferInfo.size = readAudioSampleSize;
audioBufferInfo.presentationTimeUs += audioStampTime;
audioBufferInfo.offset = 0;
audioBufferInfo.flags = videoExtractor.getSampleFlags();
mediaMuxer.writeSampleData(writeAudioTrackIndex, byteBuffer, audioBufferInfo);
audioExtractor.advance();
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
}
videoExtractor.release();
audioExtractor.release();
} catch (IOException e) {
e.printStackTrace();
}
}
}
In my application i'm trying to add filters on video.
But sometimes my app crashes and sometimes it works fine.
The error is "Failed to stop the muxer"
I got below code from https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/MediaMuxerTest.java
// Throws exception b/c start() is not called.
muxer = new MediaMuxer(outputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
muxer.addTrack(MediaFormat.createVideoFormat("video/avc", 480, 320));
try {
muxer.stop();
fail("should throw IllegalStateException.");
} catch (IllegalStateException e) {
// expected
}
according to this, you need to make sure you stated mutex before you stop it. You can keep variable like isMutextStarted like to store state. But it is just hack.
I recommand you to initalize it only muxer going to start next. Otherwise it is keep as null then your mediaMuxer.stop() and release parts are not executed.
I am using MediaCodec to convert a .wav file to .amr. I have used following code for input and getting encoded buffer. I get the encoded file but it does not play, my input is proper as i am able to play that file in Audacity. I am using EncodeDecodeTest.java from android API test. Any pointers to potential problem are appreciated.
MediaCodec codec = MediaCodec.createByCodecName(componentName);
try {
codec.configure(
format,
null /* surface */,
null /* crypto */,
MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (IllegalStateException e) {
AppLog.logString("codec '" + componentName + "' failed configuration.");
// assertTrue("codec '" + componentName + "' failed configuration.", false);
}
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
int numBytesSubmitted = 0;
boolean doneSubmittingInput = false;
int numBytesDequeued = 0;
while (true) {
int index =0 ;
if (!doneSubmittingInput) {
index = codec.dequeueInputBuffer(kTimeoutUs /* timeoutUs */);
if (index != MediaCodec.INFO_TRY_AGAIN_LATER) {
if (numBytesSubmitted >= filesize ){
codec.queueInputBuffer(
index,
0 /* offset */,
0 /* size */,
0 /* timeUs */,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
if (VERBOSE) {
AppLog.logString("queued input EOS.");
}
doneSubmittingInput = true;
} else {
int inputsize = 0;
int size = 0;
if ((filesize - numBytesSubmitted) > codecInputBuffers[index]
.limit()) {
inputsize = codecInputBuffers[index].limit();
} else
inputsize = (int) (filesize - numBytesSubmitted);
size = queueInputBuffer(codec,
codecInputBuffers, index, in, inputsize);
numBytesSubmitted += size;
/*if (VERBOSE) {
AppLog.logString("queued " + size + " bytes of input data.");
}*/
}
}else
AppLog.logString("MediaCodec.INFO_TRY_AGAIN_LATER.");
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
index = codec.dequeueOutputBuffer(info, kTimeoutUs /* timeoutUs */);
if (index == MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
AppLog.logString("INFO_OUTPUT_FORMAT_CHANGED");
MediaFormat fmt = codec.getOutputFormat();
AppLog.logString(fmt.getString(MediaFormat.KEY_MIME));
} else if (index == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
} else {
dequeueOutputBuffer(codec, codecOutputBuffers, index, info, fos);
numBytesDequeued += info.size;
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (VERBOSE) {
AppLog.logString("dequeued output EOS.");
}
break;
}
if (VERBOSE) {
AppLog.logString("dequeued " + info.size + " bytes of output data.");
}
}
}
if (VERBOSE) {
AppLog.logString("queued a total of " + numBytesSubmitted + "bytes, "
+ "dequeued " + numBytesDequeued + " bytes.");
}
codec.release();
codec = null;
queueInputBuffer and dequeueOutputBuffer are implemented as follows:
private int queueInputBuffer(
MediaCodec codec, ByteBuffer[] inputBuffers, int index, FileInputStream in, int size) {
ByteBuffer buffer = inputBuffers[index];
buffer.clear();
// int size = buffer.limit();
AppLog.logString("Size is :" + size);
byte[] data = new byte[size];
try {
in.read(data,0,size);
} catch (IOException e) {
e.printStackTrace();
}
buffer.put(data);
buffer.flip();
codec.queueInputBuffer(index, 0 /* offset */, size, 0 /* timeUs */, MediaCodec.BUFFER_FLAG_SYNC_FRAME);
return size;
}
private void dequeueOutputBuffer(
MediaCodec codec, ByteBuffer[] outputBuffers,
int index, MediaCodec.BufferInfo info, FileOutputStream fos) {
byte[] data= new byte[info.size];
ByteBuffer out = outputBuffers[index];
out.get(data);
try {
fos.write(data);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
codec.releaseOutputBuffer(index, false /* render */);
}
I'm experimenting with Android MediaCodec, with the code I got from here: https://code.google.com/p/android-source-browsing.platform--cts
When I tried to run the code "android-source-browsing.platform--cts/tests/tests/media/src/android/media/cts/DecoderTest.java", I got these errors:
D/android.media.cts.DecoderTest(6615): decode(): master = 2131034134
D/android.media.cts.DecoderTest(6615): decode(): masterFd = {AssetFileDescriptor: {ParcelFileDescriptor: FileDescriptor[50]} start=1781867 len=529200}
D/android.media.cts.DecoderTest(6615): decode(): masterLength = 529200
D/android.media.cts.DecoderTest(6615): decode(): is = android.content.res.AssetFileDescriptor$AutoCloseInputStream#417592a0
D/android.media.cts.DecoderTest(6615): decode(): bis = java.io.BufferedInputStream#41759368
I/ExtractorLoader(6615): Register extractor from [lib_k3_avi_extractor.so]
E/ExtractorLoader(6615): Open [lib_k3_mov_extractor.so] failed!
E/ExtractorLoader(6615): Open [lib_k3_flv_extractor.so] failed!
I/ExtractorLoader(6615): Register extractor from [lib_k3_m2ts_extractor.so]
E/ExtractorLoader(6615): Open [lib_k3_asf_extractor.so] failed!
E/ExtractorLoader(6615): Open [lib_k3_generic_audio_extractor.so] failed!
D/android.media.cts.DecoderTest(6615): decode(): testAssetFd = {AssetFileDescriptor: {ParcelFileDescriptor: FileDescriptor[50]} start=1591735 len=49318}
D/android.media.cts.DecoderTest(6615): decode(): testFd = FileDescriptor[50]
D/android.media.cts.DecoderTest(6615): decode(): trackCount = 1
D/android.media.cts.DecoderTest(6615): decode(): fileLength = 49318
D/android.media.cts.DecoderTest(6615): decode(): format = {durationUs=3056326, encoder-padding=1908, channel-count=2, encoder-delay=576, mime=audio/mpeg, sample-rate=44100}
D/android.media.cts.DecoderTest(6615): decode(): mime = audio/mpeg
Apparently the libraries lib_k3_avi_extractor.so, lib_k3_mov_extractor.so, etc, are not in my device. Here are my questions: What are these libraries? What are they for? Why are they not in the device? I googled for these libraries, but strangely the search came up with absolutely nothing.
The Java code that generated the errors is listed below, with my minor changes to add the log information:
private void decode(int testinput, int master, float maxerror) throws IOException {
// read master file into memory
AssetFileDescriptor masterFd = mResources.openRawResourceFd(master);
long masterLength = masterFd.getLength(); //in bytes?
short[] masterBuffer = new short[(int)(masterLength/2)];
InputStream is = masterFd.createInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
Log.d(TAG, "decode(): master = " + master);
Log.d(TAG, "decode(): masterFd = " + masterFd);
Log.d(TAG, "decode(): masterLength = " + masterLength);
Log.d(TAG, "decode(): is = " + is);
Log.d(TAG, "decode(): bis = " + bis);
for (int i = 0; i < masterBuffer.length; i++) {
int lo = bis.read();
int hi = bis.read();
if (hi >= 128) {
hi -= 256;
}
int sample = hi * 256 + lo;
masterBuffer[i] = (short) sample;
}
bis.close();
MediaExtractor extractor;
MediaCodec codec;
ByteBuffer[] codecInputBuffers;
ByteBuffer[] codecOutputBuffers;
AssetFileDescriptor testAssetFd = mResources.openRawResourceFd(testinput);
FileDescriptor testFd = testAssetFd.getFileDescriptor();
extractor = new MediaExtractor();
extractor.setDataSource(testFd, testAssetFd.getStartOffset(), testAssetFd.getLength());
//assertEquals("wrong number of tracks", 1, extractor.getTrackCount());
MediaFormat format = extractor.getTrackFormat(0);
String mime = format.getString(MediaFormat.KEY_MIME);
//assertTrue("not an audio file", mime.startsWith("audio/"));
Log.d(TAG, "decode(): testAssetFd = " + testAssetFd);
Log.d(TAG, "decode(): testFd = " + testFd);
Log.d(TAG, "decode(): trackCount = " + extractor.getTrackCount());
Log.d(TAG, "decode(): fileLength = " + testAssetFd.getLength());
Log.d(TAG, "decode(): format = " + format);
Log.d(TAG, "decode(): mime = " + mime);
codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
codec.start();
codecInputBuffers = codec.getInputBuffers();
codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
// start decoding
int numBytesDecoded = 0;
int maxdelta = 0;
long totalErrorSquared = 0;
final long kTimeOutUs = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
while (!sawOutputEOS) {
if (!sawInputEOS) {
// transfer control of input buffer from codec to Java code
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
// fill input buffer with data
int sampleSize = extractor.readSampleData(dstBuf, 0 /* offset */);
long presentationTimeUs = 0;
if (sampleSize < 0) {
Log.d(TAG, "saw input EOS.");
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
// let the codec control input buffer again
codec.queueInputBuffer(
inputBufIndex,
0 /* offset */,
sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
extractor.advance();
}
}
}
int res = codec.dequeueOutputBuffer(info, kTimeOutUs);
if (res >= 0) {
int outputBufIndex = res;
ByteBuffer buf = codecOutputBuffers[outputBufIndex];
// check the waveform matches within the specified max error
for (int i = 0; i < info.size; i += 2) {
short sample = buf.getShort(i);
int idx = (numBytesDecoded + i) / 2;
//assertTrue("decoder returned too much data", idx < masterBuffer.length);
short mastersample = masterBuffer[idx];
int d = sample - mastersample;
totalErrorSquared += d * d;
}
numBytesDecoded += info.size;
codec.releaseOutputBuffer(outputBufIndex, false /* render */);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat oformat = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + oformat);
}
}
codec.stop();
codec.release();
testAssetFd.close();
masterFd.close();
//assertEquals("wrong data size", masterLength, numBytesDecoded);
long avgErrorSquared = (totalErrorSquared / (numBytesDecoded / 2));
double rmse = Math.sqrt(avgErrorSquared);
//assertTrue("decoding error too big: " + rmse, rmse <= maxerror);
}