Develop Reference

Failed to stop the muxer

I'm facing a problem with media muxer. Why does my app crash? and what is the solution?
java.lang.IllegalStateException: Failed to stop the muxer
at android.media.MediaMuxer.nativeStop(Native Method)
at android.media.MediaMuxer.stop(MediaMuxer.java:245)
at com.app.filter.helper.config.VideoInfoUtils.combineVideo(VideoInfoUtils.java:159)
at com.app.filter.helper.filter.video.VideoFilterView.handleSaveCompleted(VideoFilterView.java:74)
at com.app.filter.helper.filter.video.VideoFilterView$ViewHandler.handleMessage(VideoFilterView.java:167)
at android.os.Handler.dispatchMessage(Handler.java:102)
at android.os.Looper.loop(Looper.java:154)
and my java code is
public class VideoInfoUtils {
public static MediaFormat getVideoInfo(String path) {
MediaExtractor audioExtractor = new MediaExtractor();
try {
audioExtractor.setDataSource(path);
} catch (IOException e) {
e.printStackTrace();
}
int selectTrack = selectTrack(audioExtractor);
if (selectTrack < 0) {
throw new RuntimeException("No video track found in " + path);
}
MediaFormat format = audioExtractor.getTrackFormat(selectTrack);
return format;
}
/**
* Selects the video track, if any.
*
* #return the track index, or -1 if no video track is found.
*/
private static int selectTrack(MediaExtractor mMediaExtractor) {
// Select the first video track we find, ignore the rest.
int numTracks = mMediaExtractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = mMediaExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
return i;
}
}
return -1;
}
#SuppressLint("NewApi")
public static void combineVideo(String videoPath, String newVideo, String output) throws IllegalArgumentException {
try {
MediaExtractor videoExtractor = new MediaExtractor();
videoExtractor.setDataSource(newVideo);
MediaFormat videoFormat = null;
int videoTrackIndex = -1;
int videoTrackCount = videoExtractor.getTrackCount();
for (int i = 0; i < videoTrackCount; i++) {
videoFormat = videoExtractor.getTrackFormat(i);
String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("video/")) {
videoTrackIndex = i;
break;
}
}
MediaExtractor audioExtractor = new MediaExtractor();
audioExtractor.setDataSource(videoPath);
MediaFormat audioFormat = null;
int audioTrackIndex = -1;
int audioTrackCount = audioExtractor.getTrackCount();
for (int i = 0; i < audioTrackCount; i++) {
audioFormat = audioExtractor.getTrackFormat(i);
String mimeType = audioFormat.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("audio/")) {
audioTrackIndex = i;
break;
}
}
videoExtractor.selectTrack(videoTrackIndex);
audioExtractor.selectTrack(audioTrackIndex);
MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
MediaMuxer mediaMuxer;
int writeVideoTrackIndex;
int writeAudioTrackIndex;
try {
mediaMuxer = new MediaMuxer(output, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
writeVideoTrackIndex = mediaMuxer.addTrack(videoFormat);
writeAudioTrackIndex = mediaMuxer.addTrack(audioFormat);
mediaMuxer.start();
} catch (IOException ioe) {
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024);
long videoStampTime = 0;
{
videoExtractor.readSampleData(byteBuffer, 0);
if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
videoExtractor.advance();
}
videoExtractor.readSampleData(byteBuffer, 0);
long secondTime = videoExtractor.getSampleTime();
videoExtractor.advance();
long thirdTime = videoExtractor.getSampleTime();
videoStampTime = Math.abs(thirdTime - secondTime);
}
videoExtractor.unselectTrack(videoTrackIndex);
videoExtractor.selectTrack(videoTrackIndex);
long audioStampTime = 0;
//获取帧之间的间隔时间
{
audioExtractor.readSampleData(byteBuffer, 0);
if (audioExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
audioExtractor.advance();
}
audioExtractor.readSampleData(byteBuffer, 0);
long secondTime = audioExtractor.getSampleTime();
audioExtractor.advance();
audioExtractor.readSampleData(byteBuffer, 0);
long thirdTime = audioExtractor.getSampleTime();
audioStampTime = Math.abs(thirdTime - secondTime);
Log.e("time", audioStampTime + "");
}
audioExtractor.unselectTrack(audioTrackIndex);
audioExtractor.selectTrack(audioTrackIndex);
while (true) {
int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer, 0);
if (readVideoSampleSize < 0) {
break;
}
videoBufferInfo.size = readVideoSampleSize;
videoBufferInfo.presentationTimeUs += videoStampTime;
videoBufferInfo.offset = 0;
videoBufferInfo.flags = videoExtractor.getSampleFlags();
mediaMuxer.writeSampleData(writeVideoTrackIndex, byteBuffer, videoBufferInfo);
videoExtractor.advance();
}
while (true) {
int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer, 0);
if (readAudioSampleSize < 0) {
break;
}
audioBufferInfo.size = readAudioSampleSize;
audioBufferInfo.presentationTimeUs += audioStampTime;
audioBufferInfo.offset = 0;
audioBufferInfo.flags = videoExtractor.getSampleFlags();
mediaMuxer.writeSampleData(writeAudioTrackIndex, byteBuffer, audioBufferInfo);
audioExtractor.advance();
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
}
videoExtractor.release();
audioExtractor.release();
} catch (IOException e) {
e.printStackTrace();
}
}
}
In my application i'm trying to add filters on video.
But sometimes my app crashes and sometimes it works fine.
The error is "Failed to stop the muxer"

I got below code from https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/MediaMuxerTest.java
// Throws exception b/c start() is not called.
muxer = new MediaMuxer(outputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
muxer.addTrack(MediaFormat.createVideoFormat("video/avc", 480, 320));
try {
muxer.stop();
fail("should throw IllegalStateException.");
} catch (IllegalStateException e) {
// expected
}
according to this, you need to make sure you stated mutex before you stop it. You can keep variable like isMutextStarted like to store state. But it is just hack.
I recommand you to initalize it only muxer going to start next. Otherwise it is keep as null then your mediaMuxer.stop() and release parts are not executed.

Convert Video to MP3

I want to Convert Video(mp4) to Mp3 format. Please if any one know then explain me.
I have tried this but its not working.
I have implement from this link but it wont work for me.
How do I extractor audio to mp3 from mp4 using java in Android?
public class AudioFromVideo {
private String audio, video;
private MediaCodec amc;
private MediaExtractor ame;
private MediaFormat amf;
private String amime;
public AudioFromVideo(String srcVideo, String destAudio) {
this.audio = destAudio;
this.video = srcVideo;
ame = new MediaExtractor();
init();
}
public void init() {
try {
ame.setDataSource(video);
amf = ame.getTrackFormat(1);
ame.selectTrack(1);
amime = amf.getString(MediaFormat.KEY_MIME);
amc = MediaCodec.createDecoderByType(amime);
amc.configure(amf, null, null, 0);
amc.start();
} catch (IOException e) {
e.printStackTrace();
}
}
public void start() {
new AudioService(amc, ame, audio).start();
}
private class AudioService extends Thread {
private MediaCodec amc;
private MediaExtractor ame;
private ByteBuffer[] aInputBuffers, aOutputBuffers;
private String destFile;
#SuppressWarnings("deprecation")
AudioService(MediaCodec amc, MediaExtractor ame, String destFile) {
this.amc = amc;
this.ame = ame;
this.destFile = destFile;
aInputBuffers = amc.getInputBuffers();
aOutputBuffers = amc.getOutputBuffers();
}
#SuppressWarnings("deprecation")
public void run() {
try {
OutputStream os = new FileOutputStream(new File(destFile));
long count = 0;
while (true) {
int inputIndex = amc.dequeueInputBuffer(0);
if (inputIndex == -1) {
continue;
}
int sampleSize = ame.readSampleData(aInputBuffers[inputIndex], 0);
if (sampleSize == -1) break;
long presentationTime = ame.getSampleTime();
int flag = ame.getSampleFlags();
ame.advance();
amc.queueInputBuffer(inputIndex, 0, sampleSize, presentationTime, flag);
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int outputIndex = amc.dequeueOutputBuffer(info, 0);
if (outputIndex >= 0) {
byte[] data = new byte[info.size];
aOutputBuffers[outputIndex].get(data, 0, data.length);
aOutputBuffers[outputIndex].clear();
os.write(data);
count += data.length;
Log.e("write", "" + count);
amc.releaseOutputBuffer(outputIndex, false);
} else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
aOutputBuffers = amc.getOutputBuffers();
} else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
}
}
os.flush();
os.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
new AudioFromVideo(videopath,audioPath).start();

You will need to extract Audio from Mp4 Videos using Android FFmpeg Library:
ffmpeg -i my_video.mp4 -ab 128k -ac 2 -ar 44100 -vn my_audio.mp3
Here is link to the library that does this:
http://writingminds.github.io/ffmpeg-android-java/
http://www.android-arsenal.com/details/1/931

It is possible to extract audio form vidoe file in android programmatically. You can do this without using any external library. It can be done by using MediaMuxer class.
new AudioExtractor().genVideoUsingMuxer("Video source Path", "Audio destination path", -1, -1, true, false);
See this post fro better understanding and complete code.
https://androidprogrammatically425516919.wordpress.com/2020/04/21/how-to-convert-video-to-audio-in-android-programmatically/

How can I rearrange the video frames in Android?

I'm developing a video editing app, but I encounter some problems.I want to rearrange the video frames. And I have got an idea.First video decoding,
and the frame decoded data stored in a list, rearrange and generate new video.
I used the MediaCodec class in Android Framework to complete codec work.But I'm not familiar with it.According to my ideas, I wrote the following code:
public class VideoProcesser {
private static final String TAG = "VideoProcesser";
private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory();
private static final String OUTPUT_VIDEO_MIME_TYPE = "video/avc";
private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps
private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps
private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames
private Context mContext;
public VideoProcesser(Context mContext) {
this.mContext = mContext;
}
public void startProcessing() {
VideoChunks longChunks = generateVideoChunks(new IntegerPair(0, 180));
VideoChunks shortChunks = generateVideoChunks(new IntegerPair(30, 60));
VideoChunks dstChunks = new VideoChunks();
dstChunks.addChunks(longChunks);
dstChunks.addChunks(shortChunks);
saveChunksToFile(dstChunks, new File(OUTPUT_FILENAME_DIR, "sample_processed.mp4"));
}
private void saveChunksToFile(VideoChunks inputData, File file) {
MediaMuxer muxer = null;
MediaCodec videoEncoder = null;
try {
MediaFormat outputFormat =
MediaFormat.createVideoFormat(OUTPUT_VIDEO_MIME_TYPE, 1280, 720);
outputFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE);
outputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
outputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
MediaCodecInfo videoEncoderInfo = selectCodec(getMimeTypeFor(outputFormat));
int colorFormat = selectColorFormat(videoEncoderInfo, OUTPUT_VIDEO_MIME_TYPE);
outputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
videoEncoder = MediaCodec.createByCodecName(videoEncoderInfo.getName());
videoEncoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
videoEncoder.start();
muxer = new MediaMuxer(file.getAbsolutePath(),
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
doSaveVideoChunks(inputData, videoEncoder, muxer);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down encoder, muxer");
if (videoEncoder != null) {
videoEncoder.stop();
videoEncoder.release();
}
if (muxer != null) {
muxer.stop();
muxer.release();
}
}
}
private void doSaveVideoChunks(VideoChunks inputData, MediaCodec videoEncoder,
MediaMuxer muxer) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoEncoderInputBuffers = videoEncoder.getInputBuffers();
ByteBuffer[] videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
MediaCodec.BufferInfo videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo();
int inputChunk = 0;
int videoEncodedFrameCount = 0;
boolean inputDone = false;
boolean videoEncodeDone = false;
boolean muxing = false;
MediaFormat encoderOutputVideoFormat = null;
int outputVideoTrack = -1;
while (!videoEncodeDone) {
Log.d(TAG, String.format("save loop: inputChunk:%s encoded:%s", inputChunk,
videoEncodedFrameCount));
if (!inputDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderInputStatus = videoEncoder.dequeueInputBuffer(TIMEOUT_USEC);
if (encoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder input buffer");
continue;
}
long time = computePresentationTime(inputChunk);
if (inputChunk == inputData.getNumChunks()) {
// End of stream -- send empty frame with EOS flag set.
videoEncoder.queueInputBuffer(encoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
Log.d(TAG, "input data EOS");
} else {
ByteBuffer encoderInputBuffer = videoEncoderInputBuffers[encoderInputStatus];
encoderInputBuffer.clear();
inputData.getChunkData(inputChunk, encoderInputBuffer);
videoEncoder.queueInputBuffer(encoderInputStatus, 0,
encoderInputBuffer.position(), time,
inputData.getChunkFlags(inputChunk));
inputChunk++;
}
}
if (!videoEncodeDone && (encoderOutputVideoFormat == null || muxing)) {
int encoderOutputStatus =
videoEncoder.dequeueOutputBuffer(videoEncoderOutputBufferInfo,
TIMEOUT_USEC);
if (encoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video encoder output buffer");
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video encoder: output buffers changed");
videoEncoderOutputBuffers = videoEncoder.getOutputBuffers();
} else if (encoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
encoderOutputVideoFormat = videoEncoder.getOutputFormat();
Log.d(TAG, "video encoder: output media format changed"
+ encoderOutputVideoFormat);
outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat);
muxing = true;
muxer.start();
} else if (encoderOutputStatus < 0) {
//ignore it
} else {
ByteBuffer encoderOutputBuffer = videoEncoderOutputBuffers[encoderOutputStatus];
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video encoder: codec config buffer");
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
continue;
// Simply ignore codec config buffers.
}
if (videoEncoderOutputBufferInfo.size != 0) {
muxer.writeSampleData(outputVideoTrack, encoderOutputBuffer,
videoEncoderOutputBufferInfo);
}
if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video encoder: EOS");
videoEncodeDone = true;
}
videoEncoder.releaseOutputBuffer(encoderOutputStatus, false);
videoEncodedFrameCount++;
}
}
}
}
private static MediaCodecInfo selectCodec(String mimeType) {
int numCodecs = MediaCodecList.getCodecCount();
for (int i = 0; i < numCodecs; i++) {
MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) {
continue;
}
String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
if (types[j].equalsIgnoreCase(mimeType)) {
return codecInfo;
}
}
}
return null;
}
private static int selectColorFormat(MediaCodecInfo codecInfo, String mimeType) {
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
for (int i = 0; i < capabilities.colorFormats.length; i++) {
int colorFormat = capabilities.colorFormats[i];
if (isRecognizedFormat(colorFormat)) {
return colorFormat;
}
}
Log.e(TAG,
"couldn't find a good color format for " + codecInfo.getName() + " / " + mimeType);
return 0; // not reached
}
private static boolean isRecognizedFormat(int colorFormat) {
switch (colorFormat) {
// these are the formats we know how to handle for this test
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
return true;
default:
return false;
}
}
private static long computePresentationTime(int frameIndex) {
return 132 + frameIndex * 1000000 / OUTPUT_VIDEO_FRAME_RATE;
}
private VideoChunks generateVideoChunks(IntegerPair segment) {
VideoChunks outputData = new VideoChunks();
MediaExtractor videoExtractor = null;
MediaCodec videoDecoder = null;
try {
videoExtractor = createVideoExtractor(R.raw.sample);
int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
MediaFormat videoInputFormat = videoExtractor.getTrackFormat(videoInputTrack);
videoDecoder = createVideoDecoder(videoInputFormat);
doGenerateVideoChunks(segment, videoExtractor, videoDecoder, outputData);
} catch (IOException e) {
e.printStackTrace();
} finally {
Log.d(TAG, "shutting down extractor, decoder");
if (videoExtractor != null) {
videoExtractor.release();
}
if (videoDecoder != null) {
videoDecoder.stop();
videoDecoder.release();
}
}
return outputData;
}
private void doGenerateVideoChunks(IntegerPair segment, MediaExtractor videoExtractor,
MediaCodec videoDecoder, VideoChunks outputData) {
final int TIMEOUT_USEC = 10000;
ByteBuffer[] videoDecoderInputBuffers = videoDecoder.getInputBuffers();
ByteBuffer[] videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
MediaCodec.BufferInfo videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo();
int videoExtractedFrameCount = 0;
int videoDecodedFrameCount = 0;
boolean videoExtracted = false;
boolean videoDecoded = false;
while (true) {
Log.d(TAG, String.format("loop: extracted:%s decoded:%s", videoExtractedFrameCount,
videoDecodedFrameCount));
while (!videoExtracted) {
int decoderInputStatus = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC);
if (decoderInputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder input buffer");
break;
}
ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputStatus];
int size = videoExtractor.readSampleData(decoderInputBuffer, 0);
long presentationTime = videoExtractor.getSampleTime();
int flags = videoExtractor.getSampleFlags();
if (size >= 0) {
videoDecoder.queueInputBuffer(decoderInputStatus, 0, size, presentationTime,
flags);
}
videoExtracted = !videoExtractor.advance();
if (videoExtracted) {
Log.d(TAG, "video extractor: EOS");
videoDecoder.queueInputBuffer(decoderInputStatus, 0, 0, 0L,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
videoExtractedFrameCount++;
break;
}
while (!videoDecoded) {
int decoderOutputStatus =
videoDecoder.dequeueOutputBuffer(videoDecoderOutputBufferInfo,
TIMEOUT_USEC);
if (decoderOutputStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "no video decoder output buffer");
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
Log.d(TAG, "video decoder: output buffers changed");
videoDecoderOutputBuffers = videoDecoder.getOutputBuffers();
break;
}
if (decoderOutputStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = videoDecoder.getOutputFormat();
Log.d(TAG, "video decoder: output format changed: " + newFormat);
break;
}
ByteBuffer decoderOutputBuffer = videoDecoderOutputBuffers[decoderOutputStatus];
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
!= 0) {
Log.d(TAG, "video decoder: codec config buffer");
videoDecoder.releaseOutputBuffer(decoderOutputStatus, false);
break;
}
boolean render = videoDecoderOutputBufferInfo.size != 0;
if (render) {
if (segment.contains(videoDecodedFrameCount)) {
outputData.addChunk(decoderOutputBuffer, videoDecoderOutputBufferInfo.flags,
videoDecoderOutputBufferInfo.presentationTimeUs);
}
}
videoDecoder.releaseOutputBuffer(decoderOutputStatus, render);
if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM)
!= 0) {
Log.d(TAG, "video decoder: EOS");
videoDecoded = true;
}
videoDecodedFrameCount++;
break;
}
if (videoDecoded) {
break;
}
}
}
private MediaCodec createVideoDecoder(MediaFormat inputFormat) throws IOException {
MediaCodec decoder = MediaCodec.createDecoderByType(getMimeTypeFor(inputFormat));
decoder.configure(inputFormat, null, null, 0);
decoder.start();
return decoder;
}
private MediaExtractor createVideoExtractor(#RawRes int resId) throws IOException {
MediaExtractor extractor = new MediaExtractor();
AssetFileDescriptor srcFd = mContext.getResources().openRawResourceFd(resId);
extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(),
srcFd.getLength());
return extractor;
}
private int getAndSelectVideoTrackIndex(MediaExtractor extractor) {
for (int index = 0; index < extractor.getTrackCount(); ++index) {
Log.d(TAG, "format for track " + index + " is " + getMimeTypeFor(
extractor.getTrackFormat(index)));
if (isVideoFormat(extractor.getTrackFormat(index))) {
extractor.selectTrack(index);
return index;
}
}
return -1;
}
private static String getMimeTypeFor(MediaFormat format) {
return format.getString(MediaFormat.KEY_MIME);
}
private static boolean isVideoFormat(MediaFormat format) {
return getMimeTypeFor(format).startsWith("video/");
}
}
However, But I got a video which is loss the original video data.Can you help me find what's wrong with the above code. Or have any good ideas.
This is my full code.
edit1: I found the problem.The size of the output video not correctly is the key to the problem.So I keep the same size with the original video, everything is ok.At Last, Don't forget the orientation of the video， otherwise your output video may be a bit strange.

Android: MediaCodec+MediaMuxer encoded audio MP4 won't play

I have the following function which takes a WAV (PCM) file and encodes it to an AAC-encoded MP4 file using Android's MediaCode and MediaMuxer classes. This is audio only. The function runs successfully and outputs a .mp4 of reasonable that is recognized as AAC-encoded. But it doesn't play on Android, web or iOS players, and crashes Audacity. Is there something I am missing? Code is shown below.
public void encode(final String from, final String to, final Callback callback) {
new Thread(new Runnable() {
#Override
public void run() {
try {
extractor.setDataSource(from);
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; ++i) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "Track " + i + " mime-type: " + mime);
if (true) {
extractor.selectTrack(i);
}
}
MediaCodec codec = MediaCodec.createEncoderByType("audio/mp4a-latm");
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_BIT_RATE, 128 * 1024);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 2);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
final MediaMuxer muxer = new MediaMuxer(to, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
final ByteBuffer byteBuffer = ByteBuffer.allocate(65536);
codec.setCallback(new MediaCodec.Callback() {
#Override
public void onInputBufferAvailable(MediaCodec codec, int bufferIndex) {
ByteBuffer inputBuffer = codec.getInputBuffer(bufferIndex);
if (isEndOfStream) {
return;
}
int sampleCapacity = inputBuffer.capacity() / 8;
if (numAvailable == 0) {
numAvailable = extractor.readSampleData(byteBuffer, 0);
if (numAvailable <= 0) {
codec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEndOfStream = true;
return;
}
long presentationTimeUs = extractor.getSampleTime();
extractor.advance();
}
if (numAvailable < sampleCapacity) {
codec.queueInputBuffer(bufferIndex, 0, numAvailable * 8, 0, 0);
numAvailable = 0;
} else {
codec.queueInputBuffer(bufferIndex, 0, sampleCapacity * 8, 0, 0);
numAvailable -= sampleCapacity;
}
}
#Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(index);
muxer.writeSampleData(audioTrackIndex,outputBuffer,info);
codec.releaseOutputBuffer(index, true);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of encoding!");
codec.stop();
codec.release();
extractor.release();
extractor = null;
muxer.stop();
muxer.release();
callback.run(true);
}
}
#Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG, "codec error", e);
}
#Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
audioTrackIndex = muxer.addTrack(format);
muxer.start();
}
});
codec.start();
} catch (IOException e) {
Log.e(TAG,"Unable to encode",e);
callback.run(false);
}
}
}).run();

You need to:
Add timestamp information correctly since media muxer need to use it to TAG packet time information.
Add logic to copy data buffer from extractor data buffer (PCM) to mediacodec input buffer, only refer to buffer index will only encode a random data buffer without initial.
Add code to apply input source property such as channels and sample rate to mediacodec. Not sure if you are intent to encode with different channels and sample rate!
Example code as below:
MediaExtractor extractor = null;
int numAvailable = 0;
boolean isEndOfStream = false;
int audioTrackIndex = 0;
long totalen = 0;
int channels = 0;
int sampleRate = 0;
public void encode(final String from, final String to) {
new Thread(new Runnable() {
#Override
public void run() {
try {
extractor = new MediaExtractor();
extractor.setDataSource(from);
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; ++i) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "Track " + i + " mime-type: " + mime);
if (true) {
extractor.selectTrack(i);
channels = extractor.getTrackFormat(i).getInteger(MediaFormat.KEY_CHANNEL_COUNT);
sampleRate = extractor.getTrackFormat(i).getInteger(MediaFormat.KEY_SAMPLE_RATE);
Log.e(TAG,"sampleRate:" + sampleRate + " channels:" + channels);
}
}
String mimeType = "audio/mp4a-latm";
MediaCodec codec = MediaCodec.createEncoderByType(mimeType);
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, mimeType);
format.setInteger(MediaFormat.KEY_BIT_RATE, 128 * 1024);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channels);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
final MediaMuxer muxer = new MediaMuxer(to, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
final ByteBuffer byteBuffer = ByteBuffer.allocate(65536);
codec.setCallback(new MediaCodec.Callback() {
#Override
public void onInputBufferAvailable(MediaCodec codec, int bufferIndex) {
ByteBuffer inputBuffer = codec.getInputBuffer(bufferIndex);
inputBuffer.clear();
if (isEndOfStream) {
return;
}
int sampleCapacity = inputBuffer.capacity();
if (numAvailable == 0) {
numAvailable = extractor.readSampleData(byteBuffer, 0);
if (numAvailable <= 0) {
codec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEndOfStream = true;
return;
}
extractor.advance();
}
long timestampUs = 1000000l * totalen / (2 * channels * sampleRate);
if (numAvailable < sampleCapacity) {
byte[] byteArray = new byte[numAvailable];
byteBuffer.get(byteArray);
inputBuffer.put(byteArray, 0, (int)numAvailable);
totalen += numAvailable;
codec.queueInputBuffer(bufferIndex, 0, numAvailable, timestampUs, 0);
numAvailable = 0;
} else {
byte[] byteArray = new byte[sampleCapacity];
byteBuffer.get(byteArray);
inputBuffer.put(byteArray, 0, (int)sampleCapacity);
totalen += sampleCapacity;
codec.queueInputBuffer(bufferIndex, 0, sampleCapacity, timestampUs, 0);
numAvailable -= sampleCapacity;
}
}
#Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(index);
muxer.writeSampleData(audioTrackIndex,outputBuffer,info);
codec.releaseOutputBuffer(index, true);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "end of encoding!");
codec.stop();
codec.release();
extractor.release();
extractor = null;
muxer.stop();
muxer.release();
//callback.run(true);
}
}
#Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG, "codec error", e);
}
#Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
audioTrackIndex = muxer.addTrack(format);
muxer.start();
}
});
codec.start();
} catch (IOException e) {
Log.e(TAG,"Unable to encode",e);
//callback.run(false);
}
}
}).run();
}
BTW,Why you need to divide 8 with the buffer length? And what's the Callback class? Please share the import module! I almost can not pass build with the callback parameter so comment it out!

You seem to be encoding your AAC into LATM format which isn't very popular. Maybe that's the reason players won't play it. Try using some other media type, audio/mp4 or audio/3gpp.
See AAC container formats.

How can I overlay one audio file over another one and save it?

What I am trying to accomplish is overlaying a vocal track over a music track to form a new song track.
Here is some code I have. I am reading the vocal.mp3 using FileInputStream and then saving it to a byte array like so...
try {
fis = new FileInputStream(myFile);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
bos = new ByteArrayOutputStream();
byte[] buf = new byte[2048];
try {
for (int readNum; (readNum = fis.read(buf)) != -1;) {
bos.write(buf, 0, readNum);
System.out.println("read " + readNum + " bytes,");
}
} catch (IOException ex) {
ex.printStackTrace();
}
bytes = bos.toByteArray();
Then... I do the same thing for the music.mp3 and read that into a separate byte array. I'm not going to bother showing the code for that since it is the same as above.
After I have the two separate byte arrays I can combine them like so...
outputStream = new ByteArrayOutputStream( );
try {
outputStream.write( bytes );
outputStream.write( bytes2 );
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
mixData = new byte[bytes.length + bytes2.length];
mixData = outputStream.toByteArray( );
And then write the combined byte array to a new song.mp3 file for saving like so...
File someFile = new File(songOutPath);
try {
fos2 = new FileOutputStream(someFile);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos2.write(mixData);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos2.flush();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fos2.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
This code will merge the two mp3 files into one...but they play one after another... I need to know if someone can help me find a way to get them to play simultaneously. This way the vocal and music track will play at the same time in a new song file that I'd generate.
UPDATE
Here is an update to the direction I am taking in my code.
I would like to call a method and pass it two filepaths for each seperate mp3 file, something like so:
mixSamples(String filePathOne, String filePathTwo)
Then in that method I would like to use media extractor to extract the data from each mp3 file and then decode each file. After the files have been decoded I would like to store each file in a short[] and then call the mix() method as seen below to mix the two short[]'s into one combined short[] and then encode that newly created array back into an mp3.
public void mixSamples(String filePathOne, String filePathTwo){
MediaCodec codec = null;
MediaExtractor extractor = new MediaExtractor();
try {
extractor.setDataSource(filePathOne);
return create(extractor);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
extractor.release();
}
// ... Do I create another extractor here for my second file?
MediaFormat format = extractor.getTrackFormat(0);
String mime = format.getString(MediaFormat.KEY_MIME);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 2);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
try {
codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null, null, 0);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
final long timeoutUs = 5000;
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while (!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if (!sawInputEOS) {
int inputBufferIndex = codec.dequeueInputBuffer(timeoutUs);
if (inputBufferIndex >= 0) {
ByteBuffer buffer = codecInputBuffers[inputBufferIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(inputBufferIndex, 0, sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
extractor.advance();
}
}
}
int outputBufferIndex = codec.dequeueOutputBuffer(info, timeoutUs);
if (outputBufferIndex >= 0) {
if (info.size > 0) {
noOutputCounter = 0;
}
ByteBuffer buffer = codecOutputBuffers[outputBufferIndex];
if (info.size > 0) {
// Do something... Maybe create my short[] here...
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
}
}
} catch (IOException e){
}finally {
codec.stop();
codec.release();
}
}
static short[] mix(short[] buffer, short[] mixWith, int numberOfMixSamples) {
final int length = Math.min(buffer.length, numberOfMixSamples);
int mixed;
for (int i = 0; i < length; i++) {
mixed = (int) buffer[i] + (int) mixWith[i];
if (mixed > 32767) mixed = 32767;
if (mixed < -32768) mixed = -32768;
buffer[i] = (short) mixed;
}
return buffer;
}

You want to use MediaCodec with MediaExtractor to decode mp3 (or any other audio format) to samples. Each sample is presented by short not byte. Eventually you would have short[] (number of samples). Once you decode both audio files, then you could mix samples together to produce new samples. Then revert process to encode to audio format using result samples. I used PCM16 as intermediate format. One of the ways to mix audio together can be this:
static short[] mix(short[] buffer, short[] mixWith, int numberOfMixSamples) {
final int length = Math.min(buffer.length, numberOfMixSamples);
int mixed;
for (int i = 0; i < length; i++) {
mixed = (int) buffer[i] + (int) mixWith[i];
if (mixed > 32767) mixed = 32767;
if (mixed < -32768) mixed = -32768;
buffer[i] = (short) mixed;
}
return buffer;
}
UPDATE
Giving code from my heart :) I am going to write articles on it later on my blog android.vladli.com. This code is for already deprecated code, it will work, and new API is slightly cleaner, even though not much different.
MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(file.getAbsolutePath());
try {
return create(extractor);
} finally {
extractor.release();
}
// ...
MediaFormat format = extractor.getTrackFormat(0);
String mime = format.getString(MediaFormat.KEY_MIME);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 2);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, 44100);
MediaCodec codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null, null, 0);
codec.start();
try {
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
final long timeoutUs = 5000;
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while (!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if (!sawInputEOS) {
int inputBufferIndex = codec.dequeueInputBuffer(timeoutUs);
if (inputBufferIndex >= 0) {
ByteBuffer buffer = codecInputBuffers[inputBufferIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(inputBufferIndex, 0, sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
extractor.advance();
}
}
}
int outputBufferIndex = codec.dequeueOutputBuffer(info, timeoutUs);
if (outputBufferIndex >= 0) {
if (info.size > 0) {
noOutputCounter = 0;
}
ByteBuffer buffer = codecOutputBuffers[outputBufferIndex];
if (info.size > 0) {
// data.writePcm16(buffer, info.offset, info.size);
// data here is my class to gather buffer (samples) in a queue for further playback. In your case can write them down into disk or do something else
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
}
}
} finally {
codec.stop();
codec.release();
}

I've made the same few years ago, I used to play 4 music at the same time. I used threads. Each thread played a music with media player and you are able to synchronise them with Cyclbarrier.
https://docs.oracle.com/javase/7/docs/api/java/util/concurrent/CyclicBarrier.html