decode h264 raw stream using mediacodec - android

I recieve h264 data from server, I want to decode this stream using mediacodec and texture view on android.I got the data from the server , parssing it to get the SPS , the PPS and the video frame data, then I passed this data to the mediacodec , but the function dequeueOutputBuffer(info, 100000) always returns -1 and I get dequeueOutputBuffer timed out.
Any help please, I'am stucked at this issues from three weeks.
this is the code used to decode the video frame.
public class H264PlayerActivity extends AppCompatActivity implements TextureView.SurfaceTextureListener {
private TextureView m_surface;// View that contains the Surface Texture
private H264Provider provider;// Object that connects to our server and gets H264 frames
private MediaCodec m_codec;// Media decoder
// private DecodeFramesTask m_frameTask;// AsyncTask that takes H264 frames and uses the decoder to update the Surface Texture
// the channel used to receive the partner's video
private ZMQ.Socket subscriber = null;
private ZMQ.Context context;
// thread handling the video reception
// byte[] byte_SPSPPS = null;
//byte[] byte_Frame = null;
public static String stringSubscribe=null;
public static String myIpAcquisition=null;
public static byte[] byte_SPSPPS = null;
public static byte[] byte_Frame = null;
boolean isIframe = false;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.h264player_activity);
Bundle extras = getIntent().getExtras();
if(extras!=null)
{
stringSubscribe=extras.getString("stringSubscribe");
myIpAcquisition=(extras.getString("myIpAcquisition"));
}
// Get a referance to the TextureView in the UI
m_surface = (TextureView) findViewById(R.id.textureView);
// Add this class as a call back so we can catch the events from the Surface Texture
m_surface.setSurfaceTextureListener(this);
}
#RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN)
#Override
// Invoked when a TextureView's SurfaceTexture is ready for use
public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
// when the surface is ready, we make a H264 provider Object. When its constructor runs it starts an AsyncTask to log into our server and start getting frames
provider = new H264Provider(stringSubscribe, myIpAcquisition,byte_SPSPPS,byte_Frame);
}
#Override
// Invoked when the SurfaceTexture's buffers size changed
public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
}
#Override
// Invoked when the specified SurfaceTexture is about to be destroyed
public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
return false;
}
#Override
// Invoked when the specified SurfaceTexture is updated through updateTexImage()
public void onSurfaceTextureUpdated(SurfaceTexture surface) {
}
private class H264Provider {
String stringSubscribe = "";
String myIpAcquisition = "";
byte[] byte_SPSPPS = null;
byte[] byte_PPS = null;
byte[] byte_Frame = null;
H264Provider(String stringSubscribe, String myIpAcquisition, byte[] byte_SPS, byte[] byte_Frame) {
this.stringSubscribe = stringSubscribe;
this.myIpAcquisition = myIpAcquisition;
this.byte_SPSPPS = byte_SPS;
this.byte_PPS = byte_PPS;
this.byte_Frame = byte_Frame;
System.out.println(" subscriber client started");
//SetUpConnection setup=new SetUpConnection();
// setup.execute();
PlayerThread mPlayer = new PlayerThread();
mPlayer.start();
}
void release(){
// close ØMQ socket
subscriber.close();
//terminate 0MQ context
context.term();
}
byte[] getCSD( ) {
return byte_SPSPPS;
}
byte[] nextFrame( ) {
return byte_Frame;
}
private class PlayerThread extends Thread
{
public PlayerThread()
{
System.out.println(" subscriber client started");
}
#TargetApi(Build.VERSION_CODES.LOLLIPOP)
#RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN)
#Override
public void run() {
/******************************************ZMQ****************************/
// Prepare our context and subscriber
ZMQ.Context context = ZMQ.context(1);
//create 0MQ socket
ZMQ.Socket subscriber = context.socket(ZMQ.SUB);
//create outgoing connection from socket
String address = "tcp://" + myIpAcquisition + ":xxxx";
Boolean bbbb = subscriber.connect(address);
subscriber.setHWM(20);// the number of messages to queue.
Log.e("zmq_tag", "connect connect " + bbbb);
//boolean bbbb = subscriber.setSocketOpt(zmq.ZMQ.ZMQ_SNDHWM, 1);
subscriber.subscribe(stringSubscribe.getBytes(ZMQ.CHARSET));
Log.e("zmq_tag", " zmq stringSubscribe " + stringSubscribe);
boolean bRun = true;
while (bRun) {
ZMsg msg = ZMsg.recvMsg(subscriber);
String string_SPS = null;
String string_PPS = null;
String SPSPPS = null;
String string_Frame = null;
if (msg != null) {
// create a video message out of the zmq message
VideoMessage oVideoMsg = VideoMessage.fromZMsg(msg);
// wait until get Iframe
String szInfoPublisher = new String(oVideoMsg.szInfoPublisher);
Log.e("zmq_tag", "szInfoPublisher " + szInfoPublisher);
if (szInfoPublisher.contains("0000000167")) {
isIframe = true;
String[] split_IFrame = szInfoPublisher.split("0000000165");
String SPS__PPS = split_IFrame[0];
String [] split_SPSPPS=SPS__PPS.split("0000000167");
SPSPPS="0000000167" + split_SPSPPS[1];
Log.e("zmq_tag", "SPS+PPS " + SPSPPS);
String iFrame = "0000000165" + split_IFrame[1];
Log.e("zmq_tag", "IFrame " + iFrame);
string_Frame = iFrame;
} else {
if ((szInfoPublisher.contains("0000000161")||szInfoPublisher.contains("0000000141")) && isIframe) {
if (szInfoPublisher.contains("0000000161"))
{
String[] split_IFrame = szInfoPublisher.split("0000000161");
String newMSG = "0000000161" + split_IFrame[1];
Log.e("zmq_tag", " P Frame " + newMSG);
string_Frame = newMSG;
} else
if (szInfoPublisher.contains("0000000141"))
{
String[] split_IFrame = szInfoPublisher.split("0000000141");
String newMSG = "0000000141" + split_IFrame[1];
Log.e("zmq_tag", " P Frame " + newMSG);
string_Frame = newMSG;
}
} else {
isIframe = false;
}
}
}
if (SPSPPS != null) {
byte_SPSPPS = SPSPPS.getBytes();
Log.e("zmq_tag", " byte_SPSPPS " + new String(byte_SPSPPS));
}
if (string_Frame != null) {
byte_Frame = string_Frame.getBytes();
Log.e("zmq_tag", " byte_Frame " + new String(byte_Frame));
}
if(SPSPPS != null) {
// Create the format settinsg for the MediaCodec
MediaFormat format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080);// MIMETYPE: a two-part identifier for file formats and format contents
// Set the PPS and SPS frame
format.setByteBuffer("csd-0", ByteBuffer.wrap(byte_SPSPPS));
// Set the buffer size
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 100000);
try {
// Get an instance of MediaCodec and give it its Mime type
m_codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
// Configure the Codec
m_codec.configure(format, new Surface(m_surface.getSurfaceTexture()), null, 0);
// Start the codec
m_codec.start();
// Create the AsyncTask to get the frames and decode them using the Codec
while (!Thread.interrupted()) {
// Get the next frame
byte[] frame = byte_Frame;
Log.e("zmq_tag", " frame " + new String(frame));
// Now we need to give it to the Codec to decode into the surface
// Get the input buffer from the decoder
int inputIndex = m_codec.dequeueInputBuffer(1);// Pass in -1 here as in this example we don't have a playback time reference
Log.e("zmq_tag", "inputIndex " + inputIndex);
// If the buffer number is valid use the buffer with that index
if (inputIndex >= 0) {
ByteBuffer buffer =m_codec.getInputBuffer(inputIndex);
buffer.put(frame);
// tell the decoder to process the frame
m_codec.queueInputBuffer(inputIndex, 0, frame.length, 0, 0);
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int outputIndex = m_codec.dequeueOutputBuffer(info, 100000);
Log.e("zmq_tag", "outputIndex " + outputIndex);
if (outputIndex >= 0) {
m_codec.releaseOutputBuffer(outputIndex, true);
}
// wait for the next frame to be ready, our server makes a frame every 250ms
try {
Thread.sleep(250);
} catch (Exception e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
// close ØMQ socket
subscriber.close();
//terminate 0MQ context
context.term();
}
}

sorry I can't comment, but I see some probable mistakes in your code :
Your KEY_MAX_INPUT_SIZE is wrong, it must be at least your Height * Width, in your case the Height * Width = 1920 * 1080 = 2073600 > 100000, you feed your decoder input buffer with data that can be > 100000, so since the decoder wants NALUs it probably wouldn't like it.
You do not clear input buffer before pushing data (realy needed?)

Related

Screen Casting starts but only thumbnail is visible and unable to play video in android 11

I'm working on a screen recorder app where if app is in backstack on android 11 and on starting of screen recording only screen casting option is visible but "app is using microphone" message is not displayed and on stopping recording only shows thumbnail and video is not played as video size is too small this is my recording code and this error is only occuring in android 11 OS other are working fine
public void startRecording() {
synchronized (sSync) {
Log.i(TAG, "startRecording: inside synchronized");
if (mMuxer == null) {
Log.i(TAG, "startRecording: muxer is null");
getScreenSize();
mMediaProjection = mMediaProjectionManager.getMediaProjection(mScreenCaptureResultCode, mScreenCaptureIntent);
DisplayManager dm = (DisplayManager) Objects.requireNonNull(MyApplication.Companion.getApplication()).getSystemService(Context.DISPLAY_SERVICE);
Display defaultDisplay;
if (dm != null) {
Log.i(TAG, "startRecording: dim is not null");
defaultDisplay = dm.getDisplay(Display.DEFAULT_DISPLAY);
} else {
throw new IllegalStateException("Cannot display manager?!?");
}
if (defaultDisplay == null) {
throw new RuntimeException("No display found.");
}
try {
mMuxer = new MediaMuxerWrapper(context, ".mp4"); // if you record audio only, ".m4a" is also OK.
if (true) {
VideoSetting videoSetting = SettingManager.getVideoProfile(context);
Log.i(TAG, "Video config: " + videoSetting);
mCurrentVideoSetting = videoSetting;
List<RenderUtil.CustomDecorator> decors = createDecorators();
// new TestMediaScreenEncoderHard(mMuxer, mMediaEncoderListener, mMediaProjection, mCurrentVideoSetting, mScreenDensity, decors);
if (MyUtils.isRunningOnEmulator()) {
new MediaScreenEncoder(mMuxer, mMediaEncoderListener, mMediaProjection, mCurrentVideoSetting, mScreenDensity, decors);
} else {
new MediaScreenEncoderHard(mMuxer, mMediaEncoderListener, mMediaProjection, mCurrentVideoSetting, mScreenDensity, decors);
}
}
if (true) {
// for audio capturing
new MediaAudioEncoder(mMuxer, mMediaEncoderListener);
Log.i(TAG, "startRecording: audio capture");
}
mMuxer.prepare();
mMuxer.startRecording();
} catch (final IOException e) {
Log.e(TAG, "startScreenRecord:", e);
}
}
}
This is my media encoder class
public class MediaAudioEncoder extends MediaEncoder {
public static final int SAMPLES_PER_FRAME = 1024; // AAC, bytes/frame/channel
public static final int FRAMES_PER_BUFFER = 25; // AAC, frame/buffer/sec
private static final String TAG = MediaAudioEncoder.class.getSimpleName();
private static final String MIME_TYPE = "audio/mp4a-latm";
private static final int SAMPLE_RATE = 44100; // 44.1[KHz] is only setting guaranteed to be available on all devices.
private static final int BIT_RATE = 64000;
private static final int[] AUDIO_SOURCES = new int[]{
MediaRecorder.AudioSource.CAMCORDER,
MediaRecorder.AudioSource.MIC,
MediaRecorder.AudioSource.DEFAULT,
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
MediaRecorder.AudioSource.VOICE_RECOGNITION,
};
private AudioThread mAudioThread = null;
public MediaAudioEncoder(final MediaMuxerWrapper muxer, final MediaEncoderListener listener) {
super(muxer, listener);
}
/**
* select the first codec that match a specific MIME type
*
* #param mimeType
* #return
*/
private static final MediaCodecInfo selectAudioCodec(final String mimeType) {
// if (DEBUG) Log.i(TAG, "selectAudioCodec:");
MediaCodecInfo result = null;
// get the list of available codecs
final int numCodecs = MediaCodecList.getCodecCount();
LOOP:
for (int i = 0; i < numCodecs; i++) {
final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
if (!codecInfo.isEncoder()) { // skipp decoder
continue;
}
final String[] types = codecInfo.getSupportedTypes();
for (int j = 0; j < types.length; j++) {
// if (DEBUG) Log.i(TAG, "supportedType:" + codecInfo.getName() + ",MIME=" + types[j]);
if (types[j].equalsIgnoreCase(mimeType)) {
if (result == null) {
result = codecInfo;
break LOOP;
}
}
}
}
return result;
}
#Override
public void prepare() throws IOException {
// if (DEBUG) Log.i(TAG, "prepare:");
mTrackIndex = -1;
mMuxerStarted = mIsEOS = false;
// prepare MediaCodec for AAC encoding of audio data from inernal mic.
final MediaCodecInfo audioCodecInfo = selectAudioCodec(MIME_TYPE);
if (audioCodecInfo == null) {
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
return;
}
// if (DEBUG) Log.i(TAG, "selected codec: " + audioCodecInfo.getName());
final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLE_RATE, 1);
audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO);
audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
// if (DEBUG) Log.i(TAG, "format: " + audioFormat);
mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE);
mMediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mMediaCodec.start();
// if (DEBUG) Log.i(TAG, "prepare finishing");
if (mListener != null) {
try {
mListener.onPrepared(this);
} catch (final Exception e) {
Log.e(TAG, "prepare:", e);
}
}
}
#Override
public void startRecording() {
super.startRecording();
// create and execute audio capturing thread using internal mic
if (mAudioThread == null) {
mAudioThread = new AudioThread();
mAudioThread.start();
}
}
#Override
protected void release() {
mAudioThread = null;
super.release();
}
/**
* Thread to capture audio data from internal mic as uncompressed 16bit PCM data
* and write them to the MediaCodec encoder
*/
private class AudioThread extends Thread {
#SuppressLint("MissingPermission")
#Override
public void run() {
try {
final int min_buffer_size = AudioRecord.getMinBufferSize(
SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT);
int buffer_size = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER;
if (buffer_size < min_buffer_size)
buffer_size = ((min_buffer_size / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
AudioRecord audioRecord = null;
for (final int source : AUDIO_SOURCES) {
try {
audioRecord = new AudioRecord(
source, SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, buffer_size);
if (audioRecord != null) {
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
audioRecord.release();
audioRecord = null;
}
}
} catch (final Exception e) {
audioRecord = null;
}
if (audioRecord != null) break;
}
if (audioRecord != null) {
try {
if (mIsEncoding) {
// if (DEBUG) Log.i(TAG, "AudioThread:start audio recording");
final ByteBuffer buf = ByteBuffer.allocateDirect(SAMPLES_PER_FRAME);
int readBytes;
audioRecord.startRecording();
try {
for (; mIsEncoding && !mRequestStop && !mIsEOS; ) {
// read audio data from internal mic
buf.clear();
readBytes = audioRecord.read(buf, SAMPLES_PER_FRAME);
if (readBytes > 0) {
// set audio data to encoder
buf.position(readBytes);
buf.flip();
encode(buf, readBytes, getPTSUs());
frameAvailableSoon();
}
}
frameAvailableSoon();
} finally {
audioRecord.stop();
}
}
} finally {
audioRecord.release();
}
} else {
Log.e(TAG, "failed to initialize AudioRecord");
}
} catch (final Exception e) {
Log.e(TAG, "AudioThread#run", e);
}
// if (DEBUG) Log.i(TAG, "AudioThread:finished");
}
}
}
These are my error logs
2022-06-09 09:40:56.785 21415-21616/com.example.recorderscreen
E/IAudioFlinger: createRecord returned error -1 2022-06-09
09:40:56.785 21415-21616/com.example.recorderscreen E/AudioRecord:
createRecord_l(436738312): AudioFlinger could not create record track,
status: -1 2022-06-09 09:40:56.785
21415-21616/com.example.recorderscreen E/AudioRecord-JNI: Error
creating AudioRecord instance: initialization check failed with status
-1. 2022-06-09 09:40:56.785 21415-21616/com.example.recorderscreen E/android.media.AudioRecord: Error code -20 when initializing native
AudioRecord object. 2022-06-09 09:40:56.785
21415-21616/com.example.recorderscreen E/MediaAudioEncoder: failed to
initialize AudioRecord

Recreate muxer for new video file while encoder works

I use CameraToMpegTest from BigFlake, but I modify it to enable start and stop muxer many time while encoder works. When the video recording is not, I still use the encoder to get frames from texture for other processing.
For first start/stop it works fine, but there is a problem with second start/stop. I get exception then close app: "stop() is called in invalid state 3", "...Caused by: java.lang.IllegalStateException: Failed to stop the muxer...". Also second video file doesn't open but it's not empty.
Is there a possibility to restart the muxer while continuously running the encoder? What am I doing wrong?
UPDATE: I find out that problem in keyframes. When I set format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 0) then all is fine. And so, how to request a keyframe when muxer starts recording again?
public class MainFrameProcessor {
public static final String TAG = "MainFrameProcessor";
public static final boolean VERBOSE = false; // lots of logging
// parameters for the encoder
private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding
private static final int FRAME_RATE = 30; // 30fps
private static final int IFRAME_INTERVAL = 5; // 5 seconds between I-frames
public static final int encWidth = 1920;
public static final int encHeight = 1080;
private static final int encBitRate = 10000000;//6000000; // Mbps //http://static.googleusercontent.com/media/source.android.com/en//compatibility/android-cdd.pdf
public ByteBuffer mPixelBuf; // used by saveFrame()
private boolean isWrite=false;
private boolean isWork=true;
// encoder / muxer state
private MediaCodec mEncoder;
private CodecInputSurface mInputSurface;
private Writer writer;
private String path;
// camera state
private Camera mCamera;
private SurfaceTextureManager mStManager;
private Handler messageHandler;
public MainFrameProcessor(String path, Handler messageHandler){
mPixelBuf = ByteBuffer.allocateDirect(encHeight * encWidth * 4);
mPixelBuf.order(ByteOrder.LITTLE_ENDIAN);
this.path=path;
this.messageHandler=messageHandler;
}
/** test entry point */
public void startProcessor() throws Throwable {
CameraToMpegWrapper.runTest(this);
}
public void stopProcessor() {
isWork=false;
}
synchronized public void release(){
stopProcessor();
releaseCamera();
releaseEncoderAndWriter();
releaseSurfaceTexture();
}
/**
* Wraps processCameraFrames(). This is necessary because SurfaceTexture will try to use
* the looper in the current thread if one exists, and the CTS tests create one on the
* test thread.
*
* The wrapper propagates exceptions thrown by the worker thread back to the caller.
*/
private static class CameraToMpegWrapper implements Runnable {
private Throwable mThrowable;
private MainFrameProcessor mTest;
private CameraToMpegWrapper(MainFrameProcessor test) {
mTest = test;
}
#Override
public void run() {
try {
mTest.processCameraFrames();
} catch (Throwable th) {
mThrowable = th;
}
}
/** Entry point. */
public static void runTest(MainFrameProcessor obj) throws Throwable {
CameraToMpegWrapper wrapper = new CameraToMpegWrapper(obj);
Thread th = new Thread(wrapper, "codec test");
th.start();
//th.join(); //http://stackoverflow.hex1.ru/questions/22457623/surfacetextures-onframeavailable-method-always-called-too-late
if (wrapper.mThrowable != null) {
throw wrapper.mThrowable;
}
}
}
boolean to_start=false;
public void startRecord(){
to_start=true;
}
boolean to_stop=false;
public void stopRecord(){
to_stop=true;
}
/**
* Tests encoding of AVC video from Camera input. The output is saved as an MP4 file.
*/
private void processCameraFrames() {
// arbitrary but popular values
Log.d(TAG, MIME_TYPE + " output " + encWidth + "x" + encHeight + " #" + encBitRate);
try {
prepareCamera(encWidth, encHeight);
prepareEncoderAndWriter(encWidth, encHeight, encBitRate);
mInputSurface.makeCurrent();
prepareSurfaceTexture();
mCamera.startPreview();
SurfaceTexture st = mStManager.getSurfaceTexture();
while (isWork) {
if (to_start){
writer.startWriter();
isWrite=true;
to_start=false;
}
if (to_stop){
isWrite=false;
writer.stopWriter();
to_stop=false;
}
// Acquire a new frame of input, and render it to the Surface. If we had a
// GLSurfaceView we could switch EGL contexts and call drawImage() a second
// time to render it on screen. The texture can be shared between contexts by
// passing the GLSurfaceView's EGLContext as eglCreateContext()'s share_context
// argument.
mStManager.awaitNewImage();
mStManager.drawImage();
synchronized (mPixelBuf) {
mPixelBuf.rewind();
GLES20.glReadPixels(0, 0, encWidth, encHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE,
mPixelBuf);
}
if (isWrite) {
if (writer.checkDurationEnd()) {
stopRecord();
Message msg = messageHandler.obtainMessage(MainActivity.MESSAGE_STOP_REC);
messageHandler.sendMessage(msg);
} else
writer.write(st, mInputSurface);
}
}
} finally {
// release everything we grabbed
release();
}
}
/**
* Configures Camera for video capture. Sets mCamera.
* <p>
* Opens a Camera and sets parameters. Does not start preview.
*/
private void prepareCamera(int encWidth, int encHeight) {
if (mCamera != null) {
throw new RuntimeException("camera already initialized");
}
Camera.CameraInfo info = new Camera.CameraInfo();
mCamera = Camera.open(); // opens first back-facing camera
if (mCamera == null) {
throw new RuntimeException("Unable to open camera");
}
Camera.Parameters parms = mCamera.getParameters();
choosePreviewSize(parms, encWidth, encHeight);
// leave the frame rate set to default
mCamera.setParameters(parms);
Camera.Size size = parms.getPreviewSize();
Log.d(TAG, "Camera preview size is " + size.width + "x" + size.height);
}
/**
* Attempts to find a preview size that matches the provided width and height (which
* specify the dimensions of the encoded video). If it fails to find a match it just
* uses the default preview size.
* <p>
* TODO: should do a best-fit match.
*/
private static void choosePreviewSize(Camera.Parameters parms, int width, int height) {
// We should make sure that the requested MPEG size is less than the preferred
// size, and has the same aspect ratio.
Camera.Size ppsfv = parms.getPreferredPreviewSizeForVideo();
if (VERBOSE && ppsfv != null) {
Log.d(TAG, "Camera preferred preview size for video is " +
ppsfv.width + "x" + ppsfv.height);
}
for (Camera.Size size : parms.getSupportedPreviewSizes()) {
if (size.width == width && size.height == height) {
parms.setPreviewSize(width, height);
return;
}
}
Log.w(TAG, "Unable to set preview size to " + width + "x" + height);
if (ppsfv != null) {
parms.setPreviewSize(ppsfv.width, ppsfv.height);
}
}
/**
* Stops camera preview, and releases the camera to the system.
*/
private void releaseCamera() {
if (VERBOSE) Log.d(TAG, "releasing camera");
if (mCamera != null) {
mCamera.stopPreview();
mCamera.release();
mCamera = null;
}
}
/**
* Configures SurfaceTexture for camera preview. Initializes mStManager, and sets the
* associated SurfaceTexture as the Camera's "preview texture".
* <p>
* Configure the EGL surface that will be used for output before calling here.
*/
private void prepareSurfaceTexture() {
mStManager = new SurfaceTextureManager(mPixelBuf);
SurfaceTexture st = mStManager.getSurfaceTexture();
try {
mCamera.setPreviewTexture(st);
} catch (IOException ioe) {
throw new RuntimeException("setPreviewTexture failed", ioe);
}
}
/**
* Releases the SurfaceTexture.
*/
private void releaseSurfaceTexture() {
if (mStManager != null) {
mStManager.release();
mStManager = null;
}
}
/**
* Configures encoder and muxer state, and prepares the input Surface. Initializes
* mEncoder, mMuxer, mInputSurface, mBufferInfo, mTrackIndex, and mMuxerStarted.
*/
private void prepareEncoderAndWriter(int width, int height, int bitRate) {
MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, width, height);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
format.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "format: " + format);
// Create a MediaCodec encoder, and configure it with our format. Get a Surface
// we can use for input and wrap it with a class that handles the EGL work.
//
// If you want to have two EGL contexts -- one for display, one for recording --
// you will likely want to defer instantiation of CodecInputSurface until after the
// "display" EGL context is created, then modify the eglCreateContext call to
// take eglGetCurrentContext() as the share_context argument.
try {
mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
} catch (IOException e) {
e.printStackTrace();
}
mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mInputSurface = new CodecInputSurface(mEncoder.createInputSurface());
mEncoder.start();
writer=new Writer(path, mEncoder);
}
/**
* Releases encoder resources.
*/
private void releaseEncoderAndWriter() {
if (VERBOSE) Log.d(TAG, "releasing encoder objects");
if (writer!=null)
writer.releaseWriter();
if (mEncoder != null) {
mEncoder.stop();
mEncoder.release();
mEncoder = null;
}
if (mInputSurface != null) {
mInputSurface.release();
mInputSurface = null;
}
}
}
Class for Muxer:
public class Writer {
private static final long DURATION_SEC = 1000; // seconds of video
private MediaCodec mEncoder;
private MediaMuxer mMuxer;
private String path;
private int mTrackIndex;
private long startWhen;
private long desiredEnd;
private boolean mMuxerStarted;
// allocate one of these up front so we don't need to do it every time
private MediaCodec.BufferInfo mBufferInfo;
boolean new_track=false;
public Writer(String path, MediaCodec mEncoder){
this.path=path;
this.mEncoder=mEncoder;
}
public void startWriter(){
// Output filename. Ideally this would use Context.getFilesDir() rather than a
// hard-coded output directory.
String outputPath= CameraUtils.createPathFile(path, "mp4");
Log.i(MainFrameProcessor.TAG, "Output file is " + outputPath);
// Create a MediaMuxer. We can't add the video track and start() the muxer here,
// because our MediaFormat doesn't have the Magic Goodies. These can only be
// obtained from the encoder after it has started processing data.
//
// We're not actually interested in multiplexing audio. We just want to convert
// the raw H.264 elementary stream we get from MediaCodec into a .mp4 file.
try {
mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException ioe) {
throw new RuntimeException("MediaMuxer creation failed", ioe);
}
mBufferInfo = new MediaCodec.BufferInfo();
mTrackIndex = -1;
mMuxerStarted = false;
new_track=true;
startWhen = System.nanoTime();
desiredEnd = startWhen + DURATION_SEC * 1000000000L;
}
synchronized public void stopWriter(){
// send end-of-stream to encoder, and drain remaining output
if (mMuxer != null) {
Log.w(MainFrameProcessor.TAG,"stop");
//if (mEncoder!=null) drainEncoder(true);
mMuxer.stop();
mMuxer.release();
mMuxer = null;
}
}
public void releaseWriter() {
stopWriter();
}
public void write(SurfaceTexture st, CodecInputSurface mInputSurface) {
// Set the presentation time stamp from the SurfaceTexture's time stamp. This
// will be used by MediaMuxer to set the PTS in the video.
if (MainFrameProcessor.VERBOSE) {
Log.d(MainFrameProcessor.TAG, "present: " +
((st.getTimestamp() - startWhen) / 1000000.0) + "ms");
}
mInputSurface.setPresentationTime(st.getTimestamp());
// Submit it to the encoder. The eglSwapBuffers call will block if the input
// is full, which would be bad if it stayed full until we dequeued an output
// buffer (which we can't do, since we're stuck here). So long as we fully drain
// the encoder before supplying additional input, the system guarantees that we
// can supply another frame without blocking.
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "sending frame to encoder");
mInputSurface.swapBuffers();
drainEncoder(false);
}
/**
* Extracts all pending data from the encoder and forwards it to the muxer.
* <p>
* If endOfStream is not set, this returns when there is no more data to drain. If it
* is set, we send EOS to the encoder, and then iterate until we see EOS on the output.
* Calling this with endOfStream set should be done once, right before stopping the muxer.
* <p>
* We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream). We're
* not recording audio.
*/
private void drainEncoder(boolean endOfStream) {
final int TIMEOUT_USEC = 10000;
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "drainEncoder(" + endOfStream + ")");
if (endOfStream) {
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "sending EOS to encoder");
mEncoder.signalEndOfInputStream();
}
ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
while (true) {
int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
Log.w("ddd", ""+encoderStatus+"; "+new_track);
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
// no output available yet
if (!endOfStream) {
break; // out of while
} else {
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "no output available, spinning to await EOS");
}
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// not expected for an encoder
encoderOutputBuffers = mEncoder.getOutputBuffers();
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// should happen before receiving buffers, and should only happen once
/*if (mMuxerStarted) {
throw new RuntimeException("format changed twice");
}
MediaFormat newFormat = mEncoder.getOutputFormat();
Log.d(MainFrameProcessor.TAG, "encoder output format changed: " + newFormat);
// now that we have the Magic Goodies, start the muxer
mTrackIndex = mMuxer.addTrack(newFormat);
mMuxer.start();
mMuxerStarted = true;*/
} else if (encoderStatus < 0) {
Log.w(MainFrameProcessor.TAG, "unexpected result from encoder.dequeueOutputBuffer: " +
encoderStatus);
// let's ignore it
} else if (new_track){
MediaFormat newFormat = mEncoder.getOutputFormat();
Log.d(MainFrameProcessor.TAG, "encoder output format changed: " + newFormat);
// now that we have the Magic Goodies, start the muxer
mTrackIndex = mMuxer.addTrack(newFormat);
mMuxer.start();
mMuxerStarted = true;
new_track=false;
} else {
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null) {
throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
" was null");
}
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
mBufferInfo.size = 0;
}
if (mBufferInfo.size != 0) {
if (!mMuxerStarted) {
throw new RuntimeException("muxer hasn't started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(mBufferInfo.offset);
encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
mMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo);
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "sent " + mBufferInfo.size + " bytes to muxer");
}
mEncoder.releaseOutputBuffer(encoderStatus, false);
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (!endOfStream) {
Log.w(MainFrameProcessor.TAG, "reached end of stream unexpectedly");
} else {
if (MainFrameProcessor.VERBOSE) Log.d(MainFrameProcessor.TAG, "end of stream reached");
}
break; // out of while
}
}
}
}
public boolean checkDurationEnd() {
return System.nanoTime() >= desiredEnd;
}
}

Slow H264 1080P#60fps Decoding on Android Lollipop 5.0.2

I'm developing a JAVA RTP Streaming App for a company project, which should be capable of joining the Multicast Server and receive the RTP Packets.Later I use the H264 Depacketizer to recreate the a complete frame from the NAL FU (Keep append the data until End Bit & Marker Bit set )
I want to decode and display a raw h264 video byte stream in Android and therefore I'm currently using the MediaCodec classes with Hardware Decoder configured.
The Application is Up and running for the Jeallybean (API 17). Various Resolutions which I need to decodes are :
480P at 30/60 FPS
720P/I at 30/60 FPS
1080P/I at 30/60 FPS
Recently, Due to System Upgrade we are porting the App to Android L Version 5.0.2. My App is not capable of playing the high resolutions videos like 720p#60fps and 1080p#60fps.
For the debugging purpose I started feeding the Elementary H264 Frames with size from the dump file to MediaCodec and found out the Video is Lagging.
There are timestamps on the sample video I used and it seems the actual time taken to proceed by 1 sec in Rendered Video is more
Below is my sample code and links to sample video
h264 video https://www.dropbox.com/s/cocjhhovihm8q25/dump60fps.h264?dl=0
h264 framesize https://www.dropbox.com/s/r146d5zederrne1/dump60fps.size?dl=0
Also as this is my question on stackoverflow, Please bear with me on Bad code formatting and Direct references.
public class MainActivity extends Activity {
static final String TAG = "MainActivity";
private PlayerThread mPlayer = null;
private static final String MIME_TYPE = "video/avc";
private byte[] mSPSPPSFrame = new byte [3000];
private byte[] sps = new byte[37];
File videoFile = null;
File videoFile1 = null;
TextView tv ;
FileInputStream videoFileStream = null;
FileInputStream videoFileStream1 = null;
int[] tall = null ;
SpeedControlCallback mspeed = new SpeedControlCallback();
int mStreamLen = 0;
FrameLayout game;
RelativeLayout rl ;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
//mVideoSurfaceView = (SurfaceView)findViewById(R.id.videoSurfaceView);
setContentView(R.layout.activity_main);
SurfaceView first = (SurfaceView) findViewById(R.id.firstSurface);
first.getHolder().addCallback(new SurfaceHolder.Callback() {
#Override
public void surfaceCreated(SurfaceHolder surfaceHolder) {
Log.d(TAG, "First surface created!");
}
#Override
public void surfaceChanged(SurfaceHolder surfaceHolder, int i, int i2, int i3) {
Log.d(TAG, "surfaceChanged()");
surfaceHolder.getSurface();
if (mPlayer == null) {
mPlayer = new PlayerThread(surfaceHolder.getSurface());
mPlayer.start();
}
}
#Override
public void surfaceDestroyed(SurfaceHolder surfaceHolder) {
Log.d(TAG, "First surface destroyed!");
}
});
tv = (TextView) findViewById(R.id.textview);
videoFile = new File("/data/local/tmp/dump60fps.h264");
videoFile1 = new File("/data/local/tmp/dump60fps.size");
}
private class PlayerThread extends Thread {
private Surface surface;
public PlayerThread(Surface surface) {
this.surface = surface;
}
#Override
public void run() {
try {
decodeVideo(0, 1920,1080, 50, surface);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (Throwable e) {
e.printStackTrace();
}
}
}
private void decodeVideo(int testinput, int width, int height,
int threshold, Surface surface) throws Throwable {
MediaCodec codec = null;
MediaFormat mFormat;
final long kTimeOutUs = 10000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
MediaFormat oformat = null;
int errors = -1;
long presentationTimeUs = 0L;
boolean mVideoStart = false;
byte[] byteArray = new byte[65525*5*3];
int i;
int sizeInBytes = 0, index, sampleSize = 0;
try {
byte[] bytes = new byte[(int) videoFile1.length()];
FileInputStream fis = new FileInputStream(videoFile1);
fis.read(bytes);
fis.close();
String[] valueStr = new String(bytes).trim().split("\\s+");
tall = new int[valueStr.length];
mStreamLen = valueStr.length;
Log.e(TAG, "++++++ Total Frames ++++++"+mStreamLen);
for ( i = 0; i < valueStr.length; i++) {
tall[i] = Integer.parseInt(valueStr[i]);
}
} catch (IOException e1) {
e1.printStackTrace();
}
index =1;
try {
videoFileStream = new FileInputStream(videoFile);
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
System.currentTimeMillis();
if (mVideoStart == false) {
try {
sizeInBytes = videoFileStream.read(mSPSPPSFrame, 0,37);
Log.e(TAG, "VideoEngine configure ."+sizeInBytes);
//for (i = 0 ; i < sizeInBytes; i++){
// Log.e(TAG, "VideoEngine ."+mSPSPPSFrame[i]);}
} catch (IOException e1) {
e1.printStackTrace();
}
sampleSize = sizeInBytes;
index++;
index++;
mFormat = MediaFormat.createVideoFormat(MIME_TYPE, 1920,1080);
mFormat.setByteBuffer("csd-0", ByteBuffer.wrap( mSPSPPSFrame,0, sizeInBytes));
codec = MediaCodec.createDecoderByType(MIME_TYPE);
codec.configure(mFormat, surface /*surface*/ , null /* crypto */, 0 /* flags */);
codec.start();
codec.getInputBuffers();
codec.getOutputBuffers();
}
// index = 0;
while (!sawOutputEOS && errors < 0) {
if (!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
//Log.d(TAG, String.format("Archana Dqing the input buffer with BufIndex #: %d",inputBufIndex));
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codec.getInputBuffers()[inputBufIndex];
/*
* Read data from file and copy to the input ByteBuffer
*/
try {
sizeInBytes = videoFileStream.read(byteArray, 0,
tall[index] /*+ 4*/);
sampleSize = tall[index]/*+ 4*/;
index++;
} catch (IOException e) {
e.printStackTrace();
}
if (sizeInBytes <= 0) {
codec.queueInputBuffer(
inputBufIndex,
0 /* offset */,
0,
presentationTimeUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM );
sawInputEOS = true;
}
else {
dstBuf.put(byteArray, 0, sizeInBytes);
if (mVideoStart == false) mVideoStart = true;
codec.queueInputBuffer(
inputBufIndex,
0 /* offset */,
sampleSize,
presentationTimeUs,
mVideoStart ? 0:MediaCodec.BUFFER_FLAG_CODEC_CONFIG );
//Log.d(TAG, String.format(" After queueing the buffer to decoder with inputbufindex and samplesize #: %d ,%d ind %d",inputBufIndex,sampleSize,index));
}
}
}
int res = codec.dequeueOutputBuffer(info, kTimeOutUs);
//Log.d(TAG, String.format(" Getting the information about decoded output buffer flags,offset,PT,size #: %d %d %d %d",info.flags,info.offset,info.presentationTimeUs,info.size));
//Log.d(TAG, String.format(" Getting the output of decoder in res #: %d",res));
if (res >= 0) {
int outputBufIndex = res;
//Log.d(TAG, "Output PTS "+info.presentationTimeUs);
//mspeed.preRender(info.presentationTimeUs);
//mspeed.setFixedPlaybackRate(25);
codec.releaseOutputBuffer(outputBufIndex, true /* render */);
//Log.d(TAG, String.format(" releaseoutputbuffer index= #: %d",outputBufIndex));
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
oformat = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + oformat);
}
}
codec.stop();
codec.release();
this.finish();
}
#Override
public boolean onCreateOptionsMenu(Menu menu) {
getMenuInflater().inflate(R.menu.activity_main, menu);
return true;
}
}
There are couples of workaround to problem with the above sample test.
Instead of feeding One Full frame to the decoder Inout, I was feeding single of NAL Units at a time. But still the playback was slow and could not match 60FPS
Google has changed the Implementation of Surface BufferQueue from Asynchronous to Synchronous.Hence when we call MediaCodec.dequeueBuffer to get decoded data, the server side (SurfaceTexture::dequeueBuffer) will wait for a buffer to be queued, and the client side waits for that, so that SurfaceTextureClient::dequeueBuffer will not return until a buffer has actually been queued on the server side. Where as in the Asynchronous Mode, a new GraphicBuffer is allocated.

Displaying h264 video from an mpegts stream over udp:// on android

Displaying h264 video from an mpegts stream over udp:// on android.
I've been trying for a few days to get this to work with no success. What I have is a device that produces a h264 video stream that it multicasts over in an mpegts container over raw udp (not rtp). I'm trying to get this to display in a custom app on android.
I read that android's built in MediaPlayer supports both h264 (avc) and mpegts, but that it does not handle udp:// streams, so I cannot use that (which would be by far the simplest). Instead, I have tried to manually parse the mpegts stream into an elementary stream and pass that to a MediaCodec that's been passed the surface of a SurfaceView. No matter what I seem to try, two things always happen (once I fix exceptions, etc):
The SurfaceView is always black.
The MediaCodec always accepts about 6-9 buffers and then dequeueInputBuffer just starts instantly failing (returning -1) and I cannot queue anything else.
I can split the mpeg stream into TS packets and then join their payloads into PES packets. I've tried passing full PES packets (minus the PES header) into MediaCodec.
I've also tried splitting the PES packets into individual NAL units by splitting on \x00\x00\x01 and passing them individually into the MediaCodec.
I've also tried holding off on passing in NAL unit until I've received the SPS NAL unit and passing that first with BUFFER_FLAG_CODEC_CONFIG.
All of these result in the same thing mentiond above. I am out of ideas about what to try, so any help would be greatly appreciated.
Some points I'm still not sure about:
Nearly all the examples I've seen get the MediaFormat from MediaExtractor, which I can't use on the stream. The few that don't use MediaExtractor explicity set csd-0 and csd-1 from bytestrings that aren't explained. I read that SPS packet can be put in the buffer instead so that's what I tried.
I'm not sure what to pass into presentationTimeUs. The TS packets have a PCR and the PES packets have a PTS, but I don't know what's expected by the api and how these relate.
I'm not sure how the data needs to be passed into MediaCodec (is this why it stops giving me buffers?). I got the idea of passing in individual NAL units from this so post:
Decoding Raw H264 stream in android?
other references I used to make this example:
MPEG-TS Format
PES Format
PES Format
code (sorry it's pretty long):
I just created a test app from the basic template in AndroidStudio, most of it is boilerplate so I'll just paste the video related stuff.
SurfaceView is defined in the xml, so grab it and get the surface when it's created/changed
public class VideoPlayer extends Activity implements SurfaceHolder.Callback {
private static final String TAG = VideoPlayer.class.getName();
PlayerThread playerThread;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_video_player);
SurfaceView view = (SurfaceView) findViewById(R.id.surface);
view.getHolder().addCallback(this);
}
...
#Override
public void surfaceCreated(SurfaceHolder surfaceHolder) {
Log.d(TAG,"surfaceCreated");
}
#Override
public void surfaceChanged(SurfaceHolder surfaceHolder, int i, int i2, int i3) {
Log.d("main","surfaceChanged");
if( playerThread == null ) {
playerThread = new PlayerThread(surfaceHolder.getSurface());
playerThread.start();
}
}
...
PlayerThread is an internal class that reads data from a multicast port and passes it to a parsing function on a background thread:
class PlayerThread extends Thread {
private final String TAG = PlayerThread.class.getName();
MediaExtractor extractor;
MediaCodec decoder;
Surface surface;
boolean running;
ByteBuffer[] inputBuffers;
public PlayerThread(Surface surface)
{
this.surface = surface;
MediaFormat format = MediaFormat.createVideoFormat("video/avc",720,480);
decoder = MediaCodec.createDecoderByType("video/avc");
decoder.configure(format, surface, null, 0);
decoder.start();
inputBuffers = decoder.getInputBuffers();
}
...
#Override
public void run() {
running = true;
try {
String mcg = "239.255.0.1";
MulticastSocket ms;
ms = new MulticastSocket(1841);
ms.joinGroup(new InetSocketAddress(mcg, 1841), NetworkInterface.getByName("eth0"));
ms.setSoTimeout(4000);
ms.setReuseAddress(true);
byte[] buffer = new byte[65535];
DatagramPacket dp = new DatagramPacket(buffer, buffer.length);
while (running) {
try {
ms.receive(dp);
parse(dp.getData());
} catch (SocketTimeoutException e) {
Log.d("thread", "timeout");
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
The receiving works fine, each datagram packet contains two TS packets. They get passed to the parse function:
boolean first = true;
ByteArrayOutputStream current = new ByteArrayOutputStream();
void parse(byte[] data) {
ByteBuffer stream = ByteBuffer.wrap(data);
// mpeg-ts stream header is 4 bytes starting with the sync byte
if( stream.get(0) != 0x47 ) {
Log.w(TAG, "got packet w/out mpegts header!");
return;
}
ByteBuffer raw = stream.duplicate();
// ts packets are 188 bytes
raw.limit(188);
TSPacket ts = new TSPacket(raw);
if( ts.pid == 0x10 ) {
processTS(ts);
}
// move to second packet
stream.position(188);
stream.limit(188*2);
if( stream.get(stream.position()) != 0x47 ) {
Log.w(TAG, "missing mpegts header!");
return;
}
raw = stream.duplicate();
raw.limit(188*2);
ts = new TSPacket(raw);
if( ts.pid == 0x10 ) {
processTS(ts);
}
}
TS packets are parsed by the TSPacket class:
public class TSPacket {
private final static String TAG = TSPacket.class.getName();
class AdaptationField {
boolean di;
boolean rai;
boolean espi;
boolean hasPcr;
boolean hasOpcr;
boolean spf;
boolean tpdf;
boolean hasExtension;
byte[] data;
public AdaptationField(ByteBuffer raw) {
// first byte is size of field minus size byte
int count = raw.get() & 0xff;
// second byte is flags
BitSet flags = BitSet.valueOf(new byte[]{ raw.get()});
di = flags.get(7);
rai = flags.get(6);
espi = flags.get(5);
hasPcr = flags.get(4);
hasOpcr = flags.get(3);
spf = flags.get(2);
tpdf = flags.get(1);
hasExtension = flags.get(0);
// the rest is 'data'
if( count > 1 ) {
data = new byte[count-1];
raw.get(data);
}
}
}
boolean tei;
boolean pus;
boolean tp;
int pid;
boolean hasAdapt;
boolean hasPayload;
int counter;
AdaptationField adaptationField;
byte[] payload;
public TSPacket(ByteBuffer raw) {
// check for sync byte
if( raw.get() != 0x47 ) {
Log.e(TAG, "missing sync byte");
throw new InvalidParameterException("missing sync byte");
}
// next 3 bits are flags
byte b = raw.get();
BitSet flags = BitSet.valueOf(new byte[] {b});
tei = flags.get(7);
pus = flags.get(6);
tp = flags.get(5);
// then 13 bits for pid
pid = ((b << 8) | (raw.get() & 0xff) ) & 0x1fff;
b = raw.get();
flags = BitSet.valueOf(new byte[]{b});
// then 4 more flags
if( flags.get(7) || flags.get(6) ) {
Log.e(TAG, "scrambled?!?!");
// todo: bail on this packet?
}
hasAdapt = flags.get(5);
hasPayload = flags.get(4);
// counter
counter = b & 0x0f;
// optional adaptation field
if( hasAdapt ) {
adaptationField = new AdaptationField(raw);
}
// optional payload field
if( hasPayload ) {
payload = new byte[raw.remaining()];
raw.get(payload);
}
}
}
Then passed to the processTS function:
// a PES packet can span multiple TS packets, so we keep track of the 'current' one
PESPacket currentPES;
void processTS(TSPacket ts) {
// payload unit start?
if( ts.pus ) {
if( currentPES != null ) {
Log.d(TAG,String.format("replacing pes: len=%d,size=%d", currentPES.length, currentPES.data.size()));
}
// start of new PES packet
currentPES = new PESPacket(ts);
} else if (currentPES != null ) {
// continued PES
currentPES.Add(ts);
} else {
// haven't got a start pes yet
return;
}
if( currentPES.isFull() ) {
long pts = currentPES.getPts();
byte[] data = currentPES.data.toByteArray();
int idx = 0;
do {
int sidx = idx;
// find next NAL prefix
idx = Utility.indexOf(data, sidx+4, data.length-(sidx+4), new byte[]{0,0,1});
byte[] NAL;
if( idx >= 0 ) {
NAL = Arrays.copyOfRange(data, sidx, idx);
} else {
NAL = Arrays.copyOfRange(data, sidx, data.length);
}
// send SPS NAL before anything else
if( first ) {
byte type = NAL[3] == 0 ? NAL[4] : NAL[3];
if( (type & 0x1f) == 7 ) {
Log.d(TAG, "found sps!");
int ibs = decoder.dequeueInputBuffer(1000);
if (ibs >= 0) {
ByteBuffer sinput = inputBuffers[ibs];
sinput.clear();
sinput.put(NAL);
decoder.queueInputBuffer(ibs, 0, NAL.length, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
Log.d(TAG, "sent sps");
first = false;
} else
Log.d(TAG, String.format("could not send sps! %d", ibs));
}
} else {
// put in decoder?
int ibs = decoder.dequeueInputBuffer(1000);
if (ibs >= 0) {
ByteBuffer sinput = inputBuffers[ibs];
sinput.clear();
sinput.put(NAL);
decoder.queueInputBuffer(ibs, 0, NAL.length, 0, 0);
Log.d(TAG, "buffa");
}
}
} while( idx >= 0 );
// finished with this pes
currentPES = null;
}
}
PES packets are parsed by the PESPacket class:
public class PESPacket {
private final static String TAG = PESPacket.class.getName();
int id;
int length;
boolean priority;
boolean dai;
boolean copyright;
boolean origOrCopy;
boolean hasPts;
boolean hasDts;
boolean hasEscr;
boolean hasEsRate;
boolean dsmtmf;
boolean acif;
boolean hasCrc;
boolean pesef;
int headerDataLength;
byte[] headerData;
ByteArrayOutputStream data = new ByteArrayOutputStream();
public PESPacket(TSPacket ts) {
if( ts == null || !ts.pus) {
Log.e(TAG, "invalid ts passed in");
throw new InvalidParameterException("invalid ts passed in");
}
ByteBuffer pes = ByteBuffer.wrap(ts.payload);
// start code
if( pes.get() != 0 || pes.get() != 0 || pes.get() != 1 ) {
Log.e(TAG, "invalid start code");
throw new InvalidParameterException("invalid start code");
}
// stream id
id = pes.get() & 0xff;
// packet length
length = pes.getShort() & 0xffff;
// this is supposedly allowed for video
if( length == 0 ) {
Log.w(TAG, "got zero-length PES?");
}
if( id != 0xe0 ) {
Log.e(TAG, String.format("unexpected stream id: 0x%x", id));
// todo: ?
}
// for 0xe0 there is an extension header starting with 2 bits '10'
byte b = pes.get();
if( (b & 0x30) != 0 ) {
Log.w(TAG, "scrambled ?!?!");
// todo: ?
}
BitSet flags = BitSet.valueOf(new byte[]{b});
priority = flags.get(3);
dai = flags.get(2);
copyright = flags.get(1);
origOrCopy = flags.get(0);
flags = BitSet.valueOf(new byte[]{pes.get()});
hasPts = flags.get(7);
hasDts = flags.get(6);
hasEscr = flags.get(5);
hasEsRate = flags.get(4);
dsmtmf = flags.get(3);
acif = flags.get(2);
hasCrc = flags.get(1);
pesef = flags.get(0);
headerDataLength = pes.get() & 0xff;
if( headerDataLength > 0 ) {
headerData = new byte[headerDataLength];
pes.get(headerData);
}
WritableByteChannel channel = Channels.newChannel(data);
try {
channel.write(pes);
} catch (IOException e) {
e.printStackTrace();
}
// length includes optional pes header,
length = length - (3 + headerDataLength);
}
public void Add(TSPacket ts) {
if( ts.pus ) {
Log.e(TAG, "don't add start of PES packet to another packet");
throw new InvalidParameterException("ts packet marked as new pes");
}
int size = data.size();
int len = length - size;
len = ts.payload.length > len ? len : ts.payload.length;
data.write(ts.payload, 0, len);
}
public boolean isFull() {
return (data.size() >= length );
}
public long getPts() {
if( !hasPts || headerDataLength < 5 )
return 0;
ByteBuffer hd = ByteBuffer.wrap(headerData);
long pts = ( ((hd.get() & 0x0e) << 29)
| ((hd.get() & 0xff) << 22)
| ((hd.get() & 0xfe) << 14)
| ((hd.get() & 0xff) << 7)
| ((hd.get() & 0xfe) >>> 1));
return pts;
}
}
So I eventually figured out that, even though I was using an output surface, I had to manually drain the output buffers. By calling decoder.dequeueOutputBuffer and then decoder.releaseOutputBuffer, the input buffers worked as expected.
I was able to also able to get output by passing in both individual NAL units as well as full access units (one per PES packet), but I got the clearest video by passing in full access units.

Playing javacv-ffmpeg decoded audio in Android with Audiotrack

i am developing android application in which i need to play AAC live audio stream coming from Red5 server.
I have successfully decoded the audio stream by using javacv-ffmpeg.
But my problem is how to play the audio from decoded samples.
I have tried by following way
int len = avcodec.avcodec_decode_audio4( audio_c, samples_frame, got_frame, pkt2);
if (len <= 0){
this.pkt2.size(0);
} else {
if (this.got_frame[0] != 0) {
long pts = avutil.av_frame_get_best_effort_timestamp(samples_frame);
int sample_format = samples_frame.format();
int planes = avutil.av_sample_fmt_is_planar(sample_format) != 0 ? samples_frame.channels() : 1;
int data_size = avutil.av_samples_get_buffer_size((IntPointer)null, audio_c.channels(), samples_frame.nb_samples(), audio_c.sample_fmt(), 1) / planes;
if ((samples_buf == null) || (samples_buf.length != planes)) {
samples_ptr = new BytePointer[planes];
samples_buf = new Buffer[planes];
}
BytePointer ptemp = samples_frame.data(0);
BytePointer[] temp_ptr = new BytePointer[1];
temp_ptr[0] = ptemp.capacity(sample_size);
ByteBuffer btemp = ptemp.asBuffer();
byte[] buftemp = new byte[sample_size];
btemp.get(buftemp, 0, buftemp.length);
play the buftemp[] with audiotrack.....
}
But only noise is heard from speakers, is there any processing is need to be done on AVFrame we get from decode_audio4(...) .
The Incoming audio stream is correctly encoded with AAC codec.
Any help, suggestion appreciated.
Thanks in advance.
You can use FFmpegFrameGrabber class to capture the stream. And extract the audio using a FloatBuffer class. This is a java example
public class PlayVideoAndAudio extends Application
{
private static final Logger LOG = Logger.getLogger(JavaFxPlayVideoAndAudio.class.getName());
private static final double SC16 = (double) 0x7FFF + 0.4999999999999999;
private static volatile Thread playThread;
public static void main(String[] args)
{
launch(args);
}
#Override
public void start(Stage primaryStage) throws Exception
{
String source = "rtsp://184.72.239.149/vod/mp4:BigBuckBunny_115k.mov";
StackPane root = new StackPane();
ImageView imageView = new ImageView();
root.getChildren().add(imageView);
imageView.fitWidthProperty().bind(primaryStage.widthProperty());
imageView.fitHeightProperty().bind(primaryStage.heightProperty());
Scene scene = new Scene(root, 640, 480);
primaryStage.setTitle("Video + audio");
primaryStage.setScene(scene);
primaryStage.show();
playThread = new Thread(() -> {
try {
FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(source);
grabber.start();
primaryStage.setWidth(grabber.getImageWidth());
primaryStage.setHeight(grabber.getImageHeight());
AudioFormat audioFormat = new AudioFormat(grabber.getSampleRate(), 16, grabber.getAudioChannels(), true, true);
DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioFormat);
SourceDataLine soundLine = (SourceDataLine) AudioSystem.getLine(info);
soundLine.open(audioFormat);
soundLine.start();
Java2DFrameConverter converter = new Java2DFrameConverter();
ExecutorService executor = Executors.newSingleThreadExecutor();
while (!Thread.interrupted()) {
Frame frame = grabber.grab();
if (frame == null) {
break;
}
if (frame.image != null) {
Image image = SwingFXUtils.toFXImage(converter.convert(frame), null);
Platform.runLater(() -> {
imageView.setImage(image);
});
} else if (frame.samples != null) {
FloatBuffer channelSamplesFloatBuffer = (FloatBuffer) frame.samples[0];
channelSamplesFloatBuffer.rewind();
ByteBuffer outBuffer = ByteBuffer.allocate(channelSamplesFloatBuffer.capacity() * 2);
for (int i = 0; i < channelSamplesFloatBuffer.capacity(); i++) {
short val = (short)((double) channelSamplesFloatBuffer.get(i) * SC16);
outBuffer.putShort(val);
}
/**
* We need this because soundLine.write ignores
* interruptions during writing.
*/
try {
executor.submit(() -> {
soundLine.write(outBuffer.array(), 0, outBuffer.capacity());
outBuffer.clear();
}).get();
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
}
}
}
executor.shutdownNow();
executor.awaitTermination(10, TimeUnit.SECONDS);
soundLine.stop();
grabber.stop();
grabber.release();
Platform.exit();
} catch (Exception exception) {
LOG.log(Level.SEVERE, null, exception);
System.exit(1);
}
});
playThread.start();
}
#Override
public void stop() throws Exception
{
playThread.interrupt();
}
}
Because, what data you are getting in buftemp[] is in this AV_SAMPLE_FMT_FLTP format, you have to change it to AV_SAMPLE_FMT_S16 format using SwrContext and then your problem will be solved.

Categories

Resources