i want to send an audio stream from PC (C++ application, using FMOD-API to decode audio data and send via UDP Socket) to an android device. The communication already works and i can hear "sound" (100ms sound, followed by 900ms silence, alternating) on the android.
I don't know why the sound is stuttering - on the PC the same audio stream is played fine in nice quality. I think the problem is on the android..
Here is the code:
DatagramSocket sock = new DatagramSocket(12345);
byte []bSockBuffer = new byte[1024];
byte []bRecvBufTmp;
int iAudioBufSize, iCurAudioBufPos = 0;
sock.setReceiveBufferSize(bSockBuffer.length);
// Audio Stream initialisieren:
iAudioBufSize = AudioTrack.getMinBufferSize(44100, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT);
AudioTrack track = new AudioTrack(AudioManager.STREAM_MUSIC, 44100, AudioFormat.CHANNEL_OUT_STEREO,
AudioFormat.ENCODING_PCM_16BIT, iAudioBufSize, AudioTrack.MODE_STREAM);
track.play();
while (true)
{
DatagramPacket pack = new DatagramPacket(bSockBuffer, bSockBuffer.length);
// Paket empfangen:
sock.receive(pack);
track.write(pack.getData(), 0, pack.getLength());
}
I'm sure to set up 'AudioTrack' object correctly, settings compare to my settings in the c++ application.
An other step was pre-buffering the received socket-data in a temporary 'byte[]' variable and writing it to the AudioTrack-object when the size of the buffer 'iAudioBufSize' was reached.
This did not helped.
Any idears?
Thanks
[EDIT]
Code of C++ Application, used sample "manualdecode" of FMOD API examples:
FMOD_RESULT F_CALLBACK pcmreadcallback(FMOD_SOUND *sound, void *data, unsigned int datalen)
{
CCtrlSocket *cClientTmp = /* Obtaining target client sock here */;
FMOD_RESULT result;
unsigned int read, uSentTmp, uSizeTmp;
EnterCriticalSection(&decodecrit);
if (!decodesound)
return (FMOD_ERR_FILE_EOF);
result = decodesound->readData(data, datalen, &read);
if (result == FMOD_ERR_FILE_EOF)
{
// Handle looping:
decodesound->seekData(0);
datalen -= read;
result = decodesound->readData((char*) data + read, datalen, &read);
}
// Split package in multiple parts:
uSentTmp = 0;
do
{
uSizeTmp = (read - uSentTmp);
if (uSizeTmp > 1024)
uSizeTmp = 1024;
uSentTmp += cClientTmp->SendAudioData((char*) data + uSentTmp, uSizeTmp);
} while (uSentTmp < read);
LeaveCriticalSection(&decodecrit);
return (FMOD_OK);
}
I've done this problem.
The mess was an entry in a logfile that has cost lots of time creating a lag :(
Now i can hear the streamed music on my android client. But there are still some lags. I've experimented a LOT of values for socket and AudioTrack buffers.
I have compared the amount of sent and received bytes: In 20 secs sending 9170000 bytes of data results in receiving 8120000 bytes on android device. At first the stream is played fast for 3 secs (that means buffer's full?). After 30 secs the stream lags (which means buffer's empty?).
In general the music quality is very good, but there is a sizzling noise all the time (which indicates lost socket packages?).
My 'PlaybackStart()' function has changed - i'm not using a PCM read callback anymore:
FMOD_RESULT CAudioStream::PlaybackStart()
{
CCtrlSocket *cClientTmp;
unsigned int read, uSentTmp, uSizeTmp;
FMOD_RESULT result;
result = system->createStream("C:\\test.mp3", FMOD_OPENONLY | FMOD_ACCURATETIME, 0, &sound);
if(result != FMOD_OK)
return (result);
int iChannels, iBits;
FMOD_SOUND_FORMAT fFormat;
FMOD_SOUND_TYPE fType;
result = sound->getFormat(&fType, &fFormat, &iChannels, &iBits);
if(result != FMOD_OK)
return (result);
void *data;
unsigned int length = 0;
int iSampleSec = 1; // Playtime
int iSampleSize = (44100 * 2 * sizeof(signed short) * iSampleSec);
int iSleep = 6; // Sleep after sending a package
DWORD dSleepTotal;
result = sound->getLength(&length, FMOD_TIMEUNIT_PCMBYTES);
if(result != FMOD_OK)
return (result);
data = malloc(iSampleSize);
if (!data)
return (FMOD_RESULT_FORCEINT);
cClientTmp = (CCtrlSocket*) CCtrlSocket::cServerSock.GetClientSock(CCtrlSocket::cServerSock.GetClientSockCount() - 1);
do
{
result = sound->readData((char*) data, iSampleSize, &read);
if ((result != FMOD_OK) && (result != FMOD_ERR_FILE_EOF))
ASSERT(FALSE);
else if (read > 0)
{
dSleepTotal = 0;
for (int i = 0; i < read; i += NET_SVR_AUDIO_BUFFER)
{
// MIN_VAL_LIMITED ((MIN_VAL(VAL1, VAL2) <= LIMIT) ? LIMIT : MIN_VAL(VAL1, VAL2))
cClientTmp->SendAudioData((char*) data + i, MIN_VAL_LIMITED(NET_SVR_AUDIO_BUFFER, (read - i), 0));
// Sleep after sending every package:
Sleep(iSleep);
dSleepTotal += iSleep;
}
if (dSleepTotal < (iSampleSec * 1000))
{
dSleepTotal = (iSampleSec * 1000) - dSleepTotal;
// Sleep after sending every second playtime:
Sleep(dSleepTotal);
}
}
} while (read > 0);
result = sound->release();
if(result != FMOD_OK)
return (result);
result = system->close();
if(result != FMOD_OK)
return (result);
result = system->release();
if(result != FMOD_OK)
return (result);
return (result);
}
I have experimented with different sleep-timings, too.
Related
I am encoding raw data on Android using ffmpeg libraries. The native code reads the audio data from an external device and encodes it into AAC format in an mp4 container. I am finding that the audio data is successfully encoded (I can play it with Groove Music, my default Windows audio player). But the metadata, as reported by ffprobe, has an incorrect duration of 0.05 secs - it's actually several seconds long. Also the bitrate is reported wrongly as around 65kbps even though I specified 192kbps.
I've tried recordings of various durations but the result is always similar - the (very small) duration and bitrate. I've tried various other audio players such as Quicktime but they play only the first 0.05 secs or so of the audio.
I've removed error-checking from the following. The actual code checks every call and no problems are reported.
Initialisation:
void AudioWriter::initialise( const char *filePath )
{
AVCodecID avCodecID = AVCodecID::AV_CODEC_ID_AAC;
int bitRate = 192000;
char *containerFormat = "mp4";
int sampleRate = 48000;
int nChannels = 2;
mAvCodec = avcodec_find_encoder(avCodecID);
mAvCodecContext = avcodec_alloc_context3(mAvCodec);
mAvCodecContext->codec_id = avCodecID;
mAvCodecContext->codec_type = AVMEDIA_TYPE_AUDIO;
mAvCodecContext->sample_fmt = AV_SAMPLE_FMT_FLTP;
mAvCodecContext->bit_rate = bitRate;
mAvCodecContext->sample_rate = sampleRate;
mAvCodecContext->channels = nChannels;
mAvCodecContext->channel_layout = AV_CH_LAYOUT_STEREO;
avcodec_open2( mAvCodecContext, mAvCodec, nullptr );
mAvFormatContext = avformat_alloc_context();
avformat_alloc_output_context2(&mAvFormatContext, nullptr, containerFormat, nullptr);
mAvFormatContext->audio_codec = mAvCodec;
mAvFormatContext->audio_codec_id = avCodecID;
mAvOutputStream = avformat_new_stream(mAvFormatContext, mAvCodec);
avcodec_parameters_from_context(mAvOutputStream->codecpar, mAvCodecContext);
if (!(mAvFormatContext->oformat->flags & AVFMT_NOFILE))
{
avio_open(&mAvFormatContext->pb, filePath, AVIO_FLAG_WRITE);
}
if ( mAvFormatContext->oformat->flags & AVFMT_GLOBALHEADER )
{
mAvCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
avformat_write_header(mAvFormatContext, NULL);
mAvAudioFrame = av_frame_alloc();
mAvAudioFrame->nb_samples = mAvCodecContext->frame_size;
mAvAudioFrame->format = mAvCodecContext->sample_fmt;
mAvAudioFrame->channel_layout = mAvCodecContext->channel_layout;
av_samples_get_buffer_size(NULL, mAvCodecContext->channels, mAvCodecContext->frame_size,
mAvCodecContext->sample_fmt, 0);
av_frame_get_buffer(mAvAudioFrame, 0);
av_frame_make_writable(mAvAudioFrame);
mAvPacket = av_packet_alloc();
}
Encoding:
// SoundRecording is a custom class with the raw samples to be encoded
bool AudioWriter::encodeToContainer( SoundRecording *soundRecording )
{
int ret;
int frameCount = mAvCodecContext->frame_size;
int nChannels = mAvCodecContext->channels;
float *buf = new float[frameCount*nChannels];
while ( soundRecording->hasReadableData() )
{
//Populate the frame
int samplesRead = soundRecording->read( buf, frameCount*nChannels );
// Planar data
int nFrames = samplesRead/nChannels;
for ( int i = 0; i < nFrames; ++i )
{
for (int c = 0; c < nChannels; ++c )
{
samples[c][i] = buf[nChannels*i +c];
}
}
// Fill a gap at the end with silence
if ( samplesRead < frameCount*nChannels )
{
for ( int i = samplesRead; i < frameCount*nChannels; ++i )
{
for (int c = 0; c < nChannels; ++c )
{
samples[c][i] = 0.0;
}
}
}
encodeFrame( mAvAudioFrame ) )
}
finish();
}
bool AudioWriter::encodeFrame( AVFrame *frame )
{
//send the frame for encoding
int ret;
if ( frame != nullptr )
{
frame->pts = mAudFrameCounter++;
}
avcodec_send_frame(mAvCodecContext, frame );
while (ret >= 0)
{
ret = avcodec_receive_packet(mAvCodecContext, mAvPacket);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF )
{
break;
}
else
if (ret < 0) {
return false;
}
av_packet_rescale_ts(mAvPacket, mAvCodecContext->time_base, mAvOutputStream->time_base);
mAvPacket->stream_index = mAvOutputStream->index;
av_interleaved_write_frame(mAvFormatContext, mAvPacket);
av_packet_unref(mAvPacket);
}
return true;
}
void AudioWriter::finish()
{
// Flush by sending a null frame
encodeFrame( nullptr );
av_write_trailer(mAvFormatContext);
}
Since the resultant file contains the recorded music, the code to manipulate the audio data seems to be correct (unless I am overwriting other memory somehow).
The inaccurate duration and bitrate suggest that information concerning time is not being properly managed. I set the pts of the frames using a simple increasing integer. I'm unclear what the code that sets the timestamp and stream index achieves - and whether it's even necessary: I copied it from supposedly working code but I've seen other code without it.
Can anyone see what I'm doing wrong?
The timestamp need to be correct. Set the time_base to 1/sample_rate and increment the timestamp by 1024 each frame. Note: 1024 is aac specific. If you change codecs, you need to change the frame size.
I'm trying to make app that transmits SuperpoweredAndroidAudioIO buffer from one android device to another. With the following code I've managed to send and receive short int buffer from the audio callback. However on the receiving side sound gets very distorted during playback.
Note: For brevity I have not included some code that doesen't seem to have to do with issue, including socket initialisation related functions. If needed I can add the code.
Sending side:
Mic.cpp
static bool SendBuffer(
int sd,
sockaddr_in address,
short int *buffer,
size_t bufferSize) {
// Send data buffer to the socket
ssize_t sentSize = sendto(sd,
buffer,
bufferSize,
0,
(struct sockaddr*)&address,
sizeof address);
// If send is failed
if (sentSize == -1){
__android_log_print(ANDROID_LOG_INFO, "Sent size ", "%i error %i",
sentSize , errno);
}
else if (sentSize > 0) {
__android_log_print(ANDROID_LOG_INFO, "DatagramSent : ", "%hi size: %hi",
buffer, sentSize);
}
return true;
}
// audio callback
static bool micProcessing(
void *clientdata,
short int *audioInputOutput,
int numberOfSamples,
int __unused samplerate) {
return SendBuffer(micSocket, dsocket, audioInputOutput, numberOfSamples);
}
// Constructor
Mic::Mic(JNIEnv *env,
unsigned int samplerate,
unsigned int buffersize,
unsigned int port){
micSocket = NewUdpSocket(env);
dsocket = initDestinationSocket(port); // where to send
__android_log_write(ANDROID_LOG_DEBUG, "Sockets", "initialised");
// init IO
microphone = new SuperpoweredAndroidAudioIO(samplerate,
buffersize,
true,
false,
micProcessing,
this,
-1,
SL_ANDROID_STREAM_MEDIA,
buffersize*2);
__android_log_write(ANDROID_LOG_DEBUG, "Mic", "initialised");
}
Receiving side consists of 2 parts: Mixer and Channel
Mixer.cpp
//audio callback
static bool mainprocess(
void *clientdata,
short int *audioInputOutput,
int numberOfSamples,
int __unused samplerate) {
return ((Mixer*)clientdata)->processMain(audioInputOutput, numberOfSamples);
}
// Setting up Mixer
Mixer::Mixer(JNIEnv *env,unsigned int samplerate, unsigned int buffersize) {
//Short int buffers for recieving
channel1 = new Channel(samplerate,buffersize);
//output buffer
outputFloat = ((float *)memalign(16, (buffersize + 16) * sizeof(float) * 2));
//volumes
outputLevel = 0.5f;
channel1level = 0.2f;
channel2level = 0.2f;
channel3level = 0.2f;
channel4level = 0.2f;
mainmixer = new SuperpoweredMonoMixer();
__android_log_print(ANDROID_LOG_INFO, "Mixer", " Created");
main = new SuperpoweredAndroidAudioIO(
samplerate,
buffersize,
false,
true,
mainprocess,
this,
-1,
SL_ANDROID_STREAM_MEDIA,
buffersize*2);
__android_log_write(ANDROID_LOG_INFO, "Main AudioIO created", " ");
main->stop();
SuperpoweredCPU::setSustainedPerformanceMode(true); // Prevents CPU drops
}
// processing.
bool Mixer::processMain(short int *outputbuffer, unsigned int numberOfSamples{
// a bit awkward
channel1->returnFloatBuffer();
inputs[0] = channel1->floatBuffer;
inputs[1] = NULL;
inputs[2] = NULL;
inputs[3] = NULL;
__android_log_print(ANDROID_LOG_INFO, "Channels are inside", " of mixer");
inputLevels[0] = channel1level;
inputLevels[1] = channel2level;
inputLevels[2] = channel3level;
inputLevels[3] = channel4level;
mainmixer->process(inputs,
outputFloat,
inputLevels,
outputLevel,
numberOfSamples);
SuperpoweredFloatToShortInt(outputFloat, outputbuffer, numberOfSamples);
return true;
}
Channel.cpp
//receiving buffer
static bool ReceiveDatagramFromSocket(int sd, short int *buffer, size_t bufferSize) {
ssize_t recvSize = recvfrom(sd, buffer, bufferSize, 0, NULL, NULL);
if (-1 == recvSize){ // If failed
__android_log_print(ANDROID_LOG_INFO, "receive failed", " %i ", errno);
}
else {
// If data is received
if (recvSize > 0) {
__android_log_print(ANDROID_LOG_INFO, "Received"," %i bytes: %hi", recvSize, buffer);
}
}
return true;
}
// init channel
Channel::Channel(unsigned int samplerate, unsigned int buffersize){
socketIn = NewUdpSocket();
BindSocketToPort(socketIn);
samplerRate = samplerate;
bufferSize = buffersize;
shortIntBuffer = (short int *)malloc((buffersize + 16) * sizeof(short int)*4);
floatBuffer = (float *)memalign(16, (buffersize + 16) * sizeof(float) * 2);
bandEQ = new Superpowered3BandEQ(samplerate);
bandEQ->enable(true);
__android_log_print(ANDROID_LOG_INFO, "Channel ", "created");
}
// this function is called from Mixer.cpp
void Channel::returnFloatBuffer(){
ReceiveDatagramFromSocket(socketIn, shortIntBuffer, bufferSize);
Converting the 16-bit integer samples to 32-bit floating point.
SuperpoweredShortIntToFloat(shortIntBuffer, floatBuffer, bufferSize, 1);
bandEQ->process(floatBuffer, floatBuffer, bufferSize );
__android_log_print(ANDROID_LOG_INFO, "EQ", " processing");
}
At first I thought that because AudioIO on both sided get initialised with different buffer sizes (different devices 240 and 512), but then I tried to hardcode 512 into both of them but it didn't help.
I also tried to increase buffer sizes in sendto() and recvfrom() up to 4096 and it made sound more recognizable but still too distorted.
I should also add that I'm a newbie in C++ and I stuck to 'naive' and 'whatever works' approaches which got me this far.
I want to understand whether I'm on the right track and what approach should I take in order to transmit audio without distortion.
There are two major problems with your approach:
Blocking functions, such as networking should be avoided from the audio processing callback. You need to perform networking (on both sides) from a different thread, and you need some buffering between the audio processing callback and the network thread to pass audio.
You need to "packetize" the transfers, you need to handle network packets on both sides. Network transfer is not fast nor reliable, so you need clever mechanisms to handle this.
In general, the implementation for such audio transfer is much, much more complex to your current code.
I am decoding a h264 video stream with the following code (original guide):
public void configure(Surface surface, int width, int height, ByteBuffer csd0) {
String VIDEO_FORMAT = "video/avc";
if (mConfigured) {
throw new IllegalStateException("Decoder is already configured");
}
MediaFormat format = MediaFormat.createVideoFormat(VIDEO_FORMAT, width, height);
// little tricky here, csd-0 is required in order to configure the codec properly
// it is basically the first sample from encoder with flag: BUFFER_FLAG_CODEC_CONFIG
format.setByteBuffer("csd-0", csd0);
try {
mCodec = MediaCodec.createDecoderByType(VIDEO_FORMAT);
} catch (IOException e) {
throw new RuntimeException("Failed to create codec", e);
}
mCodec.configure(format, surface, null, 0);
mCodec.start();
mConfigured = true;
}
#SuppressWarnings("deprecation")
public void decodeSample(byte[] data, int offset, int size, long presentationTimeUs, int flags) {
if (mConfigured && mRunning) {
int index = mCodec.dequeueInputBuffer(mTimeoutUs);
if (index >= 0) {
ByteBuffer buffer;
// since API 21 we have new API to use
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
buffer = mCodec.getInputBuffers()[index];
buffer.clear();
} else {
buffer = mCodec.getInputBuffer(index);
}
if (buffer != null) {
buffer.put(data, offset, size);
mCodec.queueInputBuffer(index, 0, size, presentationTimeUs, flags);
}
}
}
}
#Override
public void run() {
try {
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
while (mRunning) {
if (mConfigured) {
int index = mCodec.dequeueOutputBuffer(info, mTimeoutUs);
if (index >= 0) {
// setting true is telling system to render frame onto Surface
mCodec.releaseOutputBuffer(index, true);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
break;
}
}
} else {
// just waiting to be configured, then decode and render
try {
Thread.sleep(10);
} catch (InterruptedException ignore) {
}
}
}
} finally {
if (mConfigured) {
mCodec.stop();
mCodec.release();
}
}
}
I can run this on both my Nexus 6 (api 22) and Samsung galaxy core (api 16) on low and medium quality. However when I switch to high quality (720p) it crashes on the Samsung after about 30 frames (but nothing is rendered to the screen).
E/ACodec﹕ [OMX.qcom.video.decoder.avc] ERROR(0x8000100a)
E/MediaCodec﹕ Codec reported an error. (omx error 0x8000100a, internalError -2147483648)
[...]
W/System.err﹕ java.lang.IllegalStateException
W/System.err﹕ at android.media.MediaCodec.dequeueInputBuffer(Native Method)
W/System.err﹕ at com.test.stream.VideoDecoder$Worker.decodeSample(VideoDecoder.java:95)
W/System.err﹕ at com.test.stream.VideoDecoder.decodeSample(VideoDecoder.java:24)
W/System.err﹕ at com.test.stream.VideoThread.run(VideoThread.java:160)
The error above is the first error that appears, the IllegalStateException is afterwards thrown on each frame.
My question is, is this a device specific problem (because of: older api/device, less powerful, etc.) or is something actually wrong?
and how should I deal with this?
For my Android h.264 decoder i do it slightly different to your setup. I think your using more modern api level. But for me it looks more like this:
public void startDecoder() {
// Initilize codec
mediaCodec = MediaCodec.createDecoderByType("video/avc");
mediaFormat = MediaFormat.createVideoFormat("video/avc", 0, 0);
bufferInfo = new MediaCodec.BufferInfo();
// STOPS unit-tests from crashing here from mocked out android
if (mediaCodec != null) {
mediaCodec.configure(mediaFormat, targetSurface, null, 0);
mediaCodec.start();
decoderThread = new Thread(this);
decoderThread.start();
}
}
// Decoder Thread refers to this class which does the decoder/render loop:
public void run() {
//mediaCodec input + output dequeue timeouts
long kInputBufferTimeoutMs = 50;
long kOutputBufferTimeoutMs = 50;
while (running && mediaCodec != null) {
synchronized (mediaCodec) {
// stop if not running.
if (!running || mediaCodec == null)
break;
// Only push in new data if there is data available in the queue
if (naluSegmentQueue.size() > 0) {
int inputBufferIndex = mediaCodec.dequeueInputBuffer(kInputBufferTimeoutMs);
if (inputBufferIndex >= 0) {
NaluSegment segment = naluSegmentQueue.poll();
codecInputBufferAvailable(segment, mediaCodec, inputBufferIndex);
}
}
// always check if output is available.
int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, kOutputBufferTimeoutMs);
if (outputBufferIndex >= 0) {
// Try and render first
codecOuputBufferAvailable(mediaCodec, outputBufferIndex, bufferInfo);
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
// Subsequent data will conform to new format.
// Can ignore if using getOutputFormat(outputBufferId)
mediaFormat = mediaCodec.getOutputFormat();
}
}
}
}
To put data into the decoder including the parameters. I don't bother with trying to use the csd-0/1 network streams can have changing format descriptions and its easier to just let it be picked up dynamically.
private void codecInputBufferAvailable(NaluSegment segment, MediaCodec codec, int index) {
int flags = (segment.getType() == NaluType.SPS
|| segment.getType() == NaluType.PPS
|| segment.getType() == NaluType.SUPP_ENHANCEMENT) ?
MediaCodec.BUFFER_FLAG_CODEC_CONFIG : MediaCodec.BUFFER_FLAG_SYNC_FRAME;
ByteBuffer[] buffers = codec.getInputBuffers();
ByteBuffer buffer = buffers[index];
// Can throw buffer overflow exception when buffer sizes are too small.
try {
buffer.put(segment.getBuffer());
codec.queueInputBuffer(index, 0, segment.getBufferSize(), 0, flags);
} catch(Exception e) {
Log.e(TAG, "Failed to push buffer to decoder");
}
}
IMPORTANT: buffer.put(segment.getBuffer());
getBuffer() here always returns a 4 byte annexb buffer. The android decoders do not understand 3 byte nal units. So if you have a 3 byte nal unit turn it into 4 bytes magic sequence with length + 1 and 0x00, 0x00, 0x00, 0x01 as the start magic sequence the rest of the buffer should be &buffer[headerLength].
Notice the try-catch here this doesn't give a compiler warning but it can throw a buffer overflow exception here if your have a very big payload and the byte-buffer is too small.
So long as your parse out your NAL units correctly this should work for you. But for my case i noticed that the NAL units can be 3 or 4 bytes for the magic header.
/**
* H264 is comprised of NALU segments.
*
* XXXX Y ZZZZZZZZ -> XXXX Y ZZZZZZZZ -> XXXX Y ZZZZZZZZ
*
* Each segment is comprised of:
*
* XXXX -> Magic byte header (0x00, 0x00, 0x00, 0x01) NOTE: this can be either 3 of 4 bytes
* Y -> The Nalu Type
* ZZZ... -> The Payload
*
* Notice there is no nalu length specified. To parse an nalu, you must
* read until the next magic-byte-sequence AKA the next segment to figure
* out the full nalu length
**/
public static List<NaluSegment> parseNaluSegments(byte[] buffer) throws NaluBufferException {
List<NaluSegment> segmentList = new ArrayList<>();
if (buffer.length < 6) {
return segmentList;
}
int lastStartingOffset = -1;
for (int i = 0; i < buffer.length - 10; ++i) {
**if (buffer[i] == 0x00 && buffer[i+1] == 0x00 && buffer[i+2] == 0x01)** {
int naluType = (buffer[i+3] & 0x1F);
NaluSegment segment = new NaluSegment(naluType, 3, i);
**if (i > 0 && buffer[i-1] == 0x00)** {
// This is actually a 4 byte segment
int currentSegmentOffset = segment.getOffset();
segment.setHeaderSize(4);
segment.setOffset(currentSegmentOffset - 1);
}
...
Create your own nalu-segment objects and don't forget the trailing NAL.
I hope this helps.
I have found open source video player for Android, which uses ffmpeg to decode video.
I have some problems with audio, that sometimes plays with jerks, but video picture is shown well. The basic idea of player is that audio and video are decoded in two different streams, and then in the third stream the are passed back, video picture is shown on SurfaceView and video sound is passed in byte array to AudioTrack and then plays. But sometimes sound is lost or playing with jerks. Can anyone give me start point for what to do (some basic concepts). May be I should change buffer size for AudioTrack or add some flags to it. Here is a piece of code, where AudioTrack class is created.
private AudioTrack prepareAudioTrack(int sampleRateInHz,
int numberOfChannels) {
for (;;) {
int channelConfig;
if (numberOfChannels == 1) {
channelConfig = AudioFormat.CHANNEL_OUT_MONO;
} else if (numberOfChannels == 2) {
channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
} else if (numberOfChannels == 3) {
channelConfig = AudioFormat.CHANNEL_OUT_FRONT_CENTER
| AudioFormat.CHANNEL_OUT_FRONT_RIGHT
| AudioFormat.CHANNEL_OUT_FRONT_LEFT;
} else if (numberOfChannels == 4) {
channelConfig = AudioFormat.CHANNEL_OUT_QUAD;
} else if (numberOfChannels == 5) {
channelConfig = AudioFormat.CHANNEL_OUT_QUAD
| AudioFormat.CHANNEL_OUT_LOW_FREQUENCY;
} else if (numberOfChannels == 6) {
channelConfig = AudioFormat.CHANNEL_OUT_5POINT1;
} else if (numberOfChannels == 8) {
channelConfig = AudioFormat.CHANNEL_OUT_7POINT1;
} else {
channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
}
try {
Log.d("MyLog","Creating Audio player");
int minBufferSize = AudioTrack.getMinBufferSize(sampleRateInHz,
channelConfig, AudioFormat.ENCODING_PCM_16BIT);
AudioTrack audioTrack = new AudioTrack(
AudioManager.STREAM_MUSIC, sampleRateInHz,
channelConfig, AudioFormat.ENCODING_PCM_16BIT,
minBufferSize, AudioTrack.MODE_STREAM);
return audioTrack;
} catch (IllegalArgumentException e) {
if (numberOfChannels > 2) {
numberOfChannels = 2;
} else if (numberOfChannels > 1) {
numberOfChannels = 1;
} else {
throw e;
}
}
}
}
And this is a piece of native code where sound bytes are written to AudioTrack
int player_write_audio(struct DecoderData *decoder_data, JNIEnv *env,
int64_t pts, uint8_t *data, int data_size, int original_data_size) {
struct Player *player = decoder_data->player;
int stream_no = decoder_data->stream_no;
int err = ERROR_NO_ERROR;
int ret;
AVCodecContext * c = player->input_codec_ctxs[stream_no];
AVStream *stream = player->input_streams[stream_no];
LOGI(10, "player_write_audio Writing audio frame")
jbyteArray samples_byte_array = (*env)->NewByteArray(env, data_size);
if (samples_byte_array == NULL) {
err = -ERROR_NOT_CREATED_AUDIO_SAMPLE_BYTE_ARRAY;
goto end;
}
if (pts != AV_NOPTS_VALUE) {
player->audio_clock = av_rescale_q(pts, stream->time_base, AV_TIME_BASE_Q);
LOGI(9, "player_write_audio - read from pts")
} else {
int64_t sample_time = original_data_size;
sample_time *= 1000000ll;
sample_time /= c->channels;
sample_time /= c->sample_rate;
sample_time /= av_get_bytes_per_sample(c->sample_fmt);
player->audio_clock += sample_time;
LOGI(9, "player_write_audio - added")
}
enum WaitFuncRet wait_ret = player_wait_for_frame(player,
player->audio_clock + AUDIO_TIME_ADJUST_US, stream_no);
if (wait_ret == WAIT_FUNC_RET_SKIP) {
goto end;
}
LOGI(10, "player_write_audio Writing sample data")
jbyte *jni_samples = (*env)->GetByteArrayElements(env, samples_byte_array,
NULL);
memcpy(jni_samples, data, data_size);
(*env)->ReleaseByteArrayElements(env, samples_byte_array, jni_samples, 0);
LOGI(10, "player_write_audio playing audio track");
ret = (*env)->CallIntMethod(env, player->audio_track,
player->audio_track_write_method, samples_byte_array, 0, data_size);
jthrowable exc = (*env)->ExceptionOccurred(env);
if (exc) {
err = -ERROR_PLAYING_AUDIO;
LOGE(3, "Could not write audio track: reason in exception");
// TODO maybe release exc
goto free_local_ref;
}
if (ret < 0) {
err = -ERROR_PLAYING_AUDIO;
LOGE(3,
"Could not write audio track: reason: %d look in AudioTrack.write()", ret);
goto free_local_ref;
}
free_local_ref:
LOGI(10, "player_write_audio releasing local ref");
(*env)->DeleteLocalRef(env, samples_byte_array);
end: return err;
}
I will be pleased for any help!!!! Thank you very much!!!!
I had the same problem. The problem is for start point of audio data that write to audio player. In PCM data each 2 byte of data create one sample of audio base on little_endian conversion. for correct playing the PCM data samples must be correctly create an write to audio player. If the start point of reading buffer is not the first byte of sample then the samples of audio can not create correctly and sound will be destroyed. In my situation I read samples from file. In some times the start point of reading data from file had been second byte of sample and then the all data that I read from file had been decode uncorrectly. I solve the problem by checking the start point and if the start point is odd number I increase that and change it to even number.
excuse me for bad english.
I want to do some FSK Modulation over the audio port. So the problem is that my sinus wave isn't very good. It is disturb by even parts. I used the code original from http://marblemice.blogspot.com/2010/04/generate-and-play-tone-in-android.html with the further modification from Playing an arbitrary tone with Android and https://market.android.com/details?id=re.serialout&feature=search_result .
So where is the failure? What do I wrong?
private static int bitRate=300;
private static int sampleRate=48000;
private static int freq1=600;
public static void loopOnes(){
playque.add(UARTHigh);
athread.interrupt();
}
private static byte[] UARTHigh() {
int numSamples=sampleRate/bitRate;
double sample[]=new double[numSamples];
byte[] buffer=new byte[numSamples*2];
for(int i=0; i<numSamples;++i){
sample[i]=Math.sin(2*Math.PI*i*freq1/sampleRate);
}
int idx = 0;
for (final double dVal : sample) {
// scale to maximum amplitude
final short val = (short) ((dVal * 32767));
// in 16 bit wav PCM, first byte is the low order byte
buffer[idx++] = (byte) (val & 0x00ff);
buffer[idx++] = (byte) ((val & 0xff00) >>> 8);
}
return buffer;
}
private static void playSound(){
active = true;
while(active)
{
try {Thread.sleep(Long.MAX_VALUE);} catch (InterruptedException e) {
while (playque.isEmpty() == false)
{
if (atrk != null)
{
if (generatedSnd != null)
{
// Das letzte Sample erst fertig abspielen lassen
// systemClock.sleep(xx) xx könnte angepasst werden
while (atrk.getPlaybackHeadPosition() < (generatedSnd.length))
SystemClock.sleep(50); // let existing sample finish first: this can probably be set to a smarter number using the information above
}
atrk.release();
}
UpdateParameters(); // might as well do it at every iteration, it's cheap
generatedSnd = playque.poll();
length = generatedSnd.length;
if (minbufsize<length)
minbufsize=length;
atrk = new AudioTrack(AudioManager.STREAM_MUSIC,
sampleRate, AudioFormat.CHANNEL_CONFIGURATION_MONO,
AudioFormat.ENCODING_PCM_16BIT, minbufsize,
AudioTrack.MODE_STATIC);
atrk.setStereoVolume(1,1);
atrk.write(generatedSnd, 0, length);
atrk.play();
}
// Playque is Empty =>send StopBit!
// Set Loop Points
int setLoopError=atrk.setLoopPoints(0, length, -1);
atrk.play();
}
}
}
}
So the answer is to change from MODE_STATIC to MODE_STREAM and don't use Looping Points. In a new thread with low priority a busy loop writes the tracks.