Is there a way to create a video from a series of images on android? Maybe a way to extend the MediaRecorder and being able to take images as input.
I try to really create the video and store it (as an mpeg4 file for instance).
Thanks for any suggestions.
I'm also trying to do the same thing. I have been advice to use Libav.
http://libav.org/
However I need to build it with the NDK and I currently have some issues doing it.
I'm looking for some doc about it. I'll keep you posted.
I've created a post about it: Libav build for Android
You can use AnimationDrawable in an ImageView.
Add frames using the AnimationDrawable.addFrame(Drawable frame, int duration) method, and start the animation using AnimationDrawable.start().
Not sure if that's ideal, but it would work.
I use Android + NDK
AVFrame* OpenImage(const char* imageFileName)
{
AVFormatContext *pFormatCtx = avformat_alloc_context();
std::cout<<"1"<<imageFileName<<std::endl;
if( avformat_open_input(&pFormatCtx, imageFileName, NULL, NULL) < 0)
{
printf("Can't open image file '%s'\n", imageFileName);
return NULL;
}
std::cout<<"2"<<std::endl;
av_dump_format(pFormatCtx, 0, imageFileName, false);
AVCodecContext *pCodecCtx;
std::cout<<"3"<<std::endl;
pCodecCtx = pFormatCtx->streams[0]->codec;
pCodecCtx->width = W_VIDEO;
pCodecCtx->height = H_VIDEO;
//pCodecCtx->pix_fmt = AV_PIX_FMT_YUV420P;
// Find the decoder for the video stream
AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (!pCodec)
{
printf("Codec not found\n");
return NULL;
}
// Open codec
//if(avcodec_open2(pCodecCtx, pCodec)<0)
if(avcodec_open2(pCodecCtx, pCodec,NULL)<0)//check this NULL, it should be of AVDictionary **options
{
printf("Could not open codec\n");
return NULL;
}
std::cout<<"4"<<std::endl;
//
AVFrame *pFrame;
pFrame = av_frame_alloc();
if (!pFrame)
{
printf("Can't allocate memory for AVFrame\n");
return NULL;
}
printf("here");
int frameFinished;
int numBytes;
// Determine required buffer size and allocate buffer
numBytes = avpicture_get_size( pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height);
uint8_t *buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
avpicture_fill((AVPicture *) pFrame, buffer, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height);
// Read frame
AVPacket packet;
int framesNumber = 0;
while (av_read_frame(pFormatCtx, &packet) >= 0)
{
if(packet.stream_index != 0)
continue;
int ret = avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
if (ret > 0)
{
printf("Frame is decoded, size %d\n", ret);
pFrame->quality = 4;
return pFrame;
}
else
printf("Error [%d] while decoding frame: %s\n", ret, strerror(AVERROR(ret)));
}
}
int combine_images_to_video(const char * infile_dir, const char * infile_prefix, const char* infile_surname, int total_frames,const char *outfile)
{
if (total_frames <= 0){
std::cout << "Usage: cv2ff <dir_name> <prefix> <image surname> <total frames> <outfile>" << std::endl;
std::cout << "Please check that the 4th argument is integer value of total frames"<<std::endl;
return 1;
}
printf("max %d frames\n",total_frames);
char *imageFileName;
char numberChar[NUMNUMBER];
// initialize FFmpeg library
av_register_all();
// av_log_set_level(AV_LOG_DEBUG);
int ret;
const int dst_width = W_VIDEO;
const int dst_height = H_VIDEO;
const AVRational dst_fps = {30, 1};//{fps,1}
// open output format context
AVFormatContext* outctx = nullptr;
ret = avformat_alloc_output_context2(&outctx, nullptr, nullptr, outfile);
//outctx->video_codec->
if (ret < 0) {
std::cerr << "fail to avformat_alloc_output_context2(" << outfile << "): ret=" << ret;
return 2;
}
// open output IO context
ret = avio_open2(&outctx->pb, outfile, AVIO_FLAG_WRITE, nullptr, nullptr);
if (ret < 0) {
std::cerr << "fail to avio_open2: ret=" << ret;
return 2;
}
// create new video stream
AVCodec* vcodec = avcodec_find_encoder(outctx->oformat->video_codec);
AVStream* vstrm = avformat_new_stream(outctx, vcodec);
if (!vstrm) {
std::cerr << "fail to avformat_new_stream";
return 2;
}
avcodec_get_context_defaults3(vstrm->codec, vcodec);
vstrm->codec->width = dst_width;
vstrm->codec->height = dst_height;
vstrm->codec->pix_fmt = vcodec->pix_fmts[0];
vstrm->codec->time_base = vstrm->time_base = av_inv_q(dst_fps);
vstrm->r_frame_rate = vstrm->avg_frame_rate = dst_fps;
if (outctx->oformat->flags & AVFMT_GLOBALHEADER)
vstrm->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
// open video encoder
ret = avcodec_open2(vstrm->codec, vcodec, nullptr);
if (ret < 0) {
std::cerr << "fail to avcodec_open2: ret=" << ret;
return 2;
}
std::cout
<< "outfile: " << outfile << "\n"
<< "format: " << outctx->oformat->name << "\n"
<< "vcodec: " << vcodec->name << "\n"
<< "size: " << dst_width << 'x' << dst_height << "\n"
<< "fps: " << av_q2d(dst_fps) << "\n"
<< "pixfmt: " << av_get_pix_fmt_name(vstrm->codec->pix_fmt) << "\n"
<< std::flush;
// initialize sample scaler
SwsContext* swsctx = sws_getCachedContext(
nullptr, dst_width, dst_height, AV_PIX_FMT_BGR24,
dst_width, dst_height, vstrm->codec->pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr);
if (!swsctx) {
std::cerr << "fail to sws_getCachedContext";
return 2;
}
// allocate frame buffer for encoding
AVFrame* frame = av_frame_alloc();
std::vector<uint8_t> framebuf(avpicture_get_size(vstrm->codec->pix_fmt, dst_width, dst_height));
avpicture_fill(reinterpret_cast<AVPicture*>(frame), framebuf.data(), vstrm->codec->pix_fmt, dst_width, dst_height);
frame->width = dst_width;
frame->height = dst_height;
frame->format = static_cast<int>(vstrm->codec->pix_fmt);
// encoding loop
avformat_write_header(outctx, nullptr);
int64_t frame_pts = 0;
unsigned nb_frames = 0;
bool end_of_stream = false;
int got_pkt = 0;
int i =0;
imageFileName = (char *)malloc(strlen(infile_dir)+strlen(infile_prefix)+NUMNUMBER+strlen(infile_surname)+1);
do{
if(!end_of_stream){
strcpy(imageFileName,infile_dir);
//strcat(imageFileName,"/");
strcat(imageFileName,infile_prefix);
sprintf(numberChar,"%03d",i+1);
strcat(imageFileName,numberChar);
//strcat(imageFileName,".");
strcat(imageFileName,infile_surname);
__android_log_print(1, "RecordingImage", "%s", imageFileName);
std::cout<<imageFileName<<std::endl;
AVFrame* frame_from_file = OpenImage(imageFileName);
if(!frame_from_file){
std::cout<<"error OpenImage"<<std::endl;
return 5;
}
//const int Stride [] = {1920};
sws_scale(swsctx, frame_from_file->data, &STRIDE , 0, frame_from_file->height, frame->data, frame->linesize);
frame->pts = frame_pts++;
av_frame_free(&frame_from_file);
}
// encode video frame
AVPacket pkt;
pkt.data = nullptr;
pkt.size = 0;
av_init_packet(&pkt);
ret = avcodec_encode_video2(vstrm->codec, &pkt, end_of_stream ? nullptr : frame, &got_pkt);
if (ret < 0) {
std::cerr << "fail to avcodec_encode_video2: ret=" << ret << "\n";
return 2;
}
// rescale packet timestamp
pkt.duration = 1;
av_packet_rescale_ts(&pkt, vstrm->codec->time_base, vstrm->time_base);
// write packet
av_write_frame(outctx, &pkt);
std::cout << nb_frames << '\r' << std::flush; // dump progress
++nb_frames;
av_free_packet(&pkt);
i++;
if(i==total_frames-1)
end_of_stream = true;
} while (i<total_frames);
av_write_trailer(outctx);
std::cout << nb_frames << " frames encoded" << std::endl;
av_frame_free(&frame);
avcodec_close(vstrm->codec);
avio_close(outctx->pb);
avformat_free_context(outctx);
free(imageFileName);
return 0;
}
We can create video from images using ffmpeg.
Check out my post for using ffmpeg in android.
Use below command to create video from images placed in same folder
String command[]={"-y", "-r","1/5" ,"-i",src.getAbsolutePath(),
"-c:v","libx264","-vf", "fps=25","-pix_fmt","yuv420p", dest.getAbsolutePath()};
Here ,
src.getAbsolutePath() is the absolute path of all your input images.
For example,
If all your images are stored in Images folder inside Pictures directory with names
extract_picture001.jpg,extract_picture002.jpg,extract_picture003.jpg......
.
Then,
String filePrefix = "extract_picture";
String fileExtn = ".jpg";
File picDir = Environment.getExternalStoragePublicDirectory(
Environment.DIRECTORY_PICTURES);
File dir = new File(picDir, "Images");
File src = new File(dir, filePrefix + "%03d" + fileExtn);
For creating video from images placed in different folder you
have to create a text file and add image paths to it and then specify
the path of that text file as an input option.
Example,
Text File
file '/storage/emulated/0/DCIM/Camera/P_20170807_143916.jpg'
duration 2
file '/storage/emulated/0/DCIM/Pic/P_20170305_142948.jpg'
duration 5
file '/storage/emulated/0/DCIM/Camera/P_20170305_142939.jpg'
duration 6
file '/storage/emulated/0/DCIM/Pic/P_20170305_142818.jpg'
duration 2
Command
String command[] = {"-y", "-f", "concat", "-safe", "0", "-i", textFile.getAbsolutePath(), "-vsync", "vfr", "-pix_fmt", "yuv420p", dest.getAbsolutePath()};
where textFile.getAbsolutePath() is the absolute path of your text file
Check out this ffmpeg doc for more info
Related
Using FFmpeg 4.0.2 and call its ffmpeg.c's main function twice causes Android app crash (using FFmpeg shared libs and JNI)
A/libc: Fatal signal 11 (SIGSEGV), code 1, fault addr 0x0 in tid 20153
Though it works ok for FFmpeg 3.2.5
FFmpeg 4.0.2 main
int main(int argc, char **argv) {
int i, ret;
int64_t ti;
init_dynload();
register_exit(ffmpeg_cleanup);
setvbuf(stderr,NULL,_IONBF,0); /* win32 runtime needs this */
av_log_set_flags(AV_LOG_SKIP_REPEATED);
parse_loglevel(argc, argv, options);
if(argc>1 && !strcmp(argv[1], "-d")){
run_as_daemon=1;
av_log_set_callback(log_callback_null);
argc--;
argv++;
}
#if CONFIG_AVDEVICE
avdevice_register_all();
#endif
avformat_network_init();
show_banner(argc, argv, options);
/* parse options and open all input/output files */
ret = ffmpeg_parse_options(argc, argv);
if (ret < 0)
exit_program(1);
if (nb_output_files <= 0 && nb_input_files == 0) {
show_usage();
av_log(NULL, AV_LOG_WARNING, "Use -h to get full help or, even better, run 'man %s'\n", program_name);
exit_program(1);
}
/* file converter / grab */
if (nb_output_files <= 0) {
av_log(NULL, AV_LOG_FATAL, "At least one output file must be specified\n");
exit_program(1);
}
// if (nb_input_files == 0) {
// av_log(NULL, AV_LOG_FATAL, "At least one input file must be specified\n");
// exit_program(1);
// }
for (i = 0; i < nb_output_files; i++) {
if (strcmp(output_files[i]->ctx->oformat->name, "rtp"))
want_sdp = 0;
}
current_time = ti = getutime();
if (transcode() < 0)
exit_program(1);
ti = getutime() - ti;
if (do_benchmark) {
av_log(NULL, AV_LOG_INFO, "bench: utime=%0.3fs\n", ti / 1000000.0);
}
av_log(NULL, AV_LOG_DEBUG, "%"PRIu64" frames successfully decoded, %"PRIu64" decoding errors\n",
decode_error_stat[0], decode_error_stat[1]);
if ((decode_error_stat[0] + decode_error_stat[1]) * max_error_rate < decode_error_stat[1])
exit_program(69);
ffmpeg_cleanup(received_nb_signals ? 255 : main_return_code);
return main_return_code;
}
FFmpeg 3.2.5 main
int main(int argc, char **argv) {
av_log(NULL, AV_LOG_WARNING, " Command start");
int i, ret;
int64_t ti;
init_dynload();
register_exit(ffmpeg_cleanup);
setvbuf(stderr, NULL, _IONBF, 0); /* win32 runtime needs this */
av_log_set_flags(AV_LOG_SKIP_REPEATED);
parse_loglevel(argc, argv, options);
if (argc > 1 && !strcmp(argv[1], "-d")) {
run_as_daemon = 1;
av_log_set_callback(log_callback_null);
argc--;
argv++;
}
avcodec_register_all();
#if CONFIG_AVDEVICE
avdevice_register_all();
#endif
avfilter_register_all();
av_register_all();
avformat_network_init();
av_log(NULL, AV_LOG_WARNING, " Register to complete the codec");
show_banner(argc, argv, options);
/* parse options and open all input/output files */
ret = ffmpeg_parse_options(argc, argv);
if (ret < 0)
exit_program(1);
if (nb_output_files <= 0 && nb_input_files == 0) {
show_usage();
av_log(NULL, AV_LOG_WARNING, "Use -h to get full help or, even better, run 'man %s'\n",
program_name);
exit_program(1);
}
/* file converter / grab */
if (nb_output_files <= 0) {
av_log(NULL, AV_LOG_FATAL, "At least one output file must be specified\n");
exit_program(1);
}
// if (nb_input_files == 0) {
// av_log(NULL, AV_LOG_FATAL, "At least one input file must be specified\n");
// exit_program(1);
// }
for (i = 0; i < nb_output_files; i++) {
if (strcmp(output_files[i]->ctx->oformat->name, "rtp"))
want_sdp = 0;
}
current_time = ti = getutime();
if (transcode() < 0)
exit_program(1);
ti = getutime() - ti;
if (do_benchmark) {
av_log(NULL, AV_LOG_INFO, "bench: utime=%0.3fs\n", ti / 1000000.0);
}
av_log(NULL, AV_LOG_DEBUG, "%"PRIu64" frames successfully decoded, %"PRIu64" decoding errors\n",
decode_error_stat[0], decode_error_stat[1]);
if ((decode_error_stat[0] + decode_error_stat[1]) * max_error_rate < decode_error_stat[1])
exit_program(69);
exit_program(received_nb_signals ? 255 : main_return_code);
nb_filtergraphs = 0;
nb_input_streams = 0;
nb_input_files = 0;
progress_avio = NULL;
input_streams = NULL;
nb_input_streams = 0;
input_files = NULL;
nb_input_files = 0;
output_streams = NULL;
nb_output_streams = 0;
output_files = NULL;
nb_output_files = 0;
return main_return_code;
}
So what could be issue? It seems FFmpeg 4.0.2 doesn't release something (resources or its static variables to initial values after the first command)
Adding next lines from FFmpeg 3.2.5 to FFmpeg 4.0.2 to the end of main function solved the problem (I downloaded FFmpeg 3.2.5 as someone's Android project so that user added those lines)
nb_filtergraphs = 0;
nb_input_streams = 0;
nb_input_files = 0;
progress_avio = NULL;
input_streams = NULL;
nb_input_streams = 0;
input_files = NULL;
nb_input_files = 0;
output_streams = NULL;
nb_output_streams = 0;
output_files = NULL;
nb_output_files = 0;
I am trying to make 3DR texturing but it always use only vertex colors in texture.
On every frame I store frame as PNG:
RGBImage frame(t3dr_image, 4);
std::ostringstream ss;
ss << dataset_.c_str();
ss << "/";
ss << poses_.size();
ss << ".png";
frame.Write(ss.str().c_str());
poses_.push_back(t3dr_image_pose);
timestamps_.push_back(t3dr_image.timestamp);
In the method Save I am trying to process texturing:
1) I extract full mesh from context
Tango3DR_Mesh* mesh = 0;
Tango3DR_Status ret;
ret = Tango3DR_extractFullMesh(t3dr_context_, &mesh);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
2) Create texturing context using extracted mesh
Tango3DR_ConfigH textureConfig;
textureConfig = Tango3DR_Config_create(TANGO_3DR_CONFIG_TEXTURING);
ret = Tango3DR_Config_setDouble(textureConfig, "min_resolution", 0.01);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
Tango3DR_TexturingContext context;
context = Tango3DR_createTexturingContext(textureConfig, dataset.c_str(), mesh);
if (context == nullptr)
std::exit(EXIT_SUCCESS);
Tango3DR_Config_destroy(textureConfig);
3) Call Tango3DR_updateTexture with data I stored before (this does not work)
for (unsigned int i = 0; i < poses_.size(); i++) {
std::ostringstream ss;
ss << dataset_.c_str();
ss << "/";
ss << i;
ss << ".png";
RGBImage frame(ss.str());
Tango3DR_ImageBuffer image;
image.width = frame.GetWidth();
image.height = frame.GetHeight();
image.stride = frame.GetWidth() * 3;
image.timestamp = timestamps_[i];
//data are for sure in this format
image.format = TANGO_3DR_HAL_PIXEL_FORMAT_RGB_888;
image.data = frame.GetData();
ret = Tango3DR_updateTexture(context, &image, &poses_[i]);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
}
4) Texturize mesh
ret = Tango3DR_Mesh_destroy(mesh);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
mesh = 0;
ret = Tango3DR_getTexturedMesh(context, &mesh);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
5) Save it as OBJ (in the result texture are only data from vertex colors, why?)
ret = Tango3DR_Mesh_saveToObj(mesh, filename.c_str());
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
ret = Tango3DR_destroyTexturingContext(context);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
ret = Tango3DR_Mesh_destroy(mesh);
if (ret != TANGO_3DR_SUCCESS)
std::exit(EXIT_SUCCESS);
All methods returned TANGO_3DR_SUCCESS.
Full code here: https://github.com/lvonasek/tango
Thanks for reaching out and providing the detailed code breakdown.
The error is on our end - the library currently doesn't support RGB texture inputs. It assumes YUV for all input images. I've opened a ticket to track this bug and we'll fix it for the next release, by allowing RGB input and providing better return values for invalid image formats.
Edit: Found another bug on our end. The API states image_pose should be the pose of the image, but our implementation actually expects the pose of the device. I've opened a bug, and this will be fixed in next release (release-H).
You can try working around this for now by passing in the device pose without multiplying the device-to-camera extrinsic calibration, although of course that's just a temp bandaid.
I just try to make a simple demo with encoder in Gstreamer 1.0. However it does not work. Always return error when calling "push-buffer".
#define BUFFER_SIZE 1228800 //640x480x 4 byte/pixel
static void cb_need_video_data (GstElement *appsrc,guint unused_size, CustomData* user_data)
{
static gboolean white = FALSE;
static GstClockTime timestamp = 0;
GstBuffer *buffer;
guint size = BUFFER_SIZE;
GstFlowReturn ret;
user_data->num_frame++;
if (user_data->num_frame >= TOTAL_FRAMES) {
/* we are EOS, send end-of-stream and remove the source */
g_signal_emit_by_name (user_data->source, "end-of-stream", &ret);
g_print ("start TOTAL_FRAMES");
return ;
}
buffer = gst_buffer_new_allocate (NULL, size, NULL);
gst_buffer_memset (buffer, 0, white ? 0xff : 0x0, size);
white = !white;
GST_BUFFER_PTS (buffer) = timestamp;
GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale_int (1, GST_SECOND, 2);
timestamp += GST_BUFFER_DURATION (buffer);
g_signal_emit_by_name (appsrc, "push-buffer", buffer, &ret);
if (ret != GST_FLOW_OK) {
/* something wrong, stop pushing */
GST_DEBUG("QUIT MAIN LOOP %d",ret);
g_main_loop_quit (user_data->main_loop);
}
}
void gst_set_pipeline(JNIEnv* env, jobject thiz,jstring uri){
CustomData *data = GET_CUSTOM_DATA (env, thiz, custom_data_field_id);
gchar *string = NULL;
string =
g_strdup_printf ("appsrc is-live=true name=testsource "
"queue ! videoconvert ! videoscale "
"add-borders=true ! videorate ! x264enc "
"! mpegtsmux "
"! filesink "
"location=/storage/emulated/0/test.avi sync=false append=true",
data->pipeline = gst_parse_launch (string, NULL);
g_free (string);
if (data->pipeline == NULL) {
g_print ("Bad sink\n");
return;
}
data->vsource = gst_bin_get_by_name (GST_BIN (data->pipeline), "testsource");
GstCaps *caps;
caps = gst_caps_new_simple ("video/x-raw",
"format", G_TYPE_STRING, "RGB",
"width", G_TYPE_INT, 640,
"height", G_TYPE_INT, 480,
"framerate", GST_TYPE_FRACTION, 25, 1,
NULL);
gst_app_src_set_caps(GST_APP_SRC(data->vsource), caps);
g_object_set (G_OBJECT(data->vsource),"stream-type", 0, "format", GST_FORMAT_TIME, NULL);
g_signal_connect (data->vsource, "need-data", G_CALLBACK (cb_need_video_data), data);
}
When I set state to GST_STATE_PLAYING, it go into cb_need_video_data once time and get ret != GST_FLOW_OK . I think the caps and input data don't match, but I don't know why.
I'm just newbie in Gstreamer. Please help me correct above code. Thanks all
I'm using the following C function to decode packets in Android (with JNI). When I play an mp3 file the code works fine however and wma file results in choppy audio. I suspect the issue may be with the "swr_convert" function and the data_size I'm using but I'm not sure. Does anyone know why this would be happening?
int decodeFrameFromPacket(AVPacket *aPacket) {
int n;
AVPacket *pkt = aPacket;
AVFrame *decoded_frame = NULL;
int got_frame = 0;
if (aPacket->stream_index == global_audio_state->audio_stream) {
if (!decoded_frame) {
if (!(decoded_frame = avcodec_alloc_frame())) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Could not allocate audio frame\n");
return -2;
}
}
if (avcodec_decode_audio4(global_audio_state->audio_st->codec, decoded_frame, &got_frame, aPacket) < 0) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Error while decoding\n");
return -2;
}
int data_size = 0;
if (got_frame) {
/* if a frame has been decoded, output it */
data_size = av_samples_get_buffer_size(NULL, global_audio_state->audio_st->codec->channels,
decoded_frame->nb_samples,
global_audio_state->audio_st->codec->sample_fmt, 1);
}
swr_convert(global_audio_state->swr, (uint8_t **) &gAudioFrameRefBuffer, decoded_frame->nb_samples, (uint8_t const **) decoded_frame->data, decoded_frame->nb_samples);
avcodec_free_frame(&decoded_frame);
gAudioFrameDataLengthRefBuffer[0] = data_size;
return AUDIO_DATA_ID;
}
return 0;
}
I'm trying to play an audio stream with ffmpeg and opensles in android. And the problem seems to be when passing the decoded and resampled frames from ffmpeg to opensles as the sound I can hear sounds robotic and has scratching.
The decoded frames from ffmpeg:
PCM
48000 Hz
S16p
Opensles needs in this case:
PCM
48000 Hz
S16
Opensles setup:
SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 255};
SLDataFormat_PCM format_pcm = { SL_DATAFORMAT_PCM, 2 , SL_SAMPLINGRATE_48, SL_PCMSAMPLEFORMAT_FIXED_16, SL_PCMSAMPLEFORMAT_FIXED_16,
SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT, SL_BYTEORDER_LITTLEENDIAN};
SLDataSource audioSrc = {&loc_bufq, &format_pcm};
This is the pseudocode for resampling and enqueueing to opensles:
#define OPENSLES_BUFLEN 10
#define MAX_AUDIO_FRAME_SIZE 192000
DECLARE_ALIGNED(16,uint8_t,audio_buffer)[MAX_AUDIO_FRAME_SIZE * OPENSLES_BUFLEN];
int decode_audio(AVCodecContext * ctx, SwrContext *swr_context, AVPacket *packet, AVFrame * frame){
int got_frame_ptr;
int len = avcodec_decode_audio4(ctx, frame, &got_frame_ptr, packet);
if(!got_frame_ptr)
return -ERROR;
int original_data_size = av_samples_get_buffer_size(NULL, ctx->channels,
frame->nb_samples, ctx->sample_fmt, 1);
uint8_t *audio_buf;
int data_size;
if (swr_context != NULL) {
uint8_t *out[] = { audio_buffer };
int sample_per_buffer_divider = 2* av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);;
int len2 = swr_convert(swr_context, out,
sizeof(audio_buffer) / sample_per_buffer_divider,
frame->extended_data, frame->nb_samples);
if (len2 < 0) {
return -ERROR;
}
if (len2 == sizeof(audio_buffer) / sample_per_buffer_divider) {
swr_init(swr_context);
}
audio_buf = audio_buffer;
data_size = len2 * sample_per_buffer_divider;
}
else {
audio_buf = frame->data[0];
data_size = original_data_size;
}
(*opengSLESData->bqPlayerBufferQueue)->Enqueue(opengSLESData->bqPlayerBufferQueue, audio_buf, data_size)
}
I would appreciate any help, thanks.
example may help
#include "stdafx.h"
#include <iostream>
extern "C"
{
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
//#include "swscale.h"
#include "libswresample/swresample.h"
};
FILE *fin, *fout;
int ffmpeg_audio_decode( const char * inFile, const char * outFile)
{
// Initialize FFmpeg
av_register_all();
AVFrame* frame = avcodec_alloc_frame();
if (!frame)
{
std::cout << "Error allocating the frame" << std::endl;
return 1;
}
// you can change the file name "01 Push Me to the Floor.wav" to whatever the file is you're reading, like "myFile.ogg" or
// "someFile.webm" and this should still work
AVFormatContext* formatContext = NULL;
//if (avformat_open_input(&formatContext, "01 Push Me to the Floor.wav", NULL, NULL) != 0)
if (avformat_open_input(&formatContext, inFile, NULL, NULL) != 0)
{
av_free(frame);
std::cout << "Error opening the file" << std::endl;
return 1;
}
if (avformat_find_stream_info(formatContext, NULL) < 0)
{
av_free(frame);
av_close_input_file(formatContext);
std::cout << "Error finding the stream info" << std::endl;
return 1;
}
AVStream* audioStream = NULL;
// Find the audio stream (some container files can have multiple streams in them)
for (unsigned int i = 0; i < formatContext->nb_streams; ++i)
{
if (formatContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
audioStream = formatContext->streams[i];
break;
}
}
if (audioStream == NULL)
{
av_free(frame);
av_close_input_file(formatContext);
std::cout << "Could not find any audio stream in the file" << std::endl;
return 1;
}
AVCodecContext* codecContext = audioStream->codec;
codecContext->codec = avcodec_find_decoder(codecContext->codec_id);
if (codecContext->codec == NULL)
{
av_free(frame);
av_close_input_file(formatContext);
std::cout << "Couldn't find a proper decoder" << std::endl;
return 1;
}
else if (avcodec_open2(codecContext, codecContext->codec, NULL) != 0)
{
av_free(frame);
av_close_input_file(formatContext);
std::cout << "Couldn't open the context with the decoder" << std::endl;
return 1;
}
std::cout << "This stream has " << codecContext->channels << " channels and a sample rate of " << codecContext->sample_rate << "Hz" << std::endl;
std::cout << "The data is in the format " << av_get_sample_fmt_name(codecContext->sample_fmt) << std::endl;
//codecContext->sample_fmt = AV_SAMPLE_FMT_S16;
int64_t outChannelLayout = AV_CH_LAYOUT_MONO; //AV_CH_LAYOUT_STEREO;
AVSampleFormat outSampleFormat = AV_SAMPLE_FMT_S16; // Packed audio, non-planar (this is the most common format, and probably what you want; also, WAV needs it)
int outSampleRate = 8000;//44100;
// Note that AVCodecContext::channel_layout may or may not be set by libavcodec. Because of this,
// we won't use it, and will instead try to guess the layout from the number of channels.
SwrContext* swrContext = swr_alloc_set_opts(NULL,
outChannelLayout,
outSampleFormat,
outSampleRate,
av_get_default_channel_layout(codecContext->channels),
codecContext->sample_fmt,
codecContext->sample_rate,
0,
NULL);
if (swrContext == NULL)
{
av_free(frame);
avcodec_close(codecContext);
avformat_close_input(&formatContext);
std::cout << "Couldn't create the SwrContext" << std::endl;
return 1;
}
if (swr_init(swrContext) != 0)
{
av_free(frame);
avcodec_close(codecContext);
avformat_close_input(&formatContext);
swr_free(&swrContext);
std::cout << "Couldn't initialize the SwrContext" << std::endl;
return 1;
}
fout = fopen(outFile, "wb+");
AVPacket packet;
av_init_packet(&packet);
// Read the packets in a loop
while (av_read_frame(formatContext, &packet) == 0)
{
if (packet.stream_index == audioStream->index)
{
AVPacket decodingPacket = packet;
while (decodingPacket.size > 0)
{
// Try to decode the packet into a frame
int frameFinished = 0;
int result = avcodec_decode_audio4(
codecContext,
frame,
&frameFinished,
&decodingPacket);
if (result < 0 || frameFinished == 0)
{
break;
}
unsigned char buffer[100000] = {NULL};
unsigned char* pointers[SWR_CH_MAX] = {NULL};
pointers[0] = &buffer[0];
int numSamplesOut = swr_convert(
swrContext,
pointers,
outSampleRate,
(const unsigned char**)frame->extended_data,
frame->nb_samples);
fwrite(
(short *)buffer,
sizeof(short),
(size_t)numSamplesOut,
fout);
decodingPacket.size -= result;
decodingPacket.data += result;
}
}
// You *must* call av_free_packet() after each call to av_read_frame() or else you'll leak memory
av_free_packet(&packet);
}
// Some codecs will cause frames to be buffered up in the decoding process. If the CODEC_CAP_DELAY flag
// is set, there can be buffered up frames that need to be flushed, so we'll do that
if (codecContext->codec->capabilities & CODEC_CAP_DELAY)
{
av_init_packet(&packet);
// Decode all the remaining frames in the buffer, until the end is reached
int frameFinished = 0;
while (avcodec_decode_audio4(codecContext, frame, &frameFinished, &packet) >= 0 && frameFinished)
{
}
}
// Clean up!
av_free(frame);
avcodec_close(codecContext);
av_close_input_file(formatContext);
fclose(fout);
}