From f005b19ee49494059b4ad86ff95716a66eca2c00 Mon Sep 17 00:00:00 2001 From: JIe Jie Date: Fri, 23 Feb 2024 16:24:42 +0800 Subject: [PATCH] =?UTF-8?q?Audio=E6=92=AD=E6=94=BE=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/audioDecoder.h | 9 +++ include/decodeParam.h | 22 +++++-- include/mediaDecoder.h | 12 +--- include/util.h | 4 +- main.cc | 36 +++++++++-- src/audioDecoder.cc | 130 +++++++++++++++++++++++++++++++++++++- src/mediaDecoder.cc | 29 +++++---- src/tempCodeRunnerFile.cc | 0 8 files changed, 206 insertions(+), 36 deletions(-) create mode 100644 src/tempCodeRunnerFile.cc diff --git a/include/audioDecoder.h b/include/audioDecoder.h index e69de29..7adcae0 100644 --- a/include/audioDecoder.h +++ b/include/audioDecoder.h @@ -0,0 +1,9 @@ +#ifndef AUDIODECODER_H +#define AUDIODECODER_H +#include + +int RequestAudioFrame(AudioParam& param, uint8_t* audioBuffer, int bufSize); +void audioCallback(void* userdata, uint8_t* stream, int len); +void RequestAudioPacket(MediaParam& param); + +#endif diff --git a/include/decodeParam.h b/include/decodeParam.h index 9f3207f..6856ce1 100644 --- a/include/decodeParam.h +++ b/include/decodeParam.h @@ -1,5 +1,17 @@ #ifndef DECODEPARAM_H #define DECODEPARAM_H + +extern "C" { +#include "libavcodec/avcodec.h" +#include "libavformat/avformat.h" +#include "libavutil/imgutils.h" +} +#include +#include +#include +#include +using namespace std::literals::chrono_literals; + template requires std::is_same_v || std::is_same_v struct MediaQueue @@ -83,7 +95,6 @@ struct VideoParam { MediaQueue packetQueue; MediaQueue frameQueue; - AVFormatContext* fmtCtx; AVCodecContext* codecCtx; int width; int height; @@ -98,13 +109,13 @@ struct VideoParam struct AudioParam { MediaQueue packetQueue; - MediaQueue frameQueue; - AVFormatContext* fmtCtx; AVCodecContext* codecCtx; int audioStreamIndex; - + static constexpr int MAX_BUFFER_SIZE = 192000; + uint8_t* buffer = new uint8_t[MAX_BUFFER_SIZE]; + uint32_t bufferSize = 0; + uint32_t bufferIndex = 0; bool eof = false; - bool pause = false; bool quit = false; }; @@ -113,5 +124,6 @@ struct MediaParam { VideoParam videoParam; AudioParam audioParam; + AVFormatContext* fmtCtx; }; #endif \ No newline at end of file diff --git a/include/mediaDecoder.h b/include/mediaDecoder.h index f00f4c6..90fee8a 100644 --- a/include/mediaDecoder.h +++ b/include/mediaDecoder.h @@ -1,17 +1,9 @@ #ifndef DECODER_H #define DECODER_H -extern "C" { -#include "libavcodec/avcodec.h" -#include "libavformat/avformat.h" -#include "libavutil/imgutils.h" -} -#include -#include -#include #include "decodeParam.h" -void InitDecoder(const char* filepath, VideoParam& param); -void RequestPacket(MediaParam& param); +void InitDecoder(const char* filepath, MediaParam& param); +void RequestVideoPacket(MediaParam& param); void RequestVideoFrame(MediaParam& param); #endif \ No newline at end of file diff --git a/include/util.h b/include/util.h index c068791..7e0f3e5 100644 --- a/include/util.h +++ b/include/util.h @@ -8,7 +8,7 @@ #include #include enum class FileType { - MUSIC, + AUDIO, VIDEO, IMG, ERRORTYPE @@ -64,7 +64,7 @@ private: } public: static FileType GetFileType(const path& filepath) { - if (IsMusic(filepath)) return FileType::MUSIC; + if (IsMusic(filepath)) return FileType::AUDIO; if (IsVideo(filepath)) return FileType::VIDEO; if (IsImg(filepath)) return FileType::IMG; return FileType::ERRORTYPE; diff --git a/main.cc b/main.cc index 29678ec..2a2b25c 100644 --- a/main.cc +++ b/main.cc @@ -9,6 +9,7 @@ #include "mediaDecoder.h" #include "shaderService.h" #include "shader.h" +#include "audioDecoder.h" using std::cout; struct OpenglVideoParam @@ -18,11 +19,32 @@ struct OpenglVideoParam unsigned int texs[3]; }; +int InitAudio(const char* targetFilePath, MediaParam& param) +{ + InitDecoder(targetFilePath, param); + std::jthread(RequestAudioPacket, std::ref(param)).detach(); + SDL_AudioSpec des; + des.freq = param.audioParam.codecCtx->sample_rate; + des.channels = param.audioParam.codecCtx->channels; + des.format = AUDIO_S16SYS; + des.samples = 1024; + des.silence = 0; + des.userdata = &(param.audioParam); + des.callback = audioCallback; + if (SDL_OpenAudio(&des, nullptr) < 0) + { + cout << SDL_GetError() << "\n"; + return -1; + } + SDL_PauseAudio(0); + return 0; +} + int InitVideo(SDL_Window*& window, const char* targetFilepath, MediaParam& param, OpenglVideoParam& openglVideoParam, ShaderService*& shaderService) { - InitDecoder(targetFilepath, param.videoParam); + InitDecoder(targetFilepath, param); //FIX: when app exited, the fmtCtx was freed, so need notify decode thread to stop decode and exit. - std::jthread(RequestPacket, std::ref(param)).detach(); + std::jthread(RequestVideoPacket, std::ref(param)).detach(); std::jthread(RequestVideoFrame, std::ref(param)).detach(); const int client_width = param.videoParam.width / 2; const int client_height = param.videoParam.height / 2; @@ -152,7 +174,6 @@ void OpenglRenderVideo(MediaParam& param, const OpenglVideoParam& openglVideoPar glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr); } - int main(int argc, char** argv) { // Check File @@ -189,7 +210,7 @@ int main(int argc, char** argv) case FileType::VIDEO: { InitVideo(window, targetFilepath, mediaParam, openglVideoParam, shaderService); - const auto stream_frame_rate = mediaParam.videoParam.fmtCtx->streams[mediaParam.videoParam.videoStreamIndex]->avg_frame_rate; + const auto stream_frame_rate = mediaParam.fmtCtx->streams[mediaParam.videoParam.videoStreamIndex]->avg_frame_rate; framerate = static_cast(stream_frame_rate.den) / stream_frame_rate.num; break; } @@ -198,8 +219,9 @@ int main(int argc, char** argv) InitImg(window, targetFilepath, renderer, surface, texture); break; } - case FileType::MUSIC: + case FileType::AUDIO: { + InitAudio(targetFilepath, mediaParam); break; } case FileType::ERRORTYPE: @@ -245,13 +267,15 @@ int main(int argc, char** argv) case FileType::IMG: RenderPicture(window, renderer, texture); break; + case FileType::AUDIO: + break; default: break; } } avcodec_close(mediaParam.videoParam.codecCtx); - avformat_close_input(&(mediaParam.videoParam.fmtCtx)); + avformat_close_input(&(mediaParam.fmtCtx)); SDL_GL_DeleteContext(openglVideoParam.glContext); SDL_DestroyWindow(window); SDL_Quit(); diff --git a/src/audioDecoder.cc b/src/audioDecoder.cc index 459fbcc..742a908 100644 --- a/src/audioDecoder.cc +++ b/src/audioDecoder.cc @@ -1,2 +1,130 @@ -#include"audioDecoder.h" +#include "audioDecoder.h" +#include +#include + +extern "C" { +#include "libswresample/swresample.h" +} + +int RequestAudioFrame(AudioParam& param, uint8_t* audioBuffer, int bufSize) +{ + AVFrame* frame = av_frame_alloc(); + int dataSize = 0; + AVPacket packet; + SwrContext* swrCtx = nullptr; + if (param.quit) { + return -1; + } + if (!param.packetQueue.pop(&packet, true)) { + return -1; + } + int ret = avcodec_send_packet(param.codecCtx, &packet); + if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { + return -1; + } + ret = avcodec_receive_frame(param.codecCtx, frame); + if (ret < 0 && ret != AVERROR_EOF) { + av_frame_unref(frame); + return -1; + } + if (frame->best_effort_timestamp == AV_NOPTS_VALUE) + { + av_frame_unref(frame); + return -1; + } + + if (frame->channels > 0 && frame->channel_layout == 0) { + frame->channel_layout = av_get_default_channel_layout(frame->channels); + } + else if (frame->channels == 0 && frame->channel_layout > 0) { + frame->channels = av_get_channel_layout_nb_channels(frame->channel_layout); + } + + AVSampleFormat dstFormat = AV_SAMPLE_FMT_S16; + uint64_t dstLayout = av_get_default_channel_layout(frame->channels); + swrCtx = swr_alloc_set_opts(nullptr, dstLayout, dstFormat, frame->sample_rate, frame->channel_layout, (AVSampleFormat)frame->format, frame->sample_rate, 0, nullptr); + if (!swrCtx || swr_init(swrCtx) < 0) { + av_frame_unref(frame); + return -1; + } + + uint64_t dstNbSamples = av_rescale_rnd(swr_get_delay(swrCtx, frame->sample_rate) + frame->nb_samples, frame->sample_rate, frame->sample_rate, AVRounding(1)); + int nb = swr_convert(swrCtx, &audioBuffer, dstNbSamples, const_cast(frame->data), frame->nb_samples); + dataSize = frame->channels * nb * av_get_bytes_per_sample(dstFormat); + av_frame_free(&frame); + swr_free(&swrCtx); + return dataSize; +} + +void audioCallback(void* userdata, uint8_t* stream, int len) { + AudioParam* param = static_cast(userdata); + SDL_memset(stream, 0, len); + int audioSize = 0; + int len1 = 0; + while (len > 0) + { + if (param->bufferIndex >= param->bufferSize) + { + audioSize = RequestAudioFrame(*param, param->buffer, sizeof(param->buffer)); + if (audioSize < 0) + { + param->bufferSize = 0; + memset(param->buffer, 0, param->bufferSize); + } + else + { + param->bufferSize = audioSize; + } + param->bufferIndex = 0; + } + len1 = param->bufferSize - param->bufferIndex; + if (len1 > len) + len1 = len; + + SDL_MixAudio(stream, param->buffer + param->bufferIndex, len1, SDL_MIX_MAXVOLUME); + len -= len1; + stream += len1; + param->bufferIndex += len1; + } +} + +void RequestAudioPacket(MediaParam& param) { + const auto& fmtCtx = param.fmtCtx; + const auto& audioStreamIndex = param.audioParam.audioStreamIndex; + + AVPacket* packet = av_packet_alloc(); + + while (true) { + if (param.audioParam.packetQueue.isFill()) { + std::this_thread::sleep_for(100ms); + continue; + } + //FIX: + const int ret = av_read_frame(fmtCtx, packet); + if (param.audioParam.eof) { + std::this_thread::sleep_for(100ms); + av_packet_unref(packet); + return; + } + if (ret == 0) { + if (packet->stream_index == audioStreamIndex) { + param.audioParam.packetQueue.push(packet); + av_packet_unref(packet); + } + else if (ret == AVERROR_EOF) + { + param.audioParam.eof = true; + av_packet_unref(packet); + break; + } + else { + av_packet_unref(packet); + } + } + else if (param.fmtCtx->pb->error == 0) { + std::this_thread::sleep_for(100ms); + } + } + av_packet_unref(packet); +} \ No newline at end of file diff --git a/src/mediaDecoder.cc b/src/mediaDecoder.cc index aa99756..e434ce1 100644 --- a/src/mediaDecoder.cc +++ b/src/mediaDecoder.cc @@ -3,7 +3,7 @@ #include using namespace std::literals::chrono_literals; -void InitDecoder(const char* filepath, VideoParam& param) { +void InitDecoder(const char* filepath, MediaParam& param) { AVFormatContext* fmtCtx = nullptr; AVCodecContext* codecFmt = nullptr; auto ret = avformat_open_input(&fmtCtx, filepath, NULL, NULL); @@ -13,21 +13,28 @@ void InitDecoder(const char* filepath, VideoParam& param) { const auto stream = fmtCtx->streams[i]; const auto codec = avcodec_find_decoder(stream->codecpar->codec_id); if (codec->type == AVMEDIA_TYPE_VIDEO) { - param.videoStreamIndex = i; + param.videoParam.videoStreamIndex = i; codecFmt = avcodec_alloc_context3(codec); avcodec_parameters_to_context(codecFmt, stream->codecpar); avcodec_open2(codecFmt, codec, nullptr); + param.videoParam.codecCtx = codecFmt; + param.videoParam.width = codecFmt->width; + param.videoParam.height = codecFmt->height; + } + else if (codec->type == AVMEDIA_TYPE_AUDIO) + { + param.audioParam.audioStreamIndex = i; + codecFmt = avcodec_alloc_context3(codec); + avcodec_parameters_to_context(codecFmt, stream->codecpar); + avcodec_open2(codecFmt, codec, nullptr); + param.audioParam.codecCtx = codecFmt; } } - - param.codecCtx = codecFmt; param.fmtCtx = fmtCtx; - param.width = codecFmt->width; - param.height = codecFmt->height; } -void RequestPacket(MediaParam& param) { - const auto& fmtCtx = param.videoParam.fmtCtx; +void RequestVideoPacket(MediaParam& param) { + const auto& fmtCtx = param.fmtCtx; const auto& videoStreamIndex = param.videoParam.videoStreamIndex; AVPacket* packet = av_packet_alloc(); @@ -37,7 +44,7 @@ void RequestPacket(MediaParam& param) { std::this_thread::sleep_for(100ms); continue; } - //FIX: + //FIX: const int ret = av_read_frame(fmtCtx, packet); if (param.videoParam.eof) { std::this_thread::sleep_for(100ms); @@ -59,7 +66,7 @@ void RequestPacket(MediaParam& param) { av_packet_unref(packet); } } - else if (param.videoParam.fmtCtx->pb->error == 0) { + else if (param.fmtCtx->pb->error == 0) { std::this_thread::sleep_for(100ms); } } @@ -67,9 +74,7 @@ void RequestPacket(MediaParam& param) { } void RequestVideoFrame(MediaParam& param) { - const auto& fmtCtx = param.videoParam.fmtCtx; const auto& codecCtx = param.videoParam.codecCtx; - const auto& videoStreamIndex = param.videoParam.videoStreamIndex; AVPacket* packet = av_packet_alloc(); AVFrame* frame = av_frame_alloc(); while (true) { diff --git a/src/tempCodeRunnerFile.cc b/src/tempCodeRunnerFile.cc new file mode 100644 index 0000000..e69de29