compile_flags.txt
main
out.flv
+test.mkv
test.mp4
echo -xc++ > $@
pkg-config --cflags --libs $(LIBS) | xargs printf '%s\n' >> $@
+debug: main
+ gdb ./main
+
run: main
./main
-.PHONY: compile_flags.txt run
+.PHONY: compile_flags.txt debug run
--- /dev/null
+#ifndef TEST_APP_APPLICATION_H_
+#define TEST_APP_APPLICATION_H_
+
+#include <thread>
+#include <json/value.h>
+
+#include "Mixer.h"
+#include "Renderer.h"
+#include "State.h"
+#include "Stream.h"
+#include "../ffmpeg/Network.h"
+#include "../uv/Loop.h"
+#include "../ws/Connection.h"
+#include "../ws/Context.h"
+
+namespace app {
+
+class Application {
+
+public:
+ Application(int width, int height, int fps, const char *url)
+ : net()
+ , loop()
+ , ws_ctx(loop)
+ , ws_conn(ws_ctx.GetContext())
+ , stream(url, width, height, fps)
+ , mixer(stream.GetAudioPlane(), stream.GetAudioChannels(), stream.GetAudioFrameSize())
+ , renderer(stream.GetVideoPlane(), stream.GetVideoLineSize(), width, height)
+ , state(width, height) {
+ }
+ ~Application() {
+ }
+
+ Application(const Application &) = delete;
+ Application &operator =(const Application &) = delete;
+
+public:
+ void Start() {
+ ws_conn.Subscribe("ChatBotLog", &WsHandler, this);
+ stream.Start();
+
+ //Media &media = state.AddMedia("test.mp4");
+ //Clock sync_point = stream.GetVideoClock();
+ //sync_point.Advance(600);
+ //media.SetSyncPoint(sync_point);
+ //media.AddWindow({ 0, 0, 1920, 1080 }, { 600, 50, 640, 360 });
+
+ //Media &media = state.AddMedia("test.mkv");
+ //Clock sync_point = stream.GetVideoClock();
+ //sync_point.Advance(600);
+ //media.SetSyncPoint(sync_point);
+ //media.AddWindow({ 0, 0, 1280, 720 }, { 600, 50, 640, 360 });
+ }
+
+ void Step() {
+ loop.TryStep();
+
+ const int64_t target = stream.GetVideoClock().GetMS();
+ const int64_t elapsed = stream.TimeElapsedMS();
+ const int64_t difference = target - elapsed;
+
+ stream.PrepareVideoFrame();
+ state.PullVideo(stream.GetVideoClock());
+
+ if (target > 0 && difference < 0) {
+ std::cout << (difference / -1000.0) << "s behind schedule, dropping frame" << std::endl;
+ } else {
+ state.Update(stream.GetVideoClock());
+ renderer.RenderVideoFrame(state);
+ }
+
+ stream.PushVideoFrame();
+
+ while (stream.GetAudioClock().GetMS() < target) {
+ stream.PrepareAudioFrame();
+ state.PullAudio(stream.GetAudioClock(), stream.GetAudioFrameSize());
+ mixer.RenderAudioFrame(state, stream.GetAudioClock());
+ stream.PushAudioFrame();
+ }
+
+ state.Clean();
+
+ if (difference > 3000) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(difference - 3000));
+ }
+ }
+
+ void Stop() {
+ ws_ctx.Shutdown();
+ stream.Finish();
+ }
+
+private:
+ static void WsHandler(void *user, const Json::Value &json) {
+ Application *app = static_cast<Application *>(user);
+ app->HandleWebSocket(json);
+ }
+
+ void HandleWebSocket(const Json::Value &json) {
+ const std::string data_string = json["data"].asString();
+ Json::Value data;
+ Json::Reader json_reader;
+ json_reader.parse(data_string, data);
+ const std::string text = data["model"]["text"].asString();
+ const std::string channel = data["model"]["channel"]["title"].asString();
+ if (text.length() > 0) {
+ PushMessage(text, channel);
+ }
+ }
+
+ void PushMessage(const std::string &text, const std::string &channel) {
+ Message &msg = renderer.CreateMessage(state);
+ msg.SetWidth(state.GetWidth() / 2.0);
+ msg.SetText(text);
+ msg.SetChannel(channel);
+ msg.SetBirth(stream.GetVideoClock().Snapshot());
+ msg.Update(renderer.GetContext());
+ }
+
+private:
+ ffmpeg::Network net;
+ uv::Loop loop;
+ ws::Context ws_ctx;
+ ws::Connection ws_conn;
+ Stream stream;
+ Mixer mixer;
+ Renderer renderer;
+ State state;
+
+};
+
+}
+
+#endif
--- /dev/null
+#ifndef TEST_APP_AUDIOFRAMESNAPSHOT_H_
+#define TEST_APP_AUDIOFRAMESNAPSHOT_H_
+
+#include <cstdint>
+#include <vector>
+
+#include "Clock.h"
+
+namespace app {
+
+class AudioFrameSnapshot {
+
+public:
+ AudioFrameSnapshot(const float *plane, int channels, int size, const Clock &time)
+ : plane(plane, plane + (channels * size)), channels(channels), size(size), time(time) {
+ }
+ ~AudioFrameSnapshot() {
+ }
+
+public:
+ const Clock &GetStartTime() const {
+ return time;
+ }
+
+ Clock GetEndTime() const {
+ Clock end = time;
+ end.Advance(size);
+ return end;
+ }
+
+ int GetChannels() const {
+ return channels;
+ }
+
+ int GetSize() const {
+ return size;
+ }
+
+ float GetSample(int sample, int channel) const {
+ return plane[sample * channels + channel];
+ }
+
+private:
+ std::vector<float> plane;
+ int channels;
+ int size;
+ Clock time;
+
+};
+
+}
+
+#endif
#ifndef TEST_APP_CLOCK_H_
#define TEST_APP_CLOCK_H_
+#include <cstdint>
+#include <iomanip>
+#include <ostream>
extern "C" {
#include <libavutil/mathematics.h>
#include <libavutil/rational.h>
return Interpolate(from, to, from_ms, to_ms);
}
+ void Set(int64_t ts) {
+ counter = ts;
+ }
+
void Reset() {
counter = 0;
}
return av_rescale_q(counter, timebase, AVRational{1, 1000});
}
+ const AVRational &GetTimebase() const {
+ return timebase;
+ }
+
private:
AVRational timebase;
int64_t counter;
};
+inline std::ostream &operator <<(std::ostream &out, const Clock &clock) {
+ int64_t millis = clock.GetMS();
+ int64_t abs_millis = abs(millis);
+ int64_t ms = abs_millis % 1000;
+ int64_t secs = (abs_millis / 1000) % 60;
+ int64_t mins = (abs_millis / (60 * 1000)) % 60;
+ int64_t hrs = abs_millis / (60 * 60 * 1000);
+ if (millis < 0) {
+ out << '-';
+ }
+ if (hrs != 0) {
+ out << hrs << ':';
+ }
+ out << std::setw(2) << std::setfill('0') << mins << ':';
+ out << std::setw(2) << std::setfill('0') << secs << '.';
+ out << std::setw(3) << std::setfill('0') << ms;
+ return out << " (" << clock.GetCounter() << " @ " << clock.GetTimebase().num << '/' << clock.GetTimebase().den << ')';
+}
+
}
#endif
--- /dev/null
+#ifndef TEST_APP_MEDIA_H_
+#define TEST_APP_MEDIA_H_
+
+#include <unistd.h>
+#include <vector>
+
+#include "Clock.h"
+#include "Source.h"
+#include "Window.h"
+#include "../cairo/Context.h"
+#include "../gfx/Rectangle.h"
+
+namespace app {
+
+class Media {
+
+public:
+ explicit Media(const char *url)
+ : source(url)
+ , surface(source.GetVideoSurface()) {
+ }
+ ~Media() {
+ }
+
+ Media(const Media &) = delete;
+ Media &operator =(const Media &) = delete;
+
+public:
+ void SetSyncPoint(const Clock &s) {
+ sync_point = s;
+ }
+
+ void AddWindow(const gfx::Rectangle &src, const gfx::Rectangle &dst) {
+ windows.push_back({ src, dst });
+ }
+
+ void PullAudio(const Clock &clock, int frame_size) {
+ Clock diff = clock.Difference(sync_point);
+ source.SeekAudio(diff, frame_size);
+ }
+
+ void PullVideo(const Clock &clock) {
+ Clock diff = clock.Difference(sync_point);
+ source.SeekVideo(diff);
+ surface.MarkDirty();
+ }
+
+ void Render(cairo::Context &ctx) const {
+ if (!source.HasSeenVideo()) return;
+ for (const Window &win : windows) {
+ ctx.DrawSurface(surface, win.src, win.dst);
+ }
+ }
+
+ void Mix(const Clock &clock, float *plane, int channels, int frame_size) const {
+ Clock diff = clock.Difference(sync_point);
+ source.Mix(diff, plane, channels, frame_size);
+ }
+
+ bool IsEOF() const {
+ return source.IsEOF();
+ }
+
+private:
+ Source source;
+ cairo::Surface surface;
+ Clock sync_point;
+ std::vector<Window> windows;
+
+};
+
+}
+
+#endif
public:
Message(cairo::Context &ctx)
- : ctx(ctx)
- , text_layout(ctx.CreateLayout())
+ : text_layout(ctx.CreateLayout())
, channel_layout(ctx.CreateLayout())
, bg_color{0.1, 0.1, 0.1}
, text_color{1, 1, 1}
return size.h;
}
- void Update() {
- text_layout.Update();
- channel_layout.Update();
+ void Update(cairo::Context &ctx) {
+ ctx.UpdateLayout(text_layout);
+ ctx.UpdateLayout(channel_layout);
text_offset = padding.Offset();
channel_offset = text_offset;
channel_offset.y += text_layout.GetLogicalRect().height + 10.0;
size.h = channel_offset.y + channel_layout.GetLogicalRect().height + padding.bottom;
}
- void Render() {
+ void Render(cairo::Context &ctx) const {
ctx.SetSourceColor(bg_color);
ctx.Rectangle(pos, size);
ctx.Fill();
ctx.MoveTo(pos + text_offset);
ctx.SetSourceColor(text_color);
- text_layout.Render();
+ ctx.DrawLayout(text_layout);
ctx.MoveTo(pos + channel_offset);
ctx.SetSourceColor(channel_color);
- channel_layout.Render();
+ ctx.DrawLayout(channel_layout);
}
private:
- cairo::Context ctx;
pango::Layout text_layout;
pango::Layout channel_layout;
gfx::ColorRGB bg_color;
#ifndef TEST_APP_MIXER_H_
#define TEST_APP_MIXER_H_
-#include <cstdint>
-
#include "Clock.h"
+#include "State.h"
namespace app {
class Mixer {
public:
- Mixer(const Clock *clock, int16_t *plane, int channels, int frame_size)
- : clock(clock), plane(plane), channels(channels), frame_size(frame_size) {
+ Mixer(float *plane, int channels, int frame_size)
+ : plane(plane), channels(channels), frame_size(frame_size) {
}
~Mixer() {
}
Mixer &operator =(const Mixer &) = delete;
public:
- void RenderAudioFrame() {
+ void RenderAudioFrame(const State &state, const Clock &clock) {
for (int i = 0; i < frame_size; ++i) {
for (int j = 0; j < channels; ++j) {
plane[i * channels + j] = 0;
}
}
+ for (const Media &media : state.GetMedia()) {
+ media.Mix(clock, plane, channels, frame_size);
+ }
}
private:
- const Clock *clock;
- int16_t *plane;
+ float *plane;
int channels;
int frame_size;
#define TEST_APP_RENDERER_H_
#include <cstdint>
-#include <list>
extern "C" {
#include "cairo.h"
}
-#include "Clock.h"
#include "Message.h"
+#include "State.h"
#include "../cairo/Context.h"
#include "../cairo/Surface.h"
class Renderer {
public:
- Renderer(const Clock *clock, uint8_t *plane, int linesize, int width, int height)
- : clock(clock)
- , text_font("DejaVu Sans 32px")
+ Renderer(uint8_t *plane, int linesize, int width, int height)
+ : text_font("DejaVu Sans 32px")
, channel_font("DejaVu Sans 24px")
, surface(plane, linesize, CAIRO_FORMAT_ARGB32, width, height)
, ctx(surface.CreateContext())
, width(width)
, height(height) {
- PushMessage("Hello, I am a long text that should wrap eventually when it gets long enough to cross the halfway point of the total width available (not including the offset which is added afterwards).", "The Dummy Channel");
}
~Renderer() {
}
Renderer &operator =(const Renderer &) = delete;
public:
- void Update() {
- gfx::Position pos({ 50, 50 });
- for (Message &msg : msgs) {
- double distance = msg.GetHeight() + 10.0;
- Clock lifetime = clock->Difference(msg.GetBirth());
- pos.y -= lifetime.InterpolateClamp(distance + 50, 0.0, 0, 600);
- msg.SetPosition(pos);
- pos.y = std::max(pos.y + distance, 50.0);
- }
- if (msgs.size() > 1 && msgs.back().GetPosition().y > height) {
- msgs.pop_back();
- }
- }
-
- void RenderVideoFrame() {
+ void RenderVideoFrame(const State &state) {
ctx.SetSourceRGB(0, 0, 0);
ctx.Paint();
- for (Message &msg : msgs) {
- msg.Render();
+ for (const Media &media : state.GetMedia()) {
+ media.Render(ctx);
+ }
+
+ for (const Message &msg : state.GetMessages()) {
+ msg.Render(ctx);
}
surface.Flush();
}
- void PushMessage(const std::string &text, const std::string &channel) {
- msgs.emplace_front(ctx);
- Message &msg = msgs.front();
+ Message &CreateMessage(State &state) {
+ Message &msg = state.AddMessage(ctx);
msg.SetTextFont(text_font);
msg.SetChannelFont(channel_font);
- msg.SetWidth(width / 2.0);
- msg.SetText(text);
- msg.SetChannel(channel);
- msg.SetBirth(clock->Snapshot());
- msg.Update();
+ return msg;
+ }
+
+ cairo::Context &GetContext() {
+ return ctx;
}
private:
- const Clock *clock;
pango::Font text_font;
pango::Font channel_font;
cairo::Surface surface;
int width;
int height;
- std::list<Message> msgs;
-
};
}
--- /dev/null
+#ifndef TEST_APP_SOURCE_H_
+#define TEST_APP_SOURCE_H_
+
+#include <algorithm>
+#include <cairo.h>
+#include <cmath>
+#include <iostream>
+#include <list>
+extern "C" {
+#include <libavcodec/codec_id.h>
+#include <libavformat/avformat.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/pixfmt.h>
+#include <libavutil/samplefmt.h>
+}
+
+#include "AudioFrameSnapshot.h"
+#include "Clock.h"
+#include "../cairo/Surface.h"
+#include "../ffmpeg/Decoder.h"
+#include "../ffmpeg/Encoder.h"
+#include "../ffmpeg/Frame.h"
+#include "../ffmpeg/InputContext.h"
+#include "../ffmpeg/Packet.h"
+#include "../ffmpeg/Resampler.h"
+#include "../ffmpeg/Scaler.h"
+#include "../ffmpeg/Stream.h"
+
+namespace app {
+
+class Source {
+
+public:
+ explicit Source(const char *url)
+ : input(url)
+ , audio_stream(input.FindAudioStream())
+ , video_stream(input.FindVideoStream())
+ , audio_decoder(audio_stream.GetCodecId())
+ , video_decoder(video_stream.GetCodecId())
+ , audio_encoder(AV_CODEC_ID_PCM_F32LE)
+ , seen_audio(false)
+ , seen_video(false) {
+ audio_decoder.ReadParameters(audio_stream.GetParameters());
+ audio_decoder.SetTimeBase(audio_stream.GetTimeBase());
+ audio_decoder.Open();
+ video_decoder.ReadParameters(video_stream.GetParameters());
+ video_decoder.SetTimeBase(video_stream.GetTimeBase());
+ video_decoder.Open();
+ audio_encoder.SetDefaultChannelLayout(2);
+ audio_encoder.SetSampleRate(48000);
+ audio_encoder.SetSampleFormat(AV_SAMPLE_FMT_FLT);
+ audio_encoder.Open();
+ resampler.SetOpt("in_channel_count", audio_decoder.GetChannelLayout().nb_channels);
+ resampler.SetOpt("in_sample_rate", audio_decoder.GetSampleRate());
+ resampler.SetOpt("in_sample_fmt", audio_decoder.GetSampleFormat());
+ resampler.SetOpt("out_channel_count", audio_encoder.GetChannelLayout().nb_channels);
+ resampler.SetOpt("out_sample_rate", audio_encoder.GetSampleRate());
+ resampler.SetOpt("out_sample_fmt", audio_encoder.GetSampleFormat());
+ resampler.Init();
+ scaler.SetOpt("srcw", video_decoder.GetWidth());
+ scaler.SetOpt("srch", video_decoder.GetHeight());
+ scaler.SetOpt("src_format", video_decoder.GetPixelFormat());
+ scaler.SetOpt("dstw", video_decoder.GetWidth());
+ scaler.SetOpt("dsth", video_decoder.GetHeight());
+ scaler.SetOpt("dst_format", AV_PIX_FMT_BGRA);
+ scaler.Init();
+ if (audio_encoder.GetFrameSize() > 0) {
+ audio_output_frame.AllocateAudio(audio_encoder.GetFrameSize(), audio_encoder.GetSampleFormat(), audio_encoder.GetChannelLayout());
+ } else {
+ audio_output_frame.AllocateAudio(audio_decoder.GetFrameSize(), audio_encoder.GetSampleFormat(), audio_encoder.GetChannelLayout());
+ }
+ video_output_frame.AllocateImage(video_decoder.GetWidth(), video_decoder.GetHeight(), AV_PIX_FMT_BGRA);
+ audio_clock = Clock(audio_encoder.GetTimeBase());
+ video_clock = Clock(video_stream.GetTimeBase());
+ }
+ ~Source() {
+ }
+
+ Source(const Source &) = delete;
+ Source &operator =(const Source &) = delete;
+
+public:
+ void SeekAudio(const Clock &target, int frame_size) {
+ while (audio_clock.GetCounter() + audio_encoder.GetFrameSize() < target.GetCounter() + frame_size && !audio_decoder.IsEOF()) {
+ if (!ReceiveAudio()) {
+ PullPacket();
+ }
+ }
+ while (!audio_buffer.empty() && audio_buffer.front().GetEndTime().GetCounter() < target.GetCounter()) {
+ audio_buffer.pop_front();
+ }
+ }
+
+ void SeekVideo(const Clock &target) {
+ while (video_clock.GetMS() < target.GetMS() && !video_decoder.IsEOF()) {
+ if (!ReceiveVideo()) {
+ PullPacket();
+ }
+ }
+ }
+
+ bool HasSeenAudio() const {
+ return seen_audio;
+ }
+
+ bool HasSeenVideo() const {
+ return seen_video;
+ }
+
+ bool IsEOF() const {
+ return audio_decoder.IsEOF() && video_decoder.IsEOF();
+ }
+
+ const AudioFrameSnapshot &CurrentAudioFrame() const {
+ return audio_buffer.front();
+ }
+
+ void DropAudioFrame() {
+ audio_buffer.pop_front();
+ }
+
+ cairo::Surface GetVideoSurface() {
+ return cairo::Surface(
+ video_output_frame.GetDataPlane(0), video_output_frame.GetPlaneLinesize(0), CAIRO_FORMAT_ARGB32,
+ video_decoder.GetWidth(), video_decoder.GetHeight()
+ );
+ }
+
+ void Mix(const Clock &clock, float *plane, int channels, int frame_size) const {
+ int64_t out_begin = clock.GetCounter();
+ int64_t out_end = out_begin + frame_size;
+ int written = 0;
+ for (const AudioFrameSnapshot &frame : audio_buffer) {
+ int64_t frame_begin = frame.GetStartTime().GetCounter();
+ int64_t frame_end = frame_begin + frame.GetSize();
+ if (frame_begin >= out_end) continue;
+ if (frame_end < out_begin) continue;
+ int64_t src_offset = std::max(int64_t(0), out_begin - frame_begin);
+ int64_t dst_offset = std::max(int64_t(0), frame_begin - out_begin);
+ int64_t start = std::max(out_begin, frame_begin);
+ int64_t end = std::min(out_end, frame_end);
+ int64_t size = end - start;
+ int chans = std::min(channels, frame.GetChannels());
+ for (int64_t sample = 0; sample < size; ++sample) {
+ for (int channel = 0; channel < chans; ++channel) {
+ plane[(sample + dst_offset) * channels + channel] += frame.GetSample(sample + src_offset, channel);
+ }
+ }
+ written += size;
+ }
+ }
+
+private:
+ void PullPacket() {
+ if (!input.ReadPacket(packet)) {
+ // EOF
+ audio_decoder.Flush();
+ while (ReceiveAudio()) {
+ }
+ video_decoder.Flush();
+ while (ReceiveVideo()) {
+ }
+ return;
+ }
+ if (packet.GetStreamIndex() == audio_stream.GetIndex()) {
+ audio_decoder.SendPacket(packet);
+ while (ReceiveAudio()) {
+ }
+ } else if (packet.GetStreamIndex() == video_stream.GetIndex()) {
+ video_decoder.SendPacket(packet);
+ while (ReceiveVideo()) {
+ }
+ }
+ packet.Unref();
+ }
+
+ bool ReceiveAudio() {
+ bool res = audio_decoder.ReceiveFrame(audio_input_frame);
+ if (res) {
+ seen_audio = true;
+ Clock in_clock(audio_decoder.GetTimeBase());
+ in_clock.Set(audio_input_frame.GetBestEffortTimestamp());
+ int converted = resampler.Convert(audio_encoder, audio_input_frame, audio_output_frame);
+ // this may need time scaling?
+ BufferAudio(converted);
+ audio_clock.Advance(converted);
+ }
+ return res;
+ }
+
+ bool ReceiveVideo() {
+ bool res = video_decoder.ReceiveFrame(video_input_frame);
+ if (res) {
+ seen_video = true;
+ scaler.ScaleFrame(video_input_frame, video_output_frame);
+ video_clock.Set(video_input_frame.GetPacketTimestamp());
+ }
+ return res;
+ }
+
+ void BufferAudio(int size) {
+ const float *plane = reinterpret_cast<float *>(audio_output_frame.GetDataPlane(0));
+ int channels = audio_encoder.GetChannelLayout().nb_channels;
+ Clock time = audio_clock.Snapshot();
+ audio_buffer.emplace_back(plane, channels, size, time);
+ }
+
+private:
+ ffmpeg::InputContext input;
+ ffmpeg::Stream audio_stream;
+ ffmpeg::Stream video_stream;
+ ffmpeg::Decoder audio_decoder;
+ ffmpeg::Decoder video_decoder;
+ ffmpeg::Frame audio_input_frame;
+ ffmpeg::Frame video_input_frame;
+ ffmpeg::Encoder audio_encoder;
+ ffmpeg::Packet packet;
+ ffmpeg::Scaler scaler;
+ ffmpeg::Resampler resampler;
+ ffmpeg::Frame audio_output_frame;
+ ffmpeg::Frame video_output_frame;
+ std::list<AudioFrameSnapshot> audio_buffer;
+ Clock audio_clock;
+ Clock video_clock;
+ bool seen_audio;
+ bool seen_video;
+
+};
+
+}
+
+#endif
--- /dev/null
+#ifndef TEST_APP_STATE_H_
+#define TEST_APP_STATE_H_
+
+#include <list>
+#include <ostream>
+
+#include "Clock.h"
+#include "Media.h"
+#include "Message.h"
+#include "../cairo/Context.h"
+#include "../gfx/Position.h"
+
+namespace app {
+
+class State {
+
+public:
+ State(int width, int height)
+ : width(width), height(height) {
+ }
+
+public:
+ const std::list<Media> &GetMedia() const {
+ return media;
+ }
+
+ const std::list<Message> &GetMessages() const {
+ return msgs;
+ }
+
+ Media &AddMedia(const char *url) {
+ std::cout << "adding media " << url << std::endl;
+ media.emplace_back(url);
+ return media.back();
+ }
+
+ Message &AddMessage(cairo::Context &ctx) {
+ msgs.emplace_front(ctx);
+ return msgs.front();
+ }
+
+ int GetWidth() const {
+ return width;
+ }
+
+ int GetHeight() const {
+ return height;
+ }
+
+ void PullAudio(const Clock &clock, int frame_size) {
+ for (Media &m : media) {
+ m.PullAudio(clock, frame_size);
+ }
+ }
+
+ void PullVideo(const Clock &clock) {
+ for (Media &m : media) {
+ m.PullVideo(clock);
+ }
+ }
+
+ void Update(const Clock &clock) {
+ gfx::Position pos({ 50, 50 });
+ for (Message &msg : msgs) {
+ double distance = msg.GetHeight() + 10.0;
+ Clock lifetime = clock.Difference(msg.GetBirth());
+ pos.y -= lifetime.InterpolateClamp(distance + 50, 0.0, 0, 600);
+ msg.SetPosition(pos);
+ pos.y = std::max(pos.y + distance, 50.0);
+ }
+ }
+
+ void Clean() {
+ for (auto m = media.begin(); m != media.end();) {
+ if (m->IsEOF()) {
+ std::cout << "removing EOF media" << std::endl;
+ m = media.erase(m);
+ } else {
+ ++m;
+ }
+ }
+ if (msgs.size() > 1 && msgs.back().GetPosition().y > height) {
+ msgs.pop_back();
+ }
+ }
+
+private:
+ int width;
+ int height;
+
+ std::list<Media> media;
+ std::list<Message> msgs;
+
+};
+
+}
+
+#endif
#include "Clock.h"
#include "../ffmpeg/Encoder.h"
-#include "../ffmpeg/Network.h"
#include "../ffmpeg/OutputContext.h"
#include "../ffmpeg/Resampler.h"
#include "../ffmpeg/Scaler.h"
, audio_encoder(AV_CODEC_ID_AAC)
, video_encoder(AV_CODEC_ID_H264)
, scaler(width, height, AV_PIX_FMT_BGRA, width, height, video_encoder.GetPreferredPixelFormat())
- , resampler(2, 44100, AV_SAMPLE_FMT_S16, 2, 44100, audio_encoder.GetPreferredSampleFormat()) {
+ , resampler(2, 48000, AV_SAMPLE_FMT_FLT, 2, 48000, audio_encoder.GetPreferredSampleFormat()) {
audio_encoder.SetDefaultChannelLayout(2);
- audio_encoder.SetSampleRate(44100);
+ audio_encoder.SetSampleRate(48000);
audio_encoder.InferSampleFormat();
audio_encoder.SetBitRate(160 * 1000);
audio_encoder.Open();
video_input_frame.AllocateImage(width, height, AV_PIX_FMT_BGRA);
video_encoder.AllocateVideoFrame(video_output_frame);
- audio_input_frame.AllocateAudio(audio_encoder.GetFrameSize(), AV_SAMPLE_FMT_S16, audio_encoder.GetChannelLayout());
+ audio_input_frame.AllocateAudio(audio_encoder.GetFrameSize(), AV_SAMPLE_FMT_FLT, audio_encoder.GetChannelLayout());
audio_encoder.AllocateAudioFrame(audio_output_frame);
}
return video_input_frame.GetPlaneLinesize(0);
}
- int16_t *GetAudioPlane() {
- return reinterpret_cast<int16_t *>(audio_input_frame.GetDataPlane(0));
+ float *GetAudioPlane() {
+ return reinterpret_cast<float *>(audio_input_frame.GetDataPlane(0));
}
int GetAudioChannels() const {
}
private:
- ffmpeg::Network net;
ffmpeg::OutputContext output;
ffmpeg::Encoder audio_encoder;
ffmpeg::Encoder video_encoder;
--- /dev/null
+#ifndef TEST_APP_VIDEOFRAMESNAPSHOT_H_
+#define TEST_APP_VIDEOFRAMESNAPSHOT_H_
+
+#include "Clock.h"
+#include "../cairo/Surface.h"
+
+namespace app {
+
+class VideoFrameSnapshot {
+
+public:
+ VideoFrameSnapshot(const cairo::Surface &frame, const Clock &time)
+ : frame(frame), time(time) {
+ }
+ ~VideoFrameSnapshot() {
+ }
+
+private:
+ cairo::Surface frame;
+ Clock time;
+
+};
+
+}
+
+#endif
--- /dev/null
+#ifndef TEST_APP_WINDOW_H_
+#define TEST_APP_WINDOW_H_
+
+#include "../gfx/Rectangle.h"
+
+namespace app {
+
+struct Window {
+
+ gfx::Rectangle src;
+ gfx::Rectangle dst;
+
+};
+
+}
+
+#endif
--- /dev/null
+#include "Context.h"
+
+#include "Surface.h"
+
+namespace cairo {
+
+void Context::SetSourceSurface(const Surface &src, double x, double y) {
+ src.SetSource(ctx, x, y);
+}
+
+}
namespace cairo {
+class Surface;
using TextExtends = cairo_text_extents_t;
class Context {
return pango::Layout(ctx);
}
+ void DrawLayout(const pango::Layout &l) {
+ l.Render(ctx);
+ }
+
+ void UpdateLayout(pango::Layout &l) {
+ l.Update(ctx);
+ }
+
+ void DrawSurface(const Surface &src, const gfx::Position &src_offset, const gfx::Rectangle &dst_rect) {
+ SetSourceSurface(src, dst_rect.x - src_offset.x, dst_rect.y - src_offset.y);
+ Rectangle(dst_rect);
+ Fill();
+ }
+
+ void DrawSurface(const Surface &src, const gfx::Rectangle &src_rect, const gfx::Rectangle &dst_rect) {
+ double sx = dst_rect.w / src_rect.w;
+ double sy = dst_rect.h / src_rect.h;
+ Rectangle(dst_rect);
+ Save();
+ Scale(sx, sy);
+ SetSourceSurface(src, (dst_rect.x - src_rect.x) / sx, (dst_rect.y - src_rect.y) / sy);
+ Fill();
+ Restore();
+ }
+
+
void DebugPrint() {
cairo_status_t status = cairo_status(ctx);
std::cout << "cairo status: " << cairo_status_to_string(status) << std::endl;
cairo_rectangle(ctx, x, y, w, h);
}
+ void Scale(double sx, double sy) {
+ cairo_scale(ctx, sx, sy);
+ }
+
void SelectFontFace(const char *family, cairo_font_slant_t slant, cairo_font_weight_t weight) {
cairo_select_font_face(ctx, family, slant, weight);
}
cairo_set_source_rgba(ctx, r, g, b, a);
}
+ void SetSourceSurface(const Surface &srf, const gfx::Position &offset) {
+ SetSourceSurface(srf, offset.x, offset.y);
+ }
+
+ void SetSourceSurface(const Surface &, double x, double y);
+
void ShowText(const char *text) {
cairo_show_text(ctx, text);
}
cairo_stroke(ctx);
}
+ void Save() {
+ cairo_save(ctx);
+ }
+
+ void Restore() {
+ cairo_restore(ctx);
+ }
+
private:
cairo_t *ctx;
--- /dev/null
+#ifndef TEST_CAIRO_PATTERN_H_
+#define TEST_CAIRO_PATTERN_H_
+
+#include <stdexcept>
+#include <utility>
+
+#include <cairo.h>
+
+namespace cairo {
+
+class Pattern {
+
+public:
+ explicit Pattern(cairo_surface_t *s)
+ : p(cairo_pattern_create_for_surface(s)) {
+ if (!s) {
+ throw std::runtime_error("create pattern from NULLL surface");
+ }
+ if (!p) {
+ throw std::runtime_error("failed to allocate pattern");
+ }
+ }
+ ~Pattern() {
+ cairo_pattern_destroy(p);
+ }
+ Pattern(const Pattern &other): p(cairo_pattern_reference(other.p)) {
+ }
+ Pattern &operator =(const Pattern &other) {
+ Pattern temp(other);
+ Swap(temp);
+ return *this;
+ }
+ void Swap(Pattern &other) {
+ std::swap(p, other.p);
+ }
+
+private:
+ cairo_pattern_t *p;
+
+};
+
+}
+
+#endif
#include "Context.h"
#include "Error.h"
+#include "Pattern.h"
namespace cairo {
cairo_surface_destroy(s);
}
- Surface(const Surface &) = delete;
- Surface &operator =(const Surface &) = delete;
+ Surface(const Surface &other): s(other.s) {
+ cairo_surface_reference(s);
+ }
+ Surface &operator =(const Surface &other) {
+ Surface temp(other);
+ Swap(temp);
+ return *this;
+ }
+ void Swap(Surface &other) {
+ std::swap(s, other.s);
+ }
public:
Context CreateContext() {
return std::move(Context(s));
}
+ Pattern CreatePattern() {
+ return std::move(Pattern(s));
+ }
+
void Flush() {
cairo_surface_flush(s);
}
return cairo_image_surface_get_data(s);
}
+ void MarkDirty() {
+ cairo_surface_mark_dirty(s);
+ }
+
+ void SetSource(cairo_t *ctx, double x, double y) const {
+ cairo_set_source_surface(ctx, s, x, y);
+ }
+
private:
cairo_surface_t *s;
#include <libavutil/samplefmt.h>
}
+#include "Error.h"
#include "Frame.h"
namespace ffmpeg {
ctx->sample_fmt = codec->sample_fmts[0];
}
+ int GetWidth() const {
+ return ctx->width;
+ }
+
+ int GetHeight() const {
+ return ctx->height;
+ }
+
void Open() {
- if (avcodec_open2(ctx, codec, nullptr) != 0) {
- throw std::runtime_error("failed to open audio codec");
+ int res = avcodec_open2(ctx, codec, nullptr);
+ if (res != 0) {
+ throw Error("failed to open audio codec", res);
}
}
ctx->height = height;
}
+ void SetTimeBase(const AVRational &time_base) {
+ ctx->time_base = time_base;
+ }
+
bool SupportsVariableFrameSize() const {
return codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE;
}
+ void ReadParameters(const AVCodecParameters ¶ms) const {
+ int res = avcodec_parameters_to_context(ctx, ¶ms);
+ if (res != 0) {
+ throw Error("failed to copy codec params", res);
+ }
+ }
+
void WriteParameters(AVCodecParameters ¶ms) const {
int res = avcodec_parameters_from_context(¶ms, ctx);
if (res != 0) {
}
protected:
- ::AVCodecContext *ctx;
- const ::AVCodec *codec;
+ AVCodecContext *ctx;
+ const AVCodec *codec;
};
--- /dev/null
+#ifndef TEST_FFMPEG_DECODER_H_
+#define TEST_FFMPEG_DECODER_H_
+
+#include <cerrno>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
+#include <libavcodec/codec_id.h>
+#include <libavutil/error.h>
+}
+
+#include "CodecContext.h"
+#include "Error.h"
+#include "Frame.h"
+#include "Packet.h"
+
+namespace ffmpeg {
+
+class Decoder: public CodecContext {
+
+public:
+ Decoder(AVCodecID id)
+ : CodecContext(avcodec_find_decoder(id))
+ , eof(false) {
+ }
+ Decoder(const char *name)
+ : CodecContext(avcodec_find_decoder_by_name(name))
+ , eof(false) {
+ }
+
+public:
+ void SendPacket(const Packet &packet) {
+ int res = avcodec_send_packet(ctx, packet.GetPacket());
+ if (res < 0) {
+ throw Error("failed to send packet", res);
+ }
+ }
+
+ void Flush() {
+ int res = avcodec_send_packet(ctx, nullptr);
+ if (res == AVERROR_EOF) {
+ return;
+ }
+ if (res < 0) {
+ throw Error("failed to flush decoder", res);
+ }
+ }
+
+ bool ReceiveFrame(Frame &frame) {
+ int res = avcodec_receive_frame(ctx, frame.GetFrame());
+ if (res == AVERROR(EAGAIN)) {
+ return false;
+ }
+ if (res == AVERROR_EOF) {
+ eof = true;
+ return false;
+ }
+ if (res != 0) {
+ throw Error("failed to receive frame", res);
+ }
+ return true;
+ }
+
+ bool IsEOF() const {
+ return eof;
+ }
+
+private:
+ bool eof;
+
+};
+
+}
+
+#endif
public:
Encoder(AVCodecID id): CodecContext(avcodec_find_encoder(id)) {
-
}
Encoder(const char *name): CodecContext(avcodec_find_encoder_by_name(name)) {
-
}
public:
#ifndef TEST_FFMPEG_FORMATCONTEXT_H_
#define TEST_FFMPEG_FORMATCONTEXT_H_
-#include "CodecContext.h"
#include <stdexcept>
#include <utility>
extern "C" {
#include <libavformat/avformat.h>
+#include <libavutil/avutil.h>
+#include <libavutil/opt.h>
}
+#include "Error.h"
+#include "Stream.h"
+
namespace ffmpeg {
class FormatContext {
av_dump_format(ctx, index, url, is_output);
}
+ Stream FindAudioStream() {
+ int res = av_find_best_stream(ctx, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
+ if (res < 0) {
+ throw Error("audio stream not found", res);
+ }
+ return Stream(ctx->streams[res]);
+ }
+
+ Stream FindVideoStream() {
+ int res = av_find_best_stream(ctx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+ if (res < 0) {
+ throw Error("video stream not found", res);
+ }
+ return Stream(ctx->streams[res]);
+ }
+
void SetOption(const char *name, const char *value) {
int res = av_opt_set(ctx->priv_data, name, value, 0);
if (res != 0) {
public:
void AllocateAudio(int frame_size, AVSampleFormat fmt, const AVChannelLayout &layout) {
if (buffer_allocated) {
- std::cout << "freeing data" << std::endl;
av_freep(frame->data);
buffer_allocated = false;
}
return frame->linesize[num];
}
+ int64_t GetBestEffortTimestamp() const {
+ return frame->best_effort_timestamp;
+ }
+
+ int64_t GetPacketTimestamp() const {
+ return frame->pkt_dts;
+ }
+
int64_t GetPresentationTimestamp() const {
return frame->pts;
}
#ifndef TEST_FFMPEG_INPUTCONTEXT_H_
#define TEST_FFMPEG_INPUTCONTEXT_H_
-#include <stdexcept>
-
extern "C" {
#include <libavformat/avformat.h>
+#include <libavutil/error.h>
}
+#include "Error.h"
#include "FormatContext.h"
+#include "Packet.h"
namespace ffmpeg {
class InputContext: public FormatContext {
- public:
- explicit InputContext(const char *url)
- : FormatContext(avformat_alloc_context())
- , url(url) {
- if (avformat_open_input(&ctx, url, nullptr, nullptr) != 0) {
- avformat_free_context(ctx);
- throw std::runtime_error("failed to open input file");
- }
+public:
+ explicit InputContext(const char *url)
+ : FormatContext(avformat_alloc_context())
+ , url(url) {
+ int res = avformat_open_input(&ctx, url, nullptr, nullptr);
+ if (res < 0) {
+ throw Error("failed to open input file", res);
}
- ~InputContext() {
+ res = avformat_find_stream_info(ctx, nullptr);
+ if (res < 0) {
+ throw Error("unable to find stream info", res);
}
+ }
+ ~InputContext() {
+ }
- InputContext(const InputContext &) = delete;
- InputContext &operator =(const InputContext &) = delete;
+ InputContext(const InputContext &) = delete;
+ InputContext &operator =(const InputContext &) = delete;
public:
void Dump(int index) {
FormatContext::Dump(index, url, 0);
}
+ bool ReadPacket(Packet &packet) {
+ int res = av_read_frame(ctx, packet.GetPacket());
+ if (res == AVERROR_EOF) {
+ return false;
+ }
+ if (res < 0) {
+ throw Error("failed to read packet", res);
+ }
+ return true;
+ }
+
private:
const char *url;
#include <libavformat/avio.h>
}
+#include "CodecContext.h"
#include "FormatContext.h"
#include "Packet.h"
#include "Stream.h"
#define TEST_FFMPEG_PACKET_H_
#include <cstdint>
-#include <iostream>
#include <libavutil/avutil.h>
#include <libavutil/mathematics.h>
#include <stdexcept>
#include <libavutil/frame.h>
}
-#include "io.h"
+#include "CodecContext.h"
#include "Stream.h"
namespace ffmpeg {
return packet->stream_index;
}
+ void Unref() {
+ av_packet_unref(packet);
+ }
+
private:
AVPacket *packet;
#include "CodecContext.h"
#include <cstdint>
-#include <libavutil/opt.h>
-#include <libavutil/samplefmt.h>
#include <stdexcept>
extern "C" {
#include <libavutil/mathematics.h>
+#include <libavutil/opt.h>
+#include <libavutil/samplefmt.h>
#include <libswresample/swresample.h>
}
Resampler &operator =(const Resampler &) = delete;
public:
- void Convert(const CodecContext &codec, const Frame &src, Frame &dst) {
+ int Convert(const CodecContext &codec, const Frame &src, Frame &dst) {
int64_t from = swr_get_delay(ctx, codec.GetSampleRate()) + src.GetSamples();
int64_t nb_samples = av_rescale_rnd(from, codec.GetSampleRate(), codec.GetSampleRate(), AV_ROUND_UP);
int res = swr_convert(ctx, dst.GetData(), nb_samples, src.GetData(), src.GetSamples());
if (res < 0) {
throw Error("failed to resample", res);
}
+ return res;
}
void Init() {
#include <stdexcept>
extern "C" {
+#include <libavutil/opt.h>
#include <libavutil/pixfmt.h>
#include <libswscale/swscale.h>
}
}
}
+ void SetOpt(const char *name, int value) {
+ int res = av_opt_set_int(ctx, name, value, 0);
+ if (res != 0) {
+ throw Error("failed to set option", res);
+ }
+ }
+
+ void Init() {
+ int res = sws_init_context(ctx, nullptr, nullptr);
+ if (res < 0) {
+ throw Error("failed to init context", res);
+ }
+ }
+
private:
SwsContext *ctx;
#ifndef TEST_FFMPEG_STREAM_H_
#define TEST_FFMPEG_STREAM_H_
+#include <libavcodec/codec_par.h>
#include <stdexcept>
extern "C" {
}
public:
+ AVCodecID GetCodecId() const {
+ return s->codecpar->codec_id;
+ }
+
int GetIndex() const {
return s->index;
}
+ const AVCodecParameters &GetParameters() const {
+ return *s->codecpar;
+ }
+
AVRational GetTimeBase() const {
return s->time_base;
}
-#include <chrono>
#include <csignal>
-#include <cstdint>
#include <iostream>
-#include <thread>
-#include <json/json.h>
-
-extern "C" {
-#include <libavcodec/codec_id.h>
-#include <libavcodec/codec_par.h>
-#include <libavformat/avformat.h>
-#include <libavutil/buffer.h>
-#include <libavutil/mathematics.h>
-#include <libavutil/pixfmt.h>
-#include <libavutil/rational.h>
-#include <libavutil/samplefmt.h>
-#include <libavutil/timestamp.h>
-}
-
-#include "app/Mixer.h"
-#include "app/Renderer.h"
-#include "app/Stream.h"
-#include "uv/Loop.h"
-#include "ws/Connection.h"
-#include "ws/Context.h"
+#include "app/Application.h"
namespace {
running = false;
}
-void ws_handler(void *user, const Json::Value &json) {
- const std::string data_string = json["data"].asString();
- Json::Value data;
- Json::Reader json_reader;
- json_reader.parse(data_string, data);
- app::Renderer *renderer = static_cast<app::Renderer *>(user);
- const std::string text = data["model"]["text"].asString();
- const std::string channel = data["model"]["channel"]["title"].asString();
- if (text.length() > 0) {
- renderer->PushMessage(text, channel);
- }
-}
-
}
const int FPS = 60;
//const char *URL = "rtmp://localhost/horstiebot";
const char *URL = "rtmp://localhost/localhorsttv";
+ //const char *URL = "out.flv";
- uv::Loop loop;
-
- ws::Context wsctx(loop);
- ws::Connection wsconn(wsctx.GetContext());
-
- app::Stream stream(URL, WIDTH, HEIGHT, FPS);
+ app::Application app(WIDTH, HEIGHT, FPS, URL);
- running = true;
signal(SIGINT, stop);
- uint8_t *plane = stream.GetVideoPlane();
- const int linesize = stream.GetVideoLineSize();
-
- int16_t *audio_plane = stream.GetAudioPlane();
- const int audio_channels = stream.GetAudioChannels();
- const int audio_frame_size = stream.GetAudioFrameSize();
-
- app::Renderer renderer(&stream.GetVideoClock(), plane, linesize, WIDTH, HEIGHT);
- app::Mixer mixer(&stream.GetAudioClock(), audio_plane, audio_channels, audio_frame_size);
-
- wsconn.Subscribe("ChatBotLog", &ws_handler, &renderer);
-
- stream.Start();
+ app.Start();
+ running = true;
std::cout << std::endl;
while (running) {
- loop.TryStep();
-
- const int64_t target = stream.GetVideoClock().GetMS();
- const int64_t elapsed = stream.TimeElapsedMS();
- const int64_t difference = target - elapsed;
-
- stream.PrepareVideoFrame();
-
- if (target > 0 && difference < 0) {
- std::cout << (difference / -1000.0) << "s behind schedule, dropping frame" << std::endl;
- } else {
- renderer.Update();
- renderer.RenderVideoFrame();
- }
-
- stream.PushVideoFrame();
-
- while (stream.GetAudioClock().GetMS() < target) {
- stream.PrepareAudioFrame();
- mixer.RenderAudioFrame();
- stream.PushAudioFrame();
- }
-
- //if (stream.GetVideoFrameCounter() % 60 == 59) {
- // std::cout << "rendered: " << (target / 1000.0) << "s, elapsed: " << (elapsed / 1000.0) << "s, difference: " << (difference / 1000.0) << 's' << std::endl;
- //}
- if (difference > 3000) {
- std::this_thread::sleep_for(std::chrono::milliseconds(10));
- }
+ app.Step();
}
std::cout << std::endl;
- wsctx.Shutdown();
- stream.Finish();
+ app.Stop();
return 0;
}
class Layout {
public:
- explicit Layout(cairo_t *c): c(c), l(pango_cairo_create_layout(c)) {
+ explicit Layout(cairo_t *c): l(pango_cairo_create_layout(c)) {
if (!l) {
throw std::runtime_error("failed to create layout");
}
cairo_reference(c);
}
~Layout() {
- cairo_destroy(c);
g_object_unref(l);
}
- Layout(const Layout &other): c(other.c), l(other.l) {
- cairo_reference(c);
+ Layout(const Layout &other)
+ : l(other.l), ink_rect(other.ink_rect), logical_rect(other.logical_rect) {
g_object_ref(l);
}
Layout &operator =(const Layout &other) {
return *this;
}
void Swap(Layout &other) {
- std::swap(c, other.c);
std::swap(l, other.l);
+ std::swap(ink_rect, other.ink_rect);
+ std::swap(logical_rect, other.logical_rect);
}
public:
return logical_rect;
}
- void Render() {
+ void Render(cairo_t *c) const {
pango_cairo_show_layout(c, l);
}
pango_layout_set_width(l, w * 1024);
}
- void Update() {
+ void Update(cairo_t *c) {
pango_cairo_update_layout(c, l);
pango_layout_get_pixel_extents(l, &ink_rect, &logical_rect);
}
private:
- cairo_t *c;
PangoLayout *l;
PangoRectangle ink_rect;
PangoRectangle logical_rect;
projects / ffmpeg-test.git / commitdiff