OpenShot Library | libopenshot  0.5.0
FFmpegWriter.cpp
Go to the documentation of this file.
1 
12 // Copyright (c) 2008-2025 OpenShot Studios, LLC, Fabrice Bellard
13 //
14 // SPDX-License-Identifier: LGPL-3.0-or-later
15 
16 #include <algorithm>
17 #include <iostream>
18 #include <cmath>
19 #include <ctime>
20 #include <sstream>
21 #include <unistd.h>
22 
23 #include "FFmpegUtilities.h"
24 
25 #include "FFmpegWriter.h"
26 #include "Exceptions.h"
27 #include "Frame.h"
28 #include "OpenMPUtilities.h"
29 #include "Settings.h"
30 #include "ZmqLogger.h"
31 
32 using namespace openshot;
33 
34 // Multiplexer parameters temporary storage
35 AVDictionary *mux_dict = NULL;
36 
37 #if USE_HW_ACCEL
38 int hw_en_on = 1; // Is set in UI
39 int hw_en_supported = 0; // Is set by FFmpegWriter
40 AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
41 AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
42 static AVBufferRef *hw_device_ctx = NULL;
43 AVFrame *hw_frame = NULL;
44 
45 static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height)
46 {
47  AVBufferRef *hw_frames_ref;
48  AVHWFramesContext *frames_ctx = NULL;
49  int err = 0;
50 
51  if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) {
52  std::clog << "Failed to create HW frame context.\n";
53  return -1;
54  }
55  frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
56  frames_ctx->format = hw_en_av_pix_fmt;
57  frames_ctx->sw_format = AV_PIX_FMT_NV12;
58  frames_ctx->width = width;
59  frames_ctx->height = height;
60  frames_ctx->initial_pool_size = 20;
61  if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) {
62  std::clog << "Failed to initialize HW frame context. " <<
63  "Error code: " << av_err2string(err) << "\n";
64  av_buffer_unref(&hw_frames_ref);
65  return err;
66  }
67  ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref);
68  if (!ctx->hw_frames_ctx)
69  err = AVERROR(ENOMEM);
70 
71  av_buffer_unref(&hw_frames_ref);
72  return err;
73 }
74 #endif // USE_HW_ACCEL
75 
76 FFmpegWriter::FFmpegWriter(const std::string& path) :
77  path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL),
78  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
79  initial_audio_input_frame_size(0), img_convert_ctx(NULL),
80  video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0),
81  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
82  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) {
83 
84  // Disable audio & video (so they can be independently enabled)
85  info.has_audio = false;
86  info.has_video = false;
87 
88  // Initialize FFMpeg, and register all formats and codecs
89  AV_REGISTER_ALL
90 
91  // auto detect format
92  auto_detect_format();
93 }
94 
95 // Open the writer
96 void FFmpegWriter::Open() {
97  if (!is_open) {
98  // Open the writer
99  is_open = true;
100 
101  // Prepare streams (if needed)
102  if (!prepare_streams)
103  PrepareStreams();
104 
105  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
106  if (info.has_video && video_st)
107  open_video(oc, video_st);
108  if (info.has_audio && audio_st)
109  open_audio(oc, audio_st);
110 
111  // Write header (if needed)
112  if (!write_header)
113  WriteHeader();
114  }
115 }
116 
117 // auto detect format (from path)
118 void FFmpegWriter::auto_detect_format() {
119 
120  // Allocate the output media context
121  AV_OUTPUT_CONTEXT(&oc, path.c_str());
122  if (!oc) {
123  throw OutOfMemory(
124  "Could not allocate memory for AVFormatContext.", path);
125  }
126 
127  // Determine what format to use when encoding this output filename
128  oc->oformat = av_guess_format(NULL, path.c_str(), NULL);
129  if (oc->oformat == nullptr) {
130  throw InvalidFormat("Could not deduce output format from file extension.", path);
131  }
132 
133  // Update video & audio codec name
134  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video) {
135  const AVCodec *vcodec = avcodec_find_encoder(oc->oformat->video_codec);
136  info.vcodec = vcodec ? vcodec->name : std::string();
137  }
138  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio) {
139  const AVCodec *acodec = avcodec_find_encoder(oc->oformat->audio_codec);
140  info.acodec = acodec ? acodec->name : std::string();
141  }
142 }
143 
144 // initialize streams
145 void FFmpegWriter::initialize_streams() {
146  ZmqLogger::Instance()->AppendDebugMethod(
147  "FFmpegWriter::initialize_streams",
148  "oc->oformat->video_codec", oc->oformat->video_codec,
149  "oc->oformat->audio_codec", oc->oformat->audio_codec,
150  "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE);
151 
152  // Add the audio and video streams using the default format codecs and initialize the codecs
153  video_st = NULL;
154  audio_st = NULL;
155  if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video)
156  // Add video stream
157  video_st = add_video_stream();
158 
159  if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
160  // Add audio stream
161  audio_st = add_audio_stream();
162 }
163 
164 // Set video export options
165 void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) {
166  // Set the video options
167  if (codec.length() > 0) {
168  const AVCodec *new_codec;
169  // Check if the codec selected is a hardware accelerated codec
170 #if USE_HW_ACCEL
171 #if defined(__linux__)
172  if (strstr(codec.c_str(), "_vaapi") != NULL) {
173  new_codec = avcodec_find_encoder_by_name(codec.c_str());
174  hw_en_on = 1;
175  hw_en_supported = 1;
176  hw_en_av_pix_fmt = AV_PIX_FMT_VAAPI;
177  hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
178  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
179  new_codec = avcodec_find_encoder_by_name(codec.c_str());
180  hw_en_on = 1;
181  hw_en_supported = 1;
182  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
183  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
184  } else {
185  new_codec = avcodec_find_encoder_by_name(codec.c_str());
186  hw_en_on = 0;
187  hw_en_supported = 0;
188  }
189 #elif defined(_WIN32)
190  if (strstr(codec.c_str(), "_dxva2") != NULL) {
191  new_codec = avcodec_find_encoder_by_name(codec.c_str());
192  hw_en_on = 1;
193  hw_en_supported = 1;
194  hw_en_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD;
195  hw_en_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
196  } else if (strstr(codec.c_str(), "_nvenc") != NULL) {
197  new_codec = avcodec_find_encoder_by_name(codec.c_str());
198  hw_en_on = 1;
199  hw_en_supported = 1;
200  hw_en_av_pix_fmt = AV_PIX_FMT_CUDA;
201  hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA;
202  } else {
203  new_codec = avcodec_find_encoder_by_name(codec.c_str());
204  hw_en_on = 0;
205  hw_en_supported = 0;
206  }
207 #elif defined(__APPLE__)
208  if (strstr(codec.c_str(), "_videotoolbox") != NULL) {
209  new_codec = avcodec_find_encoder_by_name(codec.c_str());
210  hw_en_on = 1;
211  hw_en_supported = 1;
212  hw_en_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
213  hw_en_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
214  } else {
215  new_codec = avcodec_find_encoder_by_name(codec.c_str());
216  hw_en_on = 0;
217  hw_en_supported = 0;
218  }
219 #else // unknown OS
220  new_codec = avcodec_find_encoder_by_name(codec.c_str());
221 #endif //__linux__/_WIN32/__APPLE__
222 #else // USE_HW_ACCEL
223  new_codec = avcodec_find_encoder_by_name(codec.c_str());
224 #endif // USE_HW_ACCEL
225  if (new_codec == NULL)
226  throw InvalidCodec("A valid video codec could not be found for this file.", path);
227  else {
228  // Set video codec
229  info.vcodec = new_codec->name;
230  }
231  }
232  if (fps.num > 0) {
233  // Set frames per second (if provided)
234  info.fps.num = fps.num;
235  info.fps.den = fps.den;
236 
237  // Set the timebase (inverse of fps)
238  info.video_timebase.num = info.fps.den;
239  info.video_timebase.den = info.fps.num;
240  }
241  if (width >= 1)
242  info.width = width;
243  if (height >= 1)
244  info.height = height;
245  if (pixel_ratio.num > 0) {
246  info.pixel_ratio.num = pixel_ratio.num;
247  info.pixel_ratio.den = pixel_ratio.den;
248  }
249  if (bit_rate >= 1000) // bit_rate is the bitrate in b/s
250  info.video_bit_rate = bit_rate;
251  if ((bit_rate >= 0) && (bit_rate < 256)) // bit_rate is the bitrate in crf
252  info.video_bit_rate = bit_rate;
253 
254  info.interlaced_frame = interlaced;
255  info.top_field_first = top_field_first;
256 
257  // Calculate the DAR (display aspect ratio)
258  Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den);
259 
260  // Reduce size fraction
261  size.Reduce();
262 
263  // Set the ratio based on the reduced fraction
264  info.display_ratio.num = size.num;
265  info.display_ratio.den = size.den;
266 
267  ZmqLogger::Instance()->AppendDebugMethod(
268  "FFmpegWriter::SetVideoOptions (" + codec + ")",
269  "width", width, "height", height,
270  "size.num", size.num, "size.den", size.den,
271  "fps.num", fps.num, "fps.den", fps.den);
272 
273  // Enable / Disable video
274  info.has_video = has_video;
275 }
276 
277 // Set video export options (overloaded function)
278 void FFmpegWriter::SetVideoOptions(std::string codec, int width, int height, Fraction fps, int bit_rate) {
279  // Call full signature with some default parameters
280  FFmpegWriter::SetVideoOptions(
281  true, codec, fps, width, height,
282  openshot::Fraction(1, 1), false, true, bit_rate
283  );
284 }
285 
286 
287 // Set audio export options
288 void FFmpegWriter::SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) {
289  // Set audio options
290  if (codec.length() > 0) {
291  const AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
292  if (new_codec == NULL)
293  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
294  else {
295  // Set audio codec
296  info.acodec = new_codec->name;
297  }
298  }
299  if (sample_rate > 7999)
300  info.sample_rate = sample_rate;
301  if (channels > 0)
302  info.channels = channels;
303  if (bit_rate > 999)
304  info.audio_bit_rate = bit_rate;
305  info.channel_layout = channel_layout;
306 
307  // init resample options (if zero)
308  if (original_sample_rate == 0)
309  original_sample_rate = info.sample_rate;
310  if (original_channels == 0)
311  original_channels = info.channels;
312 
313  ZmqLogger::Instance()->AppendDebugMethod(
314  "FFmpegWriter::SetAudioOptions (" + codec + ")",
315  "sample_rate", sample_rate,
316  "channels", channels,
317  "bit_rate", bit_rate);
318 
319  // Enable / Disable audio
320  info.has_audio = has_audio;
321 }
322 
323 
324 // Set audio export options (overloaded function)
325 void FFmpegWriter::SetAudioOptions(std::string codec, int sample_rate, int bit_rate) {
326  // Call full signature with some default parameters
327  FFmpegWriter::SetAudioOptions(
328  true, codec, sample_rate, 2,
329  openshot::LAYOUT_STEREO, bit_rate
330  );
331 }
332 
333 
334 // Set custom options (some codecs accept additional params)
335 void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string value) {
336  // Declare codec context
337  AVCodecContext *c = NULL;
338  AVStream *st = NULL;
339  std::stringstream convert(value);
340 
341  if (info.has_video && stream == VIDEO_STREAM && video_st) {
342  st = video_st;
343  // Get codec context
344  c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec_ctx);
345  // Was a codec / stream found?
346  if (c) {
347  if (info.interlaced_frame) {
348  c->field_order = info.top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
349  // We only use these two version and ignore AV_FIELD_TB and AV_FIELD_BT
350  // Otherwise we would need to change the whole export window
351  }
352  }
353  } else if (info.has_audio && stream == AUDIO_STREAM && audio_st) {
354  st = audio_st;
355  // Get codec context
356  c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec_ctx);
357  } else
358  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
359 
360  // Init AVOption
361  const AVOption *option = NULL;
362 
363  // Was a codec / stream found?
364  if (c)
365  // Find AVOption (if it exists)
366  option = AV_OPTION_FIND(c->priv_data, name.c_str());
367 
368  // Was option found?
369  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
370  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" ||
371  name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp")) {
372  // Check for specific named options
373  if (name == "g")
374  // Set gop_size
375  convert >> c->gop_size;
376 
377  else if (name == "qmin")
378  // Minimum quantizer
379  convert >> c->qmin;
380 
381  else if (name == "qmax")
382  // Maximum quantizer
383  convert >> c->qmax;
384 
385  else if (name == "max_b_frames")
386  // Maximum number of B-frames between non-B-frames
387  convert >> c->max_b_frames;
388 
389  else if (name == "mb_decision")
390  // Macroblock decision mode
391  convert >> c->mb_decision;
392 
393  else if (name == "level")
394  // Set codec level
395  convert >> c->level;
396 
397  else if (name == "profile")
398  // Set codec profile
399  convert >> c->profile;
400 
401  else if (name == "slices")
402  // Indicates number of picture subdivisions
403  convert >> c->slices;
404 
405  else if (name == "rc_min_rate")
406  // Minimum bitrate
407  convert >> c->rc_min_rate;
408 
409  else if (name == "rc_max_rate")
410  // Maximum bitrate
411  convert >> c->rc_max_rate;
412 
413  else if (name == "rc_buffer_size")
414  // Buffer size
415  convert >> c->rc_buffer_size;
416 
417  else if (name == "cqp") {
418  // encode quality and special settings like lossless
419 #if USE_HW_ACCEL
420  if (hw_en_on) {
421  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
422  } else
423 #endif // USE_HW_ACCEL
424  {
425  switch (c->codec_id) {
426 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
427  // FFmpeg 4.0+
428  case AV_CODEC_ID_AV1 :
429  c->bit_rate = 0;
430  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
431  break;
432 #endif
433  case AV_CODEC_ID_VP8 :
434  c->bit_rate = 10000000;
435  av_opt_set_int(c->priv_data, "qp", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
436  break;
437  case AV_CODEC_ID_VP9 :
438  c->bit_rate = 0; // Must be zero!
439  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
440  if (std::stoi(value) == 0) {
441  av_opt_set(c->priv_data, "preset", "veryslow", 0);
442  av_opt_set_int(c->priv_data, "lossless", 1, 0);
443  }
444  break;
445  case AV_CODEC_ID_H264 :
446  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
447  if (std::stoi(value) == 0) {
448  av_opt_set(c->priv_data, "preset", "veryslow", 0);
449  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
450  }
451  break;
452  case AV_CODEC_ID_HEVC :
453  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 51), 0); // 0-51
454  if (std::stoi(value) == 0) {
455  av_opt_set(c->priv_data, "preset", "veryslow", 0);
456  av_opt_set_int(c->priv_data, "lossless", 1, 0);
457  }
458  break;
459  default:
460  // For all other codecs assume a range of 0-63
461  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value), 63), 0); // 0-63
462  c->bit_rate = 0;
463  }
464  }
465  } else if (name == "crf") {
466  // encode quality and special settings like lossless
467 #if USE_HW_ACCEL
468  if (hw_en_on) {
469  double mbs = 15000000.0;
470  if (info.video_bit_rate > 0) {
471  if (info.video_bit_rate > 42) {
472  mbs = 380000.0;
473  }
474  else {
475  mbs *= std::pow(0.912,info.video_bit_rate);
476  }
477  }
478  c->bit_rate = (int)(mbs);
479  } else
480 #endif // USE_HW_ACCEL
481  {
482  switch (c->codec_id) {
483 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
484  // FFmpeg 4.0+
485  case AV_CODEC_ID_AV1 :
486  c->bit_rate = 0;
487  // AV1 only supports "crf" quality values
488  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
489  break;
490 #endif
491  case AV_CODEC_ID_VP8 :
492  c->bit_rate = 10000000;
493  av_opt_set_int(c->priv_data, "crf", std::max(std::min(std::stoi(value), 63), 4), 0); // 4-63
494  break;
495  case AV_CODEC_ID_VP9 :
496  c->bit_rate = 0; // Must be zero!
497  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 63), 0); // 0-63
498  if (std::stoi(value) == 0) {
499  av_opt_set(c->priv_data, "preset", "veryslow", 0);
500  av_opt_set_int(c->priv_data, "lossless", 1, 0);
501  }
502  break;
503  case AV_CODEC_ID_H264 :
504  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
505  if (std::stoi(value) == 0) {
506  av_opt_set(c->priv_data, "preset", "veryslow", 0);
507  c->pix_fmt = PIX_FMT_YUV444P; // no chroma subsampling
508  }
509  break;
510  case AV_CODEC_ID_HEVC :
511  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
512  av_opt_set_int(c->priv_data, "preset", 7, 0);
513  av_opt_set_int(c->priv_data, "forced-idr",1,0);
514  av_opt_set_int(c->priv_data, "qp",std::min(std::stoi(value), 51),0);
515  }
516  else {
517  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value), 51), 0); // 0-51
518  }
519  if (std::stoi(value) == 0) {
520  av_opt_set(c->priv_data, "preset", "veryslow", 0);
521  av_opt_set_int(c->priv_data, "lossless", 1, 0);
522  }
523  break;
524  default:
525  // If this codec doesn't support crf calculate a bitrate
526  // TODO: find better formula
527  double mbs = 15000000.0;
528  if (info.video_bit_rate > 0) {
529  if (info.video_bit_rate > 42) {
530  mbs = 380000.0;
531  } else {
532  mbs *= std::pow(0.912, info.video_bit_rate);
533  }
534  }
535  c->bit_rate = (int) (mbs);
536  }
537  }
538  } else if (name == "qp") {
539  // encode quality and special settings like lossless
540 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
541  // FFmpeg 4.0+
542  switch (c->codec_id) {
543  case AV_CODEC_ID_AV1 :
544  c->bit_rate = 0;
545  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
546  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0);
547  }
548  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
549  // Set number of tiles to a fixed value
550  // TODO Let user choose number of tiles
551  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),255), 0);
552  }
553  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
554  // Set number of tiles to a fixed value
555  // TODO Let user choose number of tiles
556  // libaom doesn't have qp only crf
557  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
558  }
559  else {
560  av_opt_set_int(c->priv_data, "crf", std::min(std::stoi(value),63), 0);
561  }
562  case AV_CODEC_ID_HEVC :
563  c->bit_rate = 0;
564  if (strstr(info.vcodec.c_str(), "svt_hevc") != NULL) {
565  av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),51), 0);
566  av_opt_set_int(c->priv_data, "preset", 7, 0);
567  av_opt_set_int(c->priv_data, "forced-idr",1,0);
568  }
569  break;
570  }
571 #endif // FFmpeg 4.0+
572  } else {
573  // Set AVOption
574  AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c);
575  }
576 
577  ZmqLogger::Instance()->AppendDebugMethod(
578  "FFmpegWriter::SetOption (" + (std::string)name + ")",
579  "stream == VIDEO_STREAM", stream == VIDEO_STREAM);
580 
581  // Muxing dictionary is not part of the codec context.
582  // Just reusing SetOption function to set popular multiplexing presets.
583  } else if (name == "muxing_preset") {
584  if (value == "mp4_faststart") {
585  // 'moov' box to the beginning; only for MOV, MP4
586  av_dict_set(&mux_dict, "movflags", "faststart", 0);
587  } else if (value == "mp4_fragmented") {
588  // write selfcontained fragmented file, minimum length of the fragment 8 sec; only for MOV, MP4
589  av_dict_set(&mux_dict, "movflags", "frag_keyframe", 0);
590  av_dict_set(&mux_dict, "min_frag_duration", "8000000", 0);
591  }
592  } else {
593  throw InvalidOptions("The option is not valid for this codec.", path);
594  }
595 
596 }
597 
599 bool FFmpegWriter::IsValidCodec(std::string codec_name) {
600  // Initialize FFMpeg, and register all formats and codecs
601  AV_REGISTER_ALL
602 
603  // Find the codec (if any)
604  return avcodec_find_encoder_by_name(codec_name.c_str()) != NULL;
605 }
606 
607 // Prepare & initialize streams and open codecs
608 void FFmpegWriter::PrepareStreams() {
609  if (!info.has_audio && !info.has_video)
610  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
611 
612  ZmqLogger::Instance()->AppendDebugMethod(
613  "FFmpegWriter::PrepareStreams [" + path + "]",
614  "info.has_audio", info.has_audio,
615  "info.has_video", info.has_video);
616 
617  // Initialize the streams (i.e. add the streams)
618  initialize_streams();
619 
620  // Mark as 'prepared'
621  prepare_streams = true;
622 }
623 
624 // Write the file header (after the options are set)
625 void FFmpegWriter::WriteHeader() {
626  if (!info.has_audio && !info.has_video)
627  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
628 
629  // Open the output file, if needed
630  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
631  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
632  throw InvalidFile("Could not open or write file.", path);
633  }
634 
635  // Force the output filename (which doesn't always happen for some reason)
636  AV_SET_FILENAME(oc, path.c_str());
637 
638  // Add general metadata (if any)
639  for (auto iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
640  av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0);
641  }
642 
643  // Set multiplexing parameters (only for MP4/MOV containers)
644  AVDictionary *dict = NULL;
645  if (mux_dict) {
646  av_dict_copy(&dict, mux_dict, 0);
647  }
648 
649  // Write the stream header
650  if (avformat_write_header(oc, &dict) != 0) {
651  ZmqLogger::Instance()->AppendDebugMethod(
652  "FFmpegWriter::WriteHeader (avformat_write_header)");
653  throw InvalidFile("Could not write header to file.", path);
654  };
655 
656  // Free multiplexing dictionaries sets
657  if (dict) av_dict_free(&dict);
658  if (mux_dict) av_dict_free(&mux_dict);
659 
660  // Mark as 'written'
661  write_header = true;
662 
663  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader");
664 }
665 
666 // Add a frame to the queue waiting to be encoded.
667 void FFmpegWriter::WriteFrame(std::shared_ptr<openshot::Frame> frame) {
668  // Check for open reader (or throw exception)
669  if (!is_open)
670  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
671 
672  ZmqLogger::Instance()->AppendDebugMethod(
673  "FFmpegWriter::WriteFrame",
674  "frame->number", frame->number,
675  "is_writing", is_writing);
676 
677  // Write frames to video file
678  write_frame(frame);
679 
680  // Keep track of the last frame added
681  last_frame = frame;
682 }
683 
684 // Write all frames in the queue to the video file.
685 void FFmpegWriter::write_frame(std::shared_ptr<Frame> frame) {
686  // Flip writing flag
687  is_writing = true;
688 
689  // Create blank exception
690  bool has_error_encoding_video = false;
691 
692  // Process audio frame
693  if (info.has_audio && audio_st)
694  write_audio_packets(false, frame);
695 
696  // Process video frame
697  if (info.has_video && video_st)
698  process_video_packet(frame);
699 
700  if (info.has_video && video_st) {
701  // Does this frame's AVFrame still exist
702  if (av_frames.count(frame)) {
703  // Get AVFrame
704  AVFrame *frame_final = av_frames[frame];
705 
706  // Write frame to video file
707  if (!write_video_packet(frame, frame_final)) {
708  has_error_encoding_video = true;
709  }
710 
711  // Deallocate buffer and AVFrame
712  av_freep(&(frame_final->data[0]));
713  AV_FREE_FRAME(&frame_final);
714  av_frames.erase(frame);
715  }
716  }
717 
718  // Done writing
719  is_writing = false;
720 
721  // Raise exception from main thread
722  if (has_error_encoding_video)
723  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
724 }
725 
726 // Write a block of frames from a reader
727 void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) {
728  ZmqLogger::Instance()->AppendDebugMethod(
729  "FFmpegWriter::WriteFrame (from Reader)",
730  "start", start,
731  "length", length);
732 
733  // Loop through each frame (and encoded it)
734  for (int64_t number = start; number <= length; number++) {
735  // Get the frame
736  std::shared_ptr<Frame> f = reader->GetFrame(number);
737 
738  // Encode frame
739  WriteFrame(f);
740  }
741 }
742 
743 // Write the file trailer (after all frames are written)
744 void FFmpegWriter::WriteTrailer() {
745  // Process final audio frame (if any)
746  if (info.has_audio && audio_st)
747  write_audio_packets(true, NULL);
748 
749  // Flush encoders (who sometimes hold on to frames)
750  flush_encoders();
751 
752  /* write the trailer, if any. The trailer must be written
753  * before you close the CodecContexts open when you wrote the
754  * header; otherwise write_trailer may try to use memory that
755  * was freed on av_codec_close() */
756  av_write_trailer(oc);
757 
758  // Mark as 'written'
759  write_trailer = true;
760 
761  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer");
762 }
763 
764 // Flush encoders
765 void FFmpegWriter::flush_encoders() {
766  if (info.has_audio && audio_codec_ctx && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec_ctx)->frame_size <= 1)
767  return;
768 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
769  // FFmpeg < 4.0
770  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
771  return;
772 #else
773  if (info.has_video && video_codec_ctx && AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO)
774  return;
775 #endif
776 
777  // FLUSH VIDEO ENCODER
778  if (info.has_video) {
779  for (;;) {
780 
781  // Increment PTS (in frames and scaled to the codec's timebase)
782  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
783 
784 #if IS_FFMPEG_3_2
785  AVPacket* pkt = av_packet_alloc();
786 #else
787  AVPacket* pkt;
788  av_init_packet(pkt);
789 #endif
790  pkt->data = NULL;
791  pkt->size = 0;
792 
793  /* encode the image */
794  int got_packet = 0;
795  int error_code = 0;
796 
797 #if IS_FFMPEG_3_2
798  // Encode video packet (latest version of FFmpeg)
799  error_code = avcodec_send_frame(video_codec_ctx, NULL);
800  got_packet = 0;
801  while (error_code >= 0) {
802  error_code = avcodec_receive_packet(video_codec_ctx, pkt);
803  if (error_code == AVERROR(EAGAIN)|| error_code == AVERROR_EOF) {
804  got_packet = 0;
805  // Write packet
806  avcodec_flush_buffers(video_codec_ctx);
807  break;
808  }
809  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
810  pkt->stream_index = video_st->index;
811  error_code = av_interleaved_write_frame(oc, pkt);
812  }
813 #else // IS_FFMPEG_3_2
814 
815  // Encode video packet (older than FFmpeg 3.2)
816  error_code = avcodec_encode_video2(video_codec_ctx, pkt, NULL, &got_packet);
817 
818 #endif // IS_FFMPEG_3_2
819 
820  if (error_code < 0) {
821  ZmqLogger::Instance()->AppendDebugMethod(
822  "FFmpegWriter::flush_encoders ERROR ["
823  + av_err2string(error_code) + "]",
824  "error_code", error_code);
825  }
826  if (!got_packet) {
827  break;
828  }
829 
830  // set the timestamp
831  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
832  pkt->stream_index = video_st->index;
833 
834  // Write packet
835  error_code = av_interleaved_write_frame(oc, pkt);
836  if (error_code < 0) {
837  ZmqLogger::Instance()->AppendDebugMethod(
838  "FFmpegWriter::flush_encoders ERROR ["
839  + av_err2string(error_code) + "]",
840  "error_code", error_code);
841  }
842  }
843  }
844 
845  // FLUSH AUDIO ENCODER
846  if (info.has_audio) {
847  for (;;) {
848 #if IS_FFMPEG_3_2
849  AVPacket* pkt = av_packet_alloc();
850 #else
851  AVPacket* pkt;
852  av_init_packet(pkt);
853 #endif
854  pkt->data = NULL;
855  pkt->size = 0;
856  pkt->pts = pkt->dts = audio_timestamp;
857 
858  /* encode the image */
859  int error_code = 0;
860  int got_packet = 0;
861 #if IS_FFMPEG_3_2
862  error_code = avcodec_send_frame(audio_codec_ctx, NULL);
863 #else
864  error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, NULL, &got_packet);
865 #endif
866  if (error_code < 0) {
867  ZmqLogger::Instance()->AppendDebugMethod(
868  "FFmpegWriter::flush_encoders ERROR ["
869  + av_err2string(error_code) + "]",
870  "error_code", error_code);
871  }
872  if (!got_packet) {
873  break;
874  }
875 
876  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
877  // but it fixes lots of PTS related issues when I do this.
878  pkt->pts = pkt->dts = audio_timestamp;
879 
880  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
881  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
882 
883  // set stream
884  pkt->stream_index = audio_st->index;
885  pkt->flags |= AV_PKT_FLAG_KEY;
886 
887  // Write packet
888  error_code = av_interleaved_write_frame(oc, pkt);
889  if (error_code < 0) {
890  ZmqLogger::Instance()->AppendDebugMethod(
891  "FFmpegWriter::flush_encoders ERROR ["
892  + av_err2string(error_code) + "]",
893  "error_code", error_code);
894  }
895 
896  // Increment PTS by duration of packet
897  audio_timestamp += pkt->duration;
898 
899  // deallocate memory for packet
900  AV_FREE_PACKET(pkt);
901  }
902  }
903 
904 }
905 
906 // Close the video codec
907 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
908 {
909 #if USE_HW_ACCEL
910  if (hw_en_on && hw_en_supported) {
911  if (hw_device_ctx) {
912  av_buffer_unref(&hw_device_ctx);
913  hw_device_ctx = NULL;
914  }
915  }
916 #endif // USE_HW_ACCEL
917 
918  // Free any previous memory allocations
919  if (video_codec_ctx != nullptr) {
920  AV_FREE_CONTEXT(video_codec_ctx);
921  av_free(video_codec_ctx);
922  }
923 }
924 
925 // Close the audio codec
926 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
927 {
928  // Clear buffers
929  delete[] samples;
930  delete[] audio_outbuf;
931  delete[] audio_encoder_buffer;
932  samples = NULL;
933  audio_outbuf = NULL;
934  audio_encoder_buffer = NULL;
935 
936  // Deallocate resample buffer
937  if (avr) {
938  SWR_CLOSE(avr);
939  SWR_FREE(&avr);
940  avr = NULL;
941  }
942 
943  if (avr_planar) {
944  SWR_CLOSE(avr_planar);
945  SWR_FREE(&avr_planar);
946  avr_planar = NULL;
947  }
948 
949  // Free any previous memory allocations
950  if (audio_codec_ctx != nullptr) {
951  AV_FREE_CONTEXT(audio_codec_ctx);
952  av_free(audio_codec_ctx);
953  }
954 }
955 
956 // Close the writer
957 void FFmpegWriter::Close() {
958  // Write trailer (if needed)
959  if (!write_trailer)
960  WriteTrailer();
961 
962  // Close each codec
963  if (video_st)
964  close_video(oc, video_st);
965  if (audio_st)
966  close_audio(oc, audio_st);
967 
968  // Remove single software scaler
969  if (img_convert_ctx)
970  sws_freeContext(img_convert_ctx);
971 
972  if (!(oc->oformat->flags & AVFMT_NOFILE)) {
973  /* close the output file */
974  avio_close(oc->pb);
975  }
976 
977  // Reset frame counters
978  video_timestamp = 0;
979  audio_timestamp = 0;
980 
981  // Free the context which frees the streams too
982  avformat_free_context(oc);
983  oc = NULL;
984 
985  // Close writer
986  is_open = false;
987  prepare_streams = false;
988  write_header = false;
989  write_trailer = false;
990 
991  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close");
992 }
993 
994 // Add an AVFrame to the cache
995 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame *av_frame) {
996  // Add AVFrame to map (if it does not already exist)
997  if (!av_frames.count(frame)) {
998  // Add av_frame
999  av_frames[frame] = av_frame;
1000  } else {
1001  // Do not add, and deallocate this AVFrame
1002  AV_FREE_FRAME(&av_frame);
1003  }
1004 }
1005 
1006 // Add an audio output stream
1007 AVStream *FFmpegWriter::add_audio_stream() {
1008  // Find the audio codec
1009  const AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1010  if (codec == NULL)
1011  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
1012 
1013  // Free any previous memory allocations
1014  if (audio_codec_ctx != nullptr) {
1015  AV_FREE_CONTEXT(audio_codec_ctx);
1016  }
1017 
1018  // Create a new audio stream
1019  AVStream* st = avformat_new_stream(oc, codec);
1020  if (!st)
1021  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
1022 
1023  // Allocate a new codec context for the stream
1024  ALLOC_CODEC_CTX(audio_codec_ctx, codec, st)
1025 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1026  st->codecpar->codec_id = codec->id;
1027 #endif
1028  AVCodecContext* c = audio_codec_ctx;
1029 
1030  c->codec_id = codec->id;
1031  c->codec_type = AVMEDIA_TYPE_AUDIO;
1032 
1033  // Set the sample parameters
1034  c->bit_rate = info.audio_bit_rate;
1035 #if !HAVE_CH_LAYOUT
1036  c->channels = info.channels;
1037 #endif
1038 
1039  // Set valid sample rate (or throw error)
1040  if (codec->supported_samplerates) {
1041  int i;
1042  for (i = 0; codec->supported_samplerates[i] != 0; i++)
1043  if (info.sample_rate == codec->supported_samplerates[i]) {
1044  // Set the valid sample rate
1045  c->sample_rate = info.sample_rate;
1046  break;
1047  }
1048  if (codec->supported_samplerates[i] == 0)
1049  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
1050  } else
1051  // Set sample rate
1052  c->sample_rate = info.sample_rate;
1053 
1054  uint64_t channel_layout = info.channel_layout;
1055 #if HAVE_CH_LAYOUT
1056  // Set a valid number of channels (or throw error)
1057  AVChannelLayout ch_layout;
1058  av_channel_layout_from_mask(&ch_layout, info.channel_layout);
1059  if (codec->ch_layouts) {
1060  int i;
1061  for (i = 0; av_channel_layout_check(&codec->ch_layouts[i]); i++)
1062  if (av_channel_layout_compare(&ch_layout, &codec->ch_layouts[i])) {
1063  // Set valid channel layout
1064  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1065  break;
1066  }
1067  if (!av_channel_layout_check(&codec->ch_layouts[i]))
1068  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1069  } else
1070  // Set valid channel layout
1071  av_channel_layout_copy(&c->ch_layout, &ch_layout);
1072 #else
1073  // Set a valid number of channels (or throw error)
1074  if (codec->channel_layouts) {
1075  int i;
1076  for (i = 0; codec->channel_layouts[i] != 0; i++)
1077  if (channel_layout == codec->channel_layouts[i]) {
1078  // Set valid channel layout
1079  c->channel_layout = channel_layout;
1080  break;
1081  }
1082  if (codec->channel_layouts[i] == 0)
1083  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
1084  } else
1085  // Set valid channel layout
1086  c->channel_layout = channel_layout;
1087 #endif
1088 
1089  // Choose a valid sample_fmt
1090  if (codec->sample_fmts) {
1091  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) {
1092  // Set sample format to 1st valid format (and then exit loop)
1093  c->sample_fmt = codec->sample_fmts[i];
1094  break;
1095  }
1096  }
1097  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
1098  // Default if no sample formats found
1099  c->sample_fmt = AV_SAMPLE_FMT_S16;
1100  }
1101 
1102  // some formats want stream headers to be separate
1103  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1104 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1105  // FFmpeg 3.0+
1106  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1107 #else
1108  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1109 #endif
1110 
1111  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1112 
1113  int nb_channels;
1114  const char* nb_channels_label;
1115  const char* channel_layout_label;
1116 
1117 #if HAVE_CH_LAYOUT
1118  nb_channels = c->ch_layout.nb_channels;
1119  channel_layout = c->ch_layout.u.mask;
1120  nb_channels_label = "c->ch_layout.nb_channels";
1121  channel_layout_label = "c->ch_layout.u.mask";
1122 #else
1123  nb_channels = c->channels;
1124  nb_channels_label = "c->channels";
1125  channel_layout_label = "c->channel_layout";
1126 #endif
1127 
1128  ZmqLogger::Instance()->AppendDebugMethod(
1129  "FFmpegWriter::add_audio_stream",
1130  "c->codec_id", c->codec_id,
1131  "c->bit_rate", c->bit_rate,
1132  nb_channels_label, nb_channels,
1133  "c->sample_fmt", c->sample_fmt,
1134  channel_layout_label, channel_layout,
1135  "c->sample_rate", c->sample_rate);
1136 
1137  return st;
1138 }
1139 
1140 // Add a video output stream
1141 AVStream *FFmpegWriter::add_video_stream() {
1142  // Find the video codec
1143  const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1144  if (codec == NULL)
1145  throw InvalidCodec("A valid video codec could not be found for this file.", path);
1146 
1147  // Free any previous memory allocations
1148  if (video_codec_ctx != nullptr) {
1149  AV_FREE_CONTEXT(video_codec_ctx);
1150  }
1151 
1152  // Create a new video stream
1153  AVStream* st = avformat_new_stream(oc, codec);
1154  if (!st)
1155  throw OutOfMemory("Could not allocate memory for the video stream.", path);
1156 
1157  // Allocate a new codec context for the stream
1158  ALLOC_CODEC_CTX(video_codec_ctx, codec, st)
1159 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
1160  st->codecpar->codec_id = codec->id;
1161 #endif
1162 
1163  AVCodecContext* c = video_codec_ctx;
1164 
1165  c->codec_id = codec->id;
1166  c->codec_type = AVMEDIA_TYPE_VIDEO;
1167 
1168  // Set sample aspect ratio
1169  c->sample_aspect_ratio.num = info.pixel_ratio.num;
1170  c->sample_aspect_ratio.den = info.pixel_ratio.den;
1171 
1172  /* Init video encoder options */
1173  if (info.video_bit_rate >= 1000
1174 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1175  && c->codec_id != AV_CODEC_ID_AV1
1176 #endif
1177  ) {
1178  c->bit_rate = info.video_bit_rate;
1179  if (info.video_bit_rate >= 1500000) {
1180  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1181  c->qmin = 2;
1182  c->qmax = 30;
1183  }
1184  }
1185  // Here should be the setting for low fixed bitrate
1186  // Defaults are used because mpeg2 otherwise had problems
1187  } else {
1188  // Check if codec supports crf or qp
1189  switch (c->codec_id) {
1190 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
1191  // FFmpeg 4.0+
1192  case AV_CODEC_ID_AV1 :
1193  // TODO: Set `crf` or `qp` according to bitrate, as bitrate is not supported by these encoders yet.
1194  if (info.video_bit_rate >= 1000) {
1195  c->bit_rate = 0;
1196  if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1197  int calculated_quality = 35;
1198  if (info.video_bit_rate < 500000) calculated_quality = 50;
1199  if (info.video_bit_rate > 5000000) calculated_quality = 10;
1200  av_opt_set_int(c->priv_data, "crf", calculated_quality, 0);
1201  info.video_bit_rate = calculated_quality;
1202  } else {
1203  int calculated_quality = 50;
1204  if (info.video_bit_rate < 500000) calculated_quality = 60;
1205  if (info.video_bit_rate > 5000000) calculated_quality = 15;
1206  av_opt_set_int(c->priv_data, "qp", calculated_quality, 0);
1207  info.video_bit_rate = calculated_quality;
1208  } // medium
1209  }
1210  if (strstr(info.vcodec.c_str(), "svtav1") != NULL) {
1211  av_opt_set_int(c->priv_data, "preset", 6, 0);
1212  av_opt_set_int(c->priv_data, "forced-idr",1,0);
1213  }
1214  else if (strstr(info.vcodec.c_str(), "rav1e") != NULL) {
1215  av_opt_set_int(c->priv_data, "speed", 7, 0);
1216  av_opt_set_int(c->priv_data, "tile-rows", 2, 0);
1217  av_opt_set_int(c->priv_data, "tile-columns", 4, 0);
1218  }
1219  else if (strstr(info.vcodec.c_str(), "aom") != NULL) {
1220  // Set number of tiles to a fixed value
1221  // TODO: Allow user to chose their own number of tiles
1222  av_opt_set_int(c->priv_data, "tile-rows", 1, 0); // log2 of number of rows
1223  av_opt_set_int(c->priv_data, "tile-columns", 2, 0); // log2 of number of columns
1224  av_opt_set_int(c->priv_data, "row-mt", 1, 0); // use multiple cores
1225  av_opt_set_int(c->priv_data, "cpu-used", 3, 0); // default is 1, usable is 4
1226  }
1227  //break;
1228 #endif
1229  case AV_CODEC_ID_VP9 :
1230  case AV_CODEC_ID_HEVC :
1231  case AV_CODEC_ID_VP8 :
1232  case AV_CODEC_ID_H264 :
1233  if (info.video_bit_rate < 40) {
1234  c->qmin = 0;
1235  c->qmax = 63;
1236  } else {
1237  c->qmin = info.video_bit_rate - 5;
1238  c->qmax = 63;
1239  }
1240  break;
1241  default:
1242  // Here should be the setting for codecs that don't support crf
1243  // For now defaults are used
1244  break;
1245  }
1246  }
1247 
1248  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
1249  //invalid bitrate errors and rc buffer underflow errors, etc...
1250  //c->rc_min_rate = info.video_bit_rate;
1251  //c->rc_max_rate = info.video_bit_rate;
1252  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
1253  //if ( !c->rc_initial_buffer_occupancy )
1254  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
1255 
1256  /* resolution must be a multiple of two */
1257  // TODO: require /2 height and width
1258  c->width = info.width;
1259  c->height = info.height;
1260 
1261  /* time base: this is the fundamental unit of time (in seconds) in terms
1262  of which frame timestamps are represented. for fixed-fps content,
1263  timebase should be 1/framerate and timestamp increments should be
1264  identically 1. */
1265  c->time_base.num = info.video_timebase.num;
1266  c->time_base.den = info.video_timebase.den;
1267 // AVCodecContext->framerate was added in FFmpeg 2.6
1268 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 26, 0)
1269  c->framerate = av_inv_q(c->time_base);
1270 #endif
1271  st->avg_frame_rate = av_inv_q(c->time_base);
1272  st->time_base.num = info.video_timebase.num;
1273  st->time_base.den = info.video_timebase.den;
1274 
1275  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
1276  c->max_b_frames = 10;
1277  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
1278  /* just for testing, we also add B frames */
1279  c->max_b_frames = 2;
1280  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
1281  /* Needed to avoid using macroblocks in which some coeffs overflow.
1282  This does not happen with normal video, it just happens here as
1283  the motion of the chroma plane does not match the luma plane. */
1284  c->mb_decision = 2;
1285  // some formats want stream headers to be separate
1286  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
1287 #if (LIBAVCODEC_VERSION_MAJOR >= 57)
1288  // FFmpeg 3.0+
1289  c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
1290 #else
1291  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
1292 #endif
1293 
1294  // Find all supported pixel formats for this codec
1295  const PixelFormat *supported_pixel_formats = codec->pix_fmts;
1296  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
1297  // Assign the 1st valid pixel format (if one is missing)
1298  if (c->pix_fmt == PIX_FMT_NONE)
1299  c->pix_fmt = *supported_pixel_formats;
1300  ++supported_pixel_formats;
1301  }
1302 
1303  // Codec doesn't have any pix formats?
1304  if (c->pix_fmt == PIX_FMT_NONE) {
1305  if (oc->oformat->video_codec == AV_CODEC_ID_RAWVIDEO) {
1306  // Raw video should use RGB24
1307  c->pix_fmt = PIX_FMT_RGB24;
1308 
1309 #if (LIBAVFORMAT_VERSION_MAJOR < 58)
1310  // FFmpeg < 4.0
1311  if (strcmp(oc->oformat->name, "gif") != 0)
1312  // If not GIF format, skip the encoding process
1313  // Set raw picture flag (so we don't encode this video)
1314  oc->oformat->flags |= AVFMT_RAWPICTURE;
1315 #endif
1316  } else {
1317  // Set the default codec
1318  c->pix_fmt = PIX_FMT_YUV420P;
1319  }
1320  }
1321 
1322  AV_COPY_PARAMS_FROM_CONTEXT(st, c);
1323  ZmqLogger::Instance()->AppendDebugMethod(
1324  "FFmpegWriter::add_video_stream ("
1325  + (std::string)oc->oformat->name + " : "
1326  + (std::string)av_get_pix_fmt_name(c->pix_fmt) + ")",
1327  "c->codec_id", c->codec_id,
1328  "c->bit_rate", c->bit_rate,
1329  "c->pix_fmt", c->pix_fmt,
1330  "oc->oformat->flags", oc->oformat->flags);
1331  return st;
1332 }
1333 
1334 // open audio codec
1335 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) {
1336  const AVCodec *codec;
1337  AV_GET_CODEC_FROM_STREAM(st, audio_codec_ctx)
1338 
1339  // Audio encoding does not typically use more than 2 threads (most codecs use 1 thread)
1340  audio_codec_ctx->thread_count = std::min(FF_AUDIO_NUM_PROCESSORS, 2);
1341 
1342  // Find the audio encoder
1343  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
1344  if (!codec)
1345  codec = avcodec_find_encoder(audio_codec_ctx->codec_id);
1346  if (!codec)
1347  throw InvalidCodec("Could not find codec", path);
1348 
1349  // Init options
1350  AVDictionary *opts = NULL;
1351  av_dict_set(&opts, "strict", "experimental", 0);
1352 
1353  // Open the codec
1354  if (avcodec_open2(audio_codec_ctx, codec, &opts) < 0)
1355  throw InvalidCodec("Could not open audio codec", path);
1356  AV_COPY_PARAMS_FROM_CONTEXT(st, audio_codec_ctx);
1357 
1358  // Free options
1359  av_dict_free(&opts);
1360 
1361  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
1362  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
1363  if (audio_codec_ctx->frame_size <= 1) {
1364  // No frame size found... so calculate
1365  audio_input_frame_size = 50000 / info.channels;
1366 
1367  int s = AV_FIND_DECODER_CODEC_ID(st);
1368  switch (s) {
1369  case AV_CODEC_ID_PCM_S16LE:
1370  case AV_CODEC_ID_PCM_S16BE:
1371  case AV_CODEC_ID_PCM_U16LE:
1372  case AV_CODEC_ID_PCM_U16BE:
1373  audio_input_frame_size >>= 1;
1374  break;
1375  default:
1376  break;
1377  }
1378  } else {
1379  // Set frame size based on the codec
1380  audio_input_frame_size = audio_codec_ctx->frame_size;
1381  }
1382 
1383  // Set the initial frame size (since it might change during resampling)
1384  initial_audio_input_frame_size = audio_input_frame_size;
1385 
1386  // Allocate array for samples
1387  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1388 
1389  // Set audio output buffer (used to store the encoded audio)
1390  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1391  audio_outbuf = new uint8_t[audio_outbuf_size];
1392 
1393  // Set audio packet encoding buffer
1394  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1395  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1396 
1397  // Add audio metadata (if any)
1398  for (std::map<std::string, std::string>::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1399  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1400  }
1401 
1402  ZmqLogger::Instance()->AppendDebugMethod(
1403  "FFmpegWriter::open_audio",
1404  "audio_codec_ctx->thread_count", audio_codec_ctx->thread_count,
1405  "audio_input_frame_size", audio_input_frame_size,
1406  "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE);
1407 }
1408 
1409 // open video codec
1410 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
1411  const AVCodec *codec;
1412  AV_GET_CODEC_FROM_STREAM(st, video_codec_ctx)
1413 
1414  // Set number of threads equal to number of processors (not to exceed 16, FFmpeg doesn't recommend more than 16)
1415  video_codec_ctx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
1416 
1417 #if USE_HW_ACCEL
1418  if (hw_en_on && hw_en_supported) {
1419  //char *dev_hw = NULL;
1420  char adapter[256];
1421  char *adapter_ptr = NULL;
1422  int adapter_num;
1423  // Use the hw device given in the environment variable HW_EN_DEVICE_SET or the default if not set
1424  adapter_num = openshot::Settings::Instance()->HW_EN_DEVICE_SET;
1425  std::clog << "Encoding Device Nr: " << adapter_num << "\n";
1426  if (adapter_num < 3 && adapter_num >=0) {
1427 #if defined(__linux__)
1428  snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
1429  // Maybe 127 is better because the first card would be 1?!
1430  adapter_ptr = adapter;
1431 #elif defined(_WIN32) || defined(__APPLE__)
1432  adapter_ptr = NULL;
1433 #endif
1434  }
1435  else {
1436  adapter_ptr = NULL; // Just to be sure
1437  }
1438 // Check if it is there and writable
1439 #if defined(__linux__)
1440  if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
1441 #elif defined(_WIN32) || defined(__APPLE__)
1442  if( adapter_ptr != NULL ) {
1443 #endif
1444  ZmqLogger::Instance()->AppendDebugMethod(
1445  "Encode Device present using device",
1446  "adapter", adapter_num);
1447  }
1448  else {
1449  adapter_ptr = NULL; // use default
1450  ZmqLogger::Instance()->AppendDebugMethod(
1451  "Encode Device not present, using default");
1452  }
1453  if (av_hwdevice_ctx_create(&hw_device_ctx,
1454  hw_en_av_device_type, adapter_ptr, NULL, 0) < 0)
1455  {
1456  ZmqLogger::Instance()->AppendDebugMethod(
1457  "FFmpegWriter::open_video ERROR creating hwdevice, Codec name:",
1458  info.vcodec.c_str(), -1);
1459  throw InvalidCodec("Could not create hwdevice", path);
1460  }
1461  }
1462 #endif // USE_HW_ACCEL
1463 
1464  /* find the video encoder */
1465  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1466  if (!codec)
1467  codec = avcodec_find_encoder(AV_FIND_DECODER_CODEC_ID(st));
1468  if (!codec)
1469  throw InvalidCodec("Could not find codec", path);
1470 
1471  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1472  if (video_codec_ctx->max_b_frames && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1473  video_codec_ctx->max_b_frames = 0;
1474 
1475  // Init options
1476  AVDictionary *opts = NULL;
1477  av_dict_set(&opts, "strict", "experimental", 0);
1478 
1479 #if USE_HW_ACCEL
1480  if (hw_en_on && hw_en_supported) {
1481  video_codec_ctx->pix_fmt = hw_en_av_pix_fmt;
1482 
1483  // for the list of possible options, see the list of codec-specific options:
1484  // e.g. ffmpeg -h encoder=h264_vaapi or ffmpeg -h encoder=hevc_vaapi
1485  // and "man ffmpeg-codecs"
1486 
1487  // For VAAPI, it is safer to explicitly set rc_mode instead of relying on auto-selection
1488  // which is ffmpeg version-specific.
1489  if (hw_en_av_pix_fmt == AV_PIX_FMT_VAAPI) {
1490  int64_t qp;
1491  if (av_opt_get_int(video_codec_ctx->priv_data, "qp", 0, &qp) != 0 || qp == 0) {
1492  // unless "qp" was set for CQP, switch to VBR RC mode
1493  av_opt_set(video_codec_ctx->priv_data, "rc_mode", "VBR", 0);
1494 
1495  // In the current state (ffmpeg-4.2-4 libva-mesa-driver-19.1.5-1) to use VBR,
1496  // one has to specify both bit_rate and maxrate, otherwise a small low quality file is generated on Intel iGPU).
1497  video_codec_ctx->rc_max_rate = video_codec_ctx->bit_rate;
1498  }
1499  }
1500 
1501  switch (video_codec_ctx->codec_id) {
1502  case AV_CODEC_ID_H264:
1503  video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames
1504  video_codec_ctx->profile = FF_PROFILE_H264_BASELINE | FF_PROFILE_H264_CONSTRAINED;
1505  av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
1506  av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
1507  av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
1508  break;
1509  case AV_CODEC_ID_HEVC:
1510  // tested to work with defaults
1511  break;
1512  case AV_CODEC_ID_VP9:
1513  // tested to work with defaults
1514  break;
1515  default:
1516  ZmqLogger::Instance()->AppendDebugMethod(
1517  "No codec-specific options defined for this codec. HW encoding may fail",
1518  "codec_id", video_codec_ctx->codec_id);
1519  break;
1520  }
1521 
1522  // set hw_frames_ctx for encoder's AVCodecContext
1523  int err;
1524  if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0)
1525  {
1526  ZmqLogger::Instance()->AppendDebugMethod(
1527  "FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context",
1528  "width", info.width,
1529  "height", info.height,
1530  av_err2string(err), -1);
1531  }
1532  }
1533 #endif // USE_HW_ACCEL
1534 
1535 // Set libx265 hvc1 tag (for Apple playback compatibility).
1536 #if USE_HW_ACCEL
1537  if (!(hw_en_on && hw_en_supported) && video_codec_ctx->codec_id == AV_CODEC_ID_HEVC) {
1538  video_codec_ctx->codec_tag = MKTAG('h', 'v', 'c', '1');
1539  }
1540 #else
1541  if (video_codec_ctx->codec_id == AV_CODEC_ID_HEVC) {
1542  video_codec_ctx->codec_tag = MKTAG('h', 'v', 'c', '1');
1543  }
1544 #endif
1545 
1546  /* open the codec */
1547  if (avcodec_open2(video_codec_ctx, codec, &opts) < 0)
1548  throw InvalidCodec("Could not open video codec", path);
1549  AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx);
1550 
1551  // Free options
1552  av_dict_free(&opts);
1553 
1554  // Add video metadata (if any)
1555  for (auto iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) {
1556  av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0);
1557  }
1558 
1559  ZmqLogger::Instance()->AppendDebugMethod(
1560  "FFmpegWriter::open_video",
1561  "video_codec_ctx->thread_count", video_codec_ctx->thread_count);
1562 
1563 }
1564 
1565 // write all queued frames' audio to the video file
1566 void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr<openshot::Frame> frame) {
1567  if (!frame && !is_final)
1568  return;
1569 
1570  // Init audio buffers / variables
1571  int total_frame_samples = 0;
1572  int frame_position = 0;
1573  int channels_in_frame = 0;
1574  int sample_rate_in_frame = 0;
1575  int samples_in_frame = 0;
1576  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1577 
1578  // Create a new array (to hold all S16 audio samples, for the current queued frames
1579  unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1580  int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size);
1581  int16_t *all_resampled_samples = NULL;
1582  int16_t *final_samples_planar = NULL;
1583  int16_t *final_samples = NULL;
1584 
1585  // Get audio sample array
1586  float *frame_samples_float = NULL;
1587 
1588  // Get the audio details from this frame
1589  if (frame) {
1590  sample_rate_in_frame = frame->SampleRate();
1591  samples_in_frame = frame->GetAudioSamplesCount();
1592  channels_in_frame = frame->GetAudioChannelsCount();
1593  channel_layout_in_frame = frame->ChannelsLayout();
1594 
1595  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1596  frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame);
1597  }
1598 
1599  // Calculate total samples
1600  total_frame_samples = samples_in_frame * channels_in_frame;
1601 
1602  // Translate audio sample values back to 16 bit integers with saturation
1603  const int16_t max16 = 32767;
1604  const int16_t min16 = -32768;
1605  for (int s = 0; s < total_frame_samples; s++, frame_position++) {
1606  float valF = frame_samples_float[s] * (1 << 15);
1607  int16_t conv;
1608  if (valF > max16) {
1609  conv = max16;
1610  } else if (valF < min16) {
1611  conv = min16;
1612  } else {
1613  conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding
1614  }
1615 
1616  // Copy into buffer
1617  all_queued_samples[frame_position] = conv;
1618  }
1619 
1620  // Deallocate float array
1621  delete[] frame_samples_float;
1622 
1623 
1624  // Update total samples (since we've combined all queued frames)
1625  total_frame_samples = frame_position;
1626  int remaining_frame_samples = total_frame_samples;
1627  int samples_position = 0;
1628 
1629 
1630  ZmqLogger::Instance()->AppendDebugMethod(
1631  "FFmpegWriter::write_audio_packets",
1632  "is_final", is_final,
1633  "total_frame_samples", total_frame_samples,
1634  "channel_layout_in_frame", channel_layout_in_frame,
1635  "channels_in_frame", channels_in_frame,
1636  "samples_in_frame", samples_in_frame,
1637  "LAYOUT_MONO", LAYOUT_MONO);
1638 
1639  // Keep track of the original sample format
1640  AVSampleFormat output_sample_fmt = audio_codec_ctx->sample_fmt;
1641 
1642  AVFrame *audio_frame = NULL;
1643  if (!is_final) {
1644  // Create input frame (and allocate arrays)
1645  audio_frame = AV_ALLOCATE_FRAME();
1646  AV_RESET_FRAME(audio_frame);
1647  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1648 
1649  // Fill input frame with sample data
1650  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples, all_queued_samples_size, 0);
1651  if (error_code < 0) {
1652  ZmqLogger::Instance()->AppendDebugMethod(
1653  "FFmpegWriter::write_audio_packets ERROR ["
1654  + av_err2string(error_code) + "]",
1655  "error_code", error_code);
1656  }
1657 
1658  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1659  switch (audio_codec_ctx->sample_fmt) {
1660  case AV_SAMPLE_FMT_FLTP: {
1661  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1662  break;
1663  }
1664  case AV_SAMPLE_FMT_S32P: {
1665  output_sample_fmt = AV_SAMPLE_FMT_S32;
1666  break;
1667  }
1668  case AV_SAMPLE_FMT_S16P: {
1669  output_sample_fmt = AV_SAMPLE_FMT_S16;
1670  break;
1671  }
1672  case AV_SAMPLE_FMT_U8P: {
1673  output_sample_fmt = AV_SAMPLE_FMT_U8;
1674  break;
1675  }
1676  default: {
1677  // This is only here to silence unused-enum warnings
1678  break;
1679  }
1680  }
1681 
1682  // Update total samples & input frame size (due to bigger or smaller data types)
1683  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1684  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1685 
1686  // Create output frame (and allocate arrays)
1687  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1688  AV_RESET_FRAME(audio_converted);
1689  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1690  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1691 
1692  ZmqLogger::Instance()->AppendDebugMethod(
1693  "FFmpegWriter::write_audio_packets (1st resampling)",
1694  "in_sample_fmt", AV_SAMPLE_FMT_S16,
1695  "out_sample_fmt", output_sample_fmt,
1696  "in_sample_rate", sample_rate_in_frame,
1697  "out_sample_rate", info.sample_rate,
1698  "in_channels", channels_in_frame,
1699  "out_channels", info.channels);
1700 
1701  // setup resample context
1702  if (!avr) {
1703  avr = SWR_ALLOC();
1704 #if HAVE_CH_LAYOUT
1705  AVChannelLayout in_chlayout;
1706  AVChannelLayout out_chlayout;
1707  av_channel_layout_from_mask(&in_chlayout, channel_layout_in_frame);
1708  av_channel_layout_from_mask(&out_chlayout, info.channel_layout);
1709  av_opt_set_chlayout(avr, "in_chlayout", &in_chlayout, 0);
1710  av_opt_set_chlayout(avr, "out_chlayout", &out_chlayout, 0);
1711 #else
1712  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1713  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1714  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1715  av_opt_set_int(avr, "out_channels", info.channels, 0);
1716 #endif
1717  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1718  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1719  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1720  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1721  SWR_INIT(avr);
1722  }
1723  // Convert audio samples
1724  int nb_samples = SWR_CONVERT(
1725  avr, // audio resample context
1726  audio_converted->data, // output data pointers
1727  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1728  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1729  audio_frame->data, // input data pointers
1730  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1731  audio_frame->nb_samples // number of input samples to convert
1732  );
1733 
1734  // Set remaining samples
1735  remaining_frame_samples = total_frame_samples;
1736 
1737  // Create a new array (to hold all resampled S16 audio samples)
1738  all_resampled_samples = (int16_t *) av_malloc(
1739  sizeof(int16_t) * nb_samples * info.channels
1740  * (av_get_bytes_per_sample(output_sample_fmt) /
1741  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1742  );
1743 
1744  // Copy audio samples over original samples
1745  memcpy(all_resampled_samples, audio_converted->data[0],
1746  static_cast<size_t>(nb_samples)
1747  * info.channels
1748  * av_get_bytes_per_sample(output_sample_fmt));
1749 
1750  // Remove converted audio
1751  av_freep(&(audio_frame->data[0]));
1752  AV_FREE_FRAME(&audio_frame);
1753  av_freep(&audio_converted->data[0]);
1754  AV_FREE_FRAME(&audio_converted);
1755  all_queued_samples = NULL; // this array cleared with above call
1756 
1757  ZmqLogger::Instance()->AppendDebugMethod(
1758  "FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)",
1759  "nb_samples", nb_samples,
1760  "remaining_frame_samples", remaining_frame_samples);
1761  }
1762 
1763  // Loop until no more samples
1764  while (remaining_frame_samples > 0 || is_final) {
1765  // Get remaining samples needed for this packet
1766  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1767 
1768  // Determine how many samples we need
1769  int diff = 0;
1770  if (remaining_frame_samples >= remaining_packet_samples) {
1771  diff = remaining_packet_samples;
1772  } else {
1773  diff = remaining_frame_samples;
1774  }
1775 
1776  // Copy frame samples into the packet samples array
1777  if (!is_final)
1778  //TODO: Make this more sane
1779  memcpy(
1780  samples + (audio_input_position
1781  * (av_get_bytes_per_sample(output_sample_fmt) /
1782  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1783  ),
1784  all_resampled_samples + samples_position,
1785  static_cast<size_t>(diff)
1786  * av_get_bytes_per_sample(output_sample_fmt)
1787  );
1788 
1789  // Increment counters
1790  audio_input_position += diff;
1791  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1792  remaining_frame_samples -= diff;
1793 
1794  // Do we have enough samples to proceed?
1795  if (audio_input_position < (audio_input_frame_size * info.channels) && !is_final)
1796  // Not enough samples to encode... so wait until the next frame
1797  break;
1798 
1799  // Convert to planar (if needed by audio codec)
1800  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1801  AV_RESET_FRAME(frame_final);
1802  if (av_sample_fmt_is_planar(audio_codec_ctx->sample_fmt)) {
1803  ZmqLogger::Instance()->AppendDebugMethod(
1804  "FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)",
1805  "in_sample_fmt", output_sample_fmt,
1806  "out_sample_fmt", audio_codec_ctx->sample_fmt,
1807  "in_sample_rate", info.sample_rate,
1808  "out_sample_rate", info.sample_rate,
1809  "in_channels", info.channels,
1810  "out_channels", info.channels
1811  );
1812 
1813  // setup resample context
1814  if (!avr_planar) {
1815  avr_planar = SWR_ALLOC();
1816 #if HAVE_CH_LAYOUT
1817  AVChannelLayout layout;
1818  av_channel_layout_from_mask(&layout, info.channel_layout);
1819  av_opt_set_chlayout(avr_planar, "in_chlayout", &layout, 0);
1820  av_opt_set_chlayout(avr_planar, "out_chlayout", &layout, 0);
1821 #else
1822  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1823  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1824  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1825  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1826 #endif
1827  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1828  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec_ctx->sample_fmt, 0); // planar not allowed here
1829  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1830  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1831  SWR_INIT(avr_planar);
1832  }
1833 
1834  // Create input frame (and allocate arrays)
1835  audio_frame = AV_ALLOCATE_FRAME();
1836  AV_RESET_FRAME(audio_frame);
1837  audio_frame->nb_samples = audio_input_position / info.channels;
1838 
1839  // Create a new array
1840  final_samples_planar = (int16_t *) av_malloc(
1841  sizeof(int16_t) * audio_frame->nb_samples * info.channels
1842  * (av_get_bytes_per_sample(output_sample_fmt) /
1843  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) )
1844  );
1845 
1846  // Copy audio into buffer for frame
1847  memcpy(final_samples_planar, samples,
1848  static_cast<size_t>(audio_frame->nb_samples)
1849  * info.channels
1850  * av_get_bytes_per_sample(output_sample_fmt));
1851 
1852  // Fill input frame with sample data
1853  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt,
1854  (uint8_t *) final_samples_planar, audio_encoder_buffer_size, 0);
1855 
1856  // Create output frame (and allocate arrays)
1857  frame_final->nb_samples = audio_input_frame_size;
1858 #if HAVE_CH_LAYOUT
1859  av_channel_layout_from_mask(&frame_final->ch_layout, info.channel_layout);
1860 #else
1861  frame_final->channels = info.channels;
1862  frame_final->channel_layout = info.channel_layout;
1863 #endif
1864  frame_final->format = audio_codec_ctx->sample_fmt;
1865  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels,
1866  frame_final->nb_samples, audio_codec_ctx->sample_fmt, 0);
1867 
1868  // Convert audio samples
1869  int nb_samples = SWR_CONVERT(
1870  avr_planar, // audio resample context
1871  frame_final->data, // output data pointers
1872  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1873  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1874  audio_frame->data, // input data pointers
1875  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1876  audio_frame->nb_samples // number of input samples to convert
1877  );
1878 
1879  // Copy audio samples over original samples
1880  const auto copy_length = static_cast<size_t>(nb_samples)
1881  * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt)
1882  * info.channels;
1883 
1884  if (nb_samples > 0)
1885  memcpy(samples, frame_final->data[0], copy_length);
1886 
1887  // deallocate AVFrame
1888  av_freep(&(audio_frame->data[0]));
1889  AV_FREE_FRAME(&audio_frame);
1890  all_queued_samples = NULL; // this array cleared with above call
1891 
1892  ZmqLogger::Instance()->AppendDebugMethod(
1893  "FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)",
1894  "nb_samples", nb_samples);
1895 
1896  } else {
1897  // Create a new array
1898  const auto buf_size = static_cast<size_t>(audio_input_position)
1899  * (av_get_bytes_per_sample(audio_codec_ctx->sample_fmt) /
1900  av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)
1901  );
1902  final_samples = reinterpret_cast<int16_t*>(
1903  av_malloc(sizeof(int16_t) * buf_size));
1904 
1905  // Copy audio into buffer for frame
1906  memcpy(final_samples, samples,
1907  audio_input_position * av_get_bytes_per_sample(audio_codec_ctx->sample_fmt));
1908 
1909  // Init the nb_samples property
1910  frame_final->nb_samples = audio_input_frame_size;
1911 
1912  // Fill the final_frame AVFrame with audio (non planar)
1913 #if HAVE_CH_LAYOUT
1914  int nb_channels = audio_codec_ctx->ch_layout.nb_channels;
1915 #else
1916  int nb_channels = audio_codec_ctx->channels;
1917 #endif
1918  avcodec_fill_audio_frame(frame_final, nb_channels,
1919  audio_codec_ctx->sample_fmt, (uint8_t *) final_samples,
1920  audio_encoder_buffer_size, 0);
1921  }
1922 
1923  // Set the AVFrame's PTS
1924  frame_final->pts = audio_timestamp;
1925 
1926  // Init the packet
1927 #if IS_FFMPEG_3_2
1928  AVPacket* pkt = av_packet_alloc();
1929 #else
1930  AVPacket* pkt;
1931  av_init_packet(pkt);
1932 #endif
1933  pkt->data = audio_encoder_buffer;
1934  pkt->size = audio_encoder_buffer_size;
1935 
1936  // Set the packet's PTS prior to encoding
1937  pkt->pts = pkt->dts = audio_timestamp;
1938 
1939  /* encode the audio samples */
1940  int got_packet_ptr = 0;
1941 
1942 #if IS_FFMPEG_3_2
1943  // Encode audio (latest version of FFmpeg)
1944  int error_code;
1945  int ret = 0;
1946  int frame_finished = 0;
1947  error_code = ret = avcodec_send_frame(audio_codec_ctx, frame_final);
1948  if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1949  avcodec_send_frame(audio_codec_ctx, NULL);
1950  }
1951  else {
1952  if (ret >= 0)
1953  pkt->size = 0;
1954  ret = avcodec_receive_packet(audio_codec_ctx, pkt);
1955  if (ret >= 0)
1956  frame_finished = 1;
1957  if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1958  avcodec_flush_buffers(audio_codec_ctx);
1959  ret = 0;
1960  }
1961  if (ret >= 0) {
1962  ret = frame_finished;
1963  }
1964  }
1965  if (!pkt->data && !frame_finished)
1966  {
1967  ret = -1;
1968  }
1969  got_packet_ptr = ret;
1970 #else
1971  // Encode audio (older versions of FFmpeg)
1972  int error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, frame_final, &got_packet_ptr);
1973 #endif
1974  /* if zero size, it means the image was buffered */
1975  if (error_code == 0 && got_packet_ptr) {
1976 
1977  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1978  // but it fixes lots of PTS related issues when I do this.
1979  pkt->pts = pkt->dts = audio_timestamp;
1980 
1981  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1982  av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base);
1983 
1984  // set stream
1985  pkt->stream_index = audio_st->index;
1986  pkt->flags |= AV_PKT_FLAG_KEY;
1987 
1988  /* write the compressed frame in the media file */
1989  error_code = av_interleaved_write_frame(oc, pkt);
1990  }
1991 
1992  if (error_code < 0) {
1993  ZmqLogger::Instance()->AppendDebugMethod(
1994  "FFmpegWriter::write_audio_packets ERROR ["
1995  + av_err2string(error_code) + "]",
1996  "error_code", error_code);
1997  }
1998 
1999  // Increment PTS (no pkt.duration, so calculate with maths)
2000  audio_timestamp += FFMIN(audio_input_frame_size, audio_input_position);
2001 
2002  // deallocate AVFrame
2003  av_freep(&(frame_final->data[0]));
2004  AV_FREE_FRAME(&frame_final);
2005 
2006  // deallocate memory for packet
2007  AV_FREE_PACKET(pkt);
2008 
2009  // Reset position
2010  audio_input_position = 0;
2011  is_final = false;
2012  }
2013 
2014  // Delete arrays (if needed)
2015  if (all_resampled_samples) {
2016  av_freep(&all_resampled_samples);
2017  all_resampled_samples = NULL;
2018  }
2019  if (all_queued_samples) {
2020  av_freep(&all_queued_samples);
2021  all_queued_samples = NULL;
2022  }
2023 }
2024 
2025 // Allocate an AVFrame object
2026 AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) {
2027  // Create an RGB AVFrame
2028  AVFrame *new_av_frame = NULL;
2029 
2030  // Allocate an AVFrame structure
2031  new_av_frame = AV_ALLOCATE_FRAME();
2032  if (new_av_frame == NULL)
2033  throw OutOfMemory("Could not allocate AVFrame", path);
2034 
2035  // Determine required buffer size and allocate buffer
2036  *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height);
2037 
2038  // Create buffer (if not provided)
2039  if (!new_buffer) {
2040  // New Buffer
2041  new_buffer = (uint8_t *) av_malloc(*buffer_size * sizeof(uint8_t));
2042  // Attach buffer to AVFrame
2043  AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height);
2044  new_av_frame->width = width;
2045  new_av_frame->height = height;
2046  new_av_frame->format = pix_fmt;
2047  }
2048 
2049  // return AVFrame
2050  return new_av_frame;
2051 }
2052 
2053 // process video frame
2054 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame) {
2055  // Source dimensions (RGBA)
2056  int src_w = frame->GetWidth();
2057  int src_h = frame->GetHeight();
2058 
2059  // Skip empty frames (1×1)
2060  if (src_w == 1 && src_h == 1)
2061  return;
2062 
2063  // Point persistent_src_frame->data to RGBA pixels
2064  const uchar* pixels = frame->GetPixels();
2065  if (!persistent_src_frame) {
2066  persistent_src_frame = av_frame_alloc();
2067  if (!persistent_src_frame)
2068  throw OutOfMemory("Could not allocate persistent_src_frame", path);
2069  persistent_src_frame->format = AV_PIX_FMT_RGBA;
2070  persistent_src_frame->width = src_w;
2071  persistent_src_frame->height = src_h;
2072  persistent_src_frame->linesize[0] = src_w * 4;
2073  }
2074  persistent_src_frame->data[0] = const_cast<uint8_t*>(
2075  reinterpret_cast<const uint8_t*>(pixels)
2076  );
2077 
2078  // Prepare persistent_dst_frame + buffer on first use
2079  if (!persistent_dst_frame) {
2080  persistent_dst_frame = av_frame_alloc();
2081  if (!persistent_dst_frame)
2082  throw OutOfMemory("Could not allocate persistent_dst_frame", path);
2083 
2084  // Decide destination pixel format: NV12 if HW accel is on, else encoder’s pix_fmt
2085  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2086 #if USE_HW_ACCEL
2087  if (hw_en_on && hw_en_supported) {
2088  dst_fmt = AV_PIX_FMT_NV12;
2089  }
2090 #endif
2091  persistent_dst_frame->format = dst_fmt;
2092  persistent_dst_frame->width = info.width;
2093  persistent_dst_frame->height = info.height;
2094 
2095  persistent_dst_size = av_image_get_buffer_size(
2096  dst_fmt, info.width, info.height, 1
2097  );
2098  if (persistent_dst_size < 0)
2099  throw ErrorEncodingVideo("Invalid destination image size", -1);
2100 
2101  persistent_dst_buffer = static_cast<uint8_t*>(
2102  av_malloc(persistent_dst_size)
2103  );
2104  if (!persistent_dst_buffer)
2105  throw OutOfMemory("Could not allocate persistent_dst_buffer", path);
2106 
2107  av_image_fill_arrays(
2108  persistent_dst_frame->data,
2109  persistent_dst_frame->linesize,
2110  persistent_dst_buffer,
2111  dst_fmt,
2112  info.width,
2113  info.height,
2114  1
2115  );
2116  }
2117 
2118  // Initialize SwsContext (RGBA → dst_fmt) on first use
2119  if (!img_convert_ctx) {
2120  int flags = SWS_FAST_BILINEAR;
2121  if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
2122  flags = SWS_BICUBIC;
2123  }
2124  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2125 #if USE_HW_ACCEL
2126  if (hw_en_on && hw_en_supported) {
2127  dst_fmt = AV_PIX_FMT_NV12;
2128  }
2129 #endif
2130  img_convert_ctx = sws_getContext(
2131  src_w, src_h, AV_PIX_FMT_RGBA,
2132  info.width, info.height, dst_fmt,
2133  flags, NULL, NULL, NULL
2134  );
2135  if (!img_convert_ctx)
2136  throw ErrorEncodingVideo("Could not initialize sws context", -1);
2137  }
2138 
2139  // Scale RGBA → dst_fmt into persistent_dst_buffer
2140  sws_scale(
2141  img_convert_ctx,
2142  persistent_src_frame->data,
2143  persistent_src_frame->linesize,
2144  0, src_h,
2145  persistent_dst_frame->data,
2146  persistent_dst_frame->linesize
2147  );
2148 
2149  // Allocate a new AVFrame + buffer, then copy scaled data into it
2150  int bytes_final = 0;
2151  AVPixelFormat dst_fmt = video_codec_ctx->pix_fmt;
2152 #if USE_HW_ACCEL
2153  if (hw_en_on && hw_en_supported) {
2154  dst_fmt = AV_PIX_FMT_NV12;
2155  }
2156 #endif
2157 
2158  AVFrame* new_frame = allocate_avframe(
2159  dst_fmt,
2160  info.width,
2161  info.height,
2162  &bytes_final,
2163  nullptr
2164  );
2165  if (!new_frame)
2166  throw OutOfMemory("Could not allocate new_frame via allocate_avframe", path);
2167 
2168  // Copy persistent_dst_buffer → new_frame buffer
2169  memcpy(
2170  new_frame->data[0],
2171  persistent_dst_buffer,
2172  static_cast<size_t>(bytes_final)
2173  );
2174 
2175  // Queue the deep‐copied frame for encoding
2176  add_avframe(frame, new_frame);
2177 }
2178 
2179 // write video frame
2180 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *frame_final) {
2181 #if (LIBAVFORMAT_VERSION_MAJOR >= 58)
2182  // FFmpeg 4.0+
2183  ZmqLogger::Instance()->AppendDebugMethod(
2184  "FFmpegWriter::write_video_packet",
2185  "frame->number", frame->number,
2186  "oc->oformat->flags", oc->oformat->flags);
2187 
2188  if (AV_GET_CODEC_TYPE(video_st) == AVMEDIA_TYPE_VIDEO && AV_FIND_DECODER_CODEC_ID(video_st) == AV_CODEC_ID_RAWVIDEO) {
2189 #else
2190  // TODO: Should we have moved away from oc->oformat->flags / AVFMT_RAWPICTURE
2191  // on ffmpeg < 4.0 as well?
2192  // Does AV_CODEC_ID_RAWVIDEO not work in ffmpeg 3.x?
2193  ZmqLogger::Instance()->AppendDebugMethod(
2194  "FFmpegWriter::write_video_packet",
2195  "frame->number", frame->number,
2196  "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE);
2197 
2198  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
2199 #endif
2200  // Raw video case.
2201 #if IS_FFMPEG_3_2
2202  AVPacket* pkt = av_packet_alloc();
2203 #else
2204  AVPacket* pkt;
2205  av_init_packet(pkt);
2206 #endif
2207 
2208  av_packet_from_data(
2209  pkt, frame_final->data[0],
2210  frame_final->linesize[0] * frame_final->height);
2211 
2212  pkt->flags |= AV_PKT_FLAG_KEY;
2213  pkt->stream_index = video_st->index;
2214 
2215  // Set PTS (in frames and scaled to the codec's timebase)
2216  pkt->pts = video_timestamp;
2217 
2218  /* write the compressed frame in the media file */
2219  int error_code = av_interleaved_write_frame(oc, pkt);
2220  if (error_code < 0) {
2221  ZmqLogger::Instance()->AppendDebugMethod(
2222  "FFmpegWriter::write_video_packet ERROR ["
2223  + av_err2string(error_code) + "]",
2224  "error_code", error_code);
2225  return false;
2226  }
2227 
2228  // Deallocate packet
2229  AV_FREE_PACKET(pkt);
2230 
2231  } else
2232  {
2233 
2234 #if IS_FFMPEG_3_2
2235  AVPacket* pkt = av_packet_alloc();
2236 #else
2237  AVPacket* pkt;
2238  av_init_packet(pkt);
2239 #endif
2240  pkt->data = NULL;
2241  pkt->size = 0;
2242  pkt->pts = pkt->dts = AV_NOPTS_VALUE;
2243 
2244  // Assign the initial AVFrame PTS from the frame counter
2245  frame_final->pts = video_timestamp;
2246 #if USE_HW_ACCEL
2247  if (hw_en_on && hw_en_supported) {
2248  if (!(hw_frame = av_frame_alloc())) {
2249  std::clog << "Error code: av_hwframe_alloc\n";
2250  }
2251  if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) {
2252  std::clog << "Error code: av_hwframe_get_buffer\n";
2253  }
2254  if (!hw_frame->hw_frames_ctx) {
2255  std::clog << "Error hw_frames_ctx.\n";
2256  }
2257  hw_frame->format = AV_PIX_FMT_NV12;
2258  if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) {
2259  std::clog << "Error while transferring frame data to surface.\n";
2260  }
2261  av_frame_copy_props(hw_frame, frame_final);
2262  }
2263 #endif // USE_HW_ACCEL
2264  /* encode the image */
2265  int got_packet_ptr = 0;
2266  int error_code = 0;
2267 #if IS_FFMPEG_3_2
2268  // Write video packet
2269  int ret;
2270 
2271  #if USE_HW_ACCEL
2272  if (hw_en_on && hw_en_supported) {
2273  ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!!
2274  } else
2275  #endif // USE_HW_ACCEL
2276  {
2277  ret = avcodec_send_frame(video_codec_ctx, frame_final);
2278  }
2279  error_code = ret;
2280  if (ret < 0 ) {
2281  ZmqLogger::Instance()->AppendDebugMethod(
2282  "FFmpegWriter::write_video_packet (Frame not sent)");
2283  if (ret == AVERROR(EAGAIN) ) {
2284  std::clog << "Frame EAGAIN\n";
2285  }
2286  if (ret == AVERROR_EOF ) {
2287  std::clog << "Frame AVERROR_EOF\n";
2288  }
2289  avcodec_send_frame(video_codec_ctx, NULL);
2290  }
2291  else {
2292  while (ret >= 0) {
2293  ret = avcodec_receive_packet(video_codec_ctx, pkt);
2294 
2295  if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
2296  got_packet_ptr = 0;
2297  break;
2298  }
2299  if (ret == 0) {
2300  got_packet_ptr = 1;
2301  break;
2302  }
2303  }
2304  }
2305 #else
2306  // Write video packet (older than FFmpeg 3.2)
2307  error_code = avcodec_encode_video2(video_codec_ctx, pkt, frame_final, &got_packet_ptr);
2308  if (error_code != 0) {
2309  ZmqLogger::Instance()->AppendDebugMethod(
2310  "FFmpegWriter::write_video_packet ERROR ["
2311  + av_err2string(error_code) + "]",
2312  "error_code", error_code);
2313  }
2314  if (got_packet_ptr == 0) {
2315  ZmqLogger::Instance()->AppendDebugMethod(
2316  "FFmpegWriter::write_video_packet (Frame gotpacket error)");
2317  }
2318 #endif // IS_FFMPEG_3_2
2319 
2320  /* if zero size, it means the image was buffered */
2321  if (error_code == 0 && got_packet_ptr) {
2322  // set the timestamp
2323  av_packet_rescale_ts(pkt, video_codec_ctx->time_base, video_st->time_base);
2324  pkt->stream_index = video_st->index;
2325 
2326  /* write the compressed frame in the media file */
2327  int result = av_interleaved_write_frame(oc, pkt);
2328  if (result < 0) {
2329  ZmqLogger::Instance()->AppendDebugMethod(
2330  "FFmpegWriter::write_video_packet ERROR ["
2331  + av_err2string(result) + "]",
2332  "result", result);
2333  return false;
2334  }
2335  }
2336 
2337  // Deallocate packet
2338  AV_FREE_PACKET(pkt);
2339 #if USE_HW_ACCEL
2340  if (hw_en_on && hw_en_supported) {
2341  if (hw_frame) {
2342  av_frame_free(&hw_frame);
2343  hw_frame = NULL;
2344  }
2345  }
2346 #endif // USE_HW_ACCEL
2347  }
2348 
2349  // Increment PTS (in frames and scaled to the codec's timebase)
2350  video_timestamp += av_rescale_q(1, av_make_q(info.fps.den, info.fps.num), video_codec_ctx->time_base);
2351 
2352  // Success
2353  return true;
2354 }
2355 
2356 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
2357 void FFmpegWriter::OutputStreamInfo() {
2358  // output debug info
2359  av_dump_format(oc, 0, path.c_str(), 1);
2360 }
2361 
2362 // Set audio resample options
2363 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
2364  original_sample_rate = sample_rate;
2365  original_channels = channels;
2366 }
2367 
2368 // In FFmpegWriter.cpp
2369 void FFmpegWriter::AddSphericalMetadata(const std::string& projection, float yaw_deg, float pitch_deg, float roll_deg) {
2370  if (!oc) return;
2371  if (!info.has_video || !video_st) return;
2372 
2373  // Allow movenc.c to write out the sv3d atom
2374  oc->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
2375 
2376 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 0, 0)
2377  // Map the projection name to the enum (defaults to equirectangular)
2378  int proj = av_spherical_from_name(projection.c_str());
2379  if (proj < 0)
2380  proj = AV_SPHERICAL_EQUIRECTANGULAR;
2381 
2382  // Allocate the side‐data structure
2383  size_t sd_size = 0;
2384  AVSphericalMapping* map = av_spherical_alloc(&sd_size);
2385  if (!map) return;
2386 
2387  // Populate it
2388  map->projection = static_cast<AVSphericalProjection>(proj);
2389  // yaw/pitch/roll are 16.16 fixed point
2390  map->yaw = static_cast<int32_t>(yaw_deg * (1 << 16));
2391  map->pitch = static_cast<int32_t>(pitch_deg * (1 << 16));
2392  map->roll = static_cast<int32_t>(roll_deg * (1 << 16));
2393 
2394  av_stream_add_side_data(video_st, AV_PKT_DATA_SPHERICAL, reinterpret_cast<uint8_t*>(map), sd_size);
2395 #endif
2396 }
AUDIO_PACKET_ENCODING_SIZE
#define AUDIO_PACKET_ENCODING_SIZE
Definition: FFmpegUtilities.h:90
Settings.h
Header file for global Settings class.
openshot::AUDIO_STREAM
@ AUDIO_STREAM
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:30
hw_frame
AVFrame * hw_frame
Definition: FFmpegWriter.cpp:43
openshot::InvalidFormat
Exception when no valid format is found for a file.
Definition: Exceptions.h:202
PIX_FMT_RGB24
#define PIX_FMT_RGB24
Definition: FFmpegUtilities.h:116
AV_FIND_DECODER_CODEC_ID
#define AV_FIND_DECODER_CODEC_ID(av_stream)
Definition: FFmpegUtilities.h:211
openshot::InvalidSampleRate
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:247
FFmpegUtilities.h
Header file for FFmpegUtilities.
openshot::WriterInfo::video_bit_rate
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:43
FFmpegWriter.h
Header file for FFmpegWriter class.
openshot::InvalidCodec
Exception when no valid codec is found for a file.
Definition: Exceptions.h:172
openshot::FFmpegWriter::ResampleAudio
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
Definition: FFmpegWriter.cpp:2363
openshot::WriterInfo::display_ratio
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: WriterBase.h:45
openshot::WriterClosed
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:415
AV_COPY_PICTURE_DATA
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
Definition: FFmpegUtilities.h:223
openshot::FFmpegWriter::OutputStreamInfo
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
Definition: FFmpegWriter.cpp:2357
PixelFormat
#define PixelFormat
Definition: FFmpegUtilities.h:107
openshot::ReaderBase::GetFrame
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
AV_ALLOCATE_FRAME
#define AV_ALLOCATE_FRAME()
Definition: FFmpegUtilities.h:203
SWR_CONVERT
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
Definition: FFmpegUtilities.h:149
openshot::WriterInfo::fps
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:42
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
AV_OPTION_SET
#define AV_OPTION_SET(av_stream, priv_data, name, value, avcodec)
Definition: FFmpegUtilities.h:228
openshot::WriterInfo::audio_bit_rate
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:53
openshot::WriterInfo::channels
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:55
AV_COPY_PARAMS_FROM_CONTEXT
AV_COPY_PARAMS_FROM_CONTEXT(st, video_codec_ctx)
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
AV_GET_CODEC_FROM_STREAM
#define AV_GET_CODEC_FROM_STREAM(av_stream, codec_in)
Definition: FFmpegUtilities.h:217
openshot::FFmpegWriter::FFmpegWriter
FFmpegWriter(const std::string &path)
Constructor for FFmpegWriter. Throws an exception on failure to open path.
Definition: FFmpegWriter.cpp:76
AV_FREE_FRAME
#define AV_FREE_FRAME(av_frame)
Definition: FFmpegUtilities.h:207
AV_FREE_PACKET
#define AV_FREE_PACKET(av_packet)
Definition: FFmpegUtilities.h:208
openshot::FFmpegWriter::Open
void Open()
Open writer.
Definition: FFmpegWriter.cpp:96
openshot::FFmpegWriter::SetVideoOptions
void SetVideoOptions(bool has_video, std::string codec, openshot::Fraction fps, int width, int height, openshot::Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
Definition: FFmpegWriter.cpp:165
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
AV_GET_CODEC_PAR_CONTEXT
#define AV_GET_CODEC_PAR_CONTEXT(av_stream, av_codec)
Definition: FFmpegUtilities.h:216
openshot::WriterInfo::width
int width
The width of the video (in pixels)
Definition: WriterBase.h:40
hw_en_on
int hw_en_on
Definition: FFmpegWriter.cpp:38
openshot::WriterInfo::acodec
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:52
openshot::WriterInfo::video_timebase
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:49
openshot::LAYOUT_MONO
@ LAYOUT_MONO
Definition: ChannelLayouts.h:30
AV_GET_CODEC_ATTRIBUTES
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
Definition: FFmpegUtilities.h:218
openshot::Settings::HW_EN_DEVICE_SET
int HW_EN_DEVICE_SET
Which GPU to use to encode (0 is the first)
Definition: Settings.h:83
openshot::WriterInfo::pixel_ratio
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: WriterBase.h:44
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::WriterInfo::top_field_first
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:51
if
if(!codec) codec
AV_SET_FILENAME
#define AV_SET_FILENAME(oc, f)
Definition: FFmpegUtilities.h:201
AV_GET_IMAGE_SIZE
#define AV_GET_IMAGE_SIZE(pix_fmt, width, height)
Definition: FFmpegUtilities.h:222
ZmqLogger.h
Header file for ZeroMQ-based Logger class.
mux_dict
AVDictionary * mux_dict
Definition: FFmpegWriter.cpp:35
openshot::ErrorEncodingVideo
Exception when encoding audio packet.
Definition: Exceptions.h:142
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::FFmpegWriter::AddSphericalMetadata
void AddSphericalMetadata(const std::string &projection="equirectangular", float yaw_deg=0.0f, float pitch_deg=0.0f, float roll_deg=0.0f)
Add spherical (360°) video metadata to the video stream.
Definition: FFmpegWriter.cpp:2369
openshot::FFmpegWriter::SetOption
void SetOption(openshot::StreamType stream, std::string name, std::string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
Definition: FFmpegWriter.cpp:335
openshot::Fraction::Reduce
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:65
AV_RESET_FRAME
#define AV_RESET_FRAME(av_frame)
Definition: FFmpegUtilities.h:206
SWR_CLOSE
#define SWR_CLOSE(ctx)
Definition: FFmpegUtilities.h:152
openshot::WriterInfo::channel_layout
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:56
openshot::OutOfMemory
Exception when memory could not be allocated.
Definition: Exceptions.h:348
SWR_INIT
#define SWR_INIT(ctx)
Definition: FFmpegUtilities.h:154
hw_en_av_pix_fmt
AVPixelFormat hw_en_av_pix_fmt
Definition: FFmpegWriter.cpp:40
openshot::Settings::Instance
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:23
openshot::VIDEO_STREAM
@ VIDEO_STREAM
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:29
openshot::WriterInfo::metadata
std::map< std::string, std::string > metadata
An optional map/dictionary of video & audio metadata.
Definition: WriterBase.h:59
path
path
Definition: FFmpegWriter.cpp:1469
Frame.h
Header file for Frame class.
ALLOC_CODEC_CTX
#define ALLOC_CODEC_CTX(ctx, codec, stream)
Definition: FFmpegUtilities.h:231
openshot::InvalidFile
Exception for files that can not be found or opened.
Definition: Exceptions.h:187
openshot::ZmqLogger::Instance
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:35
openshot::FFmpegWriter::WriteFrame
void WriteFrame(std::shared_ptr< openshot::Frame > frame)
Add a frame to the stack waiting to be encoded.
Definition: FFmpegWriter.cpp:667
openshot::ZmqLogger::AppendDebugMethod
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:178
openshot::WriterInfo::has_video
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:34
openshot::WriterInfo::has_audio
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:35
PIX_FMT_YUV420P
#define PIX_FMT_YUV420P
Definition: FFmpegUtilities.h:119
PIX_FMT_YUV444P
#define PIX_FMT_YUV444P
Definition: FFmpegUtilities.h:122
AV_GET_CODEC_TYPE
#define AV_GET_CODEC_TYPE(av_stream)
Definition: FFmpegUtilities.h:210
openshot::FFmpegWriter::Close
void Close()
Close the writer.
Definition: FFmpegWriter.cpp:957
openshot::FFmpegWriter::IsValidCodec
static bool IsValidCodec(std::string codec_name)
Determine if codec name is valid.
Definition: FFmpegWriter.cpp:599
openshot::WriterInfo::height
int height
The height of the video (in pixels)
Definition: WriterBase.h:39
AV_FREE_CONTEXT
#define AV_FREE_CONTEXT(av_context)
Definition: FFmpegUtilities.h:209
OpenMPUtilities.h
Header file for OpenMPUtilities (set some common macros)
SWR_FREE
#define SWR_FREE(ctx)
Definition: FFmpegUtilities.h:153
openshot::FFmpegWriter::WriteTrailer
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
Definition: FFmpegWriter.cpp:744
PIX_FMT_NONE
#define PIX_FMT_NONE
Definition: FFmpegUtilities.h:113
FF_AUDIO_NUM_PROCESSORS
#define FF_AUDIO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:25
openshot::ReaderBase
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:75
openshot::WriterInfo::interlaced_frame
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:50
FF_VIDEO_NUM_PROCESSORS
#define FF_VIDEO_NUM_PROCESSORS
Definition: OpenMPUtilities.h:24
openshot::WriterInfo::vcodec
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:46
openshot::WriterInfo::sample_rate
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:54
openshot::InvalidChannels
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:157
AV_OPTION_FIND
#define AV_OPTION_FIND(priv_data, name)
Definition: FFmpegUtilities.h:227
codec
codec
Definition: FFmpegWriter.cpp:1465
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
hw_en_av_device_type
AVHWDeviceType hw_en_av_device_type
Definition: FFmpegWriter.cpp:41
SWR_ALLOC
#define SWR_ALLOC()
Definition: FFmpegUtilities.h:151
openshot::FFmpegWriter::SetAudioOptions
void SetAudioOptions(bool has_audio, std::string codec, int sample_rate, int channels, openshot::ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Definition: FFmpegWriter.cpp:288
AV_REGISTER_ALL
#define AV_REGISTER_ALL
Definition: FFmpegUtilities.h:198
AV_OUTPUT_CONTEXT
#define AV_OUTPUT_CONTEXT(output_context, path)
Definition: FFmpegUtilities.h:225
hw_en_supported
int hw_en_supported
Definition: FFmpegWriter.cpp:39
openshot::StreamType
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:28
openshot::FFmpegWriter::PrepareStreams
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
Definition: FFmpegWriter.cpp:608
openshot::InvalidOptions
Exception when invalid encoding options are used.
Definition: Exceptions.h:232
openshot::WriterBase::info
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:76
openshot::NoStreamsFound
Exception when no streams are found in the file.
Definition: Exceptions.h:285
MY_INPUT_BUFFER_PADDING_SIZE
#define MY_INPUT_BUFFER_PADDING_SIZE
Definition: FFmpegUtilities.h:202
AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
Definition: FFmpegUtilities.h:83
opts
AVDictionary * opts
Definition: FFmpegWriter.cpp:1476
Exceptions.h
Header file for all Exception classes.
openshot::FFmpegWriter::WriteHeader
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
Definition: FFmpegWriter.cpp:625
Generated by   doxygen 1.8.17