OpenShot Library | libopenshot 0.2.7
Frame.h
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Header file for Frame class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9/* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31#ifndef OPENSHOT_FRAME_H
32#define OPENSHOT_FRAME_H
33
34#ifdef USE_OPENCV
35 #define int64 opencv_broken_int
36 #define uint64 opencv_broken_uint
37 #include <opencv2/imgproc/imgproc.hpp>
38 #undef uint64
39 #undef int64
40#endif
41
42
43#include <iomanip>
44#include <sstream>
45#include <queue>
46#include <QApplication>
47#include <QImage>
48#include <memory>
49#include <unistd.h>
50#include "ZmqLogger.h"
51#include "ChannelLayouts.h"
52#include "AudioBufferSource.h"
53#include "AudioResampler.h"
54#include "Fraction.h"
55#include <OpenShotAudio.h>
56#ifdef USE_IMAGEMAGICK
57 #include "MagickUtilities.h"
58#endif
59
60namespace openshot
61{
62 /**
63 * @brief This class represents a single frame of video (i.e. image & audio data)
64 *
65 * FileReaders (such as FFmpegReader) use instances of this class to store the individual frames of video,
66 * which include both the image data (i.e. pixels) and audio samples. An openshot::Frame also has many debug
67 * methods, such as the ability to display the image (using X11), play the audio samples (using JUCE), or
68 * display the audio waveform as an image.
69 *
70 * FileWriters (such as FFmpegWriter) use instances of this class to create new video files, image files, or
71 * video streams. So, think of these openshot::Frame instances as the smallest unit of work in a video
72 * editor.
73 *
74 * There are many ways to create an instance of an openshot::Frame:
75 * @code
76 *
77 * // Most basic: a blank frame (all default values)
78 * Frame();
79 *
80 * // Image only settings
81 * Frame(1, // Frame number
82 * 720, // Width of image
83 * 480, // Height of image
84 * "#000000" // HTML color code of background color
85 * );
86 *
87 * // Audio only
88 * Frame(number, // Frame number
89 * 44100, // Sample rate of audio stream
90 * 2 // Number of audio channels
91 * );
92 *
93 * // Image and Audio settings (user defines all key settings)
94 * Frame(number, // Frame number
95 * 720, // Width of image
96 * 480, // Height of image
97 * "#000000" // HTML color code of background color
98 * 44100, // Sample rate of audio stream
99 * 2 // Number of audio channels
100 * );
101 *
102 * // Some methods require a shared pointer to an openshot::Frame object.
103 * auto f = std::make_shared<openshot::Frame>(1, 720, 480, "#000000", 44100, 2);
104 *
105 * @endcode
106 */
107 class Frame
108 {
109 private:
110 std::shared_ptr<QImage> image;
111 std::shared_ptr<QImage> wave_image;
112
113 std::shared_ptr<QApplication> previewApp;
114 juce::CriticalSection addingImageSection;
115 juce::CriticalSection addingAudioSection;
116 const unsigned char *qbuffer;
117 openshot::Fraction pixel_ratio;
118 int channels;
119 ChannelLayout channel_layout;
120 int width;
121 int height;
122 int sample_rate;
123 std::string color;
124 int64_t max_audio_sample; ///< The max audio sample count added to this frame
125
126#ifdef USE_OPENCV
127 cv::Mat imagecv; ///< OpenCV image. It will always be in BGR format
128#endif
129
130 /// Constrain a color value from 0 to 255
131 int constrain(int color_value);
132
133 public:
134 std::shared_ptr<juce::AudioSampleBuffer> audio;
135 int64_t number; ///< This is the frame number (starting at 1)
136 bool has_audio_data; ///< This frame has been loaded with audio data
137 bool has_image_data; ///< This frame has been loaded with pixel data
138
139
140 /// Constructor - blank frame
141 Frame();
142
143 /// Constructor - image only
144 Frame(int64_t number, int width, int height, std::string color);
145
146 /// Constructor - audio only
147 Frame(int64_t number, int samples, int channels);
148
149 /// Constructor - image & audio
150 Frame(int64_t number, int width, int height, std::string color, int samples, int channels);
151
152 /// Copy constructor
153 Frame ( const Frame &other );
154
155 /// Assignment operator
156 Frame& operator= (const Frame& other);
157
158 /// Destructor
159 virtual ~Frame();
160
161 /// Add (or replace) pixel data to the frame (based on a solid color)
162 void AddColor(int new_width, int new_height, std::string new_color);
163
164 /// Add (or replace) pixel data to the frame
165 void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_);
166
167 /// Add (or replace) pixel data to the frame
168 void AddImage(std::shared_ptr<QImage> new_image);
169
170 /// Add (or replace) pixel data to the frame (for only the odd or even lines)
171 void AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines);
172
173#ifdef USE_IMAGEMAGICK
174 /// Add (or replace) pixel data to the frame from an ImageMagick Image
175 void AddMagickImage(std::shared_ptr<Magick::Image> new_image);
176#endif
177
178 /// Add audio samples to a specific channel
179 void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource);
180
181 /// Add audio silence
182 void AddAudioSilence(int numSamples);
183
184 /// Apply gain ramp (i.e. fading volume)
185 void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain);
186
187 /// Channel Layout of audio samples. A frame needs to keep track of this, since Writers do not always
188 /// know the original channel layout of a frame's audio samples (i.e. mono, stereo, 5 point surround, etc...)
190
191 // Set the channel layout of audio samples (i.e. mono, stereo, 5 point surround, etc...)
192 void ChannelsLayout(openshot::ChannelLayout new_channel_layout) { channel_layout = new_channel_layout; };
193
194 /// Clean up buffer after QImage is deleted
195 static void cleanUpBuffer(void *info);
196
197 /// Clear the waveform image (and deallocate its memory)
198 void ClearWaveform();
199
200 /// Copy data and pointers from another Frame instance
201 void DeepCopy(const Frame& other);
202
203 /// Display the frame image to the screen (primarily used for debugging reasons)
204 void Display();
205
206 /// Display the wave form
207 void DisplayWaveform();
208
209 /// Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
210 float GetAudioSample(int channel, int sample, int magnitude_range);
211
212 /// Get an array of sample data
213 float* GetAudioSamples(int channel);
214
215 /// Get an array of sample data (all channels interleaved together), using any sample rate
216 float* GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
217
218 // Get a planar array of sample data, using any sample rate
219 float* GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler* resampler, int* sample_count);
220
221 /// Get number of audio channels
223
224 /// Get number of audio samples
226
227 juce::AudioSampleBuffer *GetAudioSampleBuffer();
228
229 /// Get the size in bytes of this frame (rough estimate)
230 int64_t GetBytes();
231
232 /// Get pointer to Qt QImage image object
233 std::shared_ptr<QImage> GetImage();
234
235#ifdef USE_IMAGEMAGICK
236 /// Get pointer to ImageMagick image object
237 std::shared_ptr<Magick::Image> GetMagickImage();
238#endif
239
240 /// Set Pixel Aspect Ratio
241 openshot::Fraction GetPixelRatio() { return pixel_ratio; };
242
243 /// Get pixel data (as packets)
244 const unsigned char* GetPixels();
245
246 /// Get pixel data (for only a single scan-line)
247 const unsigned char* GetPixels(int row);
248
249 /// Check a specific pixel color value (returns True/False)
250 bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold);
251
252 /// Get height of image
253 int GetHeight();
254
255 /// Calculate the # of samples per video frame (for the current frame number)
256 int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels);
257
258 /// Calculate the # of samples per video frame (for a specific frame number and frame rate)
259 static int GetSamplesPerFrame(int64_t frame_number, openshot::Fraction fps, int sample_rate, int channels);
260
261 /// Get an audio waveform image
262 std::shared_ptr<QImage> GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha);
263
264 /// Get an audio waveform image pixels
265 const unsigned char* GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha);
266
267 /// Get height of image
268 int GetWidth();
269
270 /// Resize audio container to hold more (or less) samples and channels
271 void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout);
272
273 /// Get the original sample rate of this frame's audio data
274 int SampleRate();
275
276 /// Set the original sample rate of this frame's audio data
277 void SampleRate(int orig_sample_rate) { sample_rate = orig_sample_rate; };
278
279 /// Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM
280 void Save(std::string path, float scale, std::string format="PNG", int quality=100);
281
282 /// Set frame number
283 void SetFrameNumber(int64_t number);
284
285 /// Set Pixel Aspect Ratio
286 void SetPixelRatio(int num, int den);
287
288 /// Thumbnail the frame image with tons of options to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG).
289 /// This method allows for masks, overlays, background color, and much more accurate resizing (including padding and centering)
290 void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
291 std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0);
292
293 /// Play audio samples for this frame
294 void Play();
295
296#ifdef USE_OPENCV
297 /// Convert Qimage to Mat
298 cv::Mat Qimage2mat( std::shared_ptr<QImage>& qimage);
299
300 /// Convert OpenCV Mat to QImage
301 std::shared_ptr<QImage> Mat2Qimage(cv::Mat img);
302
303 /// Get pointer to OpenCV Mat image object
304 cv::Mat GetImageCV();
305
306 /// Set pointer to OpenCV image object
307 void SetImageCV(cv::Mat _image);
308#endif
309 };
310
311}
312
313#endif
Header file for AudioBufferSource class.
Header file for AudioResampler class.
Header file for ChannelLayout class.
Header file for Fraction class.
Header file for MagickUtilities (IM6/IM7 compatibility overlay)
Header file for ZeroMQ-based Logger class.
This class is used to resample audio data for many sequential frames.
This class represents a fraction.
Definition: Fraction.h:48
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:108
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:88
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1085
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:938
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:737
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:371
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:453
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:172
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:961
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:567
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:130
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:416
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:913
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:137
int GetWidth()
Get height of image.
Definition: Frame.cpp:548
openshot::Fraction GetPixelRatio()
Set Pixel Aspect Ratio.
Definition: Frame.h:241
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:554
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:849
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:97
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:325
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:986
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:924
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:255
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:1014
void SampleRate(int orig_sample_rate)
Set the original sample rate of this frame's audio data.
Definition: Frame.h:277
std::shared_ptr< juce::AudioSampleBuffer > audio
Definition: Frame.h:134
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:136
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:438
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:560
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1096
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:602
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:272
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:432
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:477
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:318
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:305
virtual ~Frame()
Destructor.
Definition: Frame.cpp:120
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:900
void ChannelsLayout(openshot::ChannelLayout new_channel_layout)
Definition: Frame.h:192
Frame()
Constructor - blank frame.
Definition: Frame.cpp:69
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:756
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:891
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:426
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:863
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:262
int GetHeight()
Get height of image.
Definition: Frame.cpp:542
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:499
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:506
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:135
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:952
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...