OpenShot Library | libopenshot 0.2.7
CacheDisk.cpp
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Source file for CacheDisk class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9/* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31#include "CacheDisk.h"
32#include "Exceptions.h"
33#include "QtUtilities.h"
34#include <Qt>
35#include <QString>
36#include <QTextStream>
37
38using namespace std;
39using namespace openshot;
40
41// Default constructor, no max bytes
42CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
43 // Set cache type name
44 cache_type = "CacheDisk";
45 range_version = 0;
46 needs_range_processing = false;
47 frame_size_bytes = 0;
48 image_format = format;
49 image_quality = quality;
50 image_scale = scale;
51 max_bytes = 0;
52
53 // Init path directory
54 InitPath(cache_path);
55}
56
57// Constructor that sets the max bytes to cache
58CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
59 // Set cache type name
60 cache_type = "CacheDisk";
61 range_version = 0;
62 needs_range_processing = false;
63 frame_size_bytes = 0;
64 image_format = format;
65 image_quality = quality;
66 image_scale = scale;
67
68 // Init path directory
69 InitPath(cache_path);
70}
71
72// Initialize cache directory
73void CacheDisk::InitPath(std::string cache_path) {
74 QString qpath;
75
76 if (!cache_path.empty()) {
77 // Init QDir with cache directory
78 qpath = QString(cache_path.c_str());
79
80 } else {
81 // Init QDir with user's temp directory
82 qpath = QDir::tempPath() + QString("/preview-cache/");
83 }
84
85 // Init QDir with cache directory
86 path = QDir(qpath);
87
88 // Check if cache directory exists
89 if (!path.exists())
90 // Create
91 path.mkpath(qpath);
92}
93
94// Calculate ranges of frames
95void CacheDisk::CalculateRanges() {
96 // Only calculate when something has changed
97 if (needs_range_processing) {
98
99 // Create a scoped lock, to protect the cache from multiple threads
100 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
101
102 // Sort ordered frame #s, and calculate JSON ranges
103 std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
104
105 // Clear existing JSON variable
106 Json::Value ranges = Json::Value(Json::arrayValue);
107
108 // Increment range version
109 range_version++;
110
111 int64_t starting_frame = *ordered_frame_numbers.begin();
112 int64_t ending_frame = starting_frame;
113
114 // Loop through all known frames (in sequential order)
115 for (const auto frame_number : ordered_frame_numbers) {
116 if (frame_number - ending_frame > 1) {
117 // End of range detected
118 Json::Value range;
119
120 // Add JSON object with start/end attributes
121 // Use strings, since int64_ts are supported in JSON
122 range["start"] = std::to_string(starting_frame);
123 range["end"] = std::to_string(ending_frame);
124 ranges.append(range);
125
126 // Set new starting range
127 starting_frame = frame_number;
128 }
129
130 // Set current frame as end of range, and keep looping
131 ending_frame = frame_number;
132 }
133
134 // APPEND FINAL VALUE
135 Json::Value range;
136
137 // Add JSON object with start/end attributes
138 // Use strings, since int64_ts are supported in JSON
139 range["start"] = std::to_string(starting_frame);
140 range["end"] = std::to_string(ending_frame);
141 ranges.append(range);
142
143 // Cache range JSON as string
144 json_ranges = ranges.toStyledString();
145
146 // Reset needs_range_processing
147 needs_range_processing = false;
148 }
149}
150
151// Default destructor
153{
154 frames.clear();
155 frame_numbers.clear();
156 ordered_frame_numbers.clear();
157
158 // remove critical section
161}
162
163// Add a Frame to the cache
164void CacheDisk::Add(std::shared_ptr<Frame> frame)
165{
166 // Create a scoped lock, to protect the cache from multiple threads
167 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
168 int64_t frame_number = frame->number;
169
170 // Freshen frame if it already exists
171 if (frames.count(frame_number))
172 // Move frame to front of queue
173 MoveToFront(frame_number);
174
175 else
176 {
177 // Add frame to queue and map
178 frames[frame_number] = frame_number;
179 frame_numbers.push_front(frame_number);
180 ordered_frame_numbers.push_back(frame_number);
181 needs_range_processing = true;
182
183 // Save image to disk (if needed)
184 QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
185 frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
186 if (frame_size_bytes == 0) {
187 // Get compressed size of frame image (to correctly apply max size against)
188 QFile image_file(frame_path);
189 frame_size_bytes = image_file.size();
190 }
191
192 // Save audio data (if needed)
193 if (frame->has_audio_data) {
194 QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
195 QFile audio_file(audio_path);
196
197 if (audio_file.open(QIODevice::WriteOnly)) {
198 QTextStream audio_stream(&audio_file);
199 audio_stream << frame->SampleRate() << Qt::endl;
200 audio_stream << frame->GetAudioChannelsCount() << Qt::endl;
201 audio_stream << frame->GetAudioSamplesCount() << Qt::endl;
202 audio_stream << frame->ChannelsLayout() << Qt::endl;
203
204 // Loop through all samples
205 for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
206 {
207 // Get audio for this channel
208 float *samples = frame->GetAudioSamples(channel);
209 for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
210 audio_stream << samples[sample] << Qt::endl;
211 }
212
213 }
214
215 }
216
217 // Clean up old frames
218 CleanUp();
219 }
220}
221
222// Get a frame from the cache (or NULL shared_ptr if no frame is found)
223std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
224{
225 // Create a scoped lock, to protect the cache from multiple threads
226 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
227
228 // Does frame exists in cache?
229 if (frames.count(frame_number)) {
230 // Does frame exist on disk
231 QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
232 if (path.exists(frame_path)) {
233
234 // Load image file
235 auto image = std::make_shared<QImage>();
236 image->load(frame_path);
237
238 // Set pixel formatimage->
239 image = std::make_shared<QImage>(image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
240
241 // Create frame object
242 auto frame = std::make_shared<Frame>();
243 frame->number = frame_number;
244 frame->AddImage(image);
245
246 // Get audio data (if found)
247 QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
248 QFile audio_file(audio_path);
249 if (audio_file.exists()) {
250 // Open audio file
251 QTextStream in(&audio_file);
252 if (audio_file.open(QIODevice::ReadOnly)) {
253 int sample_rate = in.readLine().toInt();
254 int channels = in.readLine().toInt();
255 int sample_count = in.readLine().toInt();
256 int channel_layout = in.readLine().toInt();
257
258 // Set basic audio properties
259 frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
260
261 // Loop through audio samples and add to frame
262 int current_channel = 0;
263 int current_sample = 0;
264 float *channel_samples = new float[sample_count];
265 while (!in.atEnd()) {
266 // Add sample to channel array
267 channel_samples[current_sample] = in.readLine().toFloat();
268 current_sample++;
269
270 if (current_sample == sample_count) {
271 // Add audio to frame
272 frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
273
274 // Increment channel, and reset sample position
275 current_channel++;
276 current_sample = 0;
277 }
278
279 }
280 }
281 }
282
283 // return the Frame object
284 return frame;
285 }
286 }
287
288 // no Frame found
289 return std::shared_ptr<Frame>();
290}
291
292// Get the smallest frame number (or NULL shared_ptr if no frame is found)
293std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
294{
295 // Create a scoped lock, to protect the cache from multiple threads
296 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
297 std::shared_ptr<openshot::Frame> f;
298
299 // Loop through frame numbers
300 std::deque<int64_t>::iterator itr;
301 int64_t smallest_frame = -1;
302 for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
303 {
304 if (*itr < smallest_frame || smallest_frame == -1)
305 smallest_frame = *itr;
306 }
307
308 // Return frame
309 f = GetFrame(smallest_frame);
310
311 return f;
312}
313
314// Gets the maximum bytes value
316{
317 // Create a scoped lock, to protect the cache from multiple threads
318 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
319
320 int64_t total_bytes = 0;
321
322 // Loop through frames, and calculate total bytes
323 std::deque<int64_t>::reverse_iterator itr;
324 for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
325 total_bytes += frame_size_bytes;
326
327 return total_bytes;
328}
329
330// Remove a specific frame
331void CacheDisk::Remove(int64_t frame_number)
332{
333 Remove(frame_number, frame_number);
334}
335
336// Remove range of frames
337void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
338{
339 // Create a scoped lock, to protect the cache from multiple threads
340 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
341
342 // Loop through frame numbers
343 std::deque<int64_t>::iterator itr;
344 for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
345 {
346 //deque<int64_t>::iterator current = itr++;
347 if (*itr >= start_frame_number && *itr <= end_frame_number)
348 {
349 // erase frame number
350 itr = frame_numbers.erase(itr);
351 } else
352 itr++;
353 }
354
355 // Loop through ordered frame numbers
356 std::vector<int64_t>::iterator itr_ordered;
357 for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
358 {
359 if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
360 {
361 // erase frame number
362 frames.erase(*itr_ordered);
363
364 // Remove the image file (if it exists)
365 QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
366 QFile image_file(frame_path);
367 if (image_file.exists())
368 image_file.remove();
369
370 // Remove audio file (if it exists)
371 QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
372 QFile audio_file(audio_path);
373 if (audio_file.exists())
374 audio_file.remove();
375
376 itr_ordered = ordered_frame_numbers.erase(itr_ordered);
377 } else
378 itr_ordered++;
379 }
380
381 // Needs range processing (since cache has changed)
382 needs_range_processing = true;
383}
384
385// Move frame to front of queue (so it lasts longer)
386void CacheDisk::MoveToFront(int64_t frame_number)
387{
388 // Does frame exists in cache?
389 if (frames.count(frame_number))
390 {
391 // Create a scoped lock, to protect the cache from multiple threads
392 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
393
394 // Loop through frame numbers
395 std::deque<int64_t>::iterator itr;
396 for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
397 {
398 if (*itr == frame_number)
399 {
400 // erase frame number
401 frame_numbers.erase(itr);
402
403 // add frame number to 'front' of queue
404 frame_numbers.push_front(frame_number);
405 break;
406 }
407 }
408 }
409}
410
411// Clear the cache of all frames
413{
414 // Create a scoped lock, to protect the cache from multiple threads
415 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
416
417 // Clear all containers
418 frames.clear();
419 frame_numbers.clear();
420 ordered_frame_numbers.clear();
421 needs_range_processing = true;
422 frame_size_bytes = 0;
423
424 // Delete cache directory, and recreate it
425 QString current_path = path.path();
426 path.removeRecursively();
427
428 // Re-init folder
429 InitPath(current_path.toStdString());
430}
431
432// Count the frames in the queue
434{
435 // Create a scoped lock, to protect the cache from multiple threads
436 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
437
438 // Return the number of frames in the cache
439 return frames.size();
440}
441
442// Clean up cached frames that exceed the number in our max_bytes variable
443void CacheDisk::CleanUp()
444{
445 // Do we auto clean up?
446 if (max_bytes > 0)
447 {
448 // Create a scoped lock, to protect the cache from multiple threads
449 const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
450
451 while (GetBytes() > max_bytes && frame_numbers.size() > 20)
452 {
453 // Get the oldest frame number.
454 int64_t frame_to_remove = frame_numbers.back();
455
456 // Remove frame_number and frame
457 Remove(frame_to_remove);
458 }
459 }
460}
461
462// Generate JSON string of this object
463std::string CacheDisk::Json() {
464
465 // Return formatted string
466 return JsonValue().toStyledString();
467}
468
469// Generate Json::Value for this object
470Json::Value CacheDisk::JsonValue() {
471
472 // Process range data (if anything has changed)
473 CalculateRanges();
474
475 // Create root json object
476 Json::Value root = CacheBase::JsonValue(); // get parent properties
477 root["type"] = cache_type;
478 root["path"] = path.path().toStdString();
479
480 Json::Value version;
481 std::stringstream range_version_str;
482 range_version_str << range_version;
483 root["version"] = range_version_str.str();
484
485 // Parse and append range data (if any)
486 // Parse and append range data (if any)
487 try {
488 const Json::Value ranges = openshot::stringToJson(json_ranges);
489 root["ranges"] = ranges;
490 } catch (...) { }
491
492 // return JsonValue
493 return root;
494}
495
496// Load JSON string into this object
497void CacheDisk::SetJson(const std::string value) {
498
499 // Parse JSON string into JSON objects
500 try
501 {
502 const Json::Value root = openshot::stringToJson(value);
503 // Set all values that match
504 SetJsonValue(root);
505 }
506 catch (const std::exception& e)
507 {
508 // Error parsing JSON (or missing keys)
509 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
510 }
511}
512
513// Load Json::Value into this object
514void CacheDisk::SetJsonValue(const Json::Value root) {
515
516 // Close timeline before we do anything (this also removes all open and closing clips)
517 Clear();
518
519 // Set parent data
521
522 if (!root["type"].isNull())
523 cache_type = root["type"].asString();
524 if (!root["path"].isNull())
525 // Update duration of timeline
526 InitPath(root["path"].asString());
527}
Header file for CacheDisk class.
Header file for all Exception classes.
Header file for QtUtilities (compatibiity overlay)
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:49
virtual Json::Value JsonValue()=0
Generate Json::Value for this object.
Definition: CacheBase.cpp:57
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:51
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: CacheBase.cpp:70
juce::CriticalSection * cacheCriticalSection
Section lock for multiple threads.
Definition: CacheBase.h:55
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:52
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:223
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:42
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:386
std::string Json()
Generate JSON string of this object.
Definition: CacheDisk.cpp:463
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:164
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:293
Json::Value JsonValue()
Generate Json::Value for this object.
Definition: CacheDisk.cpp:470
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:412
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CacheDisk.cpp:514
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:497
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:315
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:433
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:331
Exception for invalid JSON.
Definition: Exceptions.h:206
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34