FFmpegfs Fuse Multi Media Filesystem 2.16
transcode.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 2006-2008 David Collett
3 * Copyright (C) 2008-2013 K. Henriksson
4 * Copyright (C) 2017-2024 FFmpeg support by Norbert Schlia (nschlia@oblivion-software.de)
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
19 *
20 * On Debian systems, the complete text of the GNU General Public License
21 * Version 3 can be found in `/usr/share/common-licenses/GPL-3'.
22 */
23
36#include "transcode.h"
37#include "ffmpegfs.h"
38#include "ffmpeg_transcoder.h"
39#include "buffer.h"
40#include "cache.h"
41#include "logging.h"
42#include "cache_entry.h"
43#include "thread_pool.h"
44
45#include <unistd.h>
46#include <atomic>
47
48const int GRANULARITY = 250;
49const int FRAME_TIMEOUT = 60;
54typedef struct THREAD_DATA
55{
57 std::condition_variable m_thread_running_cond;
58 std::atomic_bool m_thread_running_lock_guard;
62
63static std::unique_ptr<Cache> cache;
64static std::atomic_bool thread_exit;
66static bool transcode(std::shared_ptr<THREAD_DATA> thread_data, Cache_Entry *cache_entry, FFmpeg_Transcoder & transcoder, bool *timeout);
67static int transcoder_thread(std::shared_ptr<THREAD_DATA> thread_data);
68static bool transcode_until(Cache_Entry* cache_entry, size_t offset, size_t len, uint32_t segment_no);
69static int transcode_finish(Cache_Entry* cache_entry, FFmpeg_Transcoder & transcoder);
70
81static bool transcode_until(Cache_Entry* cache_entry, size_t offset, size_t len, uint32_t segment_no)
82{
83 size_t end = offset + len; // Cast OK: offset will never be < 0.
84 bool success = true;
85
86 if (cache_entry->is_finished() || cache_entry->m_buffer->is_segment_finished(segment_no) || cache_entry->m_buffer->tell(segment_no) >= end)
87 {
88 return true;
89 }
90
91 try
92 {
93 // Wait until decoder thread has reached the desired position
94 if (cache_entry->m_is_decoding)
95 {
96 bool reported = false;
97 while (!(cache_entry->is_finished() || cache_entry->m_buffer->is_segment_finished(segment_no) || cache_entry->m_buffer->tell(segment_no) >= end) && !cache_entry->m_cache_info.m_error)
98 {
99 if (fuse_interrupted())
100 {
101 Logging::info(cache_entry->virtname(), "The client has gone away.");
102 errno = 0; // No error
103 break;
104 }
105
106 if (thread_exit)
107 {
108 Logging::warning(cache_entry->virtname(), "Thread exit was received.");
109 errno = EINTR;
110 throw false;
111 }
112
113 if (!reported)
114 {
115 if (!segment_no)
116 {
117 Logging::trace(cache_entry->virtname(), "Cache miss at offset %1 with length %2.", offset, len);
118 }
119 else
120 {
121 Logging::trace(cache_entry->virtname(), "Cache miss at offset %1 with length %2 for segment no. %3.", offset, len, segment_no);
122 }
123 reported = true;
124 }
125 mssleep(250);
126 }
127
128 if (reported)
129 {
130 if (!segment_no)
131 {
132 Logging::trace(cache_entry->virtname(), "Cache hit at offset %1 with length %2.", offset, len);
133 }
134 else
135 {
136 Logging::trace(cache_entry->virtname(), "Cache hit at offset %1 with length %2 for segment no. %3.", offset, len, segment_no);
137 }
138 }
139 success = !cache_entry->m_cache_info.m_error;
140 }
141 }
142 catch (bool _success)
143 {
144 success = _success;
145 }
146
147 return success;
148}
149
156static int transcode_finish(Cache_Entry* cache_entry, FFmpeg_Transcoder & transcoder)
157{
158 int res = transcoder.encode_finish();
159 if (res < 0)
160 {
161 return res;
162 }
163
164 // Check encoded buffer size. Does not affect HLS segments.
165 cache_entry->m_cache_info.m_duration = transcoder.duration();
166 cache_entry->m_cache_info.m_encoded_filesize = cache_entry->m_buffer->buffer_watermark();
167 cache_entry->m_cache_info.m_video_frame_count = transcoder.video_frame_count();
168 cache_entry->m_cache_info.m_segment_count = transcoder.segment_count();
170 cache_entry->m_is_decoding = false;
171 cache_entry->m_cache_info.m_errno = 0;
172 cache_entry->m_cache_info.m_averror = 0;
173
174 Logging::debug(transcoder.virtname(), "Finishing file.");
175
176 if (!cache_entry->m_buffer->reserve(cache_entry->m_cache_info.m_encoded_filesize))
177 {
178 Logging::debug(transcoder.virtname(), "Unable to truncate the buffer.");
179 }
180
181 if (!transcoder.is_multiformat())
182 {
183 Logging::debug(transcoder.virtname(), "Predicted size: %1 Final: %2 Diff: %3 (%4%).",
185 format_size_ex(cache_entry->m_cache_info.m_encoded_filesize).c_str(),
187 ((static_cast<double>(cache_entry->m_cache_info.m_encoded_filesize) * 1000 / (static_cast<double>(cache_entry->m_cache_info.m_predicted_filesize) + 1)) + 5) / 10);
188 }
189
190 cache_entry->flush();
191
192 return 0;
193}
194
195void transcoder_cache_path(std::string * path)
196{
197 if (params.m_cachepath.size())
198 {
199 *path = params.m_cachepath;
200 }
201 else
202 {
203 if (geteuid() == 0)
204 {
205 // Running as root
206 *path = "/var/cache";
207 }
208 else
209 {
210 // Running as regular user, put cache in home dir
211 if (const char* cache_home = std::getenv("XDG_CACHE_HOME"))
212 {
213 *path = cache_home;
214 }
215 else
216 {
217 expand_path(path, "~/.cache");
218 }
219 }
220 }
221
222 append_sep(path);
223
224 *path += PACKAGE;
225
226 append_sep(path);
227}
228
230{
231 if (cache == nullptr)
232 {
233 Logging::debug(nullptr, "Creating new media file cache.");
234 cache = std::make_unique<Cache>();
235 if (cache == nullptr)
236 {
237 Logging::error(nullptr, "Unable to create new media file cache. Out of memory.");
238 std::fprintf(stderr, "ERROR: Creating new media file cache. Out of memory.\n");
239 return false;
240 }
241
242 if (!cache->load_index())
243 {
244 std::fprintf(stderr, "ERROR: Creating media file cache failed.\n");
245 return false;
246 }
247 }
248 return true;
249}
250
252{
253 if (cache != nullptr)
254 {
255 cache.reset();
256
257 Logging::debug(nullptr, "Deleting media file cache.");
258 }
259}
260
261bool transcoder_cached_filesize(LPVIRTUALFILE virtualfile, struct stat *stbuf)
262{
263 Cache_Entry* cache_entry = cache->openio(virtualfile);
264 if (cache_entry == nullptr)
265 {
266 return false;
267 }
268
269 size_t encoded_filesize = cache_entry->m_cache_info.m_encoded_filesize;
270
271 if (!encoded_filesize)
272 {
273 // If not yet encoded, return predicted file size
274 encoded_filesize = cache_entry->m_cache_info.m_predicted_filesize;
275 }
276
277 if (encoded_filesize)
278 {
279 stat_set_size(stbuf, encoded_filesize);
280 return true;
281 }
282 else
283 {
284 return false;
285 }
286}
287
288bool transcoder_set_filesize(LPVIRTUALFILE virtualfile, int64_t duration, BITRATE audio_bit_rate, int channels, int sample_rate, AVSampleFormat sample_format, BITRATE video_bit_rate, int width, int height, bool interleaved, const AVRational &framerate)
289{
290 Cache_Entry* cache_entry = cache->openio(virtualfile);
291 if (cache_entry == nullptr)
292 {
293 Logging::error(nullptr, "Out of memory getting file size.");
294 return false;
295 }
296
297 const FFmpegfs_Format *current_format = params.current_format(virtualfile);
298 if (current_format == nullptr)
299 {
300 Logging::error(cache_entry->virtname(), "Internal error getting file size.");
301 return false;
302 }
303
304 size_t filesize = 0;
305
306 if (!FFmpeg_Transcoder::audio_size(&filesize, current_format->audio_codec(), audio_bit_rate, duration, channels, sample_rate, sample_format))
307 {
308 Logging::warning(cache_entry->virtname(), "Unsupported audio codec '%1' for format %2.", get_codec_name(current_format->audio_codec()), current_format->desttype().c_str());
309 }
310
311 if (!FFmpeg_Transcoder::video_size(&filesize, current_format->video_codec(), video_bit_rate, duration, width, height, interleaved, framerate))
312 {
313 Logging::warning(cache_entry->virtname(), "Unsupported video codec '%1' for format %2.", get_codec_name(current_format->video_codec()), current_format->desttype().c_str());
314 }
315
316 if (!FFmpeg_Transcoder::total_overhead(&filesize, current_format->filetype()))
317 {
318 Logging::warning(cache_entry->virtname(), "Unsupported file type '%1' for format %2.", get_filetype_text(current_format->filetype()).c_str(), current_format->desttype().c_str());
319 }
320
321 cache_entry->m_cache_info.m_predicted_filesize = virtualfile->m_predicted_size = filesize;
322
323 Logging::trace(cache_entry->virtname(), "Predicted transcoded size of %1.", format_size_ex(cache_entry->m_cache_info.m_predicted_filesize).c_str());
324
325 return true;
326}
327
329{
330 FFmpeg_Transcoder transcoder;
331 bool success = false;
332
333 if (transcoder.open_input_file(virtualfile) >= 0)
334 {
335 if (cache_entry != nullptr)
336 {
337 cache_entry->m_cache_info.m_predicted_filesize = transcoder.predicted_filesize();
338 cache_entry->m_cache_info.m_video_frame_count = transcoder.video_frame_count();
339 cache_entry->m_cache_info.m_segment_count = transcoder.segment_count();
340 cache_entry->m_cache_info.m_duration = transcoder.duration();
341 }
342
343 Logging::trace(transcoder.filename(), "Predicted transcoded size of %1.", format_size_ex(transcoder.predicted_filesize()).c_str());
344
345 transcoder.closeio();
346
347 success = true;
348 }
349
350 return success;
351}
352
353Cache_Entry* transcoder_new(LPVIRTUALFILE virtualfile, bool begin_transcode)
354{
355 // Allocate transcoder structure
356 Cache_Entry* cache_entry = cache->openio(virtualfile);
357 if (cache_entry == nullptr)
358 {
359 return nullptr;
360 }
361
362 Logging::trace(cache_entry->filename(), "Creating transcoder object.");
363
364 try
365 {
366 cache_entry->lock();
367
368 if (!cache_entry->openio(begin_transcode))
369 {
370 throw static_cast<int>(errno);
371 }
372
374 {
375 // Disable cache
376 cache_entry->clear();
377 }
378 else if (!cache_entry->m_is_decoding && cache_entry->outdated())
379 {
380 cache_entry->clear();
381 }
382
383 if (cache_entry->m_cache_info.m_duration)
384 {
385 virtualfile->m_duration = cache_entry->m_cache_info.m_duration;
386 }
387 if (cache_entry->m_cache_info.m_predicted_filesize)
388 {
389 virtualfile->m_predicted_size = cache_entry->m_cache_info.m_predicted_filesize;
390 }
391 if (cache_entry->m_cache_info.m_video_frame_count)
392 {
393 virtualfile->m_video_frame_count = cache_entry->m_cache_info.m_video_frame_count;
394 }
395
396 if (!cache_entry->m_is_decoding && !cache_entry->is_finished_success())
397 {
398 if (begin_transcode)
399 {
400 Logging::debug(cache_entry->filename(), "Starting transcoder thread.");
401
402 // Clear cache to remove any older remains
403 cache_entry->clear();
404
405 // Must decode the file, otherwise simply use cache
406 cache_entry->m_is_decoding = true;
407
408 std::shared_ptr<THREAD_DATA> thread_data = std::make_unique<THREAD_DATA>();
409
410 thread_data->m_initialised = false;
411 thread_data->m_cache_entry = cache_entry;
412 thread_data->m_thread_running_lock_guard = false;
413
414 {
415 std::unique_lock<std::mutex> lock_thread_running_mutex(thread_data->m_thread_running_mutex);
416
417 tp->schedule_thread(std::bind(&transcoder_thread, thread_data));
418
419 // Let decoder get into gear before returning from open
420 while (!thread_data->m_thread_running_lock_guard)
421 {
422 thread_data->m_thread_running_cond.wait(lock_thread_running_mutex);
423 }
424 }
425
426 if (cache_entry->m_cache_info.m_error)
427 {
428 int ret;
429
430 Logging::trace(cache_entry->filename(), "Transcoder error!");
431
432 ret = cache_entry->m_cache_info.m_errno;
433 if (!ret)
434 {
435 ret = EIO; // Must return something, be it a simple I/O error...
436 }
437 throw ret;
438 }
439
440 Logging::debug(cache_entry->filename(), "Transcoder thread is running.");
441 }
442 else if (!cache_entry->m_cache_info.m_predicted_filesize)
443 {
444 if (!transcoder_predict_filesize(virtualfile, cache_entry))
445 {
446 throw static_cast<int>(errno);
447 }
448 }
449 }
450 else if (begin_transcode)
451 {
452 Logging::trace(cache_entry->virtname(), "Reading file from cache.");
453 }
454
455 cache_entry->unlock();
456 }
457 catch (int orgerrno)
458 {
459 cache_entry->m_is_decoding = false;
460 cache_entry->unlock();
461 cache->closeio(&cache_entry, CACHE_CLOSE_DELETE);
462 cache_entry = nullptr; // Make sure to return NULL here even if the cache could not be deleted now (still in use)
463 errno = orgerrno; // Restore last errno
464 }
465
466 return cache_entry;
467}
468
469bool transcoder_read(Cache_Entry* cache_entry, char* buff, size_t offset, size_t len, int * bytes_read, uint32_t segment_no)
470{
471 bool success = true;
472
473 if (!segment_no)
474 {
475 Logging::trace(cache_entry->virtname(), "Reading %1 bytes from offset %2 to %3.", len, offset, len + offset);
476 }
477 else
478 {
479 Logging::trace(cache_entry->virtname(), "Reading %1 bytes from offset %2 to %3 for segment no. %4.", len, offset, len + offset, segment_no);
480 }
481
482 // Store access time
483 cache_entry->update_access();
484
485 // Update read counter
486 cache_entry->update_read_count();
487
488 try
489 {
490 // Try to read requested segment, stack a seek to if if this fails.
491 // No seek if not HLS (segment_no) and not required if < MIN_SEGMENT
492 if (segment_no && !cache_entry->m_buffer->segment_exists(segment_no))
493 {
494 cache_entry->m_seek_to_no = segment_no;
495 }
496
497 // Open for reading if necessary
498 if (!cache_entry->m_buffer->open_file(segment_no, CACHE_FLAG_RO))
499 {
500 throw false;
501 }
502
503 if (!cache_entry->is_finished_success())
504 {
505 switch (params.current_format(cache_entry->virtualfile())->filetype())
506 {
507 case FILETYPE::MP3:
508 {
509 // If we are encoding to MP3 and the requested data overlaps the ID3v1 tag
510 // at the end of the file, do not encode data first up to that position.
511 // This optimises the case where applications read the end of the file
512 // first to read the ID3v1 tag.
513 if ((offset > cache_entry->m_buffer->tell(segment_no)) &&
514 (offset + len >= (cache_entry->size() - ID3V1_TAG_LENGTH)))
515 {
516 // Stuff buffer with garbage, apps won't try to play that chunk anyway.
517 std::memset(buff, 0xFF, len);
518 if (cache_entry->size() - offset == ID3V1_TAG_LENGTH)
519 {
520 std::memcpy(buff, &cache_entry->m_id3v1, std::min(len, ID3V1_TAG_LENGTH));
521 }
522
523 errno = 0;
524
525 throw true; // OK
526 }
527 break;
528 }
529 default:
530 {
531 break;
532 }
533 }
534
535 // Windows seems to access the files on Samba drives starting at the last 64K segment simply when
536 // the file is opened. Setting win_smb_fix=1 will ignore these attempts (not decode the file up
537 // to this point).
538 // Access will only be ignored if occurring at the second access.
539 if (params.m_win_smb_fix && cache_entry->read_count() == 2)
540 {
541 if ((offset > cache_entry->m_buffer->tell(segment_no)) &&
542 (len <= 65536) &&
543 check_ignore(cache_entry->size(), offset) &&
544 ((offset + len) > (cache_entry->size())))
545 {
546 Logging::warning(cache_entry->virtname(), "Ignoring Windows' groundless access to the last 8K boundary of the file.");
547
548 errno = 0;
549 *bytes_read = 0; // We've read nothing
550 len = 0;
551
552 throw true; // OK
553 }
554 }
555 }
556
557 success = transcode_until(cache_entry, offset, len, segment_no);
558
559 if (!success)
560 {
561 errno = cache_entry->m_cache_info.m_errno ? cache_entry->m_cache_info.m_errno : EIO;
562 throw false;
563 }
564
565 // truncate if we didn't actually get len
566 if (cache_entry->m_buffer->buffer_watermark(segment_no) < offset)
567 {
568 len = 0;
569 }
570 else if (cache_entry->m_buffer->buffer_watermark(segment_no) < offset + len)
571 {
572 len = cache_entry->m_buffer->buffer_watermark(segment_no) - offset;
573 }
574
575 if (len)
576 {
577 if (!cache_entry->m_buffer->copy(reinterpret_cast<uint8_t*>(buff), offset, len, segment_no))
578 {
579 len = 0;
580 // We already capped len to not overread the buffer, so it is an error if we end here.
581 throw false;
582 }
583
584 if (cache_entry->m_cache_info.m_error)
585 {
586 errno = cache_entry->m_cache_info.m_errno ? cache_entry->m_cache_info.m_errno : EIO;
587 throw false;
588 }
589 }
590
591 errno = 0;
592 }
593 catch (bool _success)
594 {
595 success = _success;
596 }
597
598 *bytes_read = static_cast<int>(len);
599
600 return success;
601}
602
603bool transcoder_read_frame(Cache_Entry* cache_entry, char* buff, size_t offset, size_t len, uint32_t frame_no, int * bytes_read, LPVIRTUALFILE virtualfile)
604{
605 bool success = false;
606
607 Logging::trace(cache_entry->virtname(), "Reading %1 bytes from offset %2 to %3 for frame no. %4.", len, offset, len + offset, frame_no);
608
609 // Store access time
610 cache_entry->update_access();
611
612 // Update read counter
613 cache_entry->update_read_count();
614
615 try
616 {
617 // Open for reading if necessary
618 if (!cache_entry->m_buffer->open_file(0, CACHE_FLAG_RO))
619 {
620 throw false;
621 }
622
623 std::vector<uint8_t> data;
624
625 // Wait until decoder thread has the requested frame available
626 if (cache_entry->m_is_decoding || cache_entry->m_suspend_timeout)
627 {
628 // Try to read requested frame, stack a seek to if if this fails.
629 if (!cache_entry->m_buffer->read_frame(&data, frame_no))
630 {
631 bool reported = false;
632
633 cache_entry->m_seek_to_no = frame_no;
634
635 int retries = TOTAL_RETRIES;
636 while (!cache_entry->m_buffer->read_frame(&data, frame_no) && !thread_exit)
637 {
638 if (errno != EAGAIN)
639 {
640 Logging::error(cache_entry->virtname(), "Reading image frame no. %1: (%2) %3", frame_no, errno, strerror(errno));
641 throw false;
642 }
643
644 if (!cache_entry->m_suspend_timeout)
645 {
646 if (errno == EAGAIN && !--retries)
647 {
648 errno = ETIMEDOUT;
649 Logging::error(cache_entry->virtname(), "Timeout reading image frame no. %1: (%2) %3", frame_no, errno, strerror(errno));
650 throw false;
651 }
652 }
653 else
654 {
655 retries = TOTAL_RETRIES;
656 }
657
658 if (thread_exit)
659 {
660 Logging::warning(cache_entry->virtname(), "Thread exit was received.");
661 errno = EINTR;
662 throw false;
663 }
664
665 if (!reported)
666 {
667 Logging::trace(cache_entry->virtname(), "Frame no. %1: Cache miss at offset %<11zu>2 (length %<6u>3).", frame_no, offset, len);
668 reported = true;
669 }
670
672 }
673 }
674
675 Logging::trace(cache_entry->virtname(), "Frame no. %1: Cache hit at offset %<11zu>2 (length %<6u>3).", frame_no, offset, len);
676
677 success = !cache_entry->m_cache_info.m_error;
678 }
679 else
680 {
681 success = cache_entry->m_buffer->read_frame(&data, frame_no);
682 if (!success)
683 {
684 Logging::error(cache_entry->virtname(), "Reading image frame no. %1: (%2) %3", frame_no, errno, strerror(errno));
685 throw false;
686 }
687 }
688
689 if (success)
690 {
691 if (data.size() < offset)
692 {
693 len = 0;
694 }
695 else if (data.size() < offset + len)
696 {
697 len = data.size() - offset;
698 }
699
700 if (len)
701 {
702 std::memcpy(buff, data.data() + offset, len);
703 }
704
705 stat_set_size(&virtualfile->m_st, data.size());
706
707 *bytes_read = static_cast<int>(len);
708 }
709 }
710 catch (bool _success)
711 {
712 success = _success;
713 }
714
715 return success;
716}
717
719{
720 cache->closeio(&cache_entry);
721}
722
724{
725 return cache_entry->size();
726}
727
728size_t transcoder_buffer_watermark(Cache_Entry* cache_entry, uint32_t segment_no)
729{
730 return cache_entry->m_buffer->buffer_watermark(segment_no);
731}
732
733size_t transcoder_buffer_tell(Cache_Entry* cache_entry, uint32_t segment_no)
734{
735 return cache_entry->m_buffer->tell(segment_no);
736}
737
739{
740 thread_exit = true;
741}
742
744{
745 if (cache != nullptr)
746 {
747 return cache->maintenance();
748 }
749 else
750 {
751 return false;
752 }
753}
754
756{
757 if (cache != nullptr)
758 {
759 return cache->clear();
760 }
761 else
762 {
763 return false;
764 }
765}
766
775static bool transcode(std::shared_ptr<THREAD_DATA> thread_data, Cache_Entry *cache_entry, FFmpeg_Transcoder & transcoder, bool *timeout)
776{
777 int averror = 0;
778 int syserror = 0;
779 bool success = true;
780
781 // Clear cache to remove any older remains
782 cache_entry->clear();
783
784 // Must decode the file, otherwise simply use cache
785 cache_entry->m_is_decoding = true;
786
787 try
788 {
789 bool unlocked = false;
790
791 Logging::info(cache_entry->filename(), "Transcoding to %1.", params.current_format(cache_entry->virtualfile())->desttype().c_str());
792
793 if (!cache_entry->openio())
794 {
795 throw (static_cast<int>(errno));
796 }
797
798 averror = transcoder.open_input_file(cache_entry->virtualfile());
799 if (averror < 0)
800 {
801 throw (static_cast<int>(errno));
802 }
803
804 if (!cache_entry->m_cache_info.m_duration)
805 {
806 cache_entry->m_cache_info.m_duration = transcoder.duration();
807 }
808
809 if (!cache_entry->m_cache_info.m_predicted_filesize)
810 {
811 cache_entry->m_cache_info.m_predicted_filesize = transcoder.predicted_filesize();
812 }
813
814 if (!cache_entry->m_cache_info.m_video_frame_count)
815 {
816 cache_entry->m_cache_info.m_video_frame_count = transcoder.video_frame_count();
817 }
818
819 if (!cache_entry->m_cache_info.m_segment_count)
820 {
821 cache_entry->m_cache_info.m_segment_count = transcoder.segment_count();
822 }
823
824 if (cache != nullptr && !cache->maintenance(transcoder.predicted_filesize()))
825 {
826 throw (static_cast<int>(errno));
827 }
828
829 averror = transcoder.open_output_file(cache_entry->m_buffer.get());
830 if (averror < 0)
831 {
832 throw (static_cast<int>(errno));
833 }
834
835 std::memcpy(&cache_entry->m_id3v1, transcoder.id3v1tag(), sizeof(ID3v1));
836
837 thread_data->m_initialised = true;
838
839 unlocked = false;
840 if ((!params.m_prebuffer_size && !params.m_prebuffer_time) || transcoder.is_frameset())
841 {
842 // Unlock frame set from beginning
843 unlocked = true;
844 thread_data->m_thread_running_lock_guard = true;
845 thread_data->m_thread_running_cond.notify_all(); // signal that we are running
846 }
847 else
848 {
850 {
851 Logging::debug(cache_entry->virtname(), "Pre-buffering up to %1.", format_time(params.m_prebuffer_time).c_str());
852 }
854 {
855 Logging::debug(cache_entry->virtname(), "Pre-buffering up to %1.", format_size(params.m_prebuffer_size).c_str());
856 }
857 }
858
859 cache_entry->m_suspend_timeout = false;
860
861 while (!cache_entry->is_finished() && !(*timeout = cache_entry->decode_timeout()) && !thread_exit)
862 {
864
865 if (cache_entry->ref_count() > 1)
866 {
867 // Set last access time
868 cache_entry->update_access(false);
869 }
870
871 if (transcoder.is_frameset())
872 {
873 uint32_t frame_no = cache_entry->m_seek_to_no;
874 if (frame_no)
875 {
876 cache_entry->m_seek_to_no = 0;
877
878 averror = transcoder.stack_seek_frame(frame_no);
879 if (averror < 0)
880 {
881 throw (static_cast<int>(errno));
882 }
883 }
884 }
885 else if (transcoder.is_hls())
886 {
887 uint32_t segment_no = cache_entry->m_seek_to_no;
888 if (segment_no)
889 {
890 cache_entry->m_seek_to_no = 0;
891
892 averror = transcoder.stack_seek_segment(segment_no);
893 if (averror < 0)
894 {
895 throw (static_cast<int>(errno));
896 }
897 }
898 }
899
900 averror = transcoder.process_single_fr(&status);
901
902 if (status == DECODER_STATUS::DEC_ERROR)
903 {
904 errno = EIO;
905 throw (static_cast<int>(errno));
906 }
907 else if (status == DECODER_STATUS::DEC_EOF)
908 {
909 cache_entry->m_suspend_timeout = true; // Suspend read_frame time out until transcoder is reopened.
910
911 averror = transcode_finish(cache_entry, transcoder);
912
913 if (averror < 0)
914 {
915 errno = EIO;
916 throw (static_cast<int>(errno));
917 }
918 }
919
920 if (!unlocked && cache_entry->m_buffer->buffer_watermark() > params.m_prebuffer_size && transcoder.pts() > static_cast<int64_t>(params.m_prebuffer_time) * AV_TIME_BASE)
921 {
922 unlocked = true;
923 Logging::debug(cache_entry->virtname(), "Pre-buffer limit reached.");
924
925 thread_data->m_thread_running_lock_guard = true;
926 thread_data->m_thread_running_cond.notify_all(); // signal that we are running
927 }
928
929 if (cache_entry->ref_count() <= 1 && cache_entry->suspend_timeout())
930 {
931 if (!unlocked && (params.m_prebuffer_size || params.m_prebuffer_time))
932 {
933 unlocked = true;
934 thread_data->m_thread_running_lock_guard = true;
935 thread_data->m_thread_running_cond.notify_all(); // signal that we are running
936 }
937
938 Logging::info(cache_entry->virtname(), "Suspend timeout. Transcoding suspended after %1 seconds inactivity.", params.m_max_inactive_suspend);
939
940 while (cache_entry->suspend_timeout() && !(*timeout = cache_entry->decode_timeout()) && !thread_exit)
941 {
943 }
944
945 if (*timeout)
946 {
947 break;
948 }
949
950 Logging::info(cache_entry->virtname(), "Transcoding resumed.");
951 }
952 }
953
954 if (!unlocked && (params.m_prebuffer_size || params.m_prebuffer_time))
955 {
956 Logging::debug(cache_entry->virtname(), "File transcode complete, releasing buffer early: Size %1.", cache_entry->m_buffer->buffer_watermark());
957 thread_data->m_thread_running_lock_guard = true;
958 thread_data->m_thread_running_cond.notify_all(); // signal that we are running
959 }
960 }
961 catch (int _syserror)
962 {
963 success = false;
964 syserror = _syserror;
965
966 if (!syserror && averror > -512)
967 {
968 // If no system error reported explicitly, and averror is a POSIX error
969 // (we simply assume that if averror < 512, I think averrors are all higher values)
970 syserror = AVUNERROR(averror);
971 }
972
973 cache_entry->m_cache_info.m_error = true;
974 if (!syserror)
975 {
976 // If system error is still zero, set to EIO to return at least anything else than success.
977 syserror = EIO;
978 }
979
980 thread_data->m_thread_running_lock_guard = true;
981 thread_data->m_thread_running_cond.notify_all(); // unlock main thread
982 }
983
984 cache_entry->m_suspend_timeout = false; // Should end that suspension; otherwise, read may hang.
985
986 cache_entry->m_cache_info.m_errno = syserror; // Preserve errno
987 cache_entry->m_cache_info.m_averror = averror; // Preserve averror
988
989 transcoder.closeio();
990
991 return success;
992}
993
999static int transcoder_thread(std::shared_ptr<THREAD_DATA> thread_data)
1000{
1001 Cache_Entry * cache_entry = thread_data->m_cache_entry;
1002 FFmpeg_Transcoder transcoder;
1003 bool timeout = false;
1004 bool success = true;
1005
1006 std::unique_lock<std::recursive_mutex> lock_active_mutex(cache_entry->m_active_mutex);
1007 std::unique_lock<std::recursive_mutex> lock_restart_mutex(cache_entry->m_restart_mutex);
1008
1009 uint32_t seek_frame = 0;
1010
1011 do
1012 {
1013 if (seek_frame)
1014 {
1015 Logging::error(transcoder.virtname(), "Transcoder completed with last seek frame to %1. Transcoder is being restarted.", seek_frame);
1016 }
1017
1018 success = transcode(thread_data, cache_entry, transcoder, &timeout);
1019
1020 seek_frame = cache_entry->m_seek_to_no != 0 ? cache_entry->m_seek_to_no.load() : transcoder.last_seek_frame_no();
1021 }
1022 while (success && !thread_exit && cache != nullptr && seek_frame);
1023
1024 cache_entry->m_is_decoding = false;
1025
1026 if (timeout || thread_exit || transcoder.have_seeked())
1027 {
1028 if (!transcoder.have_seeked())
1029 {
1030 cache_entry->m_cache_info.m_error = true;
1031 cache_entry->m_cache_info.m_errno = EIO; // Report I/O error
1032
1033 if (timeout)
1034 {
1035 Logging::warning(cache_entry->virtname(), "Timeout! Transcoding aborted after %1 seconds inactivity.", params.m_max_inactive_abort);
1036 }
1037 else if (thread_exit)
1038 {
1039 Logging::info(cache_entry->virtname(), "Thread exit! Transcoding aborted.");
1040 }
1041 else
1042 {
1043 Logging::info(cache_entry->virtname(), "Transcoding aborted.");
1044 }
1045 }
1046 else
1047 {
1048 // Must restart from scratch, but this is not an error.
1049 cache_entry->m_cache_info.m_error = false;
1050 cache_entry->m_cache_info.m_errno = 0;
1051 cache_entry->m_cache_info.m_averror = 0;
1052 }
1053 }
1054 else
1055 {
1056 cache_entry->m_cache_info.m_error = !success;
1057
1058 if (success)
1059 {
1060 Logging::info(cache_entry->virtname(), "Transcoding completed successfully.");
1061
1062 cache_entry->m_cache_info.m_errno = 0;
1063 cache_entry->m_cache_info.m_averror = 0;
1064 }
1065 else
1066 {
1067 Logging::error(cache_entry->virtname(), "Transcoding exited with error.");
1068 if (cache_entry->m_cache_info.m_errno)
1069 {
1070 Logging::error(cache_entry->virtname(), "System error: (%1) %2", cache_entry->m_cache_info.m_errno, strerror(cache_entry->m_cache_info.m_errno));
1071 }
1072
1073 if (cache_entry->m_cache_info.m_averror)
1074 {
1075 Logging::error(cache_entry->virtname(), "FFMpeg error: (%1) %2", cache_entry->m_cache_info.m_averror, ffmpeg_geterror(cache_entry->m_cache_info.m_averror).c_str());
1076 }
1077 }
1078 }
1079
1080 int _errno = cache_entry->m_cache_info.m_errno;
1081
1082 if (cache != nullptr)
1083 {
1084 cache->closeio(&cache_entry, timeout ? CACHE_CLOSE_DELETE : CACHE_CLOSE_NOOPT);
1085 }
1086
1087 thread_data.reset();
1088
1089 errno = _errno;
1090
1091 return errno;
1092}
Buffer class.
#define CACHE_FLAG_RO
Mark cache file read-only.
Definition: buffer.h:49
#define CACHE_CLOSE_DELETE
Delete cache entry, will unlink cached file! Implies CACHE_CLOSE_FREE.
Definition: buffer.h:47
#define CACHE_CLOSE_NOOPT
Dummy, do nothing special.
Definition: buffer.h:45
Data cache management.
@ FINISHED_INCOMPLETE
Transcode finished, but incomplete.
@ FINISHED_SUCCESS
Transcode finished successfully.
Cache entry
The Cache_Entry class.
Definition: cache_entry.h:49
std::recursive_mutex m_restart_mutex
Mutex while thread is restarted.
Definition: cache_entry.h:274
size_t size() const
Return size of output file, as computed by encoder.
Definition: cache_entry.cc:260
CACHE_INFO m_cache_info
Info about cached object.
Definition: cache_entry.h:277
bool update_access(bool update_database=false)
Update last access time.
Definition: cache_entry.cc:146
std::atomic_bool m_suspend_timeout
true to temporarly disable read_frame timeout
Definition: cache_entry.h:275
bool is_finished() const
Get if cache has been finished.
Definition: cache_entry.cc:435
bool suspend_timeout() const
Check for decode suspend timeout.
Definition: cache_entry.cc:301
int ref_count() const
Get the current reference counter.
Definition: cache_entry.cc:336
void lock()
Lock the access mutex.
Definition: cache_entry.cc:326
std::atomic_bool m_is_decoding
true while file is decoding
Definition: cache_entry.h:272
const char * virtname() const
Return virtual filename. Same as destination filename, but with virtual (mount) path....
Definition: cache_entry.cc:321
void clear(bool fetch_file_time=true)
Clear the cache entry.
Definition: cache_entry.cc:85
LPVIRTUALFILE virtualfile()
Get the underlying VIRTUALFILE object.
Definition: cache_entry.cc:420
bool openio(bool create_cache=true)
Open the cache file.
Definition: cache_entry.cc:160
void update_read_count()
Update read counter.
Definition: cache_entry.cc:425
const char * filename() const
Return source filename.
Definition: cache_entry.cc:311
std::recursive_mutex m_active_mutex
Mutex while thread is active.
Definition: cache_entry.h:273
ID3v1 m_id3v1
ID3v1 structure which is used to send to clients.
Definition: cache_entry.h:279
unsigned int read_count() const
Get read counter.
Definition: cache_entry.cc:430
bool flush()
Flush current memory cache to disk.
Definition: cache_entry.cc:246
bool decode_timeout() const
Check for decode timeout.
Definition: cache_entry.cc:306
void unlock()
Unlock the access mutex.
Definition: cache_entry.cc:331
bool is_finished_success() const
Get if cache has been finished and filled successfully.
Definition: cache_entry.cc:445
bool outdated() const
Check if cache entry needs to be recoded.
Definition: cache_entry.cc:351
std::atomic_uint32_t m_seek_to_no
If not 0, seeks to specified frame.
Definition: cache_entry.h:281
std::unique_ptr< Buffer > m_buffer
Buffer object.
Definition: cache_entry.h:271
The FFmpeg_Transcoder class.
static bool video_size(size_t *filesize, AVCodecID codec_id, BITRATE bit_rate, int64_t duration, int width, int height, bool interleaved, const AVRational &framerate)
Predict video file size. This may (better will surely) be inaccurate.
int stack_seek_frame(uint32_t frame_no)
Seek to a specific frame. Does not actually perform the seek, this is done asynchronously by the tran...
virtual const char * filename() const override
Return source filename.
int process_single_fr(DECODER_STATUS *status)
static bool audio_size(size_t *filesize, AVCodecID codec_id, BITRATE bit_rate, int64_t duration, int channels, int sample_rate, AVSampleFormat sample_format)
Predict audio file size. This may (better will surely) be inaccurate.
bool have_seeked() const
Check if we made a seek operation.
void closeio()
Close transcoder, free all ressources.
uint32_t video_frame_count() const
Get the number of video frames in file.
bool is_hls() const
Check for HLS format.
size_t predicted_filesize() const
Try to predict the recoded file size. This may (better will surely) be inaccurate.
static bool total_overhead(size_t *filesize, FILETYPE filetype)
Predict overhead in file size. This may (better will surely) be inaccurate.
const ID3v1 * id3v1tag() const
Assemble an ID3v1 file tag.
uint32_t segment_count() const
Get the number of HLS segments of file.
bool is_multiformat() const
Check for an export frame format.
virtual const char * virtname() const override
Return virtual filename. Same as destination filename, but with virtual (mount) path....
uint32_t last_seek_frame_no() const
Current seek frame if available.
int open_output_file(Buffer *buffer)
Open output file. Data will actually be written to buffer and copied by FUSE when accessed.
int64_t duration() const
Get the file duration.
bool is_frameset() const
Check for an export frame format.
int64_t pts() const
Get PTS (presentation time stamp) of decoded audio/video so far.
int open_input_file(LPVIRTUALFILE virtualfile, std::shared_ptr< FileIO > fio=nullptr)
int stack_seek_segment(uint32_t segment_no)
Seek to a specific HLS segment. Does not actually perform the seek, this is done asynchronously by th...
The FFmpegfs_Format class.
Definition: ffmpeg_utils.h:388
const std::string & desttype() const
Get destination type.
FILETYPE filetype() const
Get selected filetype.
AVCodecID video_codec() const
Get video codec_id.
AVCodecID audio_codec() const
Get audio codec_id.
static void warning(const T filename, const std::string &format_string, Args &&...args)
Write warning level log entry.
Definition: logging.h:220
static void debug(const T filename, const std::string &format_string, Args &&...args)
Write debug level log entry.
Definition: logging.h:182
static void trace(const T filename, const std::string &format_string, Args &&...args)
Write trace level log entry.
Definition: logging.h:163
static void info(const T filename, const std::string &format_string, Args &&...args)
Write info level log entry.
Definition: logging.h:201
static void error(const T filename, const std::string &format_string, Args &&...args)
Write error level log entry.
Definition: logging.h:239
FFmpeg transcoder.
DECODER_STATUS
Decoder status codes. Can be error, success or end of file.
@ DEC_ERROR
Decoder error, see return code.
@ DEC_EOF
Read to end of file.
@ DEC_SUCCESS
Frame decoded successfully.
bool check_ignore(size_t size, size_t offset)
For use with win_smb_fix=1: Check if this an illegal access offset by Windows.
const char * get_codec_name(AVCodecID codec_id, bool long_name)
Safe way to get the codec name. Function never fails, will return "unknown" on error.
std::string get_filetype_text(FILETYPE filetype)
Convert FILETYPE enum to human readable text.
std::string format_time(time_t value)
Format a time in format "w d m s".
std::string format_result_size_ex(size_t size_resulting, size_t size_predicted)
Format size of transcoded file including difference between predicted and resulting size.
const std::string & append_sep(std::string *path)
Add / to the path if required.
const std::string & expand_path(std::string *tgt, const std::string &src)
Expand path, e.g., expand ~/ to home directory.
void mssleep(int milliseconds)
Sleep for specified time.
std::string format_size(uint64_t value)
Format size.
void stat_set_size(struct stat *st, size_t size)
Properly fill in all size related members in stat struct.
std::string format_size_ex(uint64_t value)
Format size.
std::string ffmpeg_geterror(int errnum)
Get FFmpeg error string for errnum. Internally calls av_strerror().
#define BITRATE
For FFmpeg bit rate is an int.
Definition: ffmpeg_utils.h:145
FFMPEGFS_PARAMS params
FFmpegfs command line parameters.
Definition: ffmpegfs.cc:74
Main include for FFmpegfs project.
std::unique_ptr< thread_pool > tp
Thread pool object.
Definition: fuseops.cc:102
#define ID3V1_TAG_LENGTH
Fixed 128 bytes.
Definition: id3v1tag.h:58
Provide various log facilities to stderr, disk or syslog.
uint32_t m_video_frame_count
Number of frames in video or 0 if not a video.
Definition: cache.h:84
RESULTCODE m_result
Result code:
Definition: cache.h:86
int64_t m_duration
File duration, in AV_TIME_BASE fractional seconds.
Definition: cache.h:81
uint32_t m_segment_count
Number of segments for HLS.
Definition: cache.h:85
size_t m_encoded_filesize
Actual file size after encode.
Definition: cache.h:83
bool m_error
true if encode failed
Definition: cache.h:87
size_t m_predicted_filesize
Predicted file size.
Definition: cache.h:82
int m_averror
FFmpeg error code if encode failed.
Definition: cache.h:89
int m_errno
errno if encode failed
Definition: cache.h:88
time_t m_max_inactive_suspend
Time (seconds) that must elapse without access until transcoding is suspended.
Definition: ffmpegfs.h:243
const FFmpegfs_Format * current_format(LPCVIRTUALFILE virtualfile) const
Get FFmpegfs_Format for a virtual file.
Definition: ffmpegfs.cc:235
size_t m_prebuffer_size
Number of bytes that will be decoded before the output can be accessed.
Definition: ffmpegfs.h:246
time_t m_prebuffer_time
Playing time that will be decoded before the output can be accessed.
Definition: ffmpegfs.h:245
std::string m_cachepath
Disk cache path, defaults to $XDG_CACHE_HOME.
Definition: ffmpegfs.h:249
int m_disable_cache
Disable cache.
Definition: ffmpegfs.h:250
time_t m_max_inactive_abort
Time (seconds) that must elapse without access until transcoding is aborted.
Definition: ffmpegfs.h:244
int m_win_smb_fix
Experimental Windows fix for access to EOF at file open.
Definition: ffmpegfs.h:262
ID3 version 1 tag
Definition: id3v1tag.h:42
THREAD_DATA struct to pass data from parent to child thread.
Definition: transcode.cc:55
std::mutex m_thread_running_mutex
Mutex when thread is running.
Definition: transcode.cc:56
bool m_initialised
True when this object is completely initialised.
Definition: transcode.cc:59
std::atomic_bool m_thread_running_lock_guard
Lock guard to avoid spurious or missed unlocks.
Definition: transcode.cc:58
Cache_Entry * m_cache_entry
Cache entry object. Will not be freed by child thread.
Definition: transcode.cc:60
std::condition_variable m_thread_running_cond
Condition when thread is running.
Definition: transcode.cc:57
Virtual file definition.
Definition: fileio.h:123
size_t m_predicted_size
Use this as the size instead of computing it over and over.
Definition: fileio.h:157
struct stat m_st
stat structure with size etc.
Definition: fileio.h:153
uint32_t m_video_frame_count
Number of frames in video or 0 if not a video.
Definition: fileio.h:158
int64_t m_duration
Track/chapter duration, in AV_TIME_BASE fractional seconds.
Definition: fileio.h:156
Thread pool class implementation.
static bool transcode_until(Cache_Entry *cache_entry, size_t offset, size_t len, uint32_t segment_no)
Transcode the buffer until the buffer has enough or until an error occurs. The buffer needs at least ...
Definition: transcode.cc:81
static bool transcode(std::shared_ptr< THREAD_DATA > thread_data, Cache_Entry *cache_entry, FFmpeg_Transcoder &transcoder, bool *timeout)
Actually transcode file.
Definition: transcode.cc:775
const int TOTAL_RETRIES
Number of retries.
Definition: transcode.cc:50
struct THREAD_DATA THREAD_DATA
THREAD_DATA struct to pass data from parent to child thread.
bool transcoder_predict_filesize(LPVIRTUALFILE virtualfile, Cache_Entry *cache_entry)
Predict file size.
Definition: transcode.cc:328
static std::atomic_bool thread_exit
Used for shutdown: if true, forcibly exit all threads.
Definition: transcode.cc:64
void transcoder_cache_path(std::string *path)
Get transcoder cache path.
Definition: transcode.cc:195
Cache_Entry * transcoder_new(LPVIRTUALFILE virtualfile, bool begin_transcode)
Allocate and initialise the transcoder.
Definition: transcode.cc:353
bool transcoder_set_filesize(LPVIRTUALFILE virtualfile, int64_t duration, BITRATE audio_bit_rate, int channels, int sample_rate, AVSampleFormat sample_format, BITRATE video_bit_rate, int width, int height, bool interleaved, const AVRational &framerate)
Set the file size.
Definition: transcode.cc:288
static std::unique_ptr< Cache > cache
Global cache manager object.
Definition: transcode.cc:63
void transcoder_free()
Free transcoder.
Definition: transcode.cc:251
const int GRANULARITY
Image frame conversion: ms between checks if a picture frame is available.
Definition: transcode.cc:48
size_t transcoder_buffer_tell(Cache_Entry *cache_entry, uint32_t segment_no)
Return the current file position in the file.
Definition: transcode.cc:733
bool transcoder_read(Cache_Entry *cache_entry, char *buff, size_t offset, size_t len, int *bytes_read, uint32_t segment_no)
Read some bytes from the internal buffer and into the given buffer.
Definition: transcode.cc:469
static int transcode_finish(Cache_Entry *cache_entry, FFmpeg_Transcoder &transcoder)
Close the input file and free everything but the initial buffer.
Definition: transcode.cc:156
void transcoder_exit()
Exit transcoding.
Definition: transcode.cc:738
size_t transcoder_get_size(Cache_Entry *cache_entry)
Return size of output file, as computed by encoder.
Definition: transcode.cc:723
bool transcoder_read_frame(Cache_Entry *cache_entry, char *buff, size_t offset, size_t len, uint32_t frame_no, int *bytes_read, LPVIRTUALFILE virtualfile)
Read one image frame from the internal buffer and into the given buffer.
Definition: transcode.cc:603
bool transcoder_init()
Initialise transcoder, create cache.
Definition: transcode.cc:229
size_t transcoder_buffer_watermark(Cache_Entry *cache_entry, uint32_t segment_no)
Return the current watermark of the file while transcoding.
Definition: transcode.cc:728
bool transcoder_cache_clear()
Clear transcoder cache.
Definition: transcode.cc:755
static int transcoder_thread(std::shared_ptr< THREAD_DATA > thread_data)
Transcoding thread.
Definition: transcode.cc:999
bool transcoder_cached_filesize(LPVIRTUALFILE virtualfile, struct stat *stbuf)
Simply get encoded file size (do not create the whole encoder/decoder objects)
Definition: transcode.cc:261
bool transcoder_cache_maintenance()
Run cache maintenance.
Definition: transcode.cc:743
const int FRAME_TIMEOUT
Image frame conversion: timout seconds to wait if a picture frame is available.
Definition: transcode.cc:49
void transcoder_delete(Cache_Entry *cache_entry)
Free the cache entry structure.
Definition: transcode.cc:718
File transcoder interface (for use with by FUSE)