OpenShot Library | libopenshot  0.1.2
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), rescaler_position(0), num_of_rescalers(OPEN_MP_NUM_PROCESSORS), is_open(false),
39  seek_audio_frame_found(0), seek_video_frame_found(0), prev_samples(0), prev_pts(0),
40  pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
41  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0) {
42 
43  // Initialize FFMpeg, and register all formats and codecs
44  av_register_all();
45  avcodec_register_all();
46 
47  // Init cache
48  working_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 30, info.width, info.height, info.sample_rate, info.channels);
49  missing_frames.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
50  final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS * 2, info.width, info.height, info.sample_rate, info.channels);
51 
52  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
53  Open();
54  Close();
55 }
56 
58  if (is_open)
59  // Auto close reader if not already done
60  Close();
61 }
62 
63 // Init a collection of software rescalers (thread safe)
64 void FFmpegReader::InitScalers()
65 {
66  // Init software rescalers vector (many of them, one for each thread)
67  for (int x = 0; x < num_of_rescalers; x++)
68  {
69  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, pCodecCtx->pix_fmt, info.width,
70  info.height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL);
71 
72  // Add rescaler to vector
73  image_rescalers.push_back(img_convert_ctx);
74  }
75 }
76 
77 // This struct holds the associated video frame and starting sample # for an audio packet.
78 int AudioLocation::is_near(AudioLocation location, int samples_per_frame, int amount)
79 {
80  // Is frame even close to this one?
81  if (abs(location.frame - frame) >= 2)
82  // This is too far away to be considered
83  return false;
84 
85  int sample_diff = abs(location.sample_start - sample_start);
86  if (location.frame == frame && sample_diff >= 0 && sample_diff <= amount)
87  // close
88  return true;
89 
90  // new frame is after
91  if (location.frame > frame)
92  {
93  // remaining samples + new samples
94  sample_diff = (samples_per_frame - sample_start) + location.sample_start;
95  if (sample_diff >= 0 && sample_diff <= amount)
96  return true;
97  }
98 
99  // new frame is before
100  if (location.frame < frame)
101  {
102  // remaining new samples + old samples
103  sample_diff = (samples_per_frame - location.sample_start) + sample_start;
104  if (sample_diff >= 0 && sample_diff <= amount)
105  return true;
106  }
107 
108  // not close
109  return false;
110 }
111 
112 // Remove & deallocate all software scalers
113 void FFmpegReader::RemoveScalers()
114 {
115  // Close all rescalers
116  for (int x = 0; x < num_of_rescalers; x++)
117  sws_freeContext(image_rescalers[x]);
118 
119  // Clear vector
120  image_rescalers.clear();
121 }
122 
124 {
125  // Open reader if not already open
126  if (!is_open)
127  {
128  // Initialize format context
129  pFormatCtx = NULL;
130 
131  // Open video file
132  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
133  throw InvalidFile("File could not be opened.", path);
134 
135  // Retrieve stream information
136  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
137  throw NoStreamsFound("No streams found in file.", path);
138 
139  videoStream = -1;
140  audioStream = -1;
141  // Loop through each stream, and identify the video and audio stream index
142  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
143  {
144  // Is this a video stream?
145  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
146  videoStream = i;
147  }
148  // Is this an audio stream?
149  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
150  audioStream = i;
151  }
152  }
153  if (videoStream == -1 && audioStream == -1)
154  throw NoStreamsFound("No video or audio streams found in this file.", path);
155 
156  // Is there a video stream?
157  if (videoStream != -1)
158  {
159  // Set the stream index
160  info.video_stream_index = videoStream;
161 
162  // Set the codec and codec context pointers
163  pStream = pFormatCtx->streams[videoStream];
164  pCodecCtx = pFormatCtx->streams[videoStream]->codec;
165 
166  // Set number of threads equal to number of processors + 1
167  pCodecCtx->thread_count = OPEN_MP_NUM_PROCESSORS;
168 
169  // Find the decoder for the video stream
170  AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
171  if (pCodec == NULL) {
172  throw InvalidCodec("A valid video codec could not be found for this file.", path);
173  }
174  // Open video codec
175  if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
176  throw InvalidCodec("A video codec was found, but could not be opened.", path);
177 
178  // Update the File Info struct with video details (if a video stream is found)
179  UpdateVideoInfo();
180 
181  // Init rescalers (if video stream detected)
182  InitScalers();
183  }
184 
185  // Is there an audio stream?
186  if (audioStream != -1)
187  {
188  // Set the stream index
189  info.audio_stream_index = audioStream;
190 
191  // Get a pointer to the codec context for the audio stream
192  aStream = pFormatCtx->streams[audioStream];
193  aCodecCtx = pFormatCtx->streams[audioStream]->codec;
194 
195  // Set number of threads equal to number of processors + 1
196  aCodecCtx->thread_count = OPEN_MP_NUM_PROCESSORS;
197 
198  // Find the decoder for the audio stream
199  AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
200  if (aCodec == NULL) {
201  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
202  }
203  // Open audio codec
204  if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
205  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
206 
207  // Update the File Info struct with audio details (if an audio stream is found)
208  UpdateAudioInfo();
209  }
210 
211  // Init previous audio location to zero
212  previous_packet_location.frame = -1;
213  previous_packet_location.sample_start = 0;
214 
215  // Adjust cache size based on size of frame and audio
219 
220  // Mark as "open"
221  is_open = true;
222  }
223 }
224 
226 {
227  // Close all objects, if reader is 'open'
228  if (is_open)
229  {
230  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
231 
232  // Mark as "closed"
233  is_open = false;
234 
235  // Close the codec
236  if (info.has_video)
237  {
238  // Clear image scalers
239  RemoveScalers();
240 
241  avcodec_flush_buffers(pCodecCtx);
242  avcodec_close(pCodecCtx);
243  }
244  if (info.has_audio)
245  {
246  avcodec_flush_buffers(aCodecCtx);
247  avcodec_close(aCodecCtx);
248  }
249 
250  // Clear final cache
251  final_cache.Clear();
252  working_cache.Clear();
253  missing_frames.Clear();
254 
255  // Clear processed lists
256  {
257  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
258  processed_video_frames.clear();
259  processed_audio_frames.clear();
260  processing_video_frames.clear();
261  processing_audio_frames.clear();
262  missing_audio_frames.clear();
263  missing_video_frames.clear();
264  checked_frames.clear();
265  }
266 
267  // Close the video file
268  avformat_close_input(&pFormatCtx);
269  av_freep(&pFormatCtx);
270 
271  // Reset some variables
272  last_frame = 0;
273  largest_frame_processed = 0;
274  seek_audio_frame_found = 0;
275  seek_video_frame_found = 0;
276  current_video_frame = 0;
277  has_missing_frames = false;
278  }
279 }
280 
281 void FFmpegReader::UpdateAudioInfo()
282 {
283  // Set values of FileInfo struct
284  info.has_audio = true;
285  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
286  info.acodec = aCodecCtx->codec->name;
287  info.channels = aCodecCtx->channels;
288  if (aCodecCtx->channel_layout == 0)
289  aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );;
290  info.channel_layout = (ChannelLayout) aCodecCtx->channel_layout;
291  info.sample_rate = aCodecCtx->sample_rate;
292  info.audio_bit_rate = aCodecCtx->bit_rate;
293 
294  // Set audio timebase
295  info.audio_timebase.num = aStream->time_base.num;
296  info.audio_timebase.den = aStream->time_base.den;
297 
298  // Get timebase of audio stream (if valid) and greater than the current duration
299  if (aStream->duration > 0.0f && aStream->duration > info.duration)
300  info.duration = aStream->duration * info.audio_timebase.ToDouble();
301 
302  // Check for an invalid video length
303  if (info.has_video && info.video_length <= 0)
304  {
305  // Calculate the video length from the audio duration
307  }
308 
309  // Set video timebase (if no video stream was found)
310  if (!info.has_video)
311  {
312  // Set a few important default video settings (so audio can be divided into frames)
313  info.fps.num = 24;
314  info.fps.den = 1;
315  info.video_timebase.num = 1;
316  info.video_timebase.den = 24;
318  info.width = 720;
319  info.height = 480;
320 
321  }
322 
323 }
324 
325 void FFmpegReader::UpdateVideoInfo()
326 {
327  // Set values of FileInfo struct
328  info.has_video = true;
329  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
330  info.height = pCodecCtx->height;
331  info.width = pCodecCtx->width;
332  info.vcodec = pCodecCtx->codec->name;
333  info.video_bit_rate = pFormatCtx->bit_rate;
334  if (!check_fps)
335  {
336  // set frames per second (fps)
337  info.fps.num = pStream->avg_frame_rate.num;
338  info.fps.den = pStream->avg_frame_rate.den;
339  }
340 
341  if (pStream->sample_aspect_ratio.num != 0)
342  {
343  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
344  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
345  }
346  else if (pCodecCtx->sample_aspect_ratio.num != 0)
347  {
348  info.pixel_ratio.num = pCodecCtx->sample_aspect_ratio.num;
349  info.pixel_ratio.den = pCodecCtx->sample_aspect_ratio.den;
350  }
351  else
352  {
353  info.pixel_ratio.num = 1;
354  info.pixel_ratio.den = 1;
355  }
356 
357  info.pixel_format = pCodecCtx->pix_fmt;
358 
359  // Calculate the DAR (display aspect ratio)
361 
362  // Reduce size fraction
363  size.Reduce();
364 
365  // Set the ratio based on the reduced fraction
366  info.display_ratio.num = size.num;
367  info.display_ratio.den = size.den;
368 
369  // Set the video timebase
370  info.video_timebase.num = pStream->time_base.num;
371  info.video_timebase.den = pStream->time_base.den;
372 
373  // Set the duration in seconds, and video length (# of frames)
374  info.duration = pStream->duration * info.video_timebase.ToDouble();
375 
376  // Check for valid duration (if found)
377  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
378  // Use the format's duration
379  info.duration = pFormatCtx->duration / AV_TIME_BASE;
380 
381  // Calculate duration from filesize and bitrate (if any)
382  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
383  // Estimate from bitrate, total bytes, and framerate
385 
386  // No duration found in stream of file
387  if (info.duration <= 0.0f)
388  {
389  // No duration is found in the video stream
390  info.duration = -1;
391  info.video_length = -1;
392  is_duration_known = false;
393  }
394  else
395  {
396  // Yes, a duration was found
397  is_duration_known = true;
398 
399  // Calculate number of frames
401  }
402 
403  // Override an invalid framerate
404  if (info.fps.ToFloat() > 120.0f || (info.fps.num == 0 || info.fps.den == 0))
405  {
406  // Set a few important default video settings (so audio can be divided into frames)
407  info.fps.num = 24;
408  info.fps.den = 1;
409  info.video_timebase.num = 1;
410  info.video_timebase.den = 24;
411 
412  // Calculate number of frames
414  }
415 
416 }
417 
418 
419 tr1::shared_ptr<Frame> FFmpegReader::GetFrame(long int requested_frame) throw(OutOfBoundsFrame, ReaderClosed, TooManySeeks)
420 {
421  // Check for open reader (or throw exception)
422  if (!is_open)
423  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
424 
425  // Adjust for a requested frame that is too small or too large
426  if (requested_frame < 1)
427  requested_frame = 1;
428  if (requested_frame > info.video_length && is_duration_known)
429  requested_frame = info.video_length;
430  if (info.has_video && info.video_length == 0)
431  // Invalid duration of video file
432  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
433 
434  // Debug output
435  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
436 
437  // Check the cache for this frame
438  tr1::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
439  if (frame) {
440  // Debug output
441  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
442 
443  // Return the cached frame
444  return frame;
445  }
446  else
447  {
448  // Create a scoped lock, allowing only a single thread to run the following code at one time
449  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
450 
451  // Check the cache a 2nd time (due to a potential previous lock)
452  if (has_missing_frames)
453  CheckMissingFrame(requested_frame);
454  frame = final_cache.GetFrame(requested_frame);
455  if (frame) {
456  // Debug output
457  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
458 
459  // Return the cached frame
460  return frame;
461  }
462 
463  // Frame is not in cache
464  // Reset seek count
465  seek_count = 0;
466 
467  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
468  if (last_frame == 0 && requested_frame != 1)
469  // Get first frame
470  ReadStream(1);
471 
472  // Are we within X frames of the requested frame?
473  long int diff = requested_frame - last_frame;
474  if (diff >= 1 && diff <= 20)
475  {
476  // Continue walking the stream
477  return ReadStream(requested_frame);
478  }
479  else
480  {
481  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
482  if (enable_seek)
483  // Only seek if enabled
484  Seek(requested_frame);
485 
486  else if (!enable_seek && diff < 0)
487  {
488  // Start over, since we can't seek, and the requested frame is smaller than our position
489  Close();
490  Open();
491  }
492 
493  // Then continue walking the stream
494  return ReadStream(requested_frame);
495  }
496 
497  }
498 }
499 
500 // Read the stream until we find the requested Frame
501 tr1::shared_ptr<Frame> FFmpegReader::ReadStream(long int requested_frame)
502 {
503  // Allocate video frame
504  bool end_of_stream = false;
505  bool check_seek = false;
506  bool frame_finished = false;
507  int packet_error = -1;
508 
509  // Minimum number of packets to process (for performance reasons)
510  int packets_processed = 0;
511  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
512 
513  // Set the number of threads in OpenMP
514  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
515  // Allow nested OpenMP sections
516  omp_set_nested(true);
517 
518  // Debug output
519  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
520 
521  #pragma omp parallel
522  {
523  #pragma omp single
524  {
525  // Loop through the stream until the correct frame is found
526  while (true)
527  {
528  // Get the next packet into a local variable called packet
529  packet_error = GetNextPacket();
530 
531  // Wait if too many frames are being processed
532  while (processing_video_frames.size() + processing_audio_frames.size() >= minimum_packets)
533  usleep(2500);
534 
535  // Get the next packet (if any)
536  if (packet_error < 0)
537  {
538  // Break loop when no more packets found
539  end_of_stream = true;
540  break;
541  }
542 
543  // Debug output
544  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames.size()", processing_video_frames.size(), "processing_audio_frames.size()", processing_audio_frames.size(), "minimum_packets", minimum_packets, "packets_processed", packets_processed, "", -1);
545 
546  // Video packet
547  if (info.has_video && packet->stream_index == videoStream)
548  {
549  // Reset this counter, since we have a video packet
550  num_packets_since_video_frame = 0;
551 
552  // Check the status of a seek (if any)
553  if (is_seeking)
554  #pragma omp critical (openshot_seek)
555  check_seek = CheckSeek(true);
556  else
557  check_seek = false;
558 
559  if (check_seek) {
560  // Remove packet (since this packet is pointless)
561  RemoveAVPacket(packet);
562 
563  // Jump to the next iteration of this loop
564  continue;
565  }
566 
567  // Get the AVFrame from the current packet
568  frame_finished = GetAVFrame();
569 
570  // Check if the AVFrame is finished and set it
571  if (frame_finished)
572  {
573  // Update PTS / Frame Offset (if any)
574  UpdatePTSOffset(true);
575 
576  // Process Video Packet
577  ProcessVideoPacket(requested_frame);
578  }
579 
580  }
581  // Audio packet
582  else if (info.has_audio && packet->stream_index == audioStream)
583  {
584  // Increment this (to track # of packets since the last video packet)
585  num_packets_since_video_frame++;
586 
587  // Check the status of a seek (if any)
588  if (is_seeking)
589  #pragma omp critical (openshot_seek)
590  check_seek = CheckSeek(false);
591  else
592  check_seek = false;
593 
594  if (check_seek) {
595  // Remove packet (since this packet is pointless)
596  RemoveAVPacket(packet);
597 
598  // Jump to the next iteration of this loop
599  continue;
600  }
601 
602  // Update PTS / Frame Offset (if any)
603  UpdatePTSOffset(false);
604 
605  // Determine related video frame and starting sample # from audio PTS
606  AudioLocation location = GetAudioPTSLocation(packet->pts);
607 
608  // Process Audio Packet
609  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
610  }
611 
612  // Check if working frames are 'finished'
613  bool is_cache_found = false;
614  if (!is_seeking) {
615  // Check for any missing frames
616  CheckMissingFrame(requested_frame);
617 
618  // Check for final frames
619  CheckWorkingFrames(false, requested_frame);
620  }
621 
622  // Check if requested 'final' frame is available
623  is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
624 
625  // Increment frames processed
626  packets_processed++;
627 
628  // Break once the frame is found
629  if ((is_cache_found && packets_processed >= minimum_packets))
630  break;
631 
632  } // end while
633 
634  } // end omp single
635  } // end omp parallel
636 
637  // Debug output
638  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
639 
640  // End of stream?
641  if (end_of_stream)
642  // Mark the any other working frames as 'finished'
643  CheckWorkingFrames(end_of_stream, requested_frame);
644 
645  // Return requested frame (if found)
646  tr1::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
647  if (frame)
648  // Return prepared frame
649  return frame;
650  else {
651 
652  // Check if largest frame is still cached
653  frame = final_cache.GetFrame(largest_frame_processed);
654  if (frame) {
655  // return the largest processed frame (assuming it was the last in the video file)
656  return frame;
657  }
658  else {
659  // The largest processed frame is no longer in cache, return a blank frame
660  tr1::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
661  f->AddColor(info.width, info.height, "#000");
662  return f;
663  }
664  }
665 
666 }
667 
668 // Get the next packet (if any)
669 int FFmpegReader::GetNextPacket()
670 {
671  int found_packet = 0;
672  AVPacket *next_packet = new AVPacket();
673  found_packet = av_read_frame(pFormatCtx, next_packet);
674 
675  if (found_packet >= 0)
676  {
677  #pragma omp critical (packet_cache)
678  {
679  // Add packet to packet cache
680  packets[next_packet] = next_packet;
681 
682  // Update current packet pointer
683  packet = packets[next_packet];
684  } // end omp critical
685 
686  }
687  else
688  {
689  // Free packet, since it's unused
690  av_free_packet(next_packet);
691  delete next_packet;
692  }
693 
694  // Return if packet was found (or error number)
695  return found_packet;
696 }
697 
698 // Get an AVFrame (if any)
699 bool FFmpegReader::GetAVFrame()
700 {
701  int frameFinished = -1;
702 
703  // Decode video frame
704  AVFrame *next_frame = AV_ALLOCATE_FRAME();
705  #pragma omp critical (packet_cache)
706  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
707 
708  // is frame finished
709  if (frameFinished)
710  {
711  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
712  // must make a copy of the image data before this method is called again.
713  AVPicture *copyFrame = new AVPicture();
714  avpicture_alloc(copyFrame, pCodecCtx->pix_fmt, info.width, info.height);
715  av_picture_copy(copyFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width, info.height);
716 
717  #pragma omp critical (packet_cache)
718  {
719  // add to AVFrame cache (if frame finished)
720  frames[copyFrame] = copyFrame;
721  pFrame = frames[copyFrame];
722  }
723 
724  // Detect interlaced frame (only once)
725  if (!check_interlace)
726  {
727  check_interlace = true;
728  info.interlaced_frame = next_frame->interlaced_frame;
729  info.top_field_first = next_frame->top_field_first;
730  }
731 
732  }
733  else
734  {
735  // Remove packet (since this packet is pointless)
736  RemoveAVPacket(packet);
737  }
738 
739  // deallocate the frame
740  AV_FREE_FRAME(&next_frame);
741 
742  // Did we get a video frame?
743  return frameFinished;
744 }
745 
746 // Check the current seek position and determine if we need to seek again
747 bool FFmpegReader::CheckSeek(bool is_video)
748 {
749  // Are we seeking for a specific frame?
750  if (is_seeking)
751  {
752  // Determine if both an audio and video packet have been decoded since the seek happened.
753  // If not, allow the ReadStream method to keep looping
754  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
755  return false;
756 
757  // Check for both streams
758  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
759  return false;
760 
761  // Determine max seeked frame
762  long int max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
763  if (seek_video_frame_found > max_seeked_frame)
764  max_seeked_frame = seek_video_frame_found;
765 
766  // determine if we are "before" the requested frame
767  if (max_seeked_frame >= seeking_frame)
768  {
769  // SEEKED TOO FAR
770  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
771 
772  // Seek again... to the nearest Keyframe
773  Seek(seeking_frame - (20 * seek_count * seek_count));
774  }
775  else
776  {
777  // SEEK WORKED
778  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
779 
780  // Seek worked, and we are "before" the requested frame
781  is_seeking = false;
782  seeking_frame = 0;
783  seeking_pts = -1;
784  }
785  }
786 
787  // return the pts to seek to (if any)
788  return is_seeking;
789 }
790 
791 // Process a video packet
792 void FFmpegReader::ProcessVideoPacket(long int requested_frame)
793 {
794  // Calculate current frame #
795  long int current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
796 
797  // Track 1st video packet after a successful seek
798  if (!seek_video_frame_found && is_seeking)
799  seek_video_frame_found = current_frame;
800 
801  // Are we close enough to decode the frame? and is this frame # valid?
802  if ((!has_missing_frames and current_frame < (requested_frame - 20)) or (current_frame == -1))
803  {
804  // Remove frame and packet
805  RemoveAVFrame(pFrame);
806  RemoveAVPacket(packet);
807 
808  // Debug output
809  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
810 
811  // Skip to next frame without decoding or caching
812  return;
813  }
814 
815  // Debug output
816  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
817 
818  // Init some things local (for OpenMP)
819  PixelFormat pix_fmt = pCodecCtx->pix_fmt;
820  int height = info.height;
821  int width = info.width;
822  long int video_length = info.video_length;
823  AVPacket *my_packet = packets[packet];
824  AVPicture *my_frame = frames[pFrame];
825 
826  // Get a scaling context
827  SwsContext *img_convert_ctx = image_rescalers[rescaler_position];
828  rescaler_position++;
829  if (rescaler_position == num_of_rescalers)
830  rescaler_position = 0;
831 
832  // Add video frame to list of processing video frames
833  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
834  processing_video_frames[current_frame] = current_frame;
835 
836  #pragma omp task firstprivate(current_frame, my_packet, my_frame, height, width, video_length, pix_fmt, img_convert_ctx)
837  {
838  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
839  AVFrame *pFrameRGB = NULL;
840  int numBytes;
841  uint8_t *buffer = NULL;
842 
843  // Allocate an AVFrame structure
844  pFrameRGB = AV_ALLOCATE_FRAME();
845  if (pFrameRGB == NULL)
846  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
847 
848  // Determine required buffer size and allocate buffer
849  numBytes = avpicture_get_size(PIX_FMT_RGBA, width, height);
850  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
851 
852  // Assign appropriate parts of buffer to image planes in pFrameRGB
853  // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
854  // of AVPicture
855  avpicture_fill((AVPicture *) pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
856 
857  // Resize / Convert to RGB
858  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
859  height, pFrameRGB->data, pFrameRGB->linesize);
860 
861  // Create or get the existing frame object
862  tr1::shared_ptr<Frame> f = CreateFrame(current_frame);
863 
864  // Add Image data to frame
865  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
866 
867  // Update working cache
868  working_cache.Add(f->number, f);
869 
870  // Keep track of last last_video_frame
871  last_video_frame = f;
872 
873  // Free the RGB image
874  av_free(buffer);
875  AV_FREE_FRAME(&pFrameRGB);
876 
877  // Remove frame and packet
878  RemoveAVFrame(my_frame);
879  RemoveAVPacket(my_packet);
880 
881  // Remove video frame from list of processing video frames
882  {
883  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
884  processing_video_frames.erase(current_frame);
885  processed_video_frames[current_frame] = current_frame;
886  }
887 
888  // Debug output
889  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
890 
891  } // end omp task
892 
893 }
894 
895 // Process an audio packet
896 void FFmpegReader::ProcessAudioPacket(long int requested_frame, long int target_frame, int starting_sample)
897 {
898  // Track 1st audio packet after a successful seek
899  if (!seek_audio_frame_found && is_seeking)
900  seek_audio_frame_found = target_frame;
901 
902  // Are we close enough to decode the frame's audio?
903  if (!has_missing_frames and target_frame < (requested_frame - 20))
904  {
905  // Remove packet
906  RemoveAVPacket(packet);
907 
908  // Debug output
909  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
910 
911  // Skip to next frame without decoding or caching
912  return;
913  }
914 
915  // Init some local variables (for OpenMP)
916  AVPacket *my_packet = packets[packet];
917 
918  // Debug output
919  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
920 
921  // Init an AVFrame to hold the decoded audio samples
922  int frame_finished = 0;
923  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
924  AV_RESET_FRAME(audio_frame);
925 
926  int packet_samples = 0;
927  int data_size = 0;
928 
929  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
930  int buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE;
931  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, my_packet);
932 
933  if (frame_finished) {
934 
935  // determine how many samples were decoded
936  int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
937  int plane_size = -1;
938  data_size = av_samples_get_buffer_size(&plane_size,
939  aCodecCtx->channels,
940  audio_frame->nb_samples,
941  aCodecCtx->sample_fmt, 1);
942 
943  // Calculate total number of samples
944  packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
945  }
946 
947  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
948  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
949 
950  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
951  int adjusted_pts = packet->pts + audio_pts_offset;
952  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
953  double sample_seconds = float(pts_total) / info.sample_rate;
954 
955  // Debug output
956  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
957  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
958 
959  // DEBUG (FOR AUDIO ISSUES)
960  prev_pts = adjusted_pts;
961  pts_total += pts_remaining_samples;
962  pts_counter++;
963  prev_samples = pts_remaining_samples;
964 
965  // Add audio frame to list of processing audio frames
966  {
967  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
968  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
969  }
970 
971  while (pts_remaining_samples)
972  {
973  // Get Samples per frame (for this frame number)
974  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
975 
976  // Calculate # of samples to add to this frame
977  int samples = samples_per_frame - previous_packet_location.sample_start;
978  if (samples > pts_remaining_samples)
979  samples = pts_remaining_samples;
980 
981  // Decrement remaining samples
982  pts_remaining_samples -= samples;
983 
984  if (pts_remaining_samples > 0) {
985  // next frame
986  previous_packet_location.frame++;
987  previous_packet_location.sample_start = 0;
988 
989  // Add audio frame to list of processing audio frames
990  {
991  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
992  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
993  }
994 
995  } else {
996  // Increment sample start
997  previous_packet_location.sample_start += samples;
998  }
999  }
1000 
1001 
1002 
1003  // Process the audio samples in a separate thread (this includes resampling to 16 bit integer, and storing
1004  // in a openshot::Frame object).
1005  #pragma omp task firstprivate(requested_frame, target_frame, starting_sample, my_packet, audio_frame)
1006  {
1007  // Allocate audio buffer
1008  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE];
1009 
1010  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", aCodecCtx->sample_fmt, "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
1011 
1012  // Create output frame
1013  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1014  AV_RESET_FRAME(audio_converted);
1015  audio_converted->nb_samples = audio_frame->nb_samples;
1016  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1017 
1018  AVAudioResampleContext *avr = NULL;
1019  int nb_samples = 0;
1020  #pragma ordered
1021  {
1022  // setup resample context
1023  avr = avresample_alloc_context();
1024  av_opt_set_int(avr, "in_channel_layout", aCodecCtx->channel_layout, 0);
1025  av_opt_set_int(avr, "out_channel_layout", aCodecCtx->channel_layout, 0);
1026  av_opt_set_int(avr, "in_sample_fmt", aCodecCtx->sample_fmt, 0);
1027  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1028  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1029  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1030  av_opt_set_int(avr, "in_channels", info.channels, 0);
1031  av_opt_set_int(avr, "out_channels", info.channels, 0);
1032  int r = avresample_open(avr);
1033 
1034  // Convert audio samples
1035  nb_samples = avresample_convert(avr, // audio resample context
1036  audio_converted->data, // output data pointers
1037  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1038  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1039  audio_frame->data, // input data pointers
1040  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1041  audio_frame->nb_samples); // number of input samples to convert
1042  }
1043 
1044  // Copy audio samples over original samples
1045  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1046 
1047  // Deallocate resample buffer
1048  avresample_close(avr);
1049  avresample_free(&avr);
1050  avr = NULL;
1051 
1052  // Free AVFrames
1053  av_free(audio_converted->data[0]);
1054  AV_FREE_FRAME(&audio_converted);
1055 
1056  int starting_frame_number = -1;
1057  bool partial_frame = true;
1058  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1059  {
1060  // Array of floats (to hold samples for each channel)
1061  starting_frame_number = target_frame;
1062  int channel_buffer_size = packet_samples / info.channels;
1063  float *channel_buffer = new float[channel_buffer_size];
1064 
1065  // Init buffer array
1066  for (int z = 0; z < channel_buffer_size; z++)
1067  channel_buffer[z] = 0.0f;
1068 
1069  // Loop through all samples and add them to our Frame based on channel.
1070  // Toggle through each channel number, since channel data is stored like (left right left right)
1071  int channel = 0;
1072  int position = 0;
1073  for (int sample = 0; sample < packet_samples; sample++)
1074  {
1075  // Only add samples for current channel
1076  if (channel_filter == channel)
1077  {
1078  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1079  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1080 
1081  // Increment audio position
1082  position++;
1083  }
1084 
1085  // increment channel (if needed)
1086  if ((channel + 1) < info.channels)
1087  // move to next channel
1088  channel ++;
1089  else
1090  // reset channel
1091  channel = 0;
1092  }
1093 
1094  // Loop through samples, and add them to the correct frames
1095  int start = starting_sample;
1096  int remaining_samples = channel_buffer_size;
1097  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1098  while (remaining_samples > 0)
1099  {
1100  // Get Samples per frame (for this frame number)
1101  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1102 
1103  // Calculate # of samples to add to this frame
1104  int samples = samples_per_frame - start;
1105  if (samples > remaining_samples)
1106  samples = remaining_samples;
1107 
1108  // Create or get the existing frame object
1109  tr1::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1110 
1111  // Determine if this frame was "partially" filled in
1112  if (samples_per_frame == start + samples)
1113  partial_frame = false;
1114  else
1115  partial_frame = true;
1116 
1117  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1118  // some louder samples from maxing out at 1.0 (not sure why this happens)
1119  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1120 
1121  // Debug output
1122  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1123 
1124  // Add or update cache
1125  working_cache.Add(f->number, f);
1126 
1127  // Decrement remaining samples
1128  remaining_samples -= samples;
1129 
1130  // Increment buffer (to next set of samples)
1131  if (remaining_samples > 0)
1132  iterate_channel_buffer += samples;
1133 
1134  // Increment frame number
1135  starting_frame_number++;
1136 
1137  // Reset starting sample #
1138  start = 0;
1139  }
1140 
1141  // clear channel buffer
1142  delete[] channel_buffer;
1143  channel_buffer = NULL;
1144  iterate_channel_buffer = NULL;
1145  }
1146 
1147  // Clean up some arrays
1148  delete[] audio_buf;
1149  audio_buf = NULL;
1150 
1151  // Remove audio frame from list of processing audio frames
1152  {
1153  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1154  // Update all frames as completed
1155  for (long int f = target_frame; f < starting_frame_number; f++) {
1156  // Remove the frame # from the processing list. NOTE: If more than one thread is
1157  // processing this frame, the frame # will be in this list multiple times. We are only
1158  // removing a single instance of it here.
1159  processing_audio_frames.erase(processing_audio_frames.find(f));
1160 
1161  // Check and see if this frame is also being processed by another thread
1162  if (processing_audio_frames.count(f) == 0)
1163  // No other thread is processing it. Mark the audio as processed (final)
1164  processed_audio_frames[f] = f;
1165  }
1166 
1167  if (target_frame == starting_frame_number) {
1168  // This typically never happens, but just in case, remove the currently processing number
1169  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1170  }
1171  }
1172 
1173  // Remove this packet
1174  RemoveAVPacket(my_packet);
1175 
1176  // Debug output
1177  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1178 
1179  } // end task
1180 
1181 
1182  // TODO: Fix this bug. Wait on the task to complete. This is not ideal, but solves an issue with the
1183  // audio_frame being modified by the next call to this method. I think this is a scope issue with OpenMP.
1184  #pragma omp taskwait
1185 
1186  // Free audio frame
1187  AV_FREE_FRAME(&audio_frame);
1188 }
1189 
1190 
1191 
1192 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1193 void FFmpegReader::Seek(long int requested_frame) throw(TooManySeeks)
1194 {
1195  // Adjust for a requested frame that is too small or too large
1196  if (requested_frame < 1)
1197  requested_frame = 1;
1198  if (requested_frame > info.video_length)
1199  requested_frame = info.video_length;
1200 
1201  // Debug output
1202  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "", -1, "", -1, "", -1);
1203 
1204  // Clear working cache (since we are seeking to another location in the file)
1205  working_cache.Clear();
1206  missing_frames.Clear();
1207 
1208  // Clear processed lists
1209  {
1210  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1211  processing_audio_frames.clear();
1212  processing_video_frames.clear();
1213  processed_video_frames.clear();
1214  processed_audio_frames.clear();
1215  duplicate_video_frames.clear();
1216  missing_audio_frames.clear();
1217  missing_video_frames.clear();
1218  checked_frames.clear();
1219  }
1220 
1221  // Reset the last frame variable
1222  last_frame = 0;
1223  current_video_frame = 0;
1224  largest_frame_processed = 0;
1225  num_checks_since_final = 0;
1226  num_packets_since_video_frame = 0;
1227  has_missing_frames = false;
1228 
1229  // Increment seek count
1230  seek_count++;
1231 
1232  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1233  int buffer_amount = 6;
1234  if (requested_frame - buffer_amount < 20)
1235  {
1236  // Close and re-open file (basically seeking to frame 1)
1237  Close();
1238  Open();
1239 
1240  // Not actually seeking, so clear these flags
1241  is_seeking = false;
1242  if (seek_count == 1) {
1243  // Don't redefine this on multiple seek attempts for a specific frame
1244  seeking_frame = 1;
1245  seeking_pts = ConvertFrameToVideoPTS(1);
1246  }
1247  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1248  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1249  }
1250  else
1251  {
1252  // Seek to nearest key-frame (aka, i-frame)
1253  bool seek_worked = false;
1254  int64_t seek_target = 0;
1255 
1256  // Seek video stream (if any)
1257  if (!seek_worked && info.has_video)
1258  {
1259  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1260  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1261  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->filename);
1262  } else
1263  {
1264  // VIDEO SEEK
1265  is_video_seek = true;
1266  seek_worked = true;
1267  }
1268  }
1269 
1270  // Seek audio stream (if not already seeked... and if an audio stream is found)
1271  if (!seek_worked && info.has_audio)
1272  {
1273  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1274  if (info.has_audio && av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1275  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->filename);
1276  } else
1277  {
1278  // AUDIO SEEK
1279  is_video_seek = false;
1280  seek_worked = true;
1281  }
1282  }
1283 
1284  // Was the seek successful?
1285  if (seek_worked)
1286  {
1287  // Flush audio buffer
1288  if (info.has_audio)
1289  avcodec_flush_buffers(aCodecCtx);
1290 
1291  // Flush video buffer
1292  if (info.has_video)
1293  avcodec_flush_buffers(pCodecCtx);
1294 
1295  // Reset previous audio location to zero
1296  previous_packet_location.frame = -1;
1297  previous_packet_location.sample_start = 0;
1298 
1299  // init seek flags
1300  is_seeking = true;
1301  if (seek_count == 1) {
1302  // Don't redefine this on multiple seek attempts for a specific frame
1303  seeking_pts = seek_target;
1304  seeking_frame = requested_frame;
1305  }
1306  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1307  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1308 
1309  }
1310  else
1311  {
1312  // seek failed
1313  is_seeking = false;
1314  seeking_pts = 0;
1315  seeking_frame = 0;
1316 
1317  // dislable seeking for this reader (since it failed)
1318  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1319  enable_seek = false;
1320 
1321  // Close and re-open file (basically seeking to frame 1)
1322  Close();
1323  Open();
1324  }
1325  }
1326 }
1327 
1328 // Get the PTS for the current video packet
1329 long int FFmpegReader::GetVideoPTS()
1330 {
1331  int current_pts = 0;
1332  if(packet->dts != AV_NOPTS_VALUE)
1333  current_pts = packet->dts;
1334 
1335  // Return adjusted PTS
1336  return current_pts;
1337 }
1338 
1339 // Update PTS Offset (if any)
1340 void FFmpegReader::UpdatePTSOffset(bool is_video)
1341 {
1342  // Determine the offset between the PTS and Frame number (only for 1st frame)
1343  if (is_video)
1344  {
1345  // VIDEO PACKET
1346  if (video_pts_offset == 99999) // Has the offset been set yet?
1347  // Find the difference between PTS and frame number
1348  video_pts_offset = 0 - GetVideoPTS();
1349  }
1350  else
1351  {
1352  // AUDIO PACKET
1353  if (audio_pts_offset == 99999) // Has the offset been set yet?
1354  // Find the difference between PTS and frame number
1355  audio_pts_offset = 0 - packet->pts;
1356  }
1357 }
1358 
1359 // Convert PTS into Frame Number
1360 long int FFmpegReader::ConvertVideoPTStoFrame(long int pts)
1361 {
1362  // Apply PTS offset
1363  pts = pts + video_pts_offset;
1364  long int previous_video_frame = current_video_frame;
1365 
1366  // Get the video packet start time (in seconds)
1367  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1368 
1369  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1370  long int frame = round(video_seconds * info.fps.ToDouble()) + 1;
1371 
1372  // Keep track of the expected video frame #
1373  if (current_video_frame == 0)
1374  current_video_frame = frame;
1375  else {
1376 
1377  // Sometimes frames are duplicated due to identical (or similar) timestamps
1378  if (frame == previous_video_frame) {
1379  duplicate_video_frames.insert(pair<int,int>(frame, frame));
1380 
1381  // return -1 frame number
1382  frame = -1;
1383  }
1384  else
1385  // Increment expected frame
1386  current_video_frame++;
1387 
1388  if (current_video_frame < frame)
1389  // has missing frames
1390  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1391 
1392  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1393  // if we are missing a video frame.
1394  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1395  while (current_video_frame < frame) {
1396  if (!missing_video_frames.count(current_video_frame)) {
1397  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1398  missing_video_frames.insert(pair<long int, long int>(previous_video_frame, current_video_frame));
1399  }
1400 
1401  // Mark this reader as containing missing frames
1402  has_missing_frames = true;
1403 
1404  // Increment current frame
1405  current_video_frame++;
1406  }
1407  }
1408 
1409  // Return frame #
1410  return frame;
1411 }
1412 
1413 // Convert Frame Number into Video PTS
1414 long int FFmpegReader::ConvertFrameToVideoPTS(long int frame_number)
1415 {
1416  // Get timestamp of this frame (in seconds)
1417  double seconds = double(frame_number) / info.fps.ToDouble();
1418 
1419  // Calculate the # of video packets in this timestamp
1420  long int video_pts = round(seconds / info.video_timebase.ToDouble());
1421 
1422  // Apply PTS offset (opposite)
1423  return video_pts - video_pts_offset;
1424 }
1425 
1426 // Convert Frame Number into Video PTS
1427 long int FFmpegReader::ConvertFrameToAudioPTS(long int frame_number)
1428 {
1429  // Get timestamp of this frame (in seconds)
1430  double seconds = double(frame_number) / info.fps.ToDouble();
1431 
1432  // Calculate the # of audio packets in this timestamp
1433  long int audio_pts = round(seconds / info.audio_timebase.ToDouble());
1434 
1435  // Apply PTS offset (opposite)
1436  return audio_pts - audio_pts_offset;
1437 }
1438 
1439 // Calculate Starting video frame and sample # for an audio PTS
1440 AudioLocation FFmpegReader::GetAudioPTSLocation(long int pts)
1441 {
1442  // Apply PTS offset
1443  pts = pts + audio_pts_offset;
1444 
1445  // Get the audio packet start time (in seconds)
1446  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1447 
1448  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1449  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1450 
1451  // Frame # as a whole number (no more decimals)
1452  int whole_frame = int(frame);
1453 
1454  // Remove the whole number, and only get the decimal of the frame
1455  double sample_start_percentage = frame - double(whole_frame);
1456 
1457  // Get Samples per frame
1458  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1459 
1460  // Calculate the sample # to start on
1461  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1462 
1463  // Protect against broken (i.e. negative) timestamps
1464  if (whole_frame < 1)
1465  whole_frame = 1;
1466  if (sample_start < 0)
1467  sample_start = 0;
1468 
1469  // Prepare final audio packet location
1470  AudioLocation location = {whole_frame, sample_start};
1471 
1472  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1473  if (previous_packet_location.frame != -1 && location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1474  {
1475  int orig_frame = location.frame;
1476  int orig_start = location.sample_start;
1477 
1478  // Update sample start, to prevent gaps in audio
1479  if (previous_packet_location.sample_start <= samples_per_frame)
1480  {
1481  location.sample_start = previous_packet_location.sample_start;
1482  location.frame = previous_packet_location.frame;
1483  }
1484  else
1485  {
1486  // set to next frame (since we exceeded the # of samples on a frame)
1487  location.sample_start = 0;
1488  location.frame++;
1489  }
1490 
1491  // Debug output
1492  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1493 
1494  } else {
1495  // Debug output
1496  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1497 
1498  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1499  for (long int audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1500  if (!missing_audio_frames.count(audio_frame)) {
1501  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1502  missing_audio_frames.insert(pair<long int, long int>(previous_packet_location.frame - 1, audio_frame));
1503  }
1504  }
1505  }
1506 
1507  // Set previous location
1508  previous_packet_location = location;
1509 
1510  // Return the associated video frame and starting sample #
1511  return location;
1512 }
1513 
1514 // Create a new Frame (or return an existing one) and add it to the working queue.
1515 tr1::shared_ptr<Frame> FFmpegReader::CreateFrame(long int requested_frame)
1516 {
1517  // Check working cache
1518  tr1::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1519  if (!output)
1520  {
1521  // Create a new frame on the working cache
1522  output = tr1::shared_ptr<Frame>(new Frame(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels));
1523  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1524  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1525  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1526 
1527  working_cache.Add(requested_frame, output);
1528 
1529  // Set the largest processed frame (if this is larger)
1530  if (requested_frame > largest_frame_processed)
1531  largest_frame_processed = requested_frame;
1532  }
1533 
1534  // Return new frame
1535  return output;
1536 }
1537 
1538 // Determine if frame is partial due to seek
1539 bool FFmpegReader::IsPartialFrame(long int requested_frame) {
1540 
1541  // Sometimes a seek gets partial frames, and we need to remove them
1542  bool seek_trash = false;
1543  long int max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1544  if (seek_video_frame_found > max_seeked_frame)
1545  max_seeked_frame = seek_video_frame_found;
1546  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1547  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1548  seek_trash = true;
1549 
1550  return seek_trash;
1551 }
1552 
1553 // Check if a frame is missing and attempt to replace it's frame image (and
1554 bool FFmpegReader::CheckMissingFrame(long int requested_frame)
1555 {
1556  // Lock
1557  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1558 
1559  // Init # of times this frame has been checked so far
1560  int checked_count = 0;
1561 
1562  // Increment check count for this frame (or init to 1)
1563  if (checked_frames.count(requested_frame) == 0)
1564  checked_frames[requested_frame] = 1;
1565  else
1566  checked_frames[requested_frame]++;
1567  checked_count = checked_frames[requested_frame];
1568 
1569  // Debug output
1570  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1571 
1572 
1573  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1574  map<long int, long int>::iterator itr;
1575  bool found_missing_frame = false;
1576 
1577  for (itr = missing_video_frames.begin(); itr != missing_video_frames.end(); ++itr) {
1578  // Skip if not requested frame
1579  if (requested_frame == itr->second) {
1580 
1581  found_missing_frame = true;
1582 
1583  // Get the previous frame of this missing frame (if it's available in missing cache)
1584  tr1::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(itr->first);
1585 
1586  // Create blank missing frame
1587  tr1::shared_ptr<Frame> missing_frame = CreateFrame(itr->second);
1588 
1589  // Debug output
1590  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "previous_frame->number", itr->first, "", -1, "", -1, "", -1);
1591 
1592  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1593  if (parent_frame != NULL) {
1594  // Debug output
1595  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "previous_frame->number", parent_frame->number, "", -1, "", -1, "", -1);
1596 
1597  // Add this frame to the processed map (since it's already done)
1598  missing_frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*parent_frame->GetImage())));
1599 
1600  processed_video_frames[missing_frame->number] = missing_frame->number;
1601  processed_audio_frames[missing_frame->number] = missing_frame->number;
1602 
1603  // Move frame to final cache
1604  final_cache.Add(missing_frame->number, missing_frame);
1605 
1606  // Remove frame from working cache
1607  working_cache.Remove(missing_frame->number);
1608 
1609  // Update last_frame processed
1610  last_frame = missing_frame->number;
1611  }
1612 
1613  break; // exit looping
1614  }
1615  }
1616  if (found_missing_frame) {
1617  // Remove missing frame from map
1618  missing_video_frames.erase(itr);
1619 
1620  } else {
1621  // Look for missing audio frames
1622  found_missing_frame = false;
1623  for (itr = missing_audio_frames.begin(); itr != missing_audio_frames.end(); ++itr) {
1624  // Skip if not requested frame
1625  if (requested_frame == itr->second) {
1626 
1627  found_missing_frame = true;
1628 
1629  // Get the previous frame of this missing frame (if it's available in missing cache)
1630  tr1::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(itr->first);
1631 
1632  // Create blank missing frame
1633  tr1::shared_ptr<Frame> missing_frame = CreateFrame(itr->second);
1634 
1635  // Debug output
1636  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Audio Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "previous_frame->number", itr->first, "", -1, "", -1, "", -1);
1637 
1638  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1639  if (parent_frame != NULL) {
1640  // Debug output
1641  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "previous_frame->number", parent_frame->number, "", -1, "", -1, "", -1);
1642 
1643  // Add this frame to the processed map (since it's already done)
1644  missing_frame->AddImage(tr1::shared_ptr<QImage>(new QImage(*parent_frame->GetImage())));
1645 
1646  processed_video_frames[missing_frame->number] = missing_frame->number;
1647  processed_audio_frames[missing_frame->number] = missing_frame->number;
1648 
1649  // Move frame to final cache
1650  final_cache.Add(missing_frame->number, missing_frame);
1651 
1652  // Remove frame from working cache
1653  working_cache.Remove(missing_frame->number);
1654 
1655  // Update last_frame processed
1656  last_frame = missing_frame->number;
1657  }
1658 
1659  break; // exit looping
1660  }
1661  }
1662  if (found_missing_frame) {
1663  // Remove missing frame from map
1664  missing_audio_frames.erase(itr);
1665  }
1666  }
1667 
1668  return found_missing_frame;
1669 }
1670 
1671 // Check the working queue, and move finished frames to the finished queue
1672 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, long int requested_frame)
1673 {
1674  // Loop through all working queue frames
1675  while (true)
1676  {
1677  // Get the front frame of working cache
1678  tr1::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1679 
1680  // Was a frame found?
1681  if (!f)
1682  // No frames found
1683  break;
1684 
1685  // Check if this frame is 'missing'
1686  CheckMissingFrame(f->number);
1687 
1688  // Init # of times this frame has been checked so far
1689  int checked_count = 0;
1690 
1691  bool is_video_ready = false;
1692  bool is_audio_ready = false;
1693  { // limit scope of next few lines
1694  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1695  is_video_ready = processed_video_frames.count(f->number);
1696  is_audio_ready = processed_audio_frames.count(f->number);
1697 
1698  // Get check count for this frame
1699  checked_count = checked_frames[f->number];
1700  }
1701 
1702  if (previous_packet_location.frame == f->number && !end_of_stream)
1703  is_audio_ready = false; // don't finalize the last processed audio frame
1704  bool is_seek_trash = IsPartialFrame(f->number);
1705 
1706  // Adjust for available streams
1707  if (!info.has_video) is_video_ready = true;
1708  if (!info.has_audio) is_audio_ready = true;
1709 
1710  // Make final any frames that get stuck (for whatever reason)
1711  if (checked_count > 40 && (!is_video_ready || !is_audio_ready)) {
1712  // Debug output
1713  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames.size()", checked_frames.size(), "", -1);
1714 
1715  if (info.has_video && !is_video_ready && last_video_frame) {
1716  // Copy image from last frame
1717  f->AddImage(tr1::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1718  is_video_ready = true;
1719  }
1720 
1721  if (info.has_audio && !is_audio_ready) {
1722  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1723  // Mark audio as processed, and indicate the frame has audio data
1724  is_audio_ready = true;
1725  }
1726  }
1727 
1728  // Debug output
1729  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames.size()", checked_frames.size(), "", -1);
1730 
1731  // Check if working frame is final
1732  if ((!end_of_stream && is_video_ready && is_audio_ready && f->number <= requested_frame) || end_of_stream || is_seek_trash)
1733  {
1734  // Debug output
1735  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1, "", -1);
1736 
1737  if (!is_seek_trash)
1738  {
1739  // Reset counter since last 'final' frame
1740  num_checks_since_final = 0;
1741 
1742  // Move frame to final cache
1743  final_cache.Add(f->number, f);
1744 
1745  // Add to missing cache (if another frame depends on it)
1746  {
1747  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1748  if (!missing_video_frames.count(current_video_frame)) {
1749  // Debug output
1750  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1751 
1752  missing_frames.Add(f->number, f);
1753  }
1754  }
1755 
1756  // Remove frame from working cache
1757  working_cache.Remove(f->number);
1758 
1759  // Update last frame processed
1760  last_frame = f->number;
1761 
1762  // Remove from 'checked' count
1763  checked_frames.erase(f->number);
1764 
1765  } else {
1766  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1767  working_cache.Remove(f->number);
1768  }
1769  }
1770  else
1771  // Stop looping
1772  break;
1773  }
1774 }
1775 
1776 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1777 void FFmpegReader::CheckFPS()
1778 {
1779  check_fps = true;
1780  avpicture_alloc(pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1781 
1782  int first_second_counter = 0;
1783  int second_second_counter = 0;
1784  int third_second_counter = 0;
1785  int forth_second_counter = 0;
1786  int fifth_second_counter = 0;
1787 
1788  int iterations = 0;
1789  int threshold = 500;
1790 
1791  // Loop through the stream
1792  while (true)
1793  {
1794  // Get the next packet (if any)
1795  if (GetNextPacket() < 0)
1796  // Break loop when no more packets found
1797  break;
1798 
1799  // Video packet
1800  if (packet->stream_index == videoStream)
1801  {
1802  // Check if the AVFrame is finished and set it
1803  if (GetAVFrame())
1804  {
1805  // Update PTS / Frame Offset (if any)
1806  UpdatePTSOffset(true);
1807 
1808  // Get PTS of this packet
1809  long int pts = GetVideoPTS();
1810 
1811  // Remove pFrame
1812  RemoveAVFrame(pFrame);
1813 
1814  // remove packet
1815  RemoveAVPacket(packet);
1816 
1817  // Apply PTS offset
1818  pts += video_pts_offset;
1819 
1820  // Get the video packet start time (in seconds)
1821  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1822 
1823  // Increment the correct counter
1824  if (video_seconds <= 1.0)
1825  first_second_counter++;
1826  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1827  second_second_counter++;
1828  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1829  third_second_counter++;
1830  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1831  forth_second_counter++;
1832  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1833  fifth_second_counter++;
1834  else
1835  // Too far
1836  break;
1837  }
1838  else
1839  // remove packet
1840  RemoveAVPacket(packet);
1841  }
1842  else
1843  // remove packet
1844  RemoveAVPacket(packet);
1845 
1846  // Increment counters
1847  iterations++;
1848 
1849  // Give up (if threshold exceeded)
1850  if (iterations > threshold)
1851  break;
1852  }
1853 
1854  // Double check that all counters have greater than zero (or give up)
1855  if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1856  {
1857  // Seek to frame 1
1858  Seek(1);
1859 
1860  // exit with no changes to FPS (not enough data to calculate)
1861  return;
1862  }
1863 
1864  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1865  int avg_fps = round(sum_fps / 4.0f);
1866 
1867  // Sometimes the FPS is incorrectly detected by FFmpeg. If the 1st and 2nd seconds counters
1868  // agree with each other, we are going to adjust the FPS of this reader instance. Otherwise, print
1869  // a warning message.
1870 
1871  // Get diff from actual frame rate
1872  double fps = info.fps.ToDouble();
1873  double diff = fps - double(avg_fps);
1874 
1875  // Is difference bigger than 1 frame?
1876  if (diff <= -1 || diff >= 1)
1877  {
1878  // Compare to half the frame rate (the most common type of issue)
1879  double half_fps = Fraction(info.fps.num / 2, info.fps.den).ToDouble();
1880  diff = half_fps - double(avg_fps);
1881 
1882  // Is difference bigger than 1 frame?
1883  if (diff <= -1 || diff >= 1)
1884  {
1885  // Update FPS for this reader instance
1886  info.fps = Fraction(avg_fps, 1);
1887  }
1888  else
1889  {
1890  // Update FPS for this reader instance (to 1/2 the original framerate)
1891  info.fps = Fraction(info.fps.num / 2, info.fps.den);
1892  }
1893  }
1894 
1895  // Seek to frame 1
1896  Seek(1);
1897 }
1898 
1899 // Remove AVFrame from cache (and deallocate it's memory)
1900 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1901 {
1902  #pragma omp critical (packet_cache)
1903  {
1904  // Remove pFrame (if exists)
1905  if (frames.count(remove_frame))
1906  {
1907  // Free memory
1908  avpicture_free(frames[remove_frame]);
1909 
1910  // Remove from cache
1911  frames.erase(remove_frame);
1912 
1913  // Delete the object
1914  delete remove_frame;
1915  }
1916 
1917  } // end omp critical
1918 }
1919 
1920 // Remove AVPacket from cache (and deallocate it's memory)
1921 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1922 {
1923  #pragma omp critical (packet_cache)
1924  {
1925  // Remove packet (if any)
1926  if (packets.count(remove_packet))
1927  {
1928  // deallocate memory for packet
1929  av_free_packet(remove_packet);
1930 
1931  // Remove from cache
1932  packets.erase(remove_packet);
1933 
1934  // Delete the object
1935  delete remove_packet;
1936  }
1937 
1938  } // end omp critical
1939 }
1940 
1941 /// Get the smallest video frame that is still being processed
1942 long int FFmpegReader::GetSmallestVideoFrame()
1943 {
1944  // Loop through frame numbers
1945  map<long int, long int>::iterator itr;
1946  int smallest_frame = -1;
1947  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1948  {
1949  if (itr->first < smallest_frame || smallest_frame == -1)
1950  smallest_frame = itr->first;
1951  }
1952 
1953  // Return frame number
1954  return smallest_frame;
1955 }
1956 
1957 /// Get the smallest audio frame that is still being processed
1958 long int FFmpegReader::GetSmallestAudioFrame()
1959 {
1960  // Loop through frame numbers
1961  map<long int, long int>::iterator itr;
1962  int smallest_frame = -1;
1963  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1964  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1965  {
1966  if (itr->first < smallest_frame || smallest_frame == -1)
1967  smallest_frame = itr->first;
1968  }
1969 
1970  // Return frame number
1971  return smallest_frame;
1972 }
1973 
1974 // Generate JSON string of this object
1976 
1977  // Return formatted string
1978  return JsonValue().toStyledString();
1979 }
1980 
1981 // Generate Json::JsonValue for this object
1983 
1984  // Create root json object
1985  Json::Value root = ReaderBase::JsonValue(); // get parent properties
1986  root["type"] = "FFmpegReader";
1987  root["path"] = path;
1988 
1989  // return JsonValue
1990  return root;
1991 }
1992 
1993 // Load JSON string into this object
1994 void FFmpegReader::SetJson(string value) throw(InvalidJSON) {
1995 
1996  // Parse JSON string into JSON objects
1997  Json::Value root;
1998  Json::Reader reader;
1999  bool success = reader.parse( value, root );
2000  if (!success)
2001  // Raise exception
2002  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
2003 
2004  try
2005  {
2006  // Set all values that match
2007  SetJsonValue(root);
2008  }
2009  catch (exception e)
2010  {
2011  // Error parsing JSON (or missing keys)
2012  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
2013  }
2014 }
2015 
2016 // Load Json::JsonValue into this object
2017 void FFmpegReader::SetJsonValue(Json::Value root) throw(InvalidFile) {
2018 
2019  // Set parent data
2021 
2022  // Set data from Json (if key is found)
2023  if (!root["path"].isNull())
2024  path = root["path"].asString();
2025 
2026  // Re-Open path, and re-init everything (if needed)
2027  if (is_open)
2028  {
2029  Close();
2030  Open();
2031  }
2032 }
#define AV_RESET_FRAME(av_frame)
long long file_size
Size of file (in bytes)
Definition: ReaderBase.h:65
tr1::shared_ptr< Frame > GetFrame(long int requested_frame)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
void Add(long int frame_number, tr1::shared_ptr< Frame > frame)
Add a Frame to the cache.
Definition: Cache.cpp:57
CriticalSection processingCriticalSection
Definition: ReaderBase.h:100
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
void Remove(long int frame_number)
Remove a specific frame.
Definition: Cache.cpp:152
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:79
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:72
FFmpegReader(string path)
void SetMaxBytesFromInfo(long int number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: Cache.cpp:257
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:80
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:84
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:154
This class represents a fraction.
Definition: Fraction.h:42
Cache final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
tr1::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: Cache.cpp:112
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:104
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:153
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:108
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
int is_near(AudioLocation location, int samples_per_frame, int amount)
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:71
tr1::shared_ptr< Frame > GetFrame(long int frame_number)
Get a frame from the cache.
Definition: Cache.cpp:79
void Clear()
Clear the cache of all frames.
Definition: Cache.cpp:203
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
Definition: Exceptions.h:152
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:68
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:59
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:70
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:85
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:73
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:75
long int video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:497
long int Count()
Count the frames in the queue.
Definition: Cache.cpp:213
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81
Exception when too many seek attempts happen.
Definition: Exceptions.h:254