Add Headers and Retry Mechanism to ffprobe Utility

gallery_dl/downloader/http.py

@@ -10,12 +10,9 @@
 
 import time
 import mimetypes
-import subprocess
-import json
-from datetime import timedelta
 from requests.exceptions import RequestException, ConnectionError, Timeout
 from .common import DownloaderBase
-from .. import text, util
+from .. import text, util, ffprobe
 from ssl import SSLError
 
 
@@ -37,8 +34,6 @@ def __init__(self, job):
         self.maxsize = self.config("filesize-max")
         self.minlength = self.config("videolength-min")
         self.maxlength = self.config("videolength-max")
-        ffprobe = self.config("ffprobe-location")
-        self.ffprobe = util.expand_path(ffprobe) if ffprobe else "ffprobe"
         self.retries = self.config("retries", extractor._retries)
         self.retry_codes = self.config("retry-codes", extractor._retry_codes)
         self.timeout = self.config("timeout", extractor._timeout)
@@ -242,7 +237,7 @@ def _download_impl(self, url, pathfmt):
 
             # check video length using ffprobe request
             if (self.minlength or self.maxlength):
-                length = self._fetch_videolength(url)
+                length = ffprobe.get_video_length(self, url)
 
                 if length and self.minlength and length < self.minlength:
                     self.release_conn(response)
@@ -419,64 +414,6 @@ def _adjust_extension(pathfmt, file_header):
                     return True
         return False
 
-    def _fetch_videolength(self, url):
-        minimum_frames = 10
-        args = [
-            self.ffprobe,
-            "-v",
-            "quiet",
-            "-print_format",
-            "json",
-            "-show_format",
-            "-show_streams",
-            url,
-        ]
-
-        try:
-            result = subprocess.run(
-                args,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                text=True,
-                check=True,
-            )
-            data = json.loads(result.stdout)
-
-            video_streams = [
-                float(stream["duration"])
-                for stream in data["streams"]
-                if stream["codec_type"] == "video" and
-                "duration" in stream and
-                "avg_frame_rate" in stream and
-                self._frame_count(stream) >= minimum_frames
-            ]
-
-            if not video_streams:
-                self.log.info(
-                    "No video streams found or none with a valid duration "
-                    "and minimum frames."
-                )
-                return None
-
-            duration = timedelta(seconds=min(video_streams))
-            return duration
-
-        except subprocess.CalledProcessError as e:
-            self.log.error("ffprobe failed: %s", e.stderr)
-            return None
-        except json.JSONDecodeError:
-            self.log.error("Failed to decode ffprobe output as JSON")
-            return None
-
-    def _frame_count(self, stream):
-        """Calculates the number of frames in the video stream."""
-        try:
-            duration = float(stream["duration"])
-            avg_frame_rate = eval(stream["avg_frame_rate"])
-            return int(duration * avg_frame_rate)
-        except (ValueError, ZeroDivisionError):
-            return 0
-
 
 MIME_TYPES = {
     "image/jpeg"    : "jpg",

gallery_dl/ffprobe.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014-2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Fetch Video Length before actually downloading a whole file"""
+
+import subprocess
+import json
+import time
+from datetime import timedelta
+from . import util
+
+
+def get_video_length(obj, url):
+    minimum_frames = 10
+    data = None
+    tries = 0
+    msg = ""
+
+    ffprobe = util.expand_path(obj.config("ffprobe-location", "ffprobe"))
+
+    command = [
+        ffprobe,
+        "-v",
+        "quiet",
+        "-print_format",
+        "json",
+        "-show_format",
+        "-show_streams",
+    ]
+
+    if obj.headers:
+        for key, value in obj.headers.items():
+            command.extend(["-headers", f"{key}: {value}"])
+
+    command.append(url)
+
+    while True:
+        if tries:
+            obj.log.warning("%s (%s/%s)", msg, tries, obj.retries+1)
+            if tries > obj.retries:
+                return False
+            time.sleep(tries)
+        tries += 1
+
+        try:
+            result = subprocess.run(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
+            data = json.loads(result.stdout)
+        except subprocess.CalledProcessError as e:
+            msg = f"ffprobe failed: {e}"
+            print(e)
+            continue
+        except json.JSONDecodeError:
+            msg = "Failed to decode ffprobe output as JSON"
+            continue
+
+        # A file typically contains multiple streams (video, audio, subtitle).
+        # Here we filter out everything that is not considered a video
+        video_streams = [
+            float(stream["duration"])
+            for stream in data["streams"]
+            if stream["codec_type"] == "video" and
+            "duration" in stream and
+            "avg_frame_rate" in stream and
+            frame_count(stream) >= minimum_frames
+        ]
+
+        if not video_streams:
+            obj.log.info(
+                "No video streams found or none with a valid duration "
+                "and minimum frames."
+            )
+            return None
+
+        duration = timedelta(seconds=min(video_streams))
+        return duration
+
+
+def frame_count(stream):
+    """Calculates the number of frames in the video stream."""
+    try:
+        duration = float(stream["duration"])
+        avg_frame_rate = eval(stream["avg_frame_rate"])
+        return int(duration * avg_frame_rate)
+    except (ValueError, ZeroDivisionError):
+        return 0