diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index d7780093f8..2ab721d0e8 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -10,12 +10,9 @@ import time import mimetypes -import subprocess -import json -from datetime import timedelta from requests.exceptions import RequestException, ConnectionError, Timeout from .common import DownloaderBase -from .. import text, util +from .. import text, util, ffprobe from ssl import SSLError @@ -37,8 +34,6 @@ def __init__(self, job): self.maxsize = self.config("filesize-max") self.minlength = self.config("videolength-min") self.maxlength = self.config("videolength-max") - ffprobe = self.config("ffprobe-location") - self.ffprobe = util.expand_path(ffprobe) if ffprobe else "ffprobe" self.retries = self.config("retries", extractor._retries) self.retry_codes = self.config("retry-codes", extractor._retry_codes) self.timeout = self.config("timeout", extractor._timeout) @@ -242,7 +237,7 @@ def _download_impl(self, url, pathfmt): # check video length using ffprobe request if (self.minlength or self.maxlength): - length = self._fetch_videolength(url) + length = ffprobe.get_video_length(self, url) if length and self.minlength and length < self.minlength: self.release_conn(response) @@ -419,64 +414,6 @@ def _adjust_extension(pathfmt, file_header): return True return False - def _fetch_videolength(self, url): - minimum_frames = 10 - args = [ - self.ffprobe, - "-v", - "quiet", - "-print_format", - "json", - "-show_format", - "-show_streams", - url, - ] - - try: - result = subprocess.run( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=True, - ) - data = json.loads(result.stdout) - - video_streams = [ - float(stream["duration"]) - for stream in data["streams"] - if stream["codec_type"] == "video" and - "duration" in stream and - "avg_frame_rate" in stream and - self._frame_count(stream) >= minimum_frames - ] - - if not video_streams: - self.log.info( - "No video streams found or none with a valid duration " - "and minimum frames." - ) - return None - - duration = timedelta(seconds=min(video_streams)) - return duration - - except subprocess.CalledProcessError as e: - self.log.error("ffprobe failed: %s", e.stderr) - return None - except json.JSONDecodeError: - self.log.error("Failed to decode ffprobe output as JSON") - return None - - def _frame_count(self, stream): - """Calculates the number of frames in the video stream.""" - try: - duration = float(stream["duration"]) - avg_frame_rate = eval(stream["avg_frame_rate"]) - return int(duration * avg_frame_rate) - except (ValueError, ZeroDivisionError): - return 0 - MIME_TYPES = { "image/jpeg" : "jpg", diff --git a/gallery_dl/ffprobe.py b/gallery_dl/ffprobe.py new file mode 100644 index 0000000000..7ae2839d33 --- /dev/null +++ b/gallery_dl/ffprobe.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014-2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Fetch Video Length before actually downloading a whole file""" + +import subprocess +import json +import time +from datetime import timedelta +from . import util + + +def get_video_length(obj, url): + minimum_frames = 10 + data = None + tries = 0 + msg = "" + + ffprobe = util.expand_path(obj.config("ffprobe-location", "ffprobe")) + + command = [ + ffprobe, + "-v", + "quiet", + "-print_format", + "json", + "-show_format", + "-show_streams", + ] + + if obj.headers: + for key, value in obj.headers.items(): + command.extend(["-headers", f"{key}: {value}"]) + + command.append(url) + + while True: + if tries: + obj.log.warning("%s (%s/%s)", msg, tries, obj.retries+1) + if tries > obj.retries: + return False + time.sleep(tries) + tries += 1 + + try: + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + data = json.loads(result.stdout) + except subprocess.CalledProcessError as e: + msg = f"ffprobe failed: {e}" + print(e) + continue + except json.JSONDecodeError: + msg = "Failed to decode ffprobe output as JSON" + continue + + # A file typically contains multiple streams (video, audio, subtitle). + # Here we filter out everything that is not considered a video + video_streams = [ + float(stream["duration"]) + for stream in data["streams"] + if stream["codec_type"] == "video" and + "duration" in stream and + "avg_frame_rate" in stream and + frame_count(stream) >= minimum_frames + ] + + if not video_streams: + obj.log.info( + "No video streams found or none with a valid duration " + "and minimum frames." + ) + return None + + duration = timedelta(seconds=min(video_streams)) + return duration + + +def frame_count(stream): + """Calculates the number of frames in the video stream.""" + try: + duration = float(stream["duration"]) + avg_frame_rate = eval(stream["avg_frame_rate"]) + return int(duration * avg_frame_rate) + except (ValueError, ZeroDivisionError): + return 0