Merge 34bf80c62d into 5904853ae5

Closes #7442
Authored by: bashonly

test/test_utils.py

@@ -156,6 +156,11 @@ def test_sanitize_filename(self):
         self.assertEqual('yes no', sanitize_filename('yes? no', is_id=False))
         self.assertEqual('this - that', sanitize_filename('this: that', is_id=False))
 
+        self.assertEqual('abc_<>\\*|de', sanitize_filename('abc/<>\\*|de', keep_bad_win_chars=True, is_id=False))
+        self.assertEqual('xxx_<>\\*|', sanitize_filename('xxx/<>\\*|', keep_bad_win_chars=True, is_id=False))
+        self.assertEqual('yes? no', sanitize_filename('yes? no', keep_bad_win_chars=True, is_id=False))
+        self.assertEqual('this: that', sanitize_filename('this: that', keep_bad_win_chars=True, is_id=False))
+
         self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
         aumlaut = 'ä'
         self.assertEqual(sanitize_filename(aumlaut), aumlaut)
@@ -166,6 +171,10 @@ def test_sanitize_filename(self):
             sanitize_filename('New World record at 0:12:34'),
             'New World record at 0_12_34')
 
+        self.assertEqual(
+            sanitize_filename('New World record at 0:12:34', keep_bad_win_chars=True),
+            'New World record at 0:12:34')
+
         self.assertEqual(sanitize_filename('--gasdgf'), '--gasdgf')
         self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
         self.assertEqual(sanitize_filename('--gasdgf', is_id=False), '_-gasdgf')
@@ -217,6 +226,10 @@ def test_sanitize_ids(self):
         self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
         self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
 
+        self.assertEqual(sanitize_filename('_n_cd26wFpw', keep_bad_win_chars=True, is_id=True), '_n_cd26wFpw')
+        self.assertEqual(sanitize_filename('_BD_eEpuzXw', keep_bad_win_chars=True, is_id=True), '_BD_eEpuzXw')
+        self.assertEqual(sanitize_filename('N0Y__7-UOdI', keep_bad_win_chars=True, is_id=True), 'N0Y__7-UOdI')
+
     def test_sanitize_path(self):
         if sys.platform != 'win32':
             return

yt_dlp/YoutubeDL.py

@@ -1293,10 +1293,12 @@ def get_value(mdict):
         na = self.params.get('outtmpl_na_placeholder', 'NA')
 
         def filename_sanitizer(key, value, restricted=self.params.get('restrictfilenames')):
-            return sanitize_filename(str(value), restricted=restricted, is_id=(
-                bool(re.search(r'(^|[_.])id(\.|$)', key))
-                if 'filename-sanitization' in self.params['compat_opts']
-                else NO_DEFAULT))
+            return sanitize_filename(
+                str(value), self.params.get('keep_bad_win_chars', False), restricted=restricted,
+                is_id=(
+                    bool(re.search(r'(^|[_.])id(\.|$)', key))
+                    if 'filename-sanitization' in self.params['compat_opts']
+                    else NO_DEFAULT))
 
         sanitizer = sanitize if callable(sanitize) else filename_sanitizer
         sanitize = bool(sanitize)

yt_dlp/__init__.py

@@ -815,6 +815,7 @@ def parse_options(argv=None):
         'autonumber_start': opts.autonumber_start,
         'restrictfilenames': opts.restrictfilenames,
         'windowsfilenames': opts.windowsfilenames,
+        'keep_bad_win_chars': opts.keep_bad_win_chars,
         'ignoreerrors': opts.ignoreerrors,
         'force_generic_extractor': opts.force_generic_extractor,
         'allowed_extractors': opts.allowed_extractors or ['default'],

yt_dlp/extractor/cbc.py

@@ -151,7 +151,7 @@ def _real_extract(self, url):
 
 class CBCPlayerIE(InfoExtractor):
     IE_NAME = 'cbc.ca:player'
-    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
+    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/(?:video/)?|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
     _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
         'md5': '64d25f841ddf4ddb28a235338af32e2c',
@@ -277,6 +277,28 @@ class CBCPlayerIE(InfoExtractor):
             'location': 'Canada',
             'media_type': 'Full Program',
         },
+    }, {
+        'url': 'https://www.cbc.ca/player/play/video/1.7194274',
+        'md5': '188b96cf6bdcb2540e178a6caa957128',
+        'info_dict': {
+            'id': '2334524995812',
+            'ext': 'mp4',
+            'title': '#TheMoment a rare white spirit moose was spotted in Alberta',
+            'description': 'md5:18ae269a2d0265c5b0bbe4b2e1ac61a3',
+            'timestamp': 1714788791,
+            'duration': 77.678,
+            'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/201/543/THE_MOMENT.jpg',
+            'uploader': 'CBCC-NEW',
+            'chapters': 'count:0',
+            'upload_date': '20240504',
+            'categories': 'count:3',
+            'series': 'The National',
+            'tags': 'count:15',
+            'creators': ['encoder'],
+            'location': 'Canada',
+            'media_type': 'Excerpt',
+        },
     }, {
         'url': 'cbcplayer:1.7159484',
         'only_matching': True,

yt_dlp/extractor/common.py

@@ -1011,7 +1011,7 @@ def _request_dump_filename(self, url, video_id):
         if len(basen) > trim_length:
             h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
             basen = basen[:trim_length - len(h)] + h
-        filename = sanitize_filename(f'{basen}.dump', restricted=True)
+        filename = sanitize_filename(f'{basen}.dump', self.get_param('keep_bad_win_chars', False), restricted=True)
         # Working around MAX_PATH limitation on Windows (see
         # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
         if compat_os_name == 'nt':

yt_dlp/extractor/crunchyroll.py

@@ -53,15 +53,19 @@ def _set_auth_info(self, response):
         CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10)
 
     def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'):
-        try:  # TODO: Add impersonation support here
+        try:
             return self._download_json(
                 f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote,
-                headers=headers, data=urlencode_postdata(data))
+                headers=headers, data=urlencode_postdata(data), impersonate=True)
         except ExtractorError as error:
             if not isinstance(error.cause, HTTPError) or error.cause.status != 403:
                 raise
+            if target := error.cause.response.extensions.get('impersonate'):
+                raise ExtractorError(f'Got HTTP Error 403 when using impersonate target "{target}"')
             raise ExtractorError(
-                'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                'Request blocked by Cloudflare. '
+                'Install the required impersonation dependency if possible, '
+                'or else navigate to Crunchyroll in your browser, '
                 'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
                 'and your browser\'s User-Agent (with --user-agent)', expected=True)
 

yt_dlp/extractor/youporn.py

@@ -72,15 +72,15 @@ class YouPornIE(InfoExtractor):
             'id': '16290308',
             'age_limit': 18,
             'categories': [],
-            'description': 'md5:00ea70f642f431c379763c17c2f396bc',
+            'description': str,  # TODO: detect/remove SEO spam description in ytdl backport
             'display_id': 'tinderspecial-trailer1',
             'duration': 298.0,
             'ext': 'mp4',
             'upload_date': '20201123',
             'uploader': 'Ersties',
             'tags': [],
-            'thumbnail': 'https://fi1.ypncdn.com/202011/23/16290308/original/8/tinderspecial-trailer1-8(m=eaAaaEPbaaaa).jpg',
-            'timestamp': 1606089600,
+            'thumbnail': r're:https://.+\.jpg',
+            'timestamp': 1606147564,
             'title': 'Tinder In Real Life',
             'view_count': int,
         }
@@ -88,11 +88,17 @@ class YouPornIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
-        definitions = self._download_json(
-            f'https://www.youporn.com/api/video/media_definitions/{video_id}/', display_id or video_id)
+        self._set_cookie('.youporn.com', 'age_verified', '1')
+        webpage = self._download_webpage(f'https://www.youporn.com/watch/{video_id}', video_id)
+        definitions = self._search_json(r'\bplayervars\s*:', webpage, 'player vars', video_id)['mediaDefinitions']
 
-        def get_format_data(data, f):
-            return traverse_obj(data, lambda _, v: v['format'] == f and url_or_none(v['videoUrl']))
+        def get_format_data(data, stream_type):
+            info_url = traverse_obj(data, (lambda _, v: v['format'] == stream_type, 'videoUrl', {url_or_none}, any))
+            if not info_url:
+                return []
+            return traverse_obj(
+                self._download_json(info_url, video_id, f'Downloading {stream_type} info JSON', fatal=False),
+                lambda _, v: v['format'] == stream_type and url_or_none(v['videoUrl']))
 
         formats = []
         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
@@ -123,10 +129,6 @@ def get_format_data(data, f):
             f['height'] = height
             formats.append(f)
 
-        webpage = self._download_webpage(
-            'http://www.youporn.com/watch/%s' % video_id, display_id,
-            headers={'Cookie': 'age_verified=1'})
-
         title = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
             webpage, 'title', default=None) or self._og_search_title(

yt_dlp/options.py

@@ -267,6 +267,10 @@ def _dict_from_options_callback(
             out_dict[key] = out_dict.get(key, []) + [val] if append else val
         setattr(parser.values, option.dest, out_dict)
 
+    def _store_multiple_callback(option, opt_str, value, parser, values):
+        for key, value in values.items():
+            setattr(parser.values, key, value)
+
     def when_prefix(default):
         return {
             'default': {},
@@ -1360,7 +1364,13 @@ def _alias_callback(option, opt_str, value, parser, opts, nargs):
         help='Force filenames to be Windows-compatible')
     filesystem.add_option(
         '--no-windows-filenames',
-        action='store_false', dest='windowsfilenames',
+        action='callback', dest='keep_bad_win_chars', default=False, callback=_store_multiple_callback,
+        callback_kwargs={
+            'values': {
+                'windowsfilenames': False,
+                'keep_bad_win_chars': True
+            }
+        },
         help='Make filenames Windows-compatible only if using Windows (default)')
     filesystem.add_option(
         '--trim-filenames', '--trim-file-names', metavar='LENGTH',

yt_dlp/utils/_utils.py

@@ -609,11 +609,12 @@ def timeconvert(timestr):
     return timestamp
 
 
-def sanitize_filename(s, restricted=False, is_id=NO_DEFAULT):
+def sanitize_filename(s, keep_bad_win_chars=False, restricted=False, is_id=NO_DEFAULT):
     """Sanitizes a string so it could be used as part of a filename.
-    @param restricted   Use a stricter subset of allowed characters
-    @param is_id        Whether this is an ID that should be kept unchanged if possible.
-                        If unset, yt-dlp's new sanitization rules are in effect
+    @param keep_bad_win_chars    Whether to keep characters invalid on Windows
+    @param restricted            Use a stricter subset of allowed characters
+    @param is_id                 Whether this is an ID that should be kept unchanged if possible.
+                                 If unset, yt-dlp's new sanitization rules are in effect
     """
     if s == '':
         return ''
@@ -623,16 +624,16 @@ def replace_insane(char):
             return ACCENT_CHARS[char]
         elif not restricted and char == '\n':
             return '\0 '
-        elif is_id is NO_DEFAULT and not restricted and char in '"*:<>?|/\\':
+        elif is_id is NO_DEFAULT and not restricted and char in '"*:<>?|/\\' and not keep_bad_win_chars:
             # Replace with their full-width unicode counterparts
             return {'/': '\u29F8', '\\': '\u29f9'}.get(char, chr(ord(char) + 0xfee0))
-        elif char == '?' or ord(char) < 32 or ord(char) == 127:
+        elif (not keep_bad_win_chars and char == '?') or ord(char) < 32 or ord(char) == 127:
             return ''
-        elif char == '"':
+        elif not keep_bad_win_chars and char == '"':
             return '' if restricted else '\''
-        elif char == ':':
+        elif not keep_bad_win_chars and char == ':':
             return '\0_\0-' if restricted else '\0 \0-'
-        elif char in '\\/|*<>':
+        elif (not keep_bad_win_chars and char in '\\|*<>') or char == '/':
             return '\0_'
         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace() or ord(char) > 127):
             return '' if unicodedata.category(char)[0] in 'CM' else '\0_'
@@ -641,7 +642,8 @@ def replace_insane(char):
     # Replace look-alike Unicode glyphs
     if restricted and (is_id is NO_DEFAULT or not is_id):
         s = unicodedata.normalize('NFKC', s)
-    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)  # Handle timestamps
+    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0) if keep_bad_win_chars
+               else m.group(0).replace(':', '_'), s)  # Handle timestamps
     result = ''.join(map(replace_insane, s))
     if is_id is NO_DEFAULT:
         result = re.sub(r'(\0.)(?:(?=\1)..)+', r'\1', result)  # Remove repeated substitute chars