Merge f2767e2ce0 into f2a4983df7

README.md

@@ -1768,7 +1768,7 @@ The following extractors use this feature:
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube.py](https://github.com/yt-dlp/yt-dlp/blob/c26f9b991a0681fd3ea548d535919cec1fbbd430/yt_dlp/extractor/youtube.py#L381-L390) for list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The main clients are `web`, `ios` and `android`, with variants `_music` and `_creator` (e.g. `ios_creator`); and `mweb`, `mediaconnect`, `android_vr`, `web_safari`, `web_embedded`, `tv` and `tv_embedded` with no variants. By default, `ios,mweb` is used, and `web_creator` is added as needed for age-gated videos when account age verification is required. Similarly, the `_music` variants are added for `music.youtube.com` URLs. Some clients, such as `web` and `android`, require a `po_token` for their formats to be downloadable. Some clients, such as the `_creator` variants, will only work with authentication. You can use `all` to use all the clients, and `default` for the default clients. You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=all,-web`
+* `player_client`: Clients to extract video data from. The main clients are `web`, `ios` and `android`, with variants `_music` and `_creator` (e.g. `ios_creator`); and `mweb`, `mediaconnect`, `android_testsuite`, `android_vr`, `web_safari`, `web_embedded`, `tv` and `tv_embedded` with no variants. By default, `ios,mweb` is used, and `web_creator,mediaconnect` is added as needed for age-gated videos when account age verification is required. Similarly, the `_music` variants are added for `music.youtube.com` URLs. Some clients, such as `web` and `android`, require a `po_token` for their formats to be downloadable. Some clients, such as the `_creator` variants, will only work with authentication. You can use `all` to use all the clients, and `default` for the default clients. You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=all,-web`
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause some issues. See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) for more details
 * `player_params`: YouTube player parameters to use for player requests. Will overwrite any default ones set by yt-dlp.
 * `comment_sort`: `top` or `new` (default) - choose comment sorting mode (on YouTube's side)

devscripts/generate_aes_testdata.py

@@ -11,12 +11,13 @@ import codecs
 import subprocess
 
 from yt_dlp.aes import aes_encrypt, key_expansion
+from yt_dlp.utils import intlist_to_bytes
 
 secret_msg = b'Secret message goes here'
 
 
 def hex_str(int_list):
-    return codecs.encode(bytes(int_list), 'hex')
+    return codecs.encode(intlist_to_bytes(int_list), 'hex')
 
 
 def openssl_encode(algo, key, iv):

pyproject.toml

@@ -313,16 +313,6 @@ banned-from = [
 "yt_dlp.compat.compat_urllib_parse_urlparse".msg = "Use `urllib.parse.urlparse` instead."
 "yt_dlp.compat.compat_shlex_quote".msg = "Use `yt_dlp.utils.shell_quote` instead."
 "yt_dlp.utils.error_to_compat_str".msg = "Use `str` instead."
-"yt_dlp.utils.bytes_to_intlist".msg = "Use `list` instead."
-"yt_dlp.utils.intlist_to_bytes".msg = "Use `bytes` instead."
-"yt_dlp.utils.decodeArgument".msg = "Do not use"
-"yt_dlp.utils.decodeFilename".msg = "Do not use"
-"yt_dlp.utils.encodeFilename".msg = "Do not use"
-"yt_dlp.compat.compat_os_name".msg = "Use `os.name` instead."
-"yt_dlp.compat.compat_realpath".msg = "Use `os.path.realpath` instead."
-"yt_dlp.compat.functools".msg = "Use `functools` instead."
-"yt_dlp.utils.decodeOption".msg = "Do not use"
-"yt_dlp.utils.compiled_regex_type".msg = "Use `re.Pattern` instead."
 
 [tool.autopep8]
 max_line_length = 120

test/helper.py

@@ -9,6 +9,7 @@ import types
 
 import yt_dlp.extractor
 from yt_dlp import YoutubeDL
+from yt_dlp.compat import compat_os_name
 from yt_dlp.utils import preferredencoding, try_call, write_string, find_available_port
 
 if 'pytest' in sys.modules:
@@ -48,7 +49,7 @@ def report_warning(message, *args, **kwargs):
     Print the message to stderr, it will be prefixed with 'WARNING:'
     If stderr is a tty file the 'WARNING:' will be colored
     """
-    if sys.stderr.isatty() and os.name != 'nt':
+    if sys.stderr.isatty() and compat_os_name != 'nt':
         _msg_header = '\033[0;33mWARNING:\033[0m'
     else:
         _msg_header = 'WARNING:'

test/test_YoutubeDL.py

@@ -15,6 +15,7 @@ import json
 
 from test.helper import FakeYDL, assertRegexpMatches, try_rm
 from yt_dlp import YoutubeDL
+from yt_dlp.compat import compat_os_name
 from yt_dlp.extractor import YoutubeIE
 from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.postprocessor.common import PostProcessor
@@ -838,8 +839,8 @@ class TestYoutubeDL(unittest.TestCase):
         test('%(filesize)#D', '1Ki')
         test('%(height)5.2D', ' 1.08k')
         test('%(title4)#S', 'foo_bar_test')
-        test('%(title4).10S', ('foo ＂bar＂ ', 'foo ＂bar＂' + ('#' if os.name == 'nt' else ' ')))
-        if os.name == 'nt':
+        test('%(title4).10S', ('foo ＂bar＂ ', 'foo ＂bar＂' + ('#' if compat_os_name == 'nt' else ' ')))
+        if compat_os_name == 'nt':
             test('%(title4)q', ('"foo ""bar"" test"', None))
             test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', None))
             test('%(formats.0.id)#q', ('"id 1"', None))
@@ -902,9 +903,9 @@ class TestYoutubeDL(unittest.TestCase):
 
         # Environment variable expansion for prepare_filename
         os.environ['__yt_dlp_var'] = 'expanded'
-        envvar = '%__yt_dlp_var%' if os.name == 'nt' else '$__yt_dlp_var'
+        envvar = '%__yt_dlp_var%' if compat_os_name == 'nt' else '$__yt_dlp_var'
         test(envvar, (envvar, 'expanded'))
-        if os.name == 'nt':
+        if compat_os_name == 'nt':
             test('%s%', ('%s%', '%s%'))
             os.environ['s'] = 'expanded'
             test('%s%', ('%s%', 'expanded'))  # %s% should be expanded before escaping %s

test/test_aes.py

@@ -27,6 +27,7 @@ from yt_dlp.aes import (
     pad_block,
 )
 from yt_dlp.dependencies import Cryptodome
+from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
 
 # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
 
@@ -39,33 +40,33 @@ class TestAES(unittest.TestCase):
     def test_encrypt(self):
         msg = b'message'
         key = list(range(16))
-        encrypted = aes_encrypt(list(msg), key)
-        decrypted = bytes(aes_decrypt(encrypted, key))
+        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
+        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
         self.assertEqual(decrypted, msg)
 
     def test_cbc_decrypt(self):
         data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
-        decrypted = bytes(aes_cbc_decrypt(list(data), self.key, self.iv))
+        decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
         if Cryptodome.AES:
-            decrypted = aes_cbc_decrypt_bytes(data, bytes(self.key), bytes(self.iv))
+            decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_cbc_encrypt(self):
-        data = list(self.secret_msg)
-        encrypted = bytes(aes_cbc_encrypt(data, self.key, self.iv))
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_cbc_encrypt(data, self.key, self.iv))
         self.assertEqual(
             encrypted,
             b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd')
 
     def test_ctr_decrypt(self):
-        data = list(b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
-        decrypted = bytes(aes_ctr_decrypt(data, self.key, self.iv))
+        data = bytes_to_intlist(b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
+        decrypted = intlist_to_bytes(aes_ctr_decrypt(data, self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_ctr_encrypt(self):
-        data = list(self.secret_msg)
-        encrypted = bytes(aes_ctr_encrypt(data, self.key, self.iv))
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_ctr_encrypt(data, self.key, self.iv))
         self.assertEqual(
             encrypted,
             b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
@@ -74,19 +75,19 @@ class TestAES(unittest.TestCase):
         data = b'\x159Y\xcf5eud\x90\x9c\x85&]\x14\x1d\x0f.\x08\xb4T\xe4/\x17\xbd'
         authentication_tag = b'\xe8&I\x80rI\x07\x9d}YWuU@:e'
 
-        decrypted = bytes(aes_gcm_decrypt_and_verify(
-            list(data), self.key, list(authentication_tag), self.iv[:12]))
+        decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
+            bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
         if Cryptodome.AES:
             decrypted = aes_gcm_decrypt_and_verify_bytes(
-                data, bytes(self.key), authentication_tag, bytes(self.iv[:12]))
+                data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_gcm_aligned_decrypt(self):
         data = b'\x159Y\xcf5eud\x90\x9c\x85&]\x14\x1d\x0f'
         authentication_tag = b'\x08\xb1\x9d!&\x98\xd0\xeaRq\x90\xe6;\xb5]\xd8'
 
-        decrypted = bytes(aes_gcm_decrypt_and_verify(
+        decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
             list(data), self.key, list(authentication_tag), self.iv[:12]))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg[:16])
         if Cryptodome.AES:
@@ -95,38 +96,38 @@ class TestAES(unittest.TestCase):
             self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg[:16])
 
     def test_decrypt_text(self):
-        password = bytes(self.key).decode()
+        password = intlist_to_bytes(self.key).decode()
         encrypted = base64.b64encode(
-            bytes(self.iv[:8])
+            intlist_to_bytes(self.iv[:8])
             + b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae',
         ).decode()
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
-        password = bytes(self.key).decode()
+        password = intlist_to_bytes(self.key).decode()
         encrypted = base64.b64encode(
-            bytes(self.iv[:8])
+            intlist_to_bytes(self.iv[:8])
             + b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83',
         ).decode()
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 
     def test_ecb_encrypt(self):
-        data = list(self.secret_msg)
-        encrypted = bytes(aes_ecb_encrypt(data, self.key))
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key))
         self.assertEqual(
             encrypted,
             b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
 
     def test_ecb_decrypt(self):
-        data = list(b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
-        decrypted = bytes(aes_ecb_decrypt(data, self.key, self.iv))
+        data = bytes_to_intlist(b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
+        decrypted = intlist_to_bytes(aes_ecb_decrypt(data, self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_key_expansion(self):
         key = '4f6bdaa39e2f8cb07f5e722d9edef314'
 
-        self.assertEqual(key_expansion(list(bytearray.fromhex(key))), [
+        self.assertEqual(key_expansion(bytes_to_intlist(bytearray.fromhex(key))), [
             0x4F, 0x6B, 0xDA, 0xA3, 0x9E, 0x2F, 0x8C, 0xB0, 0x7F, 0x5E, 0x72, 0x2D, 0x9E, 0xDE, 0xF3, 0x14,
             0x53, 0x66, 0x20, 0xA8, 0xCD, 0x49, 0xAC, 0x18, 0xB2, 0x17, 0xDE, 0x35, 0x2C, 0xC9, 0x2D, 0x21,
             0x8C, 0xBE, 0xDD, 0xD9, 0x41, 0xF7, 0x71, 0xC1, 0xF3, 0xE0, 0xAF, 0xF4, 0xDF, 0x29, 0x82, 0xD5,

test/test_compat.py

@@ -12,7 +12,12 @@ import struct
 
 from yt_dlp import compat
 from yt_dlp.compat import urllib  # isort: split
-from yt_dlp.compat import compat_etree_fromstring, compat_expanduser
+from yt_dlp.compat import (
+    compat_etree_fromstring,
+    compat_expanduser,
+    compat_urllib_parse_unquote,  # noqa: TID251
+    compat_urllib_parse_urlencode,  # noqa: TID251
+)
 from yt_dlp.compat.urllib.request import getproxies
 
 
@@ -38,6 +43,39 @@ class TestCompat(unittest.TestCase):
         finally:
             os.environ['HOME'] = old_home or ''
 
+    def test_compat_urllib_parse_unquote(self):
+        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
+        self.assertEqual(compat_urllib_parse_unquote(''), '')
+        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
+        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
+        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
+        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),
+            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
+            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
+
+    def test_compat_urllib_parse_unquote_plus(self):
+        self.assertEqual(urllib.parse.unquote_plus('abc%20def'), 'abc def')
+        self.assertEqual(urllib.parse.unquote_plus('%7e/abc+def'), '~/abc def')
+
+    def test_compat_urllib_parse_urlencode(self):
+        self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({'abc': b'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({b'abc': 'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({b'abc': b'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([('abc', 'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([('abc', b'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def')
+
     def test_compat_etree_fromstring(self):
         xml = '''
             <root foo="bar" spam="中文">

test/test_downloader_http.py

@@ -15,6 +15,7 @@ import threading
 from test.helper import http_server_port, try_rm
 from yt_dlp import YoutubeDL
 from yt_dlp.downloader.http import HttpFD
+from yt_dlp.utils import encodeFilename
 from yt_dlp.utils._utils import _YDLLogger as FakeLogger
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -81,12 +82,12 @@ class TestHttpFD(unittest.TestCase):
         ydl = YoutubeDL(params)
         downloader = HttpFD(ydl, params)
         filename = 'testfile.mp4'
-        try_rm(filename)
+        try_rm(encodeFilename(filename))
         self.assertTrue(downloader.real_download(filename, {
             'url': f'http://127.0.0.1:{self.port}/{ep}',
         }), ep)
-        self.assertEqual(os.path.getsize(filename), TEST_SIZE, ep)
-        try_rm(filename)
+        self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep)
+        try_rm(encodeFilename(filename))
 
     def download_all(self, params):
         for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):

test/test_traversal.py

@@ -481,7 +481,7 @@ class TestTraversalHelpers:
             'id': 'name',
             'data': 'content',
             'url': 'url',
-        }, all, {subs_list_to_dict(lang=None)}]) == {
+        }, all, {subs_list_to_dict}]) == {
             'de': [{'url': 'https://example.com/subs/de.ass'}],
             'en': [{'data': 'content'}],
         }, 'subs with mandatory items missing should be filtered'
@@ -507,54 +507,6 @@ class TestTraversalHelpers:
             {'url': 'https://example.com/subs/en1', 'ext': 'ext'},
             {'url': 'https://example.com/subs/en2', 'ext': 'ext'},
         ]}, '`quality` key should sort subtitle list accordingly'
-        assert traverse_obj([
-            {'name': 'de', 'url': 'https://example.com/subs/de.ass'},
-            {'name': 'de'},
-            {'name': 'en', 'content': 'content'},
-            {'url': 'https://example.com/subs/en'},
-        ], [..., {
-            'id': 'name',
-            'url': 'url',
-            'data': 'content',
-        }, all, {subs_list_to_dict(lang='en')}]) == {
-            'de': [{'url': 'https://example.com/subs/de.ass'}],
-            'en': [
-                {'data': 'content'},
-                {'url': 'https://example.com/subs/en'},
-            ],
-        }, 'optionally provided lang should be used if no id available'
-        assert traverse_obj([
-            {'name': 1, 'url': 'https://example.com/subs/de1'},
-            {'name': {}, 'url': 'https://example.com/subs/de2'},
-            {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'},
-            {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'},
-        ], [..., {
-            'id': 'name',
-            'url': 'url',
-            'ext': 'ext',
-        }, all, {subs_list_to_dict(lang=None)}]) == {
-            'de': [
-                {'url': 'https://example.com/subs/de3'},
-                {'url': 'https://example.com/subs/de4'},
-            ],
-        }, 'non str types should be ignored for id and ext'
-        assert traverse_obj([
-            {'name': 1, 'url': 'https://example.com/subs/de1'},
-            {'name': {}, 'url': 'https://example.com/subs/de2'},
-            {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'},
-            {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'},
-        ], [..., {
-            'id': 'name',
-            'url': 'url',
-            'ext': 'ext',
-        }, all, {subs_list_to_dict(lang='de')}]) == {
-            'de': [
-                {'url': 'https://example.com/subs/de1'},
-                {'url': 'https://example.com/subs/de2'},
-                {'url': 'https://example.com/subs/de3'},
-                {'url': 'https://example.com/subs/de4'},
-            ],
-        }, 'non str types should be replaced by default id'
 
     def test_trim_str(self):
         with pytest.raises(TypeError):
@@ -573,7 +525,7 @@ class TestTraversalHelpers:
     def test_unpack(self):
         assert unpack(lambda *x: ''.join(map(str, x)))([1, 2, 3]) == '123'
         assert unpack(join_nonempty)([1, 2, 3]) == '1-2-3'
-        assert unpack(join_nonempty, delim=' ')([1, 2, 3]) == '1 2 3'
+        assert unpack(join_nonempty(delim=' '))([1, 2, 3]) == '1 2 3'
         with pytest.raises(TypeError):
             unpack(join_nonempty)()
         with pytest.raises(TypeError):

test/test_utils.py

@@ -21,6 +21,7 @@ import xml.etree.ElementTree
 from yt_dlp.compat import (
     compat_etree_fromstring,
     compat_HTMLParseError,
+    compat_os_name,
 )
 from yt_dlp.utils import (
     Config,
@@ -48,6 +49,7 @@ from yt_dlp.utils import (
     dfxp2srt,
     encode_base_n,
     encode_compat_str,
+    encodeFilename,
     expand_path,
     extract_attributes,
     extract_basic_auth,
@@ -67,8 +69,10 @@ from yt_dlp.utils import (
     get_elements_html_by_class,
     get_elements_text_and_html_by_attribute,
     int_or_none,
+    intlist_to_bytes,
     iri_to_uri,
     is_html,
+    join_nonempty,
     js_to_json,
     limit_length,
     locked_file,
@@ -563,10 +567,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(res_data, {'a': 'b', 'c': 'd'})
 
     def test_shell_quote(self):
-        args = ['ffmpeg', '-i', 'ñ€ß\'.mp4']
+        args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
         self.assertEqual(
             shell_quote(args),
-            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if os.name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
+            """ffmpeg -i 'ñ€ß'"'"'.mp4'""" if compat_os_name != 'nt' else '''ffmpeg -i "ñ€ß'.mp4"''')
 
     def test_float_or_none(self):
         self.assertEqual(float_or_none('42.42'), 42.42)
@@ -1306,10 +1310,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(clean_html('a:\n   "b"'), 'a: "b"')
         self.assertEqual(clean_html('a<br>\xa0b'), 'a\nb')
 
+    def test_intlist_to_bytes(self):
+        self.assertEqual(
+            intlist_to_bytes([0, 1, 127, 128, 255]),
+            b'\x00\x01\x7f\x80\xff')
+
     def test_args_to_str(self):
         self.assertEqual(
             args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
-            'foo ba/r -baz \'2 be\' \'\'' if os.name != 'nt' else 'foo ba/r -baz "2 be" ""',
+            'foo ba/r -baz \'2 be\' \'\'' if compat_os_name != 'nt' else 'foo ba/r -baz "2 be" ""',
         )
 
     def test_parse_filesize(self):
@@ -2109,7 +2118,7 @@ Line 1
         assert extract_basic_auth('http://user:@foo.bar') == ('http://foo.bar', 'Basic dXNlcjo=')
         assert extract_basic_auth('http://user:pass@foo.bar') == ('http://foo.bar', 'Basic dXNlcjpwYXNz')
 
-    @unittest.skipUnless(os.name == 'nt', 'Only relevant on Windows')
+    @unittest.skipUnless(compat_os_name == 'nt', 'Only relevant on Windows')
     def test_windows_escaping(self):
         tests = [
             'test"&',
@@ -2149,6 +2158,10 @@ Line 1
         assert int_or_none(v=10) == 10, 'keyword passed positional should call function'
         assert int_or_none(scale=0.1)(10) == 100, 'call after partial application should call the function'
 
+        assert callable(join_nonempty(delim=', ')), 'varargs positional should apply partially'
+        assert callable(join_nonempty()), 'varargs positional should apply partially'
+        assert join_nonempty(None, delim=', ') == '', 'passed varargs should call the function'
+
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/YoutubeDL.py

@@ -26,7 +26,7 @@ import unicodedata
 
 from .cache import Cache
 from .compat import urllib  # isort: split
-from .compat import urllib_req_to_req
+from .compat import compat_os_name, urllib_req_to_req
 from .cookies import CookieLoadError, LenientSimpleCookie, load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
 from .downloader.rtmp import rtmpdump_version
@@ -109,6 +109,7 @@ from .utils import (
     determine_ext,
     determine_protocol,
     encode_compat_str,
+    encodeFilename,
     escapeHTML,
     expand_path,
     extract_basic_auth,
@@ -166,7 +167,7 @@ from .utils.networking import (
 )
 from .version import CHANNEL, ORIGIN, RELEASE_GIT_HEAD, VARIANT, __version__
 
-if os.name == 'nt':
+if compat_os_name == 'nt':
     import ctypes
 
 
@@ -642,7 +643,7 @@ class YoutubeDL:
             out=stdout,
             error=sys.stderr,
             screen=sys.stderr if self.params.get('quiet') else stdout,
-            console=None if os.name == 'nt' else next(
+            console=None if compat_os_name == 'nt' else next(
                 filter(supports_terminal_sequences, (sys.stderr, sys.stdout)), None),
         )
 
@@ -951,7 +952,7 @@ class YoutubeDL:
             self._write_string(f'{self._bidi_workaround(message)}\n', self._out_files.error, only_once=only_once)
 
     def _send_console_code(self, code):
-        if os.name == 'nt' or not self._out_files.console:
+        if compat_os_name == 'nt' or not self._out_files.console:
             return
         self._write_string(code, self._out_files.console)
 
@@ -959,7 +960,7 @@ class YoutubeDL:
         if not self.params.get('consoletitle', False):
             return
         message = remove_terminal_sequences(message)
-        if os.name == 'nt':
+        if compat_os_name == 'nt':
             if ctypes.windll.kernel32.GetConsoleWindow():
                 # c_wchar_p() might not be necessary if `message` is
                 # already of type unicode()
@@ -3254,9 +3255,9 @@ class YoutubeDL:
 
         if full_filename is None:
             return
-        if not self._ensure_dir_exists(full_filename):
+        if not self._ensure_dir_exists(encodeFilename(full_filename)):
             return
-        if not self._ensure_dir_exists(temp_filename):
+        if not self._ensure_dir_exists(encodeFilename(temp_filename)):
             return
 
         if self._write_description('video', info_dict,
@@ -3288,16 +3289,16 @@ class YoutubeDL:
         if self.params.get('writeannotations', False):
             annofn = self.prepare_filename(info_dict, 'annotation')
         if annofn:
-            if not self._ensure_dir_exists(annofn):
+            if not self._ensure_dir_exists(encodeFilename(annofn)):
                 return
-            if not self.params.get('overwrites', True) and os.path.exists(annofn):
+            if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             elif not info_dict.get('annotations'):
                 self.report_warning('There are no annotations to write.')
             else:
                 try:
                     self.to_screen('[info] Writing video annotations to: ' + annofn)
-                    with open(annofn, 'w', encoding='utf-8') as annofile:
+                    with open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
                         annofile.write(info_dict['annotations'])
                 except (KeyError, TypeError):
                     self.report_warning('There are no annotations to write.')
@@ -3313,14 +3314,14 @@ class YoutubeDL:
                     f'Cannot write internet shortcut file because the actual URL of "{info_dict["webpage_url"]}" is unknown')
                 return True
             linkfn = replace_extension(self.prepare_filename(info_dict, 'link'), link_type, info_dict.get('ext'))
-            if not self._ensure_dir_exists(linkfn):
+            if not self._ensure_dir_exists(encodeFilename(linkfn)):
                 return False
-            if self.params.get('overwrites', True) and os.path.exists(linkfn):
+            if self.params.get('overwrites', True) and os.path.exists(encodeFilename(linkfn)):
                 self.to_screen(f'[info] Internet shortcut (.{link_type}) is already present')
                 return True
             try:
                 self.to_screen(f'[info] Writing internet shortcut (.{link_type}) to: {linkfn}')
-                with open(to_high_limit_path(linkfn), 'w', encoding='utf-8',
+                with open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8',
                           newline='\r\n' if link_type == 'url' else '\n') as linkfile:
                     template_vars = {'url': url}
                     if link_type == 'desktop':
@@ -3351,7 +3352,7 @@ class YoutubeDL:
 
         if self.params.get('skip_download'):
             info_dict['filepath'] = temp_filename
-            info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename))
+            info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename)))
             info_dict['__files_to_move'] = files_to_move
             replace_info_dict(self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict))
             info_dict['__write_download_archive'] = self.params.get('force_write_download_archive')
@@ -3481,7 +3482,7 @@ class YoutubeDL:
                         self.report_file_already_downloaded(dl_filename)
 
                 dl_filename = dl_filename or temp_filename
-                info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename))
+                info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename)))
 
             except network_exceptions as err:
                 self.report_error(f'unable to download video data: {err}')
@@ -4296,7 +4297,7 @@ class YoutubeDL:
         else:
             try:
                 self.to_screen(f'[info] Writing {label} description to: {descfn}')
-                with open(descfn, 'w', encoding='utf-8') as descfile:
+                with open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(ie_result['description'])
             except OSError:
                 self.report_error(f'Cannot write {label} description file {descfn}')
@@ -4380,9 +4381,7 @@ class YoutubeDL:
             return None
 
         for idx, t in list(enumerate(thumbnails))[::-1]:
-            thumb_ext = t.get('ext') or determine_ext(t['url'], 'jpg')
-            if multiple:
-                thumb_ext = f'{t["id"]}.{thumb_ext}'
+            thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
             thumb_display_id = f'{label} thumbnail {t["id"]}'
             thumb_filename = replace_extension(filename, thumb_ext, info_dict.get('ext'))
             thumb_filename_final = replace_extension(thumb_filename_base, thumb_ext, info_dict.get('ext'))
@@ -4398,7 +4397,7 @@ class YoutubeDL:
                 try:
                     uf = self.urlopen(Request(t['url'], headers=t.get('http_headers', {})))
                     self.to_screen(f'[info] Writing {thumb_display_id} to: {thumb_filename}')
-                    with open(thumb_filename, 'wb') as thumbf:
+                    with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                         shutil.copyfileobj(uf, thumbf)
                     ret.append((thumb_filename, thumb_filename_final))
                     t['filepath'] = thumb_filename

yt_dlp/__init__.py

@@ -14,6 +14,7 @@ import os
 import re
 import traceback
 
+from .compat import compat_os_name
 from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS, CookieLoadError
 from .downloader.external import get_external_downloader
 from .extractor import list_extractor_classes
@@ -43,6 +44,7 @@ from .utils import (
     GeoUtils,
     PlaylistEntries,
     SameFileError,
+    decodeOption,
     download_range_func,
     expand_path,
     float_or_none,
@@ -881,8 +883,8 @@ def parse_options(argv=None):
         'listsubtitles': opts.listsubtitles,
         'subtitlesformat': opts.subtitlesformat,
         'subtitleslangs': opts.subtitleslangs,
-        'matchtitle': opts.matchtitle,
-        'rejecttitle': opts.rejecttitle,
+        'matchtitle': decodeOption(opts.matchtitle),
+        'rejecttitle': decodeOption(opts.rejecttitle),
         'max_downloads': opts.max_downloads,
         'prefer_free_formats': opts.prefer_free_formats,
         'trim_file_name': opts.trim_file_name,
@@ -1051,7 +1053,7 @@ def _real_main(argv=None):
             ydl.warn_if_short_id(args)
 
             # Show a useful error message and wait for keypress if not launched from shell on Windows
-            if not args and os.name == 'nt' and getattr(sys, 'frozen', False):
+            if not args and compat_os_name == 'nt' and getattr(sys, 'frozen', False):
                 import ctypes.wintypes
                 import msvcrt
 

yt_dlp/aes.py

@@ -3,6 +3,7 @@ from math import ceil
 
 from .compat import compat_ord
 from .dependencies import Cryptodome
+from .utils import bytes_to_intlist, intlist_to_bytes
 
 if Cryptodome.AES:
     def aes_cbc_decrypt_bytes(data, key, iv):
@@ -16,15 +17,15 @@ if Cryptodome.AES:
 else:
     def aes_cbc_decrypt_bytes(data, key, iv):
         """ Decrypt bytes with AES-CBC using native implementation since pycryptodome is unavailable """
-        return bytes(aes_cbc_decrypt(*map(list, (data, key, iv))))
+        return intlist_to_bytes(aes_cbc_decrypt(*map(bytes_to_intlist, (data, key, iv))))
 
     def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
         """ Decrypt bytes with AES-GCM using native implementation since pycryptodome is unavailable """
-        return bytes(aes_gcm_decrypt_and_verify(*map(list, (data, key, tag, nonce))))
+        return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
 
 
 def aes_cbc_encrypt_bytes(data, key, iv, **kwargs):
-    return bytes(aes_cbc_encrypt(*map(list, (data, key, iv)), **kwargs))
+    return intlist_to_bytes(aes_cbc_encrypt(*map(bytes_to_intlist, (data, key, iv)), **kwargs))
 
 
 BLOCK_SIZE_BYTES = 16
@@ -220,7 +221,7 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce):
         j0 = [*nonce, 0, 0, 0, 1]
     else:
         fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8
-        ghash_in = nonce + [0] * fill + list((8 * len(nonce)).to_bytes(8, 'big'))
+        ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, 'big'))
         j0 = ghash(hash_subkey, ghash_in)
 
     # TODO: add nonce support to aes_ctr_decrypt
@@ -233,9 +234,9 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce):
     s_tag = ghash(
         hash_subkey,
         data
-        + [0] * pad_len                                  # pad
-        + list((0 * 8).to_bytes(8, 'big')                # length of associated data
-               + ((len(data) * 8).to_bytes(8, 'big'))),  # length of data
+        + [0] * pad_len                                         # pad
+        + bytes_to_intlist((0 * 8).to_bytes(8, 'big')           # length of associated data
+                           + ((len(data) * 8).to_bytes(8, 'big'))),  # length of data
     )
 
     if tag != aes_ctr_encrypt(s_tag, key, j0):
@@ -299,8 +300,8 @@ def aes_decrypt_text(data, password, key_size_bytes):
     """
     NONCE_LENGTH_BYTES = 8
 
-    data = list(base64.b64decode(data))
-    password = list(password.encode())
+    data = bytes_to_intlist(base64.b64decode(data))
+    password = bytes_to_intlist(password.encode())
 
     key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
     key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
@@ -309,7 +310,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     cipher = data[NONCE_LENGTH_BYTES:]
 
     decrypted_data = aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES))
-    return bytes(decrypted_data)
+    return intlist_to_bytes(decrypted_data)
 
 
 RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)

yt_dlp/compat/__init__.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import xml.etree.ElementTree as etree
 
 from .compat_utils import passthrough_module
@@ -23,14 +24,33 @@ def compat_etree_fromstring(text):
     return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
 
 
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
+def compat_shlex_quote(s):
+    from ..utils import shell_quote
+    return shell_quote(s)
+
+
 def compat_ord(c):
     return c if isinstance(c, int) else ord(c)
 
 
+if compat_os_name == 'nt' and sys.version_info < (3, 8):
+    # os.path.realpath on Windows does not follow symbolic links
+    # prior to Python 3.8 (see https://bugs.python.org/issue9949)
+    def compat_realpath(path):
+        while os.path.islink(path):
+            path = os.path.abspath(os.readlink(path))
+        return os.path.realpath(path)
+else:
+    compat_realpath = os.path.realpath
+
+
 # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
 # See https://github.com/yt-dlp/yt-dlp/issues/792
 # https://docs.python.org/3/library/os.path.html#os.path.expanduser
-if os.name in ('nt', 'ce'):
+if compat_os_name in ('nt', 'ce'):
     def compat_expanduser(path):
         HOME = os.environ.get('HOME')
         if not HOME:

yt_dlp/compat/_deprecated.py

@@ -8,14 +8,16 @@ passthrough_module(__name__, '.._legacy', callback=lambda attr: warnings.warn(
     DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=6))
 del passthrough_module
 
-import functools  # noqa: F401
-import os
+import base64
+import urllib.error
+import urllib.parse
 
+compat_str = str
 
-compat_os_name = os.name
-compat_realpath = os.path.realpath
+compat_b64decode = base64.b64decode
 
-
-def compat_shlex_quote(s):
-    from ..utils import shell_quote
-    return shell_quote(s)
+compat_urlparse = urllib.parse
+compat_parse_qs = urllib.parse.parse_qs
+compat_urllib_parse_unquote = urllib.parse.unquote
+compat_urllib_parse_urlencode = urllib.parse.urlencode
+compat_urllib_parse_urlparse = urllib.parse.urlparse

yt_dlp/compat/_legacy.py

@@ -30,7 +30,7 @@ from asyncio import run as compat_asyncio_run  # noqa: F401
 from re import Pattern as compat_Pattern  # noqa: F401
 from re import match as compat_Match  # noqa: F401
 
-from . import compat_expanduser, compat_HTMLParseError
+from . import compat_expanduser, compat_HTMLParseError, compat_realpath
 from .compat_utils import passthrough_module
 from ..dependencies import brotli as compat_brotli  # noqa: F401
 from ..dependencies import websockets as compat_websockets  # noqa: F401
@@ -78,7 +78,7 @@ compat_kwargs = lambda kwargs: kwargs
 compat_map = map
 compat_numeric_types = (int, float, complex)
 compat_os_path_expanduser = compat_expanduser
-compat_os_path_realpath = os.path.realpath
+compat_os_path_realpath = compat_realpath
 compat_print = print
 compat_shlex_split = shlex.split
 compat_socket_create_connection = socket.create_connection
@@ -104,12 +104,5 @@ compat_xml_parse_error = compat_xml_etree_ElementTree_ParseError = etree.ParseEr
 compat_xpath = lambda xpath: xpath
 compat_zip = zip
 workaround_optparse_bug9161 = lambda: None
-compat_str = str
-compat_b64decode = base64.b64decode
-compat_urlparse = urllib.parse
-compat_parse_qs = urllib.parse.parse_qs
-compat_urllib_parse_unquote = urllib.parse.unquote
-compat_urllib_parse_urlencode = urllib.parse.urlencode
-compat_urllib_parse_urlparse = urllib.parse.urlparse
 
 legacy = []

yt_dlp/compat/functools.py

@@ -0,0 +1,7 @@
+# flake8: noqa: F405
+from functools import *  # noqa: F403
+
+from .compat_utils import passthrough_module
+
+passthrough_module(__name__, 'functools')
+del passthrough_module

yt_dlp/compat/urllib/request.py

@@ -7,9 +7,9 @@ passthrough_module(__name__, 'urllib.request')
 del passthrough_module
 
 
-import os
+from .. import compat_os_name
 
-if os.name == 'nt':
+if compat_os_name == 'nt':
     # On older Python versions, proxies are extracted from Windows registry erroneously. [1]
     # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
     # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
@@ -37,4 +37,4 @@ if os.name == 'nt':
     def getproxies():
         return getproxies_environment() or getproxies_registry_patched()
 
-del os
+del compat_os_name

yt_dlp/cookies.py

@@ -25,6 +25,7 @@ from .aes import (
     aes_gcm_decrypt_and_verify_bytes,
     unpad_pkcs7,
 )
+from .compat import compat_os_name
 from .dependencies import (
     _SECRETSTORAGE_UNAVAILABLE_REASON,
     secretstorage,
@@ -342,7 +343,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
             logger.debug(f'cookie version breakdown: {counts}')
             return jar
         except PermissionError as error:
-            if os.name == 'nt' and error.errno == 13:
+            if compat_os_name == 'nt' and error.errno == 13:
                 message = 'Could not copy Chrome cookie database. See  https://github.com/yt-dlp/yt-dlp/issues/7271  for more info'
                 logger.error(message)
                 raise DownloadError(message)  # force exit

yt_dlp/downloader/common.py

@@ -20,7 +20,9 @@ from ..utils import (
     Namespace,
     RetryManager,
     classproperty,
+    decodeArgument,
     deprecation_warning,
+    encodeFilename,
     format_bytes,
     join_nonempty,
     parse_bytes,
@@ -217,7 +219,7 @@ class FileDownloader:
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
         if self.params.get('nopart', False) or filename == '-' or \
-                (os.path.exists(filename) and not os.path.isfile(filename)):
+                (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
             return filename
         return filename + '.part'
 
@@ -271,7 +273,7 @@ class FileDownloader:
         """Try to set the last-modified time of the given file."""
         if last_modified_hdr is None:
             return
-        if not os.path.isfile(filename):
+        if not os.path.isfile(encodeFilename(filename)):
             return
         timestr = last_modified_hdr
         if timestr is None:
@@ -430,13 +432,13 @@ class FileDownloader:
         """
         nooverwrites_and_exists = (
             not self.params.get('overwrites', True)
-            and os.path.exists(filename)
+            and os.path.exists(encodeFilename(filename))
         )
 
         if not hasattr(filename, 'write'):
             continuedl_and_exists = (
                 self.params.get('continuedl', True)
-                and os.path.isfile(filename)
+                and os.path.isfile(encodeFilename(filename))
                 and not self.params.get('nopart', False)
             )
 
@@ -446,7 +448,7 @@ class FileDownloader:
                 self._hook_progress({
                     'filename': filename,
                     'status': 'finished',
-                    'total_bytes': os.path.getsize(filename),
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
                 }, info_dict)
                 self._finish_multiline_status()
                 return True, False
@@ -487,7 +489,9 @@ class FileDownloader:
         if not self.params.get('verbose', False):
             return
 
-        if exe is None:
-            exe = os.path.basename(args[0])
+        str_args = [decodeArgument(a) for a in args]
 
-        self.write_debug(f'{exe} command line: {shell_quote(args)}')
+        if exe is None:
+            exe = os.path.basename(str_args[0])
+
+        self.write_debug(f'{exe} command line: {shell_quote(str_args)}')

yt_dlp/downloader/external.py

@@ -23,6 +23,7 @@ from ..utils import (
     cli_valueless_option,
     determine_ext,
     encodeArgument,
+    encodeFilename,
     find_available_port,
     remove_end,
     traverse_obj,
@@ -66,7 +67,7 @@ class ExternalFD(FragmentFD):
                 'elapsed': time.time() - started,
             }
             if filename != '-':
-                fsize = os.path.getsize(tmpfilename)
+                fsize = os.path.getsize(encodeFilename(tmpfilename))
                 self.try_rename(tmpfilename, filename)
                 status.update({
                     'downloaded_bytes': fsize,
@@ -183,9 +184,9 @@ class ExternalFD(FragmentFD):
             dest.write(decrypt_fragment(fragment, src.read()))
             src.close()
             if not self.params.get('keep_fragments', False):
-                self.try_remove(fragment_filename)
+                self.try_remove(encodeFilename(fragment_filename))
         dest.close()
-        self.try_remove(f'{tmpfilename}.frag.urls')
+        self.try_remove(encodeFilename(f'{tmpfilename}.frag.urls'))
         return 0
 
     def _call_process(self, cmd, info_dict):
@@ -619,7 +620,7 @@ class FFmpegFD(ExternalFD):
         args += self._configuration_args(('_o1', '_o', ''))
 
         args = [encodeArgument(opt) for opt in args]
-        args.append(ffpp._ffmpeg_filename_argument(tmpfilename))
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
         self._debug_cmd(args)
 
         piped = any(fmt['url'] in ('-', 'pipe:') for fmt in selected_formats)

yt_dlp/downloader/fragment.py

@@ -9,9 +9,10 @@ import time
 from .common import FileDownloader
 from .http import HttpFD
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
+from ..compat import compat_os_name
 from ..networking import Request
 from ..networking.exceptions import HTTPError, IncompleteRead
-from ..utils import DownloadError, RetryManager, traverse_obj
+from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
 from ..utils.networking import HTTPHeaderDict
 from ..utils.progress import ProgressCalculator
 
@@ -151,7 +152,7 @@ class FragmentFD(FileDownloader):
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
-                self.try_remove(ctx['fragment_filename_sanitized'])
+                self.try_remove(encodeFilename(ctx['fragment_filename_sanitized']))
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
@@ -187,7 +188,7 @@ class FragmentFD(FileDownloader):
         })
 
         if self.__do_ytdl_file(ctx):
-            ytdl_file_exists = os.path.isfile(self.ytdl_filename(ctx['filename']))
+            ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename'])))
             continuedl = self.params.get('continuedl', True)
             if continuedl and ytdl_file_exists:
                 self._read_ytdl_file(ctx)
@@ -389,7 +390,7 @@ class FragmentFD(FileDownloader):
             def __exit__(self, exc_type, exc_val, exc_tb):
                 pass
 
-        if os.name == 'nt':
+        if compat_os_name == 'nt':
             def future_result(future):
                 while True:
                     try:

yt_dlp/downloader/http.py

@@ -15,6 +15,7 @@ from ..utils import (
     ThrottledDownload,
     XAttrMetadataError,
     XAttrUnavailableError,
+    encodeFilename,
     int_or_none,
     parse_http_range,
     try_call,
@@ -57,8 +58,9 @@ class HttpFD(FileDownloader):
 
         if self.params.get('continuedl', True):
             # Establish possible resume length
-            if os.path.isfile(ctx.tmpfilename):
-                ctx.resume_len = os.path.getsize(ctx.tmpfilename)
+            if os.path.isfile(encodeFilename(ctx.tmpfilename)):
+                ctx.resume_len = os.path.getsize(
+                    encodeFilename(ctx.tmpfilename))
 
         ctx.is_resume = ctx.resume_len > 0
 
@@ -239,7 +241,7 @@ class HttpFD(FileDownloader):
                     ctx.resume_len = byte_counter
                 else:
                     try:
-                        ctx.resume_len = os.path.getsize(ctx.tmpfilename)
+                        ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename))
                     except FileNotFoundError:
                         ctx.resume_len = 0
                 raise RetryDownload(e)

yt_dlp/downloader/rtmp.py

@@ -8,6 +8,7 @@ from ..utils import (
     Popen,
     check_executable,
     encodeArgument,
+    encodeFilename,
     get_exe_version,
 )
 
@@ -178,7 +179,7 @@ class RtmpFD(FileDownloader):
             return False
 
         while retval in (RD_INCOMPLETE, RD_FAILED) and not test and not live:
-            prevsize = os.path.getsize(tmpfilename)
+            prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(f'[rtmpdump] Downloaded {prevsize} bytes')
             time.sleep(5.0)  # This seems to be needed
             args = [*basic_args, '--resume']
@@ -186,7 +187,7 @@ class RtmpFD(FileDownloader):
                 args += ['--skip', '1']
             args = [encodeArgument(a) for a in args]
             retval = run_rtmpdump(args)
-            cursize = os.path.getsize(tmpfilename)
+            cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
                 break
             # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
@@ -195,7 +196,7 @@ class RtmpFD(FileDownloader):
                 retval = RD_SUCCESS
                 break
         if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE):
-            fsize = os.path.getsize(tmpfilename)
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(f'[rtmpdump] Downloaded {fsize} bytes')
             self.try_rename(tmpfilename, filename)
             self._hook_progress({

yt_dlp/downloader/rtsp.py

@@ -2,7 +2,7 @@ import os
 import subprocess
 
 from .common import FileDownloader
-from ..utils import check_executable
+from ..utils import check_executable, encodeFilename
 
 
 class RtspFD(FileDownloader):
@@ -26,7 +26,7 @@ class RtspFD(FileDownloader):
 
         retval = subprocess.call(args)
         if retval == 0:
-            fsize = os.path.getsize(tmpfilename)
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(f'\r[{args[0]}] {fsize} bytes')
             self.try_rename(tmpfilename, filename)
             self._hook_progress({

yt_dlp/extractor/_extractors.py

@@ -208,10 +208,6 @@ from .bandcamp import (
     BandcampUserIE,
     BandcampWeeklyIE,
 )
-from .bandlab import (
-    BandlabIE,
-    BandlabPlaylistIE,
-)
 from .bannedvideo import BannedVideoIE
 from .bbc import (
     BBCIE,
@@ -1654,7 +1650,6 @@ from .radiokapital import (
     RadioKapitalIE,
     RadioKapitalShowIE,
 )
-from .radioradicale import RadioRadicaleIE
 from .radiozet import RadioZetPodcastIE
 from .radlive import (
     RadLiveChannelIE,

yt_dlp/extractor/abematv.py

@@ -6,6 +6,7 @@ import hmac
 import io
 import json
 import re
+import struct
 import time
 import urllib.parse
 import uuid
@@ -17,8 +18,10 @@ from ..networking.exceptions import TransportError
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
+    bytes_to_intlist,
     decode_base_n,
     int_or_none,
+    intlist_to_bytes,
     time_seconds,
     traverse_obj,
     update_url_query,
@@ -69,15 +72,15 @@ class AbemaLicenseRH(RequestHandler):
             })
 
         res = decode_base_n(license_response['k'], table=self._STRTABLE)
-        encvideokey = list(res.to_bytes(16, 'big'))
+        encvideokey = bytes_to_intlist(struct.pack('>QQ', res >> 64, res & 0xffffffffffffffff))
 
         h = hmac.new(
             binascii.unhexlify(self._HKEY),
             (license_response['cid'] + self.ie._DEVICE_ID).encode(),
             digestmod=hashlib.sha256)
-        enckey = list(h.digest())
+        enckey = bytes_to_intlist(h.digest())
 
-        return bytes(aes_ecb_decrypt(encvideokey, enckey))
+        return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey))
 
 
 class AbemaTVBaseIE(InfoExtractor):

yt_dlp/extractor/adn.py

@@ -11,9 +11,11 @@ from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     ass_subtitles_timecode,
+    bytes_to_intlist,
     bytes_to_long,
     float_or_none,
     int_or_none,
+    intlist_to_bytes,
     join_nonempty,
     long_to_bytes,
     parse_iso8601,
@@ -196,16 +198,16 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
         links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
         self._K = ''.join(random.choices('0123456789abcdef', k=16))
-        message = list(json.dumps({
+        message = bytes_to_intlist(json.dumps({
             'k': self._K,
             't': token,
-        }).encode())
+        }))
 
         # Sometimes authentication fails for no good reason, retry with
         # a different random padding
         links_data = None
         for _ in range(3):
-            padded_message = bytes(pkcs1pad(message, 128))
+            padded_message = intlist_to_bytes(pkcs1pad(message, 128))
             n, e = self._RSA_KEY
             encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
             authorization = base64.b64encode(encrypted_message).decode()

yt_dlp/extractor/afreecatv.py

@@ -66,14 +66,6 @@ class AfreecaTVBaseIE(InfoExtractor):
             extensions={'legacy_ssl': True}), display_id,
             'Downloading API JSON', 'Unable to download API JSON')
 
-    @staticmethod
-    def _fixup_thumb(thumb_url):
-        if not url_or_none(thumb_url):
-            return None
-        # Core would determine_ext as 'php' from the url, so we need to provide the real ext
-        # See: https://github.com/yt-dlp/yt-dlp/issues/11537
-        return [{'url': thumb_url, 'ext': 'jpg'}]
-
 
 class AfreecaTVIE(AfreecaTVBaseIE):
     IE_NAME = 'soop'
@@ -163,7 +155,7 @@ class AfreecaTVIE(AfreecaTVBaseIE):
             'uploader': ('writer_nick', {str}),
             'uploader_id': ('bj_id', {str}),
             'duration': ('total_file_duration', {int_or_none(scale=1000)}),
-            'thumbnails': ('thumb', {self._fixup_thumb}),
+            'thumbnail': ('thumb', {url_or_none}),
         })
 
         entries = []
@@ -234,7 +226,8 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
 
         return self.playlist_result(self._entries(data), video_id)
 
-    def _entries(self, data):
+    @staticmethod
+    def _entries(data):
         # 'files' is always a list with 1 element
         yield from traverse_obj(data, (
             'data', lambda _, v: v['story_type'] == 'catch',
@@ -245,7 +238,7 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
                 'title': ('title', {str}),
                 'uploader': ('writer_nick', {str}),
                 'uploader_id': ('writer_id', {str}),
-                'thumbnails': ('thumb', {self._fixup_thumb}),
+                'thumbnail': ('thumb', {url_or_none}),
                 'timestamp': ('write_timestamp', {int_or_none}),
             }))
 

yt_dlp/extractor/anvato.py

@@ -8,8 +8,10 @@ import time
 from .common import InfoExtractor
 from ..aes import aes_encrypt
 from ..utils import (
+    bytes_to_intlist,
     determine_ext,
     int_or_none,
+    intlist_to_bytes,
     join_nonempty,
     smuggle_url,
     strip_jsonp,
@@ -232,8 +234,8 @@ class AnvatoIE(InfoExtractor):
         server_time = self._server_time(access_key, video_id)
         input_data = f'{server_time}~{md5_text(video_data_url)}~{md5_text(server_time)}'
 
-        auth_secret = bytes(aes_encrypt(
-            list(input_data[:64].encode()), list(self._AUTH_KEY)))
+        auth_secret = intlist_to_bytes(aes_encrypt(
+            bytes_to_intlist(input_data[:64]), bytes_to_intlist(self._AUTH_KEY)))
         query = {
             'X-Anvato-Adst-Auth': base64.b64encode(auth_secret).decode('ascii'),
             'rtyp': 'fp',

yt_dlp/extractor/bandlab.py

@@ -1,438 +0,0 @@
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    format_field,
-    int_or_none,
-    parse_iso8601,
-    parse_qs,
-    truncate_string,
-    url_or_none,
-)
-from ..utils.traversal import traverse_obj, value
-
-
-class BandlabBaseIE(InfoExtractor):
-    def _call_api(self, endpoint, asset_id, **kwargs):
-        headers = kwargs.pop('headers', None) or {}
-        return self._download_json(
-            f'https://www.bandlab.com/api/v1.3/{endpoint}/{asset_id}',
-            asset_id, headers={
-                'accept': 'application/json',
-                'referer': 'https://www.bandlab.com/',
-                'x-client-id': 'BandLab-Web',
-                'x-client-version': '10.1.124',
-                **headers,
-            }, **kwargs)
-
-    def _parse_revision(self, revision_data, url=None):
-        return {
-            'vcodec': 'none',
-            'media_type': 'revision',
-            'extractor_key': BandlabIE.ie_key(),
-            'extractor': BandlabIE.IE_NAME,
-            **traverse_obj(revision_data, {
-                'webpage_url': (
-                    'id', ({value(url)}, {format_field(template='https://www.bandlab.com/revision/%s')}), filter, any),
-                'id': (('revisionId', 'id'), {str}, any),
-                'title': ('song', 'name', {str}),
-                'track': ('song', 'name', {str}),
-                'url': ('mixdown', 'file', {url_or_none}),
-                'thumbnail': ('song', 'picture', 'url', {url_or_none}),
-                'description': ('description', {str}),
-                'uploader': ('creator', 'name', {str}),
-                'uploader_id': ('creator', 'username', {str}),
-                'timestamp': ('createdOn', {parse_iso8601}),
-                'duration': ('mixdown', 'duration', {float_or_none}),
-                'view_count': ('counters', 'plays', {int_or_none}),
-                'like_count': ('counters', 'likes', {int_or_none}),
-                'comment_count': ('counters', 'comments', {int_or_none}),
-                'genres': ('genres', ..., 'name', {str}),
-            }),
-        }
-
-    def _parse_track(self, track_data, url=None):
-        return {
-            'vcodec': 'none',
-            'media_type': 'track',
-            'extractor_key': BandlabIE.ie_key(),
-            'extractor': BandlabIE.IE_NAME,
-            **traverse_obj(track_data, {
-                'webpage_url': (
-                    'id', ({value(url)}, {format_field(template='https://www.bandlab.com/post/%s')}), filter, any),
-                'id': (('revisionId', 'id'), {str}, any),
-                'url': ('track', 'sample', 'audioUrl', {url_or_none}),
-                'title': ('track', 'name', {str}),
-                'track': ('track', 'name', {str}),
-                'description': ('caption', {str}),
-                'thumbnail': ('track', 'picture', ('original', 'url'), {url_or_none}, any),
-                'view_count': ('counters', 'plays', {int_or_none}),
-                'like_count': ('counters', 'likes', {int_or_none}),
-                'comment_count': ('counters', 'comments', {int_or_none}),
-                'duration': ('track', 'sample', 'duration', {float_or_none}),
-                'uploader': ('creator', 'name', {str}),
-                'uploader_id': ('creator', 'username', {str}),
-                'timestamp': ('createdOn', {parse_iso8601}),
-            }),
-        }
-
-    def _parse_video(self, video_data, url=None):
-        return {
-            'media_type': 'video',
-            'extractor_key': BandlabIE.ie_key(),
-            'extractor': BandlabIE.IE_NAME,
-            **traverse_obj(video_data, {
-                'id': ('id', {str}),
-                'webpage_url': (
-                    'id', ({value(url)}, {format_field(template='https://www.bandlab.com/post/%s')}), filter, any),
-                'url': ('video', 'url', {url_or_none}),
-                'title': ('caption', {lambda x: x.replace('\n', ' ')}, {truncate_string(left=50)}),
-                'description': ('caption', {str}),
-                'thumbnail': ('video', 'picture', 'url', {url_or_none}),
-                'view_count': ('video', 'counters', 'plays', {int_or_none}),
-                'like_count': ('video', 'counters', 'likes', {int_or_none}),
-                'comment_count': ('counters', 'comments', {int_or_none}),
-                'duration': ('video', 'duration', {float_or_none}),
-                'uploader': ('creator', 'name', {str}),
-                'uploader_id': ('creator', 'username', {str}),
-            }),
-        }
-
-
-class BandlabIE(BandlabBaseIE):
-    _VALID_URL = [
-        r'https?://(?:www\.)?bandlab.com/(?P<url_type>track|post|revision)/(?P<id>[\da-f_-]+)',
-        r'https?://(?:www\.)?bandlab.com/(?P<url_type>embed)/\?(?:[^#]*&)?id=(?P<id>[\da-f-]+)',
-    ]
-    _EMBED_REGEX = [rf'<iframe[^>]+src=[\'"](?P<url>{_VALID_URL[1]})[\'"]']
-    _TESTS = [{
-        'url': 'https://www.bandlab.com/track/04b37e88dba24967b9dac8eb8567ff39_07d7f906fc96ee11b75e000d3a428fff',
-        'md5': '46f7b43367dd268bbcf0bbe466753b2c',
-        'info_dict': {
-            'id': '02d7f906-fc96-ee11-b75e-000d3a428fff',
-            'ext': 'm4a',
-            'uploader_id': 'ender_milze',
-            'track': 'sweet black',
-            'description': 'composed by juanjn3737',
-            'timestamp': 1702171963,
-            'view_count': int,
-            'like_count': int,
-            'duration': 54.629999999999995,
-            'title': 'sweet black',
-            'upload_date': '20231210',
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/',
-            'genres': ['Lofi'],
-            'uploader': 'ender milze',
-            'comment_count': int,
-            'media_type': 'revision',
-        },
-    }, {
-        # Same track as above but post URL
-        'url': 'https://www.bandlab.com/post/07d7f906-fc96-ee11-b75e-000d3a428fff',
-        'md5': '46f7b43367dd268bbcf0bbe466753b2c',
-        'info_dict': {
-            'id': '02d7f906-fc96-ee11-b75e-000d3a428fff',
-            'ext': 'm4a',
-            'uploader_id': 'ender_milze',
-            'track': 'sweet black',
-            'description': 'composed by juanjn3737',
-            'timestamp': 1702171973,
-            'view_count': int,
-            'like_count': int,
-            'duration': 54.629999999999995,
-            'title': 'sweet black',
-            'upload_date': '20231210',
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/fa082beb-b856-4730-9170-a57e4e32cc2c/',
-            'genres': ['Lofi'],
-            'uploader': 'ender milze',
-            'comment_count': int,
-            'media_type': 'revision',
-        },
-    }, {
-        # SharedKey Example
-        'url': 'https://www.bandlab.com/track/048916c2-c6da-ee11-85f9-6045bd2e11f9?sharedKey=0NNWX8qYAEmI38lWAzCNDA',
-        'md5': '15174b57c44440e2a2008be9cae00250',
-        'info_dict': {
-            'id': '038916c2-c6da-ee11-85f9-6045bd2e11f9',
-            'ext': 'm4a',
-            'comment_count': int,
-            'genres': ['Other'],
-            'uploader_id': 'user8353034818103753',
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/51b18363-da23-4b9b-a29c-2933a3e561ca/',
-            'timestamp': 1709625771,
-            'track': 'PodcastMaerchen4b',
-            'duration': 468.14,
-            'view_count': int,
-            'description': 'Podcast: Neues aus der Märchenwelt',
-            'like_count': int,
-            'upload_date': '20240305',
-            'uploader': 'Erna Wageneder',
-            'title': 'PodcastMaerchen4b',
-            'media_type': 'revision',
-        },
-    }, {
-        # Different Revision selected
-        'url': 'https://www.bandlab.com/track/130343fc-148b-ea11-96d2-0003ffd1fc09?revId=110343fc-148b-ea11-96d2-0003ffd1fc09',
-        'md5': '74e055ef9325d63f37088772fbfe4454',
-        'info_dict': {
-            'id': '110343fc-148b-ea11-96d2-0003ffd1fc09',
-            'ext': 'm4a',
-            'timestamp': 1588273294,
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/users/b612e533-e4f7-4542-9f50-3fcfd8dd822c/',
-            'description': 'Final Revision.',
-            'title': 'Replay ( Instrumental)',
-            'uploader': 'David R Sparks',
-            'uploader_id': 'davesnothome69',
-            'view_count': int,
-            'comment_count': int,
-            'track': 'Replay ( Instrumental)',
-            'genres': ['Rock'],
-            'upload_date': '20200430',
-            'like_count': int,
-            'duration': 279.43,
-            'media_type': 'revision',
-        },
-    }, {
-        # Video
-        'url': 'https://www.bandlab.com/post/5cdf9036-3857-ef11-991a-6045bd36e0d9',
-        'md5': '8caa2ef28e86c1dacf167293cfdbeba9',
-        'info_dict': {
-            'id': '5cdf9036-3857-ef11-991a-6045bd36e0d9',
-            'ext': 'mp4',
-            'duration': 44.705,
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/videos/67c6cef1-cef6-40d3-831e-a55bc1dcb972/',
-            'comment_count': int,
-            'title': 'backing vocals',
-            'uploader_id': 'marliashya',
-            'uploader': 'auraa',
-            'like_count': int,
-            'description': 'backing vocals',
-            'media_type': 'video',
-        },
-    }, {
-        # Embed Example
-        'url': 'https://www.bandlab.com/embed/?blur=false&id=014de0a4-7d82-ea11-a94c-0003ffd19c0f',
-        'md5': 'a4ad05cb68c54faaed9b0a8453a8cf4a',
-        'info_dict': {
-            'id': '014de0a4-7d82-ea11-a94c-0003ffd19c0f',
-            'ext': 'm4a',
-            'comment_count': int,
-            'genres': ['Electronic'],
-            'uploader': 'Charlie Henson',
-            'timestamp': 1587328674,
-            'upload_date': '20200419',
-            'view_count': int,
-            'track': 'Positronic Meltdown',
-            'duration': 318.55,
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/87165bc3-5439-496e-b1f7-a9f13b541ff2/',
-            'description': 'Checkout my tracks at AOMX http://aomxsounds.com/',
-            'uploader_id': 'microfreaks',
-            'title': 'Positronic Meltdown',
-            'like_count': int,
-            'media_type': 'revision',
-        },
-    }, {
-        # Track without revisions available
-        'url': 'https://www.bandlab.com/track/55767ac51789ea11a94c0003ffd1fc09_2f007b0a37b94ec7a69bc25ae15108a5',
-        'md5': 'f05d68a3769952c2d9257c473e14c15f',
-        'info_dict': {
-            'id': '55767ac51789ea11a94c0003ffd1fc09_2f007b0a37b94ec7a69bc25ae15108a5',
-            'ext': 'm4a',
-            'track': 'insame',
-            'like_count': int,
-            'duration': 84.03,
-            'title': 'insame',
-            'view_count': int,
-            'comment_count': int,
-            'uploader': 'Sorakime',
-            'uploader_id': 'sorakime',
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/users/572a351a-0f3a-4c6a-ac39-1a5defdeeb1c/',
-            'timestamp': 1691162128,
-            'upload_date': '20230804',
-            'media_type': 'track',
-        },
-    }, {
-        'url': 'https://www.bandlab.com/revision/014de0a4-7d82-ea11-a94c-0003ffd19c0f',
-        'only_matching': True,
-    }]
-    _WEBPAGE_TESTS = [{
-        'url': 'https://phantomluigi.github.io/',
-        'info_dict': {
-            'id': 'e14223c3-7871-ef11-bdfd-000d3a980db3',
-            'ext': 'm4a',
-            'view_count': int,
-            'upload_date': '20240913',
-            'uploader_id': 'phantommusicofficial',
-            'timestamp': 1726194897,
-            'uploader': 'Phantom',
-            'comment_count': int,
-            'genres': ['Progresive Rock'],
-            'description': 'md5:a38cd668f7a2843295ef284114f18429',
-            'duration': 225.23,
-            'like_count': int,
-            'title': 'Vermilion Pt. 2 (Cover)',
-            'track': 'Vermilion Pt. 2 (Cover)',
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/songs/62b10750-7aef-4f42-ad08-1af52f577e97/',
-            'media_type': 'revision',
-        },
-    }]
-
-    def _real_extract(self, url):
-        display_id, url_type = self._match_valid_url(url).group('id', 'url_type')
-
-        qs = parse_qs(url)
-        revision_id = traverse_obj(qs, (('revId', 'id'), 0, any))
-        if url_type == 'revision':
-            revision_id = display_id
-
-        revision_data = None
-        if not revision_id:
-            post_data = self._call_api(
-                'posts', display_id, note='Downloading post data',
-                query=traverse_obj(qs, {'sharedKey': ('sharedKey', 0)}))
-
-            revision_id = traverse_obj(post_data, (('revisionId', ('revision', 'id')), {str}, any))
-            revision_data = traverse_obj(post_data, ('revision', {dict}))
-
-            if not revision_data and not revision_id:
-                post_type = post_data.get('type')
-                if post_type == 'Video':
-                    return self._parse_video(post_data, url=url)
-                if post_type == 'Track':
-                    return self._parse_track(post_data, url=url)
-                raise ExtractorError(f'Could not extract data for post type {post_type!r}')
-
-        if not revision_data:
-            revision_data = self._call_api(
-                'revisions', revision_id, note='Downloading revision data', query={'edit': 'false'})
-
-        return self._parse_revision(revision_data, url=url)
-
-
-class BandlabPlaylistIE(BandlabBaseIE):
-    _VALID_URL = [
-        r'https?://(?:www\.)?bandlab.com/(?:[\w]+/)?(?P<type>albums|collections)/(?P<id>[\da-f-]+)',
-        r'https?://(?:www\.)?bandlab.com/(?P<type>embed)/collection/\?(?:[^#]*&)?id=(?P<id>[\da-f-]+)',
-    ]
-    _EMBED_REGEX = [rf'<iframe[^>]+src=[\'"](?P<url>{_VALID_URL[1]})[\'"]']
-    _TESTS = [{
-        'url': 'https://www.bandlab.com/davesnothome69/albums/89b79ea6-de42-ed11-b495-00224845aac7',
-        'info_dict': {
-            'thumbnail': 'https://bl-prod-images.azureedge.net/v1.3/albums/69507ff3-579a-45be-afca-9e87eddec944/',
-            'release_date': '20221003',
-            'title': 'Remnants',
-            'album': 'Remnants',
-            'like_count': int,
-            'album_type': 'LP',
-            'description': 'A collection of some feel good, rock hits.',
-            'comment_count': int,
-            'view_count': int,
-            'id': '89b79ea6-de42-ed11-b495-00224845aac7',
-            'uploader': 'David R Sparks',
-            'uploader_id': 'davesnothome69',
-        },
-        'playlist_count': 10,
-    }, {
-        'url': 'https://www.bandlab.com/slytheband/collections/955102d4-1040-ef11-86c3-000d3a42581b',
-        'info_dict': {
-            'id': '955102d4-1040-ef11-86c3-000d3a42581b',
-            'timestamp': 1720762659,
-            'view_count': int,
-            'title': 'My Shit 🖤',
-            'uploader_id': 'slytheband',
-            'uploader': '𝓢𝓛𝓨',
-            'upload_date': '20240712',
-            'like_count': int,
-            'thumbnail': 'https://bandlabimages.azureedge.net/v1.0/collections/2c64ca12-b180-4b76-8587-7a8da76bddc8/',
-        },
-        'playlist_count': 15,
-    }, {
-        # Embeds can contain both albums and collections with the same URL pattern. This is an album
-        'url': 'https://www.bandlab.com/embed/collection/?id=12cc6f7f-951b-ee11-907c-00224844f303',
-        'info_dict': {
-            'id': '12cc6f7f-951b-ee11-907c-00224844f303',
-            'release_date': '20230706',
-            'description': 'This is a collection of songs I created when I had an Amiga computer.',
-            'view_count': int,
-            'title': 'Mark Salud The Amiga Collection',
-            'uploader_id': 'mssirmooth1962',
-            'comment_count': int,
-            'thumbnail': 'https://bl-prod-images.azureedge.net/v1.3/albums/d618bd7b-0537-40d5-bdd8-61b066e77d59/',
-            'like_count': int,
-            'uploader': 'Mark Salud',
-            'album': 'Mark Salud The Amiga Collection',
-            'album_type': 'LP',
-        },
-        'playlist_count': 24,
-    }, {
-        # Tracks without revision id
-        'url': 'https://www.bandlab.com/embed/collection/?id=e98aafb5-d932-ee11-b8f0-00224844c719',
-        'info_dict': {
-            'like_count': int,
-            'uploader_id': 'sorakime',
-            'comment_count': int,
-            'uploader': 'Sorakime',
-            'view_count': int,
-            'description': 'md5:4ec31c568a5f5a5a2b17572ea64c3825',
-            'release_date': '20230812',
-            'title': 'Art',
-            'album': 'Art',
-            'album_type': 'Album',
-            'id': 'e98aafb5-d932-ee11-b8f0-00224844c719',
-            'thumbnail': 'https://bl-prod-images.azureedge.net/v1.3/albums/20c890de-e94a-4422-828a-2da6377a13c8/',
-        },
-        'playlist_count': 13,
-    }, {
-        'url': 'https://www.bandlab.com/albums/89b79ea6-de42-ed11-b495-00224845aac7',
-        'only_matching': True,
-    }]
-
-    def _entries(self, album_data):
-        for post in traverse_obj(album_data, ('posts', lambda _, v: v['type'])):
-            post_type = post['type']
-            if post_type == 'Revision':
-                yield self._parse_revision(post.get('revision'))
-            elif post_type == 'Track':
-                yield self._parse_track(post)
-            elif post_type == 'Video':
-                yield self._parse_video(post)
-            else:
-                self.report_warning(f'Skipping unknown post type: "{post_type}"')
-
-    def _real_extract(self, url):
-        playlist_id, playlist_type = self._match_valid_url(url).group('id', 'type')
-
-        endpoints = {
-            'albums': ['albums'],
-            'collections': ['collections'],
-            'embed': ['collections', 'albums'],
-        }.get(playlist_type)
-        for endpoint in endpoints:
-            playlist_data = self._call_api(
-                endpoint, playlist_id, note=f'Downloading {endpoint[:-1]} data',
-                fatal=False, expected_status=404)
-            if not playlist_data.get('errorCode'):
-                playlist_type = endpoint
-                break
-        if error_code := playlist_data.get('errorCode'):
-            raise ExtractorError(f'Could not find playlist data. Error code: "{error_code}"')
-
-        return self.playlist_result(
-            self._entries(playlist_data), playlist_id,
-            **traverse_obj(playlist_data, {
-                'title': ('name', {str}),
-                'description': ('description', {str}),
-                'uploader': ('creator', 'name', {str}),
-                'uploader_id': ('creator', 'username', {str}),
-                'timestamp': ('createdOn', {parse_iso8601}),
-                'release_date': ('releaseDate', {lambda x: x.replace('-', '')}, filter),
-                'thumbnail': ('picture', ('original', 'url'), {url_or_none}, any),
-                'like_count': ('counters', 'likes', {int_or_none}),
-                'comment_count': ('counters', 'comments', {int_or_none}),
-                'view_count': ('counters', 'plays', {int_or_none}),
-            }),
-            **(traverse_obj(playlist_data, {
-                'album': ('name', {str}),
-                'album_type': ('type', {str}),
-            }) if playlist_type == 'albums' else {}))

yt_dlp/extractor/chaturbate.py

@@ -5,7 +5,6 @@ from ..utils import (
     ExtractorError,
     lowercase_escape,
     url_or_none,
-    urlencode_postdata,
 )
 
 
@@ -41,48 +40,14 @@ class ChaturbateIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _ERROR_MAP = {
-        'offline': 'Room is currently offline',
-        'private': 'Room is currently in a private show',
-        'away': 'Performer is currently away',
-        'password protected': 'Room is password protected',
-        'hidden': 'Hidden session in progress',
-    }
+    _ROOM_OFFLINE = 'Room is currently offline'
 
-    def _extract_from_api(self, video_id, tld):
-        response = self._download_json(
-            f'https://chaturbate.{tld}/get_edge_hls_url_ajax/', video_id,
-            data=urlencode_postdata({'room_slug': video_id}),
-            headers={
-                **self.geo_verification_headers(),
-                'X-Requested-With': 'XMLHttpRequest',
-                'Accept': 'application/json',
-            }, fatal=False, impersonate=True) or {}
+    def _real_extract(self, url):
+        video_id, tld = self._match_valid_url(url).group('id', 'tld')
 
-        status = response.get('room_status')
-        if status != 'public':
-            if error := self._ERROR_MAP.get(status):
-                raise ExtractorError(error, expected=True)
-            self.report_warning('Falling back to webpage extraction')
-            return None
-
-        m3u8_url = response.get('url')
-        if not m3u8_url:
-            self.raise_geo_restricted()
-
-        return {
-            'id': video_id,
-            'title': video_id,
-            'thumbnail': f'https://roomimg.stream.highwebmedia.com/ri/{video_id}.jpg',
-            'is_live': True,
-            'age_limit': 18,
-            'formats': self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', live=True),
-        }
-
-    def _extract_from_webpage(self, video_id, tld):
         webpage = self._download_webpage(
             f'https://chaturbate.{tld}/{video_id}/', video_id,
-            headers=self.geo_verification_headers(), impersonate=True)
+            headers=self.geo_verification_headers())
 
         found_m3u8_urls = []
 
@@ -120,8 +85,8 @@ class ChaturbateIE(InfoExtractor):
                 webpage, 'error', group='error', default=None)
             if not error:
                 if any(p in webpage for p in (
-                        self._ERROR_MAP['offline'], 'offline_tipping', 'tip_offline')):
-                    error = self._ERROR_MAP['offline']
+                        self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
+                    error = self._ROOM_OFFLINE
             if error:
                 raise ExtractorError(error, expected=True)
             raise ExtractorError('Unable to find stream URL')
@@ -148,7 +113,3 @@ class ChaturbateIE(InfoExtractor):
             'is_live': True,
             'formats': formats,
         }
-
-    def _real_extract(self, url):
-        video_id, tld = self._match_valid_url(url).group('id', 'tld')
-        return self._extract_from_api(video_id, tld) or self._extract_from_webpage(video_id, tld)

yt_dlp/extractor/common.py

@@ -25,6 +25,7 @@ import xml.etree.ElementTree
 from ..compat import (
     compat_etree_fromstring,
     compat_expanduser,
+    compat_os_name,
     urllib_req_to_req,
 )
 from ..cookies import LenientSimpleCookie
@@ -278,7 +279,6 @@ class InfoExtractor:
     thumbnails:     A list of dictionaries, with the following entries:
                         * "id" (optional, string) - Thumbnail format ID
                         * "url"
-                        * "ext" (optional, string) - actual image extension if not given in URL
                         * "preference" (optional, int) - quality of the image
                         * "width" (optional, int)
                         * "height" (optional, int)
@@ -1028,7 +1028,7 @@ class InfoExtractor:
         filename = sanitize_filename(f'{basen}.dump', restricted=True)
         # Working around MAX_PATH limitation on Windows (see
         # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
-        if os.name == 'nt':
+        if compat_os_name == 'nt':
             absfilepath = os.path.abspath(filename)
             if len(absfilepath) > 259:
                 filename = fR'\\?\{absfilepath}'

yt_dlp/extractor/patreon.py

@@ -16,10 +16,10 @@ from ..utils import (
     parse_iso8601,
     smuggle_url,
     str_or_none,
+    traverse_obj,
     url_or_none,
     urljoin,
 )
-from ..utils.traversal import traverse_obj, value
 
 
 class PatreonBaseIE(InfoExtractor):
@@ -252,27 +252,6 @@ class PatreonIE(PatreonBaseIE):
             'thumbnail': r're:^https?://.+',
         },
         'skip': 'Patron-only content',
-    }, {
-        # Contains a comment reply in the 'included' section
-        'url': 'https://www.patreon.com/posts/114721679',
-        'info_dict': {
-            'id': '114721679',
-            'ext': 'mp4',
-            'upload_date': '20241025',
-            'uploader': 'Japanalysis',
-            'like_count': int,
-            'thumbnail': r're:^https?://.+',
-            'comment_count': int,
-            'title': 'Karasawa Part 2',
-            'description': 'Part 2 of this video https://www.youtube.com/watch?v=Azms2-VTASk',
-            'uploader_url': 'https://www.patreon.com/japanalysis',
-            'uploader_id': '80504268',
-            'channel_url': 'https://www.patreon.com/japanalysis',
-            'channel_follower_count': int,
-            'timestamp': 1729897015,
-            'channel_id': '9346307',
-        },
-        'params': {'getcomments': True},
     }]
     _RETURN_TYPE = 'video'
 
@@ -425,24 +404,26 @@ class PatreonIE(PatreonBaseIE):
                 f'posts/{post_id}/comments', post_id, query=params, note=f'Downloading comments page {page}')
 
             cursor = None
-            for comment in traverse_obj(response, (('data', 'included'), lambda _, v: v['type'] == 'comment' and v['id'])):
+            for comment in traverse_obj(response, (('data', ('included', lambda _, v: v['type'] == 'comment')), ...)):
                 count += 1
+                comment_id = comment.get('id')
+                attributes = comment.get('attributes') or {}
+                if comment_id is None:
+                    continue
                 author_id = traverse_obj(comment, ('relationships', 'commenter', 'data', 'id'))
+                author_info = traverse_obj(
+                    response, ('included', lambda _, v: v['id'] == author_id and v['type'] == 'user', 'attributes'),
+                    get_all=False, expected_type=dict, default={})
 
                 yield {
-                    **traverse_obj(comment, {
-                        'id': ('id', {str_or_none}),
-                        'text': ('attributes', 'body', {str}),
-                        'timestamp': ('attributes', 'created', {parse_iso8601}),
-                        'parent': ('relationships', 'parent', 'data', ('id', {value('root')}), {str}, any),
-                        'author_is_uploader': ('attributes', 'is_by_creator', {bool}),
-                    }),
-                    **traverse_obj(response, (
-                        'included', lambda _, v: v['id'] == author_id and v['type'] == 'user', 'attributes', {
-                            'author': ('full_name', {str}),
-                            'author_thumbnail': ('image_url', {url_or_none}),
-                        }), get_all=False),
+                    'id': comment_id,
+                    'text': attributes.get('body'),
+                    'timestamp': parse_iso8601(attributes.get('created')),
+                    'parent': traverse_obj(comment, ('relationships', 'parent', 'data', 'id'), default='root'),
+                    'author_is_uploader': attributes.get('is_by_creator'),
                     'author_id': author_id,
+                    'author': author_info.get('full_name'),
+                    'author_thumbnail': author_info.get('image_url'),
                 }
 
             if count < traverse_obj(response, ('meta', 'count')):

yt_dlp/extractor/radioradicale.py

@@ -1,105 +0,0 @@
-from .common import InfoExtractor
-from ..utils import url_or_none
-from ..utils.traversal import traverse_obj
-
-
-class RadioRadicaleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?radioradicale\.it/scheda/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'https://www.radioradicale.it/scheda/471591',
-        'md5': 'eb0fbe43a601f1a361cbd00f3c45af4a',
-        'info_dict': {
-            'id': '471591',
-            'ext': 'mp4',
-            'title': 'md5:e8fbb8de57011a3255db0beca69af73d',
-            'description': 'md5:5e15a789a2fe4d67da8d1366996e89ef',
-            'location': 'Napoli',
-            'duration': 2852.0,
-            'timestamp': 1459987200,
-            'upload_date': '20160407',
-            'thumbnail': 'https://www.radioradicale.it/photo400/0/0/9/0/1/00901768.jpg',
-        },
-    }, {
-        'url': 'https://www.radioradicale.it/scheda/742783/parlamento-riunito-in-seduta-comune-11a-della-xix-legislatura',
-        'info_dict': {
-            'id': '742783',
-            'title': 'Parlamento riunito in seduta comune (11ª della XIX legislatura)',
-            'description': '-) Votazione per l\'elezione di un giudice della Corte Costituzionale (nono scrutinio)',
-            'location': 'CAMERA',
-            'duration': 5868.0,
-            'timestamp': 1730246400,
-            'upload_date': '20241030',
-        },
-        'playlist': [{
-            'md5': 'aa48de55dcc45478e4cd200f299aab7d',
-            'info_dict': {
-                'id': '742783-0',
-                'ext': 'mp4',
-                'title': 'Parlamento riunito in seduta comune (11ª della XIX legislatura)',
-            },
-        }, {
-            'md5': 'be915c189c70ad2920e5810f32260ff5',
-            'info_dict': {
-                'id': '742783-1',
-                'ext': 'mp4',
-                'title': 'Parlamento riunito in seduta comune (11ª della XIX legislatura)',
-            },
-        }, {
-            'md5': 'f0ee4047342baf8ed3128a8417ac5e0a',
-            'info_dict': {
-                'id': '742783-2',
-                'ext': 'mp4',
-                'title': 'Parlamento riunito in seduta comune (11ª della XIX legislatura)',
-            },
-        }],
-    }]
-
-    def _entries(self, videos_info, page_id):
-        for idx, video in enumerate(traverse_obj(
-                videos_info, ('playlist', lambda _, v: v['sources']))):
-            video_id = f'{page_id}-{idx}'
-            formats = []
-            subtitles = {}
-
-            for m3u8_url in traverse_obj(video, ('sources', ..., 'src', {url_or_none})):
-                fmts, subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id)
-                formats.extend(fmts)
-                self._merge_subtitles(subs, target=subtitles)
-            for sub in traverse_obj(video, ('subtitles', ..., lambda _, v: url_or_none(v['src']))):
-                self._merge_subtitles({sub.get('srclang') or 'und': [{
-                    'url': sub['src'],
-                    'name': sub.get('label'),
-                }]}, target=subtitles)
-
-            yield {
-                'id': video_id,
-                'title': video.get('title'),
-                'formats': formats,
-                'subtitles': subtitles,
-            }
-
-    def _real_extract(self, url):
-        page_id = self._match_id(url)
-        webpage = self._download_webpage(url, page_id)
-
-        videos_info = self._search_json(
-            r'jQuery\.extend\(Drupal\.settings\s*,',
-            webpage, 'videos_info', page_id)['RRscheda']
-
-        entries = list(self._entries(videos_info, page_id))
-
-        common_info = {
-            'id': page_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'location': videos_info.get('luogo'),
-            **self._search_json_ld(webpage, page_id),
-        }
-
-        if len(entries) == 1:
-            return {
-                **entries[0],
-                **common_info,
-            }
-
-        return self.playlist_result(entries, multi_video=True, **common_info)

yt_dlp/extractor/redgifs.py

@@ -213,7 +213,7 @@ class RedGifsSearchIE(RedGifsBaseInfoExtractor):
 class RedGifsUserIE(RedGifsBaseInfoExtractor):
     IE_DESC = 'Redgifs user'
     _VALID_URL = r'https?://(?:www\.)?redgifs\.com/users/(?P<username>[^/?#]+)(?:\?(?P<query>[^#]+))?'
-    _PAGE_SIZE = 80
+    _PAGE_SIZE = 30
     _TESTS = [
         {
             'url': 'https://www.redgifs.com/users/lamsinka89',
@@ -222,7 +222,7 @@ class RedGifsUserIE(RedGifsBaseInfoExtractor):
                 'title': 'lamsinka89',
                 'description': 'RedGifs user lamsinka89, ordered by recent',
             },
-            'playlist_mincount': 391,
+            'playlist_mincount': 100,
         },
         {
             'url': 'https://www.redgifs.com/users/lamsinka89?page=3',
@@ -231,7 +231,7 @@ class RedGifsUserIE(RedGifsBaseInfoExtractor):
                 'title': 'lamsinka89',
                 'description': 'RedGifs user lamsinka89, ordered by recent',
             },
-            'playlist_count': 80,
+            'playlist_count': 30,
         },
         {
             'url': 'https://www.redgifs.com/users/lamsinka89?order=best&type=g',
@@ -240,17 +240,7 @@ class RedGifsUserIE(RedGifsBaseInfoExtractor):
                 'title': 'lamsinka89',
                 'description': 'RedGifs user lamsinka89, ordered by best',
             },
-            'playlist_mincount': 391,
-        },
-        {
-            'url': 'https://www.redgifs.com/users/ignored52',
-            'note': 'https://github.com/yt-dlp/yt-dlp/issues/7382',
-            'info_dict': {
-                'id': 'ignored52',
-                'title': 'ignored52',
-                'description': 'RedGifs user ignored52, ordered by recent',
-            },
-            'playlist_mincount': 121,
+            'playlist_mincount': 100,
         },
     ]
 

yt_dlp/extractor/shemaroome.py

@@ -1,9 +1,11 @@
 import base64
 
 from .common import InfoExtractor
-from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
+from ..aes import aes_cbc_decrypt, unpad_pkcs7
 from ..utils import (
     ExtractorError,
+    bytes_to_intlist,
+    intlist_to_bytes,
     unified_strdate,
 )
 
@@ -66,10 +68,10 @@ class ShemarooMeIE(InfoExtractor):
         data_json = self._download_json('https://www.shemaroome.com/users/user_all_lists', video_id, data=data.encode())
         if not data_json.get('status'):
             raise ExtractorError('Premium videos cannot be downloaded yet.', expected=True)
-        url_data = base64.b64decode(data_json['new_play_url'])
-        key = base64.b64decode(data_json['key'])
-        iv = bytes(16)
-        m3u8_url = unpad_pkcs7(aes_cbc_decrypt_bytes(url_data, key, iv)).decode('ascii')
+        url_data = bytes_to_intlist(base64.b64decode(data_json['new_play_url']))
+        key = bytes_to_intlist(base64.b64decode(data_json['key']))
+        iv = [0] * 16
+        m3u8_url = unpad_pkcs7(intlist_to_bytes(aes_cbc_decrypt(url_data, key, iv))).decode('ascii')
         headers = {'stream_key': data_json['stream_key']}
         formats, m3u8_subs = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, fatal=False, headers=headers)
         for fmt in formats:

yt_dlp/extractor/spankbang.py

@@ -71,11 +71,9 @@ class SpankBangIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = self._match_valid_url(url)
         video_id = mobj.group('id') or mobj.group('id_2')
-        country = self.get_param('geo_bypass_country') or 'US'
-        self._set_cookie('.spankbang.com', 'country', country.upper())
         webpage = self._download_webpage(
             url.replace(f'/{video_id}/embed', f'/{video_id}/video'),
-            video_id, impersonate=True)
+            video_id, headers={'Cookie': 'country=US'})
 
         if re.search(r'<[^>]+\b(?:id|class)=["\']video_removed', webpage):
             raise ExtractorError(

yt_dlp/extractor/youtube.py

@@ -22,7 +22,7 @@ import urllib.parse
 from .common import InfoExtractor, SearchInfoExtractor
 from .openload import PhantomJSwrapper
 from ..jsinterp import JSInterpreter
-from ..networking.exceptions import HTTPError, network_exceptions
+from ..networking.exceptions import HTTPError, TransportError, network_exceptions
 from ..utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -50,12 +50,12 @@ from ..utils import (
     parse_iso8601,
     parse_qs,
     qualities,
-    remove_end,
     remove_start,
     smuggle_url,
     str_or_none,
     str_to_int,
     strftime_or_none,
+    time_seconds,
     traverse_obj,
     try_call,
     try_get,
@@ -124,15 +124,14 @@ INNERTUBE_CLIENTS = {
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
-        'REQUIRE_AUTH': True,
     },
     'android': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'ANDROID',
-                'clientVersion': '19.44.38',
+                'clientVersion': '19.29.37',
                 'androidSdkVersion': 30,
-                'userAgent': 'com.google.android.youtube/19.44.38 (Linux; U; Android 11) gzip',
+                'userAgent': 'com.google.android.youtube/19.29.37 (Linux; U; Android 11) gzip',
                 'osName': 'Android',
                 'osVersion': '11',
             },
@@ -141,14 +140,13 @@ INNERTUBE_CLIENTS = {
         'REQUIRE_JS_PLAYER': False,
         'REQUIRE_PO_TOKEN': True,
     },
-    # This client now requires sign-in for every video
     'android_music': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'ANDROID_MUSIC',
-                'clientVersion': '7.27.52',
+                'clientVersion': '7.11.50',
                 'androidSdkVersion': 30,
-                'userAgent': 'com.google.android.apps.youtube.music/7.27.52 (Linux; U; Android 11) gzip',
+                'userAgent': 'com.google.android.apps.youtube.music/7.11.50 (Linux; U; Android 11) gzip',
                 'osName': 'Android',
                 'osVersion': '11',
             },
@@ -156,16 +154,15 @@ INNERTUBE_CLIENTS = {
         'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
         'REQUIRE_JS_PLAYER': False,
         'REQUIRE_PO_TOKEN': True,
-        'REQUIRE_AUTH': True,
     },
     # This client now requires sign-in for every video
     'android_creator': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'ANDROID_CREATOR',
-                'clientVersion': '24.45.100',
+                'clientVersion': '24.30.100',
                 'androidSdkVersion': 30,
-                'userAgent': 'com.google.android.apps.youtube.creator/24.45.100 (Linux; U; Android 11) gzip',
+                'userAgent': 'com.google.android.apps.youtube.creator/24.30.100 (Linux; U; Android 11) gzip',
                 'osName': 'Android',
                 'osVersion': '11',
             },
@@ -173,18 +170,17 @@ INNERTUBE_CLIENTS = {
         'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
         'REQUIRE_JS_PLAYER': False,
         'REQUIRE_PO_TOKEN': True,
-        'REQUIRE_AUTH': True,
     },
     # YouTube Kids videos aren't returned on this client for some reason
     'android_vr': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'ANDROID_VR',
-                'clientVersion': '1.60.19',
+                'clientVersion': '1.57.29',
                 'deviceMake': 'Oculus',
                 'deviceModel': 'Quest 3',
                 'androidSdkVersion': 32,
-                'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
+                'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.57.29 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
                 'osName': 'Android',
                 'osVersion': '12L',
             },
@@ -192,56 +188,68 @@ INNERTUBE_CLIENTS = {
         'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
         'REQUIRE_JS_PLAYER': False,
     },
+    'android_testsuite': {
+        'INNERTUBE_CONTEXT': {
+            'client': {
+                'clientName': 'ANDROID_TESTSUITE',
+                'clientVersion': '1.9',
+                'androidSdkVersion': 30,
+                'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 11) gzip',
+                'osName': 'Android',
+                'osVersion': '11',
+            },
+        },
+        'INNERTUBE_CONTEXT_CLIENT_NAME': 30,
+        'REQUIRE_JS_PLAYER': False,
+        'PLAYER_PARAMS': '2AMB',
+    },
     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
     'ios': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'IOS',
-                'clientVersion': '19.45.4',
+                'clientVersion': '19.29.1',
                 'deviceMake': 'Apple',
                 'deviceModel': 'iPhone16,2',
-                'userAgent': 'com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                'userAgent': 'com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
                 'osName': 'iPhone',
-                'osVersion': '18.1.0.22B83',
+                'osVersion': '17.5.1.21F90',
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
         'REQUIRE_JS_PLAYER': False,
     },
-    # This client now requires sign-in for every video
     'ios_music': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'IOS_MUSIC',
-                'clientVersion': '7.27.0',
+                'clientVersion': '7.08.2',
                 'deviceMake': 'Apple',
                 'deviceModel': 'iPhone16,2',
-                'userAgent': 'com.google.ios.youtubemusic/7.27.0 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                'userAgent': 'com.google.ios.youtubemusic/7.08.2 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
                 'osName': 'iPhone',
-                'osVersion': '18.1.0.22B83',
+                'osVersion': '17.5.1.21F90',
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
         'REQUIRE_JS_PLAYER': False,
-        'REQUIRE_AUTH': True,
     },
     # This client now requires sign-in for every video
     'ios_creator': {
         'INNERTUBE_CONTEXT': {
             'client': {
                 'clientName': 'IOS_CREATOR',
-                'clientVersion': '24.45.100',
+                'clientVersion': '24.30.100',
                 'deviceMake': 'Apple',
                 'deviceModel': 'iPhone16,2',
-                'userAgent': 'com.google.ios.ytcreator/24.45.100 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                'userAgent': 'com.google.ios.ytcreator/24.30.100 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)',
                 'osName': 'iPhone',
-                'osVersion': '18.1.0.22B83',
+                'osVersion': '17.5.1.21F90',
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
         'REQUIRE_JS_PLAYER': False,
-        'REQUIRE_AUTH': True,
     },
     # mweb has 'ultralow' formats
     # See: https://github.com/yt-dlp/yt-dlp/pull/557
@@ -274,10 +282,8 @@ INNERTUBE_CLIENTS = {
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 85,
-        'REQUIRE_AUTH': True,
     },
-    # This client now requires sign-in for every video
-    # It may be able to receive pre-merged video+audio 720p/1080p streams
+    # This client has pre-merged video+audio 720p/1080p streams
     'mediaconnect': {
         'INNERTUBE_CONTEXT': {
             'client': {
@@ -287,7 +293,6 @@ INNERTUBE_CLIENTS = {
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 95,
         'REQUIRE_JS_PLAYER': False,
-        'REQUIRE_AUTH': True,
     },
 }
 
@@ -316,7 +321,6 @@ def build_innertube_clients():
         ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
         ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
         ytcfg.setdefault('REQUIRE_PO_TOKEN', False)
-        ytcfg.setdefault('REQUIRE_AUTH', False)
         ytcfg.setdefault('PLAYER_PARAMS', None)
         ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
 
@@ -573,18 +577,208 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         self._check_login_required()
 
     def _perform_login(self, username, password):
-        if username.startswith('oauth'):
-            raise ExtractorError(
-                f'Login with OAuth is no longer supported. {self._youtube_login_hint}', expected=True)
+        auth_type, _, user = (username or '').partition('+')
 
-        self.report_warning(
-            f'Login with password is not supported for YouTube. {self._youtube_login_hint}')
+        if auth_type != 'oauth':
+            raise ExtractorError(self._youtube_login_hint, expected=True)
+
+        self._initialize_oauth(user, password)
+
+    '''
+    OAuth 2.0 Device Authorization Grant flow, used by the YouTube TV client (youtube.com/tv).
+
+    For more information regarding OAuth 2.0 and the Device Authorization Grant flow in general, see:
+    - https://developers.google.com/identity/protocols/oauth2/limited-input-device
+    - https://accounts.google.com/.well-known/openid-configuration
+    - https://www.rfc-editor.org/rfc/rfc8628
+    - https://www.rfc-editor.org/rfc/rfc6749
+
+    Note: The official client appears to use a proxied version of the oauth2 endpoints on youtube.com/o/oauth2,
+    which applies some modifications to the response (such as returning errors as 200 OK).
+    Since the client works with the standard API, we will use that as it is well-documented.
+    '''
+
+    _OAUTH_PROFILE = None
+    _OAUTH_ACCESS_TOKEN_CACHE = {}
+    _OAUTH_DISPLAY_ID = 'oauth'
+
+    # YouTube TV (TVHTML5) client. You can find these at youtube.com/tv
+    _OAUTH_CLIENT_ID = '861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com'
+    _OAUTH_CLIENT_SECRET = 'SboVhoG9s0rNafixCSGGKXAT'
+    _OAUTH_SCOPE = 'http://gdata.youtube.com https://www.googleapis.com/auth/youtube-paid-content'
+
+    # From https://accounts.google.com/.well-known/openid-configuration
+    # Technically, these should be fetched dynamically and not hard-coded.
+    # However, as these endpoints rarely change, we can risk saving an extra request for every invocation.
+    _OAUTH_DEVICE_AUTHORIZATION_ENDPOINT = 'https://oauth2.googleapis.com/device/code'
+    _OAUTH_TOKEN_ENDPOINT = 'https://oauth2.googleapis.com/token'
+
+    @property
+    def _oauth_cache_key(self):
+        return f'oauth_refresh_token_{self._OAUTH_PROFILE}'
+
+    def _read_oauth_error_response(self, response):
+        return traverse_obj(
+            self._webpage_read_content(response, self._OAUTH_TOKEN_ENDPOINT, self._OAUTH_DISPLAY_ID, fatal=False),
+            ({json.loads}, 'error', {str}))
+
+    def _set_oauth_info(self, token_response):
+        YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE.setdefault(self._OAUTH_PROFILE, {}).update({
+            'access_token': token_response['access_token'],
+            'token_type': token_response['token_type'],
+            'expiry': time_seconds(
+                seconds=traverse_obj(token_response, ('expires_in', {float_or_none}), default=300) - 10),
+        })
+        refresh_token = traverse_obj(token_response, ('refresh_token', {str}))
+        if refresh_token:
+            self.cache.store(self._NETRC_MACHINE, self._oauth_cache_key, refresh_token)
+            YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE[self._OAUTH_PROFILE]['refresh_token'] = refresh_token
+
+    def _initialize_oauth(self, user, refresh_token):
+        self._OAUTH_PROFILE = user or 'default'
+
+        if self._OAUTH_PROFILE in YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE:
+            self.write_debug(f'{self._OAUTH_DISPLAY_ID}: Using cached access token for profile "{self._OAUTH_PROFILE}"')
+            return
+
+        YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE[self._OAUTH_PROFILE] = {}
+
+        if refresh_token:
+            msg = f'{self._OAUTH_DISPLAY_ID}: Using password input as refresh token'
+            if self.get_param('cachedir') is not False:
+                msg += ' and caching token to disk; you should supply an empty password next time'
+            self.to_screen(msg)
+            self.cache.store(self._NETRC_MACHINE, self._oauth_cache_key, refresh_token)
+        else:
+            refresh_token = self.cache.load(self._NETRC_MACHINE, self._oauth_cache_key)
+
+        if refresh_token:
+            YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE[self._OAUTH_PROFILE]['refresh_token'] = refresh_token
+            try:
+                token_response = self._refresh_token(refresh_token)
+            except ExtractorError as e:
+                error_msg = str(e.orig_msg).replace('Failed to refresh access token: ', '')
+                self.report_warning(f'{self._OAUTH_DISPLAY_ID}: Failed to refresh access token: {error_msg}')
+                token_response = self._oauth_authorize
+        else:
+            token_response = self._oauth_authorize
+
+        self._set_oauth_info(token_response)
+        self.write_debug(f'{self._OAUTH_DISPLAY_ID}: Logged in using profile "{self._OAUTH_PROFILE}"')
+
+    def _refresh_token(self, refresh_token):
+        try:
+            token_response = self._download_json(
+                self._OAUTH_TOKEN_ENDPOINT,
+                video_id=self._OAUTH_DISPLAY_ID,
+                note='Refreshing access token',
+                data=json.dumps({
+                    'client_id': self._OAUTH_CLIENT_ID,
+                    'client_secret': self._OAUTH_CLIENT_SECRET,
+                    'refresh_token': refresh_token,
+                    'grant_type': 'refresh_token',
+                }).encode(),
+                headers={'Content-Type': 'application/json'})
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError):
+                error = self._read_oauth_error_response(e.cause.response)
+                if error == 'invalid_grant':
+                    # RFC6749 § 5.2
+                    raise ExtractorError(
+                        'Failed to refresh access token: Refresh token is invalid, revoked, or expired (invalid_grant)',
+                        expected=True, video_id=self._OAUTH_DISPLAY_ID)
+                raise ExtractorError(
+                    f'Failed to refresh access token: Authorization server returned error {error}',
+                    video_id=self._OAUTH_DISPLAY_ID)
+            raise
+        return token_response
+
+    @property
+    def _oauth_authorize(self):
+        code_response = self._download_json(
+            self._OAUTH_DEVICE_AUTHORIZATION_ENDPOINT,
+            video_id=self._OAUTH_DISPLAY_ID,
+            note='Initializing authorization flow',
+            data=json.dumps({
+                'client_id': self._OAUTH_CLIENT_ID,
+                'scope': self._OAUTH_SCOPE,
+            }).encode(),
+            headers={'Content-Type': 'application/json'})
+
+        verification_url = traverse_obj(code_response, ('verification_url', {str}))
+        user_code = traverse_obj(code_response, ('user_code', {str}))
+        if not verification_url or not user_code:
+            raise ExtractorError(
+                'Authorization server did not provide verification_url or user_code', video_id=self._OAUTH_DISPLAY_ID)
+
+        # note: The whitespace is intentional
+        self.to_screen(
+            f'{self._OAUTH_DISPLAY_ID}: To give yt-dlp access to your account, '
+            f'go to  {verification_url}  and enter code  {user_code}')
+
+        # RFC8628 § 3.5: default poll interval is 5 seconds if not provided
+        poll_interval = traverse_obj(code_response, ('interval', {int}), default=5)
+
+        for retry in self.RetryManager():
+            while True:
+                try:
+                    token_response = self._download_json(
+                        self._OAUTH_TOKEN_ENDPOINT,
+                        video_id=self._OAUTH_DISPLAY_ID,
+                        note=False,
+                        errnote='Failed to request access token',
+                        data=json.dumps({
+                            'client_id': self._OAUTH_CLIENT_ID,
+                            'client_secret': self._OAUTH_CLIENT_SECRET,
+                            'device_code': code_response['device_code'],
+                            'grant_type': 'urn:ietf:params:oauth:grant-type:device_code',
+                        }).encode(),
+                        headers={'Content-Type': 'application/json'})
+                except ExtractorError as e:
+                    if isinstance(e.cause, TransportError):
+                        retry.error = e
+                        break
+                    elif isinstance(e.cause, HTTPError):
+                        error = self._read_oauth_error_response(e.cause.response)
+                        if not error:
+                            retry.error = e
+                            break
+
+                        if error == 'authorization_pending':
+                            time.sleep(poll_interval)
+                            continue
+                        elif error == 'expired_token':
+                            raise ExtractorError(
+                                'Authorization timed out', expected=True, video_id=self._OAUTH_DISPLAY_ID)
+                        elif error == 'access_denied':
+                            raise ExtractorError(
+                                'You denied access to an account', expected=True, video_id=self._OAUTH_DISPLAY_ID)
+                        elif error == 'slow_down':
+                            # RFC8628 § 3.5: add 5 seconds to the poll interval
+                            poll_interval += 5
+                            time.sleep(poll_interval)
+                            continue
+                        else:
+                            raise ExtractorError(
+                                f'Authorization server returned an error when fetching access token: {error}',
+                                video_id=self._OAUTH_DISPLAY_ID)
+                    raise
+
+                return token_response
+
+    def _update_oauth(self):
+        token = YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE.get(self._OAUTH_PROFILE)
+        if token is None or token['expiry'] > time.time():
+            return
+
+        self._set_oauth_info(self._refresh_token(token['refresh_token']))
 
     @property
     def _youtube_login_hint(self):
-        return (f'{self._login_hint(method="cookies")}. Also see  '
-                'https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies  '
-                'for tips on effectively exporting YouTube cookies')
+        return ('Use  --username=oauth[+PROFILE] --password=""  to log in using oauth, '
+                f'or else u{self._login_hint(method="cookies")[1:]}. '
+                'See  https://github.com/yt-dlp/yt-dlp/wiki/Extractors#logging-in-with-oauth  for more on how to use oauth. '
+                'See  https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies  for help with cookies')
 
     def _check_login_required(self):
         if self._LOGIN_REQUIRED and not self.is_authenticated:
@@ -734,7 +928,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     @functools.cached_property
     def is_authenticated(self):
-        return bool(self._generate_sapisidhash_header())
+        return self._OAUTH_PROFILE or bool(self._generate_sapisidhash_header())
 
     def extract_ytcfg(self, video_id, webpage):
         if not webpage:
@@ -744,6 +938,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
                 default='{}'), video_id, fatal=False) or {}
 
+    def _generate_oauth_headers(self):
+        self._update_oauth()
+        oauth_token = YoutubeBaseInfoExtractor._OAUTH_ACCESS_TOKEN_CACHE.get(self._OAUTH_PROFILE)
+        if not oauth_token:
+            return {}
+
+        return {
+            'Authorization': f'{oauth_token["token_type"]} {oauth_token["access_token"]}',
+        }
+
     def _generate_cookie_auth_headers(self, *, ytcfg=None, account_syncid=None, session_index=None, origin=None, **kwargs):
         headers = {}
         account_syncid = account_syncid or self._extract_account_syncid(ytcfg)
@@ -773,10 +977,14 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'Origin': origin,
             'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
             'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client),
+            **self._generate_oauth_headers(),
             **self._generate_cookie_auth_headers(ytcfg=ytcfg, account_syncid=account_syncid, session_index=session_index, origin=origin),
         }
         return filter_dict(headers)
 
+    def _generate_webpage_headers(self):
+        return self._generate_oauth_headers()
+
     def _download_ytcfg(self, client, video_id):
         url = {
             'web': 'https://www.youtube.com',
@@ -786,7 +994,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         if not url:
             return {}
         webpage = self._download_webpage(
-            url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config')
+            url, video_id, fatal=False, note=f'Downloading {client.replace("_", " ").strip()} client config',
+            headers=self._generate_webpage_headers())
         return self.extract_ytcfg(video_id, webpage) or {}
 
     @staticmethod
@@ -3051,7 +3260,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             code = self._download_webpage(
                 player_url, video_id, fatal=fatal,
                 note='Downloading player ' + player_id,
-                errnote=f'Download of {player_url} failed')
+                errnote=f'Download of {player_url} failed',
+                headers=self._generate_webpage_headers())
             if code:
                 self._code_cache[player_id] = code
         return self._code_cache.get(player_id)
@@ -3334,7 +3544,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
             self._download_webpage(
                 url, video_id, f'Marking {label}watched',
-                'Unable to mark watched', fatal=False)
+                'Unable to mark watched', fatal=False,
+                headers=self._generate_webpage_headers())
 
     @classmethod
     def _extract_from_webpage(cls, url, webpage):
@@ -3848,10 +4059,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if smuggled_data.get('is_music_url') or self.is_music_url(url):
             for requested_client in requested_clients:
                 _, base_client, variant = _split_innertube_client(requested_client)
-                music_client = f'{base_client}_music' if base_client != 'mweb' else 'web_music'
+                music_client = f'{base_client}_music'
                 if variant != 'music' and music_client in INNERTUBE_CLIENTS:
-                    if not INNERTUBE_CLIENTS[music_client]['REQUIRE_AUTH'] or self.is_authenticated:
-                        requested_clients.append(music_client)
+                    requested_clients.append(music_client)
 
         return orderedSet(requested_clients)
 
@@ -3964,10 +4174,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 self.to_screen(
                     f'{video_id}: This video is age-restricted and YouTube is requiring '
                     'account age-verification; some formats may be missing', only_once=True)
-                # web_creator can work around the age-verification requirement
-                # android_vr and mediaconnect may also be able to work around age-verification
+                # web_creator and mediaconnect can work around the age-verification requirement
+                # _testsuite & _vr variants can also work around age-verification
                 # tv_embedded may(?) still work around age-verification if the video is embeddable
-                append_client('web_creator')
+                append_client('web_creator', 'mediaconnect')
 
         prs.extend(deprioritized_prs)
 
@@ -4316,7 +4526,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if pp:
                 query['pp'] = pp
             webpage = self._download_webpage(
-                webpage_url, video_id, fatal=False, query=query)
+                webpage_url, video_id, fatal=False, query=query, headers=self._generate_webpage_headers())
 
         master_ytcfg = self.extract_ytcfg(video_id, webpage) or self._get_default_ytcfg()
 
@@ -4459,9 +4669,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     self.raise_geo_restricted(subreason, countries, metadata_available=True)
                 reason += f'. {subreason}'
             if reason:
-                if 'sign in' in reason.lower():
-                    reason = remove_end(reason, 'This helps protect our community. Learn more')
-                    reason = f'{remove_end(reason.strip(), ".")}. {self._youtube_login_hint}'
                 self.raise_no_formats(reason, expected=True)
 
         keywords = get_first(video_details, 'keywords', expected_type=list) or []
@@ -5607,7 +5814,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
         webpage, data = None, None
         for retry in self.RetryManager(fatal=fatal):
             try:
-                webpage = self._download_webpage(url, item_id, note='Downloading webpage')
+                webpage = self._download_webpage(url, item_id, note='Downloading webpage', headers=self._generate_webpage_headers())
                 data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
             except ExtractorError as e:
                 if isinstance(e.cause, network_exceptions):

yt_dlp/postprocessor/common.py

@@ -9,6 +9,7 @@ from ..utils import (
     RetryManager,
     _configuration_args,
     deprecation_warning,
+    encodeFilename,
 )
 
 
@@ -150,7 +151,7 @@ class PostProcessor(metaclass=PostProcessorMetaClass):
 
     def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
         try:
-            os.utime(path, (atime, mtime))
+            os.utime(encodeFilename(path), (atime, mtime))
         except Exception:
             self.report_warning(errnote)
 

yt_dlp/postprocessor/embedthumbnail.py

@@ -12,6 +12,7 @@ from ..utils import (
     PostProcessingError,
     check_executable,
     encodeArgument,
+    encodeFilename,
     prepend_extension,
     shell_quote,
 )
@@ -67,7 +68,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             self.to_screen('There are no thumbnails on disk')
             return [], info
         thumbnail_filename = info['thumbnails'][idx]['filepath']
-        if not os.path.exists(thumbnail_filename):
+        if not os.path.exists(encodeFilename(thumbnail_filename)):
             self.report_warning('Skipping embedding the thumbnail because the file is missing.')
             return [], info
 
@@ -84,7 +85,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             thumbnail_filename = convertor.convert_thumbnail(thumbnail_filename, 'png')
             thumbnail_ext = 'png'
 
-        mtime = os.stat(filename).st_mtime
+        mtime = os.stat(encodeFilename(filename)).st_mtime
 
         success = True
         if info['ext'] == 'mp3':
@@ -153,12 +154,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 else:
                     if not prefer_atomicparsley:
                         self.to_screen('mutagen was not found. Falling back to AtomicParsley')
-                    cmd = [atomicparsley,
-                           filename,
+                    cmd = [encodeFilename(atomicparsley, True),
+                           encodeFilename(filename, True),
                            encodeArgument('--artwork'),
-                           thumbnail_filename,
+                           encodeFilename(thumbnail_filename, True),
                            encodeArgument('-o'),
-                           temp_filename]
+                           encodeFilename(temp_filename, True)]
                     cmd += [encodeArgument(o) for o in self._configuration_args('AtomicParsley')]
 
                     self._report_run('atomicparsley', filename)

yt_dlp/postprocessor/ffmpeg.py

@@ -21,6 +21,7 @@ from ..utils import (
     determine_ext,
     dfxp2srt,
     encodeArgument,
+    encodeFilename,
     filter_dict,
     float_or_none,
     is_outdated_version,
@@ -242,13 +243,13 @@ class FFmpegPostProcessor(PostProcessor):
         try:
             if self.probe_available:
                 cmd = [
-                    self.probe_executable,
+                    encodeFilename(self.probe_executable, True),
                     encodeArgument('-show_streams')]
             else:
                 cmd = [
-                    self.executable,
+                    encodeFilename(self.executable, True),
                     encodeArgument('-i')]
-            cmd.append(self._ffmpeg_filename_argument(path))
+            cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True))
             self.write_debug(f'{self.basename} command line: {shell_quote(cmd)}')
             stdout, stderr, returncode = Popen.run(
                 cmd, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -281,7 +282,7 @@ class FFmpegPostProcessor(PostProcessor):
         self.check_version()
 
         cmd = [
-            self.probe_executable,
+            encodeFilename(self.probe_executable, True),
             encodeArgument('-hide_banner'),
             encodeArgument('-show_format'),
             encodeArgument('-show_streams'),
@@ -334,9 +335,9 @@ class FFmpegPostProcessor(PostProcessor):
         self.check_version()
 
         oldest_mtime = min(
-            os.stat(path).st_mtime for path, _ in input_path_opts if path)
+            os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts if path)
 
-        cmd = [self.executable, encodeArgument('-y')]
+        cmd = [encodeFilename(self.executable, True), encodeArgument('-y')]
         # avconv does not have repeat option
         if self.basename == 'ffmpeg':
             cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')]
@@ -352,7 +353,7 @@ class FFmpegPostProcessor(PostProcessor):
                 args.append('-i')
             return (
                 [encodeArgument(arg) for arg in args]
-                + [self._ffmpeg_filename_argument(file)])
+                + [encodeFilename(self._ffmpeg_filename_argument(file), True)])
 
         for arg_type, path_opts in (('i', input_path_opts), ('o', output_path_opts)):
             cmd += itertools.chain.from_iterable(
@@ -521,8 +522,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 return [], information
             orig_path = prepend_extension(path, 'orig')
             temp_path = prepend_extension(path, 'temp')
-        if (self._nopostoverwrites and os.path.exists(new_path)
-                and os.path.exists(orig_path)):
+        if (self._nopostoverwrites and os.path.exists(encodeFilename(new_path))
+                and os.path.exists(encodeFilename(orig_path))):
             self.to_screen(f'Post-process file {new_path} exists, skipping')
             return [], information
 
@@ -837,7 +838,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
                 args.extend(['-map', f'{i}:v:0'])
         self.to_screen(f'Merging formats into "{filename}"')
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args)
-        os.rename(temp_filename, filename)
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
     def can_merge(self):
@@ -1038,7 +1039,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor):
 
     def _ffmpeg_args_for_chapter(self, number, chapter, info):
         destination = self._prepare_filename(number, chapter, info)
-        if not self._downloader._ensure_dir_exists(destination):
+        if not self._downloader._ensure_dir_exists(encodeFilename(destination)):
             return
 
         chapter['filepath'] = destination

yt_dlp/postprocessor/movefilesafterdownload.py

@@ -4,6 +4,8 @@ from .common import PostProcessor
 from ..compat import shutil
 from ..utils import (
     PostProcessingError,
+    decodeFilename,
+    encodeFilename,
     make_dir,
 )
 
@@ -19,25 +21,25 @@ class MoveFilesAfterDownloadPP(PostProcessor):
         return 'MoveFiles'
 
     def run(self, info):
-        dl_path, dl_name = os.path.split(info['filepath'])
+        dl_path, dl_name = os.path.split(encodeFilename(info['filepath']))
         finaldir = info.get('__finaldir', dl_path)
         finalpath = os.path.join(finaldir, dl_name)
         if self._downloaded:
-            info['__files_to_move'][info['filepath']] = finalpath
+            info['__files_to_move'][info['filepath']] = decodeFilename(finalpath)
 
-        make_newfilename = lambda old: os.path.join(finaldir, os.path.basename(old))
+        make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old))))
         for oldfile, newfile in info['__files_to_move'].items():
             if not newfile:
                 newfile = make_newfilename(oldfile)
-            if os.path.abspath(oldfile) == os.path.abspath(newfile):
+            if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):
                 continue
-            if not os.path.exists(oldfile):
+            if not os.path.exists(encodeFilename(oldfile)):
                 self.report_warning(f'File "{oldfile}" cannot be found')
                 continue
-            if os.path.exists(newfile):
+            if os.path.exists(encodeFilename(newfile)):
                 if self.get_param('overwrites', True):
                     self.report_warning(f'Replacing existing file "{newfile}"')
-                    os.remove(newfile)
+                    os.remove(encodeFilename(newfile))
                 else:
                     self.report_warning(
                         f'Cannot move file "{oldfile}" out of temporary directory since "{newfile}" already exists. ')

yt_dlp/postprocessor/sponskrub.py

@@ -9,6 +9,7 @@ from ..utils import (
     check_executable,
     cli_option,
     encodeArgument,
+    encodeFilename,
     prepend_extension,
     shell_quote,
     str_or_none,
@@ -51,7 +52,7 @@ class SponSkrubPP(PostProcessor):
             return [], information
 
         filename = information['filepath']
-        if not os.path.exists(filename):  # no download
+        if not os.path.exists(encodeFilename(filename)):  # no download
             return [], information
 
         if information['extractor_key'].lower() != 'youtube':
@@ -70,8 +71,8 @@ class SponSkrubPP(PostProcessor):
                 self.report_warning('If sponskrub is run multiple times, unintended parts of the video could be cut out.')
 
         temp_filename = prepend_extension(filename, self._temp_ext)
-        if os.path.exists(temp_filename):
-            os.remove(temp_filename)
+        if os.path.exists(encodeFilename(temp_filename)):
+            os.remove(encodeFilename(temp_filename))
 
         cmd = [self.path]
         if not self.cutout:

yt_dlp/postprocessor/xattrpp.py

@@ -1,6 +1,7 @@
 import os
 
 from .common import PostProcessor
+from ..compat import compat_os_name
 from ..utils import (
     PostProcessingError,
     XAttrMetadataError,
@@ -56,7 +57,7 @@ class XAttrMetadataPP(PostProcessor):
                 elif e.reason == 'VALUE_TOO_LONG':
                     self.report_warning(f'Unable to write extended attribute "{xattrname}" due to too long values.')
                 else:
-                    tip = ('You need to use NTFS' if os.name == 'nt'
+                    tip = ('You need to use NTFS' if compat_os_name == 'nt'
                            else 'You may have to enable them in your "/etc/fstab"')
                     raise PostProcessingError(f'This filesystem doesn\'t support extended attributes. {tip}')
 

yt_dlp/update.py

@@ -13,6 +13,7 @@ import sys
 from dataclasses import dataclass
 from zipimport import zipimporter
 
+from .compat import compat_realpath
 from .networking import Request
 from .networking.exceptions import HTTPError, network_exceptions
 from .utils import (
@@ -200,6 +201,8 @@ class UpdateInfo:
     binary_name: str | None = _get_binary_name()  # noqa: RUF009: Always returns the same value
     checksum: str | None = None
 
+    _has_update = True
+
 
 class Updater:
     # XXX: use class variables to simplify testing
@@ -520,7 +523,7 @@ class Updater:
     @functools.cached_property
     def filename(self):
         """Filename of the executable"""
-        return os.path.realpath(_get_variant_and_executable_path()[1])
+        return compat_realpath(_get_variant_and_executable_path()[1])
 
     @functools.cached_property
     def cmd(self):
@@ -559,14 +562,62 @@ class Updater:
             f'Unable to {action}{delim} visit  '
             f'https://github.com/{self.requested_repo}/releases/{path}', True)
 
+    # XXX: Everything below this line in this class is deprecated / for compat only
+    @property
+    def _target_tag(self):
+        """Deprecated; requested tag with 'tags/' prepended when necessary for API calls"""
+        return f'tags/{self.requested_tag}' if self.requested_tag != 'latest' else self.requested_tag
+
+    def _check_update(self):
+        """Deprecated; report whether there is an update available"""
+        return bool(self.query_update(_output=True))
+
+    def __getattr__(self, attribute: str):
+        """Compat getter function for deprecated attributes"""
+        deprecated_props_map = {
+            'check_update': '_check_update',
+            'target_tag': '_target_tag',
+            'target_channel': 'requested_channel',
+        }
+        update_info_props_map = {
+            'has_update': '_has_update',
+            'new_version': 'version',
+            'latest_version': 'requested_version',
+            'release_name': 'binary_name',
+            'release_hash': 'checksum',
+        }
+
+        if attribute not in deprecated_props_map and attribute not in update_info_props_map:
+            raise AttributeError(f'{type(self).__name__!r} object has no attribute {attribute!r}')
+
+        msg = f'{type(self).__name__}.{attribute} is deprecated and will be removed in a future version'
+        if attribute in deprecated_props_map:
+            source_name = deprecated_props_map[attribute]
+            if not source_name.startswith('_'):
+                msg += f'. Please use {source_name!r} instead'
+            source = self
+            mapping = deprecated_props_map
+
+        else:  # attribute in update_info_props_map
+            msg += '. Please call query_update() instead'
+            source = self.query_update()
+            if source is None:
+                source = UpdateInfo('', None, None, None)
+                source._has_update = False
+            mapping = update_info_props_map
+
+        deprecation_warning(msg)
+        for target_name, source_name in mapping.items():
+            value = getattr(source, source_name)
+            setattr(self, target_name, value)
+
+        return getattr(self, attribute)
+
 
 def run_update(ydl):
     """Update the program file with the latest version from the repository
     @returns    Whether there was a successful update (No update = False)
     """
-    deprecation_warning(
-        '"yt_dlp.update.run_update(ydl)" is deprecated and may be removed in a future version. '
-        'Use "yt_dlp.update.Updater(ydl).update()" instead')
     return Updater(ydl).update()
 
 

yt_dlp/utils/_deprecated.py

@@ -9,23 +9,31 @@ passthrough_module(__name__, '.._legacy', callback=lambda attr: warnings.warn(
 del passthrough_module
 
 
-import re
-import struct
+from ._utils import preferredencoding
 
 
-def bytes_to_intlist(bs):
-    if not bs:
-        return []
-    if isinstance(bs[0], int):  # Python 3
-        return list(bs)
-    else:
-        return [ord(c) for c in bs]
+def encodeFilename(s, for_subprocess=False):
+    assert isinstance(s, str)
+    return s
 
 
-def intlist_to_bytes(xs):
-    if not xs:
-        return b''
-    return struct.pack('%dB' % len(xs), *xs)
+def decodeFilename(b, for_subprocess=False):
+    return b
 
 
-compiled_regex_type = type(re.compile(''))
+def decodeArgument(b):
+    return b
+
+
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, str)
+    return optval
+
+
+def error_to_compat_str(err):
+    return str(err)

yt_dlp/utils/_legacy.py

@@ -313,30 +313,3 @@ def make_HTTPS_handler(params, **kwargs):
 
 def process_communicate_or_kill(p, *args, **kwargs):
     return Popen.communicate_or_kill(p, *args, **kwargs)
-
-
-def encodeFilename(s, for_subprocess=False):
-    assert isinstance(s, str)
-    return s
-
-
-def decodeFilename(b, for_subprocess=False):
-    return b
-
-
-def decodeArgument(b):
-    return b
-
-
-def decodeOption(optval):
-    if optval is None:
-        return optval
-    if isinstance(optval, bytes):
-        optval = optval.decode(preferredencoding())
-
-    assert isinstance(optval, str)
-    return optval
-
-
-def error_to_compat_str(err):
-    return str(err)

yt_dlp/utils/_utils.py

@@ -49,11 +49,15 @@ from ..compat import (
     compat_etree_fromstring,
     compat_expanduser,
     compat_HTMLParseError,
+    compat_os_name,
 )
 from ..dependencies import xattr
 
 __name__ = __name__.rsplit('.', 1)[0]  # noqa: A001: Pretend to be the parent module
 
+# This is not clearly defined otherwise
+compiled_regex_type = type(re.compile(''))
+
 
 class NO_DEFAULT:
     pass
@@ -212,7 +216,7 @@ def partial_application(func):
     sig = inspect.signature(func)
     required_args = [
         param.name for param in sig.parameters.values()
-        if param.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+        if param.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
         if param.default is inspect.Parameter.empty
     ]
 
@@ -870,7 +874,7 @@ class Popen(subprocess.Popen):
             kwargs.setdefault('encoding', 'utf-8')
             kwargs.setdefault('errors', 'replace')
 
-        if shell and os.name == 'nt' and kwargs.get('executable') is None:
+        if shell and compat_os_name == 'nt' and kwargs.get('executable') is None:
             if not isinstance(args, str):
                 args = shell_quote(args, shell=True)
             shell = False
@@ -1453,7 +1457,7 @@ def system_identifier():
 @functools.cache
 def get_windows_version():
     """ Get Windows version. returns () if it's not running on Windows """
-    if os.name == 'nt':
+    if compat_os_name == 'nt':
         return version_tuple(platform.win32_ver()[1])
     else:
         return ()
@@ -1466,7 +1470,7 @@ def write_string(s, out=None, encoding=None):
     if not out:
         return
 
-    if os.name == 'nt' and supports_terminal_sequences(out):
+    if compat_os_name == 'nt' and supports_terminal_sequences(out):
         s = re.sub(r'([\r\n]+)', r' \1', s)
 
     enc, buffer = None, out
@@ -1499,6 +1503,21 @@ def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
 deprecation_warning._cache = set()
 
 
+def bytes_to_intlist(bs):
+    if not bs:
+        return []
+    if isinstance(bs[0], int):  # Python 3
+        return list(bs)
+    else:
+        return [ord(c) for c in bs]
+
+
+def intlist_to_bytes(xs):
+    if not xs:
+        return b''
+    return struct.pack('%dB' % len(xs), *xs)
+
+
 class LockingUnsupportedError(OSError):
     msg = 'File locking is not supported'
 
@@ -1682,7 +1701,7 @@ _CMD_QUOTE_TRANS = str.maketrans({
 def shell_quote(args, *, shell=False):
     args = list(variadic(args))
 
-    if os.name != 'nt':
+    if compat_os_name != 'nt':
         return shlex.join(args)
 
     trans = _CMD_QUOTE_TRANS if shell else _WINDOWS_QUOTE_TRANS
@@ -4497,7 +4516,7 @@ def urshift(val, n):
 def write_xattr(path, key, value):
     # Windows: Write xattrs to NTFS Alternate Data Streams:
     # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
-    if os.name == 'nt':
+    if compat_os_name == 'nt':
         assert ':' not in key
         assert os.path.exists(path)
 
@@ -4759,12 +4778,12 @@ def jwt_decode_hs256(jwt):
     return json.loads(base64.urlsafe_b64decode(f'{payload_b64}==='))
 
 
-WINDOWS_VT_MODE = False if os.name == 'nt' else None
+WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
 
 
 @functools.cache
 def supports_terminal_sequences(stream):
-    if os.name == 'nt':
+    if compat_os_name == 'nt':
         if not WINDOWS_VT_MODE:
             return False
     elif not os.getenv('TERM'):
@@ -4818,6 +4837,7 @@ def number_of_digits(number):
     return len('%d' % number)
 
 
+@partial_application
 def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
         values = (traversal.traverse_obj(from_dict, variadic(v)) for v in values)
@@ -4858,7 +4878,7 @@ def parse_http_range(range):
 
 def read_stdin(what):
     if what:
-        eof = 'Ctrl+Z' if os.name == 'nt' else 'Ctrl+D'
+        eof = 'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'
         write_string(f'Reading {what} from STDIN - EOF ({eof}) to end:\n')
     return sys.stdin
 

yt_dlp/utils/traversal.py

@@ -332,14 +332,14 @@ class _RequiredError(ExtractorError):
 
 
 @typing.overload
-def subs_list_to_dict(*, lang: str | None = 'und', ext: str | None = None) -> collections.abc.Callable[[list[dict]], dict[str, list[dict]]]: ...
+def subs_list_to_dict(*, ext: str | None = None) -> collections.abc.Callable[[list[dict]], dict[str, list[dict]]]: ...
 
 
 @typing.overload
-def subs_list_to_dict(subs: list[dict] | None, /, *, lang: str | None = 'und', ext: str | None = None) -> dict[str, list[dict]]: ...
+def subs_list_to_dict(subs: list[dict] | None, /, *, ext: str | None = None) -> dict[str, list[dict]]: ...
 
 
-def subs_list_to_dict(subs: list[dict] | None = None, /, *, lang='und', ext=None):
+def subs_list_to_dict(subs: list[dict] | None = None, /, *, ext=None):
     """
     Convert subtitles from a traversal into a subtitle dict.
     The path should have an `all` immediately before this function.
@@ -352,7 +352,7 @@ def subs_list_to_dict(subs: list[dict] | None = None, /, *, lang='und', ext=None
     `quality`  The sort order for each subtitle
     """
     if subs is None:
-        return functools.partial(subs_list_to_dict, lang=lang, ext=ext)
+        return functools.partial(subs_list_to_dict, ext=ext)
 
     result = collections.defaultdict(list)
 
@@ -360,16 +360,10 @@ def subs_list_to_dict(subs: list[dict] | None = None, /, *, lang='und', ext=None
         if not url_or_none(sub.get('url')) and not sub.get('data'):
             continue
         sub_id = sub.pop('id', None)
-        if not isinstance(sub_id, str):
-            if not lang:
-                continue
-            sub_id = lang
-        sub_ext = sub.get('ext')
-        if not isinstance(sub_ext, str):
-            if not ext:
-                sub.pop('ext', None)
-            else:
-                sub['ext'] = ext
+        if sub_id is None:
+            continue
+        if ext is not None and not sub.get('ext'):
+            sub['ext'] = ext
         result[sub_id].append(sub)
     result = dict(result)
 
@@ -458,9 +452,9 @@ def trim_str(*, start=None, end=None):
     return trim
 
 
-def unpack(func, **kwargs):
+def unpack(func):
     @functools.wraps(func)
-    def inner(items):
+    def inner(items, **kwargs):
         return func(*items, **kwargs)
 
     return inner