linter

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>

.github/workflows/build.yml

@@ -129,7 +129,7 @@ jobs:
           brotli-python
           secretstorage
           EOF
-          sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
+          sed -E '/^(brotli|secretstorage|curl_cffi).*/d' requirements.txt >> "$reqs"
           mamba create -n build --file "$reqs"
 
       - name: Prepare
@@ -144,9 +144,6 @@ jobs:
         run: |
           unset LD_LIBRARY_PATH  # Harmful; set by setup-python
           conda activate build
-          python -m pip install -U pip setuptools wheel pip-tools
-          python -m piptools compile  -o requirements-all.txt --extra curl_cffi setup.py
-          python -m pip install -U -r requirements-all.txt
           python pyinst.py --onedir
           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
           python pyinst.py
@@ -242,17 +239,21 @@ jobs:
       # NB: Building universal2 does not work with python from actions/setup-python
       - name: Install Requirements
         run: |
-          brew install coreutils
+          brew install coreutils gnu-sed
           python3 -m pip install -U --user pip setuptools wheel delocate
+          # curl_cffi must be removed from reqs
+          gsed -i -E '/^curl_cffi.*/d' requirements.txt
           # We need to ignore wheels otherwise we break universal2 builds
-          python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
+          # PyInstaller v6 currently does not work with this
+          python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
           mkdir curl_cffi_whls curl_cffi_universal2
           python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls
           python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls
-          python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/curl_cffi* | paste -sd " " - ) -w curl_cffi_universal2
-          python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/cffi-* | paste -sd " " - ) -w curl_cffi_universal2
-          cd curl_cffi_universal2 && find curl_cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; && find cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \;
-          python3 -m pip install -U --user $(find curl_cffi-*) $(find cffi-*) && cd ..
+          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2
+          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2
+          cd curl_cffi_universal2
+          for file in *cffi-*x86_64*; do mv -n -- "${file}" "${file/x86_64/universal2}"; done
+          python3 -m pip install -U --user curl_cffi-* cffi-*
 
       - name: Prepare
         run: |
@@ -303,9 +304,8 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip setuptools wheel pip-tools
-          python3 -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
-          python3 -m pip install -U --user Pyinstaller -r requirements-all.txt
+          python3 -m pip install -U --user pip setuptools wheel
+          python3 -m pip install -U --user Pyinstaller -r requirements.txt
 
       - name: Prepare
         run: |
@@ -344,9 +344,8 @@ jobs:
           python-version: "3.8"
       - name: Install Requirements
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
-          python -m pip install -U pip setuptools wheel py2exe pip-tools
-          python -m piptools compile  -o requirements-all.txt --extra curl_cffi setup.py
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements-all.txt
+          python -m pip install -U pip setuptools wheel py2exe
+          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
 
       - name: Prepare
         run: |
@@ -395,7 +394,8 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install -U pip setuptools wheel
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+          Get-Content -Path requirements.txt | where {$_ -notlike 'curl_cffi*'} | Set-Content requirements_used.txt
+          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements_used.txt
 
       - name: Prepare
         run: |

test/test_networking.py

@@ -29,7 +29,7 @@ from http.cookiejar import CookieJar
 from test.conftest import validate_and_send
 from test.helper import FakeYDL, http_server_port
 from yt_dlp.cookies import YoutubeDLCookieJar
-from yt_dlp.dependencies import brotli, requests, urllib3, curl_cffi
+from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3
 from yt_dlp.networking import (
     HEADRequest,
     PUTRequest,
@@ -196,7 +196,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
             self._headers()
         elif self.path.startswith('/308-to-headers'):
             self.send_response(308)
-            # get server port
+            # redirect to "localhost" for testing cookie redirection handling
             self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers')
             self.send_header('Content-Length', '0')
             self.end_headers()
@@ -443,7 +443,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
     @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
     def test_request_cookie_header(self, handler):
         # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
-        with handler(verbose=True) as rh:
+        with handler() as rh:
             # Specified Cookie header should be used
             res = validate_and_send(
                 rh, Request(
@@ -942,9 +942,9 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
             # but when not impersonating don't remove std_headers
             res = validate_and_send(
                 rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower()
-            assert std_headers['user-agent'].lower() in res
-            assert std_headers['accept-language'].lower() in res
-            assert 'x-custom: test' in res
+            # std_headers should be present
+            for k, v in std_headers.items():
+                assert f'{k}: {v}'.lower() in res
 
     @pytest.mark.parametrize('raised,expected,match', [
         (lambda: curl_cffi.requests.errors.RequestsError(
@@ -957,6 +957,7 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
     @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
     def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
         import curl_cffi.requests
+
         from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter
         curl_res = curl_cffi.requests.Response()
         res = CurlCFFIResponseAdapter(curl_res)
@@ -1144,7 +1145,9 @@ class TestRequestHandlerValidation:
             ({'unsupported': 'value'}, UnsupportedRequest),
             ({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest),
             ({'impersonate': 123}, AssertionError),
-            ({'impersonate': ('chrome', None, None, None)}, False)
+            ({'impersonate': ('chrome', None, None, None)}, False),
+            ({'impersonate': (None, None, None, None)}, False),
+            ({'impersonate': ()}, False)
         ]),
         (NoCheckRH, 'http', [
             ({'cookiejar': 'notacookiejar'}, False),
@@ -1493,6 +1496,25 @@ class TestYoutubeDLNetworking:
             rh = self.build_handler(ydl, IRH)
             assert rh.impersonate == ('firefox', None, None, None)
 
+    def test_get_impersonate_targets(self):
+        handlers = []
+        for target_client in ('firefox', 'chrome', 'edge'):
+            class TestRH(ImpersonateRequestHandler):
+                def _send(self, request: Request):
+                    pass
+                _SUPPORTED_URL_SCHEMES = ('http',)
+                _SUPPORTED_IMPERSONATE_TARGET_TUPLES = [(target_client,)]
+                RH_KEY = target_client
+                RH_NAME = target_client
+            handlers.append(TestRH)
+
+        with FakeYDL() as ydl:
+            ydl._request_director = ydl.build_request_director(handlers)
+            assert set(ydl.get_impersonate_targets()) == {('firefox', 'firefox'), ('chrome', 'chrome'), ('edge', 'edge')}
+            assert ydl.impersonate_target_available(('firefox', ))
+            assert ydl.impersonate_target_available(())
+            assert not ydl.impersonate_target_available(('safari',))
+
     @pytest.mark.parametrize('proxy_key,proxy_url,expected', [
         ('http', '__noproxy__', None),
         ('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
@@ -1799,7 +1821,10 @@ class TestImpersonate:
         ('firefox:::', ('firefox', None, None, None)),
         ('firefox:120::5', ('firefox', '120', None, '5')),
         ('firefox:120:', ('firefox', '120', None, None)),
-        ('::120', None)
+        ('::120', (None, None, '120', None)),
+        (':', (None, None, None, None)),
+        (':::', (None, None, None, None)),
+        ('', (None, None, None, None)),
     ])
     def test_parse_impersonate_target(self, target, expected):
         assert parse_impersonate_target(target) == expected
@@ -1812,9 +1837,10 @@ class TestImpersonate:
         (('firefox', None, 'linux', None), 'firefox::linux'),
         (('firefox', None, None, '5'), 'firefox:::5'),
         (('firefox', '120', None, '5'), 'firefox:120::5'),
-        ((None, '120', None, None), None),
+        ((None, '120', None, None), ':120'),
         (('firefox', ), 'firefox'),
         (('firefox', None, 'linux'), 'firefox::linux'),
+        ((None, None, None, None), ''),
     ])
     def test_compile_impersonate_target(self, target_tuple, expected):
         assert compile_impersonate_target(*target_tuple) == expected

yt_dlp/YoutubeDL.py

@@ -714,13 +714,9 @@ class YoutubeDL:
             self.deprecated_feature(msg)
 
         impersonate_target = self.params.get('impersonate')
-        if impersonate_target:
+        if impersonate_target is not None:
             # This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
-            results = self._request_director.collect_from_handlers(
-                lambda x: [x.is_supported_target(impersonate_target)],
-                [lambda _, v: isinstance(v, ImpersonateRequestHandler)]
-            )
-            if not any(results):
+            if not self.impersonate_target_available(impersonate_target):
                 raise ValueError(
                     f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. '
                     f'Use --list-impersonate-targets to see available targets.')
@@ -4059,6 +4055,11 @@ class YoutubeDL:
             [lambda _, v: isinstance(v, ImpersonateRequestHandler)]
         ), key=lambda x: x[0])
 
+    def impersonate_target_available(self, target):
+        return any(self._request_director.collect_from_handlers(
+            lambda x: [x.is_supported_target(target)],
+            [lambda _, v: isinstance(v, ImpersonateRequestHandler)]))
+
     def urlopen(self, req):
         """ Start an HTTP download """
         if isinstance(req, str):

yt_dlp/__init__.py

@@ -386,7 +386,7 @@ def validate_options(opts):
                                  f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
         opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
 
-    if opts.impersonate:
+    if opts.impersonate is not None:
         target = parse_impersonate_target(opts.impersonate)
         if target is None:
             raise ValueError(f'invalid impersonate target "{opts.impersonate}"')

yt_dlp/networking/impersonate.py

@@ -8,14 +8,11 @@ from .exceptions import UnsupportedRequest
 from ..compat.types import NoneType
 from ..utils.networking import std_headers
 
-ImpersonateTarget = Tuple[str, Optional[str], Optional[str], Optional[str]]
+ImpersonateTarget = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]
 
 
 def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget):
-    if target1[0] != target2[0]:
-        return False
-
-    for i in range(1, min(len(target1), len(target2))):
+    for i in range(0, min(len(target1), len(target2))):
         if (
             target1[i]
             and target2[i]
@@ -75,13 +72,13 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
 
     def _resolve_target(self, target: ImpersonateTarget | None):
         """Resolve a target to a supported target."""
-        if not target:
+        if target is None:
             return
         for supported_target in self.get_supported_targets():
             if _target_within(target, supported_target):
                 if self.verbose:
                     self._logger.stdout(
-                        f'{self.RH_NAME}: resolved impersonate target "{target}" to "{supported_target}"')
+                        f'{self.RH_NAME}: resolved impersonate target {target} to {supported_target}')
                 return supported_target
 
     def get_supported_targets(self) -> tuple[ImpersonateTarget]:
@@ -106,7 +103,7 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
 
     def _get_impersonate_headers(self, request):
         headers = self._merge_headers(request.headers)
-        if self._get_request_target(request):
+        if self._get_request_target(request) is not None:
             # remove all headers present in std_headers
             headers.pop('User-Agent', None)
             for header in std_headers:
@@ -117,6 +114,6 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
 
 @register_preference(ImpersonateRequestHandler)
 def impersonate_preference(rh, request):
-    if request.extensions.get('impersonate') or rh.impersonate:
+    if request.extensions.get('impersonate') is not None or rh.impersonate is not None:
         return 1000
     return 0

yt_dlp/options.py

@@ -513,8 +513,8 @@ def create_parser():
     )
     network.add_option(
         '--impersonate',
-        metavar='CLIENT[:[VERSION][:[OS][:OS_VERSION]]]', dest='impersonate', default=None,
-        help='Client to impersonate for requests',
+        metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None,
+        help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client',
     )
     network.add_option(
         '--list-impersonate-targets',

yt_dlp/utils/networking.py

@@ -167,7 +167,7 @@ def normalize_url(url):
     ).geturl()
 
 
-def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]] | None:
+def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] | None:
     """
     Parse an impersonate target string into a tuple of (client, version, os, os_vers)
     If the target is invalid, return None
@@ -175,13 +175,10 @@ def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[
     client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in (
         target.split(':') + [None, None, None, None])][:4]
 
-    if client is not None:
     return client, version, os, os_vers
 
 
 def compile_impersonate_target(*args) -> str | None:
     client, version, os, os_vers = (list(args) + [None, None, None, None])[:4]
-    if not client:
-        return
     filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)]
     return ':'.join(filtered_parts).rstrip(':')