Compare commits

..

No commits in common. "5d4e5098289f1051738da58743e488de0132cacd" and "5e1200ac69b1bccb129099c37820f0ed0a2f538a" have entirely different histories.

7 changed files with 49 additions and 70 deletions

View File

@ -129,7 +129,7 @@ jobs:
brotli-python brotli-python
secretstorage secretstorage
EOF EOF
sed -E '/^(brotli|secretstorage|curl_cffi).*/d' requirements.txt >> "$reqs" sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
mamba create -n build --file "$reqs" mamba create -n build --file "$reqs"
- name: Prepare - name: Prepare
@ -144,6 +144,9 @@ jobs:
run: | run: |
unset LD_LIBRARY_PATH # Harmful; set by setup-python unset LD_LIBRARY_PATH # Harmful; set by setup-python
conda activate build conda activate build
python -m pip install -U pip setuptools wheel pip-tools
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
python -m pip install -U -r requirements-all.txt
python pyinst.py --onedir python pyinst.py --onedir
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
python pyinst.py python pyinst.py
@ -239,21 +242,17 @@ jobs:
# NB: Building universal2 does not work with python from actions/setup-python # NB: Building universal2 does not work with python from actions/setup-python
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils gnu-sed brew install coreutils
python3 -m pip install -U --user pip setuptools wheel delocate python3 -m pip install -U --user pip setuptools wheel delocate
# curl_cffi must be removed from reqs
gsed -i -E '/^curl_cffi.*/d' requirements.txt
# We need to ignore wheels otherwise we break universal2 builds # We need to ignore wheels otherwise we break universal2 builds
# PyInstaller v6 currently does not work with this python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
mkdir curl_cffi_whls curl_cffi_universal2 mkdir curl_cffi_whls curl_cffi_universal2
python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls
python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/curl_cffi* | paste -sd " " - ) -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/cffi-* | paste -sd " " - ) -w curl_cffi_universal2
cd curl_cffi_universal2 cd curl_cffi_universal2 && find curl_cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; && find cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \;
for file in *cffi-*x86_64*; do mv -n -- "${file}" "${file/x86_64/universal2}"; done python3 -m pip install -U --user $(find curl_cffi-*) $(find cffi-*) && cd ..
python3 -m pip install -U --user curl_cffi-* cffi-*
- name: Prepare - name: Prepare
run: | run: |
@ -304,8 +303,9 @@ jobs:
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils brew install coreutils
python3 -m pip install -U --user pip setuptools wheel python3 -m pip install -U --user pip setuptools wheel pip-tools
python3 -m pip install -U --user Pyinstaller -r requirements.txt python3 -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
python3 -m pip install -U --user Pyinstaller -r requirements-all.txt
- name: Prepare - name: Prepare
run: | run: |
@ -344,8 +344,9 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install Requirements - name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python -m pip install -U pip setuptools wheel py2exe python -m pip install -U pip setuptools wheel py2exe pip-tools
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements-all.txt
- name: Prepare - name: Prepare
run: | run: |
@ -394,8 +395,7 @@ jobs:
- name: Install Requirements - name: Install Requirements
run: | run: |
python -m pip install -U pip setuptools wheel python -m pip install -U pip setuptools wheel
Get-Content -Path requirements.txt | where {$_ -notlike 'curl_cffi*'} | Set-Content requirements_used.txt pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements_used.txt
- name: Prepare - name: Prepare
run: | run: |

View File

@ -29,7 +29,7 @@ from http.cookiejar import CookieJar
from test.conftest import validate_and_send from test.conftest import validate_and_send
from test.helper import FakeYDL, http_server_port from test.helper import FakeYDL, http_server_port
from yt_dlp.cookies import YoutubeDLCookieJar from yt_dlp.cookies import YoutubeDLCookieJar
from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3 from yt_dlp.dependencies import brotli, requests, urllib3, curl_cffi
from yt_dlp.networking import ( from yt_dlp.networking import (
HEADRequest, HEADRequest,
PUTRequest, PUTRequest,
@ -196,7 +196,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
self._headers() self._headers()
elif self.path.startswith('/308-to-headers'): elif self.path.startswith('/308-to-headers'):
self.send_response(308) self.send_response(308)
# redirect to "localhost" for testing cookie redirection handling # get server port
self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers') self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers')
self.send_header('Content-Length', '0') self.send_header('Content-Length', '0')
self.end_headers() self.end_headers()
@ -443,7 +443,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
def test_request_cookie_header(self, handler): def test_request_cookie_header(self, handler):
# We should accept a Cookie header being passed as in normal headers and handle it appropriately. # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
with handler() as rh: with handler(verbose=True) as rh:
# Specified Cookie header should be used # Specified Cookie header should be used
res = validate_and_send( res = validate_and_send(
rh, Request( rh, Request(
@ -942,9 +942,9 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
# but when not impersonating don't remove std_headers # but when not impersonating don't remove std_headers
res = validate_and_send( res = validate_and_send(
rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower() rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower()
# std_headers should be present assert std_headers['user-agent'].lower() in res
for k, v in std_headers.items(): assert std_headers['accept-language'].lower() in res
assert f'{k}: {v}'.lower() in res assert 'x-custom: test' in res
@pytest.mark.parametrize('raised,expected,match', [ @pytest.mark.parametrize('raised,expected,match', [
(lambda: curl_cffi.requests.errors.RequestsError( (lambda: curl_cffi.requests.errors.RequestsError(
@ -957,7 +957,6 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True) @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match): def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
import curl_cffi.requests import curl_cffi.requests
from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter
curl_res = curl_cffi.requests.Response() curl_res = curl_cffi.requests.Response()
res = CurlCFFIResponseAdapter(curl_res) res = CurlCFFIResponseAdapter(curl_res)
@ -1145,9 +1144,7 @@ class TestRequestHandlerValidation:
({'unsupported': 'value'}, UnsupportedRequest), ({'unsupported': 'value'}, UnsupportedRequest),
({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest), ({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest),
({'impersonate': 123}, AssertionError), ({'impersonate': 123}, AssertionError),
({'impersonate': ('chrome', None, None, None)}, False), ({'impersonate': ('chrome', None, None, None)}, False)
({'impersonate': (None, None, None, None)}, False),
({'impersonate': ()}, False)
]), ]),
(NoCheckRH, 'http', [ (NoCheckRH, 'http', [
({'cookiejar': 'notacookiejar'}, False), ({'cookiejar': 'notacookiejar'}, False),
@ -1496,25 +1493,6 @@ class TestYoutubeDLNetworking:
rh = self.build_handler(ydl, IRH) rh = self.build_handler(ydl, IRH)
assert rh.impersonate == ('firefox', None, None, None) assert rh.impersonate == ('firefox', None, None, None)
def test_get_impersonate_targets(self):
handlers = []
for target_client in ('firefox', 'chrome', 'edge'):
class TestRH(ImpersonateRequestHandler):
def _send(self, request: Request):
pass
_SUPPORTED_URL_SCHEMES = ('http',)
_SUPPORTED_IMPERSONATE_TARGET_TUPLES = [(target_client,)]
RH_KEY = target_client
RH_NAME = target_client
handlers.append(TestRH)
with FakeYDL() as ydl:
ydl._request_director = ydl.build_request_director(handlers)
assert set(ydl.get_impersonate_targets()) == {('firefox', 'firefox'), ('chrome', 'chrome'), ('edge', 'edge')}
assert ydl.impersonate_target_available(('firefox', ))
assert ydl.impersonate_target_available(())
assert not ydl.impersonate_target_available(('safari',))
@pytest.mark.parametrize('proxy_key,proxy_url,expected', [ @pytest.mark.parametrize('proxy_key,proxy_url,expected', [
('http', '__noproxy__', None), ('http', '__noproxy__', None),
('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'), ('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
@ -1821,10 +1799,7 @@ class TestImpersonate:
('firefox:::', ('firefox', None, None, None)), ('firefox:::', ('firefox', None, None, None)),
('firefox:120::5', ('firefox', '120', None, '5')), ('firefox:120::5', ('firefox', '120', None, '5')),
('firefox:120:', ('firefox', '120', None, None)), ('firefox:120:', ('firefox', '120', None, None)),
('::120', (None, None, '120', None)), ('::120', None)
(':', (None, None, None, None)),
(':::', (None, None, None, None)),
('', (None, None, None, None)),
]) ])
def test_parse_impersonate_target(self, target, expected): def test_parse_impersonate_target(self, target, expected):
assert parse_impersonate_target(target) == expected assert parse_impersonate_target(target) == expected
@ -1837,10 +1812,9 @@ class TestImpersonate:
(('firefox', None, 'linux', None), 'firefox::linux'), (('firefox', None, 'linux', None), 'firefox::linux'),
(('firefox', None, None, '5'), 'firefox:::5'), (('firefox', None, None, '5'), 'firefox:::5'),
(('firefox', '120', None, '5'), 'firefox:120::5'), (('firefox', '120', None, '5'), 'firefox:120::5'),
((None, '120', None, None), ':120'), ((None, '120', None, None), None),
(('firefox', ), 'firefox'), (('firefox', ), 'firefox'),
(('firefox', None, 'linux'), 'firefox::linux'), (('firefox', None, 'linux'), 'firefox::linux'),
((None, None, None, None), ''),
]) ])
def test_compile_impersonate_target(self, target_tuple, expected): def test_compile_impersonate_target(self, target_tuple, expected):
assert compile_impersonate_target(*target_tuple) == expected assert compile_impersonate_target(*target_tuple) == expected

View File

@ -714,9 +714,13 @@ class YoutubeDL:
self.deprecated_feature(msg) self.deprecated_feature(msg)
impersonate_target = self.params.get('impersonate') impersonate_target = self.params.get('impersonate')
if impersonate_target is not None: if impersonate_target:
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler # This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
if not self.impersonate_target_available(impersonate_target): results = self._request_director.collect_from_handlers(
lambda x: [x.is_supported_target(impersonate_target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
)
if not any(results):
raise ValueError( raise ValueError(
f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. ' f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. '
f'Use --list-impersonate-targets to see available targets.') f'Use --list-impersonate-targets to see available targets.')
@ -4055,11 +4059,6 @@ class YoutubeDL:
[lambda _, v: isinstance(v, ImpersonateRequestHandler)] [lambda _, v: isinstance(v, ImpersonateRequestHandler)]
), key=lambda x: x[0]) ), key=lambda x: x[0])
def impersonate_target_available(self, target):
return any(self._request_director.collect_from_handlers(
lambda x: [x.is_supported_target(target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]))
def urlopen(self, req): def urlopen(self, req):
""" Start an HTTP download """ """ Start an HTTP download """
if isinstance(req, str): if isinstance(req, str):

View File

@ -386,7 +386,7 @@ def validate_options(opts):
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}') f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
opts.cookiesfrombrowser = (browser_name, profile, keyring, container) opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
if opts.impersonate is not None: if opts.impersonate:
target = parse_impersonate_target(opts.impersonate) target = parse_impersonate_target(opts.impersonate)
if target is None: if target is None:
raise ValueError(f'invalid impersonate target "{opts.impersonate}"') raise ValueError(f'invalid impersonate target "{opts.impersonate}"')

View File

@ -8,11 +8,14 @@ from .exceptions import UnsupportedRequest
from ..compat.types import NoneType from ..compat.types import NoneType
from ..utils.networking import std_headers from ..utils.networking import std_headers
ImpersonateTarget = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] ImpersonateTarget = Tuple[str, Optional[str], Optional[str], Optional[str]]
def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget): def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget):
for i in range(0, min(len(target1), len(target2))): if target1[0] != target2[0]:
return False
for i in range(1, min(len(target1), len(target2))):
if ( if (
target1[i] target1[i]
and target2[i] and target2[i]
@ -72,13 +75,13 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _resolve_target(self, target: ImpersonateTarget | None): def _resolve_target(self, target: ImpersonateTarget | None):
"""Resolve a target to a supported target.""" """Resolve a target to a supported target."""
if target is None: if not target:
return return
for supported_target in self.get_supported_targets(): for supported_target in self.get_supported_targets():
if _target_within(target, supported_target): if _target_within(target, supported_target):
if self.verbose: if self.verbose:
self._logger.stdout( self._logger.stdout(
f'{self.RH_NAME}: resolved impersonate target {target} to {supported_target}') f'{self.RH_NAME}: resolved impersonate target "{target}" to "{supported_target}"')
return supported_target return supported_target
def get_supported_targets(self) -> tuple[ImpersonateTarget]: def get_supported_targets(self) -> tuple[ImpersonateTarget]:
@ -103,7 +106,7 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _get_impersonate_headers(self, request): def _get_impersonate_headers(self, request):
headers = self._merge_headers(request.headers) headers = self._merge_headers(request.headers)
if self._get_request_target(request) is not None: if self._get_request_target(request):
# remove all headers present in std_headers # remove all headers present in std_headers
headers.pop('User-Agent', None) headers.pop('User-Agent', None)
for header in std_headers: for header in std_headers:
@ -114,6 +117,6 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
@register_preference(ImpersonateRequestHandler) @register_preference(ImpersonateRequestHandler)
def impersonate_preference(rh, request): def impersonate_preference(rh, request):
if request.extensions.get('impersonate') is not None or rh.impersonate is not None: if request.extensions.get('impersonate') or rh.impersonate:
return 1000 return 1000
return 0 return 0

View File

@ -513,8 +513,8 @@ def create_parser():
) )
network.add_option( network.add_option(
'--impersonate', '--impersonate',
metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None, metavar='CLIENT[:[VERSION][:[OS][:OS_VERSION]]]', dest='impersonate', default=None,
help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client', help='Client to impersonate for requests',
) )
network.add_option( network.add_option(
'--list-impersonate-targets', '--list-impersonate-targets',

View File

@ -167,7 +167,7 @@ def normalize_url(url):
).geturl() ).geturl()
def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] | None: def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]] | None:
""" """
Parse an impersonate target string into a tuple of (client, version, os, os_vers) Parse an impersonate target string into a tuple of (client, version, os, os_vers)
If the target is invalid, return None If the target is invalid, return None
@ -175,10 +175,13 @@ def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str],
client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in ( client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in (
target.split(':') + [None, None, None, None])][:4] target.split(':') + [None, None, None, None])][:4]
if client is not None:
return client, version, os, os_vers return client, version, os, os_vers
def compile_impersonate_target(*args) -> str | None: def compile_impersonate_target(*args) -> str | None:
client, version, os, os_vers = (list(args) + [None, None, None, None])[:4] client, version, os, os_vers = (list(args) + [None, None, None, None])[:4]
if not client:
return
filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)] filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)]
return ':'.join(filtered_parts).rstrip(':') return ':'.join(filtered_parts).rstrip(':')