Compare commits

...

17 Commits

Author SHA1 Message Date
coletdjnz
5d4e509828
linter 2023-12-10 14:07:00 +13:00
coletdjnz
4d5a63311a
add test for ydl helper funcs 2023-12-10 14:04:37 +13:00
coletdjnz
aac33fd194
Add impersonate_target_available 2023-12-10 13:51:23 +13:00
coletdjnz
234903f3fa
Accept empty client 2023-12-10 13:47:50 +13:00
coletdjnz
bdf6273880
update doc 2023-12-10 12:23:11 +13:00
coletdjnz
c618a31fdb
revert back to 3.12 2023-12-10 12:00:50 +13:00
coletdjnz
0705da041f
try pyinstaller v5 2023-12-10 11:49:38 +13:00
coletdjnz
b9983da0b0
try python 3.11 2023-12-10 11:34:58 +13:00
coletdjnz
75fb3011ec
notlike? 2023-12-10 10:39:04 +13:00
coletdjnz
5005f4493f
POWERshell 2023-12-10 10:32:15 +13:00
coletdjnz
a210633178
exclude curl_cffi from windows32 build 2023-12-10 10:29:09 +13:00
coletdjnz
98649602d3
revert nothing like debugging in the CI 2023-12-10 10:26:45 +13:00
coletdjnz
e77b8ce51c
nothing like debugging in the CI 2023-12-10 09:58:24 +13:00
coletdjnz
4b17fe1fa1
update builds 2023-12-10 09:45:40 +13:00
coletdjnz
3d351301e0
Update .github/workflows/build.yml
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-12-09 20:41:58 +00:00
coletdjnz
5abbea2f7f
Update .github/workflows/build.yml
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-12-09 20:41:51 +00:00
coletdjnz
c86adc7c97
test cleanup 2023-12-10 09:41:04 +13:00
7 changed files with 70 additions and 49 deletions

View File

@ -129,7 +129,7 @@ jobs:
brotli-python brotli-python
secretstorage secretstorage
EOF EOF
sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs" sed -E '/^(brotli|secretstorage|curl_cffi).*/d' requirements.txt >> "$reqs"
mamba create -n build --file "$reqs" mamba create -n build --file "$reqs"
- name: Prepare - name: Prepare
@ -144,9 +144,6 @@ jobs:
run: | run: |
unset LD_LIBRARY_PATH # Harmful; set by setup-python unset LD_LIBRARY_PATH # Harmful; set by setup-python
conda activate build conda activate build
python -m pip install -U pip setuptools wheel pip-tools
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
python -m pip install -U -r requirements-all.txt
python pyinst.py --onedir python pyinst.py --onedir
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
python pyinst.py python pyinst.py
@ -242,17 +239,21 @@ jobs:
# NB: Building universal2 does not work with python from actions/setup-python # NB: Building universal2 does not work with python from actions/setup-python
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils brew install coreutils gnu-sed
python3 -m pip install -U --user pip setuptools wheel delocate python3 -m pip install -U --user pip setuptools wheel delocate
# curl_cffi must be removed from reqs
gsed -i -E '/^curl_cffi.*/d' requirements.txt
# We need to ignore wheels otherwise we break universal2 builds # We need to ignore wheels otherwise we break universal2 builds
python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt # PyInstaller v6 currently does not work with this
python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
mkdir curl_cffi_whls curl_cffi_universal2 mkdir curl_cffi_whls curl_cffi_universal2
python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls
python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/curl_cffi* | paste -sd " " - ) -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/cffi-* | paste -sd " " - ) -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2
cd curl_cffi_universal2 && find curl_cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; && find cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; cd curl_cffi_universal2
python3 -m pip install -U --user $(find curl_cffi-*) $(find cffi-*) && cd .. for file in *cffi-*x86_64*; do mv -n -- "${file}" "${file/x86_64/universal2}"; done
python3 -m pip install -U --user curl_cffi-* cffi-*
- name: Prepare - name: Prepare
run: | run: |
@ -303,9 +304,8 @@ jobs:
- name: Install Requirements - name: Install Requirements
run: | run: |
brew install coreutils brew install coreutils
python3 -m pip install -U --user pip setuptools wheel pip-tools python3 -m pip install -U --user pip setuptools wheel
python3 -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py python3 -m pip install -U --user Pyinstaller -r requirements.txt
python3 -m pip install -U --user Pyinstaller -r requirements-all.txt
- name: Prepare - name: Prepare
run: | run: |
@ -344,9 +344,8 @@ jobs:
python-version: "3.8" python-version: "3.8"
- name: Install Requirements - name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python -m pip install -U pip setuptools wheel py2exe pip-tools python -m pip install -U pip setuptools wheel py2exe
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements-all.txt
- name: Prepare - name: Prepare
run: | run: |
@ -395,7 +394,8 @@ jobs:
- name: Install Requirements - name: Install Requirements
run: | run: |
python -m pip install -U pip setuptools wheel python -m pip install -U pip setuptools wheel
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt Get-Content -Path requirements.txt | where {$_ -notlike 'curl_cffi*'} | Set-Content requirements_used.txt
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements_used.txt
- name: Prepare - name: Prepare
run: | run: |

View File

@ -29,7 +29,7 @@ from http.cookiejar import CookieJar
from test.conftest import validate_and_send from test.conftest import validate_and_send
from test.helper import FakeYDL, http_server_port from test.helper import FakeYDL, http_server_port
from yt_dlp.cookies import YoutubeDLCookieJar from yt_dlp.cookies import YoutubeDLCookieJar
from yt_dlp.dependencies import brotli, requests, urllib3, curl_cffi from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3
from yt_dlp.networking import ( from yt_dlp.networking import (
HEADRequest, HEADRequest,
PUTRequest, PUTRequest,
@ -196,7 +196,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
self._headers() self._headers()
elif self.path.startswith('/308-to-headers'): elif self.path.startswith('/308-to-headers'):
self.send_response(308) self.send_response(308)
# get server port # redirect to "localhost" for testing cookie redirection handling
self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers') self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers')
self.send_header('Content-Length', '0') self.send_header('Content-Length', '0')
self.end_headers() self.end_headers()
@ -443,7 +443,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True) @pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
def test_request_cookie_header(self, handler): def test_request_cookie_header(self, handler):
# We should accept a Cookie header being passed as in normal headers and handle it appropriately. # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
with handler(verbose=True) as rh: with handler() as rh:
# Specified Cookie header should be used # Specified Cookie header should be used
res = validate_and_send( res = validate_and_send(
rh, Request( rh, Request(
@ -942,9 +942,9 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
# but when not impersonating don't remove std_headers # but when not impersonating don't remove std_headers
res = validate_and_send( res = validate_and_send(
rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower() rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower()
assert std_headers['user-agent'].lower() in res # std_headers should be present
assert std_headers['accept-language'].lower() in res for k, v in std_headers.items():
assert 'x-custom: test' in res assert f'{k}: {v}'.lower() in res
@pytest.mark.parametrize('raised,expected,match', [ @pytest.mark.parametrize('raised,expected,match', [
(lambda: curl_cffi.requests.errors.RequestsError( (lambda: curl_cffi.requests.errors.RequestsError(
@ -957,6 +957,7 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True) @pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match): def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
import curl_cffi.requests import curl_cffi.requests
from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter
curl_res = curl_cffi.requests.Response() curl_res = curl_cffi.requests.Response()
res = CurlCFFIResponseAdapter(curl_res) res = CurlCFFIResponseAdapter(curl_res)
@ -1144,7 +1145,9 @@ class TestRequestHandlerValidation:
({'unsupported': 'value'}, UnsupportedRequest), ({'unsupported': 'value'}, UnsupportedRequest),
({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest), ({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest),
({'impersonate': 123}, AssertionError), ({'impersonate': 123}, AssertionError),
({'impersonate': ('chrome', None, None, None)}, False) ({'impersonate': ('chrome', None, None, None)}, False),
({'impersonate': (None, None, None, None)}, False),
({'impersonate': ()}, False)
]), ]),
(NoCheckRH, 'http', [ (NoCheckRH, 'http', [
({'cookiejar': 'notacookiejar'}, False), ({'cookiejar': 'notacookiejar'}, False),
@ -1493,6 +1496,25 @@ class TestYoutubeDLNetworking:
rh = self.build_handler(ydl, IRH) rh = self.build_handler(ydl, IRH)
assert rh.impersonate == ('firefox', None, None, None) assert rh.impersonate == ('firefox', None, None, None)
def test_get_impersonate_targets(self):
handlers = []
for target_client in ('firefox', 'chrome', 'edge'):
class TestRH(ImpersonateRequestHandler):
def _send(self, request: Request):
pass
_SUPPORTED_URL_SCHEMES = ('http',)
_SUPPORTED_IMPERSONATE_TARGET_TUPLES = [(target_client,)]
RH_KEY = target_client
RH_NAME = target_client
handlers.append(TestRH)
with FakeYDL() as ydl:
ydl._request_director = ydl.build_request_director(handlers)
assert set(ydl.get_impersonate_targets()) == {('firefox', 'firefox'), ('chrome', 'chrome'), ('edge', 'edge')}
assert ydl.impersonate_target_available(('firefox', ))
assert ydl.impersonate_target_available(())
assert not ydl.impersonate_target_available(('safari',))
@pytest.mark.parametrize('proxy_key,proxy_url,expected', [ @pytest.mark.parametrize('proxy_key,proxy_url,expected', [
('http', '__noproxy__', None), ('http', '__noproxy__', None),
('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'), ('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
@ -1799,7 +1821,10 @@ class TestImpersonate:
('firefox:::', ('firefox', None, None, None)), ('firefox:::', ('firefox', None, None, None)),
('firefox:120::5', ('firefox', '120', None, '5')), ('firefox:120::5', ('firefox', '120', None, '5')),
('firefox:120:', ('firefox', '120', None, None)), ('firefox:120:', ('firefox', '120', None, None)),
('::120', None) ('::120', (None, None, '120', None)),
(':', (None, None, None, None)),
(':::', (None, None, None, None)),
('', (None, None, None, None)),
]) ])
def test_parse_impersonate_target(self, target, expected): def test_parse_impersonate_target(self, target, expected):
assert parse_impersonate_target(target) == expected assert parse_impersonate_target(target) == expected
@ -1812,9 +1837,10 @@ class TestImpersonate:
(('firefox', None, 'linux', None), 'firefox::linux'), (('firefox', None, 'linux', None), 'firefox::linux'),
(('firefox', None, None, '5'), 'firefox:::5'), (('firefox', None, None, '5'), 'firefox:::5'),
(('firefox', '120', None, '5'), 'firefox:120::5'), (('firefox', '120', None, '5'), 'firefox:120::5'),
((None, '120', None, None), None), ((None, '120', None, None), ':120'),
(('firefox', ), 'firefox'), (('firefox', ), 'firefox'),
(('firefox', None, 'linux'), 'firefox::linux'), (('firefox', None, 'linux'), 'firefox::linux'),
((None, None, None, None), ''),
]) ])
def test_compile_impersonate_target(self, target_tuple, expected): def test_compile_impersonate_target(self, target_tuple, expected):
assert compile_impersonate_target(*target_tuple) == expected assert compile_impersonate_target(*target_tuple) == expected

View File

@ -714,13 +714,9 @@ class YoutubeDL:
self.deprecated_feature(msg) self.deprecated_feature(msg)
impersonate_target = self.params.get('impersonate') impersonate_target = self.params.get('impersonate')
if impersonate_target: if impersonate_target is not None:
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler # This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
results = self._request_director.collect_from_handlers( if not self.impersonate_target_available(impersonate_target):
lambda x: [x.is_supported_target(impersonate_target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
)
if not any(results):
raise ValueError( raise ValueError(
f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. ' f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. '
f'Use --list-impersonate-targets to see available targets.') f'Use --list-impersonate-targets to see available targets.')
@ -4059,6 +4055,11 @@ class YoutubeDL:
[lambda _, v: isinstance(v, ImpersonateRequestHandler)] [lambda _, v: isinstance(v, ImpersonateRequestHandler)]
), key=lambda x: x[0]) ), key=lambda x: x[0])
def impersonate_target_available(self, target):
return any(self._request_director.collect_from_handlers(
lambda x: [x.is_supported_target(target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]))
def urlopen(self, req): def urlopen(self, req):
""" Start an HTTP download """ """ Start an HTTP download """
if isinstance(req, str): if isinstance(req, str):

View File

@ -386,7 +386,7 @@ def validate_options(opts):
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}') f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
opts.cookiesfrombrowser = (browser_name, profile, keyring, container) opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
if opts.impersonate: if opts.impersonate is not None:
target = parse_impersonate_target(opts.impersonate) target = parse_impersonate_target(opts.impersonate)
if target is None: if target is None:
raise ValueError(f'invalid impersonate target "{opts.impersonate}"') raise ValueError(f'invalid impersonate target "{opts.impersonate}"')

View File

@ -8,14 +8,11 @@ from .exceptions import UnsupportedRequest
from ..compat.types import NoneType from ..compat.types import NoneType
from ..utils.networking import std_headers from ..utils.networking import std_headers
ImpersonateTarget = Tuple[str, Optional[str], Optional[str], Optional[str]] ImpersonateTarget = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]
def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget): def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget):
if target1[0] != target2[0]: for i in range(0, min(len(target1), len(target2))):
return False
for i in range(1, min(len(target1), len(target2))):
if ( if (
target1[i] target1[i]
and target2[i] and target2[i]
@ -75,13 +72,13 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _resolve_target(self, target: ImpersonateTarget | None): def _resolve_target(self, target: ImpersonateTarget | None):
"""Resolve a target to a supported target.""" """Resolve a target to a supported target."""
if not target: if target is None:
return return
for supported_target in self.get_supported_targets(): for supported_target in self.get_supported_targets():
if _target_within(target, supported_target): if _target_within(target, supported_target):
if self.verbose: if self.verbose:
self._logger.stdout( self._logger.stdout(
f'{self.RH_NAME}: resolved impersonate target "{target}" to "{supported_target}"') f'{self.RH_NAME}: resolved impersonate target {target} to {supported_target}')
return supported_target return supported_target
def get_supported_targets(self) -> tuple[ImpersonateTarget]: def get_supported_targets(self) -> tuple[ImpersonateTarget]:
@ -106,7 +103,7 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _get_impersonate_headers(self, request): def _get_impersonate_headers(self, request):
headers = self._merge_headers(request.headers) headers = self._merge_headers(request.headers)
if self._get_request_target(request): if self._get_request_target(request) is not None:
# remove all headers present in std_headers # remove all headers present in std_headers
headers.pop('User-Agent', None) headers.pop('User-Agent', None)
for header in std_headers: for header in std_headers:
@ -117,6 +114,6 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
@register_preference(ImpersonateRequestHandler) @register_preference(ImpersonateRequestHandler)
def impersonate_preference(rh, request): def impersonate_preference(rh, request):
if request.extensions.get('impersonate') or rh.impersonate: if request.extensions.get('impersonate') is not None or rh.impersonate is not None:
return 1000 return 1000
return 0 return 0

View File

@ -513,8 +513,8 @@ def create_parser():
) )
network.add_option( network.add_option(
'--impersonate', '--impersonate',
metavar='CLIENT[:[VERSION][:[OS][:OS_VERSION]]]', dest='impersonate', default=None, metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None,
help='Client to impersonate for requests', help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client',
) )
network.add_option( network.add_option(
'--list-impersonate-targets', '--list-impersonate-targets',

View File

@ -167,7 +167,7 @@ def normalize_url(url):
).geturl() ).geturl()
def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]] | None: def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] | None:
""" """
Parse an impersonate target string into a tuple of (client, version, os, os_vers) Parse an impersonate target string into a tuple of (client, version, os, os_vers)
If the target is invalid, return None If the target is invalid, return None
@ -175,13 +175,10 @@ def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[
client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in ( client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in (
target.split(':') + [None, None, None, None])][:4] target.split(':') + [None, None, None, None])][:4]
if client is not None:
return client, version, os, os_vers return client, version, os, os_vers
def compile_impersonate_target(*args) -> str | None: def compile_impersonate_target(*args) -> str | None:
client, version, os, os_vers = (list(args) + [None, None, None, None])[:4] client, version, os, os_vers = (list(args) + [None, None, None, None])[:4]
if not client:
return
filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)] filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)]
return ':'.join(filtered_parts).rstrip(':') return ':'.join(filtered_parts).rstrip(':')