Compare commits

...

17 Commits

Author SHA1 Message Date
coletdjnz
5d4e509828
linter 2023-12-10 14:07:00 +13:00
coletdjnz
4d5a63311a
add test for ydl helper funcs 2023-12-10 14:04:37 +13:00
coletdjnz
aac33fd194
Add impersonate_target_available 2023-12-10 13:51:23 +13:00
coletdjnz
234903f3fa
Accept empty client 2023-12-10 13:47:50 +13:00
coletdjnz
bdf6273880
update doc 2023-12-10 12:23:11 +13:00
coletdjnz
c618a31fdb
revert back to 3.12 2023-12-10 12:00:50 +13:00
coletdjnz
0705da041f
try pyinstaller v5 2023-12-10 11:49:38 +13:00
coletdjnz
b9983da0b0
try python 3.11 2023-12-10 11:34:58 +13:00
coletdjnz
75fb3011ec
notlike? 2023-12-10 10:39:04 +13:00
coletdjnz
5005f4493f
POWERshell 2023-12-10 10:32:15 +13:00
coletdjnz
a210633178
exclude curl_cffi from windows32 build 2023-12-10 10:29:09 +13:00
coletdjnz
98649602d3
revert nothing like debugging in the CI 2023-12-10 10:26:45 +13:00
coletdjnz
e77b8ce51c
nothing like debugging in the CI 2023-12-10 09:58:24 +13:00
coletdjnz
4b17fe1fa1
update builds 2023-12-10 09:45:40 +13:00
coletdjnz
3d351301e0
Update .github/workflows/build.yml
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-12-09 20:41:58 +00:00
coletdjnz
5abbea2f7f
Update .github/workflows/build.yml
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-12-09 20:41:51 +00:00
coletdjnz
c86adc7c97
test cleanup 2023-12-10 09:41:04 +13:00
7 changed files with 70 additions and 49 deletions

View File

@ -129,7 +129,7 @@ jobs:
brotli-python
secretstorage
EOF
sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
sed -E '/^(brotli|secretstorage|curl_cffi).*/d' requirements.txt >> "$reqs"
mamba create -n build --file "$reqs"
- name: Prepare
@ -144,9 +144,6 @@ jobs:
run: |
unset LD_LIBRARY_PATH # Harmful; set by setup-python
conda activate build
python -m pip install -U pip setuptools wheel pip-tools
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
python -m pip install -U -r requirements-all.txt
python pyinst.py --onedir
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
python pyinst.py
@ -242,17 +239,21 @@ jobs:
# NB: Building universal2 does not work with python from actions/setup-python
- name: Install Requirements
run: |
brew install coreutils
brew install coreutils gnu-sed
python3 -m pip install -U --user pip setuptools wheel delocate
# curl_cffi must be removed from reqs
gsed -i -E '/^curl_cffi.*/d' requirements.txt
# We need to ignore wheels otherwise we break universal2 builds
python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
# PyInstaller v6 currently does not work with this
python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
mkdir curl_cffi_whls curl_cffi_universal2
python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls
python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/curl_cffi* | paste -sd " " - ) -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/cffi-* | paste -sd " " - ) -w curl_cffi_universal2
cd curl_cffi_universal2 && find curl_cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; && find cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \;
python3 -m pip install -U --user $(find curl_cffi-*) $(find cffi-*) && cd ..
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2
cd curl_cffi_universal2
for file in *cffi-*x86_64*; do mv -n -- "${file}" "${file/x86_64/universal2}"; done
python3 -m pip install -U --user curl_cffi-* cffi-*
- name: Prepare
run: |
@ -303,9 +304,8 @@ jobs:
- name: Install Requirements
run: |
brew install coreutils
python3 -m pip install -U --user pip setuptools wheel pip-tools
python3 -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
python3 -m pip install -U --user Pyinstaller -r requirements-all.txt
python3 -m pip install -U --user pip setuptools wheel
python3 -m pip install -U --user Pyinstaller -r requirements.txt
- name: Prepare
run: |
@ -344,9 +344,8 @@ jobs:
python-version: "3.8"
- name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python -m pip install -U pip setuptools wheel py2exe pip-tools
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements-all.txt
python -m pip install -U pip setuptools wheel py2exe
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
- name: Prepare
run: |
@ -395,7 +394,8 @@ jobs:
- name: Install Requirements
run: |
python -m pip install -U pip setuptools wheel
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
Get-Content -Path requirements.txt | where {$_ -notlike 'curl_cffi*'} | Set-Content requirements_used.txt
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements_used.txt
- name: Prepare
run: |

View File

@ -29,7 +29,7 @@ from http.cookiejar import CookieJar
from test.conftest import validate_and_send
from test.helper import FakeYDL, http_server_port
from yt_dlp.cookies import YoutubeDLCookieJar
from yt_dlp.dependencies import brotli, requests, urllib3, curl_cffi
from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3
from yt_dlp.networking import (
HEADRequest,
PUTRequest,
@ -196,7 +196,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
self._headers()
elif self.path.startswith('/308-to-headers'):
self.send_response(308)
# get server port
# redirect to "localhost" for testing cookie redirection handling
self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers')
self.send_header('Content-Length', '0')
self.end_headers()
@ -443,7 +443,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
def test_request_cookie_header(self, handler):
# We should accept a Cookie header being passed as in normal headers and handle it appropriately.
with handler(verbose=True) as rh:
with handler() as rh:
# Specified Cookie header should be used
res = validate_and_send(
rh, Request(
@ -942,9 +942,9 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
# but when not impersonating don't remove std_headers
res = validate_and_send(
rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower()
assert std_headers['user-agent'].lower() in res
assert std_headers['accept-language'].lower() in res
assert 'x-custom: test' in res
# std_headers should be present
for k, v in std_headers.items():
assert f'{k}: {v}'.lower() in res
@pytest.mark.parametrize('raised,expected,match', [
(lambda: curl_cffi.requests.errors.RequestsError(
@ -957,6 +957,7 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
import curl_cffi.requests
from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter
curl_res = curl_cffi.requests.Response()
res = CurlCFFIResponseAdapter(curl_res)
@ -1144,7 +1145,9 @@ class TestRequestHandlerValidation:
({'unsupported': 'value'}, UnsupportedRequest),
({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest),
({'impersonate': 123}, AssertionError),
({'impersonate': ('chrome', None, None, None)}, False)
({'impersonate': ('chrome', None, None, None)}, False),
({'impersonate': (None, None, None, None)}, False),
({'impersonate': ()}, False)
]),
(NoCheckRH, 'http', [
({'cookiejar': 'notacookiejar'}, False),
@ -1493,6 +1496,25 @@ class TestYoutubeDLNetworking:
rh = self.build_handler(ydl, IRH)
assert rh.impersonate == ('firefox', None, None, None)
def test_get_impersonate_targets(self):
handlers = []
for target_client in ('firefox', 'chrome', 'edge'):
class TestRH(ImpersonateRequestHandler):
def _send(self, request: Request):
pass
_SUPPORTED_URL_SCHEMES = ('http',)
_SUPPORTED_IMPERSONATE_TARGET_TUPLES = [(target_client,)]
RH_KEY = target_client
RH_NAME = target_client
handlers.append(TestRH)
with FakeYDL() as ydl:
ydl._request_director = ydl.build_request_director(handlers)
assert set(ydl.get_impersonate_targets()) == {('firefox', 'firefox'), ('chrome', 'chrome'), ('edge', 'edge')}
assert ydl.impersonate_target_available(('firefox', ))
assert ydl.impersonate_target_available(())
assert not ydl.impersonate_target_available(('safari',))
@pytest.mark.parametrize('proxy_key,proxy_url,expected', [
('http', '__noproxy__', None),
('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
@ -1799,7 +1821,10 @@ class TestImpersonate:
('firefox:::', ('firefox', None, None, None)),
('firefox:120::5', ('firefox', '120', None, '5')),
('firefox:120:', ('firefox', '120', None, None)),
('::120', None)
('::120', (None, None, '120', None)),
(':', (None, None, None, None)),
(':::', (None, None, None, None)),
('', (None, None, None, None)),
])
def test_parse_impersonate_target(self, target, expected):
assert parse_impersonate_target(target) == expected
@ -1812,9 +1837,10 @@ class TestImpersonate:
(('firefox', None, 'linux', None), 'firefox::linux'),
(('firefox', None, None, '5'), 'firefox:::5'),
(('firefox', '120', None, '5'), 'firefox:120::5'),
((None, '120', None, None), None),
((None, '120', None, None), ':120'),
(('firefox', ), 'firefox'),
(('firefox', None, 'linux'), 'firefox::linux'),
((None, None, None, None), ''),
])
def test_compile_impersonate_target(self, target_tuple, expected):
assert compile_impersonate_target(*target_tuple) == expected

View File

@ -714,13 +714,9 @@ class YoutubeDL:
self.deprecated_feature(msg)
impersonate_target = self.params.get('impersonate')
if impersonate_target:
if impersonate_target is not None:
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
results = self._request_director.collect_from_handlers(
lambda x: [x.is_supported_target(impersonate_target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
)
if not any(results):
if not self.impersonate_target_available(impersonate_target):
raise ValueError(
f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. '
f'Use --list-impersonate-targets to see available targets.')
@ -4059,6 +4055,11 @@ class YoutubeDL:
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
), key=lambda x: x[0])
def impersonate_target_available(self, target):
return any(self._request_director.collect_from_handlers(
lambda x: [x.is_supported_target(target)],
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]))
def urlopen(self, req):
""" Start an HTTP download """
if isinstance(req, str):

View File

@ -386,7 +386,7 @@ def validate_options(opts):
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
if opts.impersonate:
if opts.impersonate is not None:
target = parse_impersonate_target(opts.impersonate)
if target is None:
raise ValueError(f'invalid impersonate target "{opts.impersonate}"')

View File

@ -8,14 +8,11 @@ from .exceptions import UnsupportedRequest
from ..compat.types import NoneType
from ..utils.networking import std_headers
ImpersonateTarget = Tuple[str, Optional[str], Optional[str], Optional[str]]
ImpersonateTarget = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]
def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget):
if target1[0] != target2[0]:
return False
for i in range(1, min(len(target1), len(target2))):
for i in range(0, min(len(target1), len(target2))):
if (
target1[i]
and target2[i]
@ -75,13 +72,13 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _resolve_target(self, target: ImpersonateTarget | None):
"""Resolve a target to a supported target."""
if not target:
if target is None:
return
for supported_target in self.get_supported_targets():
if _target_within(target, supported_target):
if self.verbose:
self._logger.stdout(
f'{self.RH_NAME}: resolved impersonate target "{target}" to "{supported_target}"')
f'{self.RH_NAME}: resolved impersonate target {target} to {supported_target}')
return supported_target
def get_supported_targets(self) -> tuple[ImpersonateTarget]:
@ -106,7 +103,7 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
def _get_impersonate_headers(self, request):
headers = self._merge_headers(request.headers)
if self._get_request_target(request):
if self._get_request_target(request) is not None:
# remove all headers present in std_headers
headers.pop('User-Agent', None)
for header in std_headers:
@ -117,6 +114,6 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
@register_preference(ImpersonateRequestHandler)
def impersonate_preference(rh, request):
if request.extensions.get('impersonate') or rh.impersonate:
if request.extensions.get('impersonate') is not None or rh.impersonate is not None:
return 1000
return 0

View File

@ -513,8 +513,8 @@ def create_parser():
)
network.add_option(
'--impersonate',
metavar='CLIENT[:[VERSION][:[OS][:OS_VERSION]]]', dest='impersonate', default=None,
help='Client to impersonate for requests',
metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None,
help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client',
)
network.add_option(
'--list-impersonate-targets',

View File

@ -167,7 +167,7 @@ def normalize_url(url):
).geturl()
def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]] | None:
def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] | None:
"""
Parse an impersonate target string into a tuple of (client, version, os, os_vers)
If the target is invalid, return None
@ -175,13 +175,10 @@ def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[
client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in (
target.split(':') + [None, None, None, None])][:4]
if client is not None:
return client, version, os, os_vers
def compile_impersonate_target(*args) -> str | None:
client, version, os, os_vers = (list(args) + [None, None, None, None])[:4]
if not client:
return
filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)]
return ':'.join(filtered_parts).rstrip(':')