mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-11-12 10:21:28 +01:00
Compare commits
No commits in common. "5d4e5098289f1051738da58743e488de0132cacd" and "5e1200ac69b1bccb129099c37820f0ed0a2f538a" have entirely different histories.
5d4e509828
...
5e1200ac69
34
.github/workflows/build.yml
vendored
34
.github/workflows/build.yml
vendored
|
@ -129,7 +129,7 @@ jobs:
|
|||
brotli-python
|
||||
secretstorage
|
||||
EOF
|
||||
sed -E '/^(brotli|secretstorage|curl_cffi).*/d' requirements.txt >> "$reqs"
|
||||
sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
|
||||
mamba create -n build --file "$reqs"
|
||||
|
||||
- name: Prepare
|
||||
|
@ -144,6 +144,9 @@ jobs:
|
|||
run: |
|
||||
unset LD_LIBRARY_PATH # Harmful; set by setup-python
|
||||
conda activate build
|
||||
python -m pip install -U pip setuptools wheel pip-tools
|
||||
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
|
||||
python -m pip install -U -r requirements-all.txt
|
||||
python pyinst.py --onedir
|
||||
(cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
|
||||
python pyinst.py
|
||||
|
@ -239,21 +242,17 @@ jobs:
|
|||
# NB: Building universal2 does not work with python from actions/setup-python
|
||||
- name: Install Requirements
|
||||
run: |
|
||||
brew install coreutils gnu-sed
|
||||
brew install coreutils
|
||||
python3 -m pip install -U --user pip setuptools wheel delocate
|
||||
# curl_cffi must be removed from reqs
|
||||
gsed -i -E '/^curl_cffi.*/d' requirements.txt
|
||||
# We need to ignore wheels otherwise we break universal2 builds
|
||||
# PyInstaller v6 currently does not work with this
|
||||
python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
|
||||
python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
|
||||
mkdir curl_cffi_whls curl_cffi_universal2
|
||||
python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls
|
||||
python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls
|
||||
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2
|
||||
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2
|
||||
cd curl_cffi_universal2
|
||||
for file in *cffi-*x86_64*; do mv -n -- "${file}" "${file/x86_64/universal2}"; done
|
||||
python3 -m pip install -U --user curl_cffi-* cffi-*
|
||||
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/curl_cffi* | paste -sd " " - ) -w curl_cffi_universal2
|
||||
python3 -m delocate.cmd.delocate_fuse $(find curl_cffi_whls/cffi-* | paste -sd " " - ) -w curl_cffi_universal2
|
||||
cd curl_cffi_universal2 && find curl_cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \; && find cffi-* -execdir bash -c 'mv -i "$1" "${1/x86_64/universal2}"' bash {} \;
|
||||
python3 -m pip install -U --user $(find curl_cffi-*) $(find cffi-*) && cd ..
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -304,8 +303,9 @@ jobs:
|
|||
- name: Install Requirements
|
||||
run: |
|
||||
brew install coreutils
|
||||
python3 -m pip install -U --user pip setuptools wheel
|
||||
python3 -m pip install -U --user Pyinstaller -r requirements.txt
|
||||
python3 -m pip install -U --user pip setuptools wheel pip-tools
|
||||
python3 -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
|
||||
python3 -m pip install -U --user Pyinstaller -r requirements-all.txt
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -344,8 +344,9 @@ jobs:
|
|||
python-version: "3.8"
|
||||
- name: Install Requirements
|
||||
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
|
||||
python -m pip install -U pip setuptools wheel py2exe
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
|
||||
python -m pip install -U pip setuptools wheel py2exe pip-tools
|
||||
python -m piptools compile -o requirements-all.txt --extra curl_cffi setup.py
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements-all.txt
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -394,8 +395,7 @@ jobs:
|
|||
- name: Install Requirements
|
||||
run: |
|
||||
python -m pip install -U pip setuptools wheel
|
||||
Get-Content -Path requirements.txt | where {$_ -notlike 'curl_cffi*'} | Set-Content requirements_used.txt
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements_used.txt
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
|
|
@ -29,7 +29,7 @@ from http.cookiejar import CookieJar
|
|||
from test.conftest import validate_and_send
|
||||
from test.helper import FakeYDL, http_server_port
|
||||
from yt_dlp.cookies import YoutubeDLCookieJar
|
||||
from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3
|
||||
from yt_dlp.dependencies import brotli, requests, urllib3, curl_cffi
|
||||
from yt_dlp.networking import (
|
||||
HEADRequest,
|
||||
PUTRequest,
|
||||
|
@ -196,7 +196,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
|
|||
self._headers()
|
||||
elif self.path.startswith('/308-to-headers'):
|
||||
self.send_response(308)
|
||||
# redirect to "localhost" for testing cookie redirection handling
|
||||
# get server port
|
||||
self.send_header('Location', f'http://localhost:{self.connection.getsockname()[1]}/headers')
|
||||
self.send_header('Content-Length', '0')
|
||||
self.end_headers()
|
||||
|
@ -443,7 +443,7 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
|
|||
@pytest.mark.parametrize('handler', ['Urllib', 'Requests', 'CurlCFFI'], indirect=True)
|
||||
def test_request_cookie_header(self, handler):
|
||||
# We should accept a Cookie header being passed as in normal headers and handle it appropriately.
|
||||
with handler() as rh:
|
||||
with handler(verbose=True) as rh:
|
||||
# Specified Cookie header should be used
|
||||
res = validate_and_send(
|
||||
rh, Request(
|
||||
|
@ -942,9 +942,9 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
|
|||
# but when not impersonating don't remove std_headers
|
||||
res = validate_and_send(
|
||||
rh, Request(f'http://127.0.0.1:{self.http_port}/headers', headers={'x-custom': 'test'})).read().decode().lower()
|
||||
# std_headers should be present
|
||||
for k, v in std_headers.items():
|
||||
assert f'{k}: {v}'.lower() in res
|
||||
assert std_headers['user-agent'].lower() in res
|
||||
assert std_headers['accept-language'].lower() in res
|
||||
assert 'x-custom: test' in res
|
||||
|
||||
@pytest.mark.parametrize('raised,expected,match', [
|
||||
(lambda: curl_cffi.requests.errors.RequestsError(
|
||||
|
@ -957,7 +957,6 @@ class TestCurlCFFIRequestHandler(TestRequestHandlerBase):
|
|||
@pytest.mark.parametrize('handler', ['CurlCFFI'], indirect=True)
|
||||
def test_response_error_mapping(self, handler, monkeypatch, raised, expected, match):
|
||||
import curl_cffi.requests
|
||||
|
||||
from yt_dlp.networking._curlcffi import CurlCFFIResponseAdapter
|
||||
curl_res = curl_cffi.requests.Response()
|
||||
res = CurlCFFIResponseAdapter(curl_res)
|
||||
|
@ -1145,9 +1144,7 @@ class TestRequestHandlerValidation:
|
|||
({'unsupported': 'value'}, UnsupportedRequest),
|
||||
({'impersonate': ('badtarget', None, None, None)}, UnsupportedRequest),
|
||||
({'impersonate': 123}, AssertionError),
|
||||
({'impersonate': ('chrome', None, None, None)}, False),
|
||||
({'impersonate': (None, None, None, None)}, False),
|
||||
({'impersonate': ()}, False)
|
||||
({'impersonate': ('chrome', None, None, None)}, False)
|
||||
]),
|
||||
(NoCheckRH, 'http', [
|
||||
({'cookiejar': 'notacookiejar'}, False),
|
||||
|
@ -1496,25 +1493,6 @@ class TestYoutubeDLNetworking:
|
|||
rh = self.build_handler(ydl, IRH)
|
||||
assert rh.impersonate == ('firefox', None, None, None)
|
||||
|
||||
def test_get_impersonate_targets(self):
|
||||
handlers = []
|
||||
for target_client in ('firefox', 'chrome', 'edge'):
|
||||
class TestRH(ImpersonateRequestHandler):
|
||||
def _send(self, request: Request):
|
||||
pass
|
||||
_SUPPORTED_URL_SCHEMES = ('http',)
|
||||
_SUPPORTED_IMPERSONATE_TARGET_TUPLES = [(target_client,)]
|
||||
RH_KEY = target_client
|
||||
RH_NAME = target_client
|
||||
handlers.append(TestRH)
|
||||
|
||||
with FakeYDL() as ydl:
|
||||
ydl._request_director = ydl.build_request_director(handlers)
|
||||
assert set(ydl.get_impersonate_targets()) == {('firefox', 'firefox'), ('chrome', 'chrome'), ('edge', 'edge')}
|
||||
assert ydl.impersonate_target_available(('firefox', ))
|
||||
assert ydl.impersonate_target_available(())
|
||||
assert not ydl.impersonate_target_available(('safari',))
|
||||
|
||||
@pytest.mark.parametrize('proxy_key,proxy_url,expected', [
|
||||
('http', '__noproxy__', None),
|
||||
('no', '127.0.0.1,foo.bar', '127.0.0.1,foo.bar'),
|
||||
|
@ -1821,10 +1799,7 @@ class TestImpersonate:
|
|||
('firefox:::', ('firefox', None, None, None)),
|
||||
('firefox:120::5', ('firefox', '120', None, '5')),
|
||||
('firefox:120:', ('firefox', '120', None, None)),
|
||||
('::120', (None, None, '120', None)),
|
||||
(':', (None, None, None, None)),
|
||||
(':::', (None, None, None, None)),
|
||||
('', (None, None, None, None)),
|
||||
('::120', None)
|
||||
])
|
||||
def test_parse_impersonate_target(self, target, expected):
|
||||
assert parse_impersonate_target(target) == expected
|
||||
|
@ -1837,10 +1812,9 @@ class TestImpersonate:
|
|||
(('firefox', None, 'linux', None), 'firefox::linux'),
|
||||
(('firefox', None, None, '5'), 'firefox:::5'),
|
||||
(('firefox', '120', None, '5'), 'firefox:120::5'),
|
||||
((None, '120', None, None), ':120'),
|
||||
((None, '120', None, None), None),
|
||||
(('firefox', ), 'firefox'),
|
||||
(('firefox', None, 'linux'), 'firefox::linux'),
|
||||
((None, None, None, None), ''),
|
||||
])
|
||||
def test_compile_impersonate_target(self, target_tuple, expected):
|
||||
assert compile_impersonate_target(*target_tuple) == expected
|
||||
|
|
|
@ -714,9 +714,13 @@ class YoutubeDL:
|
|||
self.deprecated_feature(msg)
|
||||
|
||||
impersonate_target = self.params.get('impersonate')
|
||||
if impersonate_target is not None:
|
||||
if impersonate_target:
|
||||
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
|
||||
if not self.impersonate_target_available(impersonate_target):
|
||||
results = self._request_director.collect_from_handlers(
|
||||
lambda x: [x.is_supported_target(impersonate_target)],
|
||||
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
|
||||
)
|
||||
if not any(results):
|
||||
raise ValueError(
|
||||
f'Impersonate target "{compile_impersonate_target(*self.params.get("impersonate"))}" is not available. '
|
||||
f'Use --list-impersonate-targets to see available targets.')
|
||||
|
@ -4055,11 +4059,6 @@ class YoutubeDL:
|
|||
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]
|
||||
), key=lambda x: x[0])
|
||||
|
||||
def impersonate_target_available(self, target):
|
||||
return any(self._request_director.collect_from_handlers(
|
||||
lambda x: [x.is_supported_target(target)],
|
||||
[lambda _, v: isinstance(v, ImpersonateRequestHandler)]))
|
||||
|
||||
def urlopen(self, req):
|
||||
""" Start an HTTP download """
|
||||
if isinstance(req, str):
|
||||
|
|
|
@ -386,7 +386,7 @@ def validate_options(opts):
|
|||
f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}')
|
||||
opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
|
||||
|
||||
if opts.impersonate is not None:
|
||||
if opts.impersonate:
|
||||
target = parse_impersonate_target(opts.impersonate)
|
||||
if target is None:
|
||||
raise ValueError(f'invalid impersonate target "{opts.impersonate}"')
|
||||
|
|
|
@ -8,11 +8,14 @@ from .exceptions import UnsupportedRequest
|
|||
from ..compat.types import NoneType
|
||||
from ..utils.networking import std_headers
|
||||
|
||||
ImpersonateTarget = Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]
|
||||
ImpersonateTarget = Tuple[str, Optional[str], Optional[str], Optional[str]]
|
||||
|
||||
|
||||
def _target_within(target1: ImpersonateTarget, target2: ImpersonateTarget):
|
||||
for i in range(0, min(len(target1), len(target2))):
|
||||
if target1[0] != target2[0]:
|
||||
return False
|
||||
|
||||
for i in range(1, min(len(target1), len(target2))):
|
||||
if (
|
||||
target1[i]
|
||||
and target2[i]
|
||||
|
@ -72,13 +75,13 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
|
|||
|
||||
def _resolve_target(self, target: ImpersonateTarget | None):
|
||||
"""Resolve a target to a supported target."""
|
||||
if target is None:
|
||||
if not target:
|
||||
return
|
||||
for supported_target in self.get_supported_targets():
|
||||
if _target_within(target, supported_target):
|
||||
if self.verbose:
|
||||
self._logger.stdout(
|
||||
f'{self.RH_NAME}: resolved impersonate target {target} to {supported_target}')
|
||||
f'{self.RH_NAME}: resolved impersonate target "{target}" to "{supported_target}"')
|
||||
return supported_target
|
||||
|
||||
def get_supported_targets(self) -> tuple[ImpersonateTarget]:
|
||||
|
@ -103,7 +106,7 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
|
|||
|
||||
def _get_impersonate_headers(self, request):
|
||||
headers = self._merge_headers(request.headers)
|
||||
if self._get_request_target(request) is not None:
|
||||
if self._get_request_target(request):
|
||||
# remove all headers present in std_headers
|
||||
headers.pop('User-Agent', None)
|
||||
for header in std_headers:
|
||||
|
@ -114,6 +117,6 @@ class ImpersonateRequestHandler(RequestHandler, ABC):
|
|||
|
||||
@register_preference(ImpersonateRequestHandler)
|
||||
def impersonate_preference(rh, request):
|
||||
if request.extensions.get('impersonate') is not None or rh.impersonate is not None:
|
||||
if request.extensions.get('impersonate') or rh.impersonate:
|
||||
return 1000
|
||||
return 0
|
||||
|
|
|
@ -513,8 +513,8 @@ def create_parser():
|
|||
)
|
||||
network.add_option(
|
||||
'--impersonate',
|
||||
metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None,
|
||||
help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client',
|
||||
metavar='CLIENT[:[VERSION][:[OS][:OS_VERSION]]]', dest='impersonate', default=None,
|
||||
help='Client to impersonate for requests',
|
||||
)
|
||||
network.add_option(
|
||||
'--list-impersonate-targets',
|
||||
|
|
|
@ -167,7 +167,7 @@ def normalize_url(url):
|
|||
).geturl()
|
||||
|
||||
|
||||
def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]] | None:
|
||||
def parse_impersonate_target(target: str) -> Tuple[str, Optional[str], Optional[str], Optional[str]] | None:
|
||||
"""
|
||||
Parse an impersonate target string into a tuple of (client, version, os, os_vers)
|
||||
If the target is invalid, return None
|
||||
|
@ -175,10 +175,13 @@ def parse_impersonate_target(target: str) -> Tuple[Optional[str], Optional[str],
|
|||
client, version, os, os_vers = [None if (v or '').strip() == '' else v for v in (
|
||||
target.split(':') + [None, None, None, None])][:4]
|
||||
|
||||
if client is not None:
|
||||
return client, version, os, os_vers
|
||||
|
||||
|
||||
def compile_impersonate_target(*args) -> str | None:
|
||||
client, version, os, os_vers = (list(args) + [None, None, None, None])[:4]
|
||||
if not client:
|
||||
return
|
||||
filtered_parts = [str(part) if part is not None else '' for part in (client, version, os, os_vers)]
|
||||
return ':'.join(filtered_parts).rstrip(':')
|
||||
|
|
Loading…
Reference in New Issue
Block a user