Compare commits

..

2 Commits

Author SHA1 Message Date
coletdjnz
600d7992b0
pin curl-cffi in macos build 2023-12-19 19:35:26 +13:00
coletdjnz
b086dc1114
misc fixes from review 2023-12-19 19:33:46 +13:00
8 changed files with 29 additions and 43 deletions

View File

@ -243,12 +243,10 @@ jobs:
python3 -m pip install -U --user pip setuptools wheel delocate python3 -m pip install -U --user pip setuptools wheel delocate
# curl_cffi must be removed from reqs # curl_cffi must be removed from reqs
gsed -i -E '/^curl_cffi.*/d' requirements.txt gsed -i -E '/^curl_cffi.*/d' requirements.txt
# We need to ignore wheels otherwise we break universal2 builds python3 -m pip install -U --user --no-binary :all: Pyinstaller>=6.3 -r requirements.txt
# PyInstaller v6 currently does not work with this
python3 -m pip install -U --user --no-binary :all: Pyinstaller==5.13.2 -r requirements.txt
mkdir curl_cffi_whls curl_cffi_universal2 mkdir curl_cffi_whls curl_cffi_universal2
python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_arm64 curl_cffi>=0.5.9,<0.6.0 --pre -d curl_cffi_whls
python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi --pre -d curl_cffi_whls python3 -m pip download --only-binary=:all: --platform macosx_11_0_x86_64 curl_cffi>=0.5.9,<0.6.0 --pre -d curl_cffi_whls
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi* -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2 python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi-* -w curl_cffi_universal2
cd curl_cffi_universal2 cd curl_cffi_universal2

View File

@ -476,10 +476,11 @@ If you fork the project on GitHub, you can run your fork's [build workflow](.git
--socket-timeout SECONDS Time to wait before giving up, in seconds --socket-timeout SECONDS Time to wait before giving up, in seconds
--source-address IP Client-side IP address to bind to --source-address IP Client-side IP address to bind to
--impersonate [CLIENT[:[VERSION][:[OS][:OS_VERSION]]]] --impersonate [CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]
Client to impersonate for requests. Pass in Client to impersonate for requests. E.g.
chrome, chrome:110, chrome::android. Pass in
an empty string (--impersonate "") to an empty string (--impersonate "") to
impersonate any client impersonate any client.
--list-impersonate-targets List available clients to impersonate --list-impersonate-targets List available clients to impersonate.
-4, --force-ipv4 Make all connections via IPv4 -4, --force-ipv4 Make all connections via IPv4
-6, --force-ipv6 Make all connections via IPv6 -6, --force-ipv6 Make all connections via IPv6
--enable-file-urls Enable file:// URLs. This is disabled by --enable-file-urls Enable file:// URLs. This is disabled by

View File

@ -24,7 +24,6 @@ import traceback
import unicodedata import unicodedata
from .cache import Cache from .cache import Cache
from .compat import functools, urllib # isort: split from .compat import functools, urllib # isort: split
from .compat import compat_os_name, compat_shlex_quote, urllib_req_to_req from .compat import compat_os_name, compat_shlex_quote, urllib_req_to_req
from .cookies import LenientSimpleCookie, load_cookies from .cookies import LenientSimpleCookie, load_cookies
@ -62,13 +61,7 @@ from .postprocessor import (
get_postprocessor, get_postprocessor,
) )
from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
from .update import ( from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
REPOSITORY,
_get_system_deprecation,
_make_label,
current_git_head,
detect_variant,
)
from .utils import ( from .utils import (
DEFAULT_OUTTMPL, DEFAULT_OUTTMPL,
IDENTITY, IDENTITY,
@ -714,7 +707,6 @@ class YoutubeDL:
impersonate_target = self.params.get('impersonate') impersonate_target = self.params.get('impersonate')
if impersonate_target is not None: if impersonate_target is not None:
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
if not self.impersonate_target_available(impersonate_target): if not self.impersonate_target_available(impersonate_target):
raise ValueError( raise ValueError(
f'Impersonate target "{self.params.get("impersonate")}" is not available. ' f'Impersonate target "{self.params.get("impersonate")}" is not available. '
@ -3909,9 +3901,10 @@ class YoutubeDL:
# These imports can be slow. So import them only as needed # These imports can be slow. So import them only as needed
from .extractor.extractors import _LAZY_LOADER from .extractor.extractors import _LAZY_LOADER
from .extractor.extractors import _PLUGIN_CLASSES as plugin_ies from .extractor.extractors import (
from .extractor.extractors import \ _PLUGIN_CLASSES as plugin_ies,
_PLUGIN_OVERRIDES as plugin_ie_overrides _PLUGIN_OVERRIDES as plugin_ie_overrides
)
def get_encoding(stream): def get_encoding(stream):
ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__)) ret = str(getattr(stream, 'encoding', 'missing (%s)' % type(stream).__name__))
@ -4056,6 +4049,7 @@ class YoutubeDL:
if isinstance(rh, ImpersonateRequestHandler)]), key=lambda x: x[0]) if isinstance(rh, ImpersonateRequestHandler)]), key=lambda x: x[0])
def impersonate_target_available(self, target): def impersonate_target_available(self, target):
# This assumes that all handlers that support impersonation subclass ImpersonateRequestHandler
return any( return any(
rh.is_supported_target(target) rh.is_supported_target(target)
for rh in self._request_director.handlers.values() for rh in self._request_director.handlers.values()
@ -4095,9 +4089,10 @@ class YoutubeDL:
if ( if (
'unsupported proxy type: "https"' in ue.msg.lower() 'unsupported proxy type: "https"' in ue.msg.lower()
and 'requests' not in self._request_director.handlers and 'requests' not in self._request_director.handlers
and 'curl_cffi' not in self._request_director.handlers
): ):
raise RequestError( raise RequestError(
'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests') 'To use an HTTPS proxy for this request, one of the following dependencies needs to be installed: requests, curl_cffi')
elif ( elif (
re.match(r'unsupported url scheme: "wss?"', ue.msg.lower()) re.match(r'unsupported url scheme: "wss?"', ue.msg.lower())

View File

@ -389,7 +389,7 @@ def validate_options(opts):
opts.cookiesfrombrowser = (browser_name, profile, keyring, container) opts.cookiesfrombrowser = (browser_name, profile, keyring, container)
if opts.impersonate is not None: if opts.impersonate is not None:
opts.impersonate = ImpersonateTarget.from_str(opts.impersonate) opts.impersonate = ImpersonateTarget.from_str(opts.impersonate.lower())
# MetadataParser # MetadataParser
def metadataparser_actions(f): def metadataparser_actions(f):
@ -987,19 +987,18 @@ def _real_main(argv=None):
if opts.list_impersonate_targets: if opts.list_impersonate_targets:
available_targets = ydl.get_available_impersonate_targets() available_targets = ydl.get_available_impersonate_targets()
rows = [ rows = [
[target.client, target.version, target.os, target.os_vers, handler, str(target)] [target.client, target.version, target.os, target.os_vers, handler]
for target, handler in available_targets for target, handler in available_targets
] ]
ydl.to_screen('[info] Available impersonate targets') ydl.to_screen('[info] Available impersonate targets')
ydl.to_stdout( ydl.to_stdout(
render_table(['Client', 'Version', 'OS', 'OS Version', 'Handler', 'Example'], rows) render_table(['Client', 'Version', 'OS', 'OS Version', 'Source'], rows)
) )
if not available_targets: if not available_targets:
ydl.to_stdout('You are missing dependencies for impersonation. See the README for more info.') ydl.to_stdout('You are missing dependencies for impersonation. See the README for more info.')
ydl.to_stdout( ydl.to_stdout(
'If the above table is missing targets, you may be missing dependencies for impersonation. ' 'If the above table is missing targets, you may be missing dependencies for impersonation.')
'See the documentation for more information.')
return return
if not actual_use: if not actual_use:

View File

@ -5,26 +5,20 @@ import logging
import ssl import ssl
import sys import sys
from ._helper import ( from ._helper import create_connection, select_proxy, make_socks_proxy_opts, create_socks_proxy_socket
create_connection, from .common import Response, register_rh, Features
create_socks_proxy_socket,
make_socks_proxy_opts,
select_proxy,
)
from .common import Features, Response, register_rh
from .exceptions import ( from .exceptions import (
CertificateVerifyError, CertificateVerifyError,
HTTPError, HTTPError,
ProxyError,
RequestError, RequestError,
SSLError, SSLError,
TransportError, TransportError, ProxyError,
) )
from .websocket import WebSocketRequestHandler, WebSocketResponse from .websocket import WebSocketRequestHandler, WebSocketResponse
from ..compat import functools from ..compat import functools
from ..dependencies import websockets from ..dependencies import websockets
from ..socks import ProxyError as SocksProxyError
from ..utils import int_or_none from ..utils import int_or_none
from ..socks import ProxyError as SocksProxyError
if not websockets: if not websockets:
raise ImportError('websockets is not installed') raise ImportError('websockets is not installed')
@ -104,10 +98,10 @@ class WebsocketsRH(WebSocketRequestHandler):
extensions.pop('cookiejar', None) extensions.pop('cookiejar', None)
def _send(self, request): def _send(self, request):
timeout = self._calculate_timeout(request) timeout = float(request.extensions.get('timeout') or self.timeout)
headers = self._merge_headers(request.headers) headers = self._merge_headers(request.headers)
if 'cookie' not in headers: if 'cookie' not in headers:
cookiejar = self._get_cookiejar(request) cookiejar = request.extensions.get('cookiejar') or self.cookiejar
cookie_header = cookiejar.get_cookie_header(request.url) cookie_header = cookiejar.get_cookie_header(request.url)
if cookie_header: if cookie_header:
headers['cookie'] = cookie_header headers['cookie'] = cookie_header
@ -117,7 +111,7 @@ class WebsocketsRH(WebSocketRequestHandler):
'source_address': (self.source_address, 0) if self.source_address else None, 'source_address': (self.source_address, 0) if self.source_address else None,
'timeout': timeout 'timeout': timeout
} }
proxy = select_proxy(request.url, self._get_proxies(request)) proxy = select_proxy(request.url, request.proxies or self.proxies or {})
try: try:
if proxy: if proxy:
socks_proxy_options = make_socks_proxy_opts(proxy) socks_proxy_options = make_socks_proxy_opts(proxy)

View File

@ -49,8 +49,7 @@ class ImpersonateTarget:
@classmethod @classmethod
def from_str(cls, target: str): def from_str(cls, target: str):
return ImpersonateTarget(*[ return ImpersonateTarget(*[
None if (v or '').strip() == '' else v None if (v or '').strip() == '' else v for v in target.split(':')
for v in (target.split(':') + [None, None, None, None])[:4]
]) ])
def __hash__(self): def __hash__(self):

View File

@ -514,12 +514,13 @@ def create_parser():
network.add_option( network.add_option(
'--impersonate', '--impersonate',
metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None, metavar='[CLIENT[:[VERSION][:[OS][:OS_VERSION]]]]', dest='impersonate', default=None,
help='Client to impersonate for requests. Pass in an empty string (--impersonate "") to impersonate any client', help='Client to impersonate for requests. E.g. chrome, chrome:110, chrome::android.'
' Pass in an empty string (--impersonate "") to impersonate any client.',
) )
network.add_option( network.add_option(
'--list-impersonate-targets', '--list-impersonate-targets',
dest='list_impersonate_targets', default=False, action='store_true', dest='list_impersonate_targets', default=False, action='store_true',
help='List available clients to impersonate', help='List available clients to impersonate.',
) )
network.add_option( network.add_option(
'-4', '--force-ipv4', '-4', '--force-ipv4',

View File

@ -4,7 +4,6 @@ import collections
import random import random
import urllib.parse import urllib.parse
import urllib.request import urllib.request
from typing import Optional, Tuple
from ._utils import remove_start from ._utils import remove_start