Merge 3302588c77 into 5904853ae5

Closes #7442
Authored by: bashonly

yt_dlp/extractor/cbc.py

@@ -151,7 +151,7 @@ def _real_extract(self, url):
 
 class CBCPlayerIE(InfoExtractor):
     IE_NAME = 'cbc.ca:player'
-    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
+    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/(?:video/)?|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
     _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
         'md5': '64d25f841ddf4ddb28a235338af32e2c',
@@ -277,6 +277,28 @@ class CBCPlayerIE(InfoExtractor):
             'location': 'Canada',
             'media_type': 'Full Program',
         },
+    }, {
+        'url': 'https://www.cbc.ca/player/play/video/1.7194274',
+        'md5': '188b96cf6bdcb2540e178a6caa957128',
+        'info_dict': {
+            'id': '2334524995812',
+            'ext': 'mp4',
+            'title': '#TheMoment a rare white spirit moose was spotted in Alberta',
+            'description': 'md5:18ae269a2d0265c5b0bbe4b2e1ac61a3',
+            'timestamp': 1714788791,
+            'duration': 77.678,
+            'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/201/543/THE_MOMENT.jpg',
+            'uploader': 'CBCC-NEW',
+            'chapters': 'count:0',
+            'upload_date': '20240504',
+            'categories': 'count:3',
+            'series': 'The National',
+            'tags': 'count:15',
+            'creators': ['encoder'],
+            'location': 'Canada',
+            'media_type': 'Excerpt',
+        },
     }, {
         'url': 'cbcplayer:1.7159484',
         'only_matching': True,

yt_dlp/extractor/crunchyroll.py

@@ -53,15 +53,19 @@ def _set_auth_info(self, response):
         CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10)
 
     def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'):
-        try:  # TODO: Add impersonation support here
+        try:
             return self._download_json(
                 f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote,
-                headers=headers, data=urlencode_postdata(data))
+                headers=headers, data=urlencode_postdata(data), impersonate=True)
         except ExtractorError as error:
             if not isinstance(error.cause, HTTPError) or error.cause.status != 403:
                 raise
+            if target := error.cause.response.extensions.get('impersonate'):
+                raise ExtractorError(f'Got HTTP Error 403 when using impersonate target "{target}"')
             raise ExtractorError(
-                'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                'Request blocked by Cloudflare. '
+                'Install the required impersonation dependency if possible, '
+                'or else navigate to Crunchyroll in your browser, '
                 'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
                 'and your browser\'s User-Agent (with --user-agent)', expected=True)
 

yt_dlp/extractor/facebook.py

@@ -445,7 +445,9 @@ def _perform_login(self, username, password):
         try:
             login_results = self._download_webpage(request, None,
                                                    note='Logging in', errnote='unable to fetch login page')
-            if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
+            if 'Your Request Couldn' in login_results:
+                self.raise_login_required('Failed to login with credentials', method='cookies')
+            elif re.search(r'<form[^>]*name="login"[^<]*</form>', login_results):
                 error = self._html_search_regex(
                     r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
                     login_results, 'login error', default=None, group='error')
@@ -478,13 +480,41 @@ def _perform_login(self, username, password):
             return
 
     def _extract_from_url(self, url, video_id):
+        cookies = self._get_cookies(url)
+        # user passed logged-in cookies or attempted to login
+        login_data = cookies.get('c_user') and cookies.get('xs')
+        logged_in = False
+
         webpage = self._download_webpage(
             url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
 
+        sjs_data = [self._parse_json(j, video_id, fatal=False) for j in re.findall(
+            r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
+
+        if login_data:
+            logged_in = get_first(sjs_data, (
+                'require', ..., ..., ..., '__bbox', 'define',
+                lambda _, v: 'CurrentUserInitialData' in v, ..., 'ACCOUNT_ID'), default='0') != '0'
+            if logged_in and (info := get_first(sjs_data, ('require', ..., ..., ..., '__bbox', 'require', ..., ..., ...,
+                                                           '__bbox', 'result', 'data', (('ufac_client', 'state',
+                                                                                         (('set_contact_point_state_renderer', 'title'),
+                                                                                          ('intro_state_renderer', 'header_title'))),
+                                                                                        ('epsilon_checkpoint', 'screen', 'title'))))):
+                if any(content in info for content in ['days left to appeal', 'suspended your account']):
+                    raise ExtractorError('Your account is suspended', expected=True)
+                if 'Enter mobile number' == info:
+                    raise ExtractorError('Facebook is requiring mobile number confirmation', expected=True)
+                if 'your account has been locked' in info:
+                    raise ExtractorError('Your account has been locked', expected=True)
+
+        if props := get_first(sjs_data, (
+                'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., 'rootView',
+                lambda _, v: v['title'].startswith('This content isn\'t available'))):
+            raise ExtractorError(
+                f'Content unavailable. Facebook said: {props.get("body") or props["title"]}', expected=True)
+
         def extract_metadata(webpage):
-            post_data = [self._parse_json(j, video_id, fatal=False) for j in re.findall(
-                r'data-sjs>({.*?ScheduledServerJS.*?})</script>', webpage)]
-            post = traverse_obj(post_data, (
+            post = traverse_obj(sjs_data, (
                 ..., 'require', ..., ..., ..., '__bbox', 'require', ..., ..., ..., '__bbox', 'result', 'data'), expected_type=dict) or []
             media = traverse_obj(post, (..., 'attachments', ..., lambda k, v: (
                 k == 'media' and str(v['id']) == video_id and v['__typename'] == 'Video')), expected_type=dict)
@@ -795,6 +825,10 @@ def parse_attachment(attachment, key='media'):
             video_data = extract_from_jsmods_instances(tahoe_js_data)
 
         if not video_data:
+            if not login_data:
+                raise ExtractorError('Cannot parse data. Try logging in.', expected=True)
+            if not logged_in:
+                raise ExtractorError('Failed to login with provided data.', expected=True)
             raise ExtractorError('Cannot parse data')
 
         if len(video_data) > 1:

yt_dlp/extractor/youporn.py

@@ -72,15 +72,15 @@ class YouPornIE(InfoExtractor):
             'id': '16290308',
             'age_limit': 18,
             'categories': [],
-            'description': 'md5:00ea70f642f431c379763c17c2f396bc',
+            'description': str,  # TODO: detect/remove SEO spam description in ytdl backport
             'display_id': 'tinderspecial-trailer1',
             'duration': 298.0,
             'ext': 'mp4',
             'upload_date': '20201123',
             'uploader': 'Ersties',
             'tags': [],
-            'thumbnail': 'https://fi1.ypncdn.com/202011/23/16290308/original/8/tinderspecial-trailer1-8(m=eaAaaEPbaaaa).jpg',
-            'timestamp': 1606089600,
+            'thumbnail': r're:https://.+\.jpg',
+            'timestamp': 1606147564,
             'title': 'Tinder In Real Life',
             'view_count': int,
         }
@@ -88,11 +88,17 @@ class YouPornIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
-        definitions = self._download_json(
-            f'https://www.youporn.com/api/video/media_definitions/{video_id}/', display_id or video_id)
+        self._set_cookie('.youporn.com', 'age_verified', '1')
+        webpage = self._download_webpage(f'https://www.youporn.com/watch/{video_id}', video_id)
+        definitions = self._search_json(r'\bplayervars\s*:', webpage, 'player vars', video_id)['mediaDefinitions']
 
-        def get_format_data(data, f):
-            return traverse_obj(data, lambda _, v: v['format'] == f and url_or_none(v['videoUrl']))
+        def get_format_data(data, stream_type):
+            info_url = traverse_obj(data, (lambda _, v: v['format'] == stream_type, 'videoUrl', {url_or_none}, any))
+            if not info_url:
+                return []
+            return traverse_obj(
+                self._download_json(info_url, video_id, f'Downloading {stream_type} info JSON', fatal=False),
+                lambda _, v: v['format'] == stream_type and url_or_none(v['videoUrl']))
 
         formats = []
         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
@@ -123,10 +129,6 @@ def get_format_data(data, f):
             f['height'] = height
             formats.append(f)
 
-        webpage = self._download_webpage(
-            'http://www.youporn.com/watch/%s' % video_id, display_id,
-            headers={'Cookie': 'age_verified=1'})
-
         title = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
             webpage, 'title', default=None) or self._og_search_title(