Merge 1aec76a000 into 5904853ae5

Closes #7442
Authored by: bashonly

yt_dlp/extractor/cbc.py

@@ -151,7 +151,7 @@ def _real_extract(self, url):
 
 class CBCPlayerIE(InfoExtractor):
     IE_NAME = 'cbc.ca:player'
-    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
+    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/(?:video/)?|i/caffeine/syndicate/\?mediaId=))(?P<id>(?:\d\.)?\d+)'
     _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
         'md5': '64d25f841ddf4ddb28a235338af32e2c',
@@ -277,6 +277,28 @@ class CBCPlayerIE(InfoExtractor):
             'location': 'Canada',
             'media_type': 'Full Program',
         },
+    }, {
+        'url': 'https://www.cbc.ca/player/play/video/1.7194274',
+        'md5': '188b96cf6bdcb2540e178a6caa957128',
+        'info_dict': {
+            'id': '2334524995812',
+            'ext': 'mp4',
+            'title': '#TheMoment a rare white spirit moose was spotted in Alberta',
+            'description': 'md5:18ae269a2d0265c5b0bbe4b2e1ac61a3',
+            'timestamp': 1714788791,
+            'duration': 77.678,
+            'subtitles': {'eng': [{'ext': 'vtt', 'protocol': 'm3u8_native'}]},
+            'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/201/543/THE_MOMENT.jpg',
+            'uploader': 'CBCC-NEW',
+            'chapters': 'count:0',
+            'upload_date': '20240504',
+            'categories': 'count:3',
+            'series': 'The National',
+            'tags': 'count:15',
+            'creators': ['encoder'],
+            'location': 'Canada',
+            'media_type': 'Excerpt',
+        },
     }, {
         'url': 'cbcplayer:1.7159484',
         'only_matching': True,

yt_dlp/extractor/crunchyroll.py

@@ -53,15 +53,19 @@ def _set_auth_info(self, response):
         CrunchyrollBaseIE._AUTH_EXPIRY = time_seconds(seconds=traverse_obj(response, ('expires_in', {float_or_none}), default=300) - 10)
 
     def _request_token(self, headers, data, note='Requesting token', errnote='Failed to request token'):
-        try:  # TODO: Add impersonation support here
+        try:
             return self._download_json(
                 f'{self._BASE_URL}/auth/v1/token', None, note=note, errnote=errnote,
-                headers=headers, data=urlencode_postdata(data))
+                headers=headers, data=urlencode_postdata(data), impersonate=True)
         except ExtractorError as error:
             if not isinstance(error.cause, HTTPError) or error.cause.status != 403:
                 raise
+            if target := error.cause.response.extensions.get('impersonate'):
+                raise ExtractorError(f'Got HTTP Error 403 when using impersonate target "{target}"')
             raise ExtractorError(
-                'Request blocked by Cloudflare; navigate to Crunchyroll in your browser, '
+                'Request blocked by Cloudflare. '
+                'Install the required impersonation dependency if possible, '
+                'or else navigate to Crunchyroll in your browser, '
                 'then pass the fresh cookies (with --cookies-from-browser or --cookies) '
                 'and your browser\'s User-Agent (with --user-agent)', expected=True)
 

yt_dlp/extractor/manyvids.py

@@ -1,14 +1,6 @@
-import re
+import json
 
 from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    extract_attributes,
-    int_or_none,
-    str_to_int,
-    url_or_none,
-    urlencode_postdata,
-)
 
 
 class ManyVidsIE(InfoExtractor):
@@ -16,27 +8,24 @@ class ManyVidsIE(InfoExtractor):
     _VALID_URL = r'(?i)https?://(?:www\.)?manyvids\.com/video/(?P<id>\d+)'
     _TESTS = [{
         # preview video
-        'url': 'https://www.manyvids.com/Video/133957/everthing-about-me/',
-        'md5': '03f11bb21c52dd12a05be21a5c7dcc97',
+        'url': 'https://www.manyvids.com/Video/1856601/Horny-Asian-Bunny-Needs-Cock/',
         'info_dict': {
-            'id': '133957',
+            'id': '1856601',
             'ext': 'mp4',
-            'title': 'everthing about me (Preview)',
-            'uploader': 'ellyxxix',
-            'view_count': int,
+            'title': 'Horny Asian Bunny Needs Cock',
+            'description': "This naughty little Asian bunny is bored and horny. Feed her something hard and big.",
+            'uploader': 'nicoledoshi',
             'like_count': int,
         },
     }, {
         # full video
-        'url': 'https://www.manyvids.com/Video/935718/MY-FACE-REVEAL/',
-        'md5': 'bb47bab0e0802c2a60c24ef079dfe60f',
+        'url': 'https://www.manyvids.com/Video/4957054/Latex-Dress-JOI/',
         'info_dict': {
-            'id': '935718',
+            'id': '4957054',
             'ext': 'mp4',
-            'title': 'MY FACE REVEAL',
-            'description': 'md5:ec5901d41808b3746fed90face161612',
-            'uploader': 'Sarah Calanthe',
-            'view_count': int,
+            'title': 'Latex Dress JOI',
+            'description': "My new latex dress feels so good on my skin! Just wearing it for you makes me horny. Jerk your cock while I show off my latex-clad body from all angles, rubbing my hands over my body and describing how good it feels. I can't help but touch my pussy, and I give you a countdown when I'm close so we can cum together for my new latex dress!",
+            'uploader': "Kylee Nash",
             'like_count': int,
         },
     }]
@@ -44,119 +33,33 @@ class ManyVidsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        real_url = 'https://www.manyvids.com/video/%s/gtm.js' % (video_id, )
-        try:
-            webpage = self._download_webpage(real_url, video_id)
-        except Exception:
-            # probably useless fallback
-            webpage = self._download_webpage(url, video_id)
+        infos_url = f'https://www.manyvids.com/bff/store/video/{video_id}'
+        info_webpage = self._download_webpage(infos_url, video_id, expected_status=200)
+        info: dict = json.loads(info_webpage).get('data')
 
-        info = self._search_regex(
-            r'''(<div\b[^>]*\bid\s*=\s*(['"])pageMetaDetails\2[^>]*>)''',
-            webpage, 'meta details', default='')
-        info = extract_attributes(info)
+        download_url = f'https://www.manyvids.com/bff/store/video/{video_id}/private'
+        download_webpage = self._download_webpage(download_url, video_id, expected_status=200)
+        download: dict = json.loads(download_webpage).get('data')
 
-        player = self._search_regex(
-            r'''(<div\b[^>]*\bid\s*=\s*(['"])rmpPlayerStream\2[^>]*>)''',
-            webpage, 'player details', default='')
-        player = extract_attributes(player)
+        is_free: bool = info["isFree"]
 
-        video_urls_and_ids = (
-            (info.get('data-meta-video'), 'video'),
-            (player.get('data-video-transcoded'), 'transcoded'),
-            (player.get('data-video-filepath'), 'filepath'),
-            (self._og_search_video_url(webpage, secure=False, default=None), 'og_video'),
-        )
-
-        def txt_or_none(s, default=None):
-            return (s.strip() or default) if isinstance(s, str) else default
-
-        uploader = txt_or_none(info.get('data-meta-author'))
-
-        def mung_title(s):
-            if uploader:
-                s = re.sub(r'^\s*%s\s+[|-]' % (re.escape(uploader), ), '', s)
-            return txt_or_none(s)
-
-        title = (
-            mung_title(info.get('data-meta-title'))
-            or self._html_search_regex(
-                (r'<span[^>]+class=["\']item-title[^>]+>([^<]+)',
-                 r'<h2[^>]+class=["\']h2 m-0["\'][^>]*>([^<]+)'),
-                webpage, 'title', default=None)
-            or self._html_search_meta(
-                'twitter:title', webpage, 'title', fatal=True))
-
-        title = re.sub(r'\s*[|-]\s+ManyVids\s*$', '', title) or title
-
-        if any(p in webpage for p in ('preview_videos', '_preview.mp4')):
-            title += ' (Preview)'
-
-        mv_token = self._search_regex(
-            r'data-mvtoken=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
-            'mv token', default=None, group='value')
-
-        if mv_token:
-            # Sets some cookies
-            self._download_webpage(
-                'https://www.manyvids.com/includes/ajax_repository/you_had_me_at_hello.php',
-                video_id, note='Setting format cookies', fatal=False,
-                data=urlencode_postdata({
-                    'mvtoken': mv_token,
-                    'vid': video_id,
-                }), headers={
-                    'Referer': url,
-                    'X-Requested-With': 'XMLHttpRequest'
-                })
-
-        formats = []
-        for v_url, fmt in video_urls_and_ids:
-            v_url = url_or_none(v_url)
-            if not v_url:
-                continue
-            if determine_ext(v_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    v_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': v_url,
-                    'format_id': fmt,
-                })
-
-        self._remove_duplicate_formats(formats)
-
-        for f in formats:
-            if f.get('height') is None:
-                f['height'] = int_or_none(
-                    self._search_regex(r'_(\d{2,3}[02468])_', f['url'], 'video height', default=None))
-            if '/preview/' in f['url']:
-                f['format_id'] = '_'.join(filter(None, (f.get('format_id'), 'preview')))
-                f['preference'] = -10
-            if 'transcoded' in f['format_id']:
-                f['preference'] = f.get('preference', -1) - 1
-
-        def get_likes():
-            likes = self._search_regex(
-                r'''(<a\b[^>]*\bdata-id\s*=\s*(['"])%s\2[^>]*>)''' % (video_id, ),
-                webpage, 'likes', default='')
-            likes = extract_attributes(likes)
-            return int_or_none(likes.get('data-likes'))
-
-        def get_views():
-            return str_to_int(self._html_search_regex(
-                r'''(?s)<span\b[^>]*\bclass\s*=["']views-wrapper\b[^>]+>.+?<span\b[^>]+>\s*(\d[\d,.]*)\s*</span>''',
-                webpage, 'view count', default=None))
+        title = info["title"]
+        if title and is_free is False:
+            title += " (Preview)"
 
+        if is_free:
+            download_url = download["filepath"]
+        else:
+            download_url = download.get("teaser", {}).get("filepath")
+            if download_url is None:
+                raise ValueError('this video has no preview')
+                # They are rare, but they exist. You can find such videos using the following link:
+                # https://www.manyvids.com/Vids/?content_type=1,2,3&other=blocked&search_type=0&sort=13&page=1
         return {
             'id': video_id,
+            'url': download_url,
             'title': title,
-            'formats': formats,
-            'description': txt_or_none(info.get('data-meta-description')),
-            'uploader': txt_or_none(info.get('data-meta-author')),
-            'thumbnail': (
-                url_or_none(info.get('data-meta-image'))
-                or url_or_none(player.get('data-video-screenshot'))),
-            'view_count': get_views(),
-            'like_count': get_likes(),
+            'description': info.get("description") or None,
+            'uploader': info["model"]["displayName"],
+            'like_count': int(info["likes"]),
         }

yt_dlp/extractor/youporn.py

@@ -72,15 +72,15 @@ class YouPornIE(InfoExtractor):
             'id': '16290308',
             'age_limit': 18,
             'categories': [],
-            'description': 'md5:00ea70f642f431c379763c17c2f396bc',
+            'description': str,  # TODO: detect/remove SEO spam description in ytdl backport
             'display_id': 'tinderspecial-trailer1',
             'duration': 298.0,
             'ext': 'mp4',
             'upload_date': '20201123',
             'uploader': 'Ersties',
             'tags': [],
-            'thumbnail': 'https://fi1.ypncdn.com/202011/23/16290308/original/8/tinderspecial-trailer1-8(m=eaAaaEPbaaaa).jpg',
-            'timestamp': 1606089600,
+            'thumbnail': r're:https://.+\.jpg',
+            'timestamp': 1606147564,
             'title': 'Tinder In Real Life',
             'view_count': int,
         }
@@ -88,11 +88,17 @@ class YouPornIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
-        definitions = self._download_json(
-            f'https://www.youporn.com/api/video/media_definitions/{video_id}/', display_id or video_id)
+        self._set_cookie('.youporn.com', 'age_verified', '1')
+        webpage = self._download_webpage(f'https://www.youporn.com/watch/{video_id}', video_id)
+        definitions = self._search_json(r'\bplayervars\s*:', webpage, 'player vars', video_id)['mediaDefinitions']
 
-        def get_format_data(data, f):
-            return traverse_obj(data, lambda _, v: v['format'] == f and url_or_none(v['videoUrl']))
+        def get_format_data(data, stream_type):
+            info_url = traverse_obj(data, (lambda _, v: v['format'] == stream_type, 'videoUrl', {url_or_none}, any))
+            if not info_url:
+                return []
+            return traverse_obj(
+                self._download_json(info_url, video_id, f'Downloading {stream_type} info JSON', fatal=False),
+                lambda _, v: v['format'] == stream_type and url_or_none(v['videoUrl']))
 
         formats = []
         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
@@ -123,10 +129,6 @@ def get_format_data(data, f):
             f['height'] = height
             formats.append(f)
 
-        webpage = self._download_webpage(
-            'http://www.youporn.com/watch/%s' % video_id, display_id,
-            headers={'Cookie': 'age_verified=1'})
-
         title = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
             webpage, 'title', default=None) or self._og_search_title(