Merge 773a8764c8 into 6b54cccdcb

Closes #4989
Authored by: alexhuot1, dirkf

yt_dlp/extractor/bilibili.py

@@ -93,11 +93,11 @@ def extract_formats(self, play_info):
 
         return formats
 
-    def _download_playinfo(self, video_id, cid):
+    def _download_playinfo(self, video_id, cid, headers=None):
         return self._download_json(
             'https://api.bilibili.com/x/player/playurl', video_id,
             query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
-            note=f'Downloading video formats for cid {cid}')['data']
+            note=f'Downloading video formats for cid {cid}', headers=headers)['data']
 
     def json2srt(self, json_data):
         srt_data = ''
@@ -493,7 +493,8 @@ class BiliBiliIE(BilibiliBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage, urlh = self._download_webpage_handle(url, video_id)
+        headers = self.geo_verification_headers()
+        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers)
         if not self._match_valid_url(urlh.url):
             return self.url_result(urlh.url)
 
@@ -531,7 +532,7 @@ def _real_extract(self, url):
             self._download_json(
                 'https://api.bilibili.com/x/player/pagelist', video_id,
                 fatal=False, query={'bvid': video_id, 'jsonp': 'jsonp'},
-                note='Extracting videos in anthology'),
+                note='Extracting videos in anthology', headers=headers),
             'data', expected_type=list) or []
         is_anthology = len(page_list_json) > 1
 
@@ -552,7 +553,7 @@ def _real_extract(self, url):
 
         festival_info = {}
         if is_festival:
-            play_info = self._download_playinfo(video_id, cid)
+            play_info = self._download_playinfo(video_id, cid, headers=headers)
 
             festival_info = traverse_obj(initial_state, {
                 'uploader': ('videoInfo', 'upName'),
@@ -666,14 +667,15 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
 
     def _real_extract(self, url):
         episode_id = self._match_id(url)
-        webpage = self._download_webpage(url, episode_id)
+        headers = self.geo_verification_headers()
+        webpage = self._download_webpage(url, episode_id, headers=headers)
 
         if '您所在的地区无法观看本片' in webpage:
             raise GeoRestrictedError('This video is restricted')
         elif '正在观看预览，大会员免费看全片' in webpage:
             self.raise_login_required('This video is for premium members only')
 
-        headers = {'Referer': url, **self.geo_verification_headers()}
+        headers['Referer'] = url
         play_info = self._download_json(
             'https://api.bilibili.com/pgc/player/web/v2/playurl', episode_id,
             'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
@@ -724,7 +726,7 @@ def _real_extract(self, url):
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
             'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
             '__post_extractor': self.extract_comments(aid),
-            'http_headers': headers,
+            'http_headers': {'Referer': url},
         }
 
 
@@ -1049,9 +1051,10 @@ def fetch_page(page_idx):
                     raise ExtractorError(
                         'Request is blocked by server (412), please add cookies, wait and try later.', expected=True)
                 raise
-            if response['code'] == -401:
+            if response['code'] in (-352, -401):
                 raise ExtractorError(
-                    'Request is blocked by server (401), please add cookies, wait and try later.', expected=True)
+                    f'Request is blocked by server ({-response["code"]}), '
+                    'please add cookies, wait and try later.', expected=True)
             return response['data']
 
         def get_metadata(page_data):

yt_dlp/extractor/boosty.py

@@ -1,7 +1,11 @@
+import json
+import urllib.parse
+
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..utils import (
     ExtractorError,
+    bug_reports_message,
     int_or_none,
     qualities,
     str_or_none,
@@ -162,9 +166,19 @@ def _extract_formats(self, player_urls, video_id):
 
     def _real_extract(self, url):
         user, post_id = self._match_valid_url(url).group('user', 'post_id')
+
+        auth_headers = {}
+        auth_cookie = self._get_cookies('https://boosty.to/').get('auth')
+        if auth_cookie is not None:
+            try:
+                auth_data = json.loads(urllib.parse.unquote(auth_cookie.value))
+                auth_headers['Authorization'] = f'Bearer {auth_data["accessToken"]}'
+            except (json.JSONDecodeError, KeyError):
+                self.report_warning(f'Failed to extract token from auth cookie{bug_reports_message()}')
+
         post = self._download_json(
             f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
-            note='Downloading post data', errnote='Unable to download post data')
+            note='Downloading post data', errnote='Unable to download post data', headers=auth_headers)
 
         post_title = post.get('title')
         if not post_title:
@@ -202,7 +216,9 @@ def _real_extract(self, url):
                         'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
                     }, get_all=False)})
 
-        if not entries:
+        if not entries and not post.get('hasAccess'):
+            self.raise_login_required('This post requires a subscription', metadata_available=True)
+        elif not entries:
             raise ExtractorError('No videos found', expected=True)
         if len(entries) == 1:
             return entries[0]

yt_dlp/extractor/mixch.py

@@ -1,6 +1,12 @@
 from .common import InfoExtractor
 from ..networking.exceptions import HTTPError
-from ..utils import ExtractorError, UserNotLive, int_or_none, url_or_none
+from ..utils import (
+    ExtractorError,
+    UserNotLive,
+    int_or_none,
+    str_or_none,
+    url_or_none,
+)
 from ..utils.traversal import traverse_obj
 
 
@@ -9,17 +15,20 @@ class MixchIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'https://mixch.tv/u/16236849/live',
+        'url': 'https://mixch.tv/u/16943797/live',
         'skip': 'don\'t know if this live persists',
         'info_dict': {
-            'id': '16236849',
-            'title': '24配信シェア⭕️投票🙏💦',
-            'comment_count': 13145,
-            'view_count': 28348,
-            'timestamp': 1636189377,
-            'uploader': '🦥伊咲👶🏻#フレアワ',
-            'uploader_id': '16236849',
-        }
+            'id': '16943797',
+            'ext': 'mp4',
+            'title': '#EntView #カリナ #セブチ 2024-05-05 06:58',
+            'comment_count': int,
+            'view_count': int,
+            'timestamp': 1714726805,
+            'uploader': 'Ent.View K-news🎶💕',
+            'uploader_id': '16943797',
+            'live_status': 'is_live',
+            'upload_date': '20240503',
+        },
     }, {
         'url': 'https://mixch.tv/u/16137876/live',
         'only_matching': True,
@@ -48,8 +57,20 @@ def _real_extract(self, url):
                 'protocol': 'm3u8',
             }],
             'is_live': True,
+            '__post_extractor': self.extract_comments(video_id),
         }
 
+    def _get_comments(self, video_id):
+        yield from traverse_obj(self._download_json(
+            f'https://mixch.tv/api-web/lives/{video_id}/messages', video_id,
+            note='Downloading comments', errnote='Failed to download comments'), (..., {
+                'author': ('name', {str}),
+                'author_id': ('user_id', {str_or_none}),
+                'id': ('message_id', {str}, {lambda x: x or None}),
+                'text': ('body', {str}),
+                'timestamp': ('created', {int}),
+            }))
+
 
 class MixchArchiveIE(InfoExtractor):
     IE_NAME = 'mixch:archive'

yt_dlp/extractor/tiktok.py

@@ -6,6 +6,14 @@
 import time
 import uuid
 
+from playwright.sync_api import sync_playwright
+
+from base64 import b64encode
+from urllib.parse import urlencode
+
+from Cryptodome.Cipher import AES
+from Cryptodome.Util.Padding import pad
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
 from ..networking import HEADRequest
@@ -791,104 +799,228 @@ def _real_extract(self, url):
         raise ExtractorError(f'Video not available, status code {status}', video_id=video_id)
 
 
-class TikTokUserIE(TikTokBaseIE):
+class TikTokUserIE(TikTokIE):
     IE_NAME = 'tiktok:user'
     _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P<id>[\w\.-]+)/?(?:$|[#?])'
-    _WORKING = False
+    _WORKING = True
     _TESTS = [{
-        'url': 'https://tiktok.com/@corgibobaa?lang=en',
-        'playlist_mincount': 45,
+        'url': 'https://tiktok.com/@therock?lang=en',
+        'playlist_mincount': 25,
         'info_dict': {
-            'id': '6935371178089399301',
-            'title': 'corgibobaa',
-            'thumbnail': r're:https://.+_1080x1080\.webp'
+            'id': '6745191554350760966',
+            'title': 'therock',
+            'thumbnail': r're:https://.+_100x100\.jpeg',
+            'signature': str,
+            'follower_count': int,
+            'verified': True,
+            'private': bool,
+            'following_count': int,
+            'nickname': str,
+            'like_count': int
         },
         'expected_warnings': ['Retrying']
     }, {
-        'url': 'https://www.tiktok.com/@6820838815978423302',
+        'url': 'https://www.tiktok.com/@pokemonlife22',
         'playlist_mincount': 5,
         'info_dict': {
             'id': '6820838815978423302',
-            'title': '6820838815978423302',
-            'thumbnail': r're:https://.+_1080x1080\.webp'
+            'title': 'pokemonlife22',
+            'thumbnail': r're:https://.+_100x100\.jpeg',
+            'signature': str,
+            'follower_count': int,
+            'verified': bool,
+            'private': bool,
+            'following_count': int,
+            'nickname': str,
+            'like_count': int
         },
         'expected_warnings': ['Retrying']
     }, {
         'url': 'https://www.tiktok.com/@meme',
-        'playlist_mincount': 593,
+        'playlist_mincount': 25,
         'info_dict': {
             'id': '79005827461758976',
             'title': 'meme',
-            'thumbnail': r're:https://.+_1080x1080\.webp'
+            'thumbnail': r're:https://.+_100x100\.jpeg',
+            'signature': str,
+            'follower_count': int,
+            'verified': True,
+            'private': bool,
+            'following_count': int,
+            'nickname': str,
+            'like_count': int
         },
         'expected_warnings': ['Retrying']
     }]
 
-    r'''  # TODO: Fix by adding _signature to api_url
-    def _entries(self, webpage, user_id, username):
-        secuid = self._search_regex(r'\"secUid\":\"(?P<secUid>[^\"]+)', webpage, username)
-        verifyfp_cookie = self._get_cookies('https://www.tiktok.com').get('s_v_web_id')
-        if not verifyfp_cookie:
-            raise ExtractorError('Improper cookies (missing s_v_web_id).', expected=True)
-        api_url = f'https://m.tiktok.com/api/post/item_list/?aid=1988&cookie_enabled=true&count=30&verifyFp={verifyfp_cookie.value}&secUid={secuid}&cursor='
-        cursor = '0'
-        for page in itertools.count():
-            data_json = self._download_json(api_url + cursor, username, note='Downloading Page %d' % page)
-            for video in data_json.get('itemList', []):
-                video_id = video['id']
-                video_url = f'https://www.tiktok.com/@{user_id}/video/{video_id}'
-                yield self._url_result(video_url, 'TikTok', video_id, str_or_none(video.get('desc')))
-            if not data_json.get('hasMore'):
-                break
-            cursor = data_json['cursor']
-    '''
-
-    def _video_entries_api(self, webpage, user_id, username):
-        query = {
-            'user_id': user_id,
-            'count': 21,
-            'max_cursor': 0,
-            'min_cursor': 0,
-            'retry_type': 'no_retry',
-            'device_id': ''.join(random.choices(string.digits, k=19)),  # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
+    def _generate_x_tt_params(self, secUid, device_id, cursor):
+        payload = {
+            'aid': '1988',
+            'app_name': 'tiktok_web',
+            'channel': 'tiktok_web',
+            'device_platform': 'web_pc',
+            'device_id': device_id,
+            'region': 'US',
+            'priority_region': '',
+            'os': 'windows',
+            'referer': '',
+            'root_referer': 'undefined',
+            'cookie_enabled': 'true',
+            'screen_width': '1920',
+            'screen_height': '1080',
+            'browser_language': 'en-US',
+            'browser_platform': 'Win32',
+            'browser_name': 'Mozilla',
+            'browser_version': '5.0 (Windows)',
+            'browser_online': 'true',
+            'verifyFp': 'undefined',
+            'app_language': 'en',
+            'webcast_language': 'en',
+            'tz_name': 'America/Chicago',
+            'is_page_visible': 'true',
+            'focus_state': 'false',
+            'is_fullscreen': 'false',
+            'history_len': '2',
+            'from_page': 'user',
+            'secUid': secUid,
+            'count': '30',
+            'cursor': cursor,
+            'language': 'en',
+            'userId': 'undefined',
+            'is_encryption': '1'
         }
+        # https://github.com/davidteather/TikTok-Api/issues/899#issuecomment-1175439842
+        s = urlencode(payload, doseq=True, quote_via=lambda s, *_: s)
+        key = "webapp1.0+202106".encode("utf-8")
+        cipher = AES.new(key, AES.MODE_CBC, key)
+        ct_bytes = cipher.encrypt(pad(s.encode("utf-8"), AES.block_size))
+        return b64encode(ct_bytes).decode("utf-8")
 
-        for page in itertools.count(1):
-            for retry in self.RetryManager():
-                try:
-                    post_list = self._call_api(
-                        'aweme/post', query, username, note=f'Downloading user video list page {page}',
-                        errnote='Unable to download user video list')
-                except ExtractorError as e:
-                    if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
-                        retry.error = e
-                        continue
-                    raise
-            yield from post_list.get('aweme_list', [])
-            if not post_list.get('has_more'):
+    def _video_entries_api(self, user_name, secUid):
+        cursor = '0'
+        videos = []
+        author = []
+        max = self._downloader.params.get('playlistend') or -1
+        device_id = ''.join(random.choices(string.digits, k=19))
+        self.write_debug('Launching headless browser')
+        with sync_playwright() as p:
+            browser = p.firefox.launch(args=['--mute-audio'])
+            page = browser.new_page()
+            page.goto('https://tiktok.com', wait_until='load')
+            time.sleep(2)  # it just works ok
+            for i in itertools.count(1):
+                x_tt_params = self._generate_x_tt_params(secUid, device_id, cursor)
+                self.to_screen(f'Downloading page {i}')
+                self.write_debug(f'x-tt-params: {x_tt_params}')
+                data_json = page.evaluate('([x, d]) => fetch(`https://us.tiktok.com/api/post/item_list/?aid=1988&app_language=en&app_name=tiktok_web&browser_language=en-US&browser_name=Mozilla&browser_online=true&browser_platform=Win32&browser_version=5.0%20%28Windows%29&channel=tiktok_web&cookie_enabled=true&device_id=${d}&device_platform=web_pc&focus_state=true&from_page=user&history_len=2&is_fullscreen=false&is_page_visible=true&os=windows&priority_region=&referer=&region=US&screen_height=1080&screen_width=1920`, { headers: { "x-tt-params": x } }).then(res => res.json())', [x_tt_params, device_id])
+                for video in data_json.get('itemList', []):
+                    video_id = video.get('id', '')
+                    if len(videos) == 0:
+                        author = video.get('author', [])
+                    video_url = f'https://www.tiktok.com/@{user_name}/video/{video_id}'
+                    videos.append(self.url_result(video_url, 'TikTok', video_id, str_or_none(video.get('desc'))))
+                    if max > -1 and len(videos) >= max:
+                        break
+                else:
+                    if not data_json.get('hasMore'):
+                        break
+                    cursor = data_json['cursor']
+                    continue
                 break
-            query['max_cursor'] = post_list['max_cursor']
+            browser.close()
+        return author, videos
 
-    def _entries_api(self, user_id, videos):
+    def _entries_api(self, videos):
         for video in videos:
             yield {
-                **self._parse_aweme_video_app(video),
+                **self._try_extract(video['url'], video['id']),
                 'extractor_key': TikTokIE.ie_key(),
                 'extractor': 'TikTok',
-                'webpage_url': f'https://tiktok.com/@{user_id}/video/{video["aweme_id"]}',
+                'webpage_url': video['url'],
             }
 
+    def _try_extract(self, url, video_id):
+        try:
+            return self._extract_video(url, video_id)
+        except ExtractorError as e:
+            self.report_warning(e)
+            return {}
+
+    def _extract_video(self, url, video_id):
+        try:
+            return self._extract_aweme_app(video_id)
+        except ExtractorError as e:
+            self.report_warning(f'{e}; trying with webpage')
+
+        webpage = self._download_webpage(url, video_id, headers={'User-Agent': 'User-Agent:Mozilla/5.0'})
+        next_data = self._search_nextjs_data(webpage, video_id, default='{}')
+        if next_data:
+            status = traverse_obj(next_data, ('props', 'pageProps', 'statusCode'), expected_type=int) or 0
+            video_data = traverse_obj(next_data, ('props', 'pageProps', 'itemInfo', 'itemStruct'), expected_type=dict)
+        else:
+            sigi_data = self._get_sigi_state(webpage, video_id)
+            status = traverse_obj(sigi_data, ('VideoPage', 'statusCode'), expected_type=int) or 0
+            video_data = traverse_obj(sigi_data, ('ItemModule', video_id), expected_type=dict)
+
+        if status == 0:
+            return self._parse_aweme_video_web(video_data, url)
+        elif status == 10216:
+            raise ExtractorError('This video is private', expected=True)
+        raise ExtractorError('Video not available', video_id=video_id)
+
+    def _get_frontity_state(self, webpage, user_name):
+        return traverse_obj(
+            self._parse_json(self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__FRONTITY_CONNECT_STATE__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'frontity data'), 'frontity data'),
+            ('source', 'data', f'/embed/@{user_name}'))
+
+    def _extract_secUid(self, aweme_id):
+        feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id,
+                                   note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or []
+        aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
+        if not aweme_detail:
+            raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
+        return traverse_obj(aweme_detail, ('author', 'sec_uid'))
+
     def _real_extract(self, url):
         user_name = self._match_id(url)
-        webpage = self._download_webpage(url, user_name, headers={
-            'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
-        })
-        user_id = self._html_search_regex(r'snssdk\d*://user/profile/(\d+)', webpage, 'user ID', default=None) or user_name
+        user_info = []
+        secUid = ''
 
-        videos = LazyList(self._video_entries_api(webpage, user_id, user_name))
-        thumbnail = traverse_obj(videos, (0, 'author', 'avatar_larger', 'url_list', 0))
+        try:
+            webpage = self._download_webpage(f'https://www.tiktok.com/embed/@{user_name}', user_name, note='Downloading user embed')
+            state = self._get_frontity_state(webpage, user_name)
+            user_info = state.get('userInfo')
+            latest_video = next((video for video in state.get('videoList') if len(video.get('playAddr')) > 0), None)
+            if latest_video:
+                latest_video_id = latest_video.get('id')
+                secUid = self._extract_secUid(latest_video_id)
+        except ExtractorError as e:
+            secUid = self._configuration_arg('secuid', [''], ie_key=TikTokIE, casesense=True)[0]
+            if len(secUid) == 0:
+                raise e
+            self.report_warning(f'{e}; secUid supplied, trying anyway')
 
-        return self.playlist_result(self._entries_api(user_id, videos), user_id, user_name, thumbnail=thumbnail)
+        author, response = self._video_entries_api(user_name, secUid)
+        if author.get('uniqueId', '') == user_name:
+            user_info = author
+            user_info['avatarThumbUrl'] = user_info['avatarLarger']
+
+        videos = LazyList(response)
+
+        return self.playlist_result(
+            self._entries_api(videos),
+            user_info.get('id'), user_name,
+            nickname=user_info.get('nickname', user_name),
+            thumbnail=user_info.get('avatarThumbUrl', ''),
+            verified=user_info.get('verified', False),
+            follower_count=user_info.get('followerCount', 0),
+            following_count=user_info.get('followingCount', 0),
+            like_count=user_info.get('heartCount', 0),
+            signature=user_info.get('signature', ''),
+            private=user_info.get('privateAccount', False)
+        )
 
 
 class TikTokBaseListIE(TikTokBaseIE):  # XXX: Conventionally, base classes should end with BaseIE/InfoExtractor

yt_dlp/extractor/tva.py

@@ -1,10 +1,9 @@
+import functools
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    smuggle_url,
-    strip_or_none,
-)
+from ..utils import float_or_none, int_or_none, smuggle_url, strip_or_none
+from ..utils.traversal import traverse_obj
 
 
 class TVAIE(InfoExtractor):
@@ -49,11 +48,20 @@ class QubIE(InfoExtractor):
         'info_dict': {
             'id': '6084352463001',
             'ext': 'mp4',
-            'title': 'Épisode 01',
+            'title': 'Ép 01. Mon dernier jour',
             'uploader_id': '5481942443001',
             'upload_date': '20190907',
             'timestamp': 1567899756,
             'description': 'md5:9c0d7fbb90939420c651fd977df90145',
+            'thumbnail': r're:https://.+\.jpg',
+            'episode': 'Ép 01. Mon dernier jour',
+            'episode_number': 1,
+            'tags': ['alerte amber', 'alerte amber saison 1', 'surdemande'],
+            'duration': 2625.963,
+            'season': 'Season 1',
+            'season_number': 1,
+            'series': 'Alerte Amber',
+            'channel': 'TVA',
         },
     }, {
         'url': 'https://www.qub.ca/tele/video/lcn-ca-vous-regarde-rev-30s-ap369664-1009357943',
@@ -64,22 +72,24 @@ class QubIE(InfoExtractor):
 
     def _real_extract(self, url):
         entity_id = self._match_id(url)
-        entity = self._download_json(
-            'https://www.qub.ca/proxy/pfu/content-delivery-service/v1/entities',
-            entity_id, query={'id': entity_id})
+        webpage = self._download_webpage(url, entity_id)
+        entity = self._search_nextjs_data(webpage, entity_id)['props']['initialProps']['pageProps']['fallbackData']
         video_id = entity['videoId']
         episode = strip_or_none(entity.get('name'))
 
         return {
             '_type': 'url_transparent',
+            'url': f'https://videos.tva.ca/details/_{video_id}',
+            'ie_key': TVAIE.ie_key(),
             'id': video_id,
             'title': episode,
-            # 'url': self.BRIGHTCOVE_URL_TEMPLATE % entity['referenceId'],
-            'url': 'https://videos.tva.ca/details/_' + video_id,
-            'description': entity.get('longDescription'),
-            'duration': float_or_none(entity.get('durationMillis'), 1000),
             'episode': episode,
-            'episode_number': int_or_none(entity.get('episodeNumber')),
-            # 'ie_key': 'BrightcoveNew',
-            'ie_key': TVAIE.ie_key(),
+            **traverse_obj(entity, {
+                'description': ('longDescription', {str}),
+                'duration': ('durationMillis', {functools.partial(float_or_none, scale=1000)}),
+                'channel': ('knownEntities', 'channel', 'name', {str}),
+                'series': ('knownEntities', 'videoShow', 'name', {str}),
+                'season_number': ('slug', {lambda x: re.search(r'/s(?:ai|ea)son-(\d+)/', x)}, 1, {int_or_none}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+            }),
         }

yt_dlp/extractor/vk.py

@@ -451,6 +451,7 @@ def _real_extract(self, url):
             info_page, 'view count', default=None))
 
         formats = []
+        subtitles = {}
         for format_id, format_url in data.items():
             format_url = url_or_none(format_url)
             if not format_url or not format_url.startswith(('http', '//', 'rtmp')):
@@ -462,12 +463,21 @@ def _real_extract(self, url):
                 formats.append({
                     'format_id': format_id,
                     'url': format_url,
+                    'ext': 'mp4',
+                    'source_preference': 1,
                     'height': height,
                 })
             elif format_id == 'hls':
-                formats.extend(self._extract_m3u8_formats(
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     format_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id=format_id, fatal=False, live=is_live))
+                    m3u8_id=format_id, fatal=False, live=is_live)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_id.startswith('dash_'):
+                fmts, subs = self._extract_mpd_formats_and_subtitles(
+                    format_url, video_id, mpd_id=format_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
             elif format_id == 'rtmp':
                 formats.append({
                     'format_id': format_id,
@@ -475,7 +485,6 @@ def _real_extract(self, url):
                     'ext': 'flv',
                 })
 
-        subtitles = {}
         for sub in data.get('subs') or {}:
             subtitles.setdefault(sub.get('lang', 'en'), []).append({
                 'ext': sub.get('title', '.srt').split('.')[-1],
@@ -496,6 +505,7 @@ def _real_extract(self, url):
             'comment_count': int_or_none(mv_data.get('commcount')),
             'is_live': is_live,
             'subtitles': subtitles,
+            '_format_sort_fields': ('res', 'source'),
         }
 
 

yt_dlp/extractor/yandexvideo.py

@@ -259,15 +259,15 @@ def _real_extract(self, url):
             webpage = self._download_webpage(redirect, video_id, note='Redirecting')
         data_json = self._search_json(
             r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
-        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
-                                         webpage, 'server state').replace('State', 'Settings')
+        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
                                       webpage, 'uploader', default='<a>')
         uploader_name = extract_attributes(uploader).get('aria-label')
-        video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
-        stream_urls = try_get(video_json, lambda x: x['video']['streams'])
+        item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
+        video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}
+
         formats, subtitles = [], {}
-        for s_url in stream_urls:
+        for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
             ext = determine_ext(s_url)
             if ext == 'mpd':
                 fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')