Merge 8e977a039e into 6b54cccdcb

Closes #4989
Authored by: alexhuot1, dirkf

yt_dlp/extractor/bilibili.py

@@ -93,11 +93,11 @@ def extract_formats(self, play_info):
 
         return formats
 
-    def _download_playinfo(self, video_id, cid):
+    def _download_playinfo(self, video_id, cid, headers=None):
         return self._download_json(
             'https://api.bilibili.com/x/player/playurl', video_id,
             query={'bvid': video_id, 'cid': cid, 'fnval': 4048},
-            note=f'Downloading video formats for cid {cid}')['data']
+            note=f'Downloading video formats for cid {cid}', headers=headers)['data']
 
     def json2srt(self, json_data):
         srt_data = ''
@@ -493,7 +493,8 @@ class BiliBiliIE(BilibiliBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage, urlh = self._download_webpage_handle(url, video_id)
+        headers = self.geo_verification_headers()
+        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers)
         if not self._match_valid_url(urlh.url):
             return self.url_result(urlh.url)
 
@@ -531,7 +532,7 @@ def _real_extract(self, url):
             self._download_json(
                 'https://api.bilibili.com/x/player/pagelist', video_id,
                 fatal=False, query={'bvid': video_id, 'jsonp': 'jsonp'},
-                note='Extracting videos in anthology'),
+                note='Extracting videos in anthology', headers=headers),
             'data', expected_type=list) or []
         is_anthology = len(page_list_json) > 1
 
@@ -552,7 +553,7 @@ def _real_extract(self, url):
 
         festival_info = {}
         if is_festival:
-            play_info = self._download_playinfo(video_id, cid)
+            play_info = self._download_playinfo(video_id, cid, headers=headers)
 
             festival_info = traverse_obj(initial_state, {
                 'uploader': ('videoInfo', 'upName'),
@@ -666,14 +667,15 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
 
     def _real_extract(self, url):
         episode_id = self._match_id(url)
-        webpage = self._download_webpage(url, episode_id)
+        headers = self.geo_verification_headers()
+        webpage = self._download_webpage(url, episode_id, headers=headers)
 
         if '您所在的地区无法观看本片' in webpage:
             raise GeoRestrictedError('This video is restricted')
         elif '正在观看预览，大会员免费看全片' in webpage:
             self.raise_login_required('This video is for premium members only')
 
-        headers = {'Referer': url, **self.geo_verification_headers()}
+        headers['Referer'] = url
         play_info = self._download_json(
             'https://api.bilibili.com/pgc/player/web/v2/playurl', episode_id,
             'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
@@ -724,7 +726,7 @@ def _real_extract(self, url):
             'duration': float_or_none(play_info.get('timelength'), scale=1000),
             'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
             '__post_extractor': self.extract_comments(aid),
-            'http_headers': headers,
+            'http_headers': {'Referer': url},
         }
 
 
@@ -1049,9 +1051,10 @@ def fetch_page(page_idx):
                     raise ExtractorError(
                         'Request is blocked by server (412), please add cookies, wait and try later.', expected=True)
                 raise
-            if response['code'] == -401:
+            if response['code'] in (-352, -401):
                 raise ExtractorError(
-                    'Request is blocked by server (401), please add cookies, wait and try later.', expected=True)
+                    f'Request is blocked by server ({-response["code"]}), '
+                    'please add cookies, wait and try later.', expected=True)
             return response['data']
 
         def get_metadata(page_data):

yt_dlp/extractor/boosty.py

@@ -1,7 +1,11 @@
+import json
+import urllib.parse
+
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..utils import (
     ExtractorError,
+    bug_reports_message,
     int_or_none,
     qualities,
     str_or_none,
@@ -162,9 +166,19 @@ def _extract_formats(self, player_urls, video_id):
 
     def _real_extract(self, url):
         user, post_id = self._match_valid_url(url).group('user', 'post_id')
+
+        auth_headers = {}
+        auth_cookie = self._get_cookies('https://boosty.to/').get('auth')
+        if auth_cookie is not None:
+            try:
+                auth_data = json.loads(urllib.parse.unquote(auth_cookie.value))
+                auth_headers['Authorization'] = f'Bearer {auth_data["accessToken"]}'
+            except (json.JSONDecodeError, KeyError):
+                self.report_warning(f'Failed to extract token from auth cookie{bug_reports_message()}')
+
         post = self._download_json(
             f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
-            note='Downloading post data', errnote='Unable to download post data')
+            note='Downloading post data', errnote='Unable to download post data', headers=auth_headers)
 
         post_title = post.get('title')
         if not post_title:
@@ -202,7 +216,9 @@ def _real_extract(self, url):
                         'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
                     }, get_all=False)})
 
-        if not entries:
+        if not entries and not post.get('hasAccess'):
+            self.raise_login_required('This post requires a subscription', metadata_available=True)
+        elif not entries:
             raise ExtractorError('No videos found', expected=True)
         if len(entries) == 1:
             return entries[0]

yt_dlp/extractor/mixch.py

@@ -1,6 +1,12 @@
 from .common import InfoExtractor
 from ..networking.exceptions import HTTPError
-from ..utils import ExtractorError, UserNotLive, int_or_none, url_or_none
+from ..utils import (
+    ExtractorError,
+    UserNotLive,
+    int_or_none,
+    str_or_none,
+    url_or_none,
+)
 from ..utils.traversal import traverse_obj
 
 
@@ -9,17 +15,20 @@ class MixchIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'https://mixch.tv/u/16236849/live',
+        'url': 'https://mixch.tv/u/16943797/live',
         'skip': 'don\'t know if this live persists',
         'info_dict': {
-            'id': '16236849',
-            'title': '24配信シェア⭕️投票🙏💦',
-            'comment_count': 13145,
-            'view_count': 28348,
-            'timestamp': 1636189377,
-            'uploader': '🦥伊咲👶🏻#フレアワ',
-            'uploader_id': '16236849',
-        }
+            'id': '16943797',
+            'ext': 'mp4',
+            'title': '#EntView #カリナ #セブチ 2024-05-05 06:58',
+            'comment_count': int,
+            'view_count': int,
+            'timestamp': 1714726805,
+            'uploader': 'Ent.View K-news🎶💕',
+            'uploader_id': '16943797',
+            'live_status': 'is_live',
+            'upload_date': '20240503',
+        },
     }, {
         'url': 'https://mixch.tv/u/16137876/live',
         'only_matching': True,
@@ -48,8 +57,20 @@ def _real_extract(self, url):
                 'protocol': 'm3u8',
             }],
             'is_live': True,
+            '__post_extractor': self.extract_comments(video_id),
         }
 
+    def _get_comments(self, video_id):
+        yield from traverse_obj(self._download_json(
+            f'https://mixch.tv/api-web/lives/{video_id}/messages', video_id,
+            note='Downloading comments', errnote='Failed to download comments'), (..., {
+                'author': ('name', {str}),
+                'author_id': ('user_id', {str_or_none}),
+                'id': ('message_id', {str}, {lambda x: x or None}),
+                'text': ('body', {str}),
+                'timestamp': ('created', {int}),
+            }))
+
 
 class MixchArchiveIE(InfoExtractor):
     IE_NAME = 'mixch:archive'

yt_dlp/extractor/rtp.py

@@ -1,5 +1,11 @@
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    ExtractorError,
+    RegexNotFoundError,
+    determine_ext,
+    join_nonempty,
+    js_to_json,
+)
 import re
 import json
 import urllib.parse
@@ -7,19 +13,72 @@
 
 
 class RTPIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
+    _VALID_URL = r'https?://(?:(?:(?:www\.)?rtp\.pt/play/(?P<subarea>.*/)?p(?P<program_id>[0-9]+)/)|(?:arquivos\.rtp\.pt/conteudos/))(?P<id>[^/?#]+)/?'
     _TESTS = [{
-        'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
-        'md5': 'e736ce0c665e459ddb818546220b4ef8',
+        'url': 'https://www.rtp.pt/play/p9165/e562949/por-do-sol',
         'info_dict': {
-            'id': 'e174042',
-            'ext': 'mp3',
-            'title': 'Paixões Cruzadas',
-            'description': 'As paixões musicais de António Cartaxo e António Macedo',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'id': 'e562949',
+            'ext': 'mp4',
+            'title': 'Pôr do Sol Episódio 1',
+            'description': 'Madalena Bourbon de Linhaça vive atormentada pelo segredo que esconde desde 1990. Matilde Bourbon de Linhaça sonha fugir com o seu amor proibido. O',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
         },
     }, {
-        'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
+        'url': 'https://www.rtp.pt/play/p12646/e738493/telejornal',
+        'info_dict': {
+            'id': 'e738493',
+            'ext': 'mp4',
+            'title': 'Telejornal de 01 jan 2024 PARTE 1',
+            'description': 'A mais rigorosa seleção de notícias, todos os dias às 20h00. De segunda a domingo, João Adelino Faria, José Rodrigues dos Santos e Ana Lourenço',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://www.rtp.pt/play/p6646/e457262/grande-entrevista',
+        'info_dict': {
+            'id': 'e457262',
+            'ext': 'mp4',
+            'title': 'Grande Entrevista Episódio 7 - de 19 fev 2020',
+            'description': 'Bruno Nogueira - É um dos mais originais humoristas portugueses e de maior êxito! Bruno Nogueira na Grande Entrevista com Vítor Gonçalves.',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://www.rtp.pt/play/p8064/e750623/fronteira',
+        'info_dict': {
+            'id': 'e750623',
+            'ext': 'mp4',
+            'title': 'Fronteira de 26 fev 2024',
+            'description': '1970. À aldeia de Fronteira chega um novo chefe de posto da Guarda Fiscal. Com convicções inabaláveis sobre a aplicação da Lei, rapidamente entr',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://www.rtp.pt/play/estudoemcasa/p7776/e539826/portugues-1-ano',
+        'info_dict': {
+            'id': 'e539826',
+            'ext': 'mp4',
+            'title': 'Português - 1.º ano , aula 45 - 27 abr 2021 - Estudo Em Casa - RTP',
+            'description': 'A História do Pedrito Coelho, de Beatrix Potter. O dígrafo \'lh\' - A História do Pedrito Coelho, de Beatrix Potter. O dígrafo \'lh\'.',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://www.rtp.pt/play/zigzag/p11099/e747372/coelhos-corajosos',
+        'info_dict': {
+            'id': 'e747372',
+            'ext': 'mp4',
+            'title': 'Coelhos Corajosos Episódio 1 - de 12 fev 2024 - Zig Zag Play - RTP',
+            'description': 'Boo e o seu irmão mais velho, Bop, vivem grandes aventuras com os seus amigos, e com os seus quatro irmãos pequeninos. Juntos e com muita coragem, e',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://arquivos.rtp.pt/conteudos/liga-dos-ultimos-152/',
+        'info_dict': {
+            'id': 'liga-dos-ultimos-152',
+            'ext': 'mp4',
+            'title': 'Liga dos Últimos – RTP Arquivos',
+            'description': 'Magazine desportivo, com apresentação de Álvaro Costa e comentários em estúdio do professor Hernâni Gonçalves e do sociólogo João Nuno Coelho. Destaque para os jogos de futebol das equipas dos escalões secundários de Portugal, com momentos dos jogos: Agrário de Lamas vs Pampilhoense e Apúlia vs Fragoso.',
+            'thumbnail': r're:^https?://.*\.(jpg|png)'
+        },
+    }, {
+        'url': 'https://www.rtp.pt/play/p510/aleixo-fm',
         'only_matching': True,
     }]
 
@@ -43,42 +102,78 @@ def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_meta(
-            'twitter:title', webpage, display_name='title', fatal=True)
 
-        f, config = self._search_regex(
-            r'''(?sx)
-                var\s+f\s*=\s*(?P<f>".*?"|{[^;]+?});\s*
-                var\s+player1\s+=\s+new\s+RTPPlayer\s*\((?P<config>{(?:(?!\*/).)+?})\);(?!\s*\*/)
-            ''', webpage,
-            'player config', group=('f', 'config'))
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title', default='')
 
-        f = self._parse_json(
-            f, video_id,
-            lambda data: self.__unobfuscate(data, video_id=video_id))
-        config = self._parse_json(
-            config, video_id,
-            lambda data: self.__unobfuscate(data, video_id=video_id))
+        # Raise error if episode is unavailable
+        if 'Este episódio não se encontra disponível' in title:
+            raise ExtractorError('Episode unavailable', expected=True)
+
+        # Replace irrelevant string in title
+        title = re.sub(r' -  ?RTP Play - RTP', '', title)
+
+        # Check if it's a program split in parts
+        part = self._html_search_regex(r'section\-parts.*<span.*>(.+?)</span>.*</ul>', webpage, 'part', default=None)
+
+        # Add program part identification to title if it exists
+        title = join_nonempty(title, part, delim=' ')
+
+        try:
+            # Extract f and config from page
+            f, config = self._search_regex(
+                r'''(?sx)
+                    var\s+f\s*=\s*(?P<f>".*?"|{[^;]+?});\s*
+                    var\s+player1\s+=\s+new\s+RTPPlayer\s*\((?P<config>{(?:(?!\*/).)+?})\);(?!\s*\*/)
+                ''', webpage,
+                'player config', group=('f', 'config'))
+
+            f = self._parse_json(
+                f, video_id,
+                lambda data: self.__unobfuscate(data, video_id=video_id))
+
+            config = self._parse_json(
+                config, video_id,
+                lambda data: self.__unobfuscate(data, video_id=video_id))
+
+            config['file'] = f
+        except RegexNotFoundError:
+            # Estudo em Casa / Zig Zag / RTP Arquivos pages don't include f
+            config = self._search_regex(
+                r'''(?sx)
+                    var\s+player1\s+=\s+new\s+RTPPlayer\s*\((?P<config>{(?:(?!\*/).)+?})\);(?!\s*\*/)
+                ''', webpage,
+                'just player config')
+
+            config = self._parse_json(
+                config, video_id,
+                lambda data: self.__unobfuscate(data, video_id=video_id))
 
         formats = []
-        if isinstance(f, dict):
-            f_hls = f.get('hls')
-            if f_hls is not None:
-                formats.extend(self._extract_m3u8_formats(
-                    f_hls, video_id, 'mp4', 'm3u8_native', m3u8_id='hls'))
+        file = config.get('file')
+        if isinstance(file, dict):
+            file_hls = file.get('hls')
+            file_fps = file.get('fps')
 
-            f_dash = f.get('dash')
-            if f_dash is not None:
-                formats.extend(self._extract_mpd_formats(f_dash, video_id, mpd_id='dash'))
+            if file_hls is None and file_fps is not None:
+                file_hls = file_fps.replace('drm-fps', 'hls')
+
+            formats.extend(self._extract_m3u8_formats(
+                file_hls, video_id, 'mp4', 'm3u8_native', m3u8_id='hls'))
         else:
-            formats.append({
-                'format_id': 'f',
-                'url': f,
-                'vcodec': 'none' if config.get('mediaType') == 'audio' else None,
-            })
+            ext = determine_ext(file)
+
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    file, video_id, 'mp4', 'm3u8_native', m3u8_id='hls'))
+
+            else:
+                formats.append({
+                    'format_id': 'f',
+                    'url': file,
+                    'vcodec': 'none' if config.get('mediaType') == 'audio' else None,
+                })
 
         subtitles = {}
-
         vtt = config.get('vtt')
         if vtt is not None:
             for lcode, lname, url in vtt:

yt_dlp/extractor/tva.py

@@ -1,10 +1,9 @@
+import functools
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    smuggle_url,
-    strip_or_none,
-)
+from ..utils import float_or_none, int_or_none, smuggle_url, strip_or_none
+from ..utils.traversal import traverse_obj
 
 
 class TVAIE(InfoExtractor):
@@ -49,11 +48,20 @@ class QubIE(InfoExtractor):
         'info_dict': {
             'id': '6084352463001',
             'ext': 'mp4',
-            'title': 'Épisode 01',
+            'title': 'Ép 01. Mon dernier jour',
             'uploader_id': '5481942443001',
             'upload_date': '20190907',
             'timestamp': 1567899756,
             'description': 'md5:9c0d7fbb90939420c651fd977df90145',
+            'thumbnail': r're:https://.+\.jpg',
+            'episode': 'Ép 01. Mon dernier jour',
+            'episode_number': 1,
+            'tags': ['alerte amber', 'alerte amber saison 1', 'surdemande'],
+            'duration': 2625.963,
+            'season': 'Season 1',
+            'season_number': 1,
+            'series': 'Alerte Amber',
+            'channel': 'TVA',
         },
     }, {
         'url': 'https://www.qub.ca/tele/video/lcn-ca-vous-regarde-rev-30s-ap369664-1009357943',
@@ -64,22 +72,24 @@ class QubIE(InfoExtractor):
 
     def _real_extract(self, url):
         entity_id = self._match_id(url)
-        entity = self._download_json(
-            'https://www.qub.ca/proxy/pfu/content-delivery-service/v1/entities',
-            entity_id, query={'id': entity_id})
+        webpage = self._download_webpage(url, entity_id)
+        entity = self._search_nextjs_data(webpage, entity_id)['props']['initialProps']['pageProps']['fallbackData']
         video_id = entity['videoId']
         episode = strip_or_none(entity.get('name'))
 
         return {
             '_type': 'url_transparent',
+            'url': f'https://videos.tva.ca/details/_{video_id}',
+            'ie_key': TVAIE.ie_key(),
             'id': video_id,
             'title': episode,
-            # 'url': self.BRIGHTCOVE_URL_TEMPLATE % entity['referenceId'],
-            'url': 'https://videos.tva.ca/details/_' + video_id,
-            'description': entity.get('longDescription'),
-            'duration': float_or_none(entity.get('durationMillis'), 1000),
             'episode': episode,
-            'episode_number': int_or_none(entity.get('episodeNumber')),
-            # 'ie_key': 'BrightcoveNew',
-            'ie_key': TVAIE.ie_key(),
+            **traverse_obj(entity, {
+                'description': ('longDescription', {str}),
+                'duration': ('durationMillis', {functools.partial(float_or_none, scale=1000)}),
+                'channel': ('knownEntities', 'channel', 'name', {str}),
+                'series': ('knownEntities', 'videoShow', 'name', {str}),
+                'season_number': ('slug', {lambda x: re.search(r'/s(?:ai|ea)son-(\d+)/', x)}, 1, {int_or_none}),
+                'episode_number': ('episodeNumber', {int_or_none}),
+            }),
         }

yt_dlp/extractor/vk.py

@@ -451,6 +451,7 @@ def _real_extract(self, url):
             info_page, 'view count', default=None))
 
         formats = []
+        subtitles = {}
         for format_id, format_url in data.items():
             format_url = url_or_none(format_url)
             if not format_url or not format_url.startswith(('http', '//', 'rtmp')):
@@ -462,12 +463,21 @@ def _real_extract(self, url):
                 formats.append({
                     'format_id': format_id,
                     'url': format_url,
+                    'ext': 'mp4',
+                    'source_preference': 1,
                     'height': height,
                 })
             elif format_id == 'hls':
-                formats.extend(self._extract_m3u8_formats(
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
                     format_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id=format_id, fatal=False, live=is_live))
+                    m3u8_id=format_id, fatal=False, live=is_live)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            elif format_id.startswith('dash_'):
+                fmts, subs = self._extract_mpd_formats_and_subtitles(
+                    format_url, video_id, mpd_id=format_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
             elif format_id == 'rtmp':
                 formats.append({
                     'format_id': format_id,
@@ -475,7 +485,6 @@ def _real_extract(self, url):
                     'ext': 'flv',
                 })
 
-        subtitles = {}
         for sub in data.get('subs') or {}:
             subtitles.setdefault(sub.get('lang', 'en'), []).append({
                 'ext': sub.get('title', '.srt').split('.')[-1],
@@ -496,6 +505,7 @@ def _real_extract(self, url):
             'comment_count': int_or_none(mv_data.get('commcount')),
             'is_live': is_live,
             'subtitles': subtitles,
+            '_format_sort_fields': ('res', 'source'),
         }
 
 

yt_dlp/extractor/yandexvideo.py

@@ -259,15 +259,15 @@ def _real_extract(self, url):
             webpage = self._download_webpage(redirect, video_id, note='Redirecting')
         data_json = self._search_json(
             r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
-        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
-                                         webpage, 'server state').replace('State', 'Settings')
+        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
                                       webpage, 'uploader', default='<a>')
         uploader_name = extract_attributes(uploader).get('aria-label')
-        video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
-        stream_urls = try_get(video_json, lambda x: x['video']['streams'])
+        item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
+        video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}
+
         formats, subtitles = [], {}
-        for s_url in stream_urls:
+        for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
             ext = determine_ext(s_url)
             if ext == 'mpd':
                 fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')