yt_dlp/extractor/boosty.py

@@ -19,11 +19,8 @@ class BoostyIE(InfoExtractor):
             'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
             'title': 'phasma_3',
             'channel': 'Kuplinov',
-            'channel_id': '7958701',
+            'timestamp': 1655049000,
-            'timestamp': 1655031975,
             'upload_date': '20220612',
-            'release_timestamp': 1655049000,
-            'release_date': '20220612',
             'modified_timestamp': 1668680993,
             'modified_date': '20221117',
             'tags': ['куплинов', 'phasmophobia'],
@@ -40,13 +37,11 @@ class BoostyIE(InfoExtractor):
             'id': '0c652798-3b35-471f-8b48-a76a0b28736f',
             'title': 'то что не пропустил юта6',
             'channel': 'Илья Давыдов',
-            'channel_id': '6808257',
             'timestamp': 1694017040,
             'upload_date': '20230906',
-            'release_timestamp': 1694017040,
-            'release_date': '20230906',
             'modified_timestamp': 1694071178,
             'modified_date': '20230907',
+            'tags': [],
             'like_count': int,
         },
         'playlist_count': 3,
@@ -54,15 +49,6 @@ class BoostyIE(InfoExtractor):
             'info_dict': {
                 'id': 'cc325a9f-a563-41c6-bf47-516c1b506c9a',
                 'title': 'то что не пропустил юта6',
-                'channel': 'Илья Давыдов',
-                'channel_id': '6808257',
-                'timestamp': 1694017040,
-                'upload_date': '20230906',
-                'release_timestamp': 1694017040,
-                'release_date': '20230906',
-                'modified_timestamp': 1694071178,
-                'modified_date': '20230907',
-                'like_count': int,
                 'ext': 'mp4',
                 'duration': 31204,
                 'view_count': int,
@@ -72,15 +58,6 @@ class BoostyIE(InfoExtractor):
             'info_dict': {
                 'id': 'd07b0a72-9493-4512-b54e-55ce468fd4b7',
                 'title': 'то что не пропустил юта6',
-                'channel': 'Илья Давыдов',
-                'channel_id': '6808257',
-                'timestamp': 1694017040,
-                'upload_date': '20230906',
-                'release_timestamp': 1694017040,
-                'release_date': '20230906',
-                'modified_timestamp': 1694071178,
-                'modified_date': '20230907',
-                'like_count': int,
                 'ext': 'mp4',
                 'duration': 25704,
                 'view_count': int,
@@ -90,15 +67,6 @@ class BoostyIE(InfoExtractor):
             'info_dict': {
                 'id': '4a3bba32-78c8-422a-9432-2791aff60b42',
                 'title': 'то что не пропустил юта6',
-                'channel': 'Илья Давыдов',
-                'channel_id': '6808257',
-                'timestamp': 1694017040,
-                'upload_date': '20230906',
-                'release_timestamp': 1694017040,
-                'release_date': '20230906',
-                'modified_timestamp': 1694071178,
-                'modified_date': '20230907',
-                'like_count': int,
                 'ext': 'mp4',
                 'duration': 31867,
                 'view_count': int,
@@ -110,15 +78,19 @@ class BoostyIE(InfoExtractor):
         'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
         'info_dict': {
             'id': 'EXelTnve5lY',
-            'title': 'Послание Президента Федеральному Собранию | Класс народа',
+            'title': '4Класс',
-            'upload_date': '20210425',
             'channel': 'Денис Чужой',
-            'tags': 'count:10',
+            'timestamp': 1619380873,
+            'upload_date': '20210425',
+            'modified_timestamp': 1653321155,
+            'modified_date': '20220523',
+            'tags': [],
             'like_count': int,
             'ext': 'mp4',
             'duration': 816,
             'view_count': int,
             'thumbnail': r're:^https://i\.ytimg\.com/',
+            # youtube fields
             'age_limit': 0,
             'availability': 'public',
             'categories': list,
@@ -143,67 +115,105 @@ class BoostyIE(InfoExtractor):
             f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
             note='Downloading post data', errnote='Unable to download post data')
 
-        post_title = post.get('title')
+        post_title = self._extract_title(post, post_id, url)
-        if not post_title:
+        entries = self._extract_entries(post, post_id, post_title)
-            self.report_warning('Unable to extract post title. Falling back to parsing html page')
-            webpage = self._download_webpage(url, video_id=post_id)
-            post_title = (self._og_search_title(webpage, fatal=False)
-                          or self._html_extract_title(webpage, fatal=True))
-
-        common_metadata = {
-            'title': post_title,
-            **traverse_obj(post, {
-                'channel': ('user', 'name', {str}),
-                'channel_id': ('user', 'id', {str_or_none}),
-                'timestamp': ('createdAt', {int_or_none}),
-                'release_timestamp': ('publishTime', {int_or_none}),
-                'modified_timestamp': ('updatedAt', {int_or_none}),
-                'tags': ('tags', ..., 'title', {str}),
-                'like_count': ('count', 'likes', {int_or_none}),
-            }),
-        }
-        entries = []
-        for item in traverse_obj(post, ('data', ..., {dict})):
-            item_type = item.get('type')
-            if item_type == 'video' and url_or_none(item.get('url')):
-                entries.append(self.url_result(item['url'], YoutubeIE))
-            elif item_type == 'ok_video':
-                video_id = item.get('id') or post_id
-                entries.append({
-                    'id': video_id,
-                    'formats': self._extract_formats(item.get('playerUrls'), video_id),
-                    **common_metadata,
-                    **traverse_obj(item, {
-                        'title': ('title', {str}),
-                        'duration': ('duration', {int_or_none}),
-                        'view_count': ('viewsCounter', {int_or_none}),
-                        'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
-                    }, get_all=False)})
-
         if not entries:
-            raise ExtractorError('No videos found', expected=True)
+            raise ExtractorError(
+                'no video found', video_id=post_id, expected=True)
+        result = {
+            'id': post_id,
+            'title': post_title,
+            'channel': traverse_obj(post, ('user', 'name')),
+            'timestamp': dict_get(post, ('publishTime', 'createdAt')),
+            'modified_timestamp': post.get('updatedAt'),
+            'tags': [tag['title'] for tag in post.get('tags', [])],
+            'like_count': traverse_obj(post, ('count', 'likes')),
+        }
         if len(entries) == 1:
-            return entries[0]
+            result.update(entries[0])
-        return self.playlist_result(entries, post_id, post_title, **common_metadata)
+        else:
+            result['_type'] = 'playlist'
+            result['entries'] = entries
+        return result
 
-    def _extract_formats(self, player_urls, video_id):
+    def _extract_title(self, post, post_id, url):
+        title = post.get('title')
+        if title:
+            return title
+        # falling back to parsing html page as a last resort
+        webpage = self._download_webpage(url, video_id=post_id)
+        return (
+            self._og_search_title(webpage, fatal=False)
+            or self._html_extract_title(webpage, fatal=True))
+
+    def _extract_entries(self, post, post_id, post_title):
+        entries = []
+        for item in post['data']:
+            url = None
+            formats = None
+            item_id = item.get('id', post_id)
+            item_type = item.get('type')
+            if item_type == 'video':
+                url = item.get('url')
+            elif item_type == 'ok_video':
+                formats = self._extract_formats(
+                    item.get('playerUrls', []), item_id)
+            if not url and not formats:
+                continue
+            entry = {
+                'id': item_id,
+                'title': item.get('title', post_title),
+                'duration': item.get('duration'),
+                'view_count': item.get('viewsCounter'),
+                'thumbnail': dict_get(item, ('preview', 'defaultPreview')),
+            }
+            if url:
+                entry['_type'] = 'url_transparent'
+                entry['url'] = url
+            elif formats:
+                entry['formats'] = formats
+            entries.append(entry)
+        return entries
+
+    def _extract_formats(self, player_urls, post_id):
         formats = []
-        mp4_types = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
+        for player_url in player_urls:
-        quality = qualities(mp4_types)
+            url = player_url.get('url')
-        for player_url in traverse_obj(player_urls, lambda _, v: url_or_none(v['url'])):
+            if not url:
-            url = player_url['url']
+                continue
-            format_type = player_url.get('type')
+            _type = player_url.get('type')
-            if format_type in ('hls', 'hls_live', 'live_ondemand_hls', 'live_playback_hls'):
+            if 'hls' in _type:
-                formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id='hls', fatal=False))
+                # 'hls', 'hls_live'
-            elif format_type in ('dash', 'dash_live', 'live_playback_dash'):
+                formats.extend(self._extract_m3u8_formats(
-                formats.extend(self._extract_mpd_formats(url, video_id, mpd_id='dash', fatal=False))
+                    url, video_id=post_id, fatal=False))
-            elif format_type in mp4_types:
+            elif 'dash' in _type:
+                # 'dash', 'dash_live'
+                formats.extend(self._extract_mpd_formats(
+                    url, video_id=post_id, fatal=False))
+            else:
+                # one of: 'tiny', 'low', etc.; see _get_quality for full list
                 formats.append({
                     'url': url,
                     'ext': 'mp4',
-                    'format_id': format_type,
+                    'format_id': _type,
-                    'quality': quality(format_type),
                 })
-            else:
+        for format in formats:
-                self.report_warning(f'Unknown format type: {format_type!r}')
+            # .../type/<d>/... or ...&type=<d>&...
+            format_type = self._search_regex(
+                r'\btype[/=](\d)([^\d]|$)', format['url'],
+                'format type', default=None,
+            )
+            if format_type:
+                format['quality'] = self._get_quality(format_type)
         return formats
+
+    _get_quality = staticmethod(qualities((
+        '4',  # tiny = 144p
+        '0',  # lowest = 240p
+        '1',  # low = 360p
+        '2',  # medium = 480p
+        '3',  # high = 720p
+        '5',  # full_hd = 1080p
+        '6',  # quad_hd = 1440p
+        '7',  # ultra_hd = 2160p
+    )))