Merge 8631ff86d9 into f2a4983df7

Closes #11526
Authored by: jshumphrey

yt_dlp/extractor/archiveorg.py

@@ -205,6 +205,26 @@ class ArchiveOrgIE(InfoExtractor):
                 },
             },
         ],
+    }, {
+        # The reviewbody is None for one of the reviews; just need to extract data without crashing
+        'url': 'https://archive.org/details/gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
+        'info_dict': {
+            'id': 'gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
+            'ext': 'mp3',
+            'title': 'Stuck Inside of Mobile with the Memphis Blues Again',
+            'creators': ['Grateful Dead'],
+            'duration': 338.31,
+            'track': 'Stuck Inside of Mobile with the Memphis Blues Again',
+            'description': 'md5:764348a470b986f1217ffd38d6ac7b72',
+            'display_id': 'gd95-04-02d1t04.shn',
+            'location': 'Pyramid Arena',
+            'uploader': 'jon@archive.org',
+            'album': '1995-04-02 - Pyramid Arena',
+            'upload_date': '20040519',
+            'track_number': 4,
+            'release_date': '19950402',
+            'timestamp': 1084927901,
+        },
     }]
 
     @staticmethod
@@ -335,7 +355,7 @@ class ArchiveOrgIE(InfoExtractor):
                 info['comments'].append({
                     'id': review.get('review_id'),
                     'author': review.get('reviewer'),
-                    'text': str_or_none(review.get('reviewtitle'), '') + '\n\n' + review.get('reviewbody'),
+                    'text': join_nonempty('reviewtitle', 'reviewbody', from_dict=review, delim='\n\n'),
                     'timestamp': unified_timestamp(review.get('createdate')),
                     'parent': 'root'})
 

yt_dlp/extractor/bilibili.py

@@ -33,6 +33,7 @@ from ..utils import (
     parse_qs,
     parse_resolution,
     qualities,
+    sanitize_url,
     smuggle_url,
     srt_subtitles_timecode,
     str_or_none,
@@ -41,6 +42,7 @@ from ..utils import (
     unsmuggle_url,
     url_or_none,
     urlencode_postdata,
+    value,
     variadic,
 )
 
@@ -66,6 +68,84 @@ class BilibiliBaseIE(InfoExtractor):
                 f'Format(s) {missing_formats} are missing; you have to login or '
                 f'become a premium member to download them. {self._login_hint()}')
 
+    def _extract_heatmap(self, cid):
+        heatmap_json = self._download_json(
+            'https://bvc.bilivideo.com/pbp/data', cid,
+            note='Downloading heatmap', errnote='Failed to download heatmap', fatal=False,
+            query={'cid': cid})
+        if not isinstance(heatmap_json, dict):
+            return
+        duration = self._parse_json(heatmap_json['debug'], cid).get('max_time')
+        step_sec = traverse_obj(heatmap_json, ('step_sec', {int}))
+        heatmap_data = traverse_obj(heatmap_json, ('events', 'default', {list}))
+        if not step_sec or not heatmap_data:
+            return
+        peak = max(heatmap_data)
+        if not peak:
+            return
+
+        for idx, heatmap_entry in enumerate(heatmap_data):
+            start_time = idx * step_sec
+            end_time = start_time + step_sec
+            if duration and end_time >= duration:
+                yield {
+                    'start_time': start_time,
+                    'end_time': duration,
+                    'value': heatmap_entry / peak,
+                }
+                break
+            yield {
+                'start_time': start_time,
+                'end_time': end_time,
+                'value': heatmap_entry / peak,
+            }
+
+    def _extract_storyboard(self, duration, aid=None, bvid=None, cid=None):
+        if not (video_id := aid or bvid) or not duration:
+            return
+        if storyboard_info := traverse_obj(self._download_json(
+                'https://api.bilibili.com/x/player/videoshot', video_id,
+                note='Downloading storyboard info', errnote='Failed to download storyboard info',
+                query=filter_dict({
+                    'index': 1,
+                    'aid': aid,
+                    'bvid': bvid,
+                    'cid': cid,
+                })), ('data', {lambda v: v if v.get('image') and v.get('index') else None})):
+            rows, cols = storyboard_info.get('img_x_len'), storyboard_info.get('img_y_len')
+            fragments = []
+            last_duration = 0.0
+            for i, url in enumerate(storyboard_info['image'], start=1):
+                if not rows or not cols:
+                    fragments.append({'url': sanitize_url(url)})
+                    continue
+                elif (duration_index := i * rows * cols - 1) < len(storyboard_info['index']) - 1:
+                    current_duration = traverse_obj(storyboard_info, ('index', duration_index))
+                else:
+                    current_duration = duration
+                if not current_duration or current_duration <= last_duration or current_duration > duration:
+                    break
+                fragments.append({
+                    'url': sanitize_url(url),
+                    'duration': current_duration - last_duration if current_duration is not None else None,
+                })
+            if fragments:
+                return {
+                    'format_id': 'sb',
+                    'format_note': 'storyboard',
+                    'ext': 'mhtml',
+                    'protocol': 'mhtml',
+                    'acodec': 'none',
+                    'vcodec': 'none',
+                    'url': 'about:invalid',
+                    'width': storyboard_info.get('img_x_size'),
+                    'height': storyboard_info.get('img_y_size'),
+                    'fps': len(storyboard_info['image']) * rows * cols / duration if rows and cols else None,
+                    'rows': rows,
+                    'columns': cols,
+                    'fragments': fragments,
+                }
+
     def extract_formats(self, play_info):
         format_names = {
             r['quality']: traverse_obj(r, 'new_description', 'display_desc')
@@ -287,14 +367,21 @@ class BilibiliBaseIE(InfoExtractor):
         cid_edges = self._get_divisions(video_id, graph_version, {1: {'cid': cid}}, 1)
         for cid, edges in cid_edges.items():
             play_info = self._download_playinfo(video_id, cid, headers=headers)
+            formats = self.extract_formats(play_info)
+            duration = float_or_none(play_info.get('timelength'), scale=1000)
+            if storyboard_format := self._extract_storyboard(
+                    duration=duration,
+                    bvid=video_id, cid=cid):
+                formats.append(storyboard_format)
             yield {
                 **metainfo,
                 'id': f'{video_id}_{cid}',
                 'title': f'{metainfo.get("title")} - {next(iter(edges.values())).get("title")}',
-                'formats': self.extract_formats(play_info),
+                'formats': formats,
                 'description': f'{json.dumps(edges, ensure_ascii=False)}\n{metainfo.get("description", "")}',
-                'duration': float_or_none(play_info.get('timelength'), scale=1000),
+                'duration': duration,
                 'subtitles': self.extract_subtitles(video_id, cid),
+                'heatmap': list(self._extract_heatmap(cid)),
             }
 
 
@@ -310,7 +397,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'description': '滴妹今天唱Closer給你聽! 有史以来，被推最多次也是最久的歌曲，其实歌词跟我原本想像差蛮多的，不过还是好听！ 微博@阿滴英文',
             'uploader_id': '65880958',
             'uploader': '阿滴英文',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             'duration': 554.117,
             'tags': list,
             'comment_count': int,
@@ -319,6 +406,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'like_count': int,
             'view_count': int,
             '_old_archive_ids': ['bilibili 8903802_part1'],
+            'heatmap': [],
         },
     }, {
         'note': 'old av URL version',
@@ -337,8 +425,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'comment_count': int,
             'view_count': int,
             'tags': list,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 1074402_part1'],
+            'heatmap': [],
         },
         'params': {'skip_download': True},
     }, {
@@ -356,7 +445,7 @@ class BiliBiliIE(BilibiliBaseIE):
                 'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
                 'tags': 'count:10',
                 'timestamp': 1589601697,
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
                 'uploader': '打牌还是打桩',
                 'uploader_id': '150259984',
                 'like_count': int,
@@ -366,6 +455,7 @@ class BiliBiliIE(BilibiliBaseIE):
                 'description': 'md5:e3c401cf7bc363118d1783dd74068a68',
                 'duration': 90.314,
                 '_old_archive_ids': ['bilibili 498159642_part1'],
+                'heatmap': list,
             },
         }],
     }, {
@@ -377,7 +467,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'title': '物语中的人物是如何吐槽自己的OP的 p01 Staple Stable/战场原+羽川',
             'tags': 'count:10',
             'timestamp': 1589601697,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             'uploader': '打牌还是打桩',
             'uploader_id': '150259984',
             'like_count': int,
@@ -387,6 +477,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'description': 'md5:e3c401cf7bc363118d1783dd74068a68',
             'duration': 90.314,
             '_old_archive_ids': ['bilibili 498159642_part1'],
+            'heatmap': list,
         },
     }, {
         'url': 'https://www.bilibili.com/video/av8903802/',
@@ -399,13 +490,14 @@ class BiliBiliIE(BilibiliBaseIE):
             'timestamp': 1488353834,
             'uploader_id': '65880958',
             'uploader': '阿滴英文',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             'duration': 554.117,
             'tags': list,
             'comment_count': int,
             'view_count': int,
             'like_count': int,
             '_old_archive_ids': ['bilibili 8903802_part1'],
+            'heatmap': [],
         },
         'params': {
             'skip_download': True,
@@ -428,8 +520,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'comment_count': int,
             'view_count': int,
             'like_count': int,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 463665680_part1'],
+            'heatmap': list,
         },
         'params': {'skip_download': True},
     }, {
@@ -447,8 +540,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'uploader_id': '528182630',
             'view_count': int,
             'like_count': int,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 893839363_part1'],
+            'heatmap': [],
         },
     }, {
         'note': 'newer festival video',
@@ -465,8 +559,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'uploader_id': '8469526',
             'view_count': int,
             'like_count': int,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 778246196_part1'],
+            'heatmap': list,
         },
     }, {
         'note': 'legacy flv/mp4 video',
@@ -484,8 +579,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'comment_count': int,
             'like_count': int,
             'tags': list,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 4120229_part4'],
+            'heatmap': [],
         },
         'params': {'extractor_args': {'bilibili': {'prefer_multi_flv': ['32']}}},
         'playlist_count': 19,
@@ -514,8 +610,9 @@ class BiliBiliIE(BilibiliBaseIE):
             'view_count': int,
             'like_count': int,
             'tags': list,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 15700301_part1'],
+            'heatmap': [],
         },
     }, {
         'note': 'interactive/split-path video',
@@ -533,7 +630,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'comment_count': int,
             'view_count': int,
             'like_count': int,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             '_old_archive_ids': ['bilibili 292734508_part1'],
         },
         'playlist_count': 33,
@@ -552,10 +649,33 @@ class BiliBiliIE(BilibiliBaseIE):
                 'comment_count': int,
                 'view_count': int,
                 'like_count': int,
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
                 '_old_archive_ids': ['bilibili 292734508_part1'],
+                'heatmap': [],
             },
         }],
+    }, {
+        'note': 'storyboard',
+        'url': 'https://www.bilibili.com/video/av170001/',
+        'info_dict': {
+            'id': 'BV17x411w7KC_p1',
+            'title': '【MV】保加利亚妖王AZIS视频合辑 p01 Хоп',
+            'ext': 'mhtml',
+            'upload_date': '20111109',
+            'uploader_id': '122541',
+            'view_count': int,
+            '_old_archive_ids': ['bilibili 170001_part1'],
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+            'uploader': '冰封.虾子',
+            'timestamp': 1320850533,
+            'comment_count': int,
+            'tags': ['Hop', '保加利亚妖王', '保加利亚', 'Азис', 'azis', 'mv'],
+            'description': 'md5:acfd7360b96547f031f7ebead9e66d9e',
+            'like_count': int,
+            'duration': 199.4,
+            'heatmap': list,
+        },
+        'params': {'format': 'sb', 'playlist_items': '1'},
     }, {
         'note': '301 redirect to bangumi link',
         'url': 'https://www.bilibili.com/video/BV1TE411f7f1',
@@ -574,7 +694,8 @@ class BiliBiliIE(BilibiliBaseIE):
             'duration': 1183.957,
             'timestamp': 1571648124,
             'upload_date': '20191021',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+            'heatmap': [],
         },
     }, {
         'note': 'video has subtitles, which requires login',
@@ -593,7 +714,7 @@ class BiliBiliIE(BilibiliBaseIE):
             'comment_count': int,
             'view_count': int,
             'like_count': int,
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             'subtitles': 'count:2',  # login required for CC subtitle
             '_old_archive_ids': ['bilibili 898179753_part1'],
         },
@@ -729,6 +850,9 @@ class BiliBiliIE(BilibiliBaseIE):
                 __post_extractor=self.extract_comments(aid))
         else:
             formats = self.extract_formats(play_info)
+            formats.append(self._extract_storyboard(
+                duration=float_or_none(play_info.get('timelength'), scale=1000),
+                bvid=video_id, cid=cid))
 
             if not traverse_obj(play_info, ('dash')):
                 # we only have legacy formats and need additional work
@@ -773,6 +897,7 @@ class BiliBiliIE(BilibiliBaseIE):
                         '__post_extractor': self.extract_comments(aid) if idx == 0 else None,
                     } for idx, fragment in enumerate(formats[0]['fragments'])],
                     'duration': float_or_none(play_info.get('timelength'), scale=1000),
+                    'heatmap': list(self._extract_heatmap(cid)),
                 }
             else:
                 return {
@@ -782,6 +907,7 @@ class BiliBiliIE(BilibiliBaseIE):
                     'chapters': self._get_chapters(aid, cid),
                     'subtitles': self.extract_subtitles(video_id, cid),
                     '__post_extractor': self.extract_comments(aid),
+                    'heatmap': list(self._extract_heatmap(cid)),
                 }
 
 
@@ -805,7 +931,8 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'duration': 1420.791,
             'timestamp': 1320412200,
             'upload_date': '20111104',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+            'heatmap': list,
         },
     }, {
         'url': 'https://www.bilibili.com/bangumi/play/ep267851',
@@ -824,7 +951,7 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'duration': 1425.256,
             'timestamp': 1554566400,
             'upload_date': '20190406',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
         },
         'skip': 'Geo-restricted',
     }, {
@@ -845,7 +972,8 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'duration': 1922.129,
             'timestamp': 1602853860,
             'upload_date': '20201016',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+            'heatmap': list,
         },
     }]
 
@@ -865,6 +993,8 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'Extracting episode', query={'fnval': '4048', 'ep_id': episode_id},
             headers=headers)
         premium_only = play_info.get('code') == -10403
+        episode_info = traverse_obj(play_info, ('result', 'play_view_business_info', 'episode_info'))
+        aid, cid = episode_info.get('aid'), episode_info.get('cid')
         play_info = traverse_obj(play_info, ('result', 'video_info', {dict})) or {}
 
         formats = self.extract_formats(play_info)
@@ -878,7 +1008,7 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
         episode_number, episode_info = next((
             (idx, ep) for idx, ep in enumerate(traverse_obj(
                 bangumi_info, (('episodes', ('section', ..., 'episodes')), ..., {dict})), 1)
-            if str_or_none(ep.get('id')) == episode_id), (1, {}))
+            if str_or_none(ep.get('id')) == episode_id), (1, episode_info))
 
         season_id = bangumi_info.get('season_id')
         season_number, season_title = season_id and next((
@@ -887,8 +1017,10 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             if e.get('season_id') == season_id
         ), (None, None))
 
-        aid = episode_info.get('aid')
-
+        aid, cid = episode_info.get('aid', aid), episode_info.get('cid', cid)
+        duration = float_or_none(play_info.get('timelength'), scale=1000)
+        if storyboard_format := self._extract_storyboard(duration=duration, aid=aid, cid=cid):
+            formats.append(storyboard_format)
         return {
             'id': episode_id,
             'formats': formats,
@@ -907,10 +1039,11 @@ class BiliBiliBangumiIE(BilibiliBaseIE):
             'season': str_or_none(season_title),
             'season_id': str_or_none(season_id),
             'season_number': season_number,
-            'duration': float_or_none(play_info.get('timelength'), scale=1000),
-            'subtitles': self.extract_subtitles(episode_id, episode_info.get('cid'), aid=aid),
+            'duration': duration,
+            'subtitles': self.extract_subtitles(episode_id, cid, aid=aid),
             '__post_extractor': self.extract_comments(aid),
             'http_headers': {'Referer': url},
+            'heatmap': list(self._extract_heatmap(cid)),
         }
 
 
@@ -948,7 +1081,8 @@ class BiliBiliBangumiMediaIE(BilibiliBaseIE):
                 'duration': 1525.777,
                 'timestamp': 1425074413,
                 'upload_date': '20150227',
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+                'heatmap': list,
             },
         }],
     }]
@@ -1003,7 +1137,8 @@ class BiliBiliBangumiSeasonIE(BilibiliBaseIE):
                 'duration': 1436.992,
                 'timestamp': 1343185080,
                 'upload_date': '20120725',
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
+                'heatmap': list,
             },
         }],
     }]
@@ -1037,10 +1172,14 @@ class BilibiliCheeseBaseIE(BilibiliBaseIE):
             query={'avid': aid, 'cid': cid, 'ep_id': ep_id, 'fnval': 16, 'fourk': 1},
             headers=self._HEADERS, note='Downloading playinfo')['data']
 
+        formats = self.extract_formats(play_info)
+        duration = traverse_obj(episode_info, ('duration', {int_or_none}))
+        if storyboard_format := self._extract_storyboard(duration=duration, aid=aid, cid=cid):
+            formats.append(storyboard_format)
         return {
             'id': str_or_none(ep_id),
             'episode_id': str_or_none(ep_id),
-            'formats': self.extract_formats(play_info),
+            'formats': formats,
             'extractor_key': BilibiliCheeseIE.ie_key(),
             'extractor': BilibiliCheeseIE.IE_NAME,
             'webpage_url': f'https://www.bilibili.com/cheese/play/ep{ep_id}',
@@ -1048,7 +1187,7 @@ class BilibiliCheeseBaseIE(BilibiliBaseIE):
                 'episode': ('title', {str}),
                 'title': {lambda v: v and join_nonempty('index', 'title', delim=' - ', from_dict=v)},
                 'alt_title': ('subtitle', {str}),
-                'duration': ('duration', {int_or_none}),
+                'duration': {value(duration)},
                 'episode_number': ('index', {int_or_none}),
                 'thumbnail': ('cover', {url_or_none}),
                 'timestamp': ('release_date', {int_or_none}),
@@ -1061,6 +1200,7 @@ class BilibiliCheeseBaseIE(BilibiliBaseIE):
             'subtitles': self.extract_subtitles(ep_id, cid, aid=aid),
             '__post_extractor': self.extract_comments(aid),
             'http_headers': self._HEADERS,
+            'heatmap': list(self._extract_heatmap(cid)),
         }
 
     def _download_season_info(self, query_key, video_id):
@@ -1086,8 +1226,9 @@ class BilibiliCheeseIE(BilibiliCheeseBaseIE):
             'duration': 221,
             'timestamp': 1695549606,
             'upload_date': '20230924',
-            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+            'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
             'view_count': int,
+            'heatmap': list,
         },
     }]
 
@@ -1119,8 +1260,9 @@ class BilibiliCheeseSeasonIE(BilibiliCheeseBaseIE):
                 'duration': 221,
                 'timestamp': 1695549606,
                 'upload_date': '20230924',
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
                 'view_count': int,
+                'heatmap': list,
             },
         }],
         'params': {'playlist_items': '1'},
@@ -1492,6 +1634,7 @@ class BilibiliPlaylistIE(BilibiliSpaceListBaseIE):
             'view_count': int,
             'like_count': int,
             '_old_archive_ids': ['bilibili 687146339_part1'],
+            'heatmap': [],
         },
         'params': {'noplaylist': True},
     }, {
@@ -1686,8 +1829,9 @@ class BiliBiliSearchIE(SearchInfoExtractor):
                 'comment_count': int,
                 'view_count': int,
                 'like_count': int,
-                'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)$',
+                'thumbnail': r're:https?://.*\.(?:jpg|jpeg|png)$',
                 '_old_archive_ids': ['bilibili 988222410_part1'],
+                'heatmap': [],
             },
         }],
     }]
@@ -1734,7 +1878,7 @@ class BilibiliAudioIE(BilibiliAudioBaseIE):
             'id': '1003142',
             'ext': 'm4a',
             'title': '【tsukimi】YELLOW / 神山羊',
-            'artist': 'tsukimi',
+            'artists': ['tsukimi'],
             'comment_count': int,
             'description': 'YELLOW的mp3版！',
             'duration': 183,
@@ -1746,7 +1890,7 @@ class BilibiliAudioIE(BilibiliAudioBaseIE):
             'thumbnail': r're:^https?://.+\.jpg',
             'timestamp': 1564836614,
             'upload_date': '20190803',
-            'uploader': 'tsukimi-つきみぐー',
+            'uploader': '十六夜tsukimiつきみぐ',
             'view_count': int,
         },
     }
@@ -1801,10 +1945,10 @@ class BilibiliAudioAlbumIE(BilibiliAudioBaseIE):
         'url': 'https://www.bilibili.com/audio/am10624',
         'info_dict': {
             'id': '10624',
-            'title': '每日新曲推荐（每日11:00更新）',
+            'title': '新曲推荐',
             'description': '每天11:00更新，为你推送最新音乐',
         },
-        'playlist_count': 19,
+        'playlist_mincount': 10,
     }
 
     def _real_extract(self, url):

yt_dlp/extractor/chaturbate.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.(?P<tld>com|eu|global)/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
     _TESTS = [{
         'url': 'https://www.chaturbate.com/siswet19/',
         'info_dict': {
@@ -29,15 +29,24 @@ class ChaturbateIE(InfoExtractor):
     }, {
         'url': 'https://en.chaturbate.com/siswet19/',
         'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.eu/siswet19/',
+        'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.eu/fullvideo/?b=caylin',
+        'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.global/siswet19/',
+        'only_matching': True,
     }]
 
     _ROOM_OFFLINE = 'Room is currently offline'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id, tld = self._match_valid_url(url).group('id', 'tld')
 
         webpage = self._download_webpage(
-            f'https://chaturbate.com/{video_id}/', video_id,
+            f'https://chaturbate.{tld}/{video_id}/', video_id,
             headers=self.geo_verification_headers())
 
         found_m3u8_urls = []

yt_dlp/extractor/facebook.py

@@ -563,13 +563,13 @@ class FacebookIE(InfoExtractor):
                 return extract_video_data(try_get(
                     js_data, lambda x: x['jsmods']['instances'], list) or [])
 
-        def extract_dash_manifest(video, formats):
+        def extract_dash_manifest(vid_data, formats, mpd_url=None):
             dash_manifest = traverse_obj(
-                video, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', expected_type=str)
+                vid_data, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', 'manifest_xml', expected_type=str)
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest)),
-                    mpd_url=url_or_none(video.get('dash_manifest_url'))))
+                    mpd_url=url_or_none(video.get('dash_manifest_url')) or mpd_url))
 
         def process_formats(info):
             # Downloads with browser's User-Agent are rate limited. Working around
@@ -619,9 +619,12 @@ class FacebookIE(InfoExtractor):
                         video = video['creation_story']
                         video['owner'] = traverse_obj(video, ('short_form_video_context', 'video_owner'))
                         video.update(reel_info)
-                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
+
                     formats = []
                     q = qualities(['sd', 'hd'])
+
+                    # Legacy formats extraction
+                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
                     for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
                                            ('playable_url_dash', ''), ('browser_native_hd_url', 'hd'),
                                            ('browser_native_sd_url', 'sd')):
@@ -629,7 +632,7 @@ class FacebookIE(InfoExtractor):
                         if not playable_url:
                             continue
                         if determine_ext(playable_url) == 'mpd':
-                            formats.extend(self._extract_mpd_formats(playable_url, video_id))
+                            formats.extend(self._extract_mpd_formats(playable_url, video_id, fatal=False))
                         else:
                             formats.append({
                                 'format_id': format_id,
@@ -638,6 +641,28 @@ class FacebookIE(InfoExtractor):
                                 'url': playable_url,
                             })
                     extract_dash_manifest(fmt_data, formats)
+
+                    # New videoDeliveryResponse formats extraction
+                    fmt_data = traverse_obj(video, ('videoDeliveryResponseFragment', 'videoDeliveryResponseResult'))
+                    mpd_urls = traverse_obj(fmt_data, ('dash_manifest_urls', ..., 'manifest_url', {url_or_none}))
+                    dash_manifests = traverse_obj(fmt_data, ('dash_manifests', lambda _, v: v['manifest_xml']))
+                    for idx, dash_manifest in enumerate(dash_manifests):
+                        extract_dash_manifest(dash_manifest, formats, mpd_url=traverse_obj(mpd_urls, idx))
+                    if not dash_manifests:
+                        # Only extract from MPD URLs if the manifests are not already provided
+                        for mpd_url in mpd_urls:
+                            formats.extend(self._extract_mpd_formats(mpd_url, video_id, fatal=False))
+                    for prog_fmt in traverse_obj(fmt_data, ('progressive_urls', lambda _, v: v['progressive_url'])):
+                        format_id = traverse_obj(prog_fmt, ('metadata', 'quality', {str.lower}))
+                        formats.append({
+                            'format_id': format_id,
+                            # sd, hd formats w/o resolution info should be deprioritized below DASH
+                            'quality': q(format_id) - 3,
+                            'url': prog_fmt['progressive_url'],
+                        })
+                    for m3u8_url in traverse_obj(fmt_data, ('hls_playlist_urls', ..., 'hls_playlist_url', {url_or_none})):
+                        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', fatal=False, m3u8_id='hls'))
+
                     if not formats:
                         # Do not append false positive entry w/o any formats
                         return