Merge 2761d7f271 into c699bafc50

Closes #11537

Authored by: bashonly

.github/workflows/build.yml

@@ -411,7 +411,7 @@ jobs:
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python devscripts/install_deps.py -o --include build
           python devscripts/install_deps.py --include curl-cffi
-          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.10.0-py3-none-any.whl"
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.11.1-py3-none-any.whl"
 
       - name: Prepare
         run: |
@@ -460,7 +460,7 @@ jobs:
         run: |
           python devscripts/install_deps.py -o --include build
           python devscripts/install_deps.py
-          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.10.0-py3-none-any.whl"
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.11.1-py3-none-any.whl"
 
       - name: Prepare
         run: |

pyproject.toml

@@ -83,7 +83,7 @@ test = [
     "pytest-rerunfailures~=14.0",
 ]
 pyinstaller = [
-    "pyinstaller>=6.10.0",  # Windows temp cleanup fixed in 6.10.0
+    "pyinstaller>=6.11.1",  # Windows temp cleanup fixed in 6.11.1
 ]
 
 [project.urls]

test/test_traversal.py

@@ -481,7 +481,7 @@ class TestTraversalHelpers:
             'id': 'name',
             'data': 'content',
             'url': 'url',
-        }, all, {subs_list_to_dict}]) == {
+        }, all, {subs_list_to_dict(lang=None)}]) == {
             'de': [{'url': 'https://example.com/subs/de.ass'}],
             'en': [{'data': 'content'}],
         }, 'subs with mandatory items missing should be filtered'
@@ -507,6 +507,54 @@ class TestTraversalHelpers:
             {'url': 'https://example.com/subs/en1', 'ext': 'ext'},
             {'url': 'https://example.com/subs/en2', 'ext': 'ext'},
         ]}, '`quality` key should sort subtitle list accordingly'
+        assert traverse_obj([
+            {'name': 'de', 'url': 'https://example.com/subs/de.ass'},
+            {'name': 'de'},
+            {'name': 'en', 'content': 'content'},
+            {'url': 'https://example.com/subs/en'},
+        ], [..., {
+            'id': 'name',
+            'url': 'url',
+            'data': 'content',
+        }, all, {subs_list_to_dict(lang='en')}]) == {
+            'de': [{'url': 'https://example.com/subs/de.ass'}],
+            'en': [
+                {'data': 'content'},
+                {'url': 'https://example.com/subs/en'},
+            ],
+        }, 'optionally provided lang should be used if no id available'
+        assert traverse_obj([
+            {'name': 1, 'url': 'https://example.com/subs/de1'},
+            {'name': {}, 'url': 'https://example.com/subs/de2'},
+            {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'},
+            {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'},
+        ], [..., {
+            'id': 'name',
+            'url': 'url',
+            'ext': 'ext',
+        }, all, {subs_list_to_dict(lang=None)}]) == {
+            'de': [
+                {'url': 'https://example.com/subs/de3'},
+                {'url': 'https://example.com/subs/de4'},
+            ],
+        }, 'non str types should be ignored for id and ext'
+        assert traverse_obj([
+            {'name': 1, 'url': 'https://example.com/subs/de1'},
+            {'name': {}, 'url': 'https://example.com/subs/de2'},
+            {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'},
+            {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'},
+        ], [..., {
+            'id': 'name',
+            'url': 'url',
+            'ext': 'ext',
+        }, all, {subs_list_to_dict(lang='de')}]) == {
+            'de': [
+                {'url': 'https://example.com/subs/de1'},
+                {'url': 'https://example.com/subs/de2'},
+                {'url': 'https://example.com/subs/de3'},
+                {'url': 'https://example.com/subs/de4'},
+            ],
+        }, 'non str types should be replaced by default id'
 
     def test_trim_str(self):
         with pytest.raises(TypeError):
@@ -525,7 +573,7 @@ class TestTraversalHelpers:
     def test_unpack(self):
         assert unpack(lambda *x: ''.join(map(str, x)))([1, 2, 3]) == '123'
         assert unpack(join_nonempty)([1, 2, 3]) == '1-2-3'
-        assert unpack(join_nonempty(delim=' '))([1, 2, 3]) == '1 2 3'
+        assert unpack(join_nonempty, delim=' ')([1, 2, 3]) == '1 2 3'
         with pytest.raises(TypeError):
             unpack(join_nonempty)()
         with pytest.raises(TypeError):

test/test_utils.py

@@ -72,7 +72,6 @@ from yt_dlp.utils import (
     intlist_to_bytes,
     iri_to_uri,
     is_html,
-    join_nonempty,
     js_to_json,
     limit_length,
     locked_file,
@@ -2158,10 +2157,6 @@ Line 1
         assert int_or_none(v=10) == 10, 'keyword passed positional should call function'
         assert int_or_none(scale=0.1)(10) == 100, 'call after partial application should call the function'
 
-        assert callable(join_nonempty(delim=', ')), 'varargs positional should apply partially'
-        assert callable(join_nonempty()), 'varargs positional should apply partially'
-        assert join_nonempty(None, delim=', ') == '', 'passed varargs should call the function'
-
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/YoutubeDL.py

@@ -4381,7 +4381,9 @@ class YoutubeDL:
             return None
 
         for idx, t in list(enumerate(thumbnails))[::-1]:
-            thumb_ext = (f'{t["id"]}.' if multiple else '') + determine_ext(t['url'], 'jpg')
+            thumb_ext = t.get('ext') or determine_ext(t['url'], 'jpg')
+            if multiple:
+                thumb_ext = f'{t["id"]}.{thumb_ext}'
             thumb_display_id = f'{label} thumbnail {t["id"]}'
             thumb_filename = replace_extension(filename, thumb_ext, info_dict.get('ext'))
             thumb_filename_final = replace_extension(thumb_filename_base, thumb_ext, info_dict.get('ext'))

yt_dlp/extractor/_extractors.py

@@ -1156,6 +1156,7 @@ from .mitele import MiTeleIE
 from .mixch import (
     MixchArchiveIE,
     MixchIE,
+    MixchMovieIE,
 )
 from .mixcloud import (
     MixcloudIE,
@@ -1939,9 +1940,7 @@ from .spotify import (
 )
 from .spreaker import (
     SpreakerIE,
-    SpreakerPageIE,
     SpreakerShowIE,
-    SpreakerShowPageIE,
 )
 from .springboardplatform import SpringboardPlatformIE
 from .sprout import SproutIE
@@ -2494,6 +2493,7 @@ from .xstream import XstreamIE
 from .xvideos import (
     XVideosIE,
     XVideosQuickiesIE,
+    XVideosUserIE,
 )
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (

yt_dlp/extractor/afreecatv.py

@@ -66,6 +66,14 @@ class AfreecaTVBaseIE(InfoExtractor):
             extensions={'legacy_ssl': True}), display_id,
             'Downloading API JSON', 'Unable to download API JSON')
 
+    @staticmethod
+    def _fixup_thumb(thumb_url):
+        if not url_or_none(thumb_url):
+            return None
+        # Core would determine_ext as 'php' from the url, so we need to provide the real ext
+        # See: https://github.com/yt-dlp/yt-dlp/issues/11537
+        return [{'url': thumb_url, 'ext': 'jpg'}]
+
 
 class AfreecaTVIE(AfreecaTVBaseIE):
     IE_NAME = 'soop'
@@ -155,7 +163,7 @@ class AfreecaTVIE(AfreecaTVBaseIE):
             'uploader': ('writer_nick', {str}),
             'uploader_id': ('bj_id', {str}),
             'duration': ('total_file_duration', {int_or_none(scale=1000)}),
-            'thumbnail': ('thumb', {url_or_none}),
+            'thumbnails': ('thumb', {self._fixup_thumb}),
         })
 
         entries = []
@@ -226,8 +234,7 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
 
         return self.playlist_result(self._entries(data), video_id)
 
-    @staticmethod
-    def _entries(data):
+    def _entries(self, data):
         # 'files' is always a list with 1 element
         yield from traverse_obj(data, (
             'data', lambda _, v: v['story_type'] == 'catch',
@@ -238,7 +245,7 @@ class AfreecaTVCatchStoryIE(AfreecaTVBaseIE):
                 'title': ('title', {str}),
                 'uploader': ('writer_nick', {str}),
                 'uploader_id': ('writer_id', {str}),
-                'thumbnail': ('thumb', {url_or_none}),
+                'thumbnails': ('thumb', {self._fixup_thumb}),
                 'timestamp': ('write_timestamp', {int_or_none}),
             }))
 

yt_dlp/extractor/archiveorg.py

@@ -205,6 +205,26 @@ class ArchiveOrgIE(InfoExtractor):
                 },
             },
         ],
+    }, {
+        # The reviewbody is None for one of the reviews; just need to extract data without crashing
+        'url': 'https://archive.org/details/gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
+        'info_dict': {
+            'id': 'gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
+            'ext': 'mp3',
+            'title': 'Stuck Inside of Mobile with the Memphis Blues Again',
+            'creators': ['Grateful Dead'],
+            'duration': 338.31,
+            'track': 'Stuck Inside of Mobile with the Memphis Blues Again',
+            'description': 'md5:764348a470b986f1217ffd38d6ac7b72',
+            'display_id': 'gd95-04-02d1t04.shn',
+            'location': 'Pyramid Arena',
+            'uploader': 'jon@archive.org',
+            'album': '1995-04-02 - Pyramid Arena',
+            'upload_date': '20040519',
+            'track_number': 4,
+            'release_date': '19950402',
+            'timestamp': 1084927901,
+        },
     }]
 
     @staticmethod
@@ -335,7 +355,7 @@ class ArchiveOrgIE(InfoExtractor):
                 info['comments'].append({
                     'id': review.get('review_id'),
                     'author': review.get('reviewer'),
-                    'text': str_or_none(review.get('reviewtitle'), '') + '\n\n' + review.get('reviewbody'),
+                    'text': join_nonempty('reviewtitle', 'reviewbody', from_dict=review, delim='\n\n'),
                     'timestamp': unified_timestamp(review.get('createdate')),
                     'parent': 'root'})
 

yt_dlp/extractor/chaturbate.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.(?P<tld>com|eu|global)/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
     _TESTS = [{
         'url': 'https://www.chaturbate.com/siswet19/',
         'info_dict': {
@@ -29,15 +29,24 @@ class ChaturbateIE(InfoExtractor):
     }, {
         'url': 'https://en.chaturbate.com/siswet19/',
         'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.eu/siswet19/',
+        'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.eu/fullvideo/?b=caylin',
+        'only_matching': True,
+    }, {
+        'url': 'https://chaturbate.global/siswet19/',
+        'only_matching': True,
     }]
 
     _ROOM_OFFLINE = 'Room is currently offline'
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id, tld = self._match_valid_url(url).group('id', 'tld')
 
         webpage = self._download_webpage(
-            f'https://chaturbate.com/{video_id}/', video_id,
+            f'https://chaturbate.{tld}/{video_id}/', video_id,
             headers=self.geo_verification_headers())
 
         found_m3u8_urls = []

yt_dlp/extractor/cloudflarestream.py

@@ -8,7 +8,7 @@ class CloudflareStreamIE(InfoExtractor):
     _DOMAIN_RE = r'(?:cloudflarestream\.com|(?:videodelivery|bytehighway)\.net)'
     _EMBED_RE = rf'(?:embed\.|{_SUBDOMAIN_RE}){_DOMAIN_RE}/embed/[^/?#]+\.js\?(?:[^#]+&)?video='
     _ID_RE = r'[\da-f]{32}|eyJ[\w-]+\.[\w-]+\.[\w-]+'
-    _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}{_DOMAIN_RE}/|{_EMBED_RE})(?P<id>{_ID_RE})'
+    _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}(?P<domain>{_DOMAIN_RE})/|{_EMBED_RE})(?P<id>{_ID_RE})'
     _EMBED_REGEX = [
         rf'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//{_EMBED_RE}(?:{_ID_RE})(?:(?!\1).)*)\1',
         rf'<iframe[^>]+\bsrc=["\'](?P<url>https?://{_SUBDOMAIN_RE}{_DOMAIN_RE}/[\da-f]{{32}})',
@@ -19,7 +19,7 @@ class CloudflareStreamIE(InfoExtractor):
             'id': '31c9291ab41fac05471db4e73aa11717',
             'ext': 'mp4',
             'title': '31c9291ab41fac05471db4e73aa11717',
-            'thumbnail': 'https://videodelivery.net/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://cloudflarestream.com/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg',
         },
         'params': {
             'skip_download': 'm3u8',
@@ -30,7 +30,7 @@ class CloudflareStreamIE(InfoExtractor):
             'id': '0e8e040aec776862e1d632a699edf59e',
             'ext': 'mp4',
             'title': '0e8e040aec776862e1d632a699edf59e',
-            'thumbnail': 'https://videodelivery.net/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://cloudflarestream.com/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg',
         },
     }, {
         'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1',
@@ -54,7 +54,7 @@ class CloudflareStreamIE(InfoExtractor):
             'id': 'eaef9dea5159cf968be84241b5cedfe7',
             'ext': 'mp4',
             'title': 'eaef9dea5159cf968be84241b5cedfe7',
-            'thumbnail': 'https://videodelivery.net/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://cloudflarestream.com/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg',
         },
         'params': {
             'skip_download': 'm3u8',
@@ -62,8 +62,9 @@ class CloudflareStreamIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        domain = 'bytehighway.net' if 'bytehighway.net/' in url else 'videodelivery.net'
+        video_id, domain = self._match_valid_url(url).group('id', 'domain')
+        if domain != 'bytehighway.net':
+            domain = 'cloudflarestream.com'
         base_url = f'https://{domain}/{video_id}/'
         if '.' in video_id:
             video_id = self._parse_json(base64.urlsafe_b64decode(

yt_dlp/extractor/common.py

@@ -279,6 +279,7 @@ class InfoExtractor:
     thumbnails:     A list of dictionaries, with the following entries:
                         * "id" (optional, string) - Thumbnail format ID
                         * "url"
+                        * "ext" (optional, string) - actual image extension if not given in URL
                         * "preference" (optional, int) - quality of the image
                         * "width" (optional, int)
                         * "height" (optional, int)

yt_dlp/extractor/facebook.py

@@ -563,13 +563,13 @@ class FacebookIE(InfoExtractor):
                 return extract_video_data(try_get(
                     js_data, lambda x: x['jsmods']['instances'], list) or [])
 
-        def extract_dash_manifest(video, formats):
+        def extract_dash_manifest(vid_data, formats, mpd_url=None):
             dash_manifest = traverse_obj(
-                video, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', expected_type=str)
+                vid_data, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', 'manifest_xml', expected_type=str)
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest)),
-                    mpd_url=url_or_none(video.get('dash_manifest_url'))))
+                    mpd_url=url_or_none(video.get('dash_manifest_url')) or mpd_url))
 
         def process_formats(info):
             # Downloads with browser's User-Agent are rate limited. Working around
@@ -619,9 +619,12 @@ class FacebookIE(InfoExtractor):
                         video = video['creation_story']
                         video['owner'] = traverse_obj(video, ('short_form_video_context', 'video_owner'))
                         video.update(reel_info)
-                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
+
                     formats = []
                     q = qualities(['sd', 'hd'])
+
+                    # Legacy formats extraction
+                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
                     for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
                                            ('playable_url_dash', ''), ('browser_native_hd_url', 'hd'),
                                            ('browser_native_sd_url', 'sd')):
@@ -629,7 +632,7 @@ class FacebookIE(InfoExtractor):
                         if not playable_url:
                             continue
                         if determine_ext(playable_url) == 'mpd':
-                            formats.extend(self._extract_mpd_formats(playable_url, video_id))
+                            formats.extend(self._extract_mpd_formats(playable_url, video_id, fatal=False))
                         else:
                             formats.append({
                                 'format_id': format_id,
@@ -638,6 +641,28 @@ class FacebookIE(InfoExtractor):
                                 'url': playable_url,
                             })
                     extract_dash_manifest(fmt_data, formats)
+
+                    # New videoDeliveryResponse formats extraction
+                    fmt_data = traverse_obj(video, ('videoDeliveryResponseFragment', 'videoDeliveryResponseResult'))
+                    mpd_urls = traverse_obj(fmt_data, ('dash_manifest_urls', ..., 'manifest_url', {url_or_none}))
+                    dash_manifests = traverse_obj(fmt_data, ('dash_manifests', lambda _, v: v['manifest_xml']))
+                    for idx, dash_manifest in enumerate(dash_manifests):
+                        extract_dash_manifest(dash_manifest, formats, mpd_url=traverse_obj(mpd_urls, idx))
+                    if not dash_manifests:
+                        # Only extract from MPD URLs if the manifests are not already provided
+                        for mpd_url in mpd_urls:
+                            formats.extend(self._extract_mpd_formats(mpd_url, video_id, fatal=False))
+                    for prog_fmt in traverse_obj(fmt_data, ('progressive_urls', lambda _, v: v['progressive_url'])):
+                        format_id = traverse_obj(prog_fmt, ('metadata', 'quality', {str.lower}))
+                        formats.append({
+                            'format_id': format_id,
+                            # sd, hd formats w/o resolution info should be deprioritized below DASH
+                            'quality': q(format_id) - 3,
+                            'url': prog_fmt['progressive_url'],
+                        })
+                    for m3u8_url in traverse_obj(fmt_data, ('hls_playlist_urls', ..., 'hls_playlist_url', {url_or_none})):
+                        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', fatal=False, m3u8_id='hls'))
+
                     if not formats:
                         # Do not append false positive entry w/o any formats
                         return

yt_dlp/extractor/goplay.py

@@ -5,56 +5,63 @@ import hashlib
 import hmac
 import json
 import os
+import re
+import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
+    js_to_json,
+    remove_end,
     traverse_obj,
-    unescapeHTML,
 )
 
 
 class GoPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/]+/[^/]+/|)(?P<display_id>[^/#]+)'
+    _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
 
     _NETRC_MACHINE = 'goplay'
 
     _TESTS = [{
-        'url': 'https://www.goplay.be/video/de-container-cup/de-container-cup-s3/de-container-cup-s3-aflevering-2#autoplay',
+        'url': 'https://www.goplay.be/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
         'info_dict': {
-            'id': '9c4214b8-e55d-4e4b-a446-f015f6c6f811',
+            'id': '2baa4560-87a0-421b-bffc-359914e3c387',
             'ext': 'mp4',
-            'title': 'S3 - Aflevering 2',
-            'series': 'De Container Cup',
-            'season': 'Season 3',
-            'season_number': 3,
-            'episode': 'Episode 2',
-            'episode_number': 2,
+            'title': 'S22 - Aflevering 1',
+            'description': r're:In aflevering 1 nemen Daan Alferink, Tess Elst en Xander De Rycke .{66}',
+            'series': 'De Slimste Mens ter Wereld',
+            'episode': 'Episode 1',
+            'season_number': 22,
+            'episode_number': 1,
+            'season': 'Season 22',
         },
+        'params': {'skip_download': True},
         'skip': 'This video is only available for registered users',
     }, {
-        'url': 'https://www.goplay.be/video/a-family-for-thr-holidays-s1-aflevering-1#autoplay',
+        'url': 'https://www.goplay.be/video/1917',
         'info_dict': {
-            'id': '74e3ed07-748c-49e4-85a0-393a93337dbf',
+            'id': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
             'ext': 'mp4',
-            'title': 'A Family for the Holidays',
+            'title': '1917',
+            'description': r're:Op het hoogtepunt van de Eerste Wereldoorlog krijgen twee jonge .{94}',
         },
+        'params': {'skip_download': True},
         'skip': 'This video is only available for registered users',
     }, {
         'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
         'info_dict': {
-            'id': '03eb8f2f-153e-41cb-9805-0d3a29dab656',
+            'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
             'ext': 'mp4',
             'title': 'S11 - Aflevering 1',
+            'description': r're:Tien kandidaten beginnen aan hun verovering van Amerika en ontmoeten .{102}',
             'episode': 'Episode 1',
             'series': 'De Mol',
             'season_number': 11,
             'episode_number': 1,
             'season': 'Season 11',
         },
-        'params': {
-            'skip_download': True,
-        },
+        'params': {'skip_download': True},
         'skip': 'This video is only available for registered users',
     }]
 
@@ -69,27 +76,42 @@ class GoPlayIE(InfoExtractor):
         if not self._id_token:
             raise self.raise_login_required(method='password')
 
-    def _real_extract(self, url):
-        url, display_id = self._match_valid_url(url).group(0, 'display_id')
-        webpage = self._download_webpage(url, display_id)
-        video_data_json = self._html_search_regex(r'<div\s+data-hero="([^"]+)"', webpage, 'video_data')
-        video_data = self._parse_json(unescapeHTML(video_data_json), display_id).get('data')
+    def _find_json(self, s):
+        return self._search_json(
+            r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
 
-        movie = video_data.get('movie')
-        if movie:
-            video_id = movie['videoUuid']
-            info_dict = {
-                'title': movie.get('title'),
-            }
-        else:
-            episode = traverse_obj(video_data, ('playlists', ..., 'episodes', lambda _, v: v['pageInfo']['url'] == url), get_all=False)
-            video_id = episode['videoUuid']
-            info_dict = {
-                'title': episode.get('episodeTitle'),
-                'series': traverse_obj(episode, ('program', 'title')),
-                'season_number': episode.get('seasonNumber'),
-                'episode_number': episode.get('episodeNumber'),
-            }
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        nextjs_data = traverse_obj(
+            re.findall(r'<script[^>]*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*</script>', webpage),
+            (..., {js_to_json}, {json.loads}, ..., {self._find_json}, ...))
+        meta = traverse_obj(nextjs_data, (
+            ..., lambda _, v: v['meta']['path'] == urllib.parse.urlparse(url).path, 'meta', any))
+
+        video_id = meta['uuid']
+        info_dict = traverse_obj(meta, {
+            'title': ('title', {str}),
+            'description': ('description', {str.strip}),
+        })
+
+        if traverse_obj(meta, ('program', 'subtype')) != 'movie':
+            for season_data in traverse_obj(nextjs_data, (..., 'children', ..., 'playlists', ...)):
+                episode_data = traverse_obj(
+                    season_data, ('videos', lambda _, v: v['videoId'] == video_id, any))
+                if not episode_data:
+                    continue
+
+                episode_title = traverse_obj(
+                    episode_data, 'contextualTitle', 'episodeTitle', expected_type=str)
+                info_dict.update({
+                    'title': episode_title or info_dict.get('title'),
+                    'series': remove_end(info_dict.get('title'), f' - {episode_title}'),
+                    'season_number': traverse_obj(season_data, ('season', {int_or_none})),
+                    'episode_number': traverse_obj(episode_data, ('episodeNumber', {int_or_none})),
+                })
+                break
 
         api = self._download_json(
             f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',

yt_dlp/extractor/mixch.py

@@ -12,7 +12,7 @@ from ..utils.traversal import traverse_obj
 
 class MixchIE(InfoExtractor):
     IE_NAME = 'mixch'
-    _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P<id>\d+)'
+    _VALID_URL = r'https?://mixch\.tv/u/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'https://mixch.tv/u/16943797/live',
@@ -74,7 +74,7 @@ class MixchIE(InfoExtractor):
 
 class MixchArchiveIE(InfoExtractor):
     IE_NAME = 'mixch:archive'
-    _VALID_URL = r'https?://(?:www\.)?mixch\.tv/archive/(?P<id>\d+)'
+    _VALID_URL = r'https?://mixch\.tv/archive/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'https://mixch.tv/archive/421',
@@ -116,3 +116,56 @@ class MixchArchiveIE(InfoExtractor):
             'formats': self._extract_m3u8_formats(info_json['archiveURL'], video_id),
             'thumbnail': traverse_obj(info_json, ('thumbnailURL', {url_or_none})),
         }
+
+
+class MixchMovieIE(InfoExtractor):
+    IE_NAME = 'mixch:movie'
+    _VALID_URL = r'https?://mixch\.tv/m/(?P<id>\w+)'
+
+    _TESTS = [{
+        'url': 'https://mixch.tv/m/Ve8KNkJ5',
+        'info_dict': {
+            'id': 'Ve8KNkJ5',
+            'title': '夏☀️\nムービーへのポイントは本イベントに加算されないので配信にてお願い致します🙇🏻\u200d♀️\n#TGCCAMPUS #ミス東大 #ミス東大2024 ',
+            'ext': 'mp4',
+            'uploader': 'ミス東大No.5 松藤百香🍑💫',
+            'uploader_id': '12299174',
+            'channel_follower_count': int,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'timestamp': 1724070828,
+            'uploader_url': 'https://mixch.tv/u/12299174',
+            'live_status': 'not_live',
+            'upload_date': '20240819',
+        },
+    }, {
+        'url': 'https://mixch.tv/m/61DzpIKE',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            f'https://mixch.tv/api-web/movies/{video_id}', video_id)
+        return {
+            'id': video_id,
+            'formats': [{
+                'format_id': 'mp4',
+                'url': data['movie']['file'],
+                'ext': 'mp4',
+            }],
+            **traverse_obj(data, {
+                'title': ('movie', 'title', {str}),
+                'thumbnail': ('movie', 'thumbnailURL', {url_or_none}),
+                'uploader': ('ownerInfo', 'name', {str}),
+                'uploader_id': ('ownerInfo', 'id', {int}, {str_or_none}),
+                'channel_follower_count': ('ownerInfo', 'fan', {int_or_none}),
+                'view_count': ('ownerInfo', 'view', {int_or_none}),
+                'like_count': ('movie', 'favCount', {int_or_none}),
+                'comment_count': ('movie', 'commentCount', {int_or_none}),
+                'timestamp': ('movie', 'published', {int_or_none}),
+                'uploader_url': ('ownerInfo', 'id', {lambda x: x and f'https://mixch.tv/u/{x}'}, filter),
+            }),
+            'live_status': 'not_live',
+        }

yt_dlp/extractor/rutube.py

@@ -2,15 +2,18 @@ import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    UnsupportedError,
     bool_or_none,
     determine_ext,
     int_or_none,
+    js_to_json,
     parse_qs,
-    traverse_obj,
+    str_or_none,
     try_get,
     unified_timestamp,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class RutubeBaseIE(InfoExtractor):
@@ -19,7 +22,7 @@ class RutubeBaseIE(InfoExtractor):
             query = {}
         query['format'] = 'json'
         return self._download_json(
-            f'http://rutube.ru/api/video/{video_id}/',
+            f'https://rutube.ru/api/video/{video_id}/',
             video_id, 'Downloading video JSON',
             'Unable to download video JSON', query=query)
 
@@ -61,18 +64,21 @@ class RutubeBaseIE(InfoExtractor):
             query = {}
         query['format'] = 'json'
         return self._download_json(
-            f'http://rutube.ru/api/play/options/{video_id}/',
+            f'https://rutube.ru/api/play/options/{video_id}/',
             video_id, 'Downloading options JSON',
             'Unable to download options JSON',
             headers=self.geo_verification_headers(), query=query)
 
-    def _extract_formats(self, options, video_id):
+    def _extract_formats_and_subtitles(self, options, video_id):
         formats = []
+        subtitles = {}
         for format_id, format_url in options['video_balancer'].items():
             ext = determine_ext(format_url)
             if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     format_url, video_id, f4m_id=format_id, fatal=False))
@@ -82,11 +88,19 @@ class RutubeBaseIE(InfoExtractor):
                     'format_id': format_id,
                 })
         for hls_url in traverse_obj(options, ('live_streams', 'hls', ..., 'url', {url_or_none})):
-            formats.extend(self._extract_m3u8_formats(hls_url, video_id, ext='mp4', fatal=False))
-        return formats
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls')
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+        for caption in traverse_obj(options, ('captions', lambda _, v: url_or_none(v['file']))):
+            subtitles.setdefault(caption.get('code') or 'ru', []).append({
+                'url': caption['file'],
+                'name': caption.get('langTitle'),
+            })
+        return formats, subtitles
 
-    def _download_and_extract_formats(self, video_id, query=None):
-        return self._extract_formats(
+    def _download_and_extract_formats_and_subtitles(self, video_id, query=None):
+        return self._extract_formats_and_subtitles(
             self._download_api_options(video_id, query=query), video_id)
 
 
@@ -97,8 +111,8 @@ class RutubeIE(RutubeBaseIE):
     _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//rutube\.ru/(?:play/)?embed/[\da-z]{32}.*?)\1']
 
     _TESTS = [{
-        'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
-        'md5': 'e33ac625efca66aba86cbec9851f2692',
+        'url': 'https://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
+        'md5': '3d73fdfe5bb81b9aef139e22ef3de26a',
         'info_dict': {
             'id': '3eac3b4561676c17df9132a9a1e62e3e',
             'ext': 'mp4',
@@ -111,26 +125,25 @@ class RutubeIE(RutubeBaseIE):
             'upload_date': '20131016',
             'age_limit': 0,
             'view_count': int,
-            'thumbnail': 'http://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg',
+            'thumbnail': 'https://pic.rutubelist.ru/video/d2/a0/d2a0aec998494a396deafc7ba2c82add.jpg',
             'categories': ['Новости и СМИ'],
             'chapters': [],
         },
-        'expected_warnings': ['Unable to download f4m'],
     }, {
-        'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
+        'url': 'https://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
     }, {
-        'url': 'http://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661',
+        'url': 'https://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661',
         'only_matching': True,
     }, {
-        'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252',
+        'url': 'https://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252',
         'only_matching': True,
     }, {
         'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_type=source',
         'only_matching': True,
     }, {
         'url': 'https://rutube.ru/video/private/884fb55f07a97ab673c7d654553e0f48/?p=x2QojCumHTS3rsKHWXN8Lg',
-        'md5': 'd106225f15d625538fe22971158e896f',
+        'md5': '4fce7b4fcc7b1bcaa3f45eb1e1ad0dd7',
         'info_dict': {
             'id': '884fb55f07a97ab673c7d654553e0f48',
             'ext': 'mp4',
@@ -143,11 +156,10 @@ class RutubeIE(RutubeBaseIE):
             'upload_date': '20221210',
             'age_limit': 0,
             'view_count': int,
-            'thumbnail': 'http://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg',
+            'thumbnail': 'https://pic.rutubelist.ru/video/f2/d4/f2d42b54be0a6e69c1c22539e3152156.jpg',
             'categories': ['Видеоигры'],
             'chapters': [],
         },
-        'expected_warnings': ['Unable to download f4m'],
     }, {
         'url': 'https://rutube.ru/video/c65b465ad0c98c89f3b25cb03dcc87c6/',
         'info_dict': {
@@ -156,17 +168,16 @@ class RutubeIE(RutubeBaseIE):
             'chapters': 'count:4',
             'categories': ['Бизнес и предпринимательство'],
             'description': 'md5:252feac1305257d8c1bab215cedde75d',
-            'thumbnail': 'http://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png',
+            'thumbnail': 'https://pic.rutubelist.ru/video/71/8f/718f27425ea9706073eb80883dd3787b.png',
             'duration': 782,
             'age_limit': 0,
             'uploader_id': '23491359',
             'timestamp': 1677153329,
             'view_count': int,
             'upload_date': '20230223',
-            'title': 'Бизнес с нуля: найм сотрудников. Интервью с директором строительной компании',
+            'title': 'Бизнес с нуля: найм сотрудников. Интервью с директором строительной компании #1',
             'uploader': 'Стас Быков',
         },
-        'expected_warnings': ['Unable to download f4m'],
     }, {
         'url': 'https://rutube.ru/live/video/c58f502c7bb34a8fcdd976b221fca292/',
         'info_dict': {
@@ -174,7 +185,7 @@ class RutubeIE(RutubeBaseIE):
             'ext': 'mp4',
             'categories': ['Телепередачи'],
             'description': '',
-            'thumbnail': 'http://pic.rutubelist.ru/video/14/19/14190807c0c48b40361aca93ad0867c7.jpg',
+            'thumbnail': 'https://pic.rutubelist.ru/video/14/19/14190807c0c48b40361aca93ad0867c7.jpg',
             'live_status': 'is_live',
             'age_limit': 0,
             'uploader_id': '23460655',
@@ -184,6 +195,24 @@ class RutubeIE(RutubeBaseIE):
             'title': r're:Первый канал. Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'uploader': 'Первый канал',
         },
+    }, {
+        'url': 'https://rutube.ru/play/embed/03a9cb54bac3376af4c5cb0f18444e01/',
+        'info_dict': {
+            'id': '03a9cb54bac3376af4c5cb0f18444e01',
+            'ext': 'mp4',
+            'age_limit': 0,
+            'description': '',
+            'title': 'Церемония начала торгов акциями ПАО «ЕвроТранс»',
+            'chapters': [],
+            'upload_date': '20240829',
+            'duration': 293,
+            'uploader': 'MOEX - Московская биржа',
+            'timestamp': 1724946628,
+            'thumbnail': 'https://pic.rutubelist.ru/video/2e/24/2e241fddb459baf0fa54acfca44874f4.jpg',
+            'view_count': int,
+            'uploader_id': '38420507',
+            'categories': ['Интервью'],
+        },
     }, {
         'url': 'https://rutube.ru/video/5ab908fccfac5bb43ef2b1e4182256b0/',
         'only_matching': True,
@@ -192,40 +221,46 @@ class RutubeIE(RutubeBaseIE):
         'only_matching': True,
     }]
 
-    @classmethod
-    def suitable(cls, url):
-        return False if RutubePlaylistIE.suitable(url) else super().suitable(url)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         query = parse_qs(url)
         info = self._download_and_extract_info(video_id, query)
-        info['formats'] = self._download_and_extract_formats(video_id, query)
-        return info
+        formats, subtitles = self._download_and_extract_formats_and_subtitles(video_id, query)
+        return {
+            **info,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
 
 
 class RutubeEmbedIE(RutubeBaseIE):
     IE_NAME = 'rutube:embed'
     IE_DESC = 'Rutube embedded videos'
-    _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)(?:[?#/]|$)'
 
     _TESTS = [{
-        'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
+        'url': 'https://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
         'info_dict': {
             'id': 'a10e53b86e8f349080f718582ce4c661',
             'ext': 'mp4',
             'timestamp': 1387830582,
             'upload_date': '20131223',
             'uploader_id': '297833',
-            'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89',
             'uploader': 'subziro89 ILya',
             'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89',
+            'age_limit': 0,
+            'duration': 1395,
+            'chapters': [],
+            'description': 'md5:a5acea57bbc3ccdc3cacd1f11a014b5b',
+            'view_count': int,
+            'thumbnail': 'https://pic.rutubelist.ru/video/d3/03/d3031f4670a6e6170d88fb3607948418.jpg',
+            'categories': ['Сериалы'],
         },
         'params': {
             'skip_download': True,
         },
     }, {
-        'url': 'http://rutube.ru/play/embed/8083783',
+        'url': 'https://rutube.ru/play/embed/8083783',
         'only_matching': True,
     }, {
         # private video
@@ -240,11 +275,12 @@ class RutubeEmbedIE(RutubeBaseIE):
         query = parse_qs(url)
         options = self._download_api_options(embed_id, query)
         video_id = options['effective_video']
-        formats = self._extract_formats(options, video_id)
+        formats, subtitles = self._extract_formats_and_subtitles(options, video_id)
         info = self._download_and_extract_info(video_id, query)
         info.update({
             'extractor_key': 'Rutube',
             'formats': formats,
+            'subtitles': subtitles,
         })
         return info
 
@@ -295,14 +331,14 @@ class RutubeTagsIE(RutubePlaylistBaseIE):
     IE_DESC = 'Rutube tags'
     _VALID_URL = r'https?://rutube\.ru/tags/video/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://rutube.ru/tags/video/1800/',
+        'url': 'https://rutube.ru/tags/video/1800/',
         'info_dict': {
             'id': '1800',
         },
         'playlist_mincount': 68,
     }]
 
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json'
+    _PAGE_TEMPLATE = 'https://rutube.ru/api/tags/video/%s/?page=%s&format=json'
 
 
 class RutubeMovieIE(RutubePlaylistBaseIE):
@@ -310,8 +346,8 @@ class RutubeMovieIE(RutubePlaylistBaseIE):
     IE_DESC = 'Rutube movies'
     _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P<id>\d+)'
 
-    _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
+    _MOVIE_TEMPLATE = 'https://rutube.ru/api/metainfo/tv/%s/?format=json'
+    _PAGE_TEMPLATE = 'https://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
 
     def _real_extract(self, url):
         movie_id = self._match_id(url)
@@ -327,62 +363,82 @@ class RutubePersonIE(RutubePlaylistBaseIE):
     IE_DESC = 'Rutube person videos'
     _VALID_URL = r'https?://rutube\.ru/video/person/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://rutube.ru/video/person/313878/',
+        'url': 'https://rutube.ru/video/person/313878/',
         'info_dict': {
             'id': '313878',
         },
-        'playlist_mincount': 37,
+        'playlist_mincount': 36,
     }]
 
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
+    _PAGE_TEMPLATE = 'https://rutube.ru/api/video/person/%s/?page=%s&format=json'
 
 
 class RutubePlaylistIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:playlist'
     IE_DESC = 'Rutube playlists'
-    _VALID_URL = r'https?://rutube\.ru/(?:video|(?:play/)?embed)/[\da-z]{32}/\?.*?\bpl_id=(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/plst/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'https://rutube.ru/video/cecd58ed7d531fc0f3d795d51cee9026/?pl_id=3097&pl_type=tag',
+        'url': 'https://rutube.ru/plst/308547/',
         'info_dict': {
-            'id': '3097',
+            'id': '308547',
         },
-        'playlist_count': 27,
-    }, {
-        'url': 'https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source',
-        'only_matching': True,
+        'playlist_mincount': 22,
     }]
-
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/playlist/%s/%s/?page=%s&format=json'
-
-    @classmethod
-    def suitable(cls, url):
-        from ..utils import int_or_none, parse_qs
-
-        if not super().suitable(url):
-            return False
-        params = parse_qs(url)
-        return params.get('pl_type', [None])[0] and int_or_none(params.get('pl_id', [None])[0])
-
-    def _next_page_url(self, page_num, playlist_id, item_kind):
-        return self._PAGE_TEMPLATE % (item_kind, playlist_id, page_num)
-
-    def _real_extract(self, url):
-        qs = parse_qs(url)
-        playlist_kind = qs['pl_type'][0]
-        playlist_id = qs['pl_id'][0]
-        return self._extract_playlist(playlist_id, item_kind=playlist_kind)
+    _PAGE_TEMPLATE = 'https://rutube.ru/api/playlist/custom/%s/videos?page=%s&format=json'
 
 
 class RutubeChannelIE(RutubePlaylistBaseIE):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channel'
-    _VALID_URL = r'https?://rutube\.ru/channel/(?P<id>\d+)/videos'
+    _VALID_URL = r'https?://rutube\.ru/(?:channel/(?P<id>\d+)|u/(?P<slug>\w+))(?:/(?P<section>videos|shorts|playlists))?'
     _TESTS = [{
         'url': 'https://rutube.ru/channel/639184/videos/',
         'info_dict': {
-            'id': '639184',
+            'id': '639184_videos',
         },
-        'playlist_mincount': 133,
+        'playlist_mincount': 129,
+    }, {
+        'url': 'https://rutube.ru/channel/25902603/shorts/',
+        'info_dict': {
+            'id': '25902603_shorts',
+        },
+        'playlist_mincount': 277,
+    }, {
+        'url': 'https://rutube.ru/channel/25902603/',
+        'info_dict': {
+            'id': '25902603',
+        },
+        'playlist_mincount': 406,
+    }, {
+        'url': 'https://rutube.ru/u/rutube/videos/',
+        'info_dict': {
+            'id': '23704195_videos',
+        },
+        'playlist_mincount': 113,
     }]
 
-    _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
+    _PAGE_TEMPLATE = 'https://rutube.ru/api/video/person/%s/?page=%s&format=json&origin__type=%s'
+
+    def _next_page_url(self, page_num, playlist_id, section):
+        origin_type = {
+            'videos': 'rtb,rst,ifrm,rspa',
+            'shorts': 'rshorts',
+            None: '',
+        }.get(section)
+        return self._PAGE_TEMPLATE % (playlist_id, page_num, origin_type)
+
+    def _real_extract(self, url):
+        playlist_id, slug, section = self._match_valid_url(url).group('id', 'slug', 'section')
+        if section == 'playlists':
+            raise UnsupportedError(url)
+        if slug:
+            webpage = self._download_webpage(url, slug)
+            redux_state = self._search_json(
+                r'window\.reduxState\s*=', webpage, 'redux state', slug, transform_source=js_to_json)
+            playlist_id = traverse_obj(redux_state, (
+                'api', 'queries', lambda k, _: k.startswith('channelIdBySlug'),
+                'data', 'channel_id', {int}, {str_or_none}, any))
+        playlist = self._extract_playlist(playlist_id, section=section)
+        if section:
+            playlist['id'] = f'{playlist_id}_{section}'
+        return playlist

yt_dlp/extractor/spreaker.py

@@ -2,13 +2,16 @@ import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    filter_dict,
     float_or_none,
     int_or_none,
+    parse_qs,
     str_or_none,
     try_get,
     unified_timestamp,
     url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 def _extract_episode(data, episode_id=None):
@@ -58,15 +61,10 @@ def _extract_episode(data, episode_id=None):
 
 
 class SpreakerIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        api\.spreaker\.com/
-                        (?:
-                            (?:download/)?episode|
-                            v2/episodes
-                        )/
-                        (?P<id>\d+)
-                    '''
+    _VALID_URL = [
+        r'https?://api\.spreaker\.com/(?:(?:download/)?episode|v2/episodes)/(?P<id>\d+)',
+        r'https?://(?:www\.)?spreaker\.com/episode/[^#?/]*?(?P<id>\d+)/?(?:[?#]|$)',
+    ]
     _TESTS = [{
         'url': 'https://api.spreaker.com/episode/12534508',
         'info_dict': {
@@ -83,7 +81,9 @@ class SpreakerIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
             'comment_count': int,
-            'series': 'Success With Music (SWM)',
+            'series': 'Success With Music | SWM',
+            'thumbnail': 'https://d3wo5wojvuv7l.cloudfront.net/t_square_limited_160/images.spreaker.com/original/777ce4f96b71b0e1b7c09a5e625210e3.jpg',
+            'creators': ['SWM'],
         },
     }, {
         'url': 'https://api.spreaker.com/download/episode/12534508/swm_ep15_how_to_market_your_music_part_2.mp3',
@@ -91,52 +91,75 @@ class SpreakerIE(InfoExtractor):
     }, {
         'url': 'https://api.spreaker.com/v2/episodes/12534508?export=episode_segments',
         'only_matching': True,
+    }, {
+        'note': 'episode',
+        'url': 'https://www.spreaker.com/episode/grunge-music-origins-the-raw-sound-that-defined-a-generation--60269615',
+        'info_dict': {
+            'id': '60269615',
+            'display_id': 'grunge-music-origins-the-raw-sound-that-',
+            'ext': 'mp3',
+            'title': 'Grunge Music Origins - The Raw Sound that Defined a Generation',
+            'description': str,
+            'timestamp': 1717468905,
+            'upload_date': '20240604',
+            'uploader': 'Katie Brown 2',
+            'uploader_id': '17733249',
+            'duration': 818.83,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'series': '90s Grunge',
+            'thumbnail': 'https://d3wo5wojvuv7l.cloudfront.net/t_square_limited_160/images.spreaker.com/original/bb0d4178f7cf57cc8786dedbd9c5d969.jpg',
+            'creators': ['Katie Brown 2'],
+        },
+    }, {
+        'url': 'https://www.spreaker.com/episode/60269615',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         episode_id = self._match_id(url)
         data = self._download_json(
-            f'https://api.spreaker.com/v2/episodes/{episode_id}',
-            episode_id)['response']['episode']
+            f'https://api.spreaker.com/v2/episodes/{episode_id}', episode_id,
+            query=traverse_obj(parse_qs(url), {'key': ('key', 0)}))['response']['episode']
         return _extract_episode(data, episode_id)
 
 
-class SpreakerPageIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spreaker\.com/user/[^/]+/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://www.spreaker.com/user/9780658/swm-ep15-how-to-market-your-music-part-2',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        episode_id = self._search_regex(
-            (r'data-episode_id=["\'](?P<id>\d+)',
-             r'episode_id\s*:\s*(?P<id>\d+)'), webpage, 'episode id')
-        return self.url_result(
-            f'https://api.spreaker.com/episode/{episode_id}',
-            ie=SpreakerIE.ie_key(), video_id=episode_id)
-
-
 class SpreakerShowIE(InfoExtractor):
-    _VALID_URL = r'https?://api\.spreaker\.com/show/(?P<id>\d+)'
+    _VALID_URL = [
+        r'https?://api\.spreaker\.com/show/(?P<id>\d+)',
+        r'https?://(?:www\.)?spreaker\.com/podcast/[\w-]+--(?P<id>[\d]+)',
+        r'https?://(?:www\.)?spreaker\.com/show/(?P<id>\d+)/episodes/feed',
+    ]
     _TESTS = [{
         'url': 'https://api.spreaker.com/show/4652058',
         'info_dict': {
             'id': '4652058',
         },
         'playlist_mincount': 118,
+    }, {
+        'url': 'https://www.spreaker.com/podcast/health-wealth--5918323',
+        'info_dict': {
+            'id': '5918323',
+        },
+        'playlist_mincount': 60,
+    }, {
+        'url': 'https://www.spreaker.com/show/5887186/episodes/feed',
+        'info_dict': {
+            'id': '5887186',
+        },
+        'playlist_mincount': 290,
     }]
 
-    def _entries(self, show_id):
+    def _entries(self, show_id, key=None):
         for page_num in itertools.count(1):
             episodes = self._download_json(
                 f'https://api.spreaker.com/show/{show_id}/episodes',
-                show_id, note=f'Downloading JSON page {page_num}', query={
+                show_id, note=f'Downloading JSON page {page_num}', query=filter_dict({
                     'page': page_num,
                     'max_per_page': 100,
-                })
+                    'key': key,
+                }))
             pager = try_get(episodes, lambda x: x['response']['pager'], dict)
             if not pager:
                 break
@@ -152,21 +175,5 @@ class SpreakerShowIE(InfoExtractor):
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
-        return self.playlist_result(self._entries(show_id), playlist_id=show_id)
-
-
-class SpreakerShowPageIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spreaker\.com/show/(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'https://www.spreaker.com/show/success-with-music',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        show_id = self._search_regex(
-            r'show_id\s*:\s*(?P<id>\d+)', webpage, 'show id')
-        return self.url_result(
-            f'https://api.spreaker.com/show/{show_id}',
-            ie=SpreakerShowIE.ie_key(), video_id=show_id)
+        key = traverse_obj(parse_qs(url), ('key', 0))
+        return self.playlist_result(self._entries(show_id, key), playlist_id=show_id)

yt_dlp/extractor/xvideos.py

@@ -1,14 +1,22 @@
+import functools
 import re
 import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    OnDemandPagedList,
     clean_html,
     determine_ext,
+    get_element_by_class,
+    get_element_by_id,
     int_or_none,
+    js_to_json,
     parse_duration,
+    remove_end,
+    str_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class XVideosIE(InfoExtractor):
@@ -108,9 +116,8 @@ class XVideosIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
-        if mobj:
-            raise ExtractorError(f'{self.IE_NAME} said: {clean_html(mobj.group(1))}', expected=True)
+        if inline_error := get_element_by_class('inlineError', webpage):
+            raise ExtractorError(f'{self.IE_NAME} said: {clean_html(inline_error)}', expected=True)
 
         title = self._html_search_regex(
             (r'<title>(?P<title>.+?)\s+-\s+XVID',
@@ -223,3 +230,114 @@ class XVideosQuickiesIE(InfoExtractor):
     def _real_extract(self, url):
         domain, id_ = self._match_valid_url(url).group('domain', 'id')
         return self.url_result(f'https://{domain}/video{"" if id_.isdecimal() else "."}{id_}/_', XVideosIE, id_)
+
+
+class XVideosUserIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?:.+?\.)?xvideos\.(?:com|es)/
+        (?P<page_path>(?:channels|amateur-channels|model-channels|pornstar-channels|profiles)/
+        (?P<id>[^/?#&]+))(?:(?:(?!\#quickies).)+)?$'''
+    _TESTS = [{
+        # channel; "Most viewed"
+        'url': 'https://www.xvideos.com/channels/college_girls_gone_bad#_tabVideos,rating',
+        'info_dict': {
+            'id': '70472676',
+            'display_id': 'college_girls_gone_bad',
+            'title': 'College Girls Gone Bad',
+            'description': 'Hot college girls in real sorority hazing acts!',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 99,
+    }, {
+        # channel; "New"
+        'url': 'https://www.xvideos.com/model-channels/shonariver#_tabVideos,new',
+        'info_dict': {
+            'id': '407014987',
+            'display_id': 'shonariver',
+            'title': 'Shona River',
+            'description': 'md5:ad6654037aee13535b0d15a020eb82d0',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 9,
+    }, {
+        # channel; "Most commented"
+        'url': 'https://www.xvideos.com/amateur-channels/queanfuckingcucking#_tabVideos,comments',
+        'info_dict': {
+            'id': '227800369',
+            'display_id': 'queanfuckingcucking',
+            'title': 'Queanfuckingcucking',
+            'description': 'md5:265a602186d4e811082782cd6a97b064',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 8,
+    }, {
+        # channel; "Watched recently" (default)
+        'url': 'https://www.xvideos.com/channels/girlfriendsfilmsofficial#_tabVideos',
+        'info_dict': {
+            'id': '244972019',
+            'display_id': 'girlfriendsfilmsofficial',
+            'title': 'Girlfriend\'s Films Official',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 500,
+    }, {
+        # /profiles/***
+        'url': 'https://www.xvideos.com/profiles/jacobsy',
+        'info_dict': {
+            'id': '962189',
+            'display_id': 'jacobsy',
+            'title': 'Jacobsy',
+            'description': 'fetishist and bdsm lover...',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 63,
+    }, {
+        # no description, no videos
+        'url': 'https://www.xvideos.com/profiles/espoder',
+        'info_dict': {
+            'id': '581228107',
+            'display_id': 'espoder',
+            'title': 'Espoder',
+            'thumbnails': 'count:2',
+        },
+        'playlist_count': 0,
+    }, {
+        # no description
+        'url': 'https://www.xvideos.com/profiles/alfsun',
+        'info_dict': {
+            'id': '551066909',
+            'display_id': 'alfsun',
+            'title': 'Alfsun',
+            'thumbnails': 'count:2',
+        },
+        'playlist_mincount': 3,
+    }]
+    _PAGE_SIZE = 36
+
+    def _real_extract(self, url):
+        page_path, display_id = self._match_valid_url(url).groups()
+        webpage = self._download_webpage(url, display_id)
+
+        fragment = urllib.parse.urlparse(url).fragment
+        sort_order = traverse_obj(
+            ['new', 'rating', 'comments'], (lambda _, v: v in fragment), default='best', get_all=False)
+        page_base_url = f'https://www.xvideos.com/{page_path}/videos/{sort_order}'
+
+        user_info = traverse_obj(self._search_json(
+            r'<script>.*?window\.xv\.conf\s*=', webpage, 'xv.conf',
+            display_id, transform_source=js_to_json, fatal=False), ('data', 'user'))
+        user_id = traverse_obj(user_info, ('id_user', {str_or_none})) or display_id
+
+        return self.playlist_result(
+            OnDemandPagedList(functools.partial(self._get_page, page_base_url, user_id), self._PAGE_SIZE),
+            user_id, traverse_obj(user_info, ('display', {str_or_none})),
+            remove_end(clean_html(get_element_by_id('header-about-me', webpage)), '+'),
+            display_id=(traverse_obj(user_info, ('username', {str_or_none})) or display_id),
+            thumbnails=traverse_obj(user_info, (['profile_picture_small', 'profile_picture'], {lambda x: {'url': x}})))
+
+    def _get_page(self, page_base_url, user_id, page_num):
+        page_info = self._download_json(
+            f'{page_base_url}/{page_num}', user_id, f'Downloading page {page_num + 1}')
+        yield from [self.url_result(
+            f'https://www.xvideos.com/video{video["id"]}/{video["eid"]}', ie=XVideosIE.ie_key())
+            for video in page_info['videos']]

yt_dlp/utils/_utils.py

@@ -216,7 +216,7 @@ def partial_application(func):
     sig = inspect.signature(func)
     required_args = [
         param.name for param in sig.parameters.values()
-        if param.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
+        if param.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
         if param.default is inspect.Parameter.empty
     ]
 
@@ -4837,7 +4837,6 @@ def number_of_digits(number):
     return len('%d' % number)
 
 
-@partial_application
 def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
         values = (traversal.traverse_obj(from_dict, variadic(v)) for v in values)

yt_dlp/utils/traversal.py

@@ -332,14 +332,14 @@ class _RequiredError(ExtractorError):
 
 
 @typing.overload
-def subs_list_to_dict(*, ext: str | None = None) -> collections.abc.Callable[[list[dict]], dict[str, list[dict]]]: ...
+def subs_list_to_dict(*, lang: str | None = 'und', ext: str | None = None) -> collections.abc.Callable[[list[dict]], dict[str, list[dict]]]: ...
 
 
 @typing.overload
-def subs_list_to_dict(subs: list[dict] | None, /, *, ext: str | None = None) -> dict[str, list[dict]]: ...
+def subs_list_to_dict(subs: list[dict] | None, /, *, lang: str | None = 'und', ext: str | None = None) -> dict[str, list[dict]]: ...
 
 
-def subs_list_to_dict(subs: list[dict] | None = None, /, *, ext=None):
+def subs_list_to_dict(subs: list[dict] | None = None, /, *, lang='und', ext=None):
     """
     Convert subtitles from a traversal into a subtitle dict.
     The path should have an `all` immediately before this function.
@@ -352,7 +352,7 @@ def subs_list_to_dict(subs: list[dict] | None = None, /, *, ext=None):
     `quality`  The sort order for each subtitle
     """
     if subs is None:
-        return functools.partial(subs_list_to_dict, ext=ext)
+        return functools.partial(subs_list_to_dict, lang=lang, ext=ext)
 
     result = collections.defaultdict(list)
 
@@ -360,9 +360,15 @@ def subs_list_to_dict(subs: list[dict] | None = None, /, *, ext=None):
         if not url_or_none(sub.get('url')) and not sub.get('data'):
             continue
         sub_id = sub.pop('id', None)
-        if sub_id is None:
+        if not isinstance(sub_id, str):
+            if not lang:
                 continue
-        if ext is not None and not sub.get('ext'):
+            sub_id = lang
+        sub_ext = sub.get('ext')
+        if not isinstance(sub_ext, str):
+            if not ext:
+                sub.pop('ext', None)
+            else:
                 sub['ext'] = ext
         result[sub_id].append(sub)
     result = dict(result)
@@ -452,9 +458,9 @@ def trim_str(*, start=None, end=None):
     return trim
 
 
-def unpack(func):
+def unpack(func, **kwargs):
     @functools.wraps(func)
-    def inner(items, **kwargs):
+    def inner(items):
         return func(*items, **kwargs)
 
     return inner