Merge 7fa603bcc3 into c673731061

2024-11-26 09:11:25 +01:00 · 2024-11-11 21:04:49 +01:00
9 changed files with 55 additions and 186 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -411,7 +411,7 @@ jobs:
        run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
          python devscripts/install_deps.py -o --include build
          python devscripts/install_deps.py --include curl-cffi
-          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.11.1-py3-none-any.whl"
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.10.0-py3-none-any.whl"

      - name: Prepare
        run: |
@ -460,7 +460,7 @@ jobs:
        run: |
          python devscripts/install_deps.py -o --include build
          python devscripts/install_deps.py
-          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.11.1-py3-none-any.whl"
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.10.0-py3-none-any.whl"

      - name: Prepare
        run: |
--- a/pyproject.toml
+++ b/pyproject.toml
@ -83,7 +83,7 @@ test = [
    "pytest-rerunfailures~=14.0",
 ]
 pyinstaller = [
-    "pyinstaller>=6.11.1",  # Windows temp cleanup fixed in 6.11.1
+    "pyinstaller>=6.10.0",  # Windows temp cleanup fixed in 6.10.0
 ]

 [project.urls]
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@ -1156,7 +1156,6 @@ from .mitele import MiTeleIE
 from .mixch import (
    MixchArchiveIE,
    MixchIE,
-    MixchMovieIE,
 )
 from .mixcloud import (
    MixcloudIE,
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@ -205,26 +205,6 @@ class ArchiveOrgIE(InfoExtractor):
                },
            },
        ],
-    }, {
-        # The reviewbody is None for one of the reviews; just need to extract data without crashing
-        'url': 'https://archive.org/details/gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
-        'info_dict': {
-            'id': 'gd95-04-02.sbd.11622.sbeok.shnf/gd95-04-02d1t04.shn',
-            'ext': 'mp3',
-            'title': 'Stuck Inside of Mobile with the Memphis Blues Again',
-            'creators': ['Grateful Dead'],
-            'duration': 338.31,
-            'track': 'Stuck Inside of Mobile with the Memphis Blues Again',
-            'description': 'md5:764348a470b986f1217ffd38d6ac7b72',
-            'display_id': 'gd95-04-02d1t04.shn',
-            'location': 'Pyramid Arena',
-            'uploader': 'jon@archive.org',
-            'album': '1995-04-02 - Pyramid Arena',
-            'upload_date': '20040519',
-            'track_number': 4,
-            'release_date': '19950402',
-            'timestamp': 1084927901,
-        },
    }]

    @staticmethod
@ -355,7 +335,7 @@ class ArchiveOrgIE(InfoExtractor):
                info['comments'].append({
                    'id': review.get('review_id'),
                    'author': review.get('reviewer'),
-                    'text': join_nonempty('reviewtitle', 'reviewbody', from_dict=review, delim='\n\n'),
+                    'text': str_or_none(review.get('reviewtitle'), '') + '\n\n' + review.get('reviewbody'),
                    'timestamp': unified_timestamp(review.get('createdate')),
                    'parent': 'root'})

--- a/yt_dlp/extractor/chaturbate.py
+++ b/yt_dlp/extractor/chaturbate.py
@ -9,7 +9,7 @@ from ..utils import (


 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.(?P<tld>com|eu|global)/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?:fullvideo/?\?.*?\bb=)?(?P<id>[^/?&#]+)'
    _TESTS = [{
        'url': 'https://www.chaturbate.com/siswet19/',
        'info_dict': {
@ -29,24 +29,15 @@ class ChaturbateIE(InfoExtractor):
    }, {
        'url': 'https://en.chaturbate.com/siswet19/',
        'only_matching': True,
-    }, {
-        'url': 'https://chaturbate.eu/siswet19/',
-        'only_matching': True,
-    }, {
-        'url': 'https://chaturbate.eu/fullvideo/?b=caylin',
-        'only_matching': True,
-    }, {
-        'url': 'https://chaturbate.global/siswet19/',
-        'only_matching': True,
    }]

    _ROOM_OFFLINE = 'Room is currently offline'

    def _real_extract(self, url):
-        video_id, tld = self._match_valid_url(url).group('id', 'tld')
+        video_id = self._match_id(url)

        webpage = self._download_webpage(
-            f'https://chaturbate.{tld}/{video_id}/', video_id,
+            f'https://chaturbate.com/{video_id}/', video_id,
            headers=self.geo_verification_headers())

        found_m3u8_urls = []
--- a/yt_dlp/extractor/cloudflarestream.py
+++ b/yt_dlp/extractor/cloudflarestream.py
@ -8,7 +8,7 @@ class CloudflareStreamIE(InfoExtractor):
    _DOMAIN_RE = r'(?:cloudflarestream\.com|(?:videodelivery|bytehighway)\.net)'
    _EMBED_RE = rf'(?:embed\.|{_SUBDOMAIN_RE}){_DOMAIN_RE}/embed/[^/?#]+\.js\?(?:[^#]+&)?video='
    _ID_RE = r'[\da-f]{32}|eyJ[\w-]+\.[\w-]+\.[\w-]+'
-    _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}(?P<domain>{_DOMAIN_RE})/|{_EMBED_RE})(?P<id>{_ID_RE})'
+    _VALID_URL = rf'https?://(?:{_SUBDOMAIN_RE}{_DOMAIN_RE}/|{_EMBED_RE})(?P<id>{_ID_RE})'
    _EMBED_REGEX = [
        rf'<script[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//{_EMBED_RE}(?:{_ID_RE})(?:(?!\1).)*)\1',
        rf'<iframe[^>]+\bsrc=["\'](?P<url>https?://{_SUBDOMAIN_RE}{_DOMAIN_RE}/[\da-f]{{32}})',
@ -19,7 +19,7 @@ class CloudflareStreamIE(InfoExtractor):
            'id': '31c9291ab41fac05471db4e73aa11717',
            'ext': 'mp4',
            'title': '31c9291ab41fac05471db4e73aa11717',
-            'thumbnail': 'https://cloudflarestream.com/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://videodelivery.net/31c9291ab41fac05471db4e73aa11717/thumbnails/thumbnail.jpg',
        },
        'params': {
            'skip_download': 'm3u8',
@ -30,7 +30,7 @@ class CloudflareStreamIE(InfoExtractor):
            'id': '0e8e040aec776862e1d632a699edf59e',
            'ext': 'mp4',
            'title': '0e8e040aec776862e1d632a699edf59e',
-            'thumbnail': 'https://cloudflarestream.com/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://videodelivery.net/0e8e040aec776862e1d632a699edf59e/thumbnails/thumbnail.jpg',
        },
    }, {
        'url': 'https://watch.cloudflarestream.com/9df17203414fd1db3e3ed74abbe936c1',
@ -54,7 +54,7 @@ class CloudflareStreamIE(InfoExtractor):
            'id': 'eaef9dea5159cf968be84241b5cedfe7',
            'ext': 'mp4',
            'title': 'eaef9dea5159cf968be84241b5cedfe7',
-            'thumbnail': 'https://cloudflarestream.com/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg',
+            'thumbnail': 'https://videodelivery.net/eaef9dea5159cf968be84241b5cedfe7/thumbnails/thumbnail.jpg',
        },
        'params': {
            'skip_download': 'm3u8',
@ -62,9 +62,8 @@ class CloudflareStreamIE(InfoExtractor):
    }]

    def _real_extract(self, url):
-        video_id, domain = self._match_valid_url(url).group('id', 'domain')
-        if domain != 'bytehighway.net':
-            domain = 'cloudflarestream.com'
+        video_id = self._match_id(url)
+        domain = 'bytehighway.net' if 'bytehighway.net/' in url else 'videodelivery.net'
        base_url = f'https://{domain}/{video_id}/'
        if '.' in video_id:
            video_id = self._parse_json(base64.urlsafe_b64decode(
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@ -563,13 +563,13 @@ class FacebookIE(InfoExtractor):
                return extract_video_data(try_get(
                    js_data, lambda x: x['jsmods']['instances'], list) or [])

-        def extract_dash_manifest(vid_data, formats, mpd_url=None):
+        def extract_dash_manifest(video, formats):
            dash_manifest = traverse_obj(
-                vid_data, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', 'manifest_xml', expected_type=str)
+                video, 'dash_manifest', 'playlist', 'dash_manifest_xml_string', expected_type=str)
            if dash_manifest:
                formats.extend(self._parse_mpd_formats(
                    compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest)),
-                    mpd_url=url_or_none(video.get('dash_manifest_url')) or mpd_url))
+                    mpd_url=url_or_none(video.get('dash_manifest_url'))))

        def process_formats(info):
            # Downloads with browser's User-Agent are rate limited. Working around
@ -619,12 +619,9 @@ class FacebookIE(InfoExtractor):
                        video = video['creation_story']
                        video['owner'] = traverse_obj(video, ('short_form_video_context', 'video_owner'))
                        video.update(reel_info)
-
+                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
                    formats = []
                    q = qualities(['sd', 'hd'])
-
-                    # Legacy formats extraction
-                    fmt_data = traverse_obj(video, ('videoDeliveryLegacyFields', {dict})) or video
                    for key, format_id in (('playable_url', 'sd'), ('playable_url_quality_hd', 'hd'),
                                           ('playable_url_dash', ''), ('browser_native_hd_url', 'hd'),
                                           ('browser_native_sd_url', 'sd')):
@ -632,7 +629,7 @@ class FacebookIE(InfoExtractor):
                        if not playable_url:
                            continue
                        if determine_ext(playable_url) == 'mpd':
-                            formats.extend(self._extract_mpd_formats(playable_url, video_id, fatal=False))
+                            formats.extend(self._extract_mpd_formats(playable_url, video_id))
                        else:
                            formats.append({
                                'format_id': format_id,
@ -641,28 +638,6 @@ class FacebookIE(InfoExtractor):
                                'url': playable_url,
                            })
                    extract_dash_manifest(fmt_data, formats)
-
-                    # New videoDeliveryResponse formats extraction
-                    fmt_data = traverse_obj(video, ('videoDeliveryResponseFragment', 'videoDeliveryResponseResult'))
-                    mpd_urls = traverse_obj(fmt_data, ('dash_manifest_urls', ..., 'manifest_url', {url_or_none}))
-                    dash_manifests = traverse_obj(fmt_data, ('dash_manifests', lambda _, v: v['manifest_xml']))
-                    for idx, dash_manifest in enumerate(dash_manifests):
-                        extract_dash_manifest(dash_manifest, formats, mpd_url=traverse_obj(mpd_urls, idx))
-                    if not dash_manifests:
-                        # Only extract from MPD URLs if the manifests are not already provided
-                        for mpd_url in mpd_urls:
-                            formats.extend(self._extract_mpd_formats(mpd_url, video_id, fatal=False))
-                    for prog_fmt in traverse_obj(fmt_data, ('progressive_urls', lambda _, v: v['progressive_url'])):
-                        format_id = traverse_obj(prog_fmt, ('metadata', 'quality', {str.lower}))
-                        formats.append({
-                            'format_id': format_id,
-                            # sd, hd formats w/o resolution info should be deprioritized below DASH
-                            'quality': q(format_id) - 3,
-                            'url': prog_fmt['progressive_url'],
-                        })
-                    for m3u8_url in traverse_obj(fmt_data, ('hls_playlist_urls', ..., 'hls_playlist_url', {url_or_none})):
-                        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', fatal=False, m3u8_id='hls'))
-
                    if not formats:
                        # Do not append false positive entry w/o any formats
                        return
--- a/yt_dlp/extractor/goplay.py
+++ b/yt_dlp/extractor/goplay.py
@ -5,63 +5,56 @@ import hashlib
 import hmac
 import json
 import os
-import re
-import urllib.parse

 from .common import InfoExtractor
 from ..utils import (
    ExtractorError,
-    int_or_none,
-    js_to_json,
-    remove_end,
    traverse_obj,
+    unescapeHTML,
 )


 class GoPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/?#]+/[^/?#]+/|)(?P<id>[^/#]+)'
+    _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/]+/[^/]+/|)(?P<display_id>[^/#]+)'

    _NETRC_MACHINE = 'goplay'

    _TESTS = [{
-        'url': 'https://www.goplay.be/video/de-slimste-mens-ter-wereld/de-slimste-mens-ter-wereld-s22/de-slimste-mens-ter-wereld-s22-aflevering-1',
+        'url': 'https://www.goplay.be/video/de-container-cup/de-container-cup-s3/de-container-cup-s3-aflevering-2#autoplay',
        'info_dict': {
-            'id': '2baa4560-87a0-421b-bffc-359914e3c387',
+            'id': '9c4214b8-e55d-4e4b-a446-f015f6c6f811',
            'ext': 'mp4',
-            'title': 'S22 - Aflevering 1',
-            'description': r're:In aflevering 1 nemen Daan Alferink, Tess Elst en Xander De Rycke .{66}',
-            'series': 'De Slimste Mens ter Wereld',
-            'episode': 'Episode 1',
-            'season_number': 22,
-            'episode_number': 1,
-            'season': 'Season 22',
+            'title': 'S3 - Aflevering 2',
+            'series': 'De Container Cup',
+            'season': 'Season 3',
+            'season_number': 3,
+            'episode': 'Episode 2',
+            'episode_number': 2,
        },
-        'params': {'skip_download': True},
        'skip': 'This video is only available for registered users',
    }, {
-        'url': 'https://www.goplay.be/video/1917',
+        'url': 'https://www.goplay.be/video/a-family-for-thr-holidays-s1-aflevering-1#autoplay',
        'info_dict': {
-            'id': '40cac41d-8d29-4ef5-aa11-75047b9f0907',
+            'id': '74e3ed07-748c-49e4-85a0-393a93337dbf',
            'ext': 'mp4',
-            'title': '1917',
-            'description': r're:Op het hoogtepunt van de Eerste Wereldoorlog krijgen twee jonge .{94}',
+            'title': 'A Family for the Holidays',
        },
-        'params': {'skip_download': True},
        'skip': 'This video is only available for registered users',
    }, {
        'url': 'https://www.goplay.be/video/de-mol/de-mol-s11/de-mol-s11-aflevering-1#autoplay',
        'info_dict': {
-            'id': 'ecb79672-92b9-4cd9-a0d7-e2f0250681ee',
+            'id': '03eb8f2f-153e-41cb-9805-0d3a29dab656',
            'ext': 'mp4',
            'title': 'S11 - Aflevering 1',
-            'description': r're:Tien kandidaten beginnen aan hun verovering van Amerika en ontmoeten .{102}',
            'episode': 'Episode 1',
            'series': 'De Mol',
            'season_number': 11,
            'episode_number': 1,
            'season': 'Season 11',
        },
-        'params': {'skip_download': True},
+        'params': {
+            'skip_download': True,
+        },
        'skip': 'This video is only available for registered users',
    }]

@ -76,42 +69,27 @@ class GoPlayIE(InfoExtractor):
        if not self._id_token:
            raise self.raise_login_required(method='password')

-    def _find_json(self, s):
-        return self._search_json(
-            r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None)
-
    def _real_extract(self, url):
-        display_id = self._match_id(url)
+        url, display_id = self._match_valid_url(url).group(0, 'display_id')
        webpage = self._download_webpage(url, display_id)
+        video_data_json = self._html_search_regex(r'<div\s+data-hero="([^"]+)"', webpage, 'video_data')
+        video_data = self._parse_json(unescapeHTML(video_data_json), display_id).get('data')

-        nextjs_data = traverse_obj(
-            re.findall(r'<script[^>]*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*</script>', webpage),
-            (..., {js_to_json}, {json.loads}, ..., {self._find_json}, ...))
-        meta = traverse_obj(nextjs_data, (
-            ..., lambda _, v: v['meta']['path'] == urllib.parse.urlparse(url).path, 'meta', any))
-
-        video_id = meta['uuid']
-        info_dict = traverse_obj(meta, {
-            'title': ('title', {str}),
-            'description': ('description', {str.strip}),
-        })
-
-        if traverse_obj(meta, ('program', 'subtype')) != 'movie':
-            for season_data in traverse_obj(nextjs_data, (..., 'children', ..., 'playlists', ...)):
-                episode_data = traverse_obj(
-                    season_data, ('videos', lambda _, v: v['videoId'] == video_id, any))
-                if not episode_data:
-                    continue
-
-                episode_title = traverse_obj(
-                    episode_data, 'contextualTitle', 'episodeTitle', expected_type=str)
-                info_dict.update({
-                    'title': episode_title or info_dict.get('title'),
-                    'series': remove_end(info_dict.get('title'), f' - {episode_title}'),
-                    'season_number': traverse_obj(season_data, ('season', {int_or_none})),
-                    'episode_number': traverse_obj(episode_data, ('episodeNumber', {int_or_none})),
-                })
-                break
+        movie = video_data.get('movie')
+        if movie:
+            video_id = movie['videoUuid']
+            info_dict = {
+                'title': movie.get('title'),
+            }
+        else:
+            episode = traverse_obj(video_data, ('playlists', ..., 'episodes', lambda _, v: v['pageInfo']['url'] == url), get_all=False)
+            video_id = episode['videoUuid']
+            info_dict = {
+                'title': episode.get('episodeTitle'),
+                'series': traverse_obj(episode, ('program', 'title')),
+                'season_number': episode.get('seasonNumber'),
+                'episode_number': episode.get('episodeNumber'),
+            }

        api = self._download_json(
            f'https://api.goplay.be/web/v1/videos/long-form/{video_id}',
--- a/yt_dlp/extractor/mixch.py
+++ b/yt_dlp/extractor/mixch.py
@ -12,7 +12,7 @@ from ..utils.traversal import traverse_obj

 class MixchIE(InfoExtractor):
    IE_NAME = 'mixch'
-    _VALID_URL = r'https?://mixch\.tv/u/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?mixch\.tv/u/(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://mixch.tv/u/16943797/live',
@ -74,7 +74,7 @@ class MixchIE(InfoExtractor):

 class MixchArchiveIE(InfoExtractor):
    IE_NAME = 'mixch:archive'
-    _VALID_URL = r'https?://mixch\.tv/archive/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?mixch\.tv/archive/(?P<id>\d+)'

    _TESTS = [{
        'url': 'https://mixch.tv/archive/421',
@ -116,56 +116,3 @@ class MixchArchiveIE(InfoExtractor):
            'formats': self._extract_m3u8_formats(info_json['archiveURL'], video_id),
            'thumbnail': traverse_obj(info_json, ('thumbnailURL', {url_or_none})),
        }
-
-
-class MixchMovieIE(InfoExtractor):
-    IE_NAME = 'mixch:movie'
-    _VALID_URL = r'https?://mixch\.tv/m/(?P<id>\w+)'
-
-    _TESTS = [{
-        'url': 'https://mixch.tv/m/Ve8KNkJ5',
-        'info_dict': {
-            'id': 'Ve8KNkJ5',
-            'title': '夏☀️\nムービーへのポイントは本イベントに加算されないので配信にてお願い致します🙇🏻\u200d♀️\n#TGCCAMPUS #ミス東大 #ミス東大2024 ',
-            'ext': 'mp4',
-            'uploader': 'ミス東大No.5 松藤百香🍑💫',
-            'uploader_id': '12299174',
-            'channel_follower_count': int,
-            'view_count': int,
-            'like_count': int,
-            'comment_count': int,
-            'timestamp': 1724070828,
-            'uploader_url': 'https://mixch.tv/u/12299174',
-            'live_status': 'not_live',
-            'upload_date': '20240819',
-        },
-    }, {
-        'url': 'https://mixch.tv/m/61DzpIKE',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        data = self._download_json(
-            f'https://mixch.tv/api-web/movies/{video_id}', video_id)
-        return {
-            'id': video_id,
-            'formats': [{
-                'format_id': 'mp4',
-                'url': data['movie']['file'],
-                'ext': 'mp4',
-            }],
-            **traverse_obj(data, {
-                'title': ('movie', 'title', {str}),
-                'thumbnail': ('movie', 'thumbnailURL', {url_or_none}),
-                'uploader': ('ownerInfo', 'name', {str}),
-                'uploader_id': ('ownerInfo', 'id', {int}, {str_or_none}),
-                'channel_follower_count': ('ownerInfo', 'fan', {int_or_none}),
-                'view_count': ('ownerInfo', 'view', {int_or_none}),
-                'like_count': ('movie', 'favCount', {int_or_none}),
-                'comment_count': ('movie', 'commentCount', {int_or_none}),
-                'timestamp': ('movie', 'published', {int_or_none}),
-                'uploader_url': ('ownerInfo', 'id', {lambda x: x and f'https://mixch.tv/u/{x}'}, filter),
-            }),
-            'live_status': 'not_live',
-        }