yt_dlp/extractor/_extractors.py

@@ -93,7 +93,6 @@ from .alura import (
     AluraIE,
     AluraCourseIE
 )
-from .amadeustv import AmadeusTVIE
 from .amara import AmaraIE
 from .amcnetworks import AMCNetworksIE
 from .amazon import (
@@ -906,7 +905,6 @@ from .koo import KooIE
 from .kth import KTHIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
-from .kukululive import KukuluLiveIE
 from .kusi import KUSIIE
 from .kuwo import (
     KuwoIE,
@@ -1271,7 +1269,6 @@ from .niconicochannelplus import (
     NiconicoChannelPlusChannelLivesIE,
 )
 from .ninegag import NineGagIE
-from .ninenews import NineNewsIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
 from .nitter import NitterIE
@@ -2011,7 +2008,6 @@ from .trovo import (
     TrovoChannelClipIE,
 )
 from .trtcocuk import TrtCocukVideoIE
-from .trtworld import TrtWorldIE
 from .trueid import TrueIDIE
 from .trunews import TruNewsIE
 from .truth import TruthIE

yt_dlp/extractor/abematv.py

@@ -92,8 +92,6 @@ class AbemaLicenseHandler(urllib.request.BaseHandler):
 
 
 class AbemaTVBaseIE(InfoExtractor):
-    _NETRC_MACHINE = 'abematv'
-
     _USERTOKEN = None
     _DEVICE_ID = None
     _MEDIATOKEN = None
@@ -138,15 +136,11 @@ class AbemaTVBaseIE(InfoExtractor):
         if self._USERTOKEN:
             return self._USERTOKEN
 
-        add_opener(self._downloader, AbemaLicenseHandler(self))
-
         username, _ = self._get_login_info()
-        auth_cache = username and self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19')
+        AbemaTVBaseIE._USERTOKEN = username and self.cache.load(self._NETRC_MACHINE, username)
-        AbemaTVBaseIE._USERTOKEN = auth_cache and auth_cache.get('usertoken')
         if AbemaTVBaseIE._USERTOKEN:
             # try authentication with locally stored token
             try:
-                AbemaTVBaseIE._DEVICE_ID = auth_cache.get('device_id')
                 self._get_media_token(True)
                 return
             except ExtractorError as e:
@@ -165,6 +159,7 @@ class AbemaTVBaseIE(InfoExtractor):
             })
         AbemaTVBaseIE._USERTOKEN = user_data['token']
 
+        add_opener(self._downloader, AbemaLicenseHandler(self))
         return self._USERTOKEN
 
     def _get_media_token(self, invalidate=False, to_show=True):
@@ -186,37 +181,6 @@ class AbemaTVBaseIE(InfoExtractor):
 
         return self._MEDIATOKEN
 
-    def _perform_login(self, username, password):
-        self._get_device_token()
-        if self.cache.load(self._NETRC_MACHINE, username, min_ver='2024.01.19') and self._get_media_token():
-            self.write_debug('Skipping logging in')
-            return
-
-        if '@' in username:  # don't strictly check if it's email address or not
-            ep, method = 'user/email', 'email'
-        else:
-            ep, method = 'oneTimePassword', 'userId'
-
-        login_response = self._download_json(
-            f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in',
-            data=json.dumps({
-                method: username,
-                'password': password
-            }).encode('utf-8'), headers={
-                'Authorization': f'bearer {self._get_device_token()}',
-                'Origin': 'https://abema.tv',
-                'Referer': 'https://abema.tv/',
-                'Content-Type': 'application/json',
-            })
-
-        AbemaTVBaseIE._USERTOKEN = login_response['token']
-        self._get_media_token(True)
-        auth_cache = {
-            'device_id': AbemaTVBaseIE._DEVICE_ID,
-            'usertoken': AbemaTVBaseIE._USERTOKEN,
-        }
-        self.cache.store(self._NETRC_MACHINE, username, auth_cache)
-
     def _call_api(self, endpoint, video_id, query=None, note='Downloading JSON metadata'):
         return self._download_json(
             f'https://api.abema.io/{endpoint}', video_id, query=query or {},
@@ -240,6 +204,7 @@ class AbemaTVBaseIE(InfoExtractor):
 
 class AbemaTVIE(AbemaTVBaseIE):
     _VALID_URL = r'https?://abema\.tv/(?P<type>now-on-air|video/episode|channels/.+?/slots)/(?P<id>[^?/]+)'
+    _NETRC_MACHINE = 'abematv'
     _TESTS = [{
         'url': 'https://abema.tv/video/episode/194-25_s2_p1',
         'info_dict': {
@@ -288,6 +253,33 @@ class AbemaTVIE(AbemaTVBaseIE):
     }]
     _TIMETABLE = None
 
+    def _perform_login(self, username, password):
+        self._get_device_token()
+        if self.cache.load(self._NETRC_MACHINE, username) and self._get_media_token():
+            self.write_debug('Skipping logging in')
+            return
+
+        if '@' in username:  # don't strictly check if it's email address or not
+            ep, method = 'user/email', 'email'
+        else:
+            ep, method = 'oneTimePassword', 'userId'
+
+        login_response = self._download_json(
+            f'https://api.abema.io/v1/auth/{ep}', None, note='Logging in',
+            data=json.dumps({
+                method: username,
+                'password': password
+            }).encode('utf-8'), headers={
+                'Authorization': f'bearer {self._get_device_token()}',
+                'Origin': 'https://abema.tv',
+                'Referer': 'https://abema.tv/',
+                'Content-Type': 'application/json',
+            })
+
+        AbemaTVBaseIE._USERTOKEN = login_response['token']
+        self._get_media_token(True)
+        self.cache.store(self._NETRC_MACHINE, username, AbemaTVBaseIE._USERTOKEN)
+
     def _real_extract(self, url):
         # starting download using infojson from this extractor is undefined behavior,
         # and never be fixed in the future; you must trigger downloads by directly specifying URL.

yt_dlp/extractor/amadeustv.py

@@ -1,77 +0,0 @@
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-    url_or_none,
-)
-from ..utils.traversal import traverse_obj
-
-
-class AmadeusTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?amadeus\.tv/library/(?P<id>[\da-f]+)'
-    _TESTS = [{
-        'url': 'http://www.amadeus.tv/library/65091a87ff85af59d9fc54c3',
-        'info_dict': {
-            'id': '5576678021301411311',
-            'ext': 'mp4',
-            'title': 'Jieon Park - 第五届珠海莫扎特国际青少年音乐周小提琴C组第三轮',
-            'thumbnail': 'http://1253584441.vod2.myqcloud.com/a0046a27vodtransbj1253584441/7db4af535576678021301411311/coverBySnapshot_10_0.jpg',
-            'duration': 1264.8,
-            'upload_date': '20230918',
-            'timestamp': 1695034800,
-            'display_id': '65091a87ff85af59d9fc54c3',
-            'view_count': int,
-            'description': 'md5:a0357b9c215489e2067cbae0b777bb95',
-        }
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        nuxt_data = self._search_nuxt_data(webpage, display_id, traverse=('fetch', '0'))
-        video_id = traverse_obj(nuxt_data, ('item', 'video', {str}))
-
-        if not video_id:
-            raise ExtractorError('Unable to extract actual video ID')
-
-        video_data = self._download_json(
-            f'http://playvideo.qcloud.com/getplayinfo/v2/1253584441/{video_id}',
-            video_id, headers={'Referer': 'http://www.amadeus.tv/'})
-
-        formats = []
-        for video in traverse_obj(video_data, ('videoInfo', ('sourceVideo', ('transcodeList', ...)), {dict})):
-            if not url_or_none(video.get('url')):
-                continue
-            formats.append({
-                **traverse_obj(video, {
-                    'url': 'url',
-                    'format_id': ('definition', {lambda x: f'http-{x or "0"}'}),
-                    'width': ('width', {int_or_none}),
-                    'height': ('height', {int_or_none}),
-                    'filesize': (('totalSize', 'size'), {int_or_none}),
-                    'vcodec': ('videoStreamList', 0, 'codec'),
-                    'acodec': ('audioStreamList', 0, 'codec'),
-                    'fps': ('videoStreamList', 0, 'fps', {float_or_none}),
-                }, get_all=False),
-                'http_headers': {'Referer': 'http://www.amadeus.tv/'},
-            })
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'formats': formats,
-            **traverse_obj(video_data, {
-                'title': ('videoInfo', 'basicInfo', 'name', {str}),
-                'thumbnail': ('coverInfo', 'coverUrl', {url_or_none}),
-                'duration': ('videoInfo', 'sourceVideo', ('floatDuration', 'duration'), {float_or_none}),
-            }, get_all=False),
-            **traverse_obj(nuxt_data, ('item', {
-                'title': (('title', 'title_en', 'title_cn'), {str}),
-                'description': (('description', 'description_en', 'description_cn'), {str}),
-                'timestamp': ('date', {parse_iso8601}),
-                'view_count': ('view', {int_or_none}),
-            }), get_all=False),
-        }

yt_dlp/extractor/kukululive.py

@@ -1,140 +0,0 @@
-import urllib.parse
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    filter_dict,
-    get_element_by_id,
-    int_or_none,
-    join_nonempty,
-    js_to_json,
-    qualities,
-    url_or_none,
-    urljoin,
-)
-from ..utils.traversal import traverse_obj
-
-
-class KukuluLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://live\.erinn\.biz/live\.php\?h(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'https://live.erinn.biz/live.php?h675134569',
-        'md5': 'e380fa6a47fc703d91cea913ab44ec2e',
-        'info_dict': {
-            'id': '675134569',
-            'ext': 'mp4',
-            'title': 'プロセカ',
-            'description': 'テストも兼ねたプロセカ配信。',
-            'timestamp': 1702689148,
-            'upload_date': '20231216',
-            'thumbnail': r're:^https?://.*',
-        },
-    }, {
-        'url': 'https://live.erinn.biz/live.php?h102338092',
-        'md5': 'dcf5167a934b1c60333461e13a81a6e2',
-        'info_dict': {
-            'id': '102338092',
-            'ext': 'mp4',
-            'title': 'Among Usで遊びます！！',
-            'description': 'VTuberになりましたねんねこ㌨ですよろしくお願いします',
-            'timestamp': 1704603118,
-            'upload_date': '20240107',
-            'thumbnail': r're:^https?://.*',
-        },
-    }, {
-        'url': 'https://live.erinn.biz/live.php?h878049531',
-        'only_matching': True,
-    }]
-
-    def _get_quality_meta(self, video_id, desc, code, force_h264=None):
-        desc += ' (force_h264)' if force_h264 else ''
-        qs = self._download_webpage(
-            'https://live.erinn.biz/live.player.fplayer.php', video_id,
-            f'Downloading {desc} quality metadata', f'Unable to download {desc} quality metadata',
-            query=filter_dict({
-                'hash': video_id,
-                'action': f'get{code}liveByAjax',
-                'force_h264': force_h264,
-            }))
-        return urllib.parse.parse_qs(qs)
-
-    def _add_quality_formats(self, formats, quality_meta):
-        vcodec = traverse_obj(quality_meta, ('vcodec', 0, {str}))
-        quality = traverse_obj(quality_meta, ('now_quality', 0, {str}))
-        quality_priority = qualities(('low', 'h264', 'high'))(quality)
-        if traverse_obj(quality_meta, ('hlsaddr', 0, {url_or_none})):
-            formats.append({
-                'format_id': quality,
-                'url': quality_meta['hlsaddr'][0],
-                'ext': 'mp4',
-                'vcodec': vcodec,
-                'quality': quality_priority,
-            })
-        if traverse_obj(quality_meta, ('hlsaddr_audioonly', 0, {url_or_none})):
-            formats.append({
-                'format_id': join_nonempty(quality, 'audioonly'),
-                'url': quality_meta['hlsaddr_audioonly'][0],
-                'ext': 'm4a',
-                'vcodec': 'none',
-                'quality': quality_priority,
-            })
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        html = self._download_webpage(url, video_id)
-
-        if '>タイムシフトが見つかりませんでした。<' in html:
-            raise ExtractorError('This stream has expired', expected=True)
-
-        title = clean_html(
-            get_element_by_id('livetitle', html.replace('<SPAN', '<span').replace('SPAN>', 'span>')))
-        description = self._html_search_meta('Description', html)
-        thumbnail = self._html_search_meta(['og:image', 'twitter:image'], html)
-
-        if self._search_regex(r'(var\s+timeshift\s*=\s*false)', html, 'is livestream', default=False):
-            formats = []
-            for (desc, code) in [('high', 'Z'), ('low', 'ForceLow')]:
-                quality_meta = self._get_quality_meta(video_id, desc, code)
-                self._add_quality_formats(formats, quality_meta)
-                if desc == 'high' and traverse_obj(quality_meta, ('vcodec', 0)) == 'HEVC':
-                    self._add_quality_formats(
-                        formats, self._get_quality_meta(video_id, desc, code, force_h264='1'))
-
-            return {
-                'id': video_id,
-                'title': title,
-                'description': description,
-                'thumbnail': thumbnail,
-                'is_live': True,
-                'formats': formats,
-            }
-
-        # VOD extraction
-        player_html = self._download_webpage(
-            'https://live.erinn.biz/live.timeshift.fplayer.php', video_id,
-            'Downloading player html', 'Unable to download player html', query={'hash': video_id})
-
-        sources = traverse_obj(self._search_json(
-            r'var\s+fplayer_source\s*=', player_html, 'stream data', video_id,
-            contains_pattern=r'\[(?s:.+)\]', transform_source=js_to_json), lambda _, v: v['file'])
-
-        def entries(segments, playlist=True):
-            for i, segment in enumerate(segments, 1):
-                yield {
-                    'id': f'{video_id}_{i}' if playlist else video_id,
-                    'title': f'{title} (Part {i})' if playlist else title,
-                    'description': description,
-                    'timestamp': traverse_obj(segment, ('time_start', {int_or_none})),
-                    'thumbnail': thumbnail,
-                    'formats': [{
-                        'url': urljoin('https://live.erinn.biz', segment['file']),
-                        'ext': 'mp4',
-                        'protocol': 'm3u8_native',
-                    }],
-                }
-
-        if len(sources) == 1:
-            return next(entries(sources, playlist=False))
-
-        return self.playlist_result(entries(sources), video_id, title, description, multi_video=True)

yt_dlp/extractor/ninenews.py

@@ -1,72 +0,0 @@
-from .common import InfoExtractor
-from .brightcove import BrightcoveNewIE
-from ..utils import ExtractorError
-from ..utils.traversal import traverse_obj
-
-
-class NineNewsIE(InfoExtractor):
-    IE_NAME = '9News'
-    _VALID_URL = r'https?://(?:www\.)?9news\.com\.au/(?:[\w-]+/){2,3}(?P<id>[\w-]+)/?(?:$|[?#])'
-    _TESTS = [{
-        'url': 'https://www.9news.com.au/videos/national/fair-trading-pulls-dozens-of-toys-from-shelves/clqgc7dvj000y0jnvfism0w5m',
-        'md5': 'd1a65b2e9d126e5feb9bc5cb96e62c80',
-        'info_dict': {
-            'id': '6343717246112',
-            'ext': 'mp4',
-            'title': 'Fair Trading pulls dozens of toys from shelves',
-            'description': 'Fair Trading Australia have been forced to pull dozens of toys from shelves over hazard fears.',
-            'thumbnail': 'md5:bdbe44294e2323b762d97acf8843f66c',
-            'duration': 93.44,
-            'timestamp': 1703231748,
-            'upload_date': '20231222',
-            'uploader_id': '664969388001',
-            'tags': ['networkclip', 'aunews_aunationalninenews', 'christmas presents', 'toys', 'fair trading', 'au_news'],
-        }
-    }, {
-        'url': 'https://www.9news.com.au/world/tape-reveals-donald-trump-pressured-michigan-officials-not-to-certify-2020-vote-a-new-report-says/0b8b880e-7d3c-41b9-b2bd-55bc7e492259',
-        'md5': 'a885c44d20898c3e70e9a53e8188cea1',
-        'info_dict': {
-            'id': '6343587450112',
-            'ext': 'mp4',
-            'title': 'Trump found ineligible to run for president by state court',
-            'description': 'md5:40e6e7db7a4ac6be0e960569a5af6066',
-            'thumbnail': 'md5:3e132c48c186039fd06c10787de9bff2',
-            'duration': 104.64,
-            'timestamp': 1703058034,
-            'upload_date': '20231220',
-            'uploader_id': '664969388001',
-            'tags': ['networkclip', 'aunews_aunationalninenews', 'ineligible', 'presidential candidate', 'donald trump', 'au_news'],
-        }
-    }, {
-        'url': 'https://www.9news.com.au/national/outrage-as-parents-banned-from-giving-gifts-to-kindergarten-teachers/e19b49d4-a1a4-4533-9089-6e10e2d9386a',
-        'info_dict': {
-            'id': '6343716797112',
-            'ext': 'mp4',
-            'title': 'Outrage as parents banned from giving gifts to kindergarten teachers',
-            'description': 'md5:7a8b0ed2f9e08875fd9a3e86e462bc46',
-            'thumbnail': 'md5:5ee4d66717bdd0dee9fc9a705ef041b8',
-            'duration': 91.307,
-            'timestamp': 1703229584,
-            'upload_date': '20231222',
-            'uploader_id': '664969388001',
-            'tags': ['networkclip', 'aunews_aunationalninenews', 'presents', 'teachers', 'kindergarten', 'au_news'],
-        },
-    }]
-
-    def _real_extract(self, url):
-        article_id = self._match_id(url)
-        webpage = self._download_webpage(url, article_id)
-        initial_state = self._search_json(
-            r'var\s+__INITIAL_STATE__\s*=', webpage, 'initial state', article_id)
-        video_id = traverse_obj(
-            initial_state, ('videoIndex', 'currentVideo', 'brightcoveId', {str}),
-            ('article', ..., 'media', lambda _, v: v['type'] == 'video', 'urn', {str}), get_all=False)
-        account = traverse_obj(initial_state, (
-            'videoIndex', 'config', (None, 'video'), 'account', {str}), get_all=False)
-
-        if not video_id or not account:
-            raise ExtractorError('Unable to get the required video data')
-
-        return self.url_result(
-            f'https://players.brightcove.net/{account}/default_default/index.html?videoId={video_id}',
-            BrightcoveNewIE, video_id)

yt_dlp/extractor/trtworld.py

@@ -1,101 +0,0 @@
-from .common import InfoExtractor
-from ..utils import ExtractorError, determine_ext, parse_iso8601, url_or_none
-from ..utils.traversal import traverse_obj
-
-
-class TrtWorldIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.trtworld\.com/video/[\w-]+/[\w-]+-(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'https://www.trtworld.com/video/news/turkiye-switches-to-sustainable-tourism-16067690',
-        'info_dict': {
-            'id': '16067690',
-            'ext': 'mp4',
-            'title': 'Türkiye switches to sustainable tourism',
-            'release_timestamp': 1701529569,
-            'release_date': '20231202',
-            'thumbnail': 'https://cdn-i.pr.trt.com.tr/trtworld/17647563_0-0-1920-1080.jpeg',
-            'description': 'md5:0a975c04257fb529c8f99c7b76a2cf12',
-        }
-    }, {
-        'url': 'https://www.trtworld.com/video/one-offs/frames-from-anatolia-recreating-a-james-bond-scene-in-istanbuls-grand-bazaar-14541780',
-        'info_dict': {
-            'id': '14541780',
-            'ext': 'mp4',
-            'title': 'Frames From Anatolia: Recreating a ‘James Bond’ Scene in Istanbul’s Grand Bazaar',
-            'release_timestamp': 1692440844,
-            'release_date': '20230819',
-            'thumbnail': 'https://cdn-i.pr.trt.com.tr/trtworld/16939810_0-0-1920-1080.jpeg',
-            'description': 'md5:4050e21570cc3c40b6c9badae800a94f',
-        }
-    }, {
-        'url': 'https://www.trtworld.com/video/the-newsmakers/can-sudan-find-peace-amidst-failed-transition-to-democracy-12904760',
-        'info_dict': {
-            'id': '12904760',
-            'ext': 'mp4',
-            'title': 'Can Sudan find peace amidst failed transition to democracy?',
-            'release_timestamp': 1681972747,
-            'release_date': '20230420',
-            'thumbnail': 'http://cdni0.trtworld.com/w768/q70/154214_NMYOUTUBETEMPLATE1_1681833018736.jpg'
-        }
-    }, {
-        'url': 'https://www.trtworld.com/video/africa-matters/locals-learning-to-cope-with-rising-tides-of-kenyas-great-lakes-16059545',
-        'info_dict': {
-            'id': 'zEns2dWl00w',
-            'ext': 'mp4',
-            'title': "Locals learning to cope with rising tides of Kenya's Great Lakes",
-            'thumbnail': 'https://i.ytimg.com/vi/zEns2dWl00w/maxresdefault.jpg',
-            'description': 'md5:3ad9d7c5234d752a4ead4340c79c6b8d',
-            'channel_id': 'UC7fWeaHhqgM4Ry-RMpM2YYw',
-            'channel_url': 'https://www.youtube.com/channel/UC7fWeaHhqgM4Ry-RMpM2YYw',
-            'duration': 210,
-            'view_count': int,
-            'age_limit': 0,
-            'webpage_url': 'https://www.youtube.com/watch?v=zEns2dWl00w',
-            'categories': ['News & Politics'],
-            'channel': 'TRT World',
-            'channel_follower_count': int,
-            'channel_is_verified': True,
-            'uploader': 'TRT World',
-            'uploader_id': '@trtworld',
-            'uploader_url': 'https://www.youtube.com/@trtworld',
-            'upload_date': '20231202',
-            'availability': 'public',
-            'comment_count': int,
-            'playable_in_embed': True,
-            'tags': [],
-            'live_status': 'not_live',
-            'like_count': int,
-        }
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        nuxtjs_data = self._search_nuxt_data(webpage, display_id)['videoData']['content']['platforms']
-        formats = []
-        for media_url in traverse_obj(nuxtjs_data, (
-                ('website', 'ott'), 'metadata', ('hls_url', 'url'), {url_or_none})):
-            # NB: Website sometimes serves mp4 files under `hls_url` key
-            if determine_ext(media_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(media_url, display_id, fatal=False))
-            else:
-                formats.append({
-                    'format_id': 'http',
-                    'url': media_url,
-                })
-        if not formats:
-            if youtube_id := traverse_obj(nuxtjs_data, ('youtube', 'metadata', 'youtubeId')):
-                return self.url_result(youtube_id, 'Youtube')
-            raise ExtractorError('No video found', expected=True)
-
-        return {
-            'id': display_id,
-            'formats': formats,
-            **traverse_obj(nuxtjs_data, (('website', 'ott'), {
-                'title': ('fields', 'title', 'text', {str}),
-                'description': ('fields', 'description', 'text', {str}),
-                'thumbnail': ('fields', 'thumbnail', 'url', {url_or_none}),
-                'release_timestamp': ('published', 'date', {parse_iso8601}),
-            }), get_all=False),
-        }