[ie/NYTimes] - more adjustments toNYTimesArticleIE

yt_dlp/extractor/nytimes.py

@@ -15,7 +15,6 @@ from ..utils import (
     remove_start,
     str_or_none,
     traverse_obj,
-    unified_strdate,
     url_or_none,
 )
 
@@ -23,11 +22,18 @@ from ..utils import (
 class NYTimesBaseIE(InfoExtractor):
     _GRAPHQL_API = 'https://samizdat-graphql.nytimes.com/graphql/v2'
 
+    def _extract_thumbnails(self, thumbs):
+        return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
+            'url': 'url',
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+        }))
+
     def _extract_media_from_json(self, video_id, content_media_json):
         urls = []
         formats = []
         subtitles = {}
-        for video in traverse_obj(content_media_json, ('renditions', ...)):
+        for video in traverse_obj(content_media_json, ('renditions', ..., {dict})):
             video_url = video.get('url')
             format_id = video.get('type')
             if not video_url or format_id == 'thumbs' or video_url in urls:
@@ -169,7 +175,7 @@ class NYTimesArticleIE(NYTimesBaseIE):
         # audio articles will have an url and no formats
         url = traverse_obj(block, ('fileUrl', {url_or_none}))
         if not formats and url:
-            formats.append({'url': url, 'ext': determine_ext(url)})
+            formats.append({'url': url})
 
         return {
             **details,
@@ -177,54 +183,43 @@ class NYTimesArticleIE(NYTimesBaseIE):
             'subtitles': subtitles
         }
 
-    def _extract_thumbnails(self, thumbs):
-        return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
-            'url': 'url',
-            'width': ('width', {int_or_none}),
-            'height': ('height', {int_or_none}),
-        }))
-
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
-        article_json = self._search_json(
+        art_json = self._search_json(
             r'window\.__preloadedData\s*=', webpage, 'media details', page_id,
             transform_source=lambda x: x.replace('undefined', 'null'))['initialData']['data']['article']
 
-        blocks = traverse_obj(article_json, (
+        common_info = {
+            'title': remove_end(self._html_extract_title(webpage), ' - The New York Times'),
+            'description': traverse_obj(art_json, (
+                'sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text', {str}),
+                get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage),
+            'timestamp': traverse_obj(art_json, ('firstPublished', {parse_iso8601})),
+            'creator': ', '.join(
+                traverse_obj(art_json, ('bylines', ..., 'creators', ..., 'displayName')))  # TODO: change to 'creators' (list)
+        }
+
+        blocks = traverse_obj(art_json, (
             'sprinkledBody', 'content', ..., ('ledeMedia', None),
             lambda _, v: v['__typename'] in ('Video', 'Audio')))
 
-        art_title = remove_end(self._html_extract_title(webpage), ' - The New York Times')
-        art_description = traverse_obj(
-            article_json, ('sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text'),
-            get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage)
-        art_upload_date = traverse_obj(article_json, ('firstPublished'))
-        creator = ', '.join(
-            traverse_obj(article_json, ('bylines', ..., 'creators', ..., 'displayName')))
-
         # more than 1 video in the article, treat it as a playlist
         if len(blocks) > 1:
             entries = []
             for block in blocks:
-                entries.append(merge_dicts(self._extract_content_from_block(block), {
+                entries.append(merge_dicts(
-                    'title': art_title,
+                    self._extract_content_from_block(block), common_info, {
-                    'description': art_description,
+                        'thumbnails': self._extract_thumbnails(traverse_obj(
-                    'creator': creator,  # TODO: change to 'creators' (list)
+                            block, ('promotionalMedia', 'crops', ..., 'renditions', ...)))}))
-                    'thumbnails': self._extract_thumbnails(traverse_obj(
-                        block, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
-                }))
-            return self.playlist_result(entries, page_id, art_title, art_description)
 
-        return merge_dicts(self._extract_content_from_block(blocks[0]), {
+            return self.playlist_result(entries, page_id, common_info.get('title'),
-            'title': art_title,
+                                        common_info.get('description'))
-            'description': art_description,
+
-            'creator': creator,  # TODO: change to 'creators'
+        return merge_dicts(self._extract_content_from_block(blocks[0]), common_info, {
-            'upload_date': unified_strdate(art_upload_date),
-            'timestamp': parse_iso8601(art_upload_date),
             'thumbnails': self._extract_thumbnails(traverse_obj(
                 blocks[0], ('promotionalMedia', 'crops', ..., 'renditions', ...)) or traverse_obj(
-                article_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
+                art_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
         })
 
 
@@ -277,40 +272,22 @@ class NYTimesCookingIE(InfoExtractor):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
-        json_obj = traverse_obj(
+        next_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']
-            self._search_nextjs_data(webpage, page_id), ('props', 'pageProps'), default={})
-
-        info = traverse_obj(json_obj, {
-            'id': ('recipe', 'id', {str_or_none}),
-            'title': ('recipe', 'title'),
-            'description': ('recipe', 'topnote', {clean_html}),
-            'timestamp': ('recipe', 'publishedAt'),
-            'creator': ('recipe', 'contentAttribution', 'cardByline'),
-            'upload_date': ('meta', 'jsonLD', 'video', 'uploadDate', {unified_strdate}),
-            'formats': ('recipe', 'videoSrc', {url_or_none}),
-        })
 
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            info.get('formats'), info.get('id'), 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+            next_data['recipe']['videoSrc'], page_id, 'mp4', m3u8_id='hls')
-
-        thumbnails = []
-        for image in traverse_obj(json_obj, ('recipe', 'image', 'crops', 'recipe', ...)):
-            if not url_or_none(image):
-                continue
-            thumbnails.append({
-                'url': image,
-            })
 
         return {
-            'id': info.get('id'),
+            **traverse_obj(next_data, {
-            'title': info.get('title'),
+                'id': ('recipe', 'id', {str_or_none}),
-            'description': info.get('description'),
+                'title': ('recipe', 'title'),
-            'timestamp': info.get('timestamp'),
+                'description': ('recipe', 'topnote', {clean_html}),
-            'upload_date': info.get(' upload_date'),
+                'timestamp': ('recipe', 'publishedAt'),
-            'creator': info.get('creator'),
+                'creator': ('recipe', 'contentAttribution', 'cardByline')}),
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnails': thumbnails,
+            'thumbnails': [{'url': url} for url in traverse_obj(next_data, (
+                'recipe', 'image', 'crops', 'recipe', lambda _, v: url_or_none(v)))]
         }
 
 
@@ -349,7 +326,7 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
     }]
 
     _TOKEN = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQAB'
-    _DNS_UUID = '36dd619a-56dc-595b-9e09-37f4152c7b5d'  # uuid -v5 ns:DNS scoop.nyt.net
+    _DNS_NAMESPACE = uuid.UUID('36dd619a-56dc-595b-9e09-37f4152c7b5d')
     _GRAPHQL_QUERY = '''query VideoQuery($id: String!) {
   video(id: $id) {
     ... on Video {
@@ -384,7 +361,6 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
 }'''
 
     def _build_playlist(self, media_items):
-        entries = []
         for media_id in media_items:
             json_obj = traverse_obj(self._json_from_graphql(media_id) or {}, ('data', 'video'))
 
@@ -396,53 +372,33 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
             duration = int_or_none(json_obj.get('duration'))
             formats, subtitles = self._extract_media_from_json(media_id, json_obj)
 
-            thumbnails = []
+            yield {
-            for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
-                image_url = image.get('url')
-                if not url_or_none(image_url):
-                    continue
-                thumbnails.append({
-                    'name': image.get('name'),
-                    'url': image_url,
-                    'width': int_or_none(image.get('width')),
-                    'height': int_or_none(image.get('height')),
-                })
-
-            entries.append({
                 'id': media_id,
                 'title': title,
                 'description': description,
                 'duration': duration,
-                'creator': creators,
+                'creator': creators,  # TODO: change to 'creators'
                 'formats': formats,
                 'subtitles': subtitles,
-                'thumbnails': thumbnails,
+                'thumbnails': self._extract_thumbnails(
-            })
+                    traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
-        return entries
+            }
 
     def _json_from_graphql(self, media_id):
         # reference: `id-to-uri.js`
-        namespace = uuid.UUID(self._DNS_UUID)
+        video_uuid = uuid.uuid5(self._DNS_NAMESPACE, 'video')
-        video_uuid = uuid.uuid5(namespace, 'video')
         media_uuid = uuid.uuid5(video_uuid, media_id)
 
-        payload = {
-            'query': self._GRAPHQL_QUERY,
-            'variables': {'id': f'nyt://video/{media_uuid}'}
-        }
-
-        headers = {
-            'Content-Type': 'application/json',
-            'Nyt-App-Type': 'vhs',
-            'Nyt-App-Version': 'v3.52.21',
-            'Nyt-Token': self._TOKEN,
-            'Origin': 'https://cooking.nytimes.com',
-            'Referer': 'https://www.google.com/',
-        }
-
         return self._download_json(
-            self._GRAPHQL_API, id, note='Downloading json from GRAPHQL API',
+            self._GRAPHQL_API, media_id, note='Downloading json from GRAPHQL API', data=json.dumps({
-            data=json.dumps(payload, separators=(',', ':')).encode(), headers=headers, fatal=False)
+                'query': self._GRAPHQL_QUERY, 'variables': {'id': f'nyt://video/{media_uuid}'}},
+                separators=(',', ':')).encode(), headers={
+                    'Content-Type': 'application/json',
+                    'Nyt-App-Type': 'vhs',
+                    'Nyt-App-Version': 'v3.52.21',
+                    'Nyt-Token': self._TOKEN,
+                    'Origin': 'https://cooking.nytimes.com',
+                    'Referer': 'https://www.google.com/'}, fatal=False) or {}
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
@@ -462,28 +418,17 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
             return self.playlist_result(self._build_playlist(media_items), page_id, title, description)
 
         json_obj = traverse_obj(self._json_from_graphql(lead_video_id) or {}, ('data', 'video'))
-        duration = int_or_none(json_obj.get('duration'))
 
         formats, subtitles = self._extract_media_from_json(lead_video_id, json_obj)
 
-        thumbnails = []
-        for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
-            image_url = image.get('url')
-            if not url_or_none(image_url):
-                continue
-            thumbnails.append({
-                'url': image_url,
-                'width': int_or_none(image.get('width')),
-                'height': int_or_none(image.get('height')),
-            })
-
         return {
             'id': lead_video_id,
             'title': title,
             'description': description,
-            'duration': duration,
+            'duration': int_or_none(json_obj.get('duration')),
-            'creator': creator,
+            'creator': creator,  # TODO: change to 'creators'
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnails': thumbnails,
+            'thumbnails': self._extract_thumbnails(
+                traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
         }