yt_dlp/extractor/nytimes.py

@@ -15,6 +15,7 @@ from ..utils import (
     remove_start,
     str_or_none,
     traverse_obj,
+    unified_strdate,
     url_or_none,
 )
 
@@ -22,18 +23,11 @@ from ..utils import (
 class NYTimesBaseIE(InfoExtractor):
     _GRAPHQL_API = 'https://samizdat-graphql.nytimes.com/graphql/v2'
 
-    def _extract_thumbnails(self, thumbs):
-        return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
-            'url': 'url',
-            'width': ('width', {int_or_none}),
-            'height': ('height', {int_or_none}),
-        }))
-
     def _extract_media_from_json(self, video_id, content_media_json):
         urls = []
         formats = []
         subtitles = {}
-        for video in traverse_obj(content_media_json, ('renditions', ..., {dict})):
+        for video in traverse_obj(content_media_json, ('renditions', ...)):
             video_url = video.get('url')
             format_id = video.get('type')
             if not video_url or format_id == 'thumbs' or video_url in urls:
@@ -175,7 +169,7 @@ class NYTimesArticleIE(NYTimesBaseIE):
         # audio articles will have an url and no formats
         url = traverse_obj(block, ('fileUrl', {url_or_none}))
         if not formats and url:
-            formats.append({'url': url})
+            formats.append({'url': url, 'ext': determine_ext(url)})
 
         return {
             **details,
@@ -183,43 +177,54 @@ class NYTimesArticleIE(NYTimesBaseIE):
             'subtitles': subtitles
         }
 
+    def _extract_thumbnails(self, thumbs):
+        return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
+            'url': 'url',
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+        }))
+
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
-        art_json = self._search_json(
+        article_json = self._search_json(
             r'window\.__preloadedData\s*=', webpage, 'media details', page_id,
             transform_source=lambda x: x.replace('undefined', 'null'))['initialData']['data']['article']
 
-        common_info = {
+        blocks = traverse_obj(article_json, (
-            'title': remove_end(self._html_extract_title(webpage), ' - The New York Times'),
-            'description': traverse_obj(art_json, (
-                'sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text', {str}),
-                get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage),
-            'timestamp': traverse_obj(art_json, ('firstPublished', {parse_iso8601})),
-            'creator': ', '.join(
-                traverse_obj(art_json, ('bylines', ..., 'creators', ..., 'displayName')))  # TODO: change to 'creators' (list)
-        }
-
-        blocks = traverse_obj(art_json, (
             'sprinkledBody', 'content', ..., ('ledeMedia', None),
             lambda _, v: v['__typename'] in ('Video', 'Audio')))
 
+        art_title = remove_end(self._html_extract_title(webpage), ' - The New York Times')
+        art_description = traverse_obj(
+            article_json, ('sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text'),
+            get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage)
+        art_upload_date = traverse_obj(article_json, ('firstPublished'))
+        creator = ', '.join(
+            traverse_obj(article_json, ('bylines', ..., 'creators', ..., 'displayName')))
+
         # more than 1 video in the article, treat it as a playlist
         if len(blocks) > 1:
             entries = []
             for block in blocks:
-                entries.append(merge_dicts(
+                entries.append(merge_dicts(self._extract_content_from_block(block), {
-                    self._extract_content_from_block(block), common_info, {
+                    'title': art_title,
-                        'thumbnails': self._extract_thumbnails(traverse_obj(
+                    'description': art_description,
-                            block, ('promotionalMedia', 'crops', ..., 'renditions', ...)))}))
+                    'creator': creator,  # TODO: change to 'creators' (list)
+                    'thumbnails': self._extract_thumbnails(traverse_obj(
+                        block, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
+                }))
+            return self.playlist_result(entries, page_id, art_title, art_description)
 
-            return self.playlist_result(entries, page_id, common_info.get('title'),
+        return merge_dicts(self._extract_content_from_block(blocks[0]), {
-                                        common_info.get('description'))
+            'title': art_title,
-
+            'description': art_description,
-        return merge_dicts(self._extract_content_from_block(blocks[0]), common_info, {
+            'creator': creator,  # TODO: change to 'creators'
+            'upload_date': unified_strdate(art_upload_date),
+            'timestamp': parse_iso8601(art_upload_date),
             'thumbnails': self._extract_thumbnails(traverse_obj(
                 blocks[0], ('promotionalMedia', 'crops', ..., 'renditions', ...)) or traverse_obj(
-                art_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
+                article_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
         })
 
 
@@ -272,22 +277,40 @@ class NYTimesCookingIE(InfoExtractor):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
-        next_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']
+        json_obj = traverse_obj(
+            self._search_nextjs_data(webpage, page_id), ('props', 'pageProps'), default={})
+
+        info = traverse_obj(json_obj, {
+            'id': ('recipe', 'id', {str_or_none}),
+            'title': ('recipe', 'title'),
+            'description': ('recipe', 'topnote', {clean_html}),
+            'timestamp': ('recipe', 'publishedAt'),
+            'creator': ('recipe', 'contentAttribution', 'cardByline'),
+            'upload_date': ('meta', 'jsonLD', 'video', 'uploadDate', {unified_strdate}),
+            'formats': ('recipe', 'videoSrc', {url_or_none}),
+        })
 
         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
-            next_data['recipe']['videoSrc'], page_id, 'mp4', m3u8_id='hls')
+            info.get('formats'), info.get('id'), 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        thumbnails = []
+        for image in traverse_obj(json_obj, ('recipe', 'image', 'crops', 'recipe', ...)):
+            if not url_or_none(image):
+                continue
+            thumbnails.append({
+                'url': image,
+            })
 
         return {
-            **traverse_obj(next_data, {
+            'id': info.get('id'),
-                'id': ('recipe', 'id', {str_or_none}),
+            'title': info.get('title'),
-                'title': ('recipe', 'title'),
+            'description': info.get('description'),
-                'description': ('recipe', 'topnote', {clean_html}),
+            'timestamp': info.get('timestamp'),
-                'timestamp': ('recipe', 'publishedAt'),
+            'upload_date': info.get(' upload_date'),
-                'creator': ('recipe', 'contentAttribution', 'cardByline')}),
+            'creator': info.get('creator'),
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnails': [{'url': url} for url in traverse_obj(next_data, (
+            'thumbnails': thumbnails,
-                'recipe', 'image', 'crops', 'recipe', lambda _, v: url_or_none(v)))]
         }
 
 
@@ -326,7 +349,7 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
     }]
 
     _TOKEN = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQAB'
-    _DNS_NAMESPACE = uuid.UUID('36dd619a-56dc-595b-9e09-37f4152c7b5d')
+    _DNS_UUID = '36dd619a-56dc-595b-9e09-37f4152c7b5d'  # uuid -v5 ns:DNS scoop.nyt.net
     _GRAPHQL_QUERY = '''query VideoQuery($id: String!) {
   video(id: $id) {
     ... on Video {
@@ -361,6 +384,7 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
 }'''
 
     def _build_playlist(self, media_items):
+        entries = []
         for media_id in media_items:
             json_obj = traverse_obj(self._json_from_graphql(media_id) or {}, ('data', 'video'))
 
@@ -372,33 +396,53 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
             duration = int_or_none(json_obj.get('duration'))
             formats, subtitles = self._extract_media_from_json(media_id, json_obj)
 
-            yield {
+            thumbnails = []
+            for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
+                image_url = image.get('url')
+                if not url_or_none(image_url):
+                    continue
+                thumbnails.append({
+                    'name': image.get('name'),
+                    'url': image_url,
+                    'width': int_or_none(image.get('width')),
+                    'height': int_or_none(image.get('height')),
+                })
+
+            entries.append({
                 'id': media_id,
                 'title': title,
                 'description': description,
                 'duration': duration,
-                'creator': creators,  # TODO: change to 'creators'
+                'creator': creators,
                 'formats': formats,
                 'subtitles': subtitles,
-                'thumbnails': self._extract_thumbnails(
+                'thumbnails': thumbnails,
-                    traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
+            })
-            }
+        return entries
 
     def _json_from_graphql(self, media_id):
         # reference: `id-to-uri.js`
-        video_uuid = uuid.uuid5(self._DNS_NAMESPACE, 'video')
+        namespace = uuid.UUID(self._DNS_UUID)
+        video_uuid = uuid.uuid5(namespace, 'video')
         media_uuid = uuid.uuid5(video_uuid, media_id)
 
+        payload = {
+            'query': self._GRAPHQL_QUERY,
+            'variables': {'id': f'nyt://video/{media_uuid}'}
+        }
+
+        headers = {
+            'Content-Type': 'application/json',
+            'Nyt-App-Type': 'vhs',
+            'Nyt-App-Version': 'v3.52.21',
+            'Nyt-Token': self._TOKEN,
+            'Origin': 'https://cooking.nytimes.com',
+            'Referer': 'https://www.google.com/',
+        }
+
         return self._download_json(
-            self._GRAPHQL_API, media_id, note='Downloading json from GRAPHQL API', data=json.dumps({
+            self._GRAPHQL_API, id, note='Downloading json from GRAPHQL API',
-                'query': self._GRAPHQL_QUERY, 'variables': {'id': f'nyt://video/{media_uuid}'}},
+            data=json.dumps(payload, separators=(',', ':')).encode(), headers=headers, fatal=False)
-                separators=(',', ':')).encode(), headers={
-                    'Content-Type': 'application/json',
-                    'Nyt-App-Type': 'vhs',
-                    'Nyt-App-Version': 'v3.52.21',
-                    'Nyt-Token': self._TOKEN,
-                    'Origin': 'https://cooking.nytimes.com',
-                    'Referer': 'https://www.google.com/'}, fatal=False) or {}
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
@@ -418,17 +462,28 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
             return self.playlist_result(self._build_playlist(media_items), page_id, title, description)
 
         json_obj = traverse_obj(self._json_from_graphql(lead_video_id) or {}, ('data', 'video'))
+        duration = int_or_none(json_obj.get('duration'))
 
         formats, subtitles = self._extract_media_from_json(lead_video_id, json_obj)
 
+        thumbnails = []
+        for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
+            image_url = image.get('url')
+            if not url_or_none(image_url):
+                continue
+            thumbnails.append({
+                'url': image_url,
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            })
+
         return {
             'id': lead_video_id,
             'title': title,
             'description': description,
-            'duration': int_or_none(json_obj.get('duration')),
+            'duration': duration,
-            'creator': creator,  # TODO: change to 'creators'
+            'creator': creator,
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnails': self._extract_thumbnails(
+            'thumbnails': thumbnails,
-                traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
         }