mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-10-02 23:31:22 +02:00
Compare commits
No commits in common. "b8d9fdccc0e5b53dd5441a714b074c4f19753090" and "3e632a57399786f412e06b31898de6cf55fffac9" have entirely different histories.
b8d9fdccc0
...
3e632a5739
|
@ -15,6 +15,7 @@ from ..utils import (
|
|||
remove_start,
|
||||
str_or_none,
|
||||
traverse_obj,
|
||||
unified_strdate,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
@ -22,18 +23,11 @@ from ..utils import (
|
|||
class NYTimesBaseIE(InfoExtractor):
|
||||
_GRAPHQL_API = 'https://samizdat-graphql.nytimes.com/graphql/v2'
|
||||
|
||||
def _extract_thumbnails(self, thumbs):
|
||||
return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
|
||||
'url': 'url',
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
}))
|
||||
|
||||
def _extract_media_from_json(self, video_id, content_media_json):
|
||||
urls = []
|
||||
formats = []
|
||||
subtitles = {}
|
||||
for video in traverse_obj(content_media_json, ('renditions', ..., {dict})):
|
||||
for video in traverse_obj(content_media_json, ('renditions', ...)):
|
||||
video_url = video.get('url')
|
||||
format_id = video.get('type')
|
||||
if not video_url or format_id == 'thumbs' or video_url in urls:
|
||||
|
@ -175,7 +169,7 @@ class NYTimesArticleIE(NYTimesBaseIE):
|
|||
# audio articles will have an url and no formats
|
||||
url = traverse_obj(block, ('fileUrl', {url_or_none}))
|
||||
if not formats and url:
|
||||
formats.append({'url': url})
|
||||
formats.append({'url': url, 'ext': determine_ext(url)})
|
||||
|
||||
return {
|
||||
**details,
|
||||
|
@ -183,43 +177,54 @@ class NYTimesArticleIE(NYTimesBaseIE):
|
|||
'subtitles': subtitles
|
||||
}
|
||||
|
||||
def _extract_thumbnails(self, thumbs):
|
||||
return traverse_obj(thumbs, (lambda _, v: url_or_none(v['url']), {
|
||||
'url': 'url',
|
||||
'width': ('width', {int_or_none}),
|
||||
'height': ('height', {int_or_none}),
|
||||
}))
|
||||
|
||||
def _real_extract(self, url):
|
||||
page_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, page_id)
|
||||
art_json = self._search_json(
|
||||
article_json = self._search_json(
|
||||
r'window\.__preloadedData\s*=', webpage, 'media details', page_id,
|
||||
transform_source=lambda x: x.replace('undefined', 'null'))['initialData']['data']['article']
|
||||
|
||||
common_info = {
|
||||
'title': remove_end(self._html_extract_title(webpage), ' - The New York Times'),
|
||||
'description': traverse_obj(art_json, (
|
||||
'sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text', {str}),
|
||||
get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage),
|
||||
'timestamp': traverse_obj(art_json, ('firstPublished', {parse_iso8601})),
|
||||
'creator': ', '.join(
|
||||
traverse_obj(art_json, ('bylines', ..., 'creators', ..., 'displayName'))) # TODO: change to 'creators' (list)
|
||||
}
|
||||
|
||||
blocks = traverse_obj(art_json, (
|
||||
blocks = traverse_obj(article_json, (
|
||||
'sprinkledBody', 'content', ..., ('ledeMedia', None),
|
||||
lambda _, v: v['__typename'] in ('Video', 'Audio')))
|
||||
|
||||
art_title = remove_end(self._html_extract_title(webpage), ' - The New York Times')
|
||||
art_description = traverse_obj(
|
||||
article_json, ('sprinkledBody', 'content', ..., 'summary', 'content', ..., 'text'),
|
||||
get_all=False) or self._html_search_meta(['og:description', 'twitter:description'], webpage)
|
||||
art_upload_date = traverse_obj(article_json, ('firstPublished'))
|
||||
creator = ', '.join(
|
||||
traverse_obj(article_json, ('bylines', ..., 'creators', ..., 'displayName')))
|
||||
|
||||
# more than 1 video in the article, treat it as a playlist
|
||||
if len(blocks) > 1:
|
||||
entries = []
|
||||
for block in blocks:
|
||||
entries.append(merge_dicts(
|
||||
self._extract_content_from_block(block), common_info, {
|
||||
'thumbnails': self._extract_thumbnails(traverse_obj(
|
||||
block, ('promotionalMedia', 'crops', ..., 'renditions', ...)))}))
|
||||
entries.append(merge_dicts(self._extract_content_from_block(block), {
|
||||
'title': art_title,
|
||||
'description': art_description,
|
||||
'creator': creator, # TODO: change to 'creators' (list)
|
||||
'thumbnails': self._extract_thumbnails(traverse_obj(
|
||||
block, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
|
||||
}))
|
||||
return self.playlist_result(entries, page_id, art_title, art_description)
|
||||
|
||||
return self.playlist_result(entries, page_id, common_info.get('title'),
|
||||
common_info.get('description'))
|
||||
|
||||
return merge_dicts(self._extract_content_from_block(blocks[0]), common_info, {
|
||||
return merge_dicts(self._extract_content_from_block(blocks[0]), {
|
||||
'title': art_title,
|
||||
'description': art_description,
|
||||
'creator': creator, # TODO: change to 'creators'
|
||||
'upload_date': unified_strdate(art_upload_date),
|
||||
'timestamp': parse_iso8601(art_upload_date),
|
||||
'thumbnails': self._extract_thumbnails(traverse_obj(
|
||||
blocks[0], ('promotionalMedia', 'crops', ..., 'renditions', ...)) or traverse_obj(
|
||||
art_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
|
||||
article_json, ('promotionalMedia', 'assetCrops', ..., 'renditions', ...))),
|
||||
})
|
||||
|
||||
|
||||
|
@ -272,22 +277,40 @@ class NYTimesCookingIE(InfoExtractor):
|
|||
page_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, page_id)
|
||||
|
||||
next_data = self._search_nextjs_data(webpage, page_id)['props']['pageProps']
|
||||
json_obj = traverse_obj(
|
||||
self._search_nextjs_data(webpage, page_id), ('props', 'pageProps'), default={})
|
||||
|
||||
info = traverse_obj(json_obj, {
|
||||
'id': ('recipe', 'id', {str_or_none}),
|
||||
'title': ('recipe', 'title'),
|
||||
'description': ('recipe', 'topnote', {clean_html}),
|
||||
'timestamp': ('recipe', 'publishedAt'),
|
||||
'creator': ('recipe', 'contentAttribution', 'cardByline'),
|
||||
'upload_date': ('meta', 'jsonLD', 'video', 'uploadDate', {unified_strdate}),
|
||||
'formats': ('recipe', 'videoSrc', {url_or_none}),
|
||||
})
|
||||
|
||||
formats, subtitles = self._extract_m3u8_formats_and_subtitles(
|
||||
next_data['recipe']['videoSrc'], page_id, 'mp4', m3u8_id='hls')
|
||||
info.get('formats'), info.get('id'), 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
|
||||
|
||||
thumbnails = []
|
||||
for image in traverse_obj(json_obj, ('recipe', 'image', 'crops', 'recipe', ...)):
|
||||
if not url_or_none(image):
|
||||
continue
|
||||
thumbnails.append({
|
||||
'url': image,
|
||||
})
|
||||
|
||||
return {
|
||||
**traverse_obj(next_data, {
|
||||
'id': ('recipe', 'id', {str_or_none}),
|
||||
'title': ('recipe', 'title'),
|
||||
'description': ('recipe', 'topnote', {clean_html}),
|
||||
'timestamp': ('recipe', 'publishedAt'),
|
||||
'creator': ('recipe', 'contentAttribution', 'cardByline')}),
|
||||
'id': info.get('id'),
|
||||
'title': info.get('title'),
|
||||
'description': info.get('description'),
|
||||
'timestamp': info.get('timestamp'),
|
||||
'upload_date': info.get(' upload_date'),
|
||||
'creator': info.get('creator'),
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnails': [{'url': url} for url in traverse_obj(next_data, (
|
||||
'recipe', 'image', 'crops', 'recipe', lambda _, v: url_or_none(v)))]
|
||||
'thumbnails': thumbnails,
|
||||
}
|
||||
|
||||
|
||||
|
@ -326,7 +349,7 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
|
|||
}]
|
||||
|
||||
_TOKEN = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQAB'
|
||||
_DNS_NAMESPACE = uuid.UUID('36dd619a-56dc-595b-9e09-37f4152c7b5d')
|
||||
_DNS_UUID = '36dd619a-56dc-595b-9e09-37f4152c7b5d' # uuid -v5 ns:DNS scoop.nyt.net
|
||||
_GRAPHQL_QUERY = '''query VideoQuery($id: String!) {
|
||||
video(id: $id) {
|
||||
... on Video {
|
||||
|
@ -361,6 +384,7 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
|
|||
}'''
|
||||
|
||||
def _build_playlist(self, media_items):
|
||||
entries = []
|
||||
for media_id in media_items:
|
||||
json_obj = traverse_obj(self._json_from_graphql(media_id) or {}, ('data', 'video'))
|
||||
|
||||
|
@ -372,33 +396,53 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
|
|||
duration = int_or_none(json_obj.get('duration'))
|
||||
formats, subtitles = self._extract_media_from_json(media_id, json_obj)
|
||||
|
||||
yield {
|
||||
thumbnails = []
|
||||
for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
|
||||
image_url = image.get('url')
|
||||
if not url_or_none(image_url):
|
||||
continue
|
||||
thumbnails.append({
|
||||
'name': image.get('name'),
|
||||
'url': image_url,
|
||||
'width': int_or_none(image.get('width')),
|
||||
'height': int_or_none(image.get('height')),
|
||||
})
|
||||
|
||||
entries.append({
|
||||
'id': media_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'duration': duration,
|
||||
'creator': creators, # TODO: change to 'creators'
|
||||
'creator': creators,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnails': self._extract_thumbnails(
|
||||
traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
|
||||
}
|
||||
'thumbnails': thumbnails,
|
||||
})
|
||||
return entries
|
||||
|
||||
def _json_from_graphql(self, media_id):
|
||||
# reference: `id-to-uri.js`
|
||||
video_uuid = uuid.uuid5(self._DNS_NAMESPACE, 'video')
|
||||
namespace = uuid.UUID(self._DNS_UUID)
|
||||
video_uuid = uuid.uuid5(namespace, 'video')
|
||||
media_uuid = uuid.uuid5(video_uuid, media_id)
|
||||
|
||||
payload = {
|
||||
'query': self._GRAPHQL_QUERY,
|
||||
'variables': {'id': f'nyt://video/{media_uuid}'}
|
||||
}
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Nyt-App-Type': 'vhs',
|
||||
'Nyt-App-Version': 'v3.52.21',
|
||||
'Nyt-Token': self._TOKEN,
|
||||
'Origin': 'https://cooking.nytimes.com',
|
||||
'Referer': 'https://www.google.com/',
|
||||
}
|
||||
|
||||
return self._download_json(
|
||||
self._GRAPHQL_API, media_id, note='Downloading json from GRAPHQL API', data=json.dumps({
|
||||
'query': self._GRAPHQL_QUERY, 'variables': {'id': f'nyt://video/{media_uuid}'}},
|
||||
separators=(',', ':')).encode(), headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Nyt-App-Type': 'vhs',
|
||||
'Nyt-App-Version': 'v3.52.21',
|
||||
'Nyt-Token': self._TOKEN,
|
||||
'Origin': 'https://cooking.nytimes.com',
|
||||
'Referer': 'https://www.google.com/'}, fatal=False) or {}
|
||||
self._GRAPHQL_API, id, note='Downloading json from GRAPHQL API',
|
||||
data=json.dumps(payload, separators=(',', ':')).encode(), headers=headers, fatal=False)
|
||||
|
||||
def _real_extract(self, url):
|
||||
page_id = self._match_id(url)
|
||||
|
@ -418,17 +462,28 @@ class NYTimesCookingGuidesIE(NYTimesBaseIE):
|
|||
return self.playlist_result(self._build_playlist(media_items), page_id, title, description)
|
||||
|
||||
json_obj = traverse_obj(self._json_from_graphql(lead_video_id) or {}, ('data', 'video'))
|
||||
duration = int_or_none(json_obj.get('duration'))
|
||||
|
||||
formats, subtitles = self._extract_media_from_json(lead_video_id, json_obj)
|
||||
|
||||
thumbnails = []
|
||||
for image in traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...)):
|
||||
image_url = image.get('url')
|
||||
if not url_or_none(image_url):
|
||||
continue
|
||||
thumbnails.append({
|
||||
'url': image_url,
|
||||
'width': int_or_none(image.get('width')),
|
||||
'height': int_or_none(image.get('height')),
|
||||
})
|
||||
|
||||
return {
|
||||
'id': lead_video_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'duration': int_or_none(json_obj.get('duration')),
|
||||
'creator': creator, # TODO: change to 'creators'
|
||||
'duration': duration,
|
||||
'creator': creator,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
'thumbnails': self._extract_thumbnails(
|
||||
traverse_obj(json_obj, ('promotionalMedia', 'crops', ..., 'renditions', ...))),
|
||||
'thumbnails': thumbnails,
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user