Compare commits

...

9 Commits

Author SHA1 Message Date
sfn
98cf2eede4 remove extra empty line 2023-11-19 05:53:55 +01:00
bashonly
6392d43127
cleaning up my own suggestion 2023-11-19 04:40:47 +00:00
sfn
6701623ed0 str_or_none not used 2023-11-19 04:58:50 +01:00
sfn
7cc64bc2da import join_noneempty 2023-11-19 04:57:19 +01:00
Safouane Aarab
a25cd60de6
Update yt_dlp/extractor/allstar.py
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-11-19 04:36:56 +01:00
Safouane Aarab
5e29cf0d82
Update yt_dlp/extractor/allstar.py
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-11-19 04:33:05 +01:00
sfn
77cfbfb5be set webpage_url in _parse_video_data
rename _send_query to _call_api, variables and path are required
convert timestamp to seconds
add uploader_url and upload_date to TESTS
set extractor and extractor_key to the video extractor
2023-11-19 03:00:30 +01:00
Safouane Aarab
8ad21b8e6b
Update yt_dlp/extractor/allstar.py
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-11-19 02:38:18 +01:00
Safouane Aarab
9cd1a0482f
Update yt_dlp/extractor/allstar.py
Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
2023-11-19 02:38:00 +01:00

View File

@ -5,8 +5,8 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
OnDemandPagedList, OnDemandPagedList,
UnsupportedError,
int_or_none, int_or_none,
join_nonempty,
parse_qs, parse_qs,
traverse_obj, traverse_obj,
urljoin, urljoin,
@ -60,36 +60,44 @@ _QUERIES = {
class AllstarBaseIE(InfoExtractor): class AllstarBaseIE(InfoExtractor):
@staticmethod @staticmethod
def _parse_video_data(video_data): def _parse_video_data(video_data):
def _media_url_or_none(path): def media_url_or_none(path):
return urljoin('https://media.allstar.gg/', path) return urljoin('https://media.allstar.gg/', path)
def _profile_url_or_none(path): info = traverse_obj(video_data, {
return urljoin('https://allstar.gg/u/', path)
return traverse_obj(video_data, {
'id': ('_id', {str}), 'id': ('_id', {str}),
'display_id': ('shareId', {str}), 'display_id': ('shareId', {str}),
'title': ('clipTitle', {str}), 'title': ('clipTitle', {str}),
'url': ('clipLink', {_media_url_or_none}), 'url': ('clipLink', {media_url_or_none}),
'thumbnail': ('clipImageThumb', {_media_url_or_none}), 'thumbnail': ('clipImageThumb', {media_url_or_none}),
'duration': ('clipLength', {int_or_none}), 'duration': ('clipLength', {int_or_none}),
'filesize': ('clipSizeBytes', {int_or_none}), 'filesize': ('clipSizeBytes', {int_or_none}),
'timestamp': ('createdDate', {int_or_none}), 'timestamp': ('createdDate', {functools.partial(int_or_none, scale=1000)}),
'uploader': ('username', {str}), 'uploader': ('username', {str}),
'uploader_id': ('user', '_id', {str}), 'uploader_id': ('user', '_id', {str}),
'uploader_url': ('user', '_id', {_profile_url_or_none}),
'view_count': ('views', {int_or_none}), 'view_count': ('views', {int_or_none}),
}) })
def _send_query(self, query, variables={}, path=(), video_id=None, note=None): if info.get('id') and info.get('url'):
basename = 'clip' if '/clips/' in info['url'] else 'montage'
info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'
info.update({
'extractor_key': AllstarIE.ie_key(),
'extractor': AllstarIE.IE_NAME,
'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
})
return info
def _call_api(self, query, variables, path, video_id=None, note=None):
response = self._download_json( response = self._download_json(
'https://a1.allstar.gg/graphql', video_id, note=note, 'https://a1.allstar.gg/graphql', video_id, note=note,
headers={'content-type': 'application/json'}, headers={'content-type': 'application/json'},
data=json.dumps({'variables': variables, 'query': query}).encode()) data=json.dumps({'variables': variables, 'query': query}).encode())
errors = response.get('errors') errors = traverse_obj(response, ('errors', ..., 'message', {str}))
if errors: if errors:
raise ExtractorError(errors, expected=True) raise ExtractorError('; '.join(errors))
return traverse_obj(response, path) return traverse_obj(response, path)
@ -108,8 +116,10 @@ class AllstarIE(AllstarBaseIE):
'ext': 'mp4', 'ext': 'mp4',
'duration': 20, 'duration': 20,
'filesize': 21199257, 'filesize': 21199257,
'timestamp': 1682451501555, 'timestamp': 1682451501,
'uploader_id': '62b8bdfc9021052f7905882d', 'uploader_id': '62b8bdfc9021052f7905882d',
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
'upload_date': '20230425',
'view_count': int, 'view_count': int,
} }
}, { }, {
@ -123,9 +133,11 @@ class AllstarIE(AllstarBaseIE):
'thumbnail': 'md5:90564b121f5fd7a4924920ef45614634', 'thumbnail': 'md5:90564b121f5fd7a4924920ef45614634',
'duration': 16, 'duration': 16,
'filesize': 30175859, 'filesize': 30175859,
'timestamp': 1688333419392, 'timestamp': 1688333419,
'uploader': 'cherokee', 'uploader': 'cherokee',
'uploader_id': '62b8bdfc9021052f7905882d', 'uploader_id': '62b8bdfc9021052f7905882d',
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
'upload_date': '20230702',
'view_count': int, 'view_count': int,
} }
}, { }, {
@ -137,9 +149,11 @@ class AllstarIE(AllstarBaseIE):
'url': 'md5:a3ee356022115db2b27c81321d195945', 'url': 'md5:a3ee356022115db2b27c81321d195945',
'thumbnail': 'md5:f1a5e811864e173f180b738d956356f4', 'thumbnail': 'md5:f1a5e811864e173f180b738d956356f4',
'ext': 'mp4', 'ext': 'mp4',
'timestamp': 1681810448040, 'timestamp': 1681810448,
'uploader': 'cherokee', 'uploader': 'cherokee',
'uploader_id': '62b8bdfc9021052f7905882d', 'uploader_id': '62b8bdfc9021052f7905882d',
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
'upload_date': '20230418',
'view_count': int, 'view_count': int,
} }
}, { }, {
@ -150,10 +164,12 @@ class AllstarIE(AllstarBaseIE):
'title': 'cherokee Rapid Fire Snipers Montage', 'title': 'cherokee Rapid Fire Snipers Montage',
'url': 'md5:d5672e6f88579730c2310a80fdbc4030', 'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
'thumbnail': 'md5:60872f0d236863bb9a6f3dff1623403c', 'thumbnail': 'md5:60872f0d236863bb9a6f3dff1623403c',
'uploader': 'cherokee',
'ext': 'mp4', 'ext': 'mp4',
'timestamp': 1688365434271, 'timestamp': 1688365434,
'uploader': 'cherokee',
'uploader_id': '62b8bdfc9021052f7905882d', 'uploader_id': '62b8bdfc9021052f7905882d',
'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
'upload_date': '20230703',
'view_count': int, 'view_count': int,
} }
}] }]
@ -162,9 +178,8 @@ class AllstarIE(AllstarBaseIE):
query_id, video_id = self._match_valid_url(url).group('type', 'id') query_id, video_id = self._match_valid_url(url).group('type', 'id')
return self._parse_video_data( return self._parse_video_data(
self._send_query( self._call_api(
_QUERIES.get(query_id), {'id': video_id}, _QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))
('data', 'video'), video_id))
class AllstarProfileIE(AllstarBaseIE): class AllstarProfileIE(AllstarBaseIE):
@ -173,28 +188,28 @@ class AllstarProfileIE(AllstarBaseIE):
_TESTS = [{ _TESTS = [{
'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d', 'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
'info_dict': { 'info_dict': {
'id': '62b8bdfc9021052f7905882d', 'id': '62b8bdfc9021052f7905882d-clips',
'title': 'cherokee - Clips', 'title': 'cherokee - Clips',
}, },
'playlist_mincount': 15 'playlist_mincount': 15
}, { }, {
'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips', 'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
'info_dict': { 'info_dict': {
'id': '62b8bdfc9021052f7905882d', 'id': '62b8bdfc9021052f7905882d-clips-730',
'title': 'cherokee - Clips', 'title': 'cherokee - Clips - 730',
}, },
'playlist_mincount': 15 'playlist_mincount': 15
}, { }, {
'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages', 'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
'info_dict': { 'info_dict': {
'id': '62b8bdfc9021052f7905882d', 'id': '62b8bdfc9021052f7905882d-montages',
'title': 'cherokee - Montages', 'title': 'cherokee - Montages',
}, },
'playlist_mincount': 4 'playlist_mincount': 4
}, { }, {
'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips', 'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
'info_dict': { 'info_dict': {
'id': '62b8bdfc9021052f7905882d', 'id': '62b8bdfc9021052f7905882d-mobile',
'title': 'cherokee - Mobile Clips', 'title': 'cherokee - Mobile Clips',
}, },
'playlist_mincount': 1 'playlist_mincount': 1
@ -202,50 +217,36 @@ class AllstarProfileIE(AllstarBaseIE):
_PAGE_SIZE = 10 _PAGE_SIZE = 10
@staticmethod
def _set_webpage_url(info_dict):
video_id = info_dict.get('id')
video_url = info_dict.get('url')
if video_url is None or video_id is None:
return info_dict
base_name = 'clip' if '/clips/' in video_url else 'montage'
info_dict['webpage_url'] = f'https://allstar.gg/{base_name}?{base_name}={video_id}'
info_dict['webpage_url_basename'] = base_name
return info_dict
def _get_page(self, user_id, display_id, game, query, page_num): def _get_page(self, user_id, display_id, game, query, page_num):
page_num += 1 page_num += 1
for video_data in self._send_query( for video_data in self._call_api(
query, { query, {
'user': user_id, 'user': user_id,
'page': page_num, 'page': page_num,
'game': game, 'game': game,
}, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'): }, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
yield self._set_webpage_url(self._parse_video_data(video_data)) yield self._parse_video_data(video_data)
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
profile_data = self._download_json( profile_data = self._download_json(
urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id) urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
user_id = traverse_obj(profile_data, ('data', ('_id'), {str})) user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
if not user_id:
if user_id is None: raise ExtractorError('Unable to extract the user id')
raise ExtractorError('Can not extract the user_id')
username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str})) username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
url_query = parse_qs(url) url_query = parse_qs(url)
game = traverse_obj(url_query, ('game', 0, {int})) game = traverse_obj(url_query, ('game', 0, {int_or_none}))
query_id = traverse_obj(url_query, ('view', 0), default='Clips') query_id = traverse_obj(url_query, ('view', 0), default='Clips')
if query_id not in ('Clips', 'Montages', 'Mobile Clips'): if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
raise UnsupportedError(url) raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')
return self.playlist_result( return self.playlist_result(
OnDemandPagedList( OnDemandPagedList(
functools.partial( functools.partial(
self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE), self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
user_id, f'{username or display_id} - {query_id}') playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))