Bugfix for beaa1a4455 (#9235)

[build:Makefile] Restore compatibility with GNU Make <4.0

- The != variable assignment operator is not supported by GNU Make <4.0
- $(shell) is a no-op in BSD Make, assigns an empty string to the var
- Try to assign with != and fallback to $(shell) if not assigned (?=)

- Old versions of BSD find have different -exec behavior
- Pipe to `sed` instead of using `find ... -exec dirname {}`

- BSD tar does not support --transform, --owner or --group
- Allow user to specify path to GNU tar by passing GNUTAR variable

- pandoc vars are immediately evaluated with != in gmake>=4 and bmake
- Suppress stderr output for pandoc -v in case it is not installed
- Use string comparison instead of int comparison for pandoc version

Authored by: bashonly

Makefile

@@ -37,14 +37,15 @@ BINDIR ?= $(PREFIX)/bin
 MANDIR ?= $(PREFIX)/man
 SHAREDIR ?= $(PREFIX)/share
 PYTHON ?= /usr/bin/env python3
-
-# $(shell) and $(error) are no-ops in BSD Make and the != variable assignment operator is not supported by GNU Make <4.0
-VERSION_CHECK != echo supported
-VERSION_CHECK ?= $(error GNU Make 4+ or BSD Make is required)
-CHECK_VERSION := $(VERSION_CHECK)
+GNUTAR ?= tar
 
 # set markdown input format to "markdown-smart" for pandoc version 2+ and to "markdown" for pandoc prior to version 2
-MARKDOWN != if [ "`pandoc -v | head -n1 | cut -d' ' -f2 | head -c1`" -ge "2" ]; then echo markdown-smart; else echo markdown; fi
+PANDOC_VERSION_CMD = pandoc -v 2>/dev/null | head -n1 | cut -d' ' -f2 | head -c1
+PANDOC_VERSION != $(PANDOC_VERSION_CMD)
+PANDOC_VERSION ?= $(shell $(PANDOC_VERSION_CMD))
+MARKDOWN_CMD = if [ "$(PANDOC_VERSION)" = "1" -o "$(PANDOC_VERSION)" = "0" ]; then echo markdown; else echo markdown-smart; fi
+MARKDOWN != $(MARKDOWN_CMD)
+MARKDOWN ?= $(shell $(MARKDOWN_CMD))
 
 install: lazy-extractors yt-dlp yt-dlp.1 completions
 	mkdir -p $(DESTDIR)$(BINDIR)
@@ -75,8 +76,12 @@ test:
 offlinetest: codetest
 	$(PYTHON) -m pytest -k "not download"
 
-CODE_FOLDERS != find yt_dlp -type f -name '__init__.py' -exec dirname {} \+ | grep -v '/__' | sort
-CODE_FILES != for f in $(CODE_FOLDERS) ; do echo "$$f" | sed 's,$$,/*.py,' ; done
+CODE_FOLDERS_CMD = find yt_dlp -type f -name '__init__.py' | sed 's,/__init__.py,,' | grep -v '/__' | sort
+CODE_FOLDERS != $(CODE_FOLDERS_CMD)
+CODE_FOLDERS ?= $(shell $(CODE_FOLDERS_CMD))
+CODE_FILES_CMD = for f in $(CODE_FOLDERS) ; do echo "$$f" | sed 's,$$,/*.py,' ; done
+CODE_FILES != $(CODE_FILES_CMD)
+CODE_FILES ?= $(shell $(CODE_FILES_CMD))
 yt-dlp: $(CODE_FILES)
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \
@@ -129,12 +134,14 @@ completions/fish/yt-dlp.fish: $(CODE_FILES) devscripts/fish-completion.in
 	mkdir -p completions/fish
 	$(PYTHON) devscripts/fish-completion.py
 
-_EXTRACTOR_FILES != find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py'
+_EXTRACTOR_FILES_CMD = find yt_dlp/extractor -name '*.py' -and -not -name 'lazy_extractors.py'
+_EXTRACTOR_FILES != $(_EXTRACTOR_FILES_CMD)
+_EXTRACTOR_FILES ?= $(shell $(_EXTRACTOR_FILES_CMD))
 yt_dlp/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
 yt-dlp.tar.gz: all
-	@tar -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \
+	@$(GNUTAR) -czf yt-dlp.tar.gz --transform "s|^|yt-dlp/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
 		--exclude '*.pyc' \

yt_dlp/extractor/nova.py

@@ -135,14 +135,15 @@ class NovaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
         'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
-        'md5': '249baab7d0104e186e78b0899c7d5f28',
+        'md5': 'da8f3f1fcdaf9fb0f112a32a165760a3',
         'info_dict': {
-            'id': '1757139',
-            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
+            'id': '8OvQqEvV3MW',
+            'display_id': '8OvQqEvV3MW',
             'ext': 'mp4',
             'title': 'Podzemní nemocnice v pražské Krči',
             'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
             'thumbnail': r're:^https?://.*\.(?:jpg)',
+            'duration': 151,
         }
     }, {
         'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
@@ -210,7 +211,7 @@ class NovaIE(InfoExtractor):
 
         # novaplus
         embed_id = self._search_regex(
-            r'<iframe[^>]+\bsrc=["\'](?:https?:)?//media\.cms\.nova\.cz/embed/([^/?#&]+)',
+            r'<iframe[^>]+\bsrc=["\'](?:https?:)?//media(?:tn)?\.cms\.nova\.cz/embed/([^/?#&"\']+)',
             webpage, 'embed url', default=None)
         if embed_id:
             return {

yt_dlp/extractor/onefootball.py

@@ -1,4 +1,6 @@
 from .common import InfoExtractor
+from .jwplatform import JWPlatformIE
+from ..utils import make_archive_id
 
 
 class OneFootballIE(InfoExtractor):
@@ -7,41 +9,43 @@ class OneFootballIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://onefootball.com/en/video/highlights-fc-zuerich-3-3-fc-basel-34012334',
         'info_dict': {
-            'id': '34012334',
+            'id': 'Y2VtcWAT',
             'ext': 'mp4',
             'title': 'Highlights: FC Zürich 3-3 FC Basel',
             'description': 'md5:33d9855cb790702c4fe42a513700aba8',
-            'thumbnail': 'https://photobooth-api.onefootball.com/api/screenshot/https:%2F%2Fperegrine-api.onefootball.com%2Fv2%2Fphotobooth%2Fcms%2Fen%2F34012334',
-            'timestamp': 1635874604,
-            'upload_date': '20211102'
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/Y2VtcWAT/poster.jpg?width=720',
+            'timestamp': 1635874895,
+            'upload_date': '20211102',
+            'duration': 375.0,
+            'tags': ['Football', 'Soccer', 'OneFootball'],
+            '_old_archive_ids': ['onefootball 34012334'],
         },
-        'params': {'skip_download': True}
+        'params': {'skip_download': True},
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'https://onefootball.com/en/video/klopp-fumes-at-var-decisions-in-west-ham-defeat-34041020',
         'info_dict': {
-            'id': '34041020',
+            'id': 'leVJrMho',
             'ext': 'mp4',
             'title': 'Klopp fumes at VAR decisions in West Ham defeat',
             'description': 'md5:9c50371095a01ad3f63311c73d8f51a5',
-            'thumbnail': 'https://photobooth-api.onefootball.com/api/screenshot/https:%2F%2Fperegrine-api.onefootball.com%2Fv2%2Fphotobooth%2Fcms%2Fen%2F34041020',
-            'timestamp': 1636314103,
-            'upload_date': '20211107'
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/leVJrMho/poster.jpg?width=720',
+            'timestamp': 1636315232,
+            'upload_date': '20211107',
+            'duration': 93.0,
+            'tags': ['Football', 'Soccer', 'OneFootball'],
+            '_old_archive_ids': ['onefootball 34041020'],
         },
         'params': {'skip_download': True}
     }]
 
     def _real_extract(self, url):
-        id = self._match_id(url)
-        webpage = self._download_webpage(url, id)
-        data_json = self._search_json_ld(webpage, id)
-        m3u8_url = self._html_search_regex(r'(https://cdn\.jwplayer\.com/manifests/.+\.m3u8)', webpage, 'm3u8_url')
-        formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, id)
-        return {
-            'id': id,
-            'title': data_json.get('title'),
-            'description': data_json.get('description'),
-            'thumbnail': data_json.get('thumbnail'),
-            'timestamp': data_json.get('timestamp'),
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data_json = self._search_json_ld(webpage, video_id, fatal=False)
+        data_json.pop('url', None)
+        m3u8_url = self._html_search_regex(r'(https://cdn\.jwplayer\.com/manifests/\w+\.m3u8)', webpage, 'm3u8_url')
+
+        return self.url_result(
+            m3u8_url, JWPlatformIE, video_id, _old_archive_ids=[make_archive_id(self, video_id)],
+            **data_json, url_transparent=True)

yt_dlp/extractor/screencastify.py

@@ -5,7 +5,10 @@ from ..utils import traverse_obj, update_url_query
 
 
 class ScreencastifyIE(InfoExtractor):
-    _VALID_URL = r'https?://watch\.screencastify\.com/v/(?P<id>[^/?#]+)'
+    _VALID_URL = [
+        r'https?://watch\.screencastify\.com/v/(?P<id>[^/?#]+)',
+        r'https?://app\.screencastify\.com/v[23]/watch/(?P<id>[^/?#]+)',
+    ]
     _TESTS = [{
         'url': 'https://watch.screencastify.com/v/sYVkZip3quLKhHw4Ybk8',
         'info_dict': {
@@ -19,6 +22,21 @@ class ScreencastifyIE(InfoExtractor):
         'params': {
             'skip_download': 'm3u8',
         },
+    }, {
+        'url': 'https://app.screencastify.com/v3/watch/J5N7H11wofDN1jZUCr3t',
+        'info_dict': {
+            'id': 'J5N7H11wofDN1jZUCr3t',
+            'ext': 'mp4',
+            'uploader': 'Scott Piesen',
+            'description': '',
+            'title': 'Lesson Recording 1-17 Burrr...',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }, {
+        'url': 'https://app.screencastify.com/v2/watch/BQ26VbUdfbQLhKzkktOk',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/utreon.py

@@ -10,7 +10,8 @@ from ..utils import (
 
 
 class UtreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?utreon\.com/v/(?P<id>[\w-]+)'
+    IE_NAME = 'playeur'
+    _VALID_URL = r'https?://(?:www\.)?(?:utreon|playeur)\.com/v/(?P<id>[\w-]+)'
     _TESTS = [{
         'url': 'https://utreon.com/v/z_I7ikQbuDw',
         'info_dict': {
@@ -19,8 +20,9 @@ class UtreonIE(InfoExtractor):
             'title': 'Freedom Friday meditation - Rising in the wind',
             'description': 'md5:a9bf15a42434a062fe313b938343ad1b',
             'uploader': 'Heather Dawn Elemental Health',
-            'thumbnail': 'https://data-1.utreon.com/v/MG/M2/NT/z_I7ikQbuDw/z_I7ikQbuDw_preview.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
             'release_date': '20210723',
+            'duration': 586,
         }
     }, {
         'url': 'https://utreon.com/v/jerJw5EOOVU',
@@ -28,10 +30,11 @@ class UtreonIE(InfoExtractor):
             'id': 'jerJw5EOOVU',
             'ext': 'mp4',
             'title': 'When I\'m alone, I love to reflect in peace, to make my dreams come true... [Quotes and Poems]',
-            'description': 'md5:61ee6c2da98be51b04b969ca80273aaa',
+            'description': 'md5:4026aa3a2c10169c3649926ac8ef62b6',
             'uploader': 'Frases e Poemas Quotes and Poems',
-            'thumbnail': 'https://data-1.utreon.com/v/Mz/Zh/ND/jerJw5EOOVU/jerJw5EOOVU_89af85470a4b16eededde7f8674c96d9_cover.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
             'release_date': '20210723',
+            'duration': 60,
         }
     }, {
         'url': 'https://utreon.com/v/C4ZxXhYBBmE',
@@ -39,10 +42,11 @@ class UtreonIE(InfoExtractor):
             'id': 'C4ZxXhYBBmE',
             'ext': 'mp4',
             'title': 'Biden’s Capital Gains Tax Rate to Test World’s Highest',
-            'description': 'md5:fb5a6c2e506f013cc76f133f673bc5c8',
+            'description': 'md5:995aa9ad0733c0e5863ebdeff954f40e',
             'uploader': 'Nomad Capitalist',
-            'thumbnail': 'https://data-1.utreon.com/v/ZD/k1/Mj/C4ZxXhYBBmE/C4ZxXhYBBmE_628342076198c9c06dd6b2c665978584_cover.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
             'release_date': '20210723',
+            'duration': 884,
         }
     }, {
         'url': 'https://utreon.com/v/Y-stEH-FBm8',
@@ -52,15 +56,28 @@ class UtreonIE(InfoExtractor):
             'title': 'Creeper-Chan Pranks Steve! 💚 [MINECRAFT ANIME]',
             'description': 'md5:7a48450b0d761b96dec194be0c5ecb5f',
             'uploader': 'Merryweather Comics',
-            'thumbnail': 'https://data-1.utreon.com/v/MT/E4/Zj/Y-stEH-FBm8/Y-stEH-FBm8_5290676a41a4a1096db133b09f54f77b_cover.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
             'release_date': '20210718',
-        }},
-    ]
+            'duration': 151,
+        }
+    }, {
+        'url': 'https://playeur.com/v/Wzqp-UrxSeu',
+        'info_dict': {
+            'id': 'Wzqp-UrxSeu',
+            'ext': 'mp4',
+            'title': 'Update: Clockwork Basilisk Books on the Way!',
+            'description': 'md5:d9756b0b1884c904655b0e170d17cea5',
+            'uploader': 'Forgotten Weapons',
+            'release_date': '20240208',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'duration': 262,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         json_data = self._download_json(
-            'https://api.utreon.com/v1/videos/' + video_id,
+            'https://api.playeur.com/v1/videos/' + video_id,
             video_id)
         videos_json = json_data['videos']
         formats = [{