[utils] Make restricted filenames ignore characters in Unicode categories Mark, Other

Resolves #32629

.github/workflows/ci.yml

@@ -6,9 +6,13 @@ env:
   pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
   cpython-versions: main
   test-set: core
+  # Python beta version to be built using pyenv before setup-python support
+  # Must also be included in all-cpython-versions 
+  next: 3.13
 
 on:
   push:
+    # push inputs aren't known to GitHub
     inputs:
       cpython-versions:
         type: string
@@ -17,6 +21,7 @@ on:
         type: string
         default: core
   pull_request:
+    # pull_request inputs aren't known to GitHub
     inputs:
       cpython-versions:
         type: string
@@ -56,6 +61,23 @@ jobs:
       test-set: ${{ steps.run.outputs.test-set }}
       own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
     steps:
+    # push and pull_request inputs aren't known to GitHub (pt3)
+    - name: Set push defaults
+      if: ${{ github.event_name == 'push' }}
+      env:
+        cpython-versions: all
+        test-set: core
+      run: |
+        echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
+        echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
+    - name: Get pull_request inputs
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        cpython-versions: main
+        test-set: both
+      run: |
+        echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
+        echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
     - name: Make version array
       id: run
       run: |
@@ -79,6 +101,7 @@ jobs:
         # versions with a special get-pip.py in a per-version subdirectory
         printf 'own-pip-versions=%s\n' \
           "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
+
   tests:
     name: Run tests
     needs: select
@@ -121,13 +144,20 @@ jobs:
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: sh
     steps:
+    - name: Prepare Linux
+      if: ${{ startswith(matrix.os, 'ubuntu') }}
+      shell: bash
+      run: |
+        # apt in runner, if needed, may not be up-to-date
+        sudo apt-get update
     - name: Checkout
       uses: actions/checkout@v3
     #-------- Python 3 -----
     - name: Set up supported Python ${{ matrix.python-version }}
       id: setup-python
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != env.next }}
       # wrap broken actions/setup-python@v4
+      # NB may run apt-get install in Linux
       uses: ytdl-org/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
@@ -164,23 +194,23 @@ jobs:
             'import sys' \
             'print(sys.path)' \
             | ${expected} -
-    #-------- Python 3.12 -
+    #-------- Python next (was 3.12) -
-    - name: Set up CPython 3.12 environment
+    - name: Set up CPython 3.next environment
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       shell: bash
       run: |
         PYENV_ROOT=$HOME/.local/share/pyenv
         echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
-    - name: Cache Python 3.12
+    - name: Cache Python 3.next 
-      id: cache312
+      id: cachenext
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       uses: actions/cache@v3
       with:
-        key: python-3.12
+        key: python-${{ env.next }}
         path: |
           ${{ env.PYENV_ROOT }}
-    - name: Build and set up Python 3.12
+    - name: Build and set up Python 3.next
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next && ! steps.cachenext.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -192,12 +222,13 @@ jobs:
         export PYENV_ROOT=${{ env.PYENV_ROOT }}
         export PATH=$PYENV_ROOT/bin:$PATH
         git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
-        pyenv install 3.12.0b4
+        pyenv install ${{ env.next }}
-    - name: Locate Python 3.12
+    - name: Locate Python 3.next
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       shell: bash
       run: |
-        PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
+        PYTHONHOME="$(echo "${{ env.PYENV_ROOT }}/versions/${{ env.next }}."*)"
+        test -n "$PYTHONHOME"
         echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
         echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
     #-------- Python 2.7 --
@@ -368,7 +399,7 @@ jobs:
         done
     #-------- nose --------
     - name: Install nose for Python ${{ matrix.python-version }}
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == '3.12')) }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == env.next)) }}
       shell: bash
       run: |
         echo "$PATH"

youtube_dl/extractor/youtube.py

@@ -260,16 +260,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         cookies = self._get_cookies('https://www.youtube.com/')
         if cookies.get('__Secure-3PSID'):
             return
-        consent_id = None
+        socs = cookies.get('SOCS')
-        consent = cookies.get('CONSENT')
+        if socs and not socs.value.startswith('CAA'):  # not consented
-        if consent:
+            return
-            if 'YES' in consent.value:
+        self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
-                return
-            consent_id = self._search_regex(
-                r'PENDING\+(\d+)', consent.value, 'consent', default=None)
-        if not consent_id:
-            consent_id = random.randint(100, 999)
-        self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
 
     def _real_initialize(self):
         self._initialize_consent()

youtube_dl/utils.py

@@ -2121,7 +2121,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
             return '_'
         if restricted and ord(char) > 127:
-            return '_'
+            return '' if unicodedata.category(char)[0] in 'CM' else '_'
+
         return char
 
     # Replace look-alike Unicode glyphs