mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-09-29 13:51:25 +02:00
Compare commits
15 Commits
1be72886b7
...
d0d8c02fb6
Author | SHA1 | Date | |
---|---|---|---|
|
d0d8c02fb6 | ||
|
2198b4c20e | ||
|
64d9ba3121 | ||
|
6d10cab483 | ||
|
e610afe5dd | ||
|
c2a56a59eb | ||
|
c362c47be5 | ||
|
1af9f9ff9f | ||
|
0d5a471995 | ||
|
47f7e6589b | ||
|
b9d696015a | ||
|
064981df40 | ||
|
72ce6dbdcd | ||
|
01b97742dc | ||
|
6c3e4c4dd8 |
28
.github/workflows/build.yml
vendored
28
.github/workflows/build.yml
vendored
|
@ -129,7 +129,7 @@ jobs:
|
|||
brotli-python
|
||||
secretstorage
|
||||
EOF
|
||||
sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
|
||||
python devscripts/install_deps.py --print --exclude brotli --exclude brotlicffi >> "$reqs"
|
||||
mamba create -n build --file "$reqs"
|
||||
|
||||
- name: Prepare
|
||||
|
@ -203,12 +203,13 @@ jobs:
|
|||
apt update
|
||||
apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
|
||||
python3.8 -m pip install -U pip setuptools wheel
|
||||
# Cannot access requirements.txt from the repo directory at this stage
|
||||
# Cannot access any files from the repo directory at this stage
|
||||
python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
|
||||
|
||||
run: |
|
||||
cd repo
|
||||
python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt # Cached version may be out of date
|
||||
python3.8 devscripts/install_deps.py -o --include build
|
||||
python3.8 devscripts/install_deps.py --include pyinst --include secretstorage # Cached version may be out of date
|
||||
python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
|
||||
python3.8 devscripts/make_lazy_extractors.py
|
||||
python3.8 pyinst.py
|
||||
|
@ -240,9 +241,10 @@ jobs:
|
|||
- name: Install Requirements
|
||||
run: |
|
||||
brew install coreutils
|
||||
python3 -m pip install -U --user pip setuptools wheel
|
||||
python3 devscripts/install_deps.py --user -o --include build
|
||||
python3 devscripts/install_deps.py --print --include pyinst > requirements.txt
|
||||
# We need to ignore wheels otherwise we break universal2 builds
|
||||
python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
|
||||
python3 -m pip install -U --user --no-binary :all: -r requirements.txt
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -293,8 +295,8 @@ jobs:
|
|||
- name: Install Requirements
|
||||
run: |
|
||||
brew install coreutils
|
||||
python3 -m pip install -U --user pip setuptools wheel
|
||||
python3 -m pip install -U --user Pyinstaller -r requirements.txt
|
||||
python3 devscripts/install_deps.py --user -o --include build
|
||||
python3 devscripts/install_deps.py --user --include pyinst
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -333,8 +335,9 @@ jobs:
|
|||
python-version: "3.8"
|
||||
- name: Install Requirements
|
||||
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
|
||||
python -m pip install -U pip setuptools wheel py2exe
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
|
||||
python devscripts/install_deps.py -o --include build
|
||||
python devscripts/install_deps.py --include py2exe
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
@ -342,7 +345,7 @@ jobs:
|
|||
python devscripts/make_lazy_extractors.py
|
||||
- name: Build
|
||||
run: |
|
||||
python setup.py py2exe
|
||||
python freeze.py
|
||||
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
|
||||
python pyinst.py
|
||||
python pyinst.py --onedir
|
||||
|
@ -382,8 +385,9 @@ jobs:
|
|||
architecture: "x86"
|
||||
- name: Install Requirements
|
||||
run: |
|
||||
python -m pip install -U pip setuptools wheel
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
|
||||
python devscripts/install_deps.py -o --include build
|
||||
python devscripts/install_deps.py
|
||||
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
|
|
2
.github/workflows/core.yml
vendored
2
.github/workflows/core.yml
vendored
|
@ -53,7 +53,7 @@ jobs:
|
|||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
run: python3 ./devscripts/install_deps.py --include ci
|
||||
- name: Run tests
|
||||
continue-on-error: False
|
||||
run: |
|
||||
|
|
4
.github/workflows/download.yml
vendored
4
.github/workflows/download.yml
vendored
|
@ -15,7 +15,7 @@ jobs:
|
|||
with:
|
||||
python-version: 3.9
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
run: python3 ./devscripts/install_deps.py --include ci
|
||||
- name: Run tests
|
||||
continue-on-error: true
|
||||
run: python3 ./devscripts/run_tests.py download
|
||||
|
@ -42,7 +42,7 @@ jobs:
|
|||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
run: python3 ./devscripts/install_deps.py --include ci
|
||||
- name: Run tests
|
||||
continue-on-error: true
|
||||
run: python3 ./devscripts/run_tests.py download
|
||||
|
|
6
.github/workflows/quick-test.yml
vendored
6
.github/workflows/quick-test.yml
vendored
|
@ -15,7 +15,7 @@ jobs:
|
|||
with:
|
||||
python-version: '3.8'
|
||||
- name: Install test requirements
|
||||
run: pip install pytest -r requirements.txt
|
||||
run: python3 ./devscripts/install_deps.py --include ci
|
||||
- name: Run tests
|
||||
run: |
|
||||
python3 -m yt_dlp -v || true
|
||||
|
@ -28,8 +28,8 @@ jobs:
|
|||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v4
|
||||
- name: Install flake8
|
||||
run: pip install flake8
|
||||
run: python3 -m pip install -U flake8
|
||||
- name: Make lazy extractors
|
||||
run: python devscripts/make_lazy_extractors.py
|
||||
run: python3 ./devscripts/make_lazy_extractors.py
|
||||
- name: Run flake8
|
||||
run: flake8 .
|
||||
|
|
9
.github/workflows/release.yml
vendored
9
.github/workflows/release.yml
vendored
|
@ -253,8 +253,7 @@ jobs:
|
|||
- name: Install Requirements
|
||||
run: |
|
||||
sudo apt -y install pandoc man
|
||||
python -m pip install -U pip setuptools wheel twine
|
||||
python -m pip install -U -r requirements.txt
|
||||
python -m pip install -U build setuptools wheel
|
||||
|
||||
- name: Prepare
|
||||
env:
|
||||
|
@ -272,8 +271,12 @@ jobs:
|
|||
run: |
|
||||
rm -rf dist/*
|
||||
make pypi-files
|
||||
printf '%s\n\n' \
|
||||
'Official repository: <https://github.com/yt-dlp/yt-dlp>' \
|
||||
'**PS**: Some links in this document will not work since this is a copy of the README.md from Github' > ./README.md.new
|
||||
cat ./README.md >> ./README.md.new && mv -f ./README.md.new ./README.md
|
||||
python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
|
||||
python setup.py sdist bdist_wheel
|
||||
python -m build --no-isolation .
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
|
2
Makefile
2
Makefile
|
@ -14,7 +14,7 @@ PYTHON ?= /usr/bin/env python3
|
|||
|
||||
# Keep this list in sync with MANIFEST.in
|
||||
# intended use: when building a source distribution,
|
||||
# make pypi-files && python3 -m build -s .
|
||||
# make pypi-files && python3 -m build -sn .
|
||||
pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
|
||||
completions yt-dlp.1 pyproject.toml setup.cfg devscripts/* test/*
|
||||
-mv -f pyproject.toml.old pyproject.toml
|
||||
|
|
20
README.md
20
README.md
|
@ -322,9 +322,11 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
|
|||
### Standalone PyInstaller Builds
|
||||
To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used.
|
||||
|
||||
python3 -m pip install -U pyinstaller -r requirements.txt
|
||||
python3 devscripts/make_lazy_extractors.py
|
||||
python3 pyinst.py
|
||||
```
|
||||
python3 devscripts/install_deps.py --include pyinst
|
||||
python3 devscripts/make_lazy_extractors.py
|
||||
python3 pyinst.py
|
||||
```
|
||||
|
||||
On some systems, you may need to use `py` or `python` instead of `python3`.
|
||||
|
||||
|
@ -345,18 +347,22 @@ You can also run `make yt-dlp` instead to compile only the binary without updati
|
|||
|
||||
While we provide the option to build with [py2exe](https://www.py2exe.org), it is recommended to build [using PyInstaller](#standalone-pyinstaller-builds) instead since the py2exe builds **cannot contain `pycryptodomex`/`certifi` and needs VC++14** on the target computer to run.
|
||||
|
||||
If you wish to build it anyway, install Python and py2exe, and then simply run `setup.py py2exe`
|
||||
If you wish to build it anyway, install Python and py2exe, and then simply run `py freeze.py`
|
||||
|
||||
py -m pip install -U py2exe -r requirements.txt
|
||||
py devscripts/make_lazy_extractors.py
|
||||
py setup.py py2exe
|
||||
```
|
||||
py devscripts/install_deps.py --include py2exe
|
||||
py devscripts/make_lazy_extractors.py
|
||||
py freeze.py
|
||||
```
|
||||
|
||||
### Related scripts
|
||||
|
||||
* **`devscripts/install_deps.py`** - Install dependencies for yt-dlp.
|
||||
* **`devscripts/update-version.py`** - Update the version number based on current date.
|
||||
* **`devscripts/set-variant.py`** - Set the build variant of the executable.
|
||||
* **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file.
|
||||
* **`devscripts/make_lazy_extractors.py`** - Create lazy extractors. Running this before building the binaries (any variant) will improve their startup performance. Set the environment variable `YTDLP_NO_LAZY_EXTRACTORS=1` if you wish to forcefully disable lazy extractor loading.
|
||||
* **`devscripts/include_data_files.py`** - Running this before building a wheel will ensure any generated documentation and/or shell completions will be included in the wheel.
|
||||
|
||||
Note: See their `--help` for more info.
|
||||
|
||||
|
|
0
devscripts/include_data_files.py
Normal file → Executable file
0
devscripts/include_data_files.py
Normal file → Executable file
66
devscripts/install_deps.py
Executable file
66
devscripts/install_deps.py
Executable file
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Allow execution from anywhere
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from devscripts.tomlparse import parse_toml
|
||||
from devscripts.utils import read_file
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
|
||||
parser.add_argument(
|
||||
'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
|
||||
parser.add_argument(
|
||||
'-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
|
||||
parser.add_argument(
|
||||
'-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
|
||||
parser.add_argument(
|
||||
'-o', '--only-optional', action='store_true', help='Only install optional dependencies')
|
||||
parser.add_argument(
|
||||
'-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
|
||||
parser.add_argument(
|
||||
'-u', '--user', action='store_true', help='Install with pip as --user')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
toml_data = parse_toml(read_file(args.input))
|
||||
deps = toml_data['project']['dependencies']
|
||||
targets = deps.copy() if not args.only_optional else []
|
||||
|
||||
for exclude in args.exclude or []:
|
||||
for dep in deps:
|
||||
simplified_dep = re.match(r'\w+', dep)[0]
|
||||
if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
|
||||
targets.remove(dep)
|
||||
|
||||
optional_deps = toml_data['project']['optional-dependencies']
|
||||
for include in args.include or []:
|
||||
group = optional_deps.get(include)
|
||||
if group:
|
||||
targets.extend(group)
|
||||
|
||||
if args.print:
|
||||
for target in targets:
|
||||
print(target)
|
||||
return
|
||||
|
||||
pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
|
||||
if args.user:
|
||||
pip_args.append('--user')
|
||||
pip_args.extend(targets)
|
||||
|
||||
return subprocess.call(pip_args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
172
devscripts/tomlparse.py
Executable file
172
devscripts/tomlparse.py
Executable file
|
@ -0,0 +1,172 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
WS = r'(?:[\ \t]*)'
|
||||
STRING_RE = re.compile(r'"(?:\\.|[^\\"\n])*"|\'[^\'\n]*\'')
|
||||
SINGLE_KEY_RE = re.compile(rf'{STRING_RE.pattern}|[A-Za-z0-9_-]+')
|
||||
KEY_RE = re.compile(rf'{WS}(?:{SINGLE_KEY_RE.pattern}){WS}(?:\.{WS}(?:{SINGLE_KEY_RE.pattern}){WS})*')
|
||||
EQUALS_RE = re.compile(rf'={WS}')
|
||||
WS_RE = re.compile(WS)
|
||||
|
||||
_SUBTABLE = rf'(?P<subtable>^\[(?P<is_list>\[)?(?P<path>{KEY_RE.pattern})\]\]?)'
|
||||
EXPRESSION_RE = re.compile(rf'^(?:{_SUBTABLE}|{KEY_RE.pattern}=)', re.MULTILINE)
|
||||
|
||||
LIST_WS_RE = re.compile(rf'{WS}((#[^\n]*)?\n{WS})*')
|
||||
UNKNOWN_VALUE_RE = re.compile(r'[^,}\] \t\n#]+')
|
||||
|
||||
|
||||
def parse_key(value: str):
|
||||
for match in SINGLE_KEY_RE.finditer(value):
|
||||
if match[0][0] == '"':
|
||||
yield json.loads(match[0])
|
||||
elif match[0][0] == '\'':
|
||||
yield match[0][1:-1]
|
||||
else:
|
||||
yield match[0]
|
||||
|
||||
|
||||
def get_target(root: dict, paths: list[str], is_list=False):
|
||||
target = root
|
||||
|
||||
for index, key in enumerate(paths, 1):
|
||||
use_list = is_list and index == len(paths)
|
||||
result = target.get(key)
|
||||
if result is None:
|
||||
if use_list:
|
||||
target[key] = _temp = []
|
||||
target = {}
|
||||
_temp.append(target)
|
||||
else:
|
||||
target[key] = target = {}
|
||||
|
||||
elif isinstance(result, list):
|
||||
if use_list:
|
||||
target = {}
|
||||
result.append(target)
|
||||
else:
|
||||
target = result[-1]
|
||||
|
||||
else:
|
||||
target = result
|
||||
|
||||
assert isinstance(target, dict)
|
||||
return target
|
||||
|
||||
|
||||
def parse_enclosed(data: str, index: int, end: str, ws_re: re.Pattern):
|
||||
index += 1
|
||||
|
||||
match = ws_re.match(data, index)
|
||||
assert match
|
||||
index = match.end()
|
||||
|
||||
while data[index] != end:
|
||||
index = yield True, index
|
||||
if data[index] == ',':
|
||||
index += 1
|
||||
|
||||
match = ws_re.match(data, index)
|
||||
assert match
|
||||
index = match.end()
|
||||
|
||||
yield False, index + 1
|
||||
|
||||
|
||||
def parse_value(data: str, index: int):
|
||||
if data[index] == '[':
|
||||
result = []
|
||||
|
||||
indices = parse_enclosed(data, index, ']', LIST_WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
index, value = parse_value(data, index)
|
||||
result.append(value)
|
||||
valid, index = indices.send(index)
|
||||
|
||||
return index, result
|
||||
|
||||
if data[index] == '{':
|
||||
result = {}
|
||||
|
||||
indices = parse_enclosed(data, index, '}', WS_RE)
|
||||
valid, index = next(indices)
|
||||
while valid:
|
||||
valid, index = indices.send(parse_kv_pair(data, index, result))
|
||||
|
||||
return index, result
|
||||
|
||||
if match := STRING_RE.match(data, index):
|
||||
return match.end(), json.loads(match[0]) if match[0][0] == '"' else match[0][1:-1]
|
||||
|
||||
# bool, int, float or date. Of these, only bool is used in pyproject.toml, so ignore others
|
||||
match = UNKNOWN_VALUE_RE.match(data, index)
|
||||
assert match
|
||||
if match[0] == 'true':
|
||||
value = True
|
||||
elif match[0] == 'false':
|
||||
value = False
|
||||
else:
|
||||
value = None
|
||||
|
||||
return match.end(), value
|
||||
|
||||
|
||||
def parse_kv_pair(data: str, index: int, target: dict):
|
||||
match = KEY_RE.match(data, index)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
*keys, key = parse_key(match[0])
|
||||
|
||||
match = EQUALS_RE.match(data, match.end())
|
||||
assert match
|
||||
index = match.end()
|
||||
|
||||
index, value = parse_value(data, index)
|
||||
get_target(target, keys)[key] = value
|
||||
return index
|
||||
|
||||
|
||||
def parse_toml(data: str):
|
||||
root = {}
|
||||
target = root
|
||||
|
||||
index = 0
|
||||
while True:
|
||||
match = EXPRESSION_RE.search(data, index)
|
||||
if not match:
|
||||
break
|
||||
|
||||
if match.group('subtable'):
|
||||
index = match.end()
|
||||
path, is_list = match.group('path', 'is_list')
|
||||
target = get_target(root, list(parse_key(path)), bool(is_list))
|
||||
continue
|
||||
|
||||
index = parse_kv_pair(data, match.start(), target)
|
||||
assert index is not None
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile', type=Path, help='The TOML file to read as input')
|
||||
args = parser.parse_args()
|
||||
|
||||
with args.infile.open('r', encoding='utf-8') as file:
|
||||
data = file.read()
|
||||
|
||||
print(json.dumps(parse_toml(data)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -14,12 +14,12 @@ from py2exe import freeze
|
|||
VERSION = read_version(varname='_pkg_version')
|
||||
|
||||
|
||||
def py2exe_params():
|
||||
def main():
|
||||
warnings.warn(
|
||||
'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
|
||||
'It is recommended to run "pyinst.py" to build using pyinstaller instead')
|
||||
|
||||
return {
|
||||
return freeze(**{
|
||||
'console': [{
|
||||
'script': './yt_dlp/__main__.py',
|
||||
'dest_base': 'yt-dlp',
|
||||
|
@ -51,11 +51,7 @@ def py2exe_params():
|
|||
'yt_dlp.utils._legacy', 'yt_dlp.utils._deprecated'],
|
||||
},
|
||||
'zipfile': None,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
return freeze(py2exe_params())
|
||||
})
|
||||
|
||||
|
||||
main()
|
|
@ -80,8 +80,10 @@ yt-dlp = "yt_dlp:main"
|
|||
[project.entry-points.pyinstaller40]
|
||||
hook-dirs = "yt_dlp.__pyinstaller:get_hook_dirs"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["yt_dlp"]
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["yt_dlp*"]
|
||||
exclude = ["youtube_dl*", "youtube_dlc*", "test*", "ytdlp_plugins*", "devscripts*"]
|
||||
namespaces = false # to disable scanning PEP 420 namespaces (true by default)
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {attr = "yt_dlp.version._pkg_version"}
|
||||
|
|
Loading…
Reference in New Issue
Block a user