From d0ef06593d5e7b7250ebd021bfcf0834745ef81d Mon Sep 17 00:00:00 2001 From: Simon Sawicki Date: Mon, 22 Apr 2024 22:06:33 +0200 Subject: [PATCH 01/13] Use `self.assertWarns` --- test/test_InfoExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index c633ce3e47a..744587e45b0 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -1912,7 +1912,7 @@ def test_search_nextjs_data(self): self.assertEqual(self.ie._search_nextjs_data('', None, fatal=False), {}) self.assertEqual(self.ie._search_nextjs_data('', None, default=None), None) self.assertEqual(self.ie._search_nextjs_data('', None, default={}), {}) - with self.assertRaises(DeprecationWarning): + with self.assertWarns(DeprecationWarning): self.assertEqual(self.ie._search_nextjs_data('', None, default='{}'), {}) From c36cac6b546973b7262f658d14a55403e6968ba0 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 29 Apr 2024 18:25:12 -0500 Subject: [PATCH 02/13] [Makefile] Bring `offlinetest` recipe on par with `devscripts.run_tests` Authored by: bashonly --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cef4bc6cb1a..fb4af334a6f 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ test: $(MAKE) codetest offlinetest: codetest - $(PYTHON) -m pytest -k "not download" + $(PYTHON) -m pytest -Werror -m "not download" CODE_FOLDERS_CMD = find yt_dlp -type f -name '__init__.py' | sed 's,/__init__.py,,' | grep -v '/__' | sort CODE_FOLDERS != $(CODE_FOLDERS_CMD) From b9aeb3e57495d65d27da3c47032a2d2c335347a3 Mon Sep 17 00:00:00 2001 From: bashonly Date: Mon, 29 Apr 2024 18:27:03 -0500 Subject: [PATCH 03/13] [Makefile] Add `-Werror` arg to `test` recipe Authored by: bashonly --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fb4af334a6f..ad9fa02abd4 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ codetest: flake8 . test: - $(PYTHON) -m pytest + $(PYTHON) -m pytest -Werror $(MAKE) codetest offlinetest: codetest From adb148358ed49a73e14bc7092259b76cf73dac83 Mon Sep 17 00:00:00 2001 From: bashonly Date: Tue, 30 Apr 2024 17:52:25 -0500 Subject: [PATCH 04/13] [utils] `read_batch_urls`: Fix deprecated `re.split` positional arg (3.13) Authored by: bashonly --- yt_dlp/utils/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py index b637669124f..aa7bf2ce4a3 100644 --- a/yt_dlp/utils/_utils.py +++ b/yt_dlp/utils/_utils.py @@ -2522,7 +2522,7 @@ def fixup(url): return False # "#" cannot be stripped out since it is part of the URI # However, it can be safely stripped out if following a whitespace - return re.split(r'\s#', url, 1)[0].rstrip() + return re.split(r'\s#', url, maxsplit=1)[0].rstrip() with contextlib.closing(batch_fd) as fd: return [url for url in map(fixup, fd) if url] From c91b90520ffb7503a18f3d2c075a0c751a7837e3 Mon Sep 17 00:00:00 2001 From: bashonly Date: Tue, 30 Apr 2024 17:56:00 -0500 Subject: [PATCH 05/13] [ie] Fix deprecated `re.split` positional arg (3.13) Authored by: bashonly --- yt_dlp/extractor/ceskatelevize.py | 2 +- yt_dlp/extractor/common.py | 2 +- yt_dlp/extractor/thisvid.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/ceskatelevize.py b/yt_dlp/extractor/ceskatelevize.py index 156b6a32476..5d633572962 100644 --- a/yt_dlp/extractor/ceskatelevize.py +++ b/yt_dlp/extractor/ceskatelevize.py @@ -101,7 +101,7 @@ def _real_extract(self, url): site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize') playlist_title = self._og_search_title(webpage, default=None) if site_name and playlist_title: - playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, 1)[0] + playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, maxsplit=1)[0] playlist_description = self._og_search_description(webpage, default=None) if playlist_description: playlist_description = playlist_description.replace('\xa0', ' ') diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index bebbc6b43f9..7e9da7b3b5b 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -3526,7 +3526,7 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None, # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as # of jwplayer.flash.swf rtmp_url_parts = re.split( - r'((?:mp4|mp3|flv):)', source_url, 1) + r'((?:mp4|mp3|flv):)', source_url, maxsplit=1) if len(rtmp_url_parts) == 3: rtmp_url, prefix, play_path = rtmp_url_parts a_format.update({ diff --git a/yt_dlp/extractor/thisvid.py b/yt_dlp/extractor/thisvid.py index 9d3368ed752..04b0838116c 100644 --- a/yt_dlp/extractor/thisvid.py +++ b/yt_dlp/extractor/thisvid.py @@ -134,7 +134,7 @@ def _make_playlist_result(self, url): title = re.split( r'(?i)\s*\|\s*ThisVid\.com\s*$', self._og_search_title(webpage, default=None) - or self._html_search_regex(r'(?s)]*>(.+?)]*>(.+?) Date: Tue, 30 Apr 2024 17:59:16 -0500 Subject: [PATCH 06/13] [docs] Remove `yt_dlp_linux.zip` from README Authored by: bashonly --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 37da789cf6f..dfe51c18b2e 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ File|Description [yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Win7 SP1+) standalone x86 (32-bit) binary [yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`
([Not recommended](#standalone-py2exe-builds-windows)) [yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary -[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux executable (no auto-update) [yt-dlp_linux_armv7l](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l)|Linux standalone armv7l (32-bit) binary [yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux standalone aarch64 (64-bit) binary [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update) From 97c4c860436fd4e9df52e2ee5c521844d3998ef4 Mon Sep 17 00:00:00 2001 From: sepro <4618135+seproDev@users.noreply.github.com> Date: Thu, 9 May 2024 00:32:27 +0200 Subject: [PATCH 07/13] [ie/vk] Extract additional potentially existing formats And exclude custom formats that would require custom live downloaders --- yt_dlp/extractor/vk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py index 28d5026850b..1136e41149e 100644 --- a/yt_dlp/extractor/vk.py +++ b/yt_dlp/extractor/vk.py @@ -467,13 +467,13 @@ def _real_extract(self, url): 'source_preference': 1, 'height': height, }) - elif format_id == 'hls': + elif format_id.startswith('hls') and format_id != 'hls_live_playback': fmts, subs = self._extract_m3u8_formats_and_subtitles( format_url, video_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False, live=is_live) formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) - elif format_id.startswith('dash_'): + elif format_id.startswith('dash') and format_id not in ('dash_live_playback', 'dash_uni'): fmts, subs = self._extract_mpd_formats_and_subtitles( format_url, video_id, mpd_id=format_id, fatal=False) formats.extend(fmts) From b01f22cdbf5830b3265bcabea3e16ff5d2ea3ede Mon Sep 17 00:00:00 2001 From: bashonly Date: Thu, 9 May 2024 11:05:06 -0500 Subject: [PATCH 08/13] [docs] brackets must be quoted in zsh Authored by: bashonly --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dfe51c18b2e..2ec957bab27 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ Example usage: yt-dlp --update-to nightly # To install nightly with pip: -python3 -m pip install -U --pre yt-dlp[default] +python3 -m pip install -U --pre "yt-dlp[default]" ``` ## DEPENDENCIES @@ -201,7 +201,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly The following provide support for impersonating browser requests. This may be required for some sites that employ TLS fingerprinting. * [**curl_cffi**](https://github.com/yifeikong/curl_cffi) (recommended) - Python binding for [curl-impersonate](https://github.com/lwthiker/curl-impersonate). Provides impersonation targets for Chrome, Edge and Safari. Licensed under [MIT](https://github.com/yifeikong/curl_cffi/blob/main/LICENSE) - * Can be installed with the `curl-cffi` group, e.g. `pip install yt-dlp[default,curl-cffi]` + * Can be installed with the `curl-cffi` group, e.g. `pip install "yt-dlp[default,curl-cffi]"` * Currently only included in `yt-dlp.exe` and `yt-dlp_macos` builds From 41cef9410da761158cfae6bfb7a62932d6700c22 Mon Sep 17 00:00:00 2001 From: sepro <4618135+seproDev@users.noreply.github.com> Date: Fri, 10 May 2024 16:22:57 +0200 Subject: [PATCH 09/13] Remove deprecated run_tests scripts --- devscripts/run_tests.bat | 4 ---- devscripts/run_tests.sh | 4 ---- 2 files changed, 8 deletions(-) delete mode 100644 devscripts/run_tests.bat delete mode 100755 devscripts/run_tests.sh diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat deleted file mode 100644 index 57b1f4bf465..00000000000 --- a/devscripts/run_tests.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - ->&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead -python %~dp0run_tests.py %~1 diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh deleted file mode 100755 index 123ceb1ee4f..00000000000 --- a/devscripts/run_tests.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env sh - ->&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead' -python3 devscripts/run_tests.py "$1" From 454bf0baf8d5921c50afecdcaf4df45031c704e0 Mon Sep 17 00:00:00 2001 From: bashonly Date: Fri, 10 May 2024 09:49:34 -0500 Subject: [PATCH 10/13] Remove compat `setup.py` and `pyinst.py` Authored by: bashonly --- pyinst.py | 17 ----------------- setup.py | 36 ------------------------------------ 2 files changed, 53 deletions(-) delete mode 100755 pyinst.py delete mode 100755 setup.py diff --git a/pyinst.py b/pyinst.py deleted file mode 100755 index 4a8ed2d3497..00000000000 --- a/pyinst.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 - -# Allow execution from anywhere -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -import warnings - -from bundle.pyinstaller import main - -warnings.warn(DeprecationWarning('`pyinst.py` is deprecated and will be removed in a future version. ' - 'Use `bundle.pyinstaller` instead')) - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py deleted file mode 100755 index 8d1e6d10b22..00000000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -# Allow execution from anywhere -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -import warnings - - -if sys.argv[1:2] == ['py2exe']: - warnings.warn(DeprecationWarning('`setup.py py2exe` is deprecated and will be removed in a future version. ' - 'Use `bundle.py2exe` instead')) - - import bundle.py2exe - - bundle.py2exe.main() - -elif 'build_lazy_extractors' in sys.argv: - warnings.warn(DeprecationWarning('`setup.py build_lazy_extractors` is deprecated and will be removed in a future version. ' - 'Use `devscripts.make_lazy_extractors` instead')) - - import subprocess - - os.chdir(sys.path[0]) - print('running build_lazy_extractors') - subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py']) - -else: - - print( - 'ERROR: Building by calling `setup.py` is deprecated. ' - 'Use a build frontend like `build` instead. ', - 'Refer to https://build.pypa.io for more info', file=sys.stderr) - sys.exit(1) From 04c50370ce557df243d23040684eabcf338978ef Mon Sep 17 00:00:00 2001 From: sepro <4618135+seproDev@users.noreply.github.com> Date: Sat, 18 May 2024 16:23:09 +0200 Subject: [PATCH 11/13] Remove deprecated option from example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ec957bab27..a44d65f10c9 100644 --- a/README.md +++ b/README.md @@ -1750,7 +1750,7 @@ $ yt-dlp --replace-in-metadata "title,uploader" "[ _]" "-" # EXTRACTOR ARGUMENTS -Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;include_live_dash" --extractor-args "funimation:version=uncut"` +Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;formats=incomplete" --extractor-args "funimation:version=uncut"` Note: In CLI, `ARG` can use `-` instead of `_`; e.g. `youtube:player-client"` becomes `youtube:player_client"` From 61a1e15b68af3e510d9c22b5ac518e3972728824 Mon Sep 17 00:00:00 2001 From: bashonly Date: Wed, 22 May 2024 11:46:08 -0500 Subject: [PATCH 12/13] Changelog override: Suppress "[build] Exclude `requests` from `py2exe`" Authored by: bashonly --- devscripts/changelog_override.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index 046060cb257..72dc8d43a3f 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -147,5 +147,9 @@ "action": "add", "when": "9590cc6b4768e190183d7d071a6c78170889116a", "short": "[priority] Security: [[CVE-2024-22423](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-22423)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-hjq6-52gw-2g7p)\n - The shell escape function now properly escapes `%`, `\\` and `\\n`.\n - `utils.Popen` has been patched accordingly." + }, + { + "action": "remove", + "when": "6e36d17f404556f0e3a43f441c477a71a91877d9" } ] From 7c044bd3efdb8af01ae2b0e8073a6ad15b87ebf8 Mon Sep 17 00:00:00 2001 From: bashonly Date: Sun, 26 May 2024 11:17:37 -0500 Subject: [PATCH 13/13] changelog overrides Authored by: bashonly --- devscripts/changelog_override.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json index 72dc8d43a3f..4be1e58d432 100644 --- a/devscripts/changelog_override.json +++ b/devscripts/changelog_override.json @@ -148,8 +148,20 @@ "when": "9590cc6b4768e190183d7d071a6c78170889116a", "short": "[priority] Security: [[CVE-2024-22423](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-22423)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-hjq6-52gw-2g7p)\n - The shell escape function now properly escapes `%`, `\\` and `\\n`.\n - `utils.Popen` has been patched accordingly." }, + { + "action": "change", + "when": "41ba4a808b597a3afed78c89675a30deb6844450", + "short": "[ie/tiktok] Extract via mobile API only if extractor-arg is passed (#9938)", + "authors": ["bashonly"] + }, { "action": "remove", "when": "6e36d17f404556f0e3a43f441c477a71a91877d9" + }, + { + "action": "change", + "when": "beaf832c7a9d57833f365ce18f6115b88071b296", + "short": "[ie/soundcloud] Add `formats` extractor-arg (#10004)", + "authors": ["bashonly", "Grub4K"] } ]