From 2e5bf002dad16f5ce35aa2023d392c9e518fcd8f Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 26 May 2025 14:44:35 -0500 Subject: [PATCH] [ie/go] Fix provider-locked content extraction (#13131) Closes #1770, Closes #8073 Authored by: maxbin123, bashonly Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com> --- yt_dlp/extractor/go.py | 322 +++++++++++++++++++---------------------- 1 file changed, 147 insertions(+), 175 deletions(-) diff --git a/yt_dlp/extractor/go.py b/yt_dlp/extractor/go.py index 83c1979db..4e138a828 100644 --- a/yt_dlp/extractor/go.py +++ b/yt_dlp/extractor/go.py @@ -7,161 +7,157 @@ from ..utils import ( int_or_none, join_nonempty, parse_age_limit, - remove_end, - remove_start, - traverse_obj, - try_get, unified_timestamp, urlencode_postdata, ) +from ..utils.traversal import traverse_obj class GoIE(AdobePassIE): _SITE_INFO = { 'abc': { 'brand': '001', - 'requestor_id': 'ABC', + 'requestor_id': 'dtci', + 'provider_id': 'ABC', + 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI4OTcwMjlkYS0yYjM1LTQyOWUtYWQ0NS02ZjZiZjVkZTdhOTUiLCJuYmYiOjE2MjAxNzM5NjksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNjIwMTczOTY5fQ.SC69DVJWSL8sIe-vVUrP6xS_kzHKqwz9PdKYexs_y-f7Vin6mM-7S-W1TE_-K55O0pyf-TL4xYgvm6LIye8CckG-nZfVwNPV4huduov0jmIcxCQFeUwkHULG2IaA44wfBVUBdaHgkhPweZ2amjycO_IXtez-gBXOLbE3B7Gx9j_5ISCFtyVUblThKfoGyQv6KT6t8Vpmc4ZSKCCQp74KWFFypydb9ucego1taW_nQD06Cdf4yByLd6NaTBceMcIKbug9b9gxFm3XBgJ5q3z7KGo1Kr6XalAV5j4m-fQ91wczlTilX8FM4AljMupyRM9mA_aEADILQ4hS79q4SM0w6w', }, 'freeform': { 'brand': '002', 'requestor_id': 'ABCFamily', - }, - 'watchdisneychannel': { - 'brand': '004', - 'resource_id': 'Disney', - }, - 'watchdisneyjunior': { - 'brand': '008', - 'resource_id': 'DisneyJunior', - }, - 'watchdisneyxd': { - 'brand': '009', - 'resource_id': 'DisneyXD', + 'provider_id': 'ABCFamily', + 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZWM2MGYyNC0xYzRjLTQ1NzQtYjc0Zi03ZmM4N2E5YWMzMzgiLCJuYmYiOjE1ODc2NjU5MjMsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTg3NjY1OTIzfQ.flCn3dhvmvPnWmV0JV8Fm0YFyj07yPez9-n1GFEwVIm_S2wQVWbWyJhqsAyLZVFrhOMZYTqmPS3OHxGwTwXkEYn6PD7o_vIVG3oqi-Xn1m5jRt_Gazw5qEtpat6VE7bvKGSD3ZhcidOrsCk8NcYyq75u61NHDvSl81pcedJjVRVUpsqrEwmo0aVbA0C8PX3ri0mEbGvkMKvHn8E60xp-PSE-VK8SDT0plwPu_TwUszkZ6-_I8_2xcv_WBqcXFkAVg7Q-iNJXgQvmNsrpcrYuLvi6hEH4ZLtoDcXU6MhwTQAJTiHSo8x9aHX1_qFP09CzlNOFQbC2ZEJdP9SvA53SLQ', }, 'disneynow': { - 'brand': '011', + 'brand': '011', # also: '004', '008', '009' + 'requestor_id': 'DisneyChannels', + 'provider_id': 'DisneyChannels', + 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1MzAzNTRiOS04NDNiLTRkNjAtYTQ3ZS0yNzk1MzlkOTIyNTciLCJuYmYiOjE1NTg5ODc0NDksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTU4OTg3NDQ5fQ.Jud6YS6-J2h0h6po0oMheDym0qRTJQGj4kzacrz4DFuEwhcBkkykW6pF5pKuAUJy9HCZ40oDAHe2KcTlDJjCZF5tDaUEfdihakZ9cC_rG7MU-QoRne8qaB_dPDKwGuk-ZyWD8eV3zwTJmbGo8hDxYTEU81YNCxwhyc_BPDr5TYiubbmpP3_pTnXmSpuL58isJ2peSKWlX9BacuXtBY25c_QnPFKk-_EETm7IHkTpDazde1QfHWGu4s4yJpKGk8RVVujVG6h6ELlL-ZeYLilBm7iS7h1TYG1u7fJhyZRL7isaom6NvAzsvN3ngss1fLwt8decP8wzdFHrbYTdTjW8qw', 'resource_id': 'Disney', }, - 'fxnow.fxnetworks': { - 'brand': '025', + 'fxnetworks': { + 'brand': '025', # also: '020' 'requestor_id': 'dtci', + 'provider_id': 'fx', # also 'fxx', 'fxm' + 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIzYWRhYWZiNC02OTAxLTRlYzktOTdmNy1lYWZkZTJkODJkN2EiLCJuYmYiOjE1NjIwMjQwNzYsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDI0MDc2fQ.dhKMpZK50AObbZYrMiYPSfWtzXHUaeMP3jrIY4Cgfvh0GaEgk0Mns_zp78jypFeZgRtPVleQMQDNq2YEloRLcAGqP1aa6WVDglnK77ZWUm4IKai14Rwf3A6YBhSRoO2_lMmUGkuTf6gZY-kMIPqBYKqzTQiQl4HbniPFodIzFRiuI9QJVrkoyTGrJL4oqiX08PoFI3Z-TOti1Heu3EbFC-GveQHhlinYrzU7rbiAqLEz7FImtfBDsnXX1Y3uJDLYM3Bq4Oh0nrzTv1Fd62wNsCNErHHIbELidh1zZF0ujvt7ReuZUwAitm0UhEJ7OxNOUbEQWtae6pVNscvdvTFMpg', + }, + 'nationalgeographic': { + 'brand': '026', # also '023' + 'requestor_id': 'dtci', + 'provider_id': 'ngc', # also 'ngw' + 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMzE4YTM1Ni05Mjc4LTQ4NjEtYTFmNi1jMTIzMzg1ZWMzYzMiLCJuYmYiOjE1NjIwMjM4MjgsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDIzODI4fQ.Le-2OzF9-jrhJ7ZfWtLWk5iSHGVZoxeU1w0_fO--Heli0OwRZsRq2slSmx-oZTzxuWmAgDEiBkWSDcDK6sM25DrCLsdsJa3MBuZ-slBRtH8aq3HpNoqqLkU-vg6gRUEKMtwBUtwCu_9aKUCayYtndWv4b1DjVQeSrteOW5NNudWVYleAe0kxeNJQHo5If9SCzDudKVJktFUjhNks4QPOC_uONPkRRlL9D0fNvtOY-LRFckfcHhf5z9l1iZjeukV0YhdKnuw1wyiaWrQXBUDiBfbkCRd2DM-KnelqPxfiXCaTjGKDURRBO3pz33ebge3IFXSiU5vl4qHQ8xvunzGpFw', }, } - _VALID_URL = r'''(?x) - https?:// - (?P - (?:{}\.)?go|fxnow\.fxnetworks| - (?:www\.)?(?:abc|freeform|disneynow) - )\.com/ - (?: - (?:[^/]+/)*(?P[Vv][Dd][Kk][Aa]\w+)| - (?:[^/]+/)*(?P[^/?\#]+) - ) - '''.format(r'\.|'.join(list(_SITE_INFO.keys()))) + _URL_PATH_RE = r'(?:video|episode|movies-and-specials)/(?P[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' + _VALID_URL = [ + fr'https?://(?:www\.)?(?Pabc)\.com/{_URL_PATH_RE}', + fr'https?://(?:www\.)?(?Pfreeform)\.com/{_URL_PATH_RE}', + fr'https?://(?:www\.)?(?Pdisneynow)\.com/{_URL_PATH_RE}', + fr'https?://fxnow\.(?Pfxnetworks)\.com/{_URL_PATH_RE}', + fr'https?://(?:www\.)?(?Pnationalgeographic)\.com/tv/{_URL_PATH_RE}', + ] _TESTS = [{ - 'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643', + 'url': 'https://abc.com/episode/4192c0e6-26e5-47a8-817b-ce8272b9e440/playlist/PL551127435', 'info_dict': { - 'id': 'VDKA3807643', + 'id': 'VDKA10805898', 'ext': 'mp4', - 'title': 'The Traitor in the White House', - 'description': 'md5:05b009d2d145a1e85d25111bd37222e8', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'This content is no longer available.', - }, { - 'url': 'https://disneynow.com/shows/big-hero-6-the-series', - 'info_dict': { - 'title': 'Doraemon', - 'id': 'SH55574025', - }, - 'playlist_mincount': 51, - }, { - 'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood', - 'info_dict': { - 'id': 'VDKA3609139', - 'title': 'This Guilty Blood', - 'description': 'md5:f18e79ad1c613798d95fdabfe96cd292', + 'title': 'Switch the Flip', + 'description': 'To help get Brian’s life in order, Stewie and Brian swap bodies using a machine that Stewie invents.', 'age_limit': 14, + 'duration': 1297, + 'thumbnail': r're:https?://.+/.+\.jpg', + 'series': 'Family Guy', + 'season': 'Season 16', + 'season_number': 16, + 'episode': 'Episode 17', + 'episode_number': 17, + 'timestamp': 1746082800.0, + 'upload_date': '20250501', + }, + 'params': {'skip_download': 'm3u8'}, + 'skip': 'This video requires AdobePass MSO credentials', + }, { + 'url': 'https://disneynow.com/episode/21029660-ba06-4406-adb0-a9a78f6e265e/playlist/PL553044961', + 'info_dict': { + 'id': 'VDKA39546942', + 'ext': 'mp4', + 'title': 'Zero Friends Again', + 'description': 'Relationships fray under the pressures of a difficult journey.', + 'age_limit': 0, + 'duration': 1721, + 'thumbnail': r're:https?://.+/.+\.jpg', + 'series': 'Star Wars: Skeleton Crew', + 'season': 'Season 1', + 'season_number': 1, + 'episode': 'Episode 6', + 'episode_number': 6, + 'timestamp': 1746946800.0, + 'upload_date': '20250511', + }, + 'params': {'skip_download': 'm3u8'}, + 'skip': 'This video requires AdobePass MSO credentials', + }, { + 'url': 'https://fxnow.fxnetworks.com/episode/09f4fa6f-c293-469e-aebe-32c9ca5842a7/playlist/PL554408064', + 'info_dict': { + 'id': 'VDKA38112033', + 'ext': 'mp4', + 'title': 'The Return of Jerry', + 'description': 'The vampires’ long-lost fifth roommate returns. Written by Paul Simms; directed by Kyle Newacheck.', + 'age_limit': 17, + 'duration': 1493, + 'thumbnail': r're:https?://.+/.+\.jpg', + 'series': 'What We Do in the Shadows', + 'season': 'Season 6', + 'season_number': 6, 'episode': 'Episode 1', - 'upload_date': '20170102', - 'season': 'Season 2', - 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg', - 'duration': 2544, - 'season_number': 2, - 'series': 'Shadowhunters', 'episode_number': 1, - 'timestamp': 1483387200, - 'ext': 'mp4', - }, - 'params': { - 'geo_bypass_ip_block': '3.244.239.0/24', - # m3u8 download - 'skip_download': True, + 'timestamp': 1729573200.0, + 'upload_date': '20241022', }, + 'params': {'skip_download': 'm3u8'}, + 'skip': 'This video requires AdobePass MSO credentials', }, { - 'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock', + 'url': 'https://www.freeform.com/episode/bda0eaf7-761a-4838-aa44-96f794000844/playlist/PL553044961', 'info_dict': { - 'id': 'VDKA26050359', - 'title': 'The Knock', - 'description': 'md5:0c2947e3ada4c31f28296db7db14aa64', - 'age_limit': 14, + 'id': 'VDKA39007340', 'ext': 'mp4', - 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg', - 'episode': 'Episode 12', - 'season_number': 4, - 'season': 'Season 4', - 'timestamp': 1642975200, - 'episode_number': 12, - 'upload_date': '20220123', - 'series': 'The Rookie', - 'duration': 2572, - }, - 'params': { - 'geo_bypass_ip_block': '3.244.239.0/24', - # m3u8 download - 'skip_download': True, + 'title': 'Angel\'s Landing', + 'description': 'md5:91bf084e785c968fab16734df7313446', + 'age_limit': 14, + 'duration': 2523, + 'thumbnail': r're:https?://.+/.+\.jpg', + 'series': 'How I Escaped My Cult', + 'season': 'Season 1', + 'season_number': 1, + 'episode': 'Episode 2', + 'episode_number': 2, + 'timestamp': 1740038400.0, + 'upload_date': '20250220', }, + 'params': {'skip_download': 'm3u8'}, }, { - 'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841', + 'url': 'https://www.nationalgeographic.com/tv/episode/ca694661-1186-41ae-8089-82f64d69b16d/playlist/PL554408064', 'info_dict': { - 'id': 'VDKA12782841', - 'title': 'First Look: Better Things - Season 2', - 'description': 'md5:fa73584a95761c605d9d54904e35b407', + 'id': 'VDKA39492078', 'ext': 'mp4', - 'age_limit': 14, - 'upload_date': '20170825', - 'duration': 161, - 'series': 'Better Things', - 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg', - 'timestamp': 1503661074, - }, - 'params': { - 'geo_bypass_ip_block': '3.244.239.0/24', - # m3u8 download - 'skip_download': True, + 'title': 'Heart of the Emperors', + 'description': 'md5:4fc50a2878f030bb3a7eac9124dca677', + 'age_limit': 0, + 'duration': 2775, + 'thumbnail': r're:https?://.+/.+\.jpg', + 'series': 'Secrets of the Penguins', + 'season': 'Season 1', + 'season_number': 1, + 'episode': 'Episode 1', + 'episode_number': 1, + 'timestamp': 1745204400.0, + 'upload_date': '20250421', }, + 'params': {'skip_download': 'm3u8'}, }, { - 'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding', + 'url': 'https://www.freeform.com/movies-and-specials/c38281fc-9f8f-47c7-8220-22394f9df2e1', 'only_matching': True, }, { - 'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland', - 'only_matching': True, - }, { - # brand 004 - 'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915', - 'only_matching': True, - }, { - # brand 008 - 'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013', - 'only_matching': True, - }, { - 'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013', - 'only_matching': True, - }, { - 'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner', + 'url': 'https://abc.com/video/219a454a-172c-41bf-878a-d169e6bc0bdc/playlist/PL5523098420', 'only_matching': True, }] @@ -171,58 +167,29 @@ class GoIE(AdobePassIE): f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json', display_id)['video'] + def _extract_global_var(self, name, webpage, video_id): + return self._search_json( + fr'window\[["\']{re.escape(name)}["\']\]\s*=', + webpage, f'{name.strip("_")} JSON', video_id) + def _real_extract(self, url): - mobj = self._match_valid_url(url) - sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.') - video_id, display_id = mobj.group('id', 'display_id') - site_info = self._SITE_INFO.get(sub_domain, {}) - brand = site_info.get('brand') - if not video_id or not site_info: - webpage = self._download_webpage(url, display_id or video_id) - data = self._parse_json( - self._search_regex( - r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage, - 'data', default='{}'), - display_id or video_id, fatal=False) - # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot - layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict) - video_id = None - if layout: - video_id = try_get( - layout, - (lambda x: x['videoid'], lambda x: x['video']['id']), - str) - if not video_id: - video_id = self._search_regex( - ( - # There may be inner quotes, e.g. data-video-id="'VDKA3609139'" - # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood - r'data-video-id=["\']*(VDKA\w+)', - # page.analytics.videoIdCode - r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)', - # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet - r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)', - ), webpage, 'video id', default=video_id) - if not site_info: - brand = self._search_regex( - (r'data-brand=\s*["\']\s*(\d+)', - r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand', - default='004') - site_info = next( - si for _, si in self._SITE_INFO.items() - if si.get('brand') == brand) - if not video_id: - # show extraction works for Disney, DisneyJunior and DisneyXD - # ABC and Freeform has different layout - show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id') - videos = self._extract_videos(brand, show_id=show_id) - show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False) - entries = [] - for video in videos: - entries.append(self.url_result( - video['url'], 'Go', video.get('id'), video.get('title'))) - entries.reverse() - return self.playlist_result(entries, show_id, show_title) + site, display_id = self._match_valid_url(url).group('site', 'id') + webpage = self._download_webpage(url, display_id) + config = self._extract_global_var('__CONFIG__', webpage, display_id) + data = self._extract_global_var(config['globalVar'], webpage, display_id) + video_id = traverse_obj(data, ( + 'page', 'content', 'video', 'layout', (('video', 'id'), 'videoid'), {str}, any)) + if not video_id: + video_id = self._search_regex([ + # data-track-video_id="VDKA39492078" + # data-track-video_id_code="vdka39492078" + # data-video-id="'VDKA3609139'" + r'data-(?:track-)?video[_-]id(?:_code)?=["\']*((?:vdka|VDKA)\d+)', + # page.analytics.videoIdCode + r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\d+)'], webpage, 'video ID') + + site_info = self._SITE_INFO[site] + brand = site_info['brand'] video_data = self._extract_videos(brand, video_id)[0] video_id = video_data['id'] title = video_data['title'] @@ -238,26 +205,31 @@ class GoIE(AdobePassIE): if ext == 'm3u8': video_type = video_data.get('type') data = { - 'video_id': video_data['id'], + 'video_id': video_id, 'video_type': video_type, 'brand': brand, 'device': '001', + 'app_name': 'webplayer-abc', } if video_data.get('accesslevel') == '1': - requestor_id = site_info.get('requestor_id', 'DisneyChannels') + provider_id = site_info['provider_id'] + software_statement = traverse_obj(data, ('app', 'config', ( + ('features', 'auth', 'softwareStatement'), + ('tvAuth', 'SOFTWARE_STATEMENTS', 'PRODUCTION'), + ), {str}, any)) or site_info['software_statement'] resource = site_info.get('resource_id') or self._get_mvpd_resource( - requestor_id, title, video_id, None) + provider_id, title, video_id, None) auth = self._extract_mvpd_auth( - url, video_id, requestor_id, resource) + url, video_id, site_info['requestor_id'], resource, software_statement) data.update({ 'token': auth, 'token_type': 'ap', - 'adobe_requestor_id': requestor_id, + 'adobe_requestor_id': provider_id, }) else: self._initialize_geo_bypass({'countries': ['US']}) entitlement = self._download_json( - 'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json', + 'https://prod.gatekeeper.us-abc.symphony.edgedatg.go.com/vp2/ws-secure/entitlement/2020/playmanifest_secure.json', video_id, data=urlencode_postdata(data)) errors = entitlement.get('errors', {}).get('errors', []) if errors: @@ -267,7 +239,7 @@ class GoIE(AdobePassIE): error['message'], countries=['US']) error_message = ', '.join([error['message'] for error in errors]) raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True) - asset_url += '?' + entitlement['uplynkData']['sessionKey'] + asset_url += '?' + entitlement['entitlement']['uplynkData']['sessionKey'] fmts, subs = self._extract_m3u8_formats_and_subtitles( asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False) formats.extend(fmts)