From 30d569d2acd597b8f09c843ba92471b9b453fb65 Mon Sep 17 00:00:00 2001 From: zenerdi0de <83358565+zenerdi0de@users.noreply.github.com> Date: Wed, 7 Jul 2021 04:02:56 +0530 Subject: [PATCH] [fancode] Fix extraction, support live and allow login with refresh token (#471) Authored-by: zenerdi0de --- yt_dlp/extractor/extractors.py | 6 +- yt_dlp/extractor/fancode.py | 128 ++++++++++++++++++++++++++++----- 2 files changed, 117 insertions(+), 17 deletions(-) diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index 5df1a4301..694be9e57 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -398,7 +398,11 @@ FacebookIE, FacebookPluginsVideoIE, ) -from .fancode import FancodeVodIE +from .fancode import ( + FancodeVodIE, + FancodeLiveIE +) + from .faz import FazIE from .fc2 import ( FC2IE, diff --git a/yt_dlp/extractor/fancode.py b/yt_dlp/extractor/fancode.py index 063cc0be7..fd84a6e50 100644 --- a/yt_dlp/extractor/fancode.py +++ b/yt_dlp/extractor/fancode.py @@ -7,7 +7,8 @@ from ..utils import ( parse_iso8601, ExtractorError, - try_get + try_get, + mimetype2ext ) @@ -38,16 +39,63 @@ class FancodeVodIE(InfoExtractor): 'only_matching': True, }] + _ACCESS_TOKEN = None + _NETRC_MACHINE = 'fancode' + + _LOGIN_HINT = 'Use "--user refresh --password " to login using a refresh token' + + headers = { + 'content-type': 'application/json', + 'origin': 'https://fancode.com', + 'referer': 'https://fancode.com', + } + + def _login(self): + # Access tokens are shortlived, so get them using the refresh token. + username, password = self._get_login_info() + if username == 'refresh' and password is not None: + self.report_login() + data = '''{ + "query":"mutation RefreshToken($refreshToken: String\\u0021) { refreshToken(refreshToken: $refreshToken) { accessToken }}", + "variables":{ + "refreshToken":"%s" + }, + "operationName":"RefreshToken" + }''' % password + + token_json = self.download_gql('refresh token', data, "Getting the Access token") + self._ACCESS_TOKEN = try_get(token_json, lambda x: x['data']['refreshToken']['accessToken']) + if self._ACCESS_TOKEN is None: + self.report_warning('Failed to get Access token') + else: + self.headers.update({'Authorization': 'Bearer %s' % self._ACCESS_TOKEN}) + elif username is not None: + self.report_warning(f'Login using username and password is not currently supported. {self._LOGIN_HINT}') + + def _real_initialize(self): + self._login() + + def _check_login_required(self, is_available, is_premium): + msg = None + if is_premium and self._ACCESS_TOKEN is None: + msg = f'This video is only available for registered users. {self._LOGIN_HINT}' + elif not is_available and self._ACCESS_TOKEN is not None: + msg = 'This video isn\'t available to the current logged in account' + if msg: + self.raise_login_required(msg, metadata_available=True, method=None) + + def download_gql(self, variable, data, note, fatal=False, headers=headers): + return self._download_json( + 'https://www.fancode.com/graphql', variable, + data=data.encode(), note=note, + headers=headers, fatal=fatal) + def _real_extract(self, url): BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/%s/default_default/index.html?videoId=%s' - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - brightcove_user_id = self._html_search_regex( - r'(?:https?://)?players\.brightcove\.net/(\d+)/default_default/index(?:\.min)?\.js', - webpage, 'user id') + brightcove_user_id = '6008340455001' data = '''{ "query":"query Video($id: Int\\u0021, $filter: SegmentFilter) { media(id: $id, filter: $filter) { id contentId title contentId publishedTime totalViews totalUpvotes provider thumbnail { src } mediaSource {brightcove } duration isPremium isUserEntitled tags duration }}", "variables":{ @@ -57,15 +105,9 @@ def _real_extract(self, url): } }, "operationName":"Video" - }''' % video_id + }''' % video_id - metadata_json = self._download_json( - 'https://www.fancode.com/graphql', video_id, data=data.encode(), note='Downloading metadata', - headers={ - 'content-type': 'application/json', - 'origin': 'https://fancode.com', - 'referer': url, - }) + metadata_json = self.download_gql(video_id, data, note='Downloading metadata') media = try_get(metadata_json, lambda x: x['data']['media'], dict) or {} brightcove_video_id = try_get(media, lambda x: x['mediaSource']['brightcove'], compat_str) @@ -74,8 +116,8 @@ def _real_extract(self, url): raise ExtractorError('Unable to extract brightcove Video ID') is_premium = media.get('isPremium') - if is_premium: - self.report_warning('this video requires a premium account', video_id) + + self._check_login_required(media.get('isUserEntitled'), is_premium) return { '_type': 'url_transparent', @@ -89,3 +131,57 @@ def _real_extract(self, url): 'release_timestamp': parse_iso8601(media.get('publishedTime')), 'availability': self._availability(needs_premium=is_premium), } + + +class FancodeLiveIE(FancodeVodIE): + IE_NAME = 'fancode:live' + + _VALID_URL = r'https?://(www\.)?fancode\.com/match/(?P[0-9]+).+' + + _TESTS = [{ + 'url': 'https://fancode.com/match/35328/cricket-fancode-ecs-hungary-2021-bub-vs-blb?slug=commentary', + 'info_dict': { + 'id': '35328', + 'ext': 'mp4', + 'title': 'BUB vs BLB', + "timestamp": 1624863600, + 'is_live': True, + 'upload_date': '20210628', + }, + 'skip': 'Ended' + }, { + 'url': 'https://fancode.com/match/35328/', + 'only_matching': True, + }, { + 'url': 'https://fancode.com/match/35567?slug=scorecard', + 'only_matching': True, + }] + + def _real_extract(self, url): + + id = self._match_id(url) + data = '''{ + "query":"query MatchResponse($id: Int\\u0021, $isLoggedIn: Boolean\\u0021) { match: matchWithScores(id: $id) { id matchDesc mediaId videoStreamId videoStreamUrl { ...VideoSource } liveStreams { videoStreamId videoStreamUrl { ...VideoSource } contentId } name startTime streamingStatus isPremium isUserEntitled @include(if: $isLoggedIn) status metaTags bgImage { src } sport { name slug } tour { id name } squads { name shortName } liveStreams { contentId } mediaId }}fragment VideoSource on VideoSource { title description posterUrl url deliveryType playerType}", + "variables":{ + "id":%s, + "isLoggedIn":true + }, + "operationName":"MatchResponse" + }''' % id + + info_json = self.download_gql(id, data, "Info json") + + match_info = try_get(info_json, lambda x: x['data']['match']) + + if match_info.get('status') != "LIVE": + raise ExtractorError('The stream can\'t be accessed', expected=True) + self._check_login_required(match_info.get('isUserEntitled'), True) # all live streams are premium only + + return { + 'id': id, + 'title': match_info.get('name'), + 'formats': self._extract_akamai_formats(try_get(match_info, lambda x: x['videoStreamUrl']['url']), id), + 'ext': mimetype2ext(try_get(match_info, lambda x: x['videoStreamUrl']['deliveryType'])), + 'is_live': True, + 'release_timestamp': parse_iso8601(match_info.get('startTime')) + }