From c2603313b1dd3ce9a90c5ea4a3e5ade71a1d23a2 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sun, 27 Jun 2021 02:22:32 +0300 Subject: [PATCH] [youtube_live_chat] use `clickTrackingParams` (#449) Authored by: siikamiika --- yt_dlp/downloader/youtube_live_chat.py | 47 ++++++++++++++++---------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/yt_dlp/downloader/youtube_live_chat.py b/yt_dlp/downloader/youtube_live_chat.py index 5303efd0d..35e88e367 100644 --- a/yt_dlp/downloader/youtube_live_chat.py +++ b/yt_dlp/downloader/youtube_live_chat.py @@ -44,7 +44,7 @@ def dl_fragment(url, data=None, headers=None): return self._download_fragment(ctx, url, info_dict, http_headers, data) def parse_actions_replay(live_chat_continuation): - offset = continuation_id = None + offset = continuation_id = click_tracking_params = None processed_fragment = bytearray() for action in live_chat_continuation.get('actions', []): if 'replayChatItemAction' in action: @@ -53,28 +53,34 @@ def parse_actions_replay(live_chat_continuation): processed_fragment.extend( json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n') if offset is not None: - continuation_id = try_get( + continuation = try_get( live_chat_continuation, - lambda x: x['continuations'][0]['liveChatReplayContinuationData']['continuation']) + lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict) + if continuation: + continuation_id = continuation.get('continuation') + click_tracking_params = continuation.get('clickTrackingParams') self._append_fragment(ctx, processed_fragment) - return continuation_id, offset + return continuation_id, offset, click_tracking_params def try_refresh_replay_beginning(live_chat_continuation): # choose the second option that contains the unfiltered live chat replay - refresh_continuation_id = try_get( + refresh_continuation = try_get( live_chat_continuation, - lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData']['continuation'], str) - if refresh_continuation_id: + lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict) + if refresh_continuation: # no data yet but required to call _append_fragment self._append_fragment(ctx, b'') - return refresh_continuation_id, 0 + refresh_continuation_id = refresh_continuation.get('continuation') + offset = 0 + click_tracking_params = refresh_continuation.get('trackingParams') + return refresh_continuation_id, offset, click_tracking_params return parse_actions_replay(live_chat_continuation) live_offset = 0 def parse_actions_live(live_chat_continuation): nonlocal live_offset - continuation_id = None + continuation_id = click_tracking_params = None processed_fragment = bytearray() for action in live_chat_continuation.get('actions', []): timestamp = self.parse_live_timestamp(action) @@ -95,11 +101,12 @@ def parse_actions_live(live_chat_continuation): continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict) if continuation_data: continuation_id = continuation_data.get('continuation') + click_tracking_params = continuation_data.get('clickTrackingParams') timeout_ms = int_or_none(continuation_data.get('timeoutMs')) if timeout_ms is not None: time.sleep(timeout_ms / 1000) self._append_fragment(ctx, processed_fragment) - return continuation_id, live_offset + return continuation_id, live_offset, click_tracking_params def download_and_parse_fragment(url, frag_index, request_data=None, headers=None): count = 0 @@ -107,7 +114,7 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None try: success, raw_fragment = dl_fragment(url, request_data, headers) if not success: - return False, None, None + return False, None, None, None try: data = ie._extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace')) except RegexNotFoundError: @@ -119,19 +126,19 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None lambda x: x['continuationContents']['liveChatContinuation'], dict) or {} if info_dict['protocol'] == 'youtube_live_chat_replay': if frag_index == 1: - continuation_id, offset = try_refresh_replay_beginning(live_chat_continuation) + continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation) else: - continuation_id, offset = parse_actions_replay(live_chat_continuation) + continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation) elif info_dict['protocol'] == 'youtube_live_chat': - continuation_id, offset = parse_actions_live(live_chat_continuation) - return True, continuation_id, offset + continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation) + return True, continuation_id, offset, click_tracking_params except compat_urllib_error.HTTPError as err: count += 1 if count <= fragment_retries: self.report_retry_fragment(err, frag_index, count, fragment_retries) if count > fragment_retries: self.report_error('giving up after %s fragment retries' % fragment_retries) - return False, None, None + return False, None, None, None self._prepare_and_start_frag_download(ctx) @@ -165,6 +172,7 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id frag_index = offset = 0 + click_tracking_params = None while continuation_id is not None: frag_index += 1 request_data = { @@ -173,13 +181,16 @@ def download_and_parse_fragment(url, frag_index, request_data=None, headers=None } if frag_index > 1: request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))} + if click_tracking_params: + request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params} headers = ie._generate_api_headers(ytcfg, visitor_data=visitor_data) headers.update({'content-type': 'application/json'}) fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n' - success, continuation_id, offset = download_and_parse_fragment( + success, continuation_id, offset, click_tracking_params = download_and_parse_fragment( url, frag_index, fragment_request_data, headers) else: - success, continuation_id, offset = download_and_parse_fragment(chat_page_url, frag_index) + success, continuation_id, offset, click_tracking_params = download_and_parse_fragment( + chat_page_url, frag_index) if not success: return False if test: