From c067545c175bb421fcfc255ae7df7a2fff1c5638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= Date: Fri, 2 Jan 2015 17:12:20 +0100 Subject: [PATCH 1/5] ceskatelevize: Closed captions support --- youtube_dl/extractor/ceskatelevize.py | 55 ++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py index ba8376338..345ac5e85 100644 --- a/youtube_dl/extractor/ceskatelevize.py +++ b/youtube_dl/extractor/ceskatelevize.py @@ -3,7 +3,7 @@ import re -from .common import InfoExtractor +from .subtitles import SubtitlesInfoExtractor from ..compat import ( compat_urllib_request, compat_urllib_parse, @@ -12,10 +12,11 @@ from ..utils import ( ExtractorError, float_or_none, + HEADRequest, ) -class CeskaTelevizeIE(InfoExtractor): +class CeskaTelevizeIE(SubtitlesInfoExtractor): _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P[^?#]+)' _TESTS = [ @@ -104,6 +105,15 @@ def _real_extract(self, url): duration = float_or_none(item.get('duration')) thumbnail = item.get('previewImageUrl') + # subtitles + subtitles = self.extract_subtitles(video_id, webpage) + + if self._downloader.params.get('listsubtitles', False): + self._list_available_subtitles(video_id, webpage) + return + + subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage)) + return { 'id': episode_id, 'title': title, @@ -111,4 +121,45 @@ def _real_extract(self, url): 'thumbnail': thumbnail, 'duration': duration, 'formats': formats, + 'subtitles': subtitles, } + + def _fix_subtitles(self, subtitles): + """ Convert milisecond-based subtitles to SRT """ + if subtitles is None: + return subtitles # subtitles not requested + + def _msectotimecode(msec): + """ Helper utility to convert miliseconds to timecode """ + components = [] + for divider in [1000, 60, 60, 100]: + components.append(msec % divider) + msec //= divider + return "{3:02}:{2:02}:{1:02},{0:03}".format(*components) + + def _fix_subtitle(subtitle): + for line in subtitle.splitlines(): + m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line) + if m: + yield m.group(1) + start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:]) + yield "{} --> {}".format(start, stop) + else: + yield line + + fixed_subtitles = {} + for k, v in subtitles.items(): + fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v)) + return fixed_subtitles + + def _get_available_subtitles(self, video_id, webpage): + video_id = video_id.partition('-')[0] + url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \ + 'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id) + req = HEADRequest(url) + sub = self._request_webpage( + req, video_id, + note="Checking subtitles", + errnote="No subtitles found", + fatal=False) + return {'cs': url} if sub else {} From 27a82a1b938f149b8c718e41ae57fcba01af299e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 7 Jan 2015 05:03:14 +0600 Subject: [PATCH 2/5] [ceskatelevize] Simplify --- youtube_dl/extractor/ceskatelevize.py | 32 +++++++++------------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py index 345ac5e85..0a9bd01c9 100644 --- a/youtube_dl/extractor/ceskatelevize.py +++ b/youtube_dl/extractor/ceskatelevize.py @@ -12,7 +12,6 @@ from ..utils import ( ExtractorError, float_or_none, - HEADRequest, ) @@ -105,14 +104,16 @@ def _real_extract(self, url): duration = float_or_none(item.get('duration')) thumbnail = item.get('previewImageUrl') - # subtitles - subtitles = self.extract_subtitles(video_id, webpage) + subtitles = {} + subs = item.get('subtitles') + if subs: + subtitles['cs'] = subs[0]['url'] if self._downloader.params.get('listsubtitles', False): - self._list_available_subtitles(video_id, webpage) + self._list_available_subtitles(video_id, subtitles) return - subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage)) + subtitles = self._fix_subtitles(self.extract_subtitles(video_id, subtitles)) return { 'id': episode_id, @@ -124,13 +125,14 @@ def _real_extract(self, url): 'subtitles': subtitles, } - def _fix_subtitles(self, subtitles): - """ Convert milisecond-based subtitles to SRT """ + @staticmethod + def _fix_subtitles(subtitles): + """ Convert millisecond-based subtitles to SRT """ if subtitles is None: return subtitles # subtitles not requested def _msectotimecode(msec): - """ Helper utility to convert miliseconds to timecode """ + """ Helper utility to convert milliseconds to timecode """ components = [] for divider in [1000, 60, 60, 100]: components.append(msec % divider) @@ -139,7 +141,7 @@ def _msectotimecode(msec): def _fix_subtitle(subtitle): for line in subtitle.splitlines(): - m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line) + m = re.match(r"^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$", line) if m: yield m.group(1) start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:]) @@ -151,15 +153,3 @@ def _fix_subtitle(subtitle): for k, v in subtitles.items(): fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v)) return fixed_subtitles - - def _get_available_subtitles(self, video_id, webpage): - video_id = video_id.partition('-')[0] - url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \ - 'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id) - req = HEADRequest(url) - sub = self._request_webpage( - req, video_id, - note="Checking subtitles", - errnote="No subtitles found", - fatal=False) - return {'cs': url} if sub else {} From 6309cb9b412a6e9914bd873201ad0fc38ac2659d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 7 Jan 2015 05:03:34 +0600 Subject: [PATCH 3/5] [ceskatelevize] Fix python 2.6 format issue --- youtube_dl/extractor/ceskatelevize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py index 0a9bd01c9..f70e090bb 100644 --- a/youtube_dl/extractor/ceskatelevize.py +++ b/youtube_dl/extractor/ceskatelevize.py @@ -145,7 +145,7 @@ def _fix_subtitle(subtitle): if m: yield m.group(1) start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:]) - yield "{} --> {}".format(start, stop) + yield "{0} --> {1}".format(start, stop) else: yield line From 0b54a5b10ab9da3257130b0061fb46c84b4d64ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 7 Jan 2015 05:04:15 +0600 Subject: [PATCH 4/5] [ceskatelevize] Add subtitles tests --- test/test_subtitles.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index d34565191..6336dd317 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -17,6 +17,7 @@ TEDIE, VimeoIE, WallaIE, + CeskaTelevizeIE, ) @@ -317,5 +318,32 @@ def test_nosubtitles(self): self.assertEqual(len(subtitles), 0) +class TestCeskaTelevizeSubtitles(BaseTestSubtitles): + url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky' + IE = CeskaTelevizeIE + + def test_list_subtitles(self): + self.DL.expect_warning('Automatic Captions not supported by this server') + self.DL.params['listsubtitles'] = True + info_dict = self.getInfoDict() + self.assertEqual(info_dict, None) + + def test_allsubtitles(self): + self.DL.expect_warning('Automatic Captions not supported by this server') + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(set(subtitles.keys()), set(['cs'])) + self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4') + + def test_nosubtitles(self): + self.DL.expect_warning('video doesn\'t have subtitles') + self.url = 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220' + self.DL.params['writesubtitles'] = True + self.DL.params['allsubtitles'] = True + subtitles = self.getSubtitles() + self.assertEqual(len(subtitles), 0) + + if __name__ == '__main__': unittest.main() From d6a31b17661b6d1a0d3fd987ef9570011e458e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 7 Jan 2015 05:05:18 +0600 Subject: [PATCH 5/5] Credit @oskar456 for ceskatelevize subtitles support (#4622) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index a63c97ae0..8f2010803 100644 --- a/AUTHORS +++ b/AUTHORS @@ -100,3 +100,4 @@ Cédric Luthi Thijs Vermeir Joel Leclerc Christopher Krooss +Ondřej Caletka