From a9c58ad945e88e8eadbbff9c165c19b46805063b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 13:19:58 +0200 Subject: [PATCH 01/13] Accept requested formats to be in the format 35/best (closes #1552) The format selection code is now an independent function. --- test/test_YoutubeDL.py | 23 +++++++++++++++++++++++ youtube_dl/YoutubeDL.py | 27 ++++++++++++++++++--------- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index ba6dc05bc..2073bc4df 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -94,6 +94,29 @@ def test_format_limit(self): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'format_id'], u'excellent') + def test_format_selection(self): + formats = [ + {u'format_id': u'35'}, + {u'format_id': u'47'}, + {u'format_id': u'2'}, + ] + info_dict = {u'formats': formats, u'extractor': u'test'} + + ydl = YDL({'format': u'20/47'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], u'47') + + ydl = YDL({'format': u'20/71/worst'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], u'35') + + ydl = YDL() + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], u'2') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 296c0f992..bc69214e7 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -448,6 +448,17 @@ def _fixup(r): else: raise Exception('Invalid result type: %s' % result_type) + def select_format(self, format_spec, available_formats): + if format_spec == 'best' or format_spec is None: + return available_formats[-1] + elif format_spec == 'worst': + return available_formats[0] + else: + matches = list(filter(lambda f:f['format_id'] == format_spec ,available_formats)) + if matches: + return matches[-1] + return None + def process_video_result(self, info_dict, download=True): assert info_dict.get('_type', 'video') == 'video' @@ -502,22 +513,20 @@ def _free_formats_key(f): formats = sorted(formats, key=_free_formats_key) req_format = self.params.get('format', 'best') + if req_format is None: + req_format = 'best' formats_to_download = [] - if req_format == 'best' or req_format is None: - formats_to_download = [formats[-1]] - elif req_format == 'worst': - formats_to_download = [formats[0]] # The -1 is for supporting YoutubeIE - elif req_format in ('-1', 'all'): + if req_format in ('-1', 'all'): formats_to_download = formats else: - # We can accept formats requestd in the format: 34/10/5, we pick + # We can accept formats requestd in the format: 34/5/best, we pick # the first that is available, starting from left req_formats = req_format.split('/') for rf in req_formats: - matches = filter(lambda f:f['format_id'] == rf ,formats) - if matches: - formats_to_download = [matches[0]] + selected_format = self.select_format(rf, formats) + if selected_format is not None: + formats_to_download = [selected_format] break if not formats_to_download: raise ExtractorError(u'requested format not available') From 49e86983e7639223644e1de2643745acc66f2535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 13:31:55 +0200 Subject: [PATCH 02/13] Allow to use the extension for the format selection The best format with the extension is downloaded. --- test/test_YoutubeDL.py | 17 ++++++++++++++--- youtube_dl/YoutubeDL.py | 7 ++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 2073bc4df..f8cd1bdce 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -96,9 +96,10 @@ def test_format_limit(self): def test_format_selection(self): formats = [ - {u'format_id': u'35'}, - {u'format_id': u'47'}, - {u'format_id': u'2'}, + {u'format_id': u'35', u'ext': u'mp4'}, + {u'format_id': u'45', u'ext': u'webm'}, + {u'format_id': u'47', u'ext': u'webm'}, + {u'format_id': u'2', u'ext': u'flv'}, ] info_dict = {u'formats': formats, u'extractor': u'test'} @@ -117,6 +118,16 @@ def test_format_selection(self): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'2') + ydl = YDL({'format': u'webm/mp4'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], u'47') + + ydl = YDL({'format': u'3gp/40/mp4'}) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], u'35') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bc69214e7..32f21e21a 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -454,7 +454,12 @@ def select_format(self, format_spec, available_formats): elif format_spec == 'worst': return available_formats[0] else: - matches = list(filter(lambda f:f['format_id'] == format_spec ,available_formats)) + extensions = [u'mp4', u'flv', u'webm', u'3gp'] + if format_spec in extensions: + filter_f = lambda f: f['ext'] == format_spec + else: + filter_f = lambda f: f['format_id'] == format_spec + matches = list(filter(filter_f ,available_formats)) if matches: return matches[-1] return None From 3fd39e37f2b767e4c66518a6fe0f620344c31825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 13:52:24 +0200 Subject: [PATCH 03/13] YoutubeDL: remove method that came from FileDownloader --- youtube_dl/YoutubeDL.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 32f21e21a..a837971b0 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -234,19 +234,6 @@ def report_error(self, message, tb=None): error_message = u'%s %s' % (_msg_header, message) self.trouble(error_message, tb) - def slow_down(self, start_time, byte_counter): - """Sleep if the download speed is over the rate limit.""" - rate_limit = self.params.get('ratelimit', None) - if rate_limit is None or byte_counter == 0: - return - now = time.time() - elapsed = now - start_time - if elapsed <= 0.0: - return - speed = float(byte_counter) / elapsed - if speed > rate_limit: - time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - def report_writedescription(self, descfn): """ Report that the description file is being written """ self.to_screen(u'[info] Writing video description to: ' + descfn) From 8c51aa6506c85f3826b55c35a1ebf470bd4ebe3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 14:09:38 +0200 Subject: [PATCH 04/13] The 'format' field now defaults to '{format_id} - {width}x{height}{format_note}' Following the YoutubeIE format. The 'format_note' gives additional info about the format, for example '3D' or 'DASH video'. --- youtube_dl/YoutubeDL.py | 42 ++++++++++++++++++++++------------ youtube_dl/extractor/common.py | 5 +++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a837971b0..5f70b6dac 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -473,17 +473,14 @@ def process_video_result(self, info_dict, download=True): # We check that all the formats have the format and format_id fields for (i, format) in enumerate(formats): - if format.get('format') is None: - if format.get('height') is not None: - if format.get('width') is not None: - format_desc = u'%sx%s' % (format['width'], format['height']) - else: - format_desc = u'%sp' % format['height'] - else: - format_desc = '???' - format['format'] = format_desc if format.get('format_id') is None: format['format_id'] = compat_str(i) + if format.get('format') is None: + format['format'] = u'{id} - {res}{note}'.format( + id=format['format_id'], + res=self.format_resolution(format), + note = u' ({})'.format(format['format_note']) if format.get('format_note') is not None else '', + ) if self.params.get('listformats', None): self.list_formats(info_dict) @@ -753,16 +750,31 @@ def record_download_archive(self, info_dict): with locked_file(fn, 'a', encoding='utf-8') as archive_file: archive_file.write(vid_id + u'\n') + @staticmethod + def format_resolution(format): + if format.get('height') is not None: + if format.get('width') is not None: + res = u'%sx%s' % (format['width'], format['height']) + else: + res = u'%sp' % format['height'] + else: + res = '???' + return res + def list_formats(self, info_dict): formats_s = [] for format in info_dict.get('formats', [info_dict]): - formats_s.append("%s\t:\t%s\t[%s]" % (format['format_id'], - format['ext'], - format.get('format', '???'), - ) - ) + formats_s.append(u'%-15s: %-5s %-15s[%s]' % ( + format['format_id'], + format['ext'], + format.get('format_note') or '-', + self.format_resolution(format), + ) + ) if len(formats_s) != 1: formats_s[0] += ' (worst)' formats_s[-1] += ' (best)' formats_s = "\n".join(formats_s) - self.to_screen(u"[info] Available formats for %s:\nformat code\textension\n%s" % (info_dict['id'], formats_s)) + self.to_screen(u'[info] Available formats for %s:\n' + u'format code extension note resolution\n%s' % ( + info_dict['id'], formats_s)) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index d4af3b5eb..7d7ce5d98 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -61,9 +61,12 @@ class InfoExtractor(object): * ext Will be calculated from url if missing * format A human-readable description of the format ("mp4 container with h264/opus"). - Calculated from width and height if missing. + Calculated from the format_id, width, height + and format_note fields if missing. * format_id A short description of the format ("mp4_h264_opus" or "19") + * format_note Additional info about the format + ("3D" or "DASH video") * width Width of the video, if known * height Height of the video, if known From 182a107877786516f612535a9ad8db5d7487e9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 14:40:06 +0200 Subject: [PATCH 05/13] [arte] Set the format_note and the format_id fields (closes #1628) --- youtube_dl/extractor/arte.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 5ee8a67b1..d8e32f187 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -174,12 +174,27 @@ def _match_lang(f): # Some formats use the m3u8 protocol formats = filter(lambda f: f.get('videoFormat') != 'M3U8', formats) # We order the formats by quality - formats = sorted(formats, key=lambda f: int(f.get('height',-1))) + formats = list(formats) # in python3 filter returns an iterator + if re.match(r'[A-Z]Q', formats[0]['quality']) is not None: + sort_key = lambda f: ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality']) + else: + sort_key = lambda f: int(f.get('height',-1)) + formats = sorted(formats, key=sort_key) # Prefer videos without subtitles in the same language formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None) # Pick the best quality def _format(format_info): + quality = format_info['quality'] + m_quality = re.match(r'\w*? - (\d*)p', quality) + if m_quality is not None: + quality = m_quality.group(1) + if format_info.get('versionCode') is not None: + format_id = u'%s-%s' % (quality, format_info['versionCode']) + else: + format_id = quality info = { + 'format_id': format_id, + 'format_note': format_info.get('versionLibelle'), 'width': format_info.get('width'), 'height': format_info.get('height'), } @@ -192,8 +207,6 @@ def _format(format_info): info['ext'] = determine_ext(info['url']) return info info_dict['formats'] = [_format(f) for f in formats] - # TODO: Remove when #980 has been merged - info_dict.update(info_dict['formats'][-1]) return info_dict From 685a9cd2f15d4e3b7d18df7ac639e7daf1a84483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 15:00:21 +0200 Subject: [PATCH 06/13] [googleplus] Fix upload_date extraction --- youtube_dl/extractor/googleplus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py index ab12d7e93..2570746b2 100644 --- a/youtube_dl/extractor/googleplus.py +++ b/youtube_dl/extractor/googleplus.py @@ -41,9 +41,9 @@ def _real_extract(self, url): # Extract update date upload_date = self._html_search_regex( - r'''(?x) + r'''(?x) ([0-9]{4}-[0-9]{2}-[0-9]{2})''', - webpage, u'upload date', fatal=False) + webpage, u'upload date', fatal=False, flags=re.VERBOSE) if upload_date: # Convert timestring to a format suitable for filename upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d") From cbbd9a9c6984ed42f55b13fbb97a3d7f98f77aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 15:07:33 +0200 Subject: [PATCH 07/13] Fix the duration field for the VideoDetective and InternetVideoArchive tests Also remove the use of the old format system and the comment --- youtube_dl/extractor/internetvideoarchive.py | 7 ++----- youtube_dl/extractor/videodetective.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py index 5986459d6..be8e05f53 100644 --- a/youtube_dl/extractor/internetvideoarchive.py +++ b/youtube_dl/extractor/internetvideoarchive.py @@ -19,7 +19,7 @@ class InternetVideoArchiveIE(InfoExtractor): u'info_dict': { u'title': u'SKYFALL', u'description': u'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.', - u'duration': 156, + u'duration': 153, }, } @@ -74,7 +74,7 @@ def _bp(p): }) formats = sorted(formats, key=lambda f: f['bitrate']) - info = { + return { 'id': video_id, 'title': item.find('title').text, 'formats': formats, @@ -82,6 +82,3 @@ def _bp(p): 'description': item.find('description').text, 'duration': int(attr['duration']), } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dl/extractor/videodetective.py index d89f84094..265dd5b91 100644 --- a/youtube_dl/extractor/videodetective.py +++ b/youtube_dl/extractor/videodetective.py @@ -16,7 +16,7 @@ class VideoDetectiveIE(InfoExtractor): u'info_dict': { u'title': u'KICK-ASS 2', u'description': u'md5:65ba37ad619165afac7d432eaded6013', - u'duration': 138, + u'duration': 135, }, } From f6a54188c230d76a659595640850b093f1d06b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 21 Oct 2013 16:28:55 +0200 Subject: [PATCH 08/13] [youtube] Use 'node is None' when checking if the video has automatic captions It had stopped working and it reports a FutureWarning --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index fb7c42830..a88cba2b4 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1150,7 +1150,7 @@ def _get_available_automatic_caption(self, video_id, webpage): list_page = self._download_webpage(list_url, video_id) caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8')) original_lang_node = caption_list.find('track') - if not original_lang_node or original_lang_node.attrib.get('kind') != 'asr' : + if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' : self._downloader.report_warning(u'Video doesn\'t have automatic captions') return {} original_lang = original_lang_node.attrib['lang_code'] From a6387bfd3cc65d8780bb4d6272b51c6268b45988 Mon Sep 17 00:00:00 2001 From: Joshua Elsasser Date: Fri, 5 Jul 2013 09:10:57 -0700 Subject: [PATCH 09/13] [vimeo] Implement the new format selection system (closes PR #996) Rebased and deleted some parts to use the new system instead of copying the one from YoutubeIE --- youtube_dl/extractor/vimeo.py | 49 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 2de56ac81..1125513c7 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -179,46 +179,45 @@ def _real_extract(self, url, new_video=True): # Vimeo specific: extract video codec and quality information # First consider quality, then codecs, then take everything - # TODO bind to format param - codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')] + codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')] files = { 'hd': [], 'sd': [], 'other': []} config_files = config["video"].get("files") or config["request"].get("files") for codec_name, codec_extension in codecs: - if codec_name in config_files: - if 'hd' in config_files[codec_name]: - files['hd'].append((codec_name, codec_extension, 'hd')) - elif 'sd' in config_files[codec_name]: - files['sd'].append((codec_name, codec_extension, 'sd')) + for quality in config_files.get(codec_name, []): + format_id = '-'.join((codec_name, quality)).lower() + key = quality if quality in files else 'other' + video_url = None + if isinstance(config_files[codec_name], dict): + file_info = config_files[codec_name][quality] + video_url = file_info.get('url') else: - files['other'].append((codec_name, codec_extension, config_files[codec_name][0])) + file_info = {} + if video_url is None: + video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \ + %(video_id, sig, timestamp, quality, codec_name.upper()) - for quality in ('hd', 'sd', 'other'): - if len(files[quality]) > 0: - video_quality = files[quality][0][2] - video_codec = files[quality][0][0] - video_extension = files[quality][0][1] - self.to_screen(u'%s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality)) - break - else: + files[key].append({ + 'ext': codec_extension, + 'url': video_url, + 'format_id': format_id, + 'width': file_info.get('width'), + 'height': file_info.get('height'), + }) + formats = [] + for key in ('other', 'sd', 'hd'): + formats += files[key] + if len(formats) == 0: raise ExtractorError(u'No known codec found') - video_url = None - if isinstance(config_files[video_codec], dict): - video_url = config_files[video_codec][video_quality].get("url") - if video_url is None: - video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \ - %(video_id, sig, timestamp, video_quality, video_codec.upper()) - return [{ 'id': video_id, - 'url': video_url, 'uploader': video_uploader, 'uploader_id': video_uploader_id, 'upload_date': video_upload_date, 'title': video_title, - 'ext': video_extension, 'thumbnail': video_thumbnail, 'description': video_description, + 'formats': formats, }] From 12893efe01b88a72595f50cbc692d308419001c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 22 Oct 2013 00:01:59 +0200 Subject: [PATCH 10/13] Respect the download parameter in YoutubeDL.process_video_result if the extractor handle the format selection --- youtube_dl/YoutubeDL.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 5f70b6dac..577e27b11 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -461,7 +461,8 @@ def process_video_result(self, info_dict, download=True): # This extractors handle format selection themselves if info_dict['extractor'] in [u'youtube', u'Youku', u'YouPorn', u'mixcloud']: - self.process_info(info_dict) + if download: + self.process_info(info_dict) return info_dict # We now pick which formats have to be downloaded From fe7e0c982530e62f3af001f7ac50c8c4b4abb5cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 22 Oct 2013 14:49:34 +0200 Subject: [PATCH 11/13] Style fixes in YoutubeDL.py Fixed some of the problems reported by pep8 --- youtube_dl/YoutubeDL.py | 58 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 577e27b11..f9a6da520 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -91,7 +91,7 @@ class YoutubeDL(object): downloadarchive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. - + The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, @@ -216,10 +216,10 @@ def report_warning(self, message): If stderr is a tty file the 'WARNING:' will be colored ''' if sys.stderr.isatty() and os.name != 'nt': - _msg_header=u'\033[0;33mWARNING:\033[0m' + _msg_header = u'\033[0;33mWARNING:\033[0m' else: - _msg_header=u'WARNING:' - warning_message=u'%s %s' % (_msg_header,message) + _msg_header = u'WARNING:' + warning_message = u'%s %s' % (_msg_header, message) self.to_stderr(warning_message) def report_error(self, message, tb=None): @@ -275,11 +275,11 @@ def prepare_filename(self, info_dict): if template_dict['playlist_index'] is not None: template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] - sanitize = lambda k,v: sanitize_filename( + sanitize = lambda k, v: sanitize_filename( u'NA' if v is None else compat_str(v), restricted=self.params.get('restrictfilenames'), - is_id=(k==u'id')) - template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items()) + is_id=k == u'id') + template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items()) filename = self.params['outtmpl'] % template_dict return filename @@ -315,14 +315,14 @@ def _match_entry(self, info_dict): return (u'%(title)s has already been recorded in archive' % info_dict) return None - + def extract_info(self, url, download=True, ie_key=None, extra_info={}): ''' Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. extra_info is a dict containing the extra values to add to each result ''' - + if ie_key: ies = [self.get_info_extractor(ie_key)] else: @@ -364,7 +364,7 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}): raise else: self.report_error(u'no suitable InfoExtractor: %s' % url) - + def process_ie_result(self, ie_result, download=True, extra_info={}): """ Take the result of the ie(may be modified) and resolve all unresolved @@ -388,7 +388,7 @@ def process_ie_result(self, ie_result, download=True, extra_info={}): elif result_type == 'playlist': # We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) - self.to_screen(u'[download] Downloading playlist: %s' % playlist) + self.to_screen(u'[download] Downloading playlist: %s' % playlist) playlist_results = [] @@ -406,12 +406,12 @@ def process_ie_result(self, ie_result, download=True, extra_info={}): self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % (ie_result['extractor'], playlist, n_all_entries, n_entries)) - for i,entry in enumerate(entries,1): - self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) + for i, entry in enumerate(entries, 1): + self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries)) extra = { - 'playlist': playlist, - 'playlist_index': i + playliststart, - } + 'playlist': playlist, + 'playlist_index': i + playliststart, + } if not 'extractor' in entry: # We set the extractor, if it's an url it will be set then to # the new extractor, but if it's already a video we must make @@ -446,7 +446,7 @@ def select_format(self, format_spec, available_formats): filter_f = lambda f: f['ext'] == format_spec else: filter_f = lambda f: f['format_id'] == format_spec - matches = list(filter(filter_f ,available_formats)) + matches = list(filter(filter_f, available_formats)) if matches: return matches[-1] return None @@ -480,7 +480,7 @@ def process_video_result(self, info_dict, download=True): format['format'] = u'{id} - {res}{note}'.format( id=format['format_id'], res=self.format_resolution(format), - note = u' ({})'.format(format['format_note']) if format.get('format_note') is not None else '', + note=u' ({})'.format(format['format_note']) if format.get('format_note') is not None else '', ) if self.params.get('listformats', None): @@ -607,20 +607,20 @@ def process_info(self, info_dict): if self.params.get('writeannotations', False): try: - annofn = filename + u'.annotations.xml' - self.report_writeannotations(annofn) - with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: - annofile.write(info_dict['annotations']) + annofn = filename + u'.annotations.xml' + self.report_writeannotations(annofn) + with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: + annofile.write(info_dict['annotations']) except (KeyError, TypeError): self.report_warning(u'There are no annotations to write.') except (OSError, IOError): - self.report_error(u'Cannot write annotations file: ' + annofn) - return + self.report_error(u'Cannot write annotations file: ' + annofn) + return subtitles_are_requested = any([self.params.get('writesubtitles', False), self.params.get('writeautomaticsub')]) - if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']: + if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']: # subtitles download errors are already managed as troubles in relevant IE # that way it will silently go on when used with unsupporting IE subtitles = info_dict['subtitles'] @@ -642,7 +642,7 @@ def process_info(self, info_dict): infofn = filename + u'.info.json' self.report_writeinfojson(infofn) try: - json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle']) + json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) write_json_file(json_info_dict, encodeFilename(infofn)) except (OSError, IOError): self.report_error(u'Cannot write metadata to JSON file ' + infofn) @@ -712,7 +712,7 @@ def post_process(self, filename, ie_info): keep_video = None for pp in self._pps: try: - keep_video_wish,new_info = pp.run(info) + keep_video_wish, new_info = pp.run(info) if keep_video_wish is not None: if keep_video_wish: keep_video = keep_video_wish @@ -759,7 +759,7 @@ def format_resolution(format): else: res = u'%sp' % format['height'] else: - res = '???' + res = '???' return res def list_formats(self, info_dict): @@ -773,7 +773,7 @@ def list_formats(self, info_dict): ) ) if len(formats_s) != 1: - formats_s[0] += ' (worst)' + formats_s[0] += ' (worst)' formats_s[-1] += ' (best)' formats_s = "\n".join(formats_s) self.to_screen(u'[info] Available formats for %s:\n' From ce68b5907ca3999fb3e3f0d9bcfbaa7823c46623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 22 Oct 2013 21:01:16 +0200 Subject: [PATCH 12/13] [nhl:videocenter] Fix playlist title extraction --- youtube_dl/extractor/nhl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index e8d43dd13..224f56ac8 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -90,8 +90,8 @@ def _real_extract(self, url): r'{statusIndex:0,index:0,.*?id:(.*?),'], webpage, u'category id') playlist_title = self._html_search_regex( - r'\?catid=%s">(.*?)' % cat_id, - webpage, u'playlist title', flags=re.DOTALL) + r'tab0"[^>]*?>(.*?)', + webpage, u'playlist title', flags=re.DOTALL).lower().capitalize() data = compat_urllib_parse.urlencode({ 'cid': cat_id, From b028e96144a2bf1ba84dd6d11f1cc13a2928c438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 22 Oct 2013 21:06:06 +0200 Subject: [PATCH 13/13] [arte.tv:creative] Update the title of the test --- youtube_dl/extractor/arte.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index d8e32f187..d39b48951 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -220,7 +220,7 @@ class ArteTVCreativeIE(ArteTVPlus7IE): u'url': u'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design', u'file': u'050489-002.mp4', u'info_dict': { - u'title': u'Agentur Amateur #2 - Corporate Design', + u'title': u'Agentur Amateur / Agence Amateur #2 : Corporate Design', }, }