.*?
(.+?)
',
+ webpage, u'uploader', fatal=False, flags=re.DOTALL)
return [{
'id': video_id,
@@ -4390,18 +4276,13 @@ def _real_extract(self, url):
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
webpage = self._download_webpage(webpage_url, video_id)
- mobj = re.search(r"photo_secret: '(\w+)'", webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract video secret')
- secret = mobj.group(1)
+ secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, u'secret')
first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')
- mobj = re.search(r'
- (\d+-\d+)
', first_xml)
- if mobj is None:
- raise ExtractorError(u'Unable to extract node_id')
- node_id = mobj.group(1)
+ node_id = self._html_search_regex(r'
- (\d+-\d+)
',
+ first_xml, u'node_id')
second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage')
@@ -4413,22 +4294,14 @@ def _real_extract(self, url):
raise ExtractorError(u'Unable to extract video url')
video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
- mobj = re.search(r'
(.*?)', data)
- if mobj is None:
- raise ExtractorError(u'Unable to extract video url')
- video_url = mobj.group(1)
+
+ video_url = self._html_search_regex(r'
(.*?)',
+ data, u'video URL')
return [{
'id': video_id,
@@ -4483,9 +4349,9 @@ def _real_extract(self, url):
'ext': 'mp4',
'title': video_title,
'thumbnail': thumbnail,
- 'description': description,
+ 'description': video_description,
}]
-
+
class XHamsterIE(InfoExtractor):
"""Information Extractor for xHamster"""
_VALID_URL = r'(?:http://)?(?:www.)?xhamster\.com/movies/(?P
[0-9]+)/.*\.html'
@@ -4494,8 +4360,9 @@ def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- mrss_url='http://xhamster.com/movies/%s/.html' % video_id
+ mrss_url = 'http://xhamster.com/movies/%s/.html' % video_id
webpage = self._download_webpage(mrss_url, video_id)
+
mobj = re.search(r'\'srv\': \'(?P[^\']*)\',\s*\'file\': \'(?P[^\']+)\',', webpage)
if mobj is None:
raise ExtractorError(u'Unable to extract media URL')
@@ -4505,39 +4372,33 @@ def _real_extract(self,url):
video_url = mobj.group('server')+'/key='+mobj.group('file')
video_extension = video_url.split('.')[-1]
- mobj = re.search(r'(?P<title>.+?) - xHamster\.com', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract title')
- video_title = unescapeHTML(mobj.group('title'))
+ video_title = self._html_search_regex(r'(?P<title>.+?) - xHamster\.com',
+ webpage, u'title')
- mobj = re.search(r'Description: (?P[^<]+)', webpage)
- if mobj is None:
- video_description = u''
- else:
- video_description = unescapeHTML(mobj.group('description'))
+ # Can't see the description anywhere in the UI
+ # video_description = self._html_search_regex(r'Description: (?P[^<]+)',
+ # webpage, u'description', fatal=False)
+ # if video_description: video_description = unescapeHTML(video_description)
mobj = re.search(r'hint=\'(?P[0-9]{4})-(?P[0-9]{2})-(?P[0-9]{2}) [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{3,4}\'', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract upload date')
- video_upload_date = mobj.group('upload_date_Y')+mobj.group('upload_date_m')+mobj.group('upload_date_d')
-
- mobj = re.search(r']+>(?P[^>]+)', webpage)
- if mobj is None:
- video_uploader_id = u'anonymous'
+ if mobj:
+ video_upload_date = mobj.group('upload_date_Y')+mobj.group('upload_date_m')+mobj.group('upload_date_d')
else:
- video_uploader_id = mobj.group('uploader_id')
+ video_upload_date = None
+ self._downloader.report_warning(u'Unable to extract upload date')
- mobj = re.search(r'\'image\':\'(?P[^\']+)\'', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract thumbnail URL')
- video_thumbnail = mobj.group('thumbnail')
+ video_uploader_id = self._html_search_regex(r']+>(?P[^<]+)',
+ webpage, u'uploader id', default=u'anonymous')
+
+ video_thumbnail = self._search_regex(r'\'image\':\'(?P[^\']+)\'',
+ webpage, u'thumbnail', fatal=False)
return [{
'id': video_id,
'url': video_url,
'ext': video_extension,
'title': video_title,
- 'description': video_description,
+ # 'description': video_description,
'upload_date': video_upload_date,
'uploader_id': video_uploader_id,
'thumbnail': video_thumbnail
@@ -4561,10 +4422,9 @@ def _real_extract(self, url):
cookie = urlh.headers.get('Set-Cookie', '')
self.report_extraction(track_id)
- mobj = re.search(r'', response, flags=re.MULTILINE|re.DOTALL)
- if mobj is None:
- raise ExtractorError(u'Unable to extrack tracks')
- html_tracks = mobj.group(1).strip()
+
+ html_tracks = self._html_search_regex(r'',
+ response, u'tracks', flags=re.MULTILINE|re.DOTALL).strip()
try:
track_list = json.loads(html_tracks)
track = track_list[u'tracks'][0]
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 63d9d0ae5..66ae41e31 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -12,7 +12,7 @@
import traceback
import zlib
import email.utils
-import json
+import socket
import datetime
try:
@@ -154,6 +154,9 @@ def compat_ord(c):
if type(c) is int: return c
else: return ord(c)
+# This is not clearly defined otherwise
+compiled_regex_type = type(re.compile(''))
+
std_headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
@@ -469,7 +472,11 @@ class ExtractorError(Exception):
"""Error during info extraction."""
def __init__(self, msg, tb=None):
""" tb, if given, is the original traceback (so that it can be printed out). """
+
+ if not sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+ msg = msg + u'; please report this issue on GitHub.'
super(ExtractorError, self).__init__(msg)
+
self.traceback = tb
self.exc_info = sys.exc_info() # preserve original exception