Merge branch 'master' of https://github.com/rg3/youtube-dl

Conflicts: .gitignore LATEST_VERSION Makefile youtube-dl youtube-dl.exe youtube_dl/InfoExtractors.py youtube_dl/__init__.py
2024-11-19 14:49:15 +00:00 · 2013-01-05 15:03:54 -05:00 · 2013-01-05 15:03:54 -05:00 · 258d5850c9
parent 187da2c093 8e5f761870
commit 258d5850c9
43 changed files with 6464 additions and 4994 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,17 +1,19 @@
 *.pyc
 *.pyo
 *~
+*.DS_Store
 wine-py2exe/
 py2exe.log
-youtube-dl
+*.kate-swp
+build/
+dist/
+MANIFEST
+README.txt
 youtube-dl.1
-LATEST_VERSION
-
-#OS X
-.DS_Store
-.AppleDouble
-.LSOverride
-Icon
-._*
-.Spotlight-V100
-.Trashes
+youtube-dl.bash-completion
+youtube-dl
+youtube-dl.exe
+youtube-dl.tar.gz
+.coverage
+cover/
+updates_key.pem
--- a/.tarignore
+++ b/.tarignore
@ -0,0 +1,17 @@
+updates_key.pem
+*.pyc
+*.pyo
+youtube-dl.exe
+wine-py2exe/
+py2exe.log
+*.kate-swp
+build/
+dist/
+MANIFEST
+*.DS_Store
+youtube-dl.tar.gz
+.coverage
+cover/
+__pycache__/
+.git/
+*~
--- a/.travis.yml
+++ b/.travis.yml
@ -1,9 +1,14 @@
 language: python
-#specify the python version
 python:
  - "2.6"
  - "2.7"
-#command to install the setup
-install:
-# command to run tests
-script: nosetests test --nocapture
+  - "3.3"
+script: nosetests test --verbose
+notifications:
+  email:
+    - filippo.valsorda@gmail.com
+    - phihag@phihag.de
+  irc:
+    channels:
+      - "irc.freenode.org#youtube-dl"
+    skip_join: true
--- a/14
+++ b/14
@ -0,0 +1,14 @@
+2013.01.02  Codename: GIULIA
+
+    * Add support for ComedyCentral clips <nto>
+    * Corrected Vimeo description fetching <Nick Daniels>
+    * Added the --no-post-overwrites argument <Barbu Paul - Gheorghe>
+    * --verbose offers more environment info
+    * New info_dict field: uploader_id
+    * New updates system, with signature checking
+    * New IEs: NBA, JustinTV, FunnyOrDie, TweetReel, Steam, Ustream
+    * Fixed IEs: BlipTv
+    * Fixed for Python 3 IEs: Xvideo, Youku, XNXX, Dailymotion, Vimeo, InfoQ
+    * Simplified IEs and test code
+    * Various (Python 3 and other) fixes
+    * Revamped and expanded tests
--- a/1
+++ b/1
@ -0,0 +1 @@
+2012.10.09
--- a/24
+++ b/24
@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>
--- a/MANIFEST.in
+++ b/MANIFEST.in
@ -0,0 +1,3 @@
+include README.md
+include test/*.py
+include test/*.json
--- a/50
+++ b/50
@ -1,8 +1,7 @@
-all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
-# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion

 clean:
-	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION youtube_dl/*.pyc
+	rm -rf youtube-dl youtube-dl.exe youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/

 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
@ -17,43 +16,32 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl

-.PHONY: all clean install youtube-dl.bash-completion
-# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+test:
+	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
+	nosetests --verbose test
+
+.PHONY: all clean install test

 youtube-dl: youtube_dl/*.py
-	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	zip --quiet youtube-dl youtube_dl/*.py
+	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
 	echo '#!/usr/bin/env python' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl

-youtube-dl.exe: youtube_dl/*.py
-	bash devscripts/wine-py2exe.sh build_exe.py
-
 README.md: youtube_dl/*.py
-	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
-		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
-		footer=$$(sed -e '1,/.*# FAQ/ d' README.md) && \
-		echo "$${header}" > README.md && \
-		echo >> README.md && \
-		echo '# OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '# FAQ' >> README.md && \
-		echo "$${footer}" >> README.md
+	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py

-youtube-dl.1: 
-	pandoc -s -w man README.md -o youtube-dl.1
+README.txt: README.md
+	pandoc -f markdown -t plain README.md -o README.txt

-youtube-dl.bash-completion: 
-	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
-		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
-		echo "$${content}" > youtube-dl.bash-completion
+youtube-dl.1: README.md
+	pandoc -s -f markdown -t man README.md -o youtube-dl.1

-LATEST_VERSION: youtube_dl/__init__.py
-	python -m youtube_dl --version > LATEST_VERSION
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in
+	python devscripts/bash-completion.py

-test:
-	nosetests2 --nocapture test
-
-.PHONY: default compile update update-latest update-readme test clean
+youtube-dl.tar.gz: all
+	tar -cvzf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" \
+		--exclude-from=".tarignore" -- .
--- a/README.md
+++ b/README.md
@ -1,4 +1,4 @@
-% youtube-dl(1)
+% YOUTUBE-DL(1)

 # NAME
 youtube-dl
@ -20,6 +20,11 @@ # OPTIONS
    -i, --ignore-errors      continue on download errors
    -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
    -R, --retries RETRIES    number of retries (default is 10)
+    --buffer-size SIZE       size of download buffer (e.g. 1024 or 16k) (default
+                             is 1024)
+    --no-resize-buffer       do not automatically adjust the buffer size. By
+                             default, the buffer size is automatically resized
+                             from an initial value of SIZE.
    --dump-user-agent        display the current browser identification
    --user-agent UA          specify a custom user agent
    --list-extractors        List all supported extractors and the URLs they
@ -37,16 +42,22 @@ # OPTIONS
  Filesystem Options:
    -t, --title              use title in file name
    --id                     use video ID in file name
-    -l, --literal            use literal title in file name
+    -l, --literal            [deprecated] alias of --title
    -A, --auto-number        number downloaded files starting from 00000
-    -o, --output TEMPLATE    output filename template. Use %(stitle)s to get the
+    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
                             title, %(uploader)s for the uploader name,
-                             %(autonumber)s to get an automatically incremented
-                             number, %(ext)s for the filename extension,
-                             %(upload_date)s for the upload date (YYYYMMDD),
-                             %(extractor)s for the provider (youtube, metacafe,
-                             etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout.
+                             %(uploader_id)s for the uploader nickname if
+                             different, %(autonumber)s to get an automatically
+                             incremented number, %(ext)s for the filename
+                             extension, %(upload_date)s for the upload date
+                             (YYYYMMDD), %(extractor)s for the provider
+                             (youtube, metacafe, etc), %(id)s for the video id
+                             and %% for a literal percent. Use - to output to
+                             stdout. Can also be used to download to a different
+                             directory, for example with -o '/my/downloads/%(upl
+                             oader)s/%(title)s-%(id)s.%(ext)s' .
+    --restrict-filenames     Restrict filenames to only ASCII characters, and
+                             avoid "&" and spaces in filenames
    -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
    -w, --no-overwrites      do not overwrite files
    -c, --continue           resume partially downloaded files
@ -101,6 +112,34 @@ # OPTIONS
                             specific bitrate like 128K (default 5)
    -k, --keep-video         keeps the video file on disk after the post-
                             processing; the video is erased by default
+    --no-post-overwrites     do not overwrite post-processed files; the post-
+                             processed files are overwritten by default
+
+# CONFIGURATION
+
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+
+# OUTPUT TEMPLATE
+
+The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:
+
+ - `id`: The sequence will be replaced by the video identifier.
+ - `url`: The sequence will be replaced by the video URL.
+ - `uploader`: The sequence will be replaced by the nickname of the person who uploaded the video.
+ - `upload_date`: The sequence will be replaced by the upload date in YYYYMMDD format.
+ - `title`: The sequence will be replaced by the video title.
+ - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
+ - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
+ - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
+
+The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment).
+
+In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
+
+    $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+    youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
+    $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+    youtube-dl_test_video_.mp4          # A simple file name

 # FAQ

@ -137,17 +176,9 @@ ### SyntaxError: Non-ASCII character ###

 means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.

-To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
-
-	git clone git://github.com/rg3/youtube-dl.git
-	cd youtube-dl
-	python -m youtube_dl --help
-
-Please note that Python 2.5 is not supported anymore.
-
 ### What is this binary file? Where has the code gone?

-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repo to see the code. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make compile`.
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.

 ### The exe throws a *Runtime error from Visual C++*

@ -166,6 +197,9 @@ # BUGS
 Please include:

 * Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+* If possible re-run the command with `--verbose`, and include the full output, it is really helpful to us.
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.
--- a/bin/youtube-dl
+++ b/bin/youtube-dl
@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import youtube_dl
+
+if __name__ == '__main__':
+    youtube_dl.main()
--- a/build_exe.py
+++ b/build_exe.py
@ -1,48 +0,0 @@
-from distutils.core import setup
-import py2exe
-import sys, os
-
-"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
-
-# If run without args, build executables
-if len(sys.argv) == 1:
-    sys.argv.append("py2exe")
-
-# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
-sys.path.append('./youtube_dl')
-
-options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-console = [{
-    "script":"./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
-}]
-
-init_file = open('./youtube_dl/__init__.py')
-for line in init_file.readlines():
-    if line.startswith('__version__'):
-        version = line[11:].strip(" ='\n")
-        break
-else:
-    version = ''
-
-setup(name='youtube-dl',
-      version=version,
-      description='Small command-line program to download videos from YouTube.com and other video sites',
-      url='https://github.com/rg3/youtube-dl',
-      packages=['youtube_dl'],
-      
-      console = console,
-      options = {"py2exe": options},
-      zipfile = None,
-)
-
-import shutil
-shutil.rmtree("build")
-
--- a/devscripts/bash-completion.in
+++ b/devscripts/bash-completion.in
@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="{{flags}}"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
+
+def build_completion(opt_parser):
+    opts_flag = []
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            #for every long flag
+            opts_flag.append(option.get_opt_string())
+    with open(BASH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    with open(BASH_COMPLETION_FILE, "w") as f:
+        #just using the special char
+        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)
--- a/devscripts/gh-pages/add-version.py
+++ b/devscripts/gh-pages/add-version.py
@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import hashlib
+import urllib.request
+
+if len(sys.argv) <= 1:
+	print('Specify the version number as parameter')
+	sys.exit()
+version = sys.argv[1]
+
+with open('update/LATEST_VERSION', 'w') as f:
+	f.write(version)
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+new_version = {}
+
+filenames = {'bin': 'youtube-dl', 'exe': 'youtube-dl.exe', 'tar': 'youtube-dl-%s.tar.gz' % version}
+for key, filename in filenames.items():
+	print('Downloading and checksumming %s...' %filename)
+	url = 'http://youtube-dl.org/downloads/%s/%s' % (version, filename)
+	data = urllib.request.urlopen(url).read()
+	sha256sum = hashlib.sha256(data).hexdigest()
+	new_version[key] = (url, sha256sum)
+
+versions_info['versions'][version] = new_version
+versions_info['latest'] = version
+
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)
--- a/devscripts/gh-pages/generate-download.py
+++ b/devscripts/gh-pages/generate-download.py
@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import hashlib
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+import json
+
+versions_info = json.load(open('update/versions.json'))
+version = versions_info['latest']
+URL = versions_info['versions'][version]['bin'][0]
+
+data = urllib.request.urlopen(URL).read()
+
+# Read template page
+with open('download.html.in', 'r', encoding='utf-8') as tmplf:
+    template = tmplf.read()
+
+md5sum = hashlib.md5(data).hexdigest()
+sha1sum = hashlib.sha1(data).hexdigest()
+sha256sum = hashlib.sha256(data).hexdigest()
+template = template.replace('@PROGRAM_VERSION@', version)
+template = template.replace('@PROGRAM_URL@', URL)
+template = template.replace('@PROGRAM_MD5SUM@', md5sum)
+template = template.replace('@PROGRAM_SHA1SUM@', sha1sum)
+template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
+template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
+template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])
+template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0])
+template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1])
+with open('download.html', 'w', encoding='utf-8') as dlf:
+    dlf.write(template)
--- a/devscripts/gh-pages/sign-versions.py
+++ b/devscripts/gh-pages/sign-versions.py
@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import rsa
+import json
+from binascii import hexlify
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+print('Enter the PKCS1 private key, followed by a blank line:')
+privkey = ''
+while True:
+	try:
+		line = input()
+	except EOFError:
+		break
+	if line == '':
+		break
+	privkey += line + '\n'
+privkey = bytes(privkey, 'ascii')
+privkey = rsa.PrivateKey.load_pkcs1(privkey)
+
+signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
+print('signature: ' + signature)
+
+versions_info['signature'] = signature
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import with_statement
+
+import datetime
+import glob
+import io # For Python 2 compatibilty
+import os
+import re
+
+year = str(datetime.datetime.now().year)
+for fn in glob.glob('*.html*'):
+    with io.open(fn, encoding='utf-8') as f:
+        content = f.read()
+    newc = re.sub(u'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', u'Copyright © 2006-' + year, content)
+    if content != newc:
+        tmpFn = fn + '.part'
+        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
+            outf.write(newc)
+        os.rename(tmpFn, fn)
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@ -0,0 +1,20 @@
+import sys
+import re
+
+README_FILE = 'README.md'
+helptext = sys.stdin.read()
+
+with open(README_FILE) as f:
+    oldreadme = f.read()
+
+header = oldreadme[:oldreadme.index('# OPTIONS')]
+footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+
+options = helptext[helptext.index('  General Options:')+19:]
+options = re.sub(r'^  (\w.+)$', r'## \1', options, flags=re.M)
+options = '# OPTIONS\n' + options + '\n'
+
+with open(README_FILE, 'w') as f:
+    f.write(header)
+    f.write(options)
+    f.write(footer)
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@ -1,11 +1,85 @@
 #!/bin/sh

+# IMPORTANT: the following assumptions are made
+# * the GH repo is on the origin remote
+# * the gh-pages branch is named so locally
+# * the git config user.signingkey is properly set
+
+# You will need
+# pip install coverage nose rsa
+
+# TODO
+# release notes
+# make hash on local files
+
+set -e
+
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
-sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
-make all
-git add -A
+if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
+
+echo "\n### First of all, testing..."
+make clean
+nosetests --with-coverage --cover-package=youtube_dl --cover-html test || exit 1
+
+echo "\n### Changing version in version.py..."
+sed -i~ "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+
+echo "\n### Committing CHANGELOG README.md and youtube_dl/version.py..."
+make README.md
+git add CHANGELOG README.md youtube_dl/version.py
 git commit -m "release $version"
-git tag -m "Release $version" "$version"
+
+echo "\n### Now tagging, signing and pushing..."
+git tag -s -m "Release $version" "$version"
+git show "$version"
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+MASTER=$(git rev-parse --abbrev-ref HEAD)
+git push origin $MASTER:master
+git push origin "$version"
+
+echo "\n### OK, now it is time to build the binaries..."
+REV=$(git rev-parse HEAD)
+make youtube-dl youtube-dl.tar.gz
+wget "http://jeromelaheurte.net:8142/download/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe || \
+	wget "http://jeromelaheurte.net:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+mkdir -p "update_staging/$version"
+mv youtube-dl youtube-dl.exe "update_staging/$version"
+mv youtube-dl.tar.gz "update_staging/$version/youtube-dl-$version.tar.gz"
+RELEASE_FILES=youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz
+(cd update_staging/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
+(cd update_staging/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
+(cd update_staging/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
+(cd update_staging/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
+git checkout HEAD -- youtube-dl youtube-dl.exe
+
+echo "\n### Signing and uploading the new binaries to youtube-dl.org..."
+for f in $RELEASE_FILES; do gpg --detach-sig "update_staging/$version/$f"; done
+scp -r "update_staging/$version" ytdl@youtube-dl.org:html/downloads/
+rm -r update_staging
+
+echo "\n### Now switching to gh-pages..."
+git checkout gh-pages
+git checkout "$MASTER" -- devscripts/gh-pages/
+git reset devscripts/gh-pages/
+devscripts/gh-pages/add-version.py $version
+devscripts/gh-pages/sign-versions.py < updates_key.pem
+devscripts/gh-pages/generate-download.py
+devscripts/gh-pages/update-copyright.py
+rm -r test_coverage
+mv cover test_coverage
+git add *.html *.html.in update test_coverage
+git commit -m "release $version"
+git show HEAD
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+git push origin gh-pages
+
+echo "\n### DONE!"
+rm -r devscripts
+git checkout $MASTER
--- a/devscripts/transition_helper.py
+++ b/devscripts/transition_helper.py
@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+try:
+	raw_input()
+except NameError: # Python 3
+	input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+try:
+    urlh = compat_urllib_request.urlopen(BIN_URL)
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    with open(filename, 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')
--- a/devscripts/transition_helper_exe/setup.py
+++ b/devscripts/transition_helper_exe/setup.py
@ -0,0 +1,12 @@
+from distutils.core import setup
+import py2exe
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)
--- a/devscripts/transition_helper_exe/youtube-dl.py
+++ b/devscripts/transition_helper_exe/youtube-dl.py
@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+import sys, os
+import urllib2
+import json, hashlib
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
+
+raw_input()
+
+filename = sys.argv[0]
+
+UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+JSON_URL = UPDATE_URL + 'versions.json'
+UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+exe = os.path.abspath(filename)
+directory = os.path.dirname(exe)
+if not os.access(directory, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % directory)
+
+try:
+    versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
+    versions_info = json.loads(versions_info)
+except:
+    sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
+if not 'signature' in versions_info:
+    sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+signature = versions_info['signature']
+del versions_info['signature']
+if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
+    sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
+
+version = versions_info['versions'][versions_info['latest']]
+
+try:
+    urlh = urllib2.urlopen(version['exe'][0])
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+if newcontent_hash != version['exe'][1]:
+    sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
+
+try:
+    with open(exe + '.new', 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit(u'ERROR: unable to write the new version')
+
+try:
+    bat = os.path.join(directory, 'youtube-dl-updater.bat')
+    b = open(bat, 'w')
+    b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+    \n""" %(exe, exe, bat))
+    b.close()
+
+    os.startfile(bat)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')
--- a/setup.py
+++ b/setup.py
@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+from distutils.core import setup
+import pkg_resources
+import sys
+
+try:
+    import py2exe
+    """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+except ImportError:
+    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+        print("Cannot import py2exe", file=sys.stderr)
+        exit(1)
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+py2exe_console = [{
+    "script": "./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+py2exe_params = {
+    'console': py2exe_console,
+    'options': { "py2exe": py2exe_options },
+    'zipfile': None
+}
+
+if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+    params = py2exe_params
+else:
+    params = {
+        'scripts': ['bin/youtube-dl'],
+        'data_files': [('etc/bash_completion.d', ['youtube-dl.bash-completion']), # Installing system-wide would require sudo...
+                       ('share/doc/youtube_dl', ['README.txt']),
+                       ('share/man/man1/', ['youtube-dl.1'])]
+    }
+
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec'))
+
+setup(
+    name = 'youtube_dl',
+    version = __version__,
+    description = 'YouTube video downloader',
+    long_description = 'Small command-line program to download videos from YouTube.com and other video sites.',
+    url = 'https://github.com/rg3/youtube-dl',
+    author = 'Ricardo Garcia',
+    maintainer = 'Philipp Hagemeister',
+    maintainer_email = 'phihag@phihag.de',
+    packages = ['youtube_dl'],
+
+    # Provokes warning on most systems (why?!)
+    #test_suite = 'nose.collector',
+    #test_requires = ['nosetest'],
+
+    classifiers = [
+        "Topic :: Multimedia :: Video",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "License :: Public Domain",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3"
+    ],
+
+    **params
+)
--- a/test/parameters.json
+++ b/test/parameters.json
@ -1 +1,40 @@
-{"username": null, "listformats": null, "skip_download": false, "usenetrc": false, "max_downloads": null, "noprogress": false, "forcethumbnail": false, "forceformat": false, "format_limit": null, "ratelimit": null, "nooverwrites": false, "forceurl": false, "writeinfojson": false, "simulate": false, "playliststart": 1, "continuedl": true, "password": null, "prefer_free_formats": false, "nopart": false, "retries": 10, "updatetime": true, "consoletitle": false, "verbose": true, "forcefilename": false, "ignoreerrors": false, "logtostderr": false, "format": null, "subtitleslang": null, "quiet": false, "outtmpl": "%(id)s.%(ext)s", "rejecttitle": null, "playlistend": -1, "writedescription": false, "forcetitle": false, "forcedescription": false, "writesubtitles": false, "matchtitle": null}
+{
+    "consoletitle": false, 
+    "continuedl": true, 
+    "forcedescription": false, 
+    "forcefilename": false, 
+    "forceformat": false, 
+    "forcethumbnail": false, 
+    "forcetitle": false, 
+    "forceurl": false, 
+    "format": null, 
+    "format_limit": null, 
+    "ignoreerrors": false, 
+    "listformats": null, 
+    "logtostderr": false, 
+    "matchtitle": null, 
+    "max_downloads": null, 
+    "nooverwrites": false, 
+    "nopart": false, 
+    "noprogress": false, 
+    "outtmpl": "%(id)s.%(ext)s", 
+    "password": null, 
+    "playlistend": -1, 
+    "playliststart": 1, 
+    "prefer_free_formats": false, 
+    "quiet": false, 
+    "ratelimit": null, 
+    "rejecttitle": null, 
+    "retries": 10, 
+    "simulate": false, 
+    "skip_download": false, 
+    "subtitleslang": null, 
+    "test": true, 
+    "updatetime": true, 
+    "usenetrc": false, 
+    "username": null, 
+    "verbose": true, 
+    "writedescription": false, 
+    "writeinfojson": true, 
+    "writesubtitles": false
+}
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestAllURLsMatching(unittest.TestCase):
+    def test_youtube_playlist_matching(self):
+        self.assertTrue(YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE().suitable(u'PL63F0C78739B09958'))
+        self.assertFalse(YoutubePlaylistIE().suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_matching(self):
+        self.assertTrue(YoutubeIE().suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_extract(self):
+        self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc'), 'BaW_jenozKc')
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_download.py
+++ b/test/test_download.py
@ -1,93 +1,125 @@
-#!/usr/bin/env python2
-import unittest
+#!/usr/bin/env python
+
+import errno
 import hashlib
+import io
 import os
 import json
+import unittest
+import sys
+import hashlib
+import socket

-from youtube_dl.FileDownloader import FileDownloader
-from youtube_dl.InfoExtractors  import YoutubeIE, DailymotionIE
-from youtube_dl.InfoExtractors import  MetacafeIE, BlipTVIE
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+def _try_rm(filename):
+    """ Remove a file if it exists """
+    try:
+        os.remove(filename)
+    except OSError as ose:
+        if ose.errno != errno.ENOENT:
+            raise
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        self.to_stderr = self.to_screen
+        self.processed_info_dicts = []
+        return youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+    def process_info(self, info_dict):
+        self.processed_info_dicts.append(info_dict)
+        return youtube_dl.FileDownloader.process_info(self, info_dict)
+
+def _file_md5(fn):
+    with open(fn, 'rb') as f:
+        return hashlib.md5(f.read()).hexdigest()
+
+with io.open(DEF_FILE, encoding='utf-8') as deff:
+    defs = json.load(deff)
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)


-class DownloadTest(unittest.TestCase):
-	PARAMETERS_FILE = "test/parameters.json"
-	#calculated with md5sum:
-	#md5sum (GNU coreutils) 8.19
+class TestDownload(unittest.TestCase):
+    def setUp(self):
+        self.parameters = parameters
+        self.defs = defs

-	YOUTUBE_SIZE = 1993883
-	YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
-	YOUTUBE_FILE = "BaW_jenozKc.mp4"
+### Dynamically generate tests
+def generator(test_case):

-	DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
-	DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
-	DAILYMOTION_FILE = "x33vw9.mp4"
+    def test_template(self):
+        ie = getattr(youtube_dl.InfoExtractors, test_case['name'] + 'IE')
+        if not ie._WORKING:
+            print('Skipping: IE marked as not _WORKING')
+            return
+        if 'playlist' not in test_case and not test_case['file']:
+            print('Skipping: No output file specified')
+            return
+        if 'skip' in test_case:
+            print('Skipping: {0}'.format(test_case['skip']))
+            return

-	METACAFE_SIZE = 5754305
-	METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
-	METACAFE_FILE = "_aUehQsCQtM.flv"
+        params = self.parameters.copy()
+        params.update(test_case.get('params', {}))

-	BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
-	BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
-	BLIP_FILE = "5779306.m4v"
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie())
+        for ien in test_case.get('add_ie', []):
+            fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())

-	XVIDEO_MD5 = ""
-	XVIDEO_URL = ""
-	XVIDEO_FILE = ""
+        test_cases = test_case.get('playlist', [test_case])
+        for tc in test_cases:
+            _try_rm(tc['file'])
+            _try_rm(tc['file'] + '.part')
+            _try_rm(tc['file'] + '.info.json')
+        try:
+            fd.download([test_case['url']])
+
+            for tc in test_cases:
+                if not test_case.get('params', {}).get('skip_download', False):
+                    self.assertTrue(os.path.exists(tc['file']))
+                self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
+                if 'md5' in tc:
+                    md5_for_file = _file_md5(tc['file'])
+                    self.assertEqual(md5_for_file, tc['md5'])
+                with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
+                    info_dict = json.load(infof)
+                for (info_field, value) in tc.get('info_dict', {}).items():
+                    if value.startswith('md5:'):
+                        md5_info_value = hashlib.md5(info_dict.get(info_field, '')).hexdigest()
+                        self.assertEqual(value[3:], md5_info_value)
+                    else:
+                        self.assertEqual(value, info_dict.get(info_field))
+        finally:
+            for tc in test_cases:
+                _try_rm(tc['file'])
+                _try_rm(tc['file'] + '.part')
+                _try_rm(tc['file'] + '.info.json')
+
+    return test_template
+
+### And add them to TestDownload
+for test_case in defs:
+    test_method = generator(test_case)
+    test_method.__name__ = "test_{0}".format(test_case["name"])
+    setattr(TestDownload, test_method.__name__, test_method)
+    del test_method


-	def test_youtube(self):
-		#let's download a file from youtube
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(YoutubeIE())
-		fd.download([DownloadTest.YOUTUBE_URL])
-		self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
-
-	def test_dailymotion(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(DailymotionIE())
-		fd.download([DownloadTest.DAILYMOTION_URL])
-		self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
-		md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
-
-	def test_metacafe(self):
-		#this emulate a skip,to be 2.6 compatible
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(MetacafeIE())
-		fd.add_info_extractor(YoutubeIE())
-		fd.download([DownloadTest.METACAFE_URL])
-		self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
-
-	def test_blip(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(BlipTVIE())
-		fd.download([DownloadTest.BLIP_URL])
-		self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
-		md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
-
-	def tearDown(self):
-		if os.path.exists(DownloadTest.YOUTUBE_FILE):
-			os.remove(DownloadTest.YOUTUBE_FILE)
-		if os.path.exists(DownloadTest.DAILYMOTION_FILE):
-			os.remove(DownloadTest.DAILYMOTION_FILE)
-		if os.path.exists(DownloadTest.METACAFE_FILE):
-			os.remove(DownloadTest.METACAFE_FILE)
-		if os.path.exists(DownloadTest.BLIP_FILE):
-			os.remove(DownloadTest.BLIP_FILE)
-
-def md5_for_file(filename, block_size=2**20):
-    with open(filename) as f:
-        md5 = hashlib.md5()
-        while True:
-            data = f.read(block_size)
-            if not data:
-                break
-            md5.update(data)
-            return md5.hexdigest()
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_execution.py
+++ b/test/test_execution.py
@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+    _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+    def test_import(self):
+        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+    def test_module_exec(self):
+        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+    def test_main_exec(self):
+        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_utils.py
+++ b/test/test_utils.py
@ -1,15 +1,25 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python

 # Various small unit tests

+import sys
 import unittest

+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 #from youtube_dl.utils import htmlentity_transform
 from youtube_dl.utils import timeconvert
 from youtube_dl.utils import sanitize_filename
 from youtube_dl.utils import unescapeHTML
 from youtube_dl.utils import orderedSet

+if sys.version_info < (3, 0):
+    _compat_str = lambda b: b.decode('unicode-escape')
+else:
+    _compat_str = lambda s: s
+

 class TestUtil(unittest.TestCase):
    def test_timeconvert(self):
@ -17,24 +27,64 @@ def test_timeconvert(self):
        self.assertTrue(timeconvert('bougrg') is None)

    def test_sanitize_filename(self):
-		self.assertEqual(sanitize_filename(u'abc'), u'abc')
-		self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e')
+        self.assertEqual(sanitize_filename('abc'), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')

-		self.assertEqual(sanitize_filename(u'123'), u'123')
+        self.assertEqual(sanitize_filename('123'), '123')

-		self.assertEqual(u'abc-de', sanitize_filename(u'abc/de'))
-		self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
+        self.assertEqual('abc_de', sanitize_filename('abc/de'))
+        self.assertFalse('/' in sanitize_filename('abc/de///'))

-		self.assertEqual(u'abc-de', sanitize_filename(u'abc/<>\\*|de'))
-		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
-		self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
-		self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
+        self.assertEqual('yes no', sanitize_filename('yes? no'))
+        self.assertEqual('this - that', sanitize_filename('this: that'))

-		self.assertEqual(sanitize_filename(u'ä'), u'ä')
-		self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица')
+        self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
+        aumlaut = _compat_str('\xe4')
+        self.assertEqual(sanitize_filename(aumlaut), aumlaut)
+        tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
+        self.assertEqual(sanitize_filename(tests), tests)

-		for forbidden in u'"\0\\/':
-			self.assertTrue(forbidden not in sanitize_filename(forbidden))
+        forbidden = '"\0\\/'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc))
+
+    def test_sanitize_filename_restricted(self):
+        self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
+
+        self.assertEqual(sanitize_filename('123', restricted=True), '123')
+
+        self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
+        self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
+
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
+        self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
+        self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
+
+        tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c')
+        self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
+        self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '')  # No empty filename
+
+        forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+
+        # Handle a common case more neatly
+        self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
+        self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
+        # .. but make sure the file name is never empty
+        self.assertTrue(sanitize_filename('-', restricted=True) != '')
+        self.assertTrue(sanitize_filename(':', restricted=True) != '')
+
+    def test_sanitize_ids(self):
+        self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+        self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')

    def test_ordered_set(self):
        self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
@ -44,4 +94,7 @@ def test_ordered_set(self):
        self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])

    def test_unescape_html(self):
-		self.assertEqual(unescapeHTML(u"%20;"), u"%20;")
+        self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_write_info_json.py
+++ b/test/test_write_info_json.py
@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import json
+import os
+import sys
+import unittest
+
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    params = json.load(pf)
+params['writeinfojson'] = True
+params['skip_download'] = True
+params['writedescription'] = True
+
+TEST_ID = 'BaW_jenozKc'
+INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
+DESCRIPTION_FILE = TEST_ID + '.mp4.description'
+EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
+
+This is a test video for youtube-dl.
+
+For more information, contact phihag@phihag.de .'''
+
+class TestInfoJSON(unittest.TestCase):
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+    def test_info_json(self):
+        ie = youtube_dl.InfoExtractors.YoutubeIE()
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie)
+        fd.download([TEST_ID])
+        self.assertTrue(os.path.exists(INFO_JSON_FILE))
+        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
+            jd = json.load(jsonf)
+        self.assertEqual(jd['upload_date'], u'20121002')
+        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
+        self.assertEqual(jd['id'], TEST_ID)
+        self.assertEqual(jd['extractor'], 'youtube')
+        self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
+        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
+
+        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
+        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
+            descr = descf.read()
+        self.assertEqual(descr, EXPECTED_DESCRIPTION)
+
+    def tearDown(self):
+        if os.path.exists(INFO_JSON_FILE):
+            os.remove(INFO_JSON_FILE)
+        if os.path.exists(DESCRIPTION_FILE):
+            os.remove(DESCRIPTION_FILE)
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeUserIE,YoutubePlaylistIE
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+class TestYoutubeLists(unittest.TestCase):
+    def test_youtube_playlist(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertEqual(DL.result, [
+            ['http://www.youtube.com/watch?v=bV9L5Ht9LgY'],
+            ['http://www.youtube.com/watch?v=FXxLjLQi3Fg'],
+            ['http://www.youtube.com/watch?v=tU3Bgo5qJZE']
+        ])
+
+    def test_youtube_playlist_long(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
+        self.assertTrue(len(DL.result) >= 799)
+
+    def test_youtube_course(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        # TODO find a > 100 (paginating?) videos course
+        IE.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+        self.assertEqual(DL.result[0], ['http://www.youtube.com/watch?v=j9WZyLZCBzs'])
+        self.assertEqual(len(DL.result), 25)
+        self.assertEqual(DL.result[-1], ['http://www.youtube.com/watch?v=rYefUsYuEp0'])
+
+    def test_youtube_channel(self):
+        # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long
+        pass # TODO
+
+    def test_youtube_user(self):
+        DL = FakeDownloader()
+        IE = YoutubeUserIE(DL)
+        IE.extract('https://www.youtube.com/user/TheLinuxFoundation')
+        self.assertTrue(len(DL.result) >= 320)
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_youtube_subtitles.py
+++ b/test/test_youtube_subtitles.py
@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+import io
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestYoutubeSubtitles(unittest.TestCase):
+    def test_youtube_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), 'c3228550d59116f3c29fba370b55d033')
+
+    def test_youtube_subtitles_it(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), '132a88a0daf8e1520f393eb58f1f646a')
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/tests.json
+++ b/test/tests.json
@ -0,0 +1,164 @@
+[
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=BaW_jenozKc",
+    "file":  "BaW_jenozKc.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐",
+      "uploader": "Philipp Hagemeister",
+      "uploader_id": "phihag",
+      "upload_date": "20121002",
+      "description": "test chars:  \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
+    }
+  },
+  {
+    "name": "Dailymotion",
+    "md5":  "392c4b85a60a90dc4792da41ce3144eb",
+    "url":  "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech",
+    "file":  "x33vw9.mp4"
+  },
+  {
+    "name": "Metacafe",
+    "add_ie": ["Youtube"],
+    "url":  "http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
+    "file":  "_aUehQsCQtM.flv"
+  },
+  {
+    "name": "BlipTV",
+    "md5":  "b2d849efcf7ee18917e4b4d9ff37cafe",
+    "url":  "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352",
+    "file":  "5779306.m4v"
+  },
+  {
+    "name": "XVideos",
+    "md5":  "1d0c835822f0a71a7bf011855db929d0",
+    "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
+    "file":  "939581.flv"
+  },
+  {
+    "name": "Vimeo",
+    "md5":  "8879b6cc097e987f02484baf890129e5",
+    "url":  "http://vimeo.com/56015672",
+    "file": "56015672.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video - ★ \" ' 幸 / \\ ä ↭ 𝕐",
+      "uploader": "Filippo Valsorda",
+      "uploader_id": "user7108434",
+      "upload_date": "20121220",
+      "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: ★ \" ' 幸 / \\ ä ↭ 𝕐"
+    }
+  },
+  {
+    "name": "Soundcloud",
+    "md5":  "ebef0a451b909710ed1d7787dddbf0d7",
+    "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
+    "file":  "62986583.mp3"
+  },
+  {
+    "name": "StanfordOpenClassroom",
+    "md5":  "544a9468546059d4e80d76265b0443b8",
+    "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
+    "file":  "PracticalUnix_intro-environment.mp4"
+  },
+  {
+    "name": "XNXX",
+    "md5":  "0831677e2b4761795f68d417e0b7b445",
+    "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
+    "file":  "1135332.flv"
+  },
+  {
+    "name": "Youku",
+    "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+    "file": "XNDgyMDQ2NTQw_part00.flv",
+    "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
+    "params": { "test": false }
+  },
+  {
+    "name": "NBA",
+    "url": "http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html",
+    "file": "0021200253-okc-bkn-recap.nba.mp4",
+    "md5": "c0edcfc37607344e2ff8f13c378c88a4"
+  },
+  {
+    "name": "JustinTV",
+    "url": "http://www.twitch.tv/thegamedevhub/b/296128360",
+    "file": "296128360.flv",
+    "md5": "ecaa8a790c22a40770901460af191c9a"
+  },
+  {
+    "name": "MyVideo",
+    "url": "http://www.myvideo.de/watch/8229274/bowling_fail_or_win",
+    "file": "8229274.flv",
+    "md5": "2d2753e8130479ba2cb7e0a37002053e"
+  },
+  {
+    "name": "Escapist",
+    "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
+    "file": "6618-Breaking-Down-Baldurs-Gate.flv",
+    "md5": "c6793dbda81388f4264c1ba18684a74d",
+    "skip": "Fails with timeout on Travis"
+  },
+  {
+    "name": "GooglePlus",
+    "url": "https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH",
+    "file": "ZButuJc6CtH.flv"
+  },
+  {
+    "name": "FunnyOrDie",
+    "url": "http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version",
+    "file": "0732f586d7.mp4",
+    "md5": "f647e9e90064b53b6e046e75d0241fbd"
+  },
+  {
+    "name": "TweetReel",
+    "url": "http://tweetreel.com/?77smq",
+    "file": "77smq.mov",
+    "md5": "56b4d9ca9de467920f3f99a6d91255d6",
+    "info_dict": {
+        "uploader": "itszero",
+        "uploader_id": "itszero",
+        "upload_date": "20091225",
+        "description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D"
+    }
+  },
+  {
+    "name": "Steam",
+    "url": "http://store.steampowered.com/video/105600/",
+    "playlist": [
+      {
+        "file": "81300.flv",
+        "md5": "f870007cee7065d7c76b88f0a45ecc07",
+        "info_dict": {
+            "title": "Terraria 1.1 Trailer"
+        }
+      },
+      {
+        "file": "80859.flv",
+        "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+        "info_dict": {
+          "title": "Terraria Trailer"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Ustream",
+    "url": "http://www.ustream.tv/recorded/20274954",
+    "file": "20274954.flv",
+    "md5": "088f151799e8f572f84eb62f17d73e5c",
+    "info_dict": {
+        "title": "Young Americans for Liberty February 7, 2012 2:28 AM"
+    }
+  },
+  {
+    "name": "InfoQ",
+    "url": "http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things",
+    "file": "12-jan-pythonthings.mp4",
+    "info_dict": {
+      "title": "A Few of My Favorite [Python] Things"
+    },
+    "params": {
+      "skip_download": true
+    }
+  }
+]
--- a/BIN
+++ b/BIN
--- a/youtube-dl.bash-completion
+++ b/youtube-dl.bash-completion
@ -1,14 +0,0 @@
-__youtube-dl()
-{
-    local cur prev opts
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
-
-    if [[ ${cur} == * ]] ; then
-        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        return 0
-    fi
-}
-
-complete -F __youtube-dl youtube-dl
--- a/youtube-dl.exe
+++ b/youtube-dl.exe
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@ -1,20 +1,22 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-

-import httplib
+from __future__ import absolute_import
+
 import math
+import io
 import os
 import re
 import socket
 import subprocess
 import sys
 import time
-import urllib2
+import traceback

 if os.name == 'nt':
    import ctypes

-from utils import *
+from .utils import *


 class FileDownloader(object):
@ -57,10 +59,13 @@ class FileDownloader(object):
    format:            Video format code.
    format_limit:      Highest quality format to try.
    outtmpl:           Template for output names.
+    restrictfilenames: Do not allow "&" and spaces in file names
    ignoreerrors:      Do not stop on download errors.
    ratelimit:         Download speed limit, in bytes/sec.
    nooverwrites:      Prevent overwriting files.
    retries:           Number of times to retry for HTTP error 5xx
+    buffersize:        Size of download buffer in bytes.
+    noresizebuffer:    Do not automatically resize the download buffer.
    continuedl:        Try to continue downloads if possible.
    noprogress:        Do not print the progress bar.
    playliststart:     Playlist item to start at.
@ -75,6 +80,7 @@ class FileDownloader(object):
    writeinfojson:     Write the video description to a .info.json file
    writesubtitles:    Write the video subtitles to a .srt file
    subtitleslang:     Language of the subtitles to download
+    test:              Download only first bytes to test the downloader.
    """

    params = None
@ -93,6 +99,9 @@ def __init__(self, params):
        self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
        self.params = params

+        if '%(stitle)s' in self.params['outtmpl']:
+            self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+
    @staticmethod
    def format_bytes(bytes):
        if bytes is None:
@ -102,7 +111,7 @@ def format_bytes(bytes):
        if bytes == 0.0:
            exponent = 0
        else:
-			exponent = long(math.log(bytes, 1024.0))
+            exponent = int(math.log(bytes, 1024.0))
        suffix = 'bkMGTPEZY'[exponent]
        converted = float(bytes) / float(1024 ** exponent)
        return '%.2f%s' % (converted, suffix)
@ -121,7 +130,7 @@ def calc_eta(start, now, total, current):
        if current == 0 or dif < 0.001: # One millisecond
            return '--:--'
        rate = float(current) / dif
-		eta = long((float(total) - float(current)) / rate)
+        eta = int((float(total) - float(current)) / rate)
        (eta_mins, eta_secs) = divmod(eta, 60)
        if eta_mins > 99:
            return '--:--'
@ -139,23 +148,23 @@ def best_block_size(elapsed_time, bytes):
        new_min = max(bytes / 2.0, 1.0)
        new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
        if elapsed_time < 0.001:
-			return long(new_max)
+            return int(new_max)
        rate = bytes / elapsed_time
        if rate > new_max:
-			return long(new_max)
+            return int(new_max)
        if rate < new_min:
-			return long(new_min)
-		return long(rate)
+            return int(new_min)
+        return int(rate)

    @staticmethod
    def parse_bytes(bytestr):
-		"""Parse a string indicating a byte quantity into a long integer."""
+        """Parse a string indicating a byte quantity into an integer."""
        matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
        if matchobj is None:
            return None
        number = float(matchobj.group(1))
        multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
-		return long(round(number * multiplier))
+        return int(round(number * multiplier))

    def add_info_extractor(self, ie):
        """Add an InfoExtractor object to the end of the list."""
@ -173,14 +182,18 @@ def to_screen(self, message, skip_eol=False):
        if not self.params.get('quiet', False):
            terminator = [u'\n', u''][skip_eol]
            output = message + terminator
-			if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+            if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
                output = output.encode(preferredencoding(), 'ignore')
            self._screen_file.write(output)
            self._screen_file.flush()

    def to_stderr(self, message):
        """Print message to stderr."""
-		print >>sys.stderr, message.encode(preferredencoding())
+        assert type(message) == type(u'')
+        output = message + u'\n'
+        if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+            output = output.encode(preferredencoding())
+        sys.stderr.write(output)

    def to_cons_title(self, message):
        """Set console/terminal window title to message."""
@ -195,17 +208,24 @@ def to_cons_title(self, message):

    def fixed_template(self):
        """Checks if the output template is fixed."""
-		return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
+        return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)

-	def trouble(self, message=None):
+    def trouble(self, message=None, tb=None):
        """Determine action to take when a download problem appears.

        Depending on if the downloader has been configured to ignore
        download errors or not, this method may throw an exception or
        not when errors are found, after printing the message.
+
+        tb, if given, is additional traceback information.
        """
        if message is not None:
            self.to_stderr(message)
+        if self.params.get('verbose'):
+            if tb is None:
+                tb_data = traceback.format_list(traceback.extract_stack())
+                tb = u''.join(tb_data)
+            self.to_stderr(tb)
        if not self.params.get('ignoreerrors', False):
            raise DownloadError(message)
        self._download_retcode = 1
@ -240,7 +260,7 @@ def try_rename(self, old_filename, new_filename):
            if old_filename == new_filename:
                return
            os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
-		except (IOError, OSError), err:
+        except (IOError, OSError) as err:
            self.trouble(u'ERROR: unable to rename file')

    def try_utime(self, filename, last_modified_hdr):
@ -298,7 +318,7 @@ def report_file_already_downloaded(self, file_name):
        """Report file has already been fully downloaded."""
        try:
            self.to_screen(u'[download] %s has already been downloaded' % file_name)
-		except (UnicodeEncodeError), err:
+        except (UnicodeEncodeError) as err:
            self.to_screen(u'[download] The file has already been downloaded')

    def report_unable_to_resume(self):
@ -320,11 +340,19 @@ def prepare_filename(self, info_dict):
        """Generate the output filename."""
        try:
            template_dict = dict(info_dict)
-			template_dict['epoch'] = unicode(long(time.time()))
-			template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
+
+            template_dict['epoch'] = int(time.time())
+            template_dict['autonumber'] = u'%05d' % self._num_downloads
+
+            sanitize = lambda k,v: sanitize_filename(
+                u'NA' if v is None else compat_str(v),
+                restricted=self.params.get('restrictfilenames'),
+                is_id=(k==u'id'))
+            template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
+
            filename = self.params['outtmpl'] % template_dict
            return filename
-		except (ValueError, KeyError), err:
+        except (ValueError, KeyError) as err:
            self.trouble(u'ERROR: invalid system charset or erroneous output template')
            return None

@ -347,7 +375,11 @@ def _match_entry(self, info_dict):
    def process_info(self, info_dict):
        """Process a single dictionary returned by an InfoExtractor."""

-		info_dict['stitle'] = sanitize_filename(info_dict['title'])
+        # Keep for backwards compatibility
+        info_dict['stitle'] = info_dict['title']
+
+        if not 'format' in info_dict:
+            info_dict['format'] = info_dict['ext']

        reason = self._match_entry(info_dict)
        if reason is not None:
@ -363,17 +395,17 @@ def process_info(self, info_dict):

        # Forced printings
        if self.params.get('forcetitle', False):
-			print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(info_dict['title'])
        if self.params.get('forceurl', False):
-			print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(info_dict['url'])
        if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
-			print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(info_dict['thumbnail'])
        if self.params.get('forcedescription', False) and 'description' in info_dict:
-			print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(info_dict['description'])
        if self.params.get('forcefilename', False) and filename is not None:
-			print filename.encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(filename)
        if self.params.get('forceformat', False):
-			print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
+            compat_print(info_dict['format'])

        # Do nothing else if in simulate mode
        if self.params.get('simulate', False):
@ -386,19 +418,16 @@ def process_info(self, info_dict):
            dn = os.path.dirname(encodeFilename(filename))
            if dn != '' and not os.path.exists(dn): # dn is already encoded
                os.makedirs(dn)
-		except (OSError, IOError), err:
-			self.trouble(u'ERROR: unable to create directory ' + unicode(err))
+        except (OSError, IOError) as err:
+            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
            return

        if self.params.get('writedescription', False):
            try:
                descfn = filename + u'.description'
                self.report_writedescription(descfn)
-				descfile = open(encodeFilename(descfn), 'wb')
-				try:
-					descfile.write(info_dict['description'].encode('utf-8'))
-				finally:
-					descfile.close()
+                with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+                    descfile.write(info_dict['description'])
            except (OSError, IOError):
                self.trouble(u'ERROR: Cannot write description file ' + descfn)
                return
@ -409,11 +438,8 @@ def process_info(self, info_dict):
            try:
                srtfn = filename.rsplit('.', 1)[0] + u'.srt'
                self.report_writesubtitles(srtfn)
-				srtfile = open(encodeFilename(srtfn), 'wb')
-				try:
-					srtfile.write(info_dict['subtitles'].encode('utf-8'))
-				finally:
-					srtfile.close()
+                with io.open(encodeFilename(srtfn), 'w', encoding='utf-8') as srtfile:
+                    srtfile.write(info_dict['subtitles'])
            except (OSError, IOError):
                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
                return
@ -422,17 +448,8 @@ def process_info(self, info_dict):
            infofn = filename + u'.info.json'
            self.report_writeinfojson(infofn)
            try:
-				json.dump
-			except (NameError,AttributeError):
-				self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
-				return
-			try:
-				infof = open(encodeFilename(infofn), 'wb')
-				try:
-					json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
-					json.dump(json_info_dict, infof)
-				finally:
-					infof.close()
+                json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
+                write_json_file(json_info_dict, encodeFilename(infofn))
            except (OSError, IOError):
                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
                return
@ -443,19 +460,19 @@ def process_info(self, info_dict):
            else:
                try:
                    success = self._do_download(filename, info_dict)
-				except (OSError, IOError), err:
-					raise UnavailableVideoError
-				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                except (OSError, IOError) as err:
+                    raise UnavailableVideoError()
+                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
                    return
-				except (ContentTooShortError, ), err:
+                except (ContentTooShortError, ) as err:
                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                    return

            if success:
                try:
                    self.post_process(filename, info_dict)
-				except (PostProcessingError), err:
+                except (PostProcessingError) as err:
                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
                    return

@ -471,11 +488,30 @@ def download(self, url_list):
                if not ie.suitable(url):
                    continue

+                # Warn if the _WORKING attribute is False
+                if not ie.working():
+                    self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
+                                   u'and will probably not work. If you want to go on, use the -i option.')
+
                # Suitable InfoExtractor found
                suitable_found = True

                # Extract information from URL and process it
+                try:
                    videos = ie.extract(url)
+                except ExtractorError as de: # An error we somewhat expected
+                    self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
+                    break
+                except Exception as e:
+                    if self.params.get('ignoreerrors', False):
+                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
+                        break
+                    else:
+                        raise
+
+                if len(videos or []) > 1 and self.fixed_template():
+                    raise SameFileError(self.params['outtmpl'])
+
                for video in videos or []:
                    video['extractor'] = ie.IE_NAME
                    try:
@ -501,7 +537,7 @@ def post_process(self, filename, ie_info):
            if info is None:
                break

-	def _download_with_rtmpdump(self, filename, url, player_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url):
        self.report_destination(filename)
        tmpfilename = self.temp_name(filename)

@ -515,7 +551,11 @@ def _download_with_rtmpdump(self, filename, url, player_url):
        # Download using rtmpdump. rtmpdump returns exit code 2 when
        # the connection was interrumpted and resuming appears to be
        # possible. This is part of rtmpdump's normal usage, AFAIK.
-		basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
+        basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
+        if player_url is not None:
+            basic_args += ['-W', player_url]
+        if page_url is not None:
+            basic_args += ['--pageUrl', page_url]
        args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
        if self.params.get('verbose', False):
            try:
@ -548,7 +588,6 @@ def _download_with_rtmpdump(self, filename, url, player_url):

    def _do_download(self, filename, info_dict):
        url = info_dict['url']
-		player_url = info_dict.get('player_url', None)

        # Check file already present
        if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
@ -557,15 +596,20 @@ def _do_download(self, filename, info_dict):

        # Attempt to download using rtmpdump
        if url.startswith('rtmp'):
-			return self._download_with_rtmpdump(filename, url, player_url)
+            return self._download_with_rtmpdump(filename, url,
+                                                info_dict.get('player_url', None),
+                                                info_dict.get('page_url', None))

        tmpfilename = self.temp_name(filename)
        stream = None

        # Do not include the Accept-Encoding header
        headers = {'Youtubedl-no-compression': 'True'}
-		basic_request = urllib2.Request(url, None, headers)
-		request = urllib2.Request(url, None, headers)
+        basic_request = compat_urllib_request.Request(url, None, headers)
+        request = compat_urllib_request.Request(url, None, headers)
+
+        if self.params.get('test', False):
+            request.add_header('Range','bytes=0-10240')

        # Establish possible resume length
        if os.path.isfile(encodeFilename(tmpfilename)):
@ -589,9 +633,9 @@ def _do_download(self, filename, info_dict):
            try:
                if count == 0 and 'urlhandle' in info_dict:
                    data = info_dict['urlhandle']
-				data = urllib2.urlopen(request)
+                data = compat_urllib_request.urlopen(request)
                break
-			except (urllib2.HTTPError, ), err:
+            except (compat_urllib_error.HTTPError, ) as err:
                if (err.code < 500 or err.code >= 600) and err.code != 416:
                    # Unexpected HTTP error
                    raise
@ -599,15 +643,15 @@ def _do_download(self, filename, info_dict):
                    # Unable to resume (requested range not satisfiable)
                    try:
                        # Open the connection again without the range header
-						data = urllib2.urlopen(basic_request)
+                        data = compat_urllib_request.urlopen(basic_request)
                        content_length = data.info()['Content-Length']
-					except (urllib2.HTTPError, ), err:
+                    except (compat_urllib_error.HTTPError, ) as err:
                        if err.code < 500 or err.code >= 600:
                            raise
                    else:
                        # Examine the reported length
                        if (content_length is not None and
-								(resume_len - 100 < long(content_length) < resume_len + 100)):
+                                (resume_len - 100 < int(content_length) < resume_len + 100)):
                            # The file had already been fully downloaded.
                            # Explanation to the above condition: in issue #175 it was revealed that
                            # YouTube sometimes adds or removes a few bytes from the end of the file,
@ -634,10 +678,10 @@ def _do_download(self, filename, info_dict):

        data_len = data.info().get('Content-length', None)
        if data_len is not None:
-			data_len = long(data_len) + resume_len
+            data_len = int(data_len) + resume_len
        data_len_str = self.format_bytes(data_len)
        byte_counter = 0 + resume_len
-		block_size = 1024
+        block_size = self.params.get('buffersize', 1024)
        start = time.time()
        while True:
            # Download and write
@ -655,14 +699,15 @@ def _do_download(self, filename, info_dict):
                    assert stream is not None
                    filename = self.undo_temp_name(tmpfilename)
                    self.report_destination(filename)
-				except (OSError, IOError), err:
+                except (OSError, IOError) as err:
                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
                    return False
            try:
                stream.write(data_block)
-			except (IOError, OSError), err:
+            except (IOError, OSError) as err:
                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
                return False
+            if not self.params.get('noresizebuffer', False):
                block_size = self.best_block_size(after - before, len(data_block))

            # Progress message
@ -683,7 +728,7 @@ def _do_download(self, filename, info_dict):
        stream.close()
        self.report_finish()
        if data_len is not None and byte_counter != data_len:
-			raise ContentTooShortError(byte_counter, long(data_len))
+            raise ContentTooShortError(byte_counter, int(data_len))
        self.try_rename(tmpfilename, filename)

        # Update file modification time
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
--- a/youtube_dl/PostProcessor.py
+++ b/youtube_dl/PostProcessor.py
@ -1,12 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-

+from __future__ import absolute_import
+
 import os
 import subprocess
 import sys
 import time

-from utils import *
+from .utils import *


 class PostProcessor(object):
@ -60,13 +62,14 @@ def __init__(self, message):
        self.message = message

 class FFmpegExtractAudioPP(PostProcessor):
-	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False, nopostoverwrites=False):
        PostProcessor.__init__(self, downloader)
        if preferredcodec is None:
            preferredcodec = 'best'
        self._preferredcodec = preferredcodec
        self._preferredquality = preferredquality
        self._keepvideo = keepvideo
+        self._nopostoverwrites = nopostoverwrites
        self._exes = self.detect_executables()

    @staticmethod
@ -84,14 +87,14 @@ def get_audio_codec(self, path):
        if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
        try:
            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
-			handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
            output = handle.communicate()[0]
            if handle.wait() != 0:
                return None
        except (IOError, OSError):
            return None
        audio_codec = None
-		for line in output.split('\n'):
+        for line in output.decode('ascii', 'ignore').split('\n'):
            if line.startswith('codec_name='):
                audio_codec = line.split('=')[1].strip()
            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
@ -169,8 +172,11 @@ def run(self, information):

        prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
        new_path = prefix + sep + extension
-		self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
        try:
+            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
+                self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
+            else:
+                self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
                self.run_ffmpeg(path, new_path, acodec, more_opts)
        except:
            etype,e,tb = sys.exc_info()
--- a/youtube_dl/init.py
+++ b/youtube_dl/init.py
@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-

 from __future__ import with_statement
+from __future__ import absolute_import

 __authors__  = (
    'Ricardo Garcia Gonzalez',
@ -18,17 +19,13 @@
    'Ori Avtalion',
    'shizeeg',
    'Filippo Valsorda',
+    'Christian Albrecht',
+    'Dave Vasilevsky',
+    'Jaime Marquínez Ferrándiz',
    )

 __license__ = 'Public Domain'
-__version__ = '2012.10.09'

-UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
-UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
-UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
-
-
-import cookielib
 import getpass
 import optparse
 import os
@ -37,77 +34,15 @@
 import socket
 import subprocess
 import sys
-import urllib2
 import warnings
+import platform

-from utils import *
-from FileDownloader import *
-from InfoExtractors import *
-from PostProcessor import *
-
-def updateSelf(downloader, filename):
-	''' Update the program file with the latest version from the repository '''
-	# Note: downloader only used for options
-	
-	if not os.access(filename, os.W_OK):
-		sys.exit('ERROR: no write permissions on %s' % filename)
-
-	downloader.to_screen(u'Updating to latest version...')
-
-	urlv = urllib2.urlopen(UPDATE_URL_VERSION)
-	newversion = urlv.read().strip()
-	if newversion == __version__:
-		downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
-		return
-	urlv.close()
-
-	if hasattr(sys, "frozen"): #py2exe
-		exe = os.path.abspath(filename)
-		directory = os.path.dirname(exe)
-		if not os.access(directory, os.W_OK):
-			sys.exit('ERROR: no write permissions on %s' % directory)
-			
-		try:
-			urlh = urllib2.urlopen(UPDATE_URL_EXE)
-			newcontent = urlh.read()
-			urlh.close()
-			with open(exe + '.new', 'wb') as outf:
-				outf.write(newcontent)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to download latest version')
-			
-		try:
-			bat = os.path.join(directory, 'youtube-dl-updater.bat')
-			b = open(bat, 'w')
-			
-			print >> b, """
-echo Updating youtube-dl...
-ping 127.0.0.1 -n 5 -w 1000 > NUL
-move /Y "%s.new" "%s"
-del "%s"
-			""" %(exe, exe, bat)
-			
-			b.close()
-			
-			os.startfile(bat)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to overwrite current version')
-
-	else:
-		try:
-			urlh = urllib2.urlopen(UPDATE_URL)
-			newcontent = urlh.read()
-			urlh.close()
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to download latest version')
-
-		try:
-			with open(filename, 'wb') as outf:
-				outf.write(newcontent)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to overwrite current version')
-
-	downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+from .utils import *
+from .update import update_self
+from .version import __version__
+from .FileDownloader import *
+from .InfoExtractors import gen_extractors
+from .PostProcessor import *

 def parseOpts():
    def _readOptions(filename_bytes):
@ -128,9 +63,12 @@ def _format_option_string(option):

        opts = []

-		if option._short_opts: opts.append(option._short_opts[0])
-		if option._long_opts: opts.append(option._long_opts[0])
-		if len(opts) > 1: opts.insert(1, ', ')
+        if option._short_opts:
+            opts.append(option._short_opts[0])
+        if option._long_opts:
+            opts.append(option._long_opts[0])
+        if len(opts) > 1:
+            opts.insert(1, ', ')

        if option.takes_value(): opts.append(' %s' % option.metavar)

@ -189,6 +127,11 @@ def _find_term_columns():
            dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
    general.add_option('-R', '--retries',
            dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
+    general.add_option('--buffer-size',
+            dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
+    general.add_option('--no-resize-buffer',
+            action='store_true', dest='noresizebuffer',
+            help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
    general.add_option('--dump-user-agent',
            action='store_true', dest='dump_user_agent',
            help='display the current browser identification', default=False)
@ -197,6 +140,7 @@ def _find_term_columns():
    general.add_option('--list-extractors',
            action='store_true', dest='list_extractors',
            help='List all supported extractors and the URLs they would handle', default=False)
+    general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)

    selection.add_option('--playlist-start',
            dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
@ -268,12 +212,15 @@ def _find_term_columns():
    filesystem.add_option('--id',
            action='store_true', dest='useid', help='use video ID in file name', default=False)
    filesystem.add_option('-l', '--literal',
-			action='store_true', dest='useliteral', help='use literal title in file name', default=False)
+            action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
    filesystem.add_option('-A', '--auto-number',
            action='store_true', dest='autonumber',
            help='number downloaded files starting from 00000', default=False)
    filesystem.add_option('-o', '--output',
-			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
+    filesystem.add_option('--restrict-filenames',
+            action='store_true', dest='restrictfilenames',
+            help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
    filesystem.add_option('-a', '--batch-file',
            dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
    filesystem.add_option('-w', '--no-overwrites',
@ -306,6 +253,8 @@ def _find_term_columns():
            help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
    postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
            help='keeps the video file on disk after the post-processing; the video is erased by default')
+    postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
+            help='do not overwrite post-processed files; the post-processed files are overwritten by default')


    parser.add_option_group(general)
@ -326,59 +275,18 @@ def _find_term_columns():

    return parser, opts, args

-def gen_extractors():
-	""" Return a list of an instance of every supported extractor.
-	The order does matter; the first extractor matched is the one handling the URL.
-	"""
-	return [
-		YoutubePlaylistIE(),
-		YoutubeChannelIE(),
-		YoutubeUserIE(),
-		YoutubeSearchIE(),
-		YoutubeIE(),
-		MetacafeIE(),
-		DailymotionIE(),
-		GoogleIE(),
-		GoogleSearchIE(),
-		PhotobucketIE(),
-		YahooIE(),
-		YahooSearchIE(),
-		DepositFilesIE(),
-		FacebookIE(),
-		BlipTVUserIE(),
-		BlipTVIE(),
-		VimeoIE(),
-		MyVideoIE(),
-		ComedyCentralIE(),
-		EscapistIE(),
-		CollegeHumorIE(),
-		XVideosIE(),
-		SoundcloudIE(),
-		InfoQIE(),
-		MixcloudIE(),
-		StanfordOpenClassroomIE(),
-		MTVIE(),
-		YoukuIE(),
-		XNXXIE(),
-		GooglePlusIE(),
-		PornotubeIE(),
-		YouPornIE(),
-		YouJizzIE(),
-		GenericIE()
-	]
-
 def _real_main():
    parser, opts, args = parseOpts()

    # Open appropriate CookieJar
    if opts.cookiefile is None:
-		jar = cookielib.CookieJar()
+        jar = compat_cookiejar.CookieJar()
    else:
        try:
-			jar = cookielib.MozillaCookieJar(opts.cookiefile)
+            jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
            if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
                jar.load()
-		except (IOError, OSError), err:
+        except (IOError, OSError) as err:
            sys.exit(u'ERROR: unable to open cookie file')
    # Set user agent
    if opts.user_agent is not None:
@ -386,7 +294,7 @@ def _real_main():

    # Dump user agent
    if opts.dump_user_agent:
-		print std_headers['User-Agent']
+        print(std_headers['User-Agent'])
        sys.exit(0)

    # Batch file verification
@ -403,22 +311,22 @@ def _real_main():
        except IOError:
            sys.exit(u'ERROR: batch file could not be read')
    all_urls = batchurls + args
-	all_urls = map(lambda url: url.strip(), all_urls)
+    all_urls = [url.strip() for url in all_urls]

    # General configuration
-	cookie_processor = urllib2.HTTPCookieProcessor(jar)
-	proxy_handler = urllib2.ProxyHandler()
-	opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-	urllib2.install_opener(opener)
+    cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+    proxy_handler = compat_urllib_request.ProxyHandler()
+    opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+    compat_urllib_request.install_opener(opener)
    socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)

    extractors = gen_extractors()

    if opts.list_extractors:
        for ie in extractors:
-			print(ie.IE_NAME)
-			matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
-			all_urls = filter(lambda url: url not in matchedUrls, all_urls)
+            print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+            matchedUrls = [url for url in all_urls if ie.suitable(url)]
+            all_urls = [url for url in all_urls if url not in matchedUrls]
            for mu in matchedUrls:
                print(u'  ' + mu)
        sys.exit(0)
@ -428,14 +336,10 @@ def _real_main():
        parser.error(u'using .netrc conflicts with giving username/password')
    if opts.password is not None and opts.username is None:
        parser.error(u'account username missing')
-	if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber or opts.useid):
-		parser.error(u'using output template conflicts with using title, literal title, video ID or auto number')
-	if opts.usetitle and opts.useliteral:
-		parser.error(u'using title conflicts with using literal title')
+    if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
+        parser.error(u'using output template conflicts with using title, video ID or auto number')
    if opts.usetitle and opts.useid:
        parser.error(u'using title conflicts with using video ID')
-	if opts.useliteral and opts.useid:
-		parser.error(u'using literal title conflicts with using video ID')
    if opts.username is not None and opts.password is None:
        opts.password = getpass.getpass(u'Type account password and press return:')
    if opts.ratelimit is not None:
@ -445,20 +349,25 @@ def _real_main():
        opts.ratelimit = numeric_limit
    if opts.retries is not None:
        try:
-			opts.retries = long(opts.retries)
-		except (TypeError, ValueError), err:
+            opts.retries = int(opts.retries)
+        except (TypeError, ValueError) as err:
            parser.error(u'invalid retry count specified')
+    if opts.buffersize is not None:
+        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
+        if numeric_buffersize is None:
+            parser.error(u'invalid buffer size specified')
+        opts.buffersize = numeric_buffersize
    try:
        opts.playliststart = int(opts.playliststart)
        if opts.playliststart <= 0:
            raise ValueError(u'Playlist start must be positive')
-	except (TypeError, ValueError), err:
+    except (TypeError, ValueError) as err:
        parser.error(u'invalid playlist start number specified')
    try:
        opts.playlistend = int(opts.playlistend)
        if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
            raise ValueError(u'Playlist end must be greater than playlist start')
-	except (TypeError, ValueError), err:
+    except (TypeError, ValueError) as err:
        parser.error(u'invalid playlist end number specified')
    if opts.extractaudio:
        if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
@ -468,6 +377,18 @@ def _real_main():
        if not opts.audioquality.isdigit():
            parser.error(u'invalid audio quality specified')

+    if sys.version_info < (3,):
+        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
+        if opts.outtmpl is not None:
+            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
+    outtmpl =((opts.outtmpl is not None and opts.outtmpl)
+            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
+            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
+            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
+            or (opts.useid and u'%(id)s.%(ext)s')
+            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+            or u'%(id)s.%(ext)s')
    # File downloader
    fd = FileDownloader({
        'usenetrc': opts.usenetrc,
@ -485,21 +406,14 @@ def _real_main():
        'format': opts.format,
        'format_limit': opts.format_limit,
        'listformats': opts.listformats,
-		'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
-			or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
-			or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
-			or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-			or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
-			or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-			or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
-			or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
-			or (opts.useid and u'%(id)s.%(ext)s')
-			or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
-			or u'%(id)s.%(ext)s'),
+        'outtmpl': outtmpl,
+        'restrictfilenames': opts.restrictfilenames,
        'ignoreerrors': opts.ignoreerrors,
        'ratelimit': opts.ratelimit,
        'nooverwrites': opts.nooverwrites,
        'retries': opts.retries,
+        'buffersize': opts.buffersize,
+        'noresizebuffer': opts.noresizebuffer,
        'continuedl': opts.continue_dl,
        'noprogress': opts.noprogress,
        'playliststart': opts.playliststart,
@ -517,9 +431,21 @@ def _real_main():
        'max_downloads': opts.max_downloads,
        'prefer_free_formats': opts.prefer_free_formats,
        'verbose': opts.verbose,
+        'test': opts.test,
        })

    if opts.verbose:
+        fd.to_screen(u'[debug] youtube-dl version ' + __version__)
+        try:
+            sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                  cwd=os.path.dirname(os.path.abspath(__file__)))
+            out, err = sp.communicate()
+            out = out.decode().strip()
+            if re.match('[0-9a-f]+', out):
+                fd.to_screen(u'[debug] Git HEAD: ' + out)
+        except:
+            pass
+        fd.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform()))
        fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))

    for extractor in extractors:
@ -527,11 +453,11 @@ def _real_main():

    # PostProcessors
    if opts.extractaudio:
-		fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
+        fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo, nopostoverwrites=opts.nopostoverwrites))

    # Update version
    if opts.update_self:
-		updateSelf(fd, sys.argv[0])
+        update_self(fd.to_screen, opts.verbose, sys.argv[0])

    # Maybe do nothing
    if len(all_urls) < 1:
@ -550,7 +476,7 @@ def _real_main():
    if opts.cookiefile is not None:
        try:
            jar.save()
-		except (IOError, OSError), err:
+        except (IOError, OSError) as err:
            sys.exit(u'ERROR: unable to save cookie jar')

    sys.exit(retcode)
--- a/youtube_dl/main.py
+++ b/youtube_dl/main.py
@ -1,7 +1,17 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-

-import __init__
+# Execute with
+# $ python youtube_dl/__main__.py (2.6+)
+# $ python -m youtube_dl          (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, "frozen"):
+    # direct call of __main__.py
+    import os.path
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl

 if __name__ == '__main__':
-	__init__.main()
+    youtube_dl.main()
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@ -0,0 +1,160 @@
+import json
+import traceback
+import hashlib
+from zipimport import zipimporter
+
+from .utils import *
+from .version import __version__
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+def update_self(to_screen, verbose, filename):
+    """Update the program file with the latest version from the repository"""
+
+    UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+    JSON_URL = UPDATE_URL + 'versions.json'
+    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+        to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.')
+        return
+
+    # Check if there is a new version
+    try:
+        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+        return
+    if newversion == __version__:
+        to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+        return
+
+    # Download and check versions info
+    try:
+        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = json.loads(versions_info)
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
+        return
+    if not 'signature' in versions_info:
+        to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+        return
+    signature = versions_info['signature']
+    del versions_info['signature']
+    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
+        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
+        return
+
+    to_screen(u'Updating to version ' + versions_info['latest'] + '...')
+    version = versions_info['versions'][versions_info['latest']]
+    if version.get('notes'):
+        to_screen(u'PLEASE NOTE:')
+        for note in version['notes']:
+            to_screen(note)
+
+    if not os.access(filename, os.W_OK):
+        to_screen(u'ERROR: no write permissions on %s' % filename)
+        return
+
+    # Py2EXE
+    if hasattr(sys, "frozen"):
+        exe = os.path.abspath(filename)
+        directory = os.path.dirname(exe)
+        if not os.access(directory, os.W_OK):
+            to_screen(u'ERROR: no write permissions on %s' % directory)
+            return
+
+        try:
+            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['exe'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(exe + '.new', 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to write the new version')
+            return
+
+        try:
+            bat = os.path.join(directory, 'youtube-dl-updater.bat')
+            b = open(bat, 'w')
+            b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+            \n""" %(exe, exe, bat))
+            b.close()
+
+            os.startfile(bat)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    # Zip unix package
+    elif isinstance(globals().get('__loader__'), zipimporter):
+        try:
+            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['bin'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(filename, 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@ -2,21 +2,151 @@
 # -*- coding: utf-8 -*-

 import gzip
-import htmlentitydefs
-import HTMLParser
+import io
+import json
 import locale
 import os
 import re
 import sys
+import traceback
 import zlib
-import urllib2
 import email.utils
 import json

 try:
-	import cStringIO as StringIO
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+try:
+    import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_error
+
+try:
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
 except ImportError:
-	import StringIO
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+    def _unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = _unquote(name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = _unquote(value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
+
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr

 std_headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
@ -32,19 +162,35 @@ def preferredencoding():
    Returns the best encoding scheme for the system, based on
    locale.getpreferredencoding() and some further tweaks.
    """
-	def yield_preferredencoding():
    try:
        pref = locale.getpreferredencoding()
        u'TEST'.encode(pref)
    except:
        pref = 'UTF-8'
-		while True:
-			yield pref
-	return yield_preferredencoding().next()

+    return pref
+
+if sys.version_info < (3,0):
+    def compat_print(s):
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
+
+# In Python 2.x, json.dump expects a bytestream.
+# In Python 3.x, it writes to a character stream
+if sys.version_info < (3,0):
+    def write_json_file(obj, fn):
+        with open(fn, 'wb') as f:
+            json.dump(obj, f)
+else:
+    def write_json_file(obj, fn):
+        with open(fn, 'w', encoding='utf-8') as f:
+            json.dump(obj, f)

 def htmlentity_transform(matchobj):
-	"""Transforms an HTML entity to a Unicode character.
+    """Transforms an HTML entity to a character.

    This function receives a match object and is intended to be used with
    the re.sub() function.
@ -52,11 +198,10 @@ def htmlentity_transform(matchobj):
    entity = matchobj.group(1)

    # Known non-numeric HTML entity
-	if entity in htmlentitydefs.name2codepoint:
-		return unichr(htmlentitydefs.name2codepoint[entity])
+    if entity in compat_html_entities.name2codepoint:
+        return compat_chr(compat_html_entities.name2codepoint[entity])

-	# Unicode character
-	mobj = re.match(ur'(?u)#(x?\d+)', entity)
+    mobj = re.match(u'(?u)#(x?\\d+)', entity)
    if mobj is not None:
        numstr = mobj.group(1)
        if numstr.startswith(u'x'):
@ -64,28 +209,28 @@ def htmlentity_transform(matchobj):
            numstr = u'0%s' % numstr
        else:
            base = 10
-		return unichr(long(numstr, base))
+        return compat_chr(int(numstr, base))

    # Unknown entity in name, return its literal representation
    return (u'&%s;' % entity)

-HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class IDParser(HTMLParser.HTMLParser):
-	"""Modified HTMLParser that isolates a tag with the specified id"""
-	def __init__(self, id):
-		self.id = id
+compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+class AttrParser(compat_html_parser.HTMLParser):
+    """Modified HTMLParser that isolates a tag with the specified attribute"""
+    def __init__(self, attribute, value):
+        self.attribute = attribute
+        self.value = value
        self.result = None
        self.started = False
        self.depth = {}
        self.html = None
        self.watch_startpos = False
        self.error_count = 0
-		HTMLParser.HTMLParser.__init__(self)
+        compat_html_parser.HTMLParser.__init__(self)

    def error(self, message):
-		#print >> sys.stderr, self.getpos()
        if self.error_count > 10 or self.started:
-			raise HTMLParser.HTMLParseError(message, self.getpos())
+            raise compat_html_parser.HTMLParseError(message, self.getpos())
        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
        self.error_count += 1
        self.goahead(1)
@ -99,7 +244,7 @@ def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if self.started:
            self.find_startpos(None)
-		if 'id' in attrs and attrs['id'] == self.id:
+        if self.attribute in attrs and attrs[self.attribute] == self.value:
            self.result = [tag]
            self.started = True
            self.watch_startpos = True
@ -124,8 +269,10 @@ def find_startpos(self, x):
    handle_decl = handle_pi = unknown_decl = find_startpos

    def get_result(self):
-		if self.result == None: return None
-		if len(self.result) != 3: return None
+        if self.result is None:
+            return None
+        if len(self.result) != 3:
+            return None
        lines = self.html.split('\n')
        lines = lines[self.result[1][0]-1:self.result[2][0]]
        lines[0] = lines[0][self.result[1][1]:]
@ -135,11 +282,15 @@ def get_result(self):
        return '\n'.join(lines).strip()

 def get_element_by_id(id, html):
-	"""Return the content of the tag with the specified id in the passed HTML document"""
-	parser = IDParser(id)
+    """Return the content of the tag with the specified ID in the passed HTML document"""
+    return get_element_by_attribute("id", id, html)
+
+def get_element_by_attribute(attribute, value, html):
+    """Return the content of the tag with the specified attribute in the passed HTML document"""
+    parser = AttrParser(attribute, value)
    try:
        parser.loads(html)
-	except HTMLParser.HTMLParseError:
+    except compat_html_parser.HTMLParseError:
        pass
    return parser.get_result()

@ -148,7 +299,8 @@ def clean_html(html):
    """Clean an HTML snippet into a readable string"""
    # Newline vs <br />
    html = html.replace('\n', ' ')
-	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
    # Strip html tags
    html = re.sub('<.*?>', '', html)
    # Replace html entities
@ -174,9 +326,9 @@ def sanitize_open(filename, open_mode):
            return (sys.stdout, filename)
        stream = open(encodeFilename(filename), open_mode)
        return (stream, filename)
-	except (IOError, OSError), err:
+    except (IOError, OSError) as err:
        # In case of error, try to remove win32 forbidden chars
-		filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
+        filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)

        # An exception here should be caught in the caller
        stream = open(encodeFilename(filename), open_mode)
@ -191,23 +343,37 @@ def timeconvert(timestr):
        timestamp = email.utils.mktime_tz(timetuple)
    return timestamp

-def sanitize_filename(s):
-	"""Sanitizes a string so it could be used as part of a filename."""
+def sanitize_filename(s, restricted=False, is_id=False):
+    """Sanitizes a string so it could be used as part of a filename.
+    If restricted is set, use a stricter subset of allowed characters.
+    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
+    """
    def replace_insane(char):
        if char == '?' or ord(char) < 32 or ord(char) == 127:
            return ''
        elif char == '"':
-			return '\''
+            return '' if restricted else '\''
        elif char == ':':
-			return ' -'
+            return '_-' if restricted else ' -'
        elif char in '\\/|*<>':
-			return '-'
+            return '_'
+        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
+            return '_'
+        if restricted and ord(char) > 127:
+            return '_'
        return char

    result = u''.join(map(replace_insane, s))
-	while '--' in result:
-		result = result.replace('--', '-')
-	return result.strip('-')
+    if not is_id:
+        while '__' in result:
+            result = result.replace('__', '_')
+        result = result.strip('_')
+        # Common case of "Foreign band name - English song title"
+        if restricted and result.startswith('-_'):
+            result = result[2:]
+        if not result:
+            result = '_'
+    return result

 def orderedSet(iterable):
    """ Remove all duplicates from the input iterable """
@ -219,20 +385,24 @@ def orderedSet(iterable):

 def unescapeHTML(s):
    """
-	@param s a string (of type unicode)
+    @param s a string
    """
    assert type(s) == type(u'')

-	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
+    result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
    return result

 def encodeFilename(s):
    """
-	@param s The name of the file (of type unicode)
+    @param s The name of the file
    """

    assert type(s) == type(u'')

+    # Python 3 has a Unicode API
+    if sys.version_info >= (3, 0):
+        return s
+
    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
@ -241,6 +411,20 @@ def encodeFilename(s):
    else:
        return s.encode(sys.getfilesystemencoding(), 'ignore')

+
+class ExtractorError(Exception):
+    """Error during info extraction."""
+    def __init__(self, msg, tb=None):
+        """ tb, if given, is the original traceback (so that it can be printed out). """
+        super(ExtractorError, self).__init__(msg)
+        self.traceback = tb
+
+    def format_traceback(self):
+        if self.traceback is None:
+            return None
+        return u''.join(traceback.format_tb(self.traceback))
+
+
 class DownloadError(Exception):
    """Download Error exception.

@ -297,15 +481,7 @@ def __init__(self, downloaded, expected):
        self.downloaded = downloaded
        self.expected = expected

-
-class Trouble(Exception):
-	"""Trouble helper exception
-	
-	This is an exception to be handled with
-	FileDownloader.trouble
-	"""
-
-class YoutubeDLHandler(urllib2.HTTPHandler):
+class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
    """Handler for HTTP requests and responses.

    This class, when installed with an OpenerDirector, automatically adds
@ -332,9 +508,9 @@ def deflate(data):

    @staticmethod
    def addinfourl_wrapper(stream, headers, url, code):
-		if hasattr(urllib2.addinfourl, 'getcode'):
-			return urllib2.addinfourl(stream, headers, url, code)
-		ret = urllib2.addinfourl(stream, headers, url)
+        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
+            return compat_urllib_request.addinfourl(stream, headers, url, code)
+        ret = compat_urllib_request.addinfourl(stream, headers, url)
        ret.code = code
        return ret

@ -353,12 +529,15 @@ def http_response(self, req, resp):
        old_resp = resp
        # gzip
        if resp.headers.get('Content-encoding', '') == 'gzip':
-			gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
+            gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
            resp.msg = old_resp.msg
        # deflate
        if resp.headers.get('Content-encoding', '') == 'deflate':
-			gz = StringIO.StringIO(self.deflate(resp.read()))
+            gz = io.BytesIO(self.deflate(resp.read()))
            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
            resp.msg = old_resp.msg
        return resp
+
+    https_request = http_request
+    https_response = http_response
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@ -0,0 +1,2 @@
+
+__version__ = '2013.01.02'