From 37e3cbe22e0bfa6b98a6343be88e1c8c2c7ac41f Mon Sep 17 00:00:00 2001
From: anovicecodemonkey <anovicecodemonkey13435@mailinator.com>
Date: Sun, 1 Jun 2014 01:16:35 +0930
Subject: [PATCH] Move duplicate check to generic.py

---
 youtube_dl/extractor/common.py  | 10 ----------
 youtube_dl/extractor/generic.py | 24 ++++++++++++++++++++++++
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 26dd9882f..db472aace 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -343,16 +343,6 @@ def url_result(url, ie=None, video_id=None):
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None):
         """Returns a playlist"""
-        # Ensure we don't have any duplicates in the playlist
-        seen = set()
-        new_list = []
-        for url in entries:
-            theurl = tuple(url.items())
-            if theurl not in seen:
-             seen.add(theurl)
-             new_list.append(url)
-             entries = new_list
-
         video_info = {'_type': 'playlist',
                       'entries': entries}
         if playlist_id:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c1e533821..dfa8d6153 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -494,6 +494,14 @@ def _real_extract(self, url):
         if matches:
             urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
                      for tuppl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -503,6 +511,14 @@ def _real_extract(self, url):
         if matches:
             urlrs = [self.url_result(unescapeHTML(tuppl[1]))
                      for tuppl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -615,6 +631,14 @@ def _real_extract(self, url):
         if matches:
             urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
                      for eurl in matches]
+            # First, ensure we have a duplicate free list of entries
+            seen = set()
+            new_list = []
+            theurl = tuple(url.items())
+            if theurl not in seen:
+                seen.add(theurl)
+                new_list.append(url)
+                urlrs = new_list
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)