From 3a9192a9d9266477e14b5bed496288975557f836 Mon Sep 17 00:00:00 2001
From: Jashandeep Sohi <jashandeep.s.sohi@gmail.com>
Date: Fri, 28 Nov 2014 17:18:17 -0800
Subject: [PATCH] Added support for plugin/local extractors.

Currently, creating new extractors is a bit tedious for non developers.
You have to get the source, edit the package and then build it for your distro
(or possibly push the updates upstream). Although this centralized extractor
approach has it's upsides (e.g. more supported sites for everyone), sometimes
you just want to write an extractor quickly for local use.

I've added a system for writing plugin extractors and have them load at runtime.
Extractor classes are placed in .py files in
"$XDG_CONFIG_HOME/youtube-dl/extractors/" or "~/.config/youtube-dl/extractors".
At runtime these files are compiled and executed in an independent global
namespace. This global namespace is scanned for classes ending in "IE", which
are put in the 'youtube_dl.extractor' global namespace, init-ed and returned.

Plugin extractors are placed before regular extractors so that they have
precedence in case of overwrites.
---
 youtube_dl/__init__.py           | 10 ++++++++--
 youtube_dl/extractor/__init__.py | 23 +++++++++++++++++++++++
 youtube_dl/options.py            | 21 +++++++++++++++++++++
 3 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 77b3384a0..8aa48f1cd 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -38,7 +38,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors
+from .extractor import gen_extractors, gen_plugin_extractors
 from .YoutubeDL import YoutubeDL
 from .postprocessor import (
     AtomicParsleyPP,
@@ -105,7 +105,13 @@ def _real_main(argv=None):
     _enc = preferredencoding()
     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 
-    extractors = gen_extractors()
+    # Load plugin extractors
+    if os.path.isdir(opts.plugin_extractors_dir) and not opts.ignore_plugin_extractors:
+      extractors = gen_plugin_extractors(opts.plugin_extractors_dir)
+    else:
+      extractors = []
+    
+    extractors += gen_extractors()
 
     if opts.list_extractors:
         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9387feef1..ac22d5d2b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -523,6 +523,7 @@ from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
 )
+import os
 
 _ALL_CLASSES = [
     klass
@@ -538,7 +539,29 @@ def gen_extractors():
     """
     return [klass() for klass in _ALL_CLASSES]
 
+def gen_plugin_extractors(plugin_dir, verbosity = False):
+    """ Return a list of an instance of every plugin extractor found in the
+    plugin directory. Scan every .py file in plugin_dir for classes with names
+    ending in 'IE'.
+    """
+    classes = []
+    
+    for file_ in os.listdir(plugin_dir):
+      if not file_.endswith(".py"):
+        continue
+      
+      file_globals = {}
+      exec(open(os.path.join(plugin_dir, file_), "r").read(), file_globals)
+          
+      for name, class_ in file_globals.items():
+        if name.endswith("IE"):
+          #Add the class to this modules globals so that get_info_extractor works
+          globals()[name] = class_
+          classes.append(class_)
+          
+    return [class_() for class_ in classes]
 
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
+
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2e8c71508..cb1531700 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -68,6 +68,18 @@ def parseOpts(overrideArguments=None):
 
         return userConf
 
+    def _get_default_plugin_extractors_dir():
+        """ Return the default plugin extractors directory.
+        If XDG_CONFIG_HOME is set, then the location is XDG_CONFIG_HOME/youtube-dl/extractors,
+        otherwise it is ~/.config/youtube-dl/extractors.
+        """
+        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
+        if xdg_config_home:
+          return os.path.join(xdg_config_home, 'youtube-dl', 'extractors')
+        else:
+          return os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'extractors')
+          
+          
     def _format_option_string(option):
         ''' ('-o', '--option') -> -o, --format METAVAR'''
 
@@ -148,6 +160,15 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
+    general.add_option(
+        '--plugin-extractors-dir', default=_get_default_plugin_extractors_dir(),
+        metavar='PATH',
+        action='store',
+        help='All .py files in this directory are scanned for extractor classes ending in IE and are loaded. (default: %default)')
+    general.add_option(
+        '--ignore-plugin-extractors',
+        action='store_true', default=False,
+        help='Do not load extractors in the plugin directory')
     general.add_option(
         '--proxy', dest='proxy',
         default=None, metavar='URL',