From f7c1e3455ddddcd15767df452d3c4d616bcd65eb Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 20 Jul 2015 08:56:33 -0500
Subject: [PATCH] --fallback-generic: try to download with generic on failure

* does not retry if generic fails, or if a url from generic fails
  (to prevent silly generic->embed->fail->generic->embed->fail->shenanigans)

* 'failed' extra_info propagates to playlist entries, to prevent
  the above as well

* extra_info re-initialized for every argument in .download()
  (it seems to persist to the next argument)

* 'arg_url' extra_info added, to store the requested url:

    youtube-dl http://www.telegraph.co.uk/news/newstopics/howaboutthat/11750410/Clever-dog-uses-boat-to-fetch-ball-without-getting-wet.html \
    --custom-meta 'description=\n arg_url=%(arg_url)s\n webpage_url=%(webpage_url)s'

    arg_url=http://www.telegraph.co.uk/news/newstopics/howaboutthat/11750410/Clever-dog-uses-boat-to-fetch-ball-without-getting-wet.html
    webpage_url=http://player.ooyala.com/player.js?embedCode=VpMjlidjp_25xhKrv4mmLPgL4hRArBZG
---
 youtube_dl/YoutubeDL.py | 18 +++++++++++++++---
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 00af78e06..2d1b53e16 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -140,6 +140,7 @@ class YoutubeDL(object):
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
     force_generic_extractor: Force downloader to use the generic extractor
+    fallback_generic:  Try again with generic extractor
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
@@ -644,6 +645,8 @@ class YoutubeDL(object):
         else:
             ies = self._ies
 
+        fallback_ok = self.params.get('fallback_generic', False) and extra_info.get('extractor_key') != 'Generic' and not extra_info.get('failed')
+
         for ie in ies:
             if not ie.suitable(url):
                 continue
@@ -668,8 +671,13 @@ class YoutubeDL(object):
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
-                self.report_error(compat_str(de), de.format_traceback())
-                break
+                if fallback_ok and ie.ie_key() != 'Generic':
+                    self.report_warning('[' + ie.IE_NAME + '] ' + compat_str(de))
+                    extra_info['failed'] = True  # extra precaution against multiple fallbacks
+                    continue  # should continue to generic
+                else:
+                    self.report_error(compat_str(de), de.format_traceback())
+                    break
             except MaxDownloadsReached:
                 raise
             except Exception as e:
@@ -816,6 +824,8 @@ class YoutubeDL(object):
                     'webpage_url': ie_result['webpage_url'],
                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
                     'extractor_key': ie_result['extractor_key'],
+                    'arg_url': extra_info.get('arg_url', ie_result['webpage_url']),
+                    'failed': extra_info.get('failed'),
                 }
 
                 reason = self._match_entry(entry, incomplete=True)
@@ -842,6 +852,8 @@ class YoutubeDL(object):
                         'webpage_url': ie_result['webpage_url'],
                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
                         'extractor_key': ie_result['extractor_key'],
+                        'arg_url': extra_info.get('arg_url', ie_result['webpage_url']),
+                        'failed': extra_info.get('failed'),
                     }
                 )
                 return r
@@ -1500,7 +1512,7 @@ class YoutubeDL(object):
             try:
                 # It also downloads the videos
                 res = self.extract_info(
-                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
+                    url, force_generic_extractor=self.params.get('force_generic_extractor', False), extra_info={'arg_url': url, 'failed': None})
             except UnavailableVideoError:
                 self.report_error('unable to download video')
             except MaxDownloadsReached:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 55b22c889..819b36e97 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -297,6 +297,7 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'force_generic_extractor': opts.force_generic_extractor,
+        'fallback_generic': opts.fallback_generic,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
         'retries': opts_retries,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 85365d769..e3894c683 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -154,6 +154,10 @@ def parseOpts(overrideArguments=None):
         '--force-generic-extractor',
         action='store_true', dest='force_generic_extractor', default=False,
         help='Force extraction to use the generic extractor')
+    general.add_option(
+        '--fallback-generic',
+        action='store_true', dest='fallback_generic', default=False,
+        help='Try the generic extractor if a site-specific extractor fails')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',