interactiveshell.py

# -*- coding: utf-8 -*-
"""Main IPython class."""

#-----------------------------------------------------------------------------
#  Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
#  Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

from __future__ import absolute_import, print_function

import __future__
import abc
import ast
import atexit
import functools
import os
import re
import runpy
import sys
import tempfile
import traceback
import types
import subprocess
import warnings
from io import open as io_open

from pickleshare import PickleShareDB

from traitlets.config.configurable import SingletonConfigurable
from IPython.core import oinspect
from IPython.core import magic
from IPython.core import page
from IPython.core import prefilter
from IPython.core import shadowns
from IPython.core import ultratb
from IPython.core.alias import Alias, AliasManager
from IPython.core.autocall import ExitAutocall
from IPython.core.builtin_trap import BuiltinTrap
from IPython.core.events import EventManager, available_events
from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
from IPython.core.debugger import Pdb
from IPython.core.display_trap import DisplayTrap
from IPython.core.displayhook import DisplayHook
from IPython.core.displaypub import DisplayPublisher
from IPython.core.error import InputRejected, UsageError
from IPython.core.extensions import ExtensionManager
from IPython.core.formatters import DisplayFormatter
from IPython.core.history import HistoryManager
from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
from IPython.core.logger import Logger
from IPython.core.macro import Macro
from IPython.core.payload import PayloadManager
from IPython.core.prefilter import PrefilterManager
from IPython.core.profiledir import ProfileDir
from IPython.core.usage import default_banner
from IPython.testing.skipdoctest import skip_doctest_py2, skip_doctest
from IPython.utils import PyColorize
from IPython.utils import io
from IPython.utils import py3compat
from IPython.utils import openpy
from IPython.utils.decorators import undoc
from IPython.utils.io import ask_yes_no
from IPython.utils.ipstruct import Struct
from IPython.paths import get_ipython_dir
from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
from IPython.utils.process import system, getoutput
from IPython.utils.py3compat import (builtin_mod, unicode_type, string_types,
                                     with_metaclass, iteritems)
from IPython.utils.strdispatch import StrDispatch
from IPython.utils.syspathcontext import prepended_to_syspath
from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
from IPython.utils.tempdir import TemporaryDirectory
from traitlets import (
    Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
    observe, default,
)
from warnings import warn
from logging import error
import IPython.core.hooks

try:
    import docrepr.sphinxify as sphx

    def sphinxify(doc):
        with TemporaryDirectory() as dirname:
            return {
                'text/html': sphx.sphinxify(doc, dirname),
                'text/plain': doc
            }
except ImportError:
    sphinxify = None


class ProvisionalWarning(DeprecationWarning):
    """
    Warning class for unstable features
    """
    pass

#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------

# compiled regexps for autoindent management
dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')

#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------

@undoc
def softspace(file, newvalue):
    """Copied from code.py, to remove the dependency"""

    oldvalue = 0
    try:
        oldvalue = file.softspace
    except AttributeError:
        pass
    try:
        file.softspace = newvalue
    except (AttributeError, TypeError):
        # "attribute-less object" or "read-only attributes"
        pass
    return oldvalue

@undoc
def no_op(*a, **kw): pass


class SpaceInInput(Exception): pass


def get_default_colors():
    "DEPRECATED"
    warn('get_default_color is Deprecated, and is `Neutral` on all platforms.',
            DeprecationWarning, stacklevel=2)
    return 'Neutral'


class SeparateUnicode(Unicode):
    r"""A Unicode subclass to validate separate_in, separate_out, etc.

    This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
    """

    def validate(self, obj, value):
        if value == '0': value = ''
        value = value.replace('\\n','\n')
        return super(SeparateUnicode, self).validate(obj, value)


@undoc
class DummyMod(object):
    """A dummy module used for IPython's interactive module when
    a namespace must be assigned to the module's __dict__."""
    pass


class ExecutionResult(object):
    """The result of a call to :meth:`InteractiveShell.run_cell`

    Stores information about what took place.
    """
    execution_count = None
    error_before_exec = None
    error_in_exec = None
    result = None

    @property
    def success(self):
        return (self.error_before_exec is None) and (self.error_in_exec is None)

    def raise_error(self):
        """Reraises error if `success` is `False`, otherwise does nothing"""
        if self.error_before_exec is not None:
            raise self.error_before_exec
        if self.error_in_exec is not None:
            raise self.error_in_exec

    def __repr__(self):
        return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s result=%s>' %\
                (self.__class__.__qualname__, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.result))


class InteractiveShell(SingletonConfigurable):
    """An enhanced, interactive shell for Python."""

    _instance = None
    
    ast_transformers = List([], help=
        """
        A list of ast.NodeTransformer subclass instances, which will be applied
        to user input before code is run.
        """
    ).tag(config=True)

    autocall = Enum((0,1,2), default_value=0, help=
        """
        Make IPython automatically call any callable object even if you didn't
        type explicit parentheses. For example, 'str 43' becomes 'str(43)'
        automatically. The value can be '0' to disable the feature, '1' for
        'smart' autocall, where it is not applied if there are no more
        arguments on the line, and '2' for 'full' autocall, where all callable
        objects are automatically called (even if no arguments are present).
        """
    ).tag(config=True)
    # TODO: remove all autoindent logic and put into frontends.
    # We can't do this yet because even runlines uses the autoindent.
    autoindent = Bool(True, help=
        """
        Autoindent IPython code entered interactively.
        """
    ).tag(config=True)

    automagic = Bool(True, help=
        """
        Enable magic commands to be called without the leading %.
        """
    ).tag(config=True)
    
    banner1 = Unicode(default_banner,
        help="""The part of the banner to be printed before the profile"""
    ).tag(config=True)
    banner2 = Unicode('',
        help="""The part of the banner to be printed after the profile"""
    ).tag(config=True)

    cache_size = Integer(1000, help=
        """
        Set the size of the output cache.  The default is 1000, you can
        change it permanently in your config file.  Setting it to 0 completely
        disables the caching system, and the minimum value accepted is 20 (if
        you provide a value less than 20, it is reset to 0 and a warning is
        issued).  This limit is defined because otherwise you'll spend more
        time re-flushing a too small cache than working
        """
    ).tag(config=True)
    color_info = Bool(True, help=
        """
        Use colors for displaying information about objects. Because this
        information is passed through a pager (like 'less'), and some pagers
        get confused with color codes, this capability can be turned off.
        """
    ).tag(config=True)
    colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
                             default_value='Neutral',
        help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
    ).tag(config=True)
    debug = Bool(False).tag(config=True)
    deep_reload = Bool(False, help=
        """
        **Deprecated**

        Will be removed in IPython 6.0

        Enable deep (recursive) reloading by default. IPython can use the
        deep_reload module which reloads changes in modules recursively (it
        replaces the reload() function, so you don't need to change anything to
        use it). `deep_reload` forces a full reload of modules whose code may
        have changed, which the default reload() function does not.  When
        deep_reload is off, IPython will use the normal reload(), but
        deep_reload will still be available as dreload().
        """
    ).tag(config=True)
    disable_failing_post_execute = Bool(False,
        help="Don't call post-execute functions that have failed in the past."
    ).tag(config=True)
    display_formatter = Instance(DisplayFormatter, allow_none=True)
    displayhook_class = Type(DisplayHook)
    display_pub_class = Type(DisplayPublisher)
    
    sphinxify_docstring = Bool(False, help=
        """
        Enables rich html representation of docstrings. (This requires the
        docrepr module).
        """).tag(config=True)

    @observe("sphinxify_docstring")
    def _sphinxify_docstring_changed(self, change):
        if change['new']:
            warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)

    enable_html_pager = Bool(False, help=
        """
        (Provisional API) enables html representation in mime bundles sent
        to pagers.
        """).tag(config=True)

    @observe("enable_html_pager")
    def _enable_html_pager_changed(self, change):
        if change['new']:
            warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)

    data_pub_class = None

    exit_now = Bool(False)
    exiter = Instance(ExitAutocall)
    @default('exiter')
    def _exiter_default(self):
        return ExitAutocall(self)
    # Monotonically increasing execution counter
    execution_count = Integer(1)
    filename = Unicode("<ipython console>")
    ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__

    # Input splitter, to transform input line by line and detect when a block
    # is ready to be executed.
    input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                              (), {'line_input_checker': True})

    # This InputSplitter instance is used to transform completed cells before
    # running them. It allows cell magics to contain blank lines.
    input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                                         (), {'line_input_checker': False})

    logstart = Bool(False, help=
        """
        Start logging to the default log file in overwrite mode.
        Use `logappend` to specify a log file to **append** logs to.
        """
    ).tag(config=True)
    logfile = Unicode('', help=
        """
        The name of the logfile to use.
        """
    ).tag(config=True)
    logappend = Unicode('', help=
        """
        Start logging to the given file in append mode.
        Use `logfile` to specify a log file to **overwrite** logs to.
        """
    ).tag(config=True)
    object_info_string_level = Enum((0,1,2), default_value=0,
    ).tag(config=True)
    pdb = Bool(False, help=
        """
        Automatically call the pdb debugger after every exception.
        """
    ).tag(config=True)
    display_page = Bool(False,
        help="""If True, anything that would be passed to the pager
        will be displayed as regular output instead."""
    ).tag(config=True)

    # deprecated prompt traits:
    
    prompt_in1 = Unicode('In [\\#]: ',
        help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
    ).tag(config=True)
    prompt_in2 = Unicode('   .\\D.: ',
        help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
    ).tag(config=True)
    prompt_out = Unicode('Out[\\#]: ',
        help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
    ).tag(config=True)
    prompts_pad_left = Bool(True,
        help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
    ).tag(config=True)
    
    @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
    def _prompt_trait_changed(self, change):
        name = change['name']
        warn("InteractiveShell.{name} is deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly.".format(
                name=name)
        )
        # protect against weird cases where self.config may not exist:

    show_rewritten_input = Bool(True,
        help="Show rewritten input, e.g. for autocall."
    ).tag(config=True)

    quiet = Bool(False).tag(config=True)

    history_length = Integer(10000,
        help='Total length of command history'
    ).tag(config=True)

    history_load_length = Integer(1000, help=
        """
        The number of saved history entries to be loaded
        into the history buffer at startup.
        """
    ).tag(config=True)

    ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
                                  default_value='last_expr',
                                  help="""
        'all', 'last', 'last_expr' or 'none', specifying which nodes should be
        run interactively (displaying output from expressions)."""
    ).tag(config=True)

    # TODO: this part of prompt management should be moved to the frontends.
    # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
    separate_in = SeparateUnicode('\n').tag(config=True)
    separate_out = SeparateUnicode('').tag(config=True)
    separate_out2 = SeparateUnicode('').tag(config=True)
    wildcards_case_sensitive = Bool(True).tag(config=True)
    xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
                            default_value='Context').tag(config=True)

    # Subcomponents of InteractiveShell
    alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
    prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
    builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
    display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
    extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
    payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
    history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
    magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)

    profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
    @property
    def profile(self):
        if self.profile_dir is not None:
            name = os.path.basename(self.profile_dir.location)
            return name.replace('profile_','')


    # Private interface
    _post_execute = Dict()

    # Tracks any GUI loop loaded for pylab
    pylab_gui_select = None

    last_execution_succeeded = Bool(True, help='Did last executed command succeeded')

    def __init__(self, ipython_dir=None, profile_dir=None,
                 user_module=None, user_ns=None,
                 custom_exceptions=((), None), **kwargs):

        # This is where traits with a config_key argument are updated
        # from the values on config.
        super(InteractiveShell, self).__init__(**kwargs)
        if 'PromptManager' in self.config:
            warn('As of IPython 5.0 `PromptManager` config will have no effect'
                 ' and has been replaced by TerminalInteractiveShell.prompts_class')
        self.configurables = [self]

        # These are relatively independent and stateless
        self.init_ipython_dir(ipython_dir)
        self.init_profile_dir(profile_dir)
        self.init_instance_attrs()
        self.init_environment()
        
        # Check if we're in a virtualenv, and set up sys.path.
        self.init_virtualenv()

        # Create namespaces (user_ns, user_global_ns, etc.)
        self.init_create_namespaces(user_module, user_ns)
        # This has to be done after init_create_namespaces because it uses
        # something in self.user_ns, but before init_sys_modules, which
        # is the first thing to modify sys.
        # TODO: When we override sys.stdout and sys.stderr before this class
        # is created, we are saving the overridden ones here. Not sure if this
        # is what we want to do.
        self.save_sys_module_state()
        self.init_sys_modules()

        # While we're trying to have each part of the code directly access what
        # it needs without keeping redundant references to objects, we have too
        # much legacy code that expects ip.db to exist.
        self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))

        self.init_history()
        self.init_encoding()
        self.init_prefilter()

        self.init_syntax_highlighting()
        self.init_hooks()
        self.init_events()
        self.init_pushd_popd_magic()
        # self.init_traceback_handlers use to be here, but we moved it below
        # because it and init_io have to come after init_readline.
        self.init_user_ns()
        self.init_logger()
        self.init_builtins()

        # The following was in post_config_initialization
        self.init_inspector()
        # init_readline() must come before init_io(), because init_io uses
        # readline related things.
        self.init_readline()
        # We save this here in case user code replaces raw_input, but it needs
        # to be after init_readline(), because PyPy's readline works by replacing
        # raw_input.
        if py3compat.PY3:
            self.raw_input_original = input
        else:
            self.raw_input_original = raw_input
        # init_completer must come after init_readline, because it needs to
        # know whether readline is present or not system-wide to configure the
        # completers, since the completion machinery can now operate
        # independently of readline (e.g. over the network)
        self.init_completer()
        # TODO: init_io() needs to happen before init_traceback handlers
        # because the traceback handlers hardcode the stdout/stderr streams.
        # This logic in in debugger.Pdb and should eventually be changed.
        self.init_io()
        self.init_traceback_handlers(custom_exceptions)
        self.init_prompts()
        self.init_display_formatter()
        self.init_display_pub()
        self.init_data_pub()
        self.init_displayhook()
        self.init_magics()
        self.init_alias()
        self.init_logstart()
        self.init_pdb()
        self.init_extension_manager()
        self.init_payload()
        self.init_deprecation_warnings()
        self.hooks.late_startup_hook()
        self.events.trigger('shell_initialized', self)
        atexit.register(self.atexit_operations)

    def get_ipython(self):
        """Return the currently running IPython instance."""
        return self

    #-------------------------------------------------------------------------
    # Trait changed handlers
    #-------------------------------------------------------------------------
    @observe('ipython_dir')
    def _ipython_dir_changed(self, change):
        ensure_dir_exists(change['new'])

    def set_autoindent(self,value=None):
        """Set the autoindent flag.

        If called with no arguments, it acts as a toggle."""
        if value is None:
            self.autoindent = not self.autoindent
        else:
            self.autoindent = value

    #-------------------------------------------------------------------------
    # init_* methods called by __init__
    #-------------------------------------------------------------------------

    def init_ipython_dir(self, ipython_dir):
        if ipython_dir is not None:
            self.ipython_dir = ipython_dir
            return

        self.ipython_dir = get_ipython_dir()

    def init_profile_dir(self, profile_dir):
        if profile_dir is not None:
            self.profile_dir = profile_dir
            return
        self.profile_dir =\
            ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')

    def init_instance_attrs(self):
        self.more = False

        # command compiler
        self.compile = CachingCompiler()

        # Make an empty namespace, which extension writers can rely on both
        # existing and NEVER being used by ipython itself.  This gives them a
        # convenient location for storing additional information and state
        # their extensions may require, without fear of collisions with other
        # ipython names that may develop later.
        self.meta = Struct()

        # Temporary files used for various purposes.  Deleted at exit.
        self.tempfiles = []
        self.tempdirs = []

        # keep track of where we started running (mainly for crash post-mortem)
        # This is not being used anywhere currently.
        self.starting_dir = py3compat.getcwd()

        # Indentation management
        self.indent_current_nsp = 0

        # Dict to track post-execution functions that have been registered
        self._post_execute = {}

    def init_environment(self):
        """Any changes we need to make to the user's environment."""
        pass

    def init_encoding(self):
        # Get system encoding at startup time.  Certain terminals (like Emacs
        # under Win32 have it set to None, and we need to have a known valid
        # encoding to use in the raw_input() method
        try:
            self.stdin_encoding = sys.stdin.encoding or 'ascii'
        except AttributeError:
            self.stdin_encoding = 'ascii'

    def init_syntax_highlighting(self):
        # Python source parser/formatter for syntax highlighting
        pyformat = PyColorize.Parser().format
        self.pycolorize = lambda src: pyformat(src,'str',self.colors)

    def refresh_style(self):
        # No-op here, used in subclass
        pass

    def init_pushd_popd_magic(self):
        # for pushd/popd management
        self.home_dir = get_home_dir()

        self.dir_stack = []

    def init_logger(self):
        self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                             logmode='rotate')

    def init_logstart(self):
        """Initialize logging in case it was requested at the command line.
        """
        if self.logappend:
            self.magic('logstart %s append' % self.logappend)
        elif self.logfile:
            self.magic('logstart %s' % self.logfile)
        elif self.logstart:
            self.magic('logstart')

    def init_deprecation_warnings(self):
        """
        register default filter for deprecation warning.

        This will allow deprecation warning of function used interactively to show
        warning to users, and still hide deprecation warning from libraries import.
        """
        warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))

    def init_builtins(self):
        # A single, static flag that we set to True.  Its presence indicates
        # that an IPython shell has been created, and we make no attempts at
        # removing on exit or representing the existence of more than one
        # IPython at a time.
        builtin_mod.__dict__['__IPYTHON__'] = True

        self.builtin_trap = BuiltinTrap(shell=self)

    def init_inspector(self):
        # Object inspector
        self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                            PyColorize.ANSICodeColors,
                                            'NoColor',
                                            self.object_info_string_level)

    def init_io(self):
        # This will just use sys.stdout and sys.stderr. If you want to
        # override sys.stdout and sys.stderr themselves, you need to do that
        # *before* instantiating this class, because io holds onto
        # references to the underlying streams.
        io.stdout = io.IOStream(sys.stdout)
        io.stderr = io.IOStream(sys.stderr)

    def init_prompts(self):
        # Set system prompts, so that scripts can decide if they are running
        # interactively.
        sys.ps1 = 'In : '
        sys.ps2 = '...: '
        sys.ps3 = 'Out: '

    def init_display_formatter(self):
        self.display_formatter = DisplayFormatter(parent=self)
        self.configurables.append(self.display_formatter)

    def init_display_pub(self):
        self.display_pub = self.display_pub_class(parent=self)
        self.configurables.append(self.display_pub)

    def init_data_pub(self):
        if not self.data_pub_class:
            self.data_pub = None
            return
        self.data_pub = self.data_pub_class(parent=self)
        self.configurables.append(self.data_pub)

    def init_displayhook(self):
        # Initialize displayhook, set in/out prompts and printing system
        self.displayhook = self.displayhook_class(
            parent=self,
            shell=self,
            cache_size=self.cache_size,
        )
        self.configurables.append(self.displayhook)
        # This is a context manager that installs/revmoes the displayhook at
        # the appropriate time.
        self.display_trap = DisplayTrap(hook=self.displayhook)

    def init_virtualenv(self):
        """Add a virtualenv to sys.path so the user can import modules from it.
        This isn't perfect: it doesn't use the Python interpreter with which the
        virtualenv was built, and it ignores the --no-site-packages option. A
        warning will appear suggesting the user installs IPython in the
        virtualenv, but for many cases, it probably works well enough.
        
        Adapted from code snippets online.
        
        http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
        """
        if 'VIRTUAL_ENV' not in os.environ:
            # Not in a virtualenv
            return
        
        # venv detection:
        # stdlib venv may symlink sys.executable, so we can't use realpath.
        # but others can symlink *to* the venv Python, so we can't just use sys.executable.
        # So we just check every item in the symlink tree (generally <= 3)
        p = os.path.normcase(sys.executable)
        paths = [p]
        while os.path.islink(p):
            p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
            paths.append(p)
        p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
        if any(p.startswith(p_venv) for p in paths):
            # Running properly in the virtualenv, don't need to do anything
            return
        
        warn("Attempting to work in a virtualenv. If you encounter problems, please "
             "install IPython inside the virtualenv.")
        if sys.platform == "win32":
            virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages') 
        else:
            virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
                       'python%d.%d' % sys.version_info[:2], 'site-packages')
        
        import site
        sys.path.insert(0, virtual_env)
        site.addsitedir(virtual_env)

    #-------------------------------------------------------------------------
    # Things related to injections into the sys module
    #-------------------------------------------------------------------------

    def save_sys_module_state(self):
        """Save the state of hooks in the sys module.

        This has to be called after self.user_module is created.
        """
        self._orig_sys_module_state = {'stdin': sys.stdin,
                                       'stdout': sys.stdout,
                                       'stderr': sys.stderr,
                                       'excepthook': sys.excepthook}
        self._orig_sys_modules_main_name = self.user_module.__name__
        self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)

    def restore_sys_module_state(self):
        """Restore the state of the sys module."""
        try:
            for k, v in iteritems(self._orig_sys_module_state):
                setattr(sys, k, v)
        except AttributeError:
            pass
        # Reset what what done in self.init_sys_modules
        if self._orig_sys_modules_main_mod is not None:
            sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod

    #-------------------------------------------------------------------------
    # Things related to the banner
    #-------------------------------------------------------------------------
    
    @property
    def banner(self):
        banner = self.banner1
        if self.profile and self.profile != 'default':
            banner += '\nIPython profile: %s\n' % self.profile
        if self.banner2:
            banner += '\n' + self.banner2
        return banner

    def show_banner(self, banner=None):
        if banner is None:
            banner = self.banner
        sys.stdout.write(banner)
    
    #-------------------------------------------------------------------------
    # Things related to hooks
    #-------------------------------------------------------------------------

    def init_hooks(self):
        # hooks holds pointers used for user-side customizations
        self.hooks = Struct()

        self.strdispatchers = {}

        # Set all default hooks, defined in the IPython.hooks module.
        hooks = IPython.core.hooks
        for hook_name in hooks.__all__:
            # default hooks have priority 100, i.e. low; user hooks should have
            # 0-100 priority
            self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
        
        if self.display_page:
            self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
    
    def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
                 _warn_deprecated=True):
        """set_hook(name,hook) -> sets an internal IPython hook.

        IPython exposes some of its internal API as user-modifiable hooks.  By
        adding your function to one of these hooks, you can modify IPython's
        behavior to call at runtime your own routines."""

        # At some point in the future, this should validate the hook before it
        # accepts it.  Probably at least check that the hook takes the number
        # of args it's supposed to.

        f = types.MethodType(hook,self)

        # check if the hook is for strdispatcher first
        if str_key is not None:
            sdp = self.strdispatchers.get(name, StrDispatch())
            sdp.add_s(str_key, f, priority )
            self.strdispatchers[name] = sdp
            return
        if re_key is not None:
            sdp = self.strdispatchers.get(name, StrDispatch())
            sdp.add_re(re.compile(re_key), f, priority )
            self.strdispatchers[name] = sdp
            return

        dp = getattr(self.hooks, name, None)
        if name not in IPython.core.hooks.__all__:
            print("Warning! Hook '%s' is not one of %s" % \
                  (name, IPython.core.hooks.__all__ ))

        if _warn_deprecated and (name in IPython.core.hooks.deprecated):
            alternative = IPython.core.hooks.deprecated[name]
            warn("Hook {} is deprecated. Use {} instead.".format(name, alternative))

        if not dp:
            dp = IPython.core.hooks.CommandChainDispatcher()

        try:
            dp.add(f,priority)
        except AttributeError:
            # it was not commandchain, plain old func - replace
            dp = f

        setattr(self.hooks,name, dp)

    #-------------------------------------------------------------------------
    # Things related to events
    #-------------------------------------------------------------------------

    def init_events(self):
        self.events = EventManager(self, available_events)

        self.events.register("pre_execute", self._clear_warning_registry)

    def register_post_execute(self, func):
        """DEPRECATED: Use ip.events.register('post_run_cell', func)
        
        Register a function for calling after code execution.
        """
        warn("ip.register_post_execute is deprecated, use "
             "ip.events.register('post_run_cell', func) instead.")
        self.events.register('post_run_cell', func)
    
    def _clear_warning_registry(self):
        # clear the warning registry, so that different code blocks with
        # overlapping line number ranges don't cause spurious suppression of
        # warnings (see gh-6611 for details)
        if "__warningregistry__" in self.user_global_ns:
            del self.user_global_ns["__warningregistry__"]

    #-------------------------------------------------------------------------
    # Things related to the "main" module
    #-------------------------------------------------------------------------

    def new_main_mod(self, filename, modname):
        """Return a new 'main' module object for user code execution.
        
        ``filename`` should be the path of the script which will be run in the
        module. Requests with the same filename will get the same module, with
        its namespace cleared.
        
        ``modname`` should be the module name - normally either '__main__' or
        the basename of the file without the extension.
        
        When scripts are executed via %run, we must keep a reference to their
        __main__ module around so that Python doesn't
        clear it, rendering references to module globals useless.

        This method keeps said reference in a private dict, keyed by the
        absolute path of the script. This way, for multiple executions of the
        same script we only keep one copy of the namespace (the last one),
        thus preventing memory leaks from old references while allowing the
        objects from the last execution to be accessible.
        """
        filename = os.path.abspath(filename)
        try:
            main_mod = self._main_mod_cache[filename]
        except KeyError:
            main_mod = self._main_mod_cache[filename] = types.ModuleType(
                        py3compat.cast_bytes_py2(modname),
                        doc="Module created for script run in IPython")
        else:
            main_mod.__dict__.clear()
            main_mod.__name__ = modname
        
        main_mod.__file__ = filename
        # It seems pydoc (and perhaps others) needs any module instance to
        # implement a __nonzero__ method
        main_mod.__nonzero__ = lambda : True
        
        return main_mod

    def clear_main_mod_cache(self):
        """Clear the cache of main modules.

        Mainly for use by utilities like %reset.

        Examples
        --------

        In [15]: import IPython

        In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')

        In [17]: len(_ip._main_mod_cache) > 0
        Out[17]: True

        In [18]: _ip.clear_main_mod_cache()

        In [19]: len(_ip._main_mod_cache) == 0
        Out[19]: True
        """
        self._main_mod_cache.clear()

    #-------------------------------------------------------------------------
    # Things related to debugging
    #-------------------------------------------------------------------------

    def init_pdb(self):
        # Set calling of pdb on exceptions
        # self.call_pdb is a property
        self.call_pdb = self.pdb

    def _get_call_pdb(self):
        return self._call_pdb

    def _set_call_pdb(self,val):

        if val not in (0,1,False,True):
            raise ValueError('new call_pdb value must be boolean')

        # store value in instance
        self._call_pdb = val

        # notify the actual exception handlers
        self.InteractiveTB.call_pdb = val

    call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                        'Control auto-activation of pdb at exceptions')

    def debugger(self,force=False):
        """Call the pdb debugger.

        Keywords:

          - force(False): by default, this routine checks the instance call_pdb
            flag and does not actually invoke the debugger if the flag is false.
            The 'force' option forces the debugger to activate even if the flag
            is false.
        """

        if not (force or self.call_pdb):
            return

        if not hasattr(sys,'last_traceback'):
            error('No traceback has been produced, nothing to debug.')
            return

        self.InteractiveTB.debugger(force=True)

    #-------------------------------------------------------------------------
    # Things related to IPython's various namespaces
    #-------------------------------------------------------------------------
    default_user_namespaces = True

    def init_create_namespaces(self, user_module=None, user_ns=None):
        # Create the namespace where the user will operate.  user_ns is
        # normally the only one used, and it is passed to the exec calls as
        # the locals argument.  But we do carry a user_global_ns namespace
        # given as the exec 'globals' argument,  This is useful in embedding
        # situations where the ipython shell opens in a context where the
        # distinction between locals and globals is meaningful.  For
        # non-embedded contexts, it is just the same object as the user_ns dict.

        # FIXME. For some strange reason, __builtins__ is showing up at user
        # level as a dict instead of a module. This is a manual fix, but I
        # should really track down where the problem is coming from. Alex
        # Schmolck reported this problem first.