multienginetest.py

# encoding: utf-8

""""""

__docformat__ = "restructuredtext en"

#-------------------------------------------------------------------------------
#  Copyright (C) 2008  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------------

from twisted.internet import defer

from IPython.kernel import engineservice as es
from IPython.kernel import multiengine as me
from IPython.kernel import newserialized
from IPython.testing import util
from IPython.testing.parametric import parametric, Parametric
from IPython.kernel import newserialized
from IPython.kernel.util import printer
from IPython.kernel.error import (InvalidEngineID,
    NoEnginesRegistered,
    CompositeError,
    InvalidDeferredID)
from IPython.kernel.tests.engineservicetest import validCommands, invalidCommands
from IPython.kernel.core.interpreter import Interpreter


#-------------------------------------------------------------------------------
# Base classes and utilities
#-------------------------------------------------------------------------------

class IMultiEngineBaseTestCase(object):
    """Basic utilities for working with multiengine tests.

    Some subclass should define:

    * self.multiengine
    * self.engines to keep track of engines for clean up"""

    def createShell(self):
        return Interpreter()

    def addEngine(self, n=1):
        for i in range(n):
            e = es.EngineService()
            e.startService()
            regDict = self.controller.register_engine(es.QueuedEngine(e), None)
            e.id = regDict['id']
            self.engines.append(e)


def testf(x):
    return 2.0*x


globala = 99


def testg(x):
    return  globala*x


def isdid(did):
    if not isinstance(did, str):
        return False
    if not len(did)==40:
        return False
    return True


def _raise_it(f):
    try:
        f.raiseException()
    except CompositeError, e:
        e.raise_exception()

#-------------------------------------------------------------------------------
# IMultiEngineTestCase
#-------------------------------------------------------------------------------

class IMultiEngineTestCase(IMultiEngineBaseTestCase):
    """A test for any object that implements IEngineMultiplexer.

    self.multiengine must be defined and implement IEngineMultiplexer.
    """

    def testIMultiEngineInterface(self):
        """Does self.engine claim to implement IEngineCore?"""
        self.assert_(me.IEngineMultiplexer.providedBy(self.multiengine))
        self.assert_(me.IMultiEngine.providedBy(self.multiengine))

    def testIEngineMultiplexerInterfaceMethods(self):
        """Does self.engine have the methods and attributes in IEngineCore."""
        for m in list(me.IEngineMultiplexer):
            self.assert_(hasattr(self.multiengine, m))

    def testIEngineMultiplexerDeferreds(self):
        self.addEngine(1)
        d= self.multiengine.execute('a=5', targets=0)
        d.addCallback(lambda _: self.multiengine.push(dict(a=5),targets=0))
        d.addCallback(lambda _: self.multiengine.push(dict(a=5, b='asdf', c=[1,2,3]),targets=0))
        d.addCallback(lambda _: self.multiengine.pull(('a','b','c'),targets=0))
        d.addCallback(lambda _: self.multiengine.get_result(targets=0))
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.keys(targets=0))
        d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)),targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a',targets=0))
        d.addCallback(lambda _: self.multiengine.clear_queue(targets=0))
        d.addCallback(lambda _: self.multiengine.queue_status(targets=0))
        return d

    def testInvalidEngineID(self):
         self.addEngine(1)
         badID = 100
         d = self.multiengine.execute('a=5', targets=badID)
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.reset(targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.keys(targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
         d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
         return d

    def testNoEnginesRegistered(self):
        badID = 'all'
        d= self.multiengine.execute('a=5', targets=badID)
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.get_result(targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.reset(targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.keys(targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
        d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
        return d

    def runExecuteAll(self, d, cmd, shell):
        actual = shell.execute(cmd)
        d.addCallback(lambda _: self.multiengine.execute(cmd))
        def compare(result):
            for r in result:
                actual['id'] = r['id']
                self.assertEquals(r, actual)
        d.addCallback(compare)

    def testExecuteAll(self):
        self.addEngine(4)
        d= defer.Deferred()
        shell = Interpreter()
        for cmd in validCommands:
                self.runExecuteAll(d, cmd, shell)
        d.callback(None)
        return d

    # The following two methods show how to do parametrized
    # tests.  This is really slick!  Same is used above.
    def runExecuteFailures(self, cmd, exc):
        self.addEngine(4)
        d= self.multiengine.execute(cmd)
        d.addErrback(lambda f: self.assertRaises(exc, _raise_it, f))
        return d

    @parametric
    def testExecuteFailuresMultiEng(cls):
        return [(cls.runExecuteFailures,cmd,exc) for
                cmd,exc in invalidCommands]

    def testPushPull(self):
        self.addEngine(1)
        objs = [10,"hi there",1.2342354,{"p":(1,2)}]
        d= self.multiengine.push(dict(key=objs[0]), targets=0)
        d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
        d.addCallback(lambda r: self.assertEquals(r, [objs[0]]))
        d.addCallback(lambda _: self.multiengine.push(dict(key=objs[1]), targets=0))
        d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
        d.addCallback(lambda r: self.assertEquals(r, [objs[1]]))
        d.addCallback(lambda _: self.multiengine.push(dict(key=objs[2]), targets=0))
        d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
        d.addCallback(lambda r: self.assertEquals(r, [objs[2]]))
        d.addCallback(lambda _: self.multiengine.push(dict(key=objs[3]), targets=0))
        d.addCallback(lambda _: self.multiengine.pull('key', targets=0))
        d.addCallback(lambda r: self.assertEquals(r, [objs[3]]))
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.pull('a', targets=0))
        d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
        d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=20)))
        d.addCallback(lambda _: self.multiengine.pull(('a','b')))
        d.addCallback(lambda r: self.assertEquals(r, [[10,20]]))
        return d

    def testPushPullAll(self):
        self.addEngine(4)
        d= self.multiengine.push(dict(a=10))
        d.addCallback(lambda _: self.multiengine.pull('a'))
        d.addCallback(lambda r: self.assert_(r==[10,10,10,10]))
        d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20)))
        d.addCallback(lambda _: self.multiengine.pull(('a','b')))
        d.addCallback(lambda r: self.assert_(r==4*[[10,20]]))
        d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20), targets=0))
        d.addCallback(lambda _: self.multiengine.pull(('a','b'), targets=0))
        d.addCallback(lambda r: self.assert_(r==[[10,20]]))
        d.addCallback(lambda _: self.multiengine.push(dict(a=None, b=None), targets=0))
        d.addCallback(lambda _: self.multiengine.pull(('a','b'), targets=0))
        d.addCallback(lambda r: self.assert_(r==[[None,None]]))
        return d

    def testPushPullSerialized(self):
        self.addEngine(1)
        objs = [10,"hi there",1.2342354,{"p":(1,2)}]
        d= self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[0])), targets=0)
        d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, objs[0]))
        d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[1])), targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, objs[1]))
        d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[2])), targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, objs[2]))
        d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[3])), targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, objs[3]))
        d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=range(5)), targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized(('a','b'), targets=0))
        d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
        d.addCallback(lambda r: self.assertEquals(r, [10, range(5)]))
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
        d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
        return d

        objs = [10,"hi there",1.2342354,{"p":(1,2)}]
        d= defer.succeed(None)
        for o in objs:
            self.multiengine.push_serialized(0, key=newserialized.serialize(o))
            value = self.multiengine.pull_serialized(0, 'key')
            value.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
            d = self.assertDeferredEquals(value,o,d)
        return d

    def runGetResultAll(self, d, cmd, shell):
        actual = shell.execute(cmd)
        d.addCallback(lambda _: self.multiengine.execute(cmd))
        d.addCallback(lambda _: self.multiengine.get_result())
        def compare(result):
            for r in result:
                actual['id'] = r['id']
                self.assertEquals(r, actual)
        d.addCallback(compare)

    def testGetResultAll(self):
        self.addEngine(4)
        d= defer.Deferred()
        shell = Interpreter()
        for cmd in validCommands:
                self.runGetResultAll(d, cmd, shell)
        d.callback(None)
        return d

    def testGetResultDefault(self):
        self.addEngine(1)
        target = 0
        cmd = 'a=5'
        shell = self.createShell()
        shellResult = shell.execute(cmd)
        def popit(dikt, key):
            dikt.pop(key)
            return dikt
        d= self.multiengine.execute(cmd, targets=target)
        d.addCallback(lambda _: self.multiengine.get_result(targets=target))
        d.addCallback(lambda r: self.assertEquals(shellResult, popit(r[0],'id')))
        return d

    def testGetResultFailure(self):
        self.addEngine(1)
        d= self.multiengine.get_result(None, targets=0)
        d.addErrback(lambda f: self.assertRaises(IndexError, _raise_it, f))
        d.addCallback(lambda _: self.multiengine.get_result(10, targets=0))
        d.addErrback(lambda f: self.assertRaises(IndexError, _raise_it, f))
        return d

    def testPushFunction(self):
        self.addEngine(1)
        d= self.multiengine.push_function(dict(f=testf), targets=0)
        d.addCallback(lambda _: self.multiengine.execute('result = f(10)', targets=0))
        d.addCallback(lambda _: self.multiengine.pull('result', targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0], testf(10)))
        d.addCallback(lambda _: self.multiengine.push(dict(globala=globala), targets=0))
        d.addCallback(lambda _: self.multiengine.push_function(dict(g=testg), targets=0))
        d.addCallback(lambda _: self.multiengine.execute('result = g(10)', targets=0))
        d.addCallback(lambda _: self.multiengine.pull('result', targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0], testg(10)))
        return d

    def testPullFunction(self):
        self.addEngine(1)
        d= self.multiengine.push(dict(a=globala), targets=0)
        d.addCallback(lambda _: self.multiengine.push_function(dict(f=testf), targets=0))
        d.addCallback(lambda _: self.multiengine.pull_function('f', targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0](10), testf(10)))
        d.addCallback(lambda _: self.multiengine.execute("def g(x): return x*x", targets=0))
        d.addCallback(lambda _: self.multiengine.pull_function(('f','g'),targets=0))
        d.addCallback(lambda r: self.assertEquals((r[0][0](10),r[0][1](10)), (testf(10), 100)))
        return d

    def testPushFunctionAll(self):
        self.addEngine(4)
        d= self.multiengine.push_function(dict(f=testf))
        d.addCallback(lambda _: self.multiengine.execute('result = f(10)'))
        d.addCallback(lambda _: self.multiengine.pull('result'))
        d.addCallback(lambda r: self.assertEquals(r, 4*[testf(10)]))
        d.addCallback(lambda _: self.multiengine.push(dict(globala=globala)))
        d.addCallback(lambda _: self.multiengine.push_function(dict(testg=testg)))
        d.addCallback(lambda _: self.multiengine.execute('result = testg(10)'))
        d.addCallback(lambda _: self.multiengine.pull('result'))
        d.addCallback(lambda r: self.assertEquals(r, 4*[testg(10)]))
        return d

    def testPullFunctionAll(self):
        self.addEngine(4)
        d= self.multiengine.push_function(dict(f=testf))
        d.addCallback(lambda _: self.multiengine.pull_function('f'))
        d.addCallback(lambda r: self.assertEquals([func(10) for func in r], 4*[testf(10)]))
        return d

    def testGetIDs(self):
        self.addEngine(1)
        d= self.multiengine.get_ids()
        d.addCallback(lambda r: self.assertEquals(r, [0]))
        d.addCallback(lambda _: self.addEngine(3))
        d.addCallback(lambda _: self.multiengine.get_ids())
        d.addCallback(lambda r: self.assertEquals(r, [0,1,2,3]))
        return d

    def testClearQueue(self):
        self.addEngine(4)
        d= self.multiengine.clear_queue()
        d.addCallback(lambda r: self.assertEquals(r,4*[None]))
        return d

    def testQueueStatus(self):
        self.addEngine(4)
        d= self.multiengine.queue_status(targets=0)
        d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
        return d

    def testGetSetProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.get_properties())
        d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
        d.addCallback(lambda r: self.multiengine.get_properties(('c',)))
        d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
        d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False)))
        d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))
        return d

    def testClearProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.clear_properties())
        d.addCallback(lambda r: self.multiengine.get_properties())
        d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))
        return d

    def testDelHasProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.del_properties(('b','e')))
        d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))
        return d

Parametric(IMultiEngineTestCase)

#-------------------------------------------------------------------------------
# ISynchronousMultiEngineTestCase
#-------------------------------------------------------------------------------

class ISynchronousMultiEngineTestCase(IMultiEngineBaseTestCase):

    def testISynchronousMultiEngineInterface(self):
        """Does self.engine claim to implement IEngineCore?"""
        self.assert_(me.ISynchronousEngineMultiplexer.providedBy(self.multiengine))
        self.assert_(me.ISynchronousMultiEngine.providedBy(self.multiengine))

    def testExecute(self):
        self.addEngine(4)
        execute = self.multiengine.execute
        d= execute('a=5', targets=0, block=True)
        d.addCallback(lambda r: self.assert_(len(r)==1))
        d.addCallback(lambda _: execute('b=10'))
        d.addCallback(lambda r: self.assert_(len(r)==4))
        d.addCallback(lambda _: execute('c=30', block=False))
        d.addCallback(lambda did: self.assert_(isdid(did)))
        d.addCallback(lambda _: execute('d=[0,1,2]', block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assert_(len(r)==4))
        return d

    def testPushPull(self):
        data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
        self.addEngine(4)
        push = self.multiengine.push
        pull = self.multiengine.pull
        d= push({'data':data}, targets=0)
        d.addCallback(lambda r: pull('data', targets=0))
        d.addCallback(lambda r: self.assertEqual(r,[data]))
        d.addCallback(lambda _: push({'data':data}))
        d.addCallback(lambda r: pull('data'))
        d.addCallback(lambda r: self.assertEqual(r,4*[data]))
        d.addCallback(lambda _: push({'data':data}, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda _: pull('data', block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEqual(r,4*[data]))
        d.addCallback(lambda _: push(dict(a=10,b=20)))
        d.addCallback(lambda _: pull(('a','b')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[10,20]]))
        return d

    def testPushPullFunction(self):
        self.addEngine(4)
        pushf = self.multiengine.push_function
        pullf = self.multiengine.pull_function
        push = self.multiengine.push
        pull = self.multiengine.pull
        execute = self.multiengine.execute
        d= pushf({'testf':testf}, targets=0)
        d.addCallback(lambda r: pullf('testf', targets=0))
        d.addCallback(lambda r: self.assertEqual(r[0](1.0), testf(1.0)))
        d.addCallback(lambda _: execute('r = testf(10)', targets=0))
        d.addCallback(lambda _: pull('r', targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0], testf(10)))
        d.addCallback(lambda _: pushf({'testf':testf}, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda _: pullf('testf', block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEqual(r[0](1.0), testf(1.0)))
        d.addCallback(lambda _: execute("def g(x): return x*x", targets=0))
        d.addCallback(lambda _: pullf(('testf','g'),targets=0))
        d.addCallback(lambda r: self.assertEquals((r[0][0](10),r[0][1](10)), (testf(10), 100)))
        return d

    def testGetResult(self):
        shell = Interpreter()
        result1 = shell.execute('a=10')
        result1['id'] = 0
        result2 = shell.execute('b=20')
        result2['id'] = 0
        execute= self.multiengine.execute
        get_result = self.multiengine.get_result
        self.addEngine(1)
        d= execute('a=10')
        d.addCallback(lambda _: get_result())
        d.addCallback(lambda r: self.assertEquals(r[0], result1))
        d.addCallback(lambda _: execute('b=20'))
        d.addCallback(lambda _: get_result(1))
        d.addCallback(lambda r: self.assertEquals(r[0], result1))
        d.addCallback(lambda _: get_result(2, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r[0], result2))
        return d

    def testResetAndKeys(self):
        self.addEngine(1)

        #Blocking mode
        d= self.multiengine.push(dict(a=10, b=20, c=range(10)), targets=0)
        d.addCallback(lambda _: self.multiengine.keys(targets=0))
        def keys_found(keys):
            self.assert_('a' in keys[0])
            self.assert_('b' in keys[0])
            self.assert_('b' in keys[0])
        d.addCallback(keys_found)
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.keys(targets=0))
        def keys_not_found(keys):
            self.assert_('a' not in keys[0])
            self.assert_('b' not in keys[0])
            self.assert_('b' not in keys[0])
        d.addCallback(keys_not_found)

        #Non-blocking mode
        d.addCallback(lambda _: self.multiengine.push(dict(a=10, b=20, c=range(10)), targets=0))
        d.addCallback(lambda _: self.multiengine.keys(targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        def keys_found(keys):
            self.assert_('a' in keys[0])
            self.assert_('b' in keys[0])
            self.assert_('b' in keys[0])
        d.addCallback(keys_found)
        d.addCallback(lambda _: self.multiengine.reset(targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda _: self.multiengine.keys(targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        def keys_not_found(keys):
            self.assert_('a' not in keys[0])
            self.assert_('b' not in keys[0])
            self.assert_('b' not in keys[0])
        d.addCallback(keys_not_found)

        return d

    def testPushPullSerialized(self):
        self.addEngine(1)
        dikt = dict(a=10,b='hi there',c=1.2345,d={'p':(1,2)})
        sdikt = {}
        for k,v in dikt.iteritems():
            sdikt[k] = newserialized.serialize(v)
        d= self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0)
        d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
        d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0))
        d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
        d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
        d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))

        #Non-blocking mode
        d.addCallback(lambda r: self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
        d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
        d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
        d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
        d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
        d.addCallback(lambda _: self.multiengine.reset(targets=0))
        d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
        return d

    def testClearQueue(self):
        self.addEngine(4)
        d= self.multiengine.clear_queue()
        d.addCallback(lambda r: self.multiengine.clear_queue(block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r,4*[None]))
        return d

    def testQueueStatus(self):
        self.addEngine(4)
        d= self.multiengine.queue_status(targets=0)
        d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
        d.addCallback(lambda r: self.multiengine.queue_status(targets=0, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assert_(isinstance(r[0],tuple)))
        return d

    def testGetIDs(self):
        self.addEngine(1)
        d= self.multiengine.get_ids()
        d.addCallback(lambda r: self.assertEquals(r, [0]))
        d.addCallback(lambda _: self.addEngine(3))
        d.addCallback(lambda _: self.multiengine.get_ids())
        d.addCallback(lambda r: self.assertEquals(r, [0,1,2,3]))
        return d

    def testGetSetProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.get_properties())
        d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
        d.addCallback(lambda r: self.multiengine.get_properties(('c',)))
        d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
        d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False)))
        d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))

        #Non-blocking
        d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.get_properties(block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, 4*[dikt]))
        d.addCallback(lambda r: self.multiengine.get_properties(('c',), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}]))
        d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd'), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)]))
        return d

    def testClearProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.clear_properties())
        d.addCallback(lambda r: self.multiengine.get_properties())
        d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))

        #Non-blocking
        d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.clear_properties(block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.get_properties(block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, 4*[{}]))
        return d

    def testDelHasProperties(self):
        self.addEngine(4)
        dikt = dict(a=5, b='asdf', c=True, d=None, e=range(5))
        d= self.multiengine.set_properties(dikt)
        d.addCallback(lambda r: self.multiengine.del_properties(('b','e')))
        d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))

        #Non-blocking
        d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.del_properties(('b','e'), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.has_properties(('a','b','c','d','e'), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[True, False, True, True, False]]))
        return d

    def test_clear_pending_deferreds(self):
        self.addEngine(4)
        did_list = []
        d= self.multiengine.execute('a=10',block=False)
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.push(dict(b=10),block=False))
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.pull(('a','b'),block=False))
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.clear_pending_deferreds())
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[0],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[1],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[2],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        return d

#-------------------------------------------------------------------------------
# Coordinator test cases
#-------------------------------------------------------------------------------

class IMultiEngineCoordinatorTestCase(object):

    def testScatterGather(self):
        self.addEngine(4)
        d= self.multiengine.scatter('a', range(16))
        d.addCallback(lambda r: self.multiengine.gather('a'))
        d.addCallback(lambda r: self.assertEquals(r, range(16)))
        d.addCallback(lambda _: self.multiengine.gather('asdf'))
        d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
        return d

    def testScatterGatherNumpy(self):
        try:
            import numpy
            from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
        except:
            return
        else:
            self.addEngine(4)
            a = numpy.arange(16)
            d = self.multiengine.scatter('a', a)
            d.addCallback(lambda r: self.multiengine.gather('a'))
            d.addCallback(lambda r: assert_array_equal(r, a))
            return d

    def testMap(self):
        self.addEngine(4)
        def f(x):
            return x**2
        data = range(16)
        d= self.multiengine.map(f, data)
        d.addCallback(lambda r: self.assertEquals(r,[f(x) for x in data]))
        return d


class ISynchronousMultiEngineCoordinatorTestCase(IMultiEngineCoordinatorTestCase):

    def testScatterGatherNonblocking(self):
        self.addEngine(4)
        d= self.multiengine.scatter('a', range(16), block=False)
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.gather('a', block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r, range(16)))
        return d
    
    def testScatterGatherNumpyNonblocking(self):
        try:
            import numpy
            from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
        except:
            return
        else:
            self.addEngine(4)
            a = numpy.arange(16)
            d = self.multiengine.scatter('a', a, block=False)
            d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
            d.addCallback(lambda r: self.multiengine.gather('a', block=False))
            d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
            d.addCallback(lambda r: assert_array_equal(r, a))
            return d
    
    def test_clear_pending_deferreds(self):
        self.addEngine(4)
        did_list = []
        d= self.multiengine.scatter('a',range(16),block=False)
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.gather('a',block=False))
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.map(lambda x: x, range(16),block=False))
        d.addCallback(lambda did: did_list.append(did))
        d.addCallback(lambda _: self.multiengine.clear_pending_deferreds())
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[0],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[1],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        d.addCallback(lambda _: self.multiengine.get_pending_deferred(did_list[2],True))
        d.addErrback(lambda f: self.assertRaises(InvalidDeferredID, f.raiseException))
        return d

#-------------------------------------------------------------------------------
# Extras test cases
#-------------------------------------------------------------------------------

class IMultiEngineExtrasTestCase(object):

    def testZipPull(self):
        self.addEngine(4)
        d= self.multiengine.push(dict(a=10,b=20))
        d.addCallback(lambda r: self.multiengine.zip_pull(('a','b')))
        d.addCallback(lambda r: self.assert_(r, [4*[10],4*[20]]))
        return d

    def testRun(self):
        self.addEngine(4)
        import tempfile
        fname = tempfile.mktemp('foo.py')
        f= open(fname, 'w')
        f.write('a = 10\nb=30')
        f.close()
        d= self.multiengine.run(fname)
        d.addCallback(lambda r: self.multiengine.pull(('a','b')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[10,30]]))
        return d


class ISynchronousMultiEngineExtrasTestCase(IMultiEngineExtrasTestCase):

    def testZipPullNonblocking(self):
        self.addEngine(4)
        d= self.multiengine.push(dict(a=10,b=20))
        d.addCallback(lambda r: self.multiengine.zip_pull(('a','b'), block=False))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assert_(r, [4*[10],4*[20]]))
        return d

    def testRunNonblocking(self):
        self.addEngine(4)
        import tempfile
        fname = tempfile.mktemp('foo.py')
        f= open(fname, 'w')
        f.write('a = 10\nb=30')
        f.close()
        d= self.multiengine.run(fname, block=False)
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.multiengine.pull(('a','b')))
        d.addCallback(lambda r: self.assertEquals(r, 4*[[10,30]]))
        return d


#-------------------------------------------------------------------------------
# IFullSynchronousMultiEngineTestCase
#-------------------------------------------------------------------------------

class IFullSynchronousMultiEngineTestCase(ISynchronousMultiEngineTestCase,
    ISynchronousMultiEngineCoordinatorTestCase,
    ISynchronousMultiEngineExtrasTestCase):
    pass