pwordfreq.py

#!/usr/bin/env python
"""Parallel word frequency counter."""


from itertools import repeat
from wordfreq import print_wordfreq, wordfreq

def pwordfreq(rc, text):
    """Parallel word frequency counter.
    
    rc - An IPython RemoteController
    text - The name of a string on the engines to do the freq count on.
    """
    
    rc.execute('freqs = wordfreq(%s)' %text)
    freqs_list = rc.pull('freqs')
    word_set = set()
    for f in freqs_list:
        word_set.update(f.keys())
    freqs = dict(zip(word_set, repeat(0)))
    for f in freqs_list:
        for word, count in f.iteritems():
            freqs[word] += count
    return freqs

if __name__ == '__main__':
    # Create a MultiEngineClient
    from IPython.kernel import client
    ipc = client.MultiEngineClient()
    
    # Run the wordfreq script on the engines.
    ipc.run('wordfreq.py')

    # Run the serial version
    print "Serial word frequency count:"
    text = open('davinci.txt').read()
    freqs = wordfreq(text)
    print_wordfreq(freqs, 10)
    
    # The parallel version
    print "\nParallel word frequency count:"
    files = ['davinci%i.txt' % i for i in range(4)]
    ipc.scatter('textfile', files)
    ipc.execute('text = open(textfile[0]).read()')
    pfreqs = pwordfreq(ipc,'text')
    print_wordfreq(freqs)