Python Text Analysis

import operator

import time

import string

import re

from collections import defaultdict

filepath = '/Users/JoJo/Desktop/Projects/650/Week 6/'

f = open(filepath + 'beauty&thebeast.txt', 'rb')

start = time.time()

beautybeast = defaultdict(int)

punc = string.punctuation

for line in f:
    cln_line = re.sub('[' + punc + ']', '', line.decode('utf-8'))
    spl_line = cln_line.split()
    for word in spl_line:
        lower_word = word.lower()

beautybeast.setdefault(lower_word, 0)

0

beautybeast[lower_word] += 1

sorted_beautybeast = sorted(beautybeast.items(), key=operator.itemgetter(1), reverse=True)

elapsed = time.time() - start

print('Run took', elapsed, ' seconds.')

Run took 507.92300391197205  seconds.

print('Number of distinct words:', len(sorted_beautybeast))

Number of distinct words: 1

top_n = 10

for pair in range(top_n):
    print(sorted_beautybeast[pair])

('end', 1)



---------------------------------------------------------------------------

IndexError                                Traceback (most recent call last)

<ipython-input-72-c3fcfe71def9> in <module>()
      1 for pair in range(top_n):
----> 2     print(sorted_beautybeast[pair])


IndexError: list index out of range

filepath = '/Users/JoJo/Desktop/Projects/650/Week 6/'

f = open(filepath + 'peterpan.txt', 'rb')

start = time.time()

peterpan = defaultdict(int)

punc = string.punctuation

for line in f:
    cln_line = re.sub('[' + punc + ']', '', line.decode('utf-8'))
    spl_line = cln_line.split()
    for word in spl_line:
        lower_word = word.lower()
        if lower_word in peterpan:
            peterpan[lower_word] += 1
        else:
            peterpan[lower_word] = 1

sorted_peterpan = sorted(peterpan.items(), key=operator.itemgetter(1), reverse=True)

elapsed = time.time() - start

print('Run took', elapsed, ' seconds.')

Run took 1085.910615682602  seconds.

print('Number of distinct words: ', len(sorted_peterpan))

Number of distinct words:  1

top_n = 10

y[]

  File "<ipython-input-91-2b8b5195ff99>", line 1
    y[]
      ^
SyntaxError: invalid syntax