统计单词出现频率
#http://stackoverflow.com/questions/4215472/python-take-max-n-elements-from-some-listimport re, collectionsimport heapqignore_words = ['the','and','to','in','a','that','he','was','it','his','of', 'is', 'with', 'as', 'i', 'had', 'for', 'at', 'by', 'on','not', 'be', 'from', 'but', 's', 'you', 'or', 'her', 'him', 'which']def words(text): return re.findall('[a-z]+', text.lower())def train(features): model = collections.defaultdict(lambda: 1) for f in features: model[f] += 1 return modelimport timestarttime = time.time()f = file('/duitang/data/nltk_data/big.txt').read()endtime = time.time()exe_time = (endtime - starttime)*1000print 'read',exe_timestarttime = time.time()f = words(f)endtime = time.time()exe_time = (endtime - starttime)*1000print 're',exe_timestarttime = time.time()f = train(f)endtime = time.time()exe_time = (endtime - starttime)*1000print 'dict',exe_timestarttime = time.time()max_list=heapq.nlargest(40,f,key=f.get)nmax_list = []for m in max_list: if m in ignore_words: continue nmax_list.append(m)print nmax_listendtime = time.time()exe_time = (endtime - starttime)*1000print 'sort',exe_time
?
