帮我加快这段代码的速度-Python
|
伙计们,我正在编写此程序,该程序会处理一系列推文并返回使用最多的单词。
我想使其更快,但是我想知道您是否可以帮助指出一些我可以提高速度的问题或领域。
谢谢
见下面的代码
#import string
import re
from string import punctuation
from operator import itemgetter
import pprint
class Tweet:
def __init__(self, timestamp, userId, message):
self.timestamp = timestamp
self.userId = userId
self.message = message
def getDate(self):
tokens = re.split(\' \', self.timestamp)
return tokens[0]
def __repr__(self):
return \"[timestamp=%s userId=%s message=%s]\" % (self.timestamp, self.userId, self.message)
outfile = file
def readOneTweet(file):
\"\"\" Reads a single tweet from the file, and returns the string containing the tweet.
This will often just be a single line from the file, but may be more if it ends with a slash.
\"\"\"
lineBuffer = \"\"
while True:
# TODO: read the line and strip it
rawLine = file.readline().strip(\'\\n\')
if (len(rawLine)== 0):
break
lineBuffer +=rawLine
if (rawLine[(len(rawLine)-1)]!= \"\\\\\"):
break
return lineBuffer
def readTweets():
tweets = []
inputfile = raw_input(\"Enter filename: \")
# move the try / except around a single tweet.
# so that we can keep going if we encounter a line with an error.
try:
f = open(inputfile , \"r\")
while True:
tweet = readOneTweet(f) # readOneTweet is method
if not tweet:
break
try:
lineStrip = tweet.rstrip()
split_word = re.split(\'\\t\', lineStrip.lower()) #(\'/([^a-z])([A-Z]) ([0-9])/n:.;\\]+/\', line.lower())
tweetTime = split_word[1]
userId = split_word[0]
message = split_word[2]
tweets.append(Tweet(tweetTime, userId, message))
if len(tweets) % 10000 == 0:
print \'read\', len(tweets), \'tweets\'
except IndexError, e:
print \"bad tweet\", tweet
except IOError:
print \"file not found!\"
return tweets
######################DATA ##############
\"\"\"
- Need to separate tweets
- Obtain information about each tweet - UserID, Time, words
\"\"\"
def writeWordFile(word):
toWrite = \'test.txt\'
fileHandle = open ( toWrite, \'w\' )
for i in word:
fileHandle.write (i)
def dailyMessages(twt):
dailyMsg =dict ()
for i in twt:
date =i.getDate()
#print i.message
#dailyMsg[date] =messageList
if dailyMsg.has_key(date):
dailyMsg[date].append(twt)
else:
dailyMsg[date] =[twt]
#for k, v in dailyMsg.items():
#print k, v, \'\\n\'
return dailyMsg
\"\"\"
Takes dailyTweets and perform word coun.
\"\"\"
def dailyWord(tweetsByDay):
dailyTweetsWordCount = { }
for date in tweetsByDay.keys():
dayTweets =tweetsByDay[date]
if len(dayTweets) != 0:
count = wordCount(dayTweets)
dailyTweetsWordCount[date] = count
return dailyTweetsWordCount
def wordCount(tweets):
\"\"\"Takes a list of tweets and returns a dictionary of counts for words\"\"\"
N = 100
# we\'ll build a dictionary mapping each word to a SET of users who have used it
wordTweeters = {}
for tweet in tweets:
# print tweet
for i in tweet:
for word in i.message.split():
if not wordTweeters.has_key(word):
wordTweeters[word] = set()
wordTweeters[word].add(i.userId)
# we\'ll build up a dictionary mapping each word to the number of users who have used it.
p = dict ()
#print wordTweeters
for day in wordTweeters.keys():
usersNo = len (wordTweeters[day])
p[day] = usersNo
#print wordTweeters
return p #wordTweeters, p
def searchForMemes(tweetUserCounts):
for key in tweetsUserCounts.keys():
# for pmeme in tweetUserCounts
pass
\"\"\"Takes information returned by daily word\"\"\"
def isMeme(word, day1Count, day2Count, day3Count):
#takes the daily count
# check if it is a meme
#First - check count
#check count in different days
# determine the if it qualifies as a tweet
# if not drop it do not do below checks
#Second - time stamp
#CHECK ITS TIME TRACK
#How is the count of over time
# rise and fall
#
#Third - user id
# check if is form different users
#how many of those counts are from different users
pass
def dayUserCount(z,word, d1, d2, d3):
\"\"\" assume dictionary will be input\"\"\"
# check if the word exist in the dictionary
if z.has_key(d1):
date1 =z[d1]
#print value.keys()
if date1.has_key(word):
print date1
c1 =date1[word]
else:
print \"word not used in %s\"%d1
c1 =0
else:
print \'date does not exist\'
if z.has_key(d2):
#print value.keys()
date2 =z[d2]
if date2.has_key(word):
print date2
c2 =date2[word]
else:
print \"word not used in %s\"%d2
c2 =0
else:
print \'date does not exist\'
if z.has_key(d3):
date3 = z[d3]
if date3.has_key(word):
print date3
c3 =date3[word]
else:
print \"word not used in %s\" %d3
c3 =0
else:
print \'date does not exist\'
result = \"Word: %s , %s count: %s, %s count: %s, %s count: %s\"%(word,d1,c1,d2,c2, d3,c3)
return result
# supportive functions
def hashtag(tw):
hasht =[]
for word in tw.split():
if word.startswith(\'#\'):
hasht.append(word)
return hasht
def httpTag(tw):
http =[]
for word in tw.split():
if word.startswith(\'http\'):
http.append(word)
return http
def reply(tw):
reply =[]
for word in tw.split():
if word.startswith(\'@\'):
reply.append(word)
return reply
def reTweet(tw):
rt =[]
for word in tw.split():
if word.startswith(\'rt\') or word.startswith(\'RT\'):
rt.append(word)
return rt
\"\"\"
Old functions
\"\"\"
def writeToFile(tweet):
#filename = test.txt
filek = open(\'test.txt\', \'w\')
print \"writing on the file: \"
filek.write(tweet)
# print \" _____--------______\" + tweet
filek.close()
# count word frequency.
def f2count():
N = 100000000000
words = {}
words_gen = (word.strip(punctuation).lower()
for line in open(\'c.txt\')
for word in line.split())
for word in words_gen:
words[word] = words.get(word, 0) + 1
top_words = sorted(words.iteritems(),
key=lambda(word, count): (-count, word))[:N]
for word, frequency in top_words:
print \"%s: %d\" % (word, frequency)
没有找到相关结果
已邀请:
3 个回复
穗揣
可以写成
切勿使用
作为索引,而应使用
。 我不知道为什么你只用6英镑就用5英镑。 不要使用
,请使用
。 当您遍历
时,您确实应该这样做:
这样,您不必手动将值绑定到键。 那仅仅是一个开始。还有更多的问题需要解决。我认为您真的只需要精通Python即可-从阅读代码中可以清楚地了解到,要么Python不是您的第一语言,要么您是从没有教您如何编写好它的资源中学到的。例如,为什么要在条件语句两边加上括号?在Python中这不是必需的(尽管这是来自类似Algol的语言(如C或Java)的工件。为什么用
代替
?最好用第二种方式写一个空的字典。您可能会发现有关惯用Python的本教程很有帮助。
藕挝
可以并联运行。由于每个推文都不直接依赖于另一个推文,因此没有理由依次遍历该列表。将推文列表分成较小的列表,然后在每个子列表上创建一个线程。他们的子词典创建完成后,您可以做一些工作将它们全部合并为一个词典。 编辑: 有关如何使列表求和并行化的示例。您将更改线程的主体以执行任何任务。
你换
3-我很想回答新注册的人员,这些人员在stackoverflow上遇到了沉重的问题,然后又消失了,有时甚至不发表任何新闻或发表评论。打扰一下,如果你不想这么做的话 编辑 writeToFile f2count 必须从清单中删除。