library.py 6.65 KB
Newer Older
1
# -*- coding: utf-8 -*-
Janez K's avatar
Janez K committed
2
3
4
5
6
7
'''
Streaming widgets librarby

@author: Janez Kranjc <janez.kranjc@ijs.si>
'''

Janez K's avatar
Janez K committed
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
def streaming_collect_and_display_tweets(input_dict,widget,stream=None):
    from streams.models import StreamWidgetData
    if stream is None:
        return {}
    else:
        try:
            swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
            data = swd.value
        except Exception as e:
            swd = StreamWidgetData()
            swd.stream = stream
            swd.widget = widget
            data = []
            swd.value = data
            swd.save()
        swd.value = input_dict['ltw']+swd.value
        swd.save()
        return {}

Janez K's avatar
Janez K committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
def streaming_sentiment_graph(input_dict,widget,stream=None):
    from streams.models import StreamWidgetData
    if stream is None:
        return {}
    else:
        try:
            swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
            data = swd.value
        except Exception as e:
            swd = StreamWidgetData()
            swd.stream = stream
            swd.widget = widget
            data = []
            swd.value = data
            swd.save()
        swd.value = input_dict['ltw']+swd.value
        swd.save()
        return {}

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
def streaming_tweet_sentiment_service(input_dict,widget,stream=None):
    import pickle
    from pysimplesoap.client import SoapClient, SoapFault

    client = SoapClient(location = "http://batman.ijs.si:8008/",action = 'http://batman.ijs.si:8008/',namespace = "http://example.com/tweetsentiment.wsdl",soap_ns='soap',trace = False,ns = False)

    list_of_tweets = input_dict['ltw']

    new_list_of_tweets = []

    for tweet in list_of_tweets:
        new_list_of_tweets.append({'id':tweet['id'],'text':tweet['text'],'language':tweet['lang']})

    pickled_list_of_tweets = pickle.dumps(new_list_of_tweets)

    response = client.TweetSentimentService(tweets=pickled_list_of_tweets)

    new_ltw = pickle.loads(unicode(response.TweetSentimentResult))

    i=0
    for new_tweet in new_ltw:
        list_of_tweets[i]['sentiment']=new_tweet['sentiment']
        list_of_tweets[i]['lang']=new_tweet['language']
        list_of_tweets[i]['reliability']=new_tweet['reliability']
        i = i + 1

Janez K's avatar
Janez K committed
72
    output_dict = {}
73
74
75

    output_dict['ltw'] = list_of_tweets

Janez K's avatar
Janez K committed
76
77
    return output_dict

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def streaming_twitter(input_dict,widget,stream=None):
    import tweepy
    from streams.models import StreamWidgetData
    from streams.models import HaltStream

    if input_dict['cfauth']=="true":
        consumer_key="zmK41mqxU3ZNJTFQpYwTdg"
        consumer_secret="9StnKNAe20ebDOREQjsVjAjBEiz5R9feZJTGUYWqLo"
        access_token="45210078-VydgdJMwhWYjZRvlNbrKj6jfqicUIsdMnRbnaPElL"
        access_token_secret="uLvIN3MMxFSxdK4M8P5RYojjUkbc2reqNydYtpT7Ks"
    else:
        consumer_key = input_dict['ck']
        consumer_secret = input_dict['cs']
        access_token = input_dict['at']
        access_token_secret = input_dict['as']

    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    api = tweepy.API(auth)

    query = input_dict['query']

    rate_limit_status = api.rate_limit_status()

    if rate_limit_status['resources']['search']['/search/tweets']['remaining']>0:

        if stream is None:
            try:
                ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100)
            except Exception as e:
                raise HaltStream("The Twitter API returned an error: "+str(e))
        else:
            try:
                swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
                data = swd.value
            except Exception as e:
                swd = StreamWidgetData()
                swd.stream = stream
                swd.widget = widget
                data = {}
                swd.value = data
                swd.save()
            if data.has_key(query):
                since_id = data[query]
                try:
                    ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100,since_id=since_id)
                except Exception as e:
                    raise HaltStream("The Twitter API returned an error: "+str(e))
            else:
                try:
                    ltw = api.new_search(q=input_dict['query'],geocode=input_dict['geocode'],count=100)
                except Exception as e:
                    raise HaltStream("The Twitter API returned an error: "+str(e))
            if len(ltw)>0:
                data[query]=ltw[0].id
                swd.value = data
                swd.save()
    else:
        import datetime
        import time
        current_time = int(time.mktime(datetime.datetime.now().timetuple()))
        remaining = rate_limit_status['resources']['search']['/search/tweets']['reset']-current_time
        if input_dict['cfauth']=="true":
            raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds. Try using your own credentials.")
        else:
            raise HaltStream("The twitter API limit has been reached. Try again in "+str(remaining)+" seconds.")

    output_dict = {}

    tweets = []

    for tw in ltw:
        tweet = {}
        tweet['id'] = tw.id
        tweet['created_at'] = tw.created_at
        tweet['text'] = unicode(tw.text).encode("utf-8")
Janez K's avatar
Janez K committed
155
156
157
158
        try:
            tweet['user'] = tw.user['screen_name']
        except:
            tweet['user'] = ""
159
160
161
162
163
164
165
166
167
        tweet['lang'] = tw.lang
        tweets.append(tweet)

    if len(tweets)>0 or stream is None:
        output_dict['ltw']=tweets
        return output_dict
    else:
        raise HaltStream("No new results, halting stream.")

Janez K's avatar
Janez K committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
def streaming_rss_reader(input_dict,widget,stream=None):
    import feedparser
    from streams.models import StreamWidgetData
    feed = feedparser.parse(input_dict['url'])
    output_dict = {}
    if stream is None:
        output_dict['url'] = feed['items'][0]['link']
    else:
        try:
            swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
            data = swd.value
        except:
            swd = StreamWidgetData()
            swd.stream = stream
            swd.widget = widget
            data = []
            swd.value = data
            swd.save()
        feed_length = len(feed['items'])
        feed['items'].reverse()
        for item in feed['items']:
Janez K's avatar
Janez K committed
189
190
            if item['link'] not in data:
                data.append(item['link'])
Janez K's avatar
Janez K committed
191
192
193
194
195
                swd.value = data
                swd.save()
                output_dict['url'] = item['link']
                break
        else:
Janez K's avatar
Janez K committed
196
197
198
            from streams.models import HaltStream
            raise HaltStream("Halting stream.")
    return output_dict