twitter streams

streams/management/commands/list_streams.py

+from django.core.management.base import NoArgsCommand
+from streams.models import *
+
+class Command(NoArgsCommand):
+    help = 'see a list of all streams'
+    option_list = NoArgsCommand.option_list
+    def handle_noargs(self, **options):
+        import time
+        self.stdout.write("Working on streams...")
+        self.stdout.flush()
+        while True:
+            streams = Stream.objects.filter(active=True)
+            for stream in streams:
+                #self.stdout.write(u"\nChecking stream "+unicode(stream)+"...\n")
+                #self.stdout.flush()
+                #preverimo ce je ze dost casa pretekl
+                import django
+                now = django.utils.timezone.now()
+                delta = now - stream.last_executed
+                delta_seconds = delta.seconds + delta.days * 86400
+                if delta_seconds > stream.period:
+                    stream.last_executed = now
+                    stream.save()
+                    self.stdout.write(u"<"+str(now)+">Executing "+unicode(stream)+"...")
+                    self.stdout.flush()
+                    stream.execute()
+                    self.stdout.write("done!\n")
+                    self.stdout.flush()
+                    #print stream.execute()
+            time.sleep(1)
+            #self.stdout.write(".")
+            #self.stdout.flush()
+

streams/management/commands/run_streams.py

@@ -11,6 +11,8 @@ class Command(NoArgsCommand):
         while True:
             streams = Stream.objects.filter(active=True)
             for stream in streams:
+                #self.stdout.write(u"\nChecking stream "+unicode(stream)+"...\n")
+                #self.stdout.flush()
                 #preverimo ce je ze dost casa pretekl
                 import django
                 now = django.utils.timezone.now()
@@ -19,8 +21,16 @@ class Command(NoArgsCommand):
                 if delta_seconds > stream.period:
                     stream.last_executed = now
                     stream.save()
-                    #self.stdout.write("-")
-                    #self.stdout.flush()
+                    self.stdout.write(u"Executing "+unicode(stream)+"...")
+                    self.stdout.flush()
+                    try:
+                        stream.execute()
+                    except:
+                        import traceback
+                        self.stdout.write("\n ERROR in executing stream:\n")
+                        traceback.print_exc(file=self.stdout)
+                    self.stdout.write("done!\n")
+                    self.stdout.flush()
                     #print stream.execute()
             time.sleep(1)
             #self.stdout.write(".")

streams/models.py

@@ -17,6 +17,12 @@ class Stream(models.Model):
     period = models.IntegerField(default=60)
     active = models.BooleanField(default=False)
 
+    def stream_visualization_widgets(self):
+        return self.workflow.widgets.all().exclude(abstract_widget__streaming_visualization_view='')
+
+    def reset(self):
+        self.widget_data.all().delete()
+
     def execute(self,workflow=None,outputs={}):
         if workflow is None:
             workflow = self.workflow

streams/templates/streams/base.html

+{% extends "website/base.html" %}
+{% load url from future %}
+
+{% block "streamnav" %}
+
+<div class="navbar">
+  <div class="navbar-inner">
+    <a class="brand">{{stream}}</a>
+    <ul class="nav">
+        {% for w in stream.stream_visualization_widgets %}
+        <li {% if widget.pk == w.pk %}class="active"{% endif %}><a href="{% url 'stream widget visualization' stream.pk w.pk %}">{{w}}</a></li>
+      {% endfor %}
+    </ul>
+  </div>
+</div>
+
+{% endblock %}

website/static/website-css/style.css

-.brand {
-    font-family: 'Condiment', cursive;
-}
\ No newline at end of file

website/templates/website/base.html

@@ -26,6 +26,11 @@
     <link href="{{STATIC_URL}}bootstrap/css/bootstrap-responsive.css" rel="stylesheet">
     <link href='http://fonts.googleapis.com/css?family=Condiment' rel='stylesheet' type='text/css'>
 
+        <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+    <script src="{{STATIC_URL}}bootstrap/js/bootstrap.js"></script>
+    <script src="{{STATIC_URL}}js/fancybox/jquery.fancybox-1.3.4.js"></script>
+    <script src="{{STATIC_URL}}js/signuplogin.js"></script>
+
     <!-- Le fav and touch icons -->
     <link rel="shortcut icon" href="images/favicon.ico">
     <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
@@ -44,15 +49,15 @@
     var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
   })();
 
-</script>    
-    
+</script>
+
   </head>
 
   <body>
 
 
-    
-       <div class="navbar navbar-fixed-top">
+
+       <div class="navbar navbar-fixed-top navbar-inverse">
        <div class="navbar-inner">
         <div class="container">
 
@@ -65,9 +70,9 @@
           <div class="nav-collapse">
             <ul class="nav pull-right">
 
-              <li class="active"><a href="{% url 'website index' %}">Home</a></li>
-              <li><a href="{% url 'the index' %}" class="{% if not user.is_authenticated %}must_login{% endif %}">Start working</a></li>
-              <li><a href="{% url 'existing workflows' %}">Explore workflows</a></li>
+              <li {% block "homeactive" %}{% endblock %}><a href="{% url 'website index' %}">Home</a></li>
+              <li {% block "workflowactive" %}{% endblock %}><a href="{% url 'the index' %}" class="{% if not user.is_authenticated %}must_login{% endif %}">Start working</a></li>
+              <li {% block "exploreactive" %}{% endblock %}><a href="{% url 'existing workflows' %}">Explore workflows</a></li>
               <li><a class="{% if not user.is_authenticated %}must_login{% endif %}" href="{% if user.is_authenticated %}{% url 'logout' %}{% endif %}{% if not user.is_authenticated %}{% url 'website index' %}{% endif %}">{% if not user.is_authenticated %}Log in{% endif %}{% if user.is_authenticated %}Log out{% endif %}</a></li>
             </ul>
           </div><!--/.nav-collapse -->
@@ -77,56 +82,61 @@
     </div>
 
     <div class="container">
-    {% block 'container' %} 
-      
+
+    {% block "streamnav" %}
+
+    {% endblock %}
+
+    {% block 'container' %}
+
       {% endblock %}
 
       <hr>
 
       <footer>
-        
-    
+
+
         <div class="row">
-        
+
             <div class="span3" style="text-align:center;">
-        
+
 				<a class="img" href="http://project-first.eu/">
 					<img alt="FIRST Logo" src="{{STATIC_URL}}website-images/FIRSTLogoSmall.gif" style="vertical-align: middle;">
 				</a>
-                
+
             </div>
             <div class="span3" style="text-align:center;">
-            
+
 				<a class="img" href="http://www.ijs.si/">
 					<img alt="JSI Logo" src="{{STATIC_URL}}website-images/JSILogoSmall.gif">
 				</a>
 
             </div>
-            <div class="span3" style="text-align:center;">                
-                
+            <div class="span3" style="text-align:center;">
+
 				<a class="img" href="http://www.focproject.net/">
 					<img alt="FOC Logo" src="{{STATIC_URL}}website-images/foc_logo_200x50.gif" width="200" height="50">
 				</a>
-                
+
             </div>
             <div class="span3" style="text-align:center;">
 
-            
+
 				<a class="img" href="http://cordis.europa.eu/info-management/">
 					<img alt="EU Logo" src="{{STATIC_URL}}website-images/EULogoSmall.gif" style="vertical-align: middle;">
 				</a>
-                
-            </div>                
-                
+
+            </div>
+
         </div>
-        
+
         <hr>
-        
+
 			<p><center>&copy; 2012 <a href="http://kt.ijs.si">Department of Knowledge Technologies</a>. Co-funded by the <a href="http://cordis.europa.eu/info-management/">European Union</a>.<br><br><a href="http://kt.ijs.si/janez_kranjc/">About the author</a>.<br>
             </center></p>
-                    
-  
-        
+
+
+
       </footer>
 
     </div> <!-- /container -->
@@ -134,12 +144,8 @@
     <!-- Le javascript
     ================================================== -->
     <!-- Placed at the end of the document so the pages load faster -->
-    <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
-    <script src="{{STATIC_URL}}bootstrap/js/bootstrap.js"></script>
-    <script src="{{STATIC_URL}}js/fancybox/jquery.fancybox-1.3.4.js"></script>
-    <script src="{{STATIC_URL}}js/signuplogin.js"></script>
     <div style="position:absolute;top:50%;left:50%;display:none;z-index:20;" class="ajax-loader">
         <img src="{{STATIC_URL}}images/ajax-loader-big.gif">
     </div>
   </body>
-</html>
\ No newline at end of file
+</html>

website/templates/website/existing.html

 {% extends 'website/base.html' %}
+{% block "exploreactive" %}class="active"{% endblock %}
 {% block 'container' %}
 
     {% for w in workflows %}
@@ -31,13 +32,13 @@
 
           <h2>Try it live</h2>
           <p>By clicking on the button below you will launch the workflow and be able to run it or edit it.</p>
-       
+
           <a href="{{w.get_copy_url}}" class="btn btn-primary btn-large launch {% if not user.is_authenticated %}must_login{% endif %}" onClick="$('.launch').hide();$('.ajax-loader').show()">Launch workflow!{% if w.user == user %} (Clicking this will create a new copy){% endif %}</a>
           {% if w.user == user %}<a href="{{w.get_absolute_url}}" class="btn btn-large launch {% if not user.is_authenticated %}must_login{% endif %}" onClick="$('.launch').hide();$('.ajax-loader').show()">This is your workflow. Click here to edit it!</a>{% endif %}
 
           <hr>
 
-         
+
     </section>
     {% endfor %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}

website/templates/website/index.html

 {% extends "website/base.html" %}
 {% load url from future %}
+{% block "homeactive" %}class="active"{% endblock %}
 {% block 'container' %}
       <div class="hero-unit">
         <h1>ClowdFlows</h1>
@@ -10,26 +11,26 @@
 
       <!-- Example row of columns -->
       <div class="row">
-      
+
         <div class="span4">
           <h2>Work anywhere</h2>
            <p>The non local nature of the application allows you to work anywhere at anytime! No installation required!</p>
            <p>It's a web application, and your work is saved on the server.</p>
-       </div>      
-      
+       </div>
+
         <div class="span4">
           <h2>Absolutely free</h2>
            <p>You can use the platform for free! Just register and you're good to go!</p>
-       </div> 
-       
+       </div>
+
         <div class="span4">
 
           <h2>Powerful workflow controls</h2>
           <p>Organize your workflows with powerful tools that allow packing workflows into workflow elements and even running them in <strong>for loops</strong>.</p>
-        </div>       
+        </div>
 
       </div>
-      
+
       <div class="row">
         <div class="span4">
           <h2>Web services in workflows</h2>
@@ -41,12 +42,12 @@
           <h2>Data mining algorithms</h2>
            <p>Unleash the power of data mining by using Weka's algorithms that have been exposed as WSDL web services. Construct trees, build models and experiment!</p>
        </div>
-       
+
         <div class="span4">
 
           <h2>Contribute and share your work</h2>
           <p>Have you constructed an interesting workflow? Share it with other people, so that they may use it to create or improve their workflows!</p>
-        </div>       
+        </div>
 
-      </div>     
-{% endblock %}
\ No newline at end of file
+      </div>
+{% endblock %}

workflows/streaming/library.py

@@ -24,6 +24,25 @@ def streaming_collect_and_display_tweets(input_dict,widget,stream=None):
         swd.save()
         return {}
 
+def streaming_sentiment_graph(input_dict,widget,stream=None):
+    from streams.models import StreamWidgetData
+    if stream is None:
+        return {}
+    else:
+        try:
+            swd = StreamWidgetData.objects.get(stream=stream,widget=widget)
+            data = swd.value
+        except Exception as e:
+            swd = StreamWidgetData()
+            swd.stream = stream
+            swd.widget = widget
+            data = []
+            swd.value = data
+            swd.save()
+        swd.value = input_dict['ltw']+swd.value
+        swd.save()
+        return {}
+
 def streaming_tweet_sentiment_service(input_dict,widget,stream=None):
     import pickle
     from pysimplesoap.client import SoapClient, SoapFault
@@ -133,7 +152,10 @@ def streaming_twitter(input_dict,widget,stream=None):
         tweet['id'] = tw.id
         tweet['created_at'] = tw.created_at
         tweet['text'] = unicode(tw.text).encode("utf-8")
-        tweet['user'] = tw.user
+        try:
+            tweet['user'] = tw.user['screen_name']
+        except:
+            tweet['user'] = ""
         tweet['lang'] = tw.lang
         tweets.append(tweet)