Commit c22245e8 authored by Janez K's avatar Janez K

dodal haltstream exception

parent 5fa82106
...@@ -186,10 +186,10 @@ INSTALLED_APPS_CONCUR = ( ...@@ -186,10 +186,10 @@ INSTALLED_APPS_CONCUR = (
INSTALLED_APPS_WORKFLOWS_SUB = ( INSTALLED_APPS_WORKFLOWS_SUB = (
'workflows.base', 'workflows.base',
'workflows.latino', #'workflows.latino',
'workflows.decision_support', 'workflows.decision_support',
'workflows.segmine', 'workflows.segmine',
'workflows.subgroup_discovery', #'workflows.subgroup_discovery',
'workflows.nlp', 'workflows.nlp',
'workflows.nl_toolkit', 'workflows.nl_toolkit',
'workflows.ilp', 'workflows.ilp',
...@@ -197,8 +197,8 @@ INSTALLED_APPS_WORKFLOWS_SUB = ( ...@@ -197,8 +197,8 @@ INSTALLED_APPS_WORKFLOWS_SUB = (
'workflows.cforange', 'workflows.cforange',
'workflows.perfeval', 'workflows.perfeval',
'workflows.mysql', 'workflows.mysql',
'workflows.lemmagen', #'workflows.lemmagen',
'workflows.crossbee', #'workflows.crossbee',
'workflows.streaming', 'workflows.streaming',
#WORKFLOWS_SUBAPP_PLACEHOLDER #WORKFLOWS_SUBAPP_PLACEHOLDER
) )
......
...@@ -16,4 +16,4 @@ class Command(NoArgsCommand): ...@@ -16,4 +16,4 @@ class Command(NoArgsCommand):
stream.last_executed = now stream.last_executed = now
stream.save() stream.save()
print stream.execute() print stream.execute()
...@@ -5,6 +5,9 @@ import workflows.library ...@@ -5,6 +5,9 @@ import workflows.library
from picklefield.fields import PickledObjectField from picklefield.fields import PickledObjectField
class HaltStream(Exception):
pass
# Create your models here. # Create your models here.
class Stream(models.Model): class Stream(models.Model):
...@@ -13,13 +16,13 @@ class Stream(models.Model): ...@@ -13,13 +16,13 @@ class Stream(models.Model):
last_executed = models.DateTimeField(auto_now_add=True) last_executed = models.DateTimeField(auto_now_add=True)
period = models.IntegerField(default=60) period = models.IntegerField(default=60)
active = models.BooleanField(default=False) active = models.BooleanField(default=False)
def execute(self,workflow=None,outputs={}): def execute(self,workflow=None,outputs={}):
if workflow is None: if workflow is None:
workflow = self.workflow workflow = self.workflow
ready_to_run = [] ready_to_run = []
widgets = workflow.widgets.all() widgets = workflow.widgets.all()
#get unfinished #get unfinished
if workflow.is_for_loop(): if workflow.is_for_loop():
fi = workflow.widgets.filter(type='for_input')[0] fi = workflow.widgets.filter(type='for_input')[0]
...@@ -34,10 +37,10 @@ class Stream(models.Model): ...@@ -34,10 +37,10 @@ class Stream(models.Model):
input_list = [] input_list = []
else: else:
input_list = [0] input_list = [0]
#print input_list #print input_list
for for_input in input_list: for for_input in input_list:
#print for_input #print for_input
finished = [] finished = []
...@@ -48,9 +51,9 @@ class Stream(models.Model): ...@@ -48,9 +51,9 @@ class Stream(models.Model):
for w in unfinished_list: for w in unfinished_list:
# prepare all the inputs for this widget # prepare all the inputs for this widget
input_dict = {} input_dict = {}
output_dict = {} output_dict = {}
finish = True finish = True
for i in w.inputs.all(): for i in w.inputs.all():
#gremo pogledat ce obstaja povezava in ce obstaja gremo value prebrat iz outputa #gremo pogledat ce obstaja povezava in ce obstaja gremo value prebrat iz outputa
if not i.parameter: if not i.parameter:
...@@ -65,25 +68,25 @@ class Stream(models.Model): ...@@ -65,25 +68,25 @@ class Stream(models.Model):
if not i.variable in input_dict: if not i.variable in input_dict:
input_dict[i.variable]=[] input_dict[i.variable]=[]
if not i.value==None: if not i.value==None:
input_dict[i.variable].append(i.value) input_dict[i.variable].append(i.value)
if w.type == 'regular': if w.type == 'regular':
function_to_call = getattr(workflows.library,w.abstract_widget.action) function_to_call = getattr(workflows.library,w.abstract_widget.action)
if w.abstract_widget.wsdl != '': if w.abstract_widget.wsdl != '':
input_dict['wsdl']=w.abstract_widget.wsdl input_dict['wsdl']=w.abstract_widget.wsdl
input_dict['wsdl_method']=w.abstract_widget.wsdl_method input_dict['wsdl_method']=w.abstract_widget.wsdl_method
try: try:
if w.abstract_widget.has_progress_bar: if w.abstract_widget.has_progress_bar:
output_dict = function_to_call(input_dict,w) output_dict = function_to_call(input_dict,w)
elif w.abstract_widget.is_streaming: elif w.abstract_widget.is_streaming:
output_dict = function_to_call(input_dict,w,self) output_dict = function_to_call(input_dict,w,self)
else: else:
output_dict = function_to_call(input_dict) output_dict = function_to_call(input_dict)
except: except HaltStream:
halted.append(w) halted.append(w)
finish=False finish=False
if w.type == 'subprocess': if w.type == 'subprocess':
new_outputs = self.execute(workflow=w.workflow_link,outputs=outputs) new_outputs = self.execute(workflow=w.workflow_link,outputs=outputs)
for o in w.outputs.all(): for o in w.outputs.all():
...@@ -91,18 +94,18 @@ class Stream(models.Model): ...@@ -91,18 +94,18 @@ class Stream(models.Model):
outputs[o.pk]=new_outputs[o.pk] outputs[o.pk]=new_outputs[o.pk]
except: except:
outputs[o.pk]=(o.variable,None) outputs[o.pk]=(o.variable,None)
if w.type == 'for_input': if w.type == 'for_input':
for o in w.outputs.all(): for o in w.outputs.all():
outputs[o.pk]=(o.variable,for_input) outputs[o.pk]=(o.variable,for_input)
#print outputs[o.pk] #print outputs[o.pk]
output_dict[o.variable]=for_input output_dict[o.variable]=for_input
if w.type == 'for_output': if w.type == 'for_output':
for i in w.inputs.all(): for i in w.inputs.all():
outputs[i.outer_output.pk][1].append(input_dict[i.variable]) outputs[i.outer_output.pk][1].append(input_dict[i.variable])
output_dict[i.variable]=input_dict[i.variable] output_dict[i.variable]=input_dict[i.variable]
if w.type == 'input': if w.type == 'input':
for o in w.outputs.all(): for o in w.outputs.all():
value = None value = None
...@@ -112,17 +115,17 @@ class Stream(models.Model): ...@@ -112,17 +115,17 @@ class Stream(models.Model):
except: except:
value = None value = None
output_dict[o.variable]=value output_dict[o.variable]=value
if finish: if finish:
if w.type == 'output': if w.type == 'output':
for i in w.inputs.all(): for i in w.inputs.all():
outputs[i.outer_output.pk]=(i.outer_output.variable,input_dict[i.variable]) outputs[i.outer_output.pk]=(i.outer_output.variable,input_dict[i.variable])
if w.type != 'subprocess': if w.type != 'subprocess':
for o in w.outputs.all(): for o in w.outputs.all():
outputs[o.pk]=(o.variable,output_dict[o.variable]) outputs[o.pk]=(o.variable,output_dict[o.variable])
finished.append(w.pk) finished.append(w.pk)
unfinished_list = [] unfinished_list = []
for w in widgets: for w in widgets:
...@@ -139,14 +142,14 @@ class Stream(models.Model): ...@@ -139,14 +142,14 @@ class Stream(models.Model):
if len(unfinished_list)==0: if len(unfinished_list)==0:
loop = False loop = False
return outputs return outputs
def __unicode__(self): def __unicode__(self):
return unicode(self.workflow)+' stream' return unicode(self.workflow)+' stream'
class StreamWidgetData(models.Model): class StreamWidgetData(models.Model):
stream = models.ForeignKey(Stream, related_name="widget_data") stream = models.ForeignKey(Stream, related_name="widget_data")
widget = models.ForeignKey(Widget, related_name="stream_data") widget = models.ForeignKey(Widget, related_name="stream_data")
value = PickledObjectField(null=True) value = PickledObjectField(null=True)
\ No newline at end of file
...@@ -4,6 +4,10 @@ Streaming widgets librarby ...@@ -4,6 +4,10 @@ Streaming widgets librarby
@author: Janez Kranjc <janez.kranjc@ijs.si> @author: Janez Kranjc <janez.kranjc@ijs.si>
''' '''
def streaming_twitter(input_dict,widget,stream=None):
output_dict = {}
return output_dict
def streaming_rss_reader(input_dict,widget,stream=None): def streaming_rss_reader(input_dict,widget,stream=None):
import feedparser import feedparser
from streams.models import StreamWidgetData from streams.models import StreamWidgetData
...@@ -25,12 +29,13 @@ def streaming_rss_reader(input_dict,widget,stream=None): ...@@ -25,12 +29,13 @@ def streaming_rss_reader(input_dict,widget,stream=None):
feed_length = len(feed['items']) feed_length = len(feed['items'])
feed['items'].reverse() feed['items'].reverse()
for item in feed['items']: for item in feed['items']:
if item['id'] not in data: if item['link'] not in data:
data.append(item['id']) data.append(item['link'])
swd.value = data swd.value = data
swd.save() swd.save()
output_dict['url'] = item['link'] output_dict['url'] = item['link']
break break
else: else:
raise Exception("Halting stream.") from streams.models import HaltStream
return output_dict raise HaltStream("Halting stream.")
\ No newline at end of file return output_dict
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment