Commit 73b1243d authored by Janez K's avatar Janez K

Merge branch 'dev' into streams

parents c22245e8 a0ef382b
...@@ -15,16 +15,19 @@ from mothra.settings import USE_CONCURRENCY ...@@ -15,16 +15,19 @@ from mothra.settings import USE_CONCURRENCY
if USE_CONCURRENCY: if USE_CONCURRENCY:
from workflows.tasks import runWidgetAsync, runForLoopIteration from workflows.tasks import runWidgetAsync, runForLoopIteration
class WidgetException(Exception):
pass
class Connection(models.Model): class Connection(models.Model):
output = models.ForeignKey("Output",related_name="connections") output = models.ForeignKey("Output",related_name="connections")
input = models.ForeignKey("Input",related_name="connections") input = models.ForeignKey("Input",related_name="connections")
workflow = models.ForeignKey("Workflow",related_name="connections") workflow = models.ForeignKey("Workflow",related_name="connections")
class Category(models.Model): class Category(models.Model):
name = models.CharField(max_length=50) name = models.CharField(max_length=50)
parent = models.ForeignKey('self',related_name="children",null=True,blank=True) parent = models.ForeignKey('self',related_name="children",null=True,blank=True)
user = models.ForeignKey(User,null=True,blank=True,related_name="categories") user = models.ForeignKey(User,null=True,blank=True,related_name="categories")
workflow = models.ForeignKey('Workflow',null=True,blank=True,related_name="categories") workflow = models.ForeignKey('Workflow',null=True,blank=True,related_name="categories")
order = models.PositiveIntegerField(default=1) order = models.PositiveIntegerField(default=1)
...@@ -48,14 +51,14 @@ class Workflow(models.Model): ...@@ -48,14 +51,14 @@ class Workflow(models.Model):
description = models.TextField(blank=True,default='') description = models.TextField(blank=True,default='')
widget = models.OneToOneField('Widget',related_name="workflow_link",blank=True,null=True) widget = models.OneToOneField('Widget',related_name="workflow_link",blank=True,null=True)
template_parent = models.ForeignKey('Workflow',blank=True,null=True,default=None,on_delete=models.SET_NULL) template_parent = models.ForeignKey('Workflow',blank=True,null=True,default=None,on_delete=models.SET_NULL)
def is_for_loop(self): def is_for_loop(self):
if self.widgets.filter(type='for_input').count()>0: if self.widgets.filter(type='for_input').count()>0:
return True return True
else: else:
return False return False
def get_ready_to_run(self): def get_ready_to_run(self):
widgets = self.widgets.all() widgets = self.widgets.all()
unfinished_list = [] unfinished_list = []
...@@ -70,7 +73,7 @@ class Workflow(models.Model): ...@@ -70,7 +73,7 @@ class Workflow(models.Model):
if ready_to_run: if ready_to_run:
unfinished_list.append(w.id) unfinished_list.append(w.id)
return unfinished_list return unfinished_list
def get_runnable_widgets(self): def get_runnable_widgets(self):
widgets = self.widgets.all() widgets = self.widgets.all()
unfinished_list = [] unfinished_list = []
...@@ -93,7 +96,7 @@ class Workflow(models.Model): ...@@ -93,7 +96,7 @@ class Workflow(models.Model):
outer_output = fo.inputs.all()[0].outer_output outer_output = fo.inputs.all()[0].outer_output
outer_output.value=[] outer_output.value=[]
outer_output.save() outer_output.save()
input_list = fi.outputs.all()[0].outer_input.value input_list = fi.outputs.all()[0].outer_input.value
progress_total = len(input_list) progress_total = len(input_list)
current_iteration = 0 current_iteration = 0
...@@ -112,7 +115,7 @@ class Workflow(models.Model): ...@@ -112,7 +115,7 @@ class Workflow(models.Model):
for w in unfinished_list: for w in unfinished_list:
w.run(True) w.run(True)
total = self.widgets.count() total = self.widgets.count()
completed = self.widgets.filter(finished=True).count() completed = self.widgets.filter(finished=True).count()
self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total))) self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total)))
self.widget.save() self.widget.save()
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
...@@ -148,7 +151,7 @@ class Workflow(models.Model): ...@@ -148,7 +151,7 @@ class Workflow(models.Model):
completed = self.widgets.filter(finished=True).count() completed = self.widgets.filter(finished=True).count()
if self.widget: if self.widget:
self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total))) self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total)))
self.widget.save() self.widget.save()
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
except: except:
raise raise
...@@ -189,7 +192,7 @@ class Workflow(models.Model): ...@@ -189,7 +192,7 @@ class Workflow(models.Model):
self.widget.progress = (int)(((completed*1.0)/total)*100) self.widget.progress = (int)(((completed*1.0)/total)*100)
self.widget.save() self.widget.save()
is_running=True is_running=True
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
while len(unfinished_list)==0 and is_running: while len(unfinished_list)==0 and is_running:
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
is_running = False is_running = False
...@@ -201,32 +204,32 @@ class Workflow(models.Model): ...@@ -201,32 +204,32 @@ class Workflow(models.Model):
completed = self.widgets.filter(finished=True).count() completed = self.widgets.filter(finished=True).count()
if self.widget: if self.widget:
self.widget.progress = (int)(((completed*1.0)/total)*100) self.widget.progress = (int)(((completed*1.0)/total)*100)
self.widget.save() self.widget.save()
time.sleep(1) time.sleep(1)
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
except: except:
raise raise
def rename(self,new_name): def rename(self,new_name):
self.name = new_name self.name = new_name
self.save() self.save()
@models.permalink @models.permalink
def get_absolute_url(self): def get_absolute_url(self):
return ('open workflow', [str(self.id)]) return ('open workflow', [str(self.id)])
@models.permalink @models.permalink
def get_copy_url(self): def get_copy_url(self):
return ('copy workflow', [str(self.id)]) return ('copy workflow', [str(self.id)])
@models.permalink @models.permalink
def get_info_url(self): def get_info_url(self):
return ('workflow information', [str(self.id)]) return ('workflow information', [str(self.id)])
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class Meta: class Meta:
ordering = ['name'] ordering = ['name']
class AbstractWidget(models.Model): class AbstractWidget(models.Model):
name = models.CharField(max_length=200,help_text='Name is the name that will be displayed in the widget repository and under the actual widget itself.') name = models.CharField(max_length=200,help_text='Name is the name that will be displayed in the widget repository and under the actual widget itself.')
...@@ -240,21 +243,21 @@ class AbstractWidget(models.Model): ...@@ -240,21 +243,21 @@ class AbstractWidget(models.Model):
interactive = models.BooleanField(default=False,help_text='The widget can be interactive. This means that when a user executes the widget, the action will perform, then the interaction view will be executed and finally the Post interact action will be executed.') interactive = models.BooleanField(default=False,help_text='The widget can be interactive. This means that when a user executes the widget, the action will perform, then the interaction view will be executed and finally the Post interact action will be executed.')
interaction_view = models.CharField(max_length=200,blank=True,default='') interaction_view = models.CharField(max_length=200,blank=True,default='')
post_interact_action = models.CharField(max_length=200,blank=True,default='') post_interact_action = models.CharField(max_length=200,blank=True,default='')
image = ThumbnailField(blank=True,null=True,upload_to="images",size=(34,34),help_text='Image and Treeview image are deprecated and will be phased out soon. Please use the static image field.') image = ThumbnailField(blank=True,null=True,upload_to="images",size=(34,34),help_text='Image and Treeview image are deprecated and will be phased out soon. Please use the static image field.')
treeview_image = ThumbnailField(blank=True,null=True,upload_to="treeview",size=(16,16)) treeview_image = ThumbnailField(blank=True,null=True,upload_to="treeview",size=(16,16))
static_image = models.CharField(max_length=250,blank=True,default='',help_text='In the static image field just enter the filename of the image (without the path). The path will be $package_name$/icons/widget/$filename$ and $package_name$/icons/treeview/$filename$ where the treeview image is the small image that appears in the treeview on the left side and the widget image is the actual normal sized icon for the widget. IMPORTANT: the static image field only works if the package is set.') static_image = models.CharField(max_length=250,blank=True,default='',help_text='In the static image field just enter the filename of the image (without the path). The path will be $package_name$/icons/widget/$filename$ and $package_name$/icons/treeview/$filename$ where the treeview image is the small image that appears in the treeview on the left side and the widget image is the actual normal sized icon for the widget. IMPORTANT: the static image field only works if the package is set.')
has_progress_bar = models.BooleanField(default=False,help_text='The flag has progress bar determines if the widget implements a progress bar.') has_progress_bar = models.BooleanField(default=False,help_text='The flag has progress bar determines if the widget implements a progress bar.')
is_streaming = models.BooleanField(default=False,help_text='The is streaming flag is currently under construction, please do not use it yet.') is_streaming = models.BooleanField(default=False,help_text='The is streaming flag is currently under construction, please do not use it yet.')
order = models.PositiveIntegerField(default=1,help_text='The Order determines the order in which the widget will be displayed in the repository. This is set automatically when sorting widgets in a single category from the admin.') order = models.PositiveIntegerField(default=1,help_text='The Order determines the order in which the widget will be displayed in the repository. This is set automatically when sorting widgets in a single category from the admin.')
uid = models.CharField(max_length=250,blank=True,default='',help_text='UID is set automatically when you export a package with the -u switch.') uid = models.CharField(max_length=250,blank=True,default='',help_text='UID is set automatically when you export a package with the -u switch.')
package = models.CharField(max_length=150,blank=True,default='',help_text='Package is the package name. You are encouraged to use packages.') package = models.CharField(max_length=150,blank=True,default='',help_text='Package is the package name. You are encouraged to use packages.')
class Meta: class Meta:
ordering = ('order','name',) ordering = ('order','name',)
...@@ -298,17 +301,17 @@ class AbstractInput(models.Model): ...@@ -298,17 +301,17 @@ class AbstractInput(models.Model):
('file', 'File'), ('file', 'File'),
) )
parameter_type = models.CharField(max_length=50,choices=PARAMETER_CHOICES,blank=True,null=True) parameter_type = models.CharField(max_length=50,choices=PARAMETER_CHOICES,blank=True,null=True)
order = models.PositiveIntegerField(default=1) order = models.PositiveIntegerField(default=1)
uid = models.CharField(max_length=250,blank=True,default='') uid = models.CharField(max_length=250,blank=True,default='')
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class Meta: class Meta:
ordering = ('order',) ordering = ('order',)
class AbstractOption(models.Model): class AbstractOption(models.Model):
abstract_input = models.ForeignKey(AbstractInput,related_name="options") abstract_input = models.ForeignKey(AbstractInput,related_name="options")
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
...@@ -318,9 +321,9 @@ class AbstractOption(models.Model): ...@@ -318,9 +321,9 @@ class AbstractOption(models.Model):
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class Meta: class Meta:
ordering = ['name'] ordering = ['name']
class AbstractOutput(models.Model): class AbstractOutput(models.Model):
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
...@@ -328,14 +331,14 @@ class AbstractOutput(models.Model): ...@@ -328,14 +331,14 @@ class AbstractOutput(models.Model):
description = models.TextField(blank=True) description = models.TextField(blank=True)
variable = models.CharField(max_length=50,help_text='The variable attribute of both the input and the output are important because this is how the data will be accessed in the python function that is executed when the widget runs.') variable = models.CharField(max_length=50,help_text='The variable attribute of both the input and the output are important because this is how the data will be accessed in the python function that is executed when the widget runs.')
widget = models.ForeignKey(AbstractWidget,related_name="outputs") widget = models.ForeignKey(AbstractWidget,related_name="outputs")
order = models.PositiveIntegerField(default=1) order = models.PositiveIntegerField(default=1)
uid = models.CharField(max_length=250,blank=True,default='') uid = models.CharField(max_length=250,blank=True,default='')
class Meta: class Meta:
ordering = ('order',) ordering = ('order',)
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
...@@ -356,23 +359,23 @@ class Widget(models.Model): ...@@ -356,23 +359,23 @@ class Widget(models.Model):
('output', 'Output widget'), ('output', 'Output widget'),
) )
type = models.CharField(max_length=50,choices=WIDGET_CHOICES,default='regular') type = models.CharField(max_length=50,choices=WIDGET_CHOICES,default='regular')
progress = models.IntegerField(default=0) progress = models.IntegerField(default=0)
def is_visualization(self): def is_visualization(self):
try: try:
if self.abstract_widget.visualization_view != '': if self.abstract_widget.visualization_view != '':
return True return True
except: except:
return False return False
def ready_to_run(self): def ready_to_run(self):
cons = Connection.objects.filter(input__widget=self) cons = Connection.objects.filter(input__widget=self)
for c in cons: for c in cons:
if not c.output.widget.finished: if not c.output.widget.finished:
return False return False
return True return True
def unfinish(self): def unfinish(self):
if self.finished or self.error: if self.finished or self.error:
self.finished=False self.finished=False
...@@ -390,7 +393,7 @@ class Widget(models.Model): ...@@ -390,7 +393,7 @@ class Widget(models.Model):
w.save() w.save()
if w.type=='subprocess': if w.type=='subprocess':
w.subunfinish() w.subunfinish()
def subunfinish(self): def subunfinish(self):
if self.type == 'subprocess': if self.type == 'subprocess':
for w in self.workflow_link.widgets.all(): for w in self.workflow_link.widgets.all():
...@@ -399,7 +402,7 @@ class Widget(models.Model): ...@@ -399,7 +402,7 @@ class Widget(models.Model):
w.save() w.save()
if w.type=='subprocess': if w.type=='subprocess':
w.subunfinish() w.subunfinish()
def rename(self,new_name): def rename(self,new_name):
self.name = new_name self.name = new_name
self.save() self.save()
...@@ -418,17 +421,17 @@ class Widget(models.Model): ...@@ -418,17 +421,17 @@ class Widget(models.Model):
inp.save() inp.save()
inp.outer_output.name = self.name inp.outer_output.name = self.name
inp.outer_output.short_name = self.name[:3] inp.outer_output.short_name = self.name[:3]
inp.outer_output.save() inp.outer_output.save()
try: try:
w_link = self.workflow_link w_link = self.workflow_link
w_link.name=new_name w_link.name=new_name
w_link.save() w_link.save()
except Workflow.DoesNotExist: except Workflow.DoesNotExist:
pass pass
def run(self,offline): def run(self,offline):
if not self.ready_to_run(): if not self.ready_to_run():
raise Exception("The prerequisites for running this widget have not been met.") raise WidgetException("The prerequisites for running this widget have not been met.")
self.running=True self.running=True
self.save() self.save()
if self.type == 'regular' or self.type == 'subprocess': if self.type == 'regular' or self.type == 'subprocess':
...@@ -537,12 +540,12 @@ class Widget(models.Model): ...@@ -537,12 +540,12 @@ class Widget(models.Model):
i.outer_output.value = i.value i.outer_output.value = i.value
i.outer_output.save() i.outer_output.save()
self.finished=True self.finished=True
self.finished=True self.finished=True
self.running=False self.running=False
self.error=False self.error=False
self.save() self.save()
return None return None
def reset(self,offline): def reset(self,offline):
for i in self.inputs.defer("value").all(): for i in self.inputs.defer("value").all():
if not i.parameter: if not i.parameter:
...@@ -558,7 +561,7 @@ class Widget(models.Model): ...@@ -558,7 +561,7 @@ class Widget(models.Model):
def run_post(self,request): def run_post(self,request):
if not self.ready_to_run(): if not self.ready_to_run():
raise Exception("The prerequisites for running this widget have not been met.") raise WidgetException("The prerequisites for running this widget have not been met.")
self.running=True self.running=True
self.save() self.save()
function_to_call = getattr(workflows.library,self.abstract_widget.post_interact_action) function_to_call = getattr(workflows.library,self.abstract_widget.post_interact_action)
...@@ -605,11 +608,11 @@ class Widget(models.Model): ...@@ -605,11 +608,11 @@ class Widget(models.Model):
cons = Connection.objects.filter(output__widget=self) cons = Connection.objects.filter(output__widget=self)
for c in cons: for c in cons:
c.input.widget.unfinish() c.input.widget.unfinish()
return outputs return outputs
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class Input(models.Model): class Input(models.Model):
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
short_name = models.CharField(max_length=3) short_name = models.CharField(max_length=3)
...@@ -632,17 +635,17 @@ class Input(models.Model): ...@@ -632,17 +635,17 @@ class Input(models.Model):
class Meta: class Meta:
ordering = ('order',) ordering = ('order',)
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class Option(models.Model): class Option(models.Model):
input = models.ForeignKey(Input,related_name="options") input = models.ForeignKey(Input,related_name="options")
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
value = models.TextField(blank=True,null=True) value = models.TextField(blank=True,null=True)
class Meta: class Meta:
ordering = ['name'] ordering = ['name']
class Output(models.Model): class Output(models.Model):
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
...@@ -657,14 +660,14 @@ class Output(models.Model): ...@@ -657,14 +660,14 @@ class Output(models.Model):
class Meta: class Meta:
ordering = ('order',) ordering = ('order',)
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
class UserProfile(models.Model): class UserProfile(models.Model):
user = models.OneToOneField(User,related_name="userprofile") user = models.OneToOneField(User,related_name="userprofile")
active_workflow = models.ForeignKey(Workflow,related_name="users",null=True,blank=True,on_delete=models.SET_NULL) active_workflow = models.ForeignKey(Workflow,related_name="users",null=True,blank=True,on_delete=models.SET_NULL)
def __unicode__(self): def __unicode__(self):
return unicode(self.user) return unicode(self.user)
...@@ -764,4 +767,4 @@ def copy_workflow(old, user, parent_widget_conversion={},parent_input_conversion ...@@ -764,4 +767,4 @@ def copy_workflow(old, user, parent_widget_conversion={},parent_input_conversion
for widget in old.widgets.filter(type='subprocess'): for widget in old.widgets.filter(type='subprocess'):
#tuki mormo vse subprocesse zrihtat #tuki mormo vse subprocesse zrihtat
copy_workflow(widget.workflow_link, user, widget_conversion, input_conversion, output_conversion, Widget.objects.get(pk=widget_conversion[widget.id])) copy_workflow(widget.workflow_link, user, widget_conversion, input_conversion, output_conversion, Widget.objects.get(pk=widget_conversion[widget.id]))
return w return w
\ No newline at end of file
...@@ -62,7 +62,7 @@ def new_workflow(request): ...@@ -62,7 +62,7 @@ def new_workflow(request):
request.user.userprofile.active_workflow = w request.user.userprofile.active_workflow = w
request.user.userprofile.save() request.user.userprofile.save()
return redirect('the index') return redirect('the index')
@login_required @login_required
def open_workflow(request,workflow_id): def open_workflow(request,workflow_id):
w = get_object_or_404(Workflow, pk=workflow_id) w = get_object_or_404(Workflow, pk=workflow_id)
...@@ -70,9 +70,9 @@ def open_workflow(request,workflow_id): ...@@ -70,9 +70,9 @@ def open_workflow(request,workflow_id):
request.user.userprofile.active_workflow = w request.user.userprofile.active_workflow = w
request.user.userprofile.save() request.user.userprofile.save()
else: else:
return HttpResponse(status=400) return HttpResponse(status=400)
return redirect('the index') return redirect('the index')
@login_required @login_required
def widget_progress(request): def widget_progress(request):
w = get_object_or_404(Widget, pk=request.GET['widget_id']) w = get_object_or_404(Widget, pk=request.GET['widget_id'])
...@@ -82,7 +82,7 @@ def widget_progress(request): ...@@ -82,7 +82,7 @@ def widget_progress(request):
if w.progress==100: if w.progress==100:
return HttpResponse("100") return HttpResponse("100")
return HttpResponse("-1") return HttpResponse("-1")
@login_required @login_required
def add_widget(request): def add_widget(request):
if request.is_ajax() or DEBUG: if request.is_ajax() or DEBUG:
...@@ -172,7 +172,7 @@ def add_widget(request): ...@@ -172,7 +172,7 @@ def add_widget(request):
j.widget = w j.widget = w
j.required = i.required j.required = i.required
j.parameter = i.parameter j.parameter = i.parameter
j.parameter_type = i.parameter_type j.parameter_type = i.parameter_type
j.value = i.value j.value = i.value
j.multi_id = i.multi_id j.multi_id = i.multi_id
j.save() j.save()
...@@ -192,13 +192,13 @@ def add_widget(request): ...@@ -192,13 +192,13 @@ def add_widget(request):
j.save() j.save()
w.defered_outputs = w.outputs.defer("value").all() w.defered_outputs = w.outputs.defer("value").all()
w.defered_inputs = w.inputs.defer("value").all() w.defered_inputs = w.inputs.defer("value").all()
return render(request, 'widgets.html', {'widgets':[w,]}) return render(request, 'widgets.html', {'widgets':[w,]})
elif aw.type=='subprocess': elif aw.type=='subprocess':
workflow = get_object_or_404(Workflow, pk=request.POST['active_workflow']) workflow = get_object_or_404(Workflow, pk=request.POST['active_workflow'])
if (workflow.user==request.user): if (workflow.user==request.user):
widget_conversion = {} widget_conversion = {}
input_conversion = {} input_conversion = {}
output_conversion = {} output_conversion = {}
w = Widget() w = Widget()
w.workflow = workflow w.workflow = workflow
w.x = int(request.POST['scrollLeft'])+50 w.x = int(request.POST['scrollLeft'])+50
...@@ -220,7 +220,7 @@ def add_widget(request): ...@@ -220,7 +220,7 @@ def add_widget(request):
j.widget = w j.widget = w
j.required = i.required j.required = i.required
j.parameter = i.parameter j.parameter = i.parameter