Commit 528fec6b authored by Janez K's avatar Janez K

importing and exporting workflows, fixed bug with subprocesses

parent 1c6e78a7
from django import forms
import json
from workflows.models import AbstractWidget
class ImportForm(forms.Form):
data = forms.CharField(widget=forms.Textarea(attrs={'class':'formfieldclass'}))
def clean_data(self):
def get_all_abstract_widgets(jsondata,awset):
widgets = jsondata.get('widgets',None)
for w in widgets:
if w['abstract_widget']:
awset.add((w['abstract_widget'],w['abstract_widget_package'],w['name']))
if w['workflow']:
awset = get_all_abstract_widgets(w['workflow'],awset)
return awset
data = self.cleaned_data['data']
try:
d = json.loads(data)
except:
raise forms.ValidationError("Please enter valid json.")
try:
abstract_widgets = get_all_abstract_widgets(d,set())
except:
raise forms.ValidationError("Please enter valid workflow data.")
abstract_widgets = list(abstract_widgets)
for aw in abstract_widgets:
if AbstractWidget.objects.filter(uid=aw[0],package=aw[1]).count()==0:
raise forms.ValidationError("The widget "+aw[2]+" from tha package "+aw[1]+" is missing in this installation. Cannot import!")
return data
\ No newline at end of file
...@@ -155,4 +155,11 @@ header h1 { ...@@ -155,4 +155,11 @@ header h1 {
.popover-title { .popover-title {
background-color:rgba(240,240,240,0.8); background-color:rgba(240,240,240,0.8);
}
.formfieldclass {
display:block;
width:100%;
height:500px;
font-family:monospace;
} }
\ No newline at end of file
{% extends 'website/base.html' %}
{% load url from future %}
{% block "yourworkflowsactive" %}class="active"{% endblock %}
{% block "container" %}
<div class="container">
<h1>Export workflow</h1>
<textarea style="width:100%;height:600px;font-family:monospace;">{{exported_data}}</textarea>
<h2>What to do with this?</h2>
<p>You can copy and paste this into another installation of ClowdFlows. Just make sure that the other installation has all the appropriate packages imported.</p>
</div>
{% endblock %}
{% extends 'website/base.html' %}
{% load url from future %}
{% block "yourworkflowsactive" %}class="active"{% endblock %}
{% block "container" %}
<div class="container">
<h1>Import workflow</h1>
<p>Paste your workflow data in the box below:</p>
<form method="post">
{%csrf_token%}
{{form.as_p}}
<button type="submit" class="btn btn-primary">Import</button>
</form>
</div>
{% endblock %}
...@@ -28,13 +28,15 @@ ...@@ -28,13 +28,15 @@
{% else %}<a href="{% url 'start stream' w.pk %}" class="btn btn-info btn-xs">Start stream mining</a>{% endif %} {% else %}<a href="{% url 'start stream' w.pk %}" class="btn btn-info btn-xs">Start stream mining</a>{% endif %}
{% else %}<span class="label label-default"><span class="glyphicon glyphicon-remove"></span> No streaming widgets</span>{% endif %} {% else %}<span class="label label-default"><span class="glyphicon glyphicon-remove"></span> No streaming widgets</span>{% endif %}
</td> </td>
<td><a href="{{w.get_absolute_url}}">Edit</a> | <a href="{{w.get_copy_url}}">Open as new</a> | <a href="javascript:;" rel="{{w.pk}}" class="delete_workflow">Delete</a> | {% if not w.public %}<a href="{% url 'make public' w.pk %}">Make public</a>{% else %}<a href="{% url 'make private' w.pk %}">Make private</a>{% endif %}</td> <td><a href="{{w.get_absolute_url}}">Edit</a> | <a href="{{w.get_copy_url}}">Open as new</a> | <a href="javascript:;" rel="{{w.pk}}" class="delete_workflow">Delete</a> | {% if not w.public %}<a href="{% url 'make public' w.pk %}">Make public</a>{% else %}<a href="{% url 'make private' w.pk %}">Make private</a>{% endif %} | <a href="{{w.get_export_url}}">Export</a></td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
<a href="{% url "import workflow" %}">Import a workflow from another installation of ClowdFlows</a>
</div> </div>
......
...@@ -17,6 +17,8 @@ urlpatterns = patterns('', ...@@ -17,6 +17,8 @@ urlpatterns = patterns('',
url(r'^make-private/(?P<workflow_id>[0-9]+)/$', 'website.views.make_private', name='make private'), url(r'^make-private/(?P<workflow_id>[0-9]+)/$', 'website.views.make_private', name='make private'),
url(r'^make-public/(?P<workflow_id>[0-9]+)/$', 'website.views.make_public', name='make public'), url(r'^make-public/(?P<workflow_id>[0-9]+)/$', 'website.views.make_public', name='make public'),
url(r'^export-workflow/(?P<workflow_id>[0-9]+)/$', 'website.views.export_workflow', name='export workflow'),
url(r'^import-workflow/$', 'website.views.import_workflow', name='import workflow'),
url(r'^workflow/(?P<workflow_id>[0-9]+)/$', 'website.views.workflow_information', name='workflow information'), url(r'^workflow/(?P<workflow_id>[0-9]+)/$', 'website.views.workflow_information', name='workflow information'),
url(r'^editor/$', 'website.views.editor', name='editor'), url(r'^editor/$', 'website.views.editor', name='editor'),
......
...@@ -19,6 +19,10 @@ from django.template.loader import get_template ...@@ -19,6 +19,10 @@ from django.template.loader import get_template
from django.template import TemplateDoesNotExist from django.template import TemplateDoesNotExist
import os import os
import json
from website.forms import ImportForm
def index(request): def index(request):
return render(request, 'website/index.html') return render(request, 'website/index.html')
...@@ -191,3 +195,21 @@ def editor(request): ...@@ -191,3 +195,21 @@ def editor(request):
tutorial = False tutorial = False
return render(request, 'website/editor.html', {'tutorial':tutorial}) return render(request, 'website/editor.html', {'tutorial':tutorial})
@login_required
def export_workflow(request,workflow_id):
w = get_object_or_404(Workflow, pk=workflow_id)
exported_data = json.dumps(w.export(),indent=2)
return render(request,'website/export_workflow.html',{"workflow":w,'exported_data':exported_data})
@login_required
def import_workflow(request):
if request.method == "POST":
form = ImportForm(request.POST)
if form.is_valid():
new_workflow = Workflow()
new_workflow.user = request.user
new_workflow.import_from_json(json.loads(form.cleaned_data['data']),{},{})
return redirect(new_workflow.get_absolute_url())
else:
form = ImportForm()
return render(request,'website/import_workflow.html',{"form":form})
\ No newline at end of file
...@@ -33,6 +33,11 @@ class Connection(models.Model): ...@@ -33,6 +33,11 @@ class Connection(models.Model):
d['input_id']=self.input.pk d['input_id']=self.input.pk
return d return d
def import_from_json(self,json_data,input_conversion,output_conversion):
self.output_id = output_conversion[json_data['output_id']]
self.input_id = input_conversion[json_data['input_id']]
self.save()
class Category(models.Model): class Category(models.Model):
name = models.CharField(max_length=50) name = models.CharField(max_length=50)
parent = models.ForeignKey('self',related_name="children",null=True,blank=True) parent = models.ForeignKey('self',related_name="children",null=True,blank=True)
...@@ -70,6 +75,25 @@ class Workflow(models.Model): ...@@ -70,6 +75,25 @@ class Workflow(models.Model):
widget = models.OneToOneField('Widget',related_name="workflow_link",blank=True,null=True) widget = models.OneToOneField('Widget',related_name="workflow_link",blank=True,null=True)
template_parent = models.ForeignKey('Workflow',blank=True,null=True,default=None,on_delete=models.SET_NULL) template_parent = models.ForeignKey('Workflow',blank=True,null=True,default=None,on_delete=models.SET_NULL)
def import_from_json(self,json_data,input_conversion,output_conversion):
self.name = json_data['name']
self.description = json_data['description']
self.save()
for widget in json_data['widgets']:
w = Widget()
w.workflow = self
w.import_from_json(widget,input_conversion,output_conversion)
if widget['workflow']:
subwidget_workflow = Workflow()
subwidget_workflow.user = self.user
subwidget_workflow.import_from_json(widget['workflow'],input_conversion,output_conversion)
subwidget_workflow.widget = w
subwidget_workflow.save()
for connection in json_data['connections']:
c = Connection()
c.workflow = self
c.import_from_json(connection,input_conversion,output_conversion)
def export(self): def export(self):
""" Exports the workflow to a dictionary that can be imported """ """ Exports the workflow to a dictionary that can be imported """
d = {} d = {}
...@@ -132,11 +156,11 @@ class Workflow(models.Model): ...@@ -132,11 +156,11 @@ class Workflow(models.Model):
unfinished_list = [] unfinished_list = []
for w in widgets: for w in widgets:
if not w.finished and not w.running: if not w.finished and not w.running:
""" if widget isn't finished and is not running than true"""
ready_to_run = True ready_to_run = True
connections = self.connections.filter(input__widget=w) connections = self.connections.filter(input__widget=w)
for c in connections: for c in connections:
if not c.output.widget.finished: if not c.output.widget.finished:
#print c.output.widget
ready_to_run = False ready_to_run = False
break break
if ready_to_run: if ready_to_run:
...@@ -147,7 +171,7 @@ class Workflow(models.Model): ...@@ -147,7 +171,7 @@ class Workflow(models.Model):
""" Method runs the workflow for loop. The use of [0] at the end of lines is because """ Method runs the workflow for loop. The use of [0] at the end of lines is because
there can be only one for loop in one workflow. This way we take the first one. """ there can be only one for loop in one workflow. This way we take the first one. """
#clear for_input and for_output #clear for_input and for_output
print("run_for_loop") #print("run_for_loop")
fi = self.widgets.filter(type='for_input')[0] fi = self.widgets.filter(type='for_input')[0]
fo = self.widgets.filter(type='for_output')[0] fo = self.widgets.filter(type='for_output')[0]
outer_output = fo.inputs.all()[0].outer_output outer_output = fo.inputs.all()[0].outer_output
...@@ -221,7 +245,7 @@ class Workflow(models.Model): ...@@ -221,7 +245,7 @@ class Workflow(models.Model):
def run_cross_validation(self): def run_cross_validation(self):
""" Method runs cross_validation. """ """ Method runs cross_validation. """
#clear for_input and for_output #clear for_input and for_output
print("run_cross_validation") #print("run_cross_validation")
import random as rand import random as rand
fi = self.widgets.filter(type='cv_input')[0] fi = self.widgets.filter(type='cv_input')[0]
fo = self.widgets.filter(type='cv_output')[0] fo = self.widgets.filter(type='cv_output')[0]
...@@ -334,6 +358,7 @@ class Workflow(models.Model): ...@@ -334,6 +358,7 @@ class Workflow(models.Model):
def run(self): def run(self):
if not USE_CONCURRENCY or not self.widget: if not USE_CONCURRENCY or not self.widget:
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
#print unfinished_list
try: try:
total = self.widgets.count() total = self.widgets.count()
completed = self.widgets.filter(finished=True).count() completed = self.widgets.filter(finished=True).count()
...@@ -346,6 +371,7 @@ class Workflow(models.Model): ...@@ -346,6 +371,7 @@ class Workflow(models.Model):
self.widget.progress = (int)(((completed*1.0)/total)*100) self.widget.progress = (int)(((completed*1.0)/total)*100)
self.widget.save() self.widget.save()
unfinished_list = self.get_runnable_widgets() unfinished_list = self.get_runnable_widgets()
#print unfinished_list
except: except:
raise raise
else: else:
...@@ -399,6 +425,10 @@ class Workflow(models.Model): ...@@ -399,6 +425,10 @@ class Workflow(models.Model):
def get_info_url(self): def get_info_url(self):
return ('workflow information', [str(self.id)]) return ('workflow information', [str(self.id)])
@models.permalink
def get_export_url(self):
return ('export workflow', [str(self.id)])
def __unicode__(self): def __unicode__(self):
return unicode(self.name) return unicode(self.name)
...@@ -564,6 +594,25 @@ class Widget(models.Model): ...@@ -564,6 +594,25 @@ class Widget(models.Model):
progress = models.IntegerField(default=0) progress = models.IntegerField(default=0)
def import_from_json(self,json_data,input_conversion,output_conversion):
self.x = json_data['x']
self.y = json_data['y']
self.name = json_data['name']
if json_data['abstract_widget']:
aw = AbstractWidget.objects.get(uid=json_data['abstract_widget'],package=json_data['abstract_widget_package'])
self.abstract_widget = aw
self.type = json_data['type']
self.save()
for i in json_data['inputs']:
new_i = Input()
new_i.widget = self
new_i.import_from_json(i,input_conversion,output_conversion)
for o in json_data['outputs']:
new_o = Output()
new_o.widget = self
new_o.import_from_json(o,input_conversion,output_conversion)
def export(self): def export(self):
d = {} d = {}
try: try:
...@@ -580,16 +629,17 @@ class Widget(models.Model): ...@@ -580,16 +629,17 @@ class Widget(models.Model):
if self.abstract_widget: if self.abstract_widget:
if self.abstract_widget.uid: if self.abstract_widget.uid:
d['abstract_widget']=self.abstract_widget.uid d['abstract_widget']=self.abstract_widget.uid
d['abstract_widget_package']=self.abstract_widget.package
else: else:
raise Exception("Cannot export a widget that doesn't have a UID. ("+str(self.name)+")") raise Exception("Cannot export a widget that doesn't have a UID. ("+str(self.name)+")")
else: else:
d['abstract_widget']=None d['abstract_widget']=None
d['finished']=self.finished #d['finished']=self.finished
d['error']=self.error #d['error']=self.error
d['running']=self.running #d['running']=self.running
d['interaction_waiting']=self.interaction_waiting #d['interaction_waiting']=self.interaction_waiting
d['type']=self.type d['type']=self.type
d['progress']=self.progress #d['progress']=self.progress
d['inputs']=[] d['inputs']=[]
d['outputs']=[] d['outputs']=[]
for i in self.inputs.all(): for i in self.inputs.all():
...@@ -663,7 +713,7 @@ class Widget(models.Model): ...@@ -663,7 +713,7 @@ class Widget(models.Model):
def proper_run(self,offline): def proper_run(self,offline):
""" This is the real start. """ """ This is the real start. """
print("proper_run_widget") #print("proper_run_widget")
if not self.ready_to_run(): if not self.ready_to_run():
raise WidgetException("The prerequisites for running this widget have not been met.") raise WidgetException("The prerequisites for running this widget have not been met.")
self.running=True self.running=True
...@@ -715,7 +765,7 @@ class Widget(models.Model): ...@@ -715,7 +765,7 @@ class Widget(models.Model):
if self.workflow_link.is_for_loop(): if self.workflow_link.is_for_loop():
""" if this is object is a for loop than true and run; """ if this is object is a for loop than true and run;
else false and run workflow """ else false and run workflow """
print("proper_run_is_for_loop") #print("proper_run_is_for_loop")
self.workflow_link.run_for_loop() self.workflow_link.run_for_loop()
#print self.outputs.all()[0].value #print self.outputs.all()[0].value
elif self.workflow_link.is_cross_validation(): elif self.workflow_link.is_cross_validation():
...@@ -844,6 +894,7 @@ class Widget(models.Model): ...@@ -844,6 +894,7 @@ class Widget(models.Model):
self.finished=True self.finished=True
self.running=False self.running=False
self.error=False self.error=False
self.save()
return None return None
def reset(self,offline): def reset(self,offline):
...@@ -996,6 +1047,29 @@ class Input(models.Model): ...@@ -996,6 +1047,29 @@ class Input(models.Model):
class Meta: class Meta:
ordering = ('order',) ordering = ('order',)
def import_from_json(self,json_data,input_conversion,output_conversion):
self.name = json_data['name']
self.short_name = json_data['short_name']
self.description = json_data['description']
self.variable = json_data['variable']
self.required = json_data['required']
self.parameter = json_data['parameter']
self.multi_id = json_data['multi_id']
self.parameter_type = json_data['parameter_type']
self.order = json_data['order']
self.save()
input_conversion[json_data['pk']]=self.pk
for o in json_data['options']:
o = Option()
o.input = self
o.import_from_json(json_data,input_conversion,output_conversion)
o.save()
if json_data['outer_output']:
self.outer_output = Output.objects.get(pk=output_conversion[json_data['outer_output']])
self.outer_output.inner_input = self
self.outer_output.save()
self.save()
def export(self): def export(self):
d = {} d = {}
d['name']=self.name d['name']=self.name
...@@ -1073,6 +1147,11 @@ class Option(models.Model): ...@@ -1073,6 +1147,11 @@ class Option(models.Model):
name = models.CharField(max_length=200) name = models.CharField(max_length=200)
value = models.TextField(blank=True,null=True) value = models.TextField(blank=True,null=True)
def import_from_json(self,json_data,input_conversion,output_conversion):
self.name = json_data['name']
self.value = json_data['value']
self.save()
def export(self): def export(self):
d = {} d = {}
d['name']=self.name d['name']=self.name
...@@ -1093,6 +1172,20 @@ class Output(models.Model): ...@@ -1093,6 +1172,20 @@ class Output(models.Model):
outer_input = models.ForeignKey(Input,related_name="inner_output_rel",blank=True,null=True) #za subprocess outer_input = models.ForeignKey(Input,related_name="inner_output_rel",blank=True,null=True) #za subprocess
order = models.PositiveIntegerField(default=1) order = models.PositiveIntegerField(default=1)
def import_from_json(self,json_data,input_conversion,output_conversion):
self.name = json_data['name']
self.short_name = json_data['short_name']
self.description = json_data['description']
self.variable = json_data['variable']
self.order = json_data['order']
self.save()
output_conversion[json_data['pk']]=self.pk
if json_data['outer_input']:
self.outer_input = Input.objects.get(pk=input_conversion[json_data['outer_input']])
self.outer_input.inner_output = self
self.outer_input.save()
self.save()
def export(self): def export(self):
d = {} d = {}
......
...@@ -29,6 +29,8 @@ from mothra.settings import DEBUG, FILES_FOLDER ...@@ -29,6 +29,8 @@ from mothra.settings import DEBUG, FILES_FOLDER
#ostalo #ostalo
import os import os
import json
from workflows import module_importer from workflows import module_importer
def setattr_local(name, value, package): def setattr_local(name, value, package):
setattr(sys.modules[__name__], name, value) setattr(sys.modules[__name__], name, value)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment