models.py 36.4 KB
Newer Older
Janez K's avatar
Janez K committed
1 2 3 4 5 6 7 8 9 10 11 12
from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save
import workflows.library

import time

from picklefield.fields import PickledObjectField

from workflows.thumbs import ThumbnailField

from mothra.settings import DEBUG
Janez K's avatar
Janez K committed
13
from mothra.settings import USE_CONCURRENCY
Janez K's avatar
Janez K committed
14

Janez K's avatar
Janez K committed
15
if USE_CONCURRENCY:
Janez K's avatar
Janez K committed
16 17
    from workflows.tasks import runWidgetAsync, runForLoopIteration

18 19
from workflows.tasks import executeWidgetFunction, executeWidgetProgressBar, executeWidgetStreaming, executeWidgetWithRequest

Janez K's avatar
Janez K committed
20 21 22
class WidgetException(Exception):
    pass

Janez K's avatar
Janez K committed
23 24 25 26
class Connection(models.Model):
    output = models.ForeignKey("Output",related_name="connections")
    input = models.ForeignKey("Input",related_name="connections")
    workflow = models.ForeignKey("Workflow",related_name="connections")
Janez K's avatar
Janez K committed
27

Janez K's avatar
Janez K committed
28 29 30 31
class Category(models.Model):
    name = models.CharField(max_length=50)
    parent = models.ForeignKey('self',related_name="children",null=True,blank=True)
    user = models.ForeignKey(User,null=True,blank=True,related_name="categories")
Janez K's avatar
Janez K committed
32

Janez K's avatar
Janez K committed
33
    workflow = models.ForeignKey('Workflow',null=True,blank=True,related_name="categories")
34 35 36

    order = models.PositiveIntegerField(default=1)

Janez K's avatar
Janez K committed
37 38
    uid = models.CharField(max_length=250,blank=True,default='')

Janez K's avatar
Janez K committed
39 40
    class Meta:
        verbose_name_plural = "categories"
41 42
        ordering = ('order','name',)

Janez K's avatar
Janez K committed
43 44 45 46 47 48 49 50 51 52 53 54 55
    def __unicode__(self):
        if self.parent is None:
            return unicode(self.name)
        else:
            return unicode(unicode(self.parent)+" :: "+self.name)

class Workflow(models.Model):
    name = models.CharField(max_length=200,default='Untitled workflow')
    user = models.ForeignKey(User,related_name="workflows")
    public = models.BooleanField(default=False)
    description = models.TextField(blank=True,default='')
    widget = models.OneToOneField('Widget',related_name="workflow_link",blank=True,null=True)
    template_parent = models.ForeignKey('Workflow',blank=True,null=True,default=None,on_delete=models.SET_NULL)
Janez K's avatar
Janez K committed
56

Janez K's avatar
Janez K committed
57 58 59 60 61
    def can_be_streaming(self):
        if self.widgets.filter(abstract_widget__is_streaming=True).count()>0:
            return True
        else:
            return False
Janez K's avatar
Janez K committed
62

Janez K's avatar
Janez K committed
63 64 65 66
    def is_for_loop(self):
        if self.widgets.filter(type='for_input').count()>0:
            return True
        else:
Janez K's avatar
Janez K committed
67 68
            return False

Janez K's avatar
Janez K committed
69 70 71 72 73 74 75 76 77 78 79 80 81 82
    def get_ready_to_run(self):
        widgets = self.widgets.all()
        unfinished_list = []
        for w in widgets:
            if not w.finished and not w.running:
                ready_to_run = True
                connections = self.connections.filter(input__widget=w)
                for c in connections:
                    if not c.output.widget.finished:
                        ready_to_run = False
                        break
                if ready_to_run:
                    unfinished_list.append(w.id)
        return unfinished_list
Janez K's avatar
Janez K committed
83

Janez K's avatar
Janez K committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
    def get_runnable_widgets(self):
        widgets = self.widgets.all()
        unfinished_list = []
        for w in widgets:
            if not w.finished and not w.running:
                ready_to_run = True
                connections = self.connections.filter(input__widget=w)
                for c in connections:
                    if not c.output.widget.finished:
                        ready_to_run = False
                        break
                if ready_to_run:
                    unfinished_list.append(w)
        return unfinished_list

    def run_for_loop(self):
        #clear for_input and for_output
        fi = self.widgets.filter(type='for_input')[0]
        fo = self.widgets.filter(type='for_output')[0]
        outer_output = fo.inputs.all()[0].outer_output
        outer_output.value=[]
        outer_output.save()
Janez K's avatar
Janez K committed
106

Janez K's avatar
Janez K committed
107 108 109 110 111 112 113 114 115 116 117
        input_list = fi.outputs.all()[0].outer_input.value
        progress_total = len(input_list)
        current_iteration = 0
        for i in input_list:
            fi.unfinish()
            fo.unfinish()
            proper_output = fi.outputs.all()[0]
            proper_output.value = i
            proper_output.save()
            fi.finished=True
            fi.save()
Janez K's avatar
Janez K committed
118
            if not USE_CONCURRENCY or 1==1:
Janez K's avatar
Janez K committed
119 120 121 122 123 124
                unfinished_list = self.get_runnable_widgets()
                try:
                    while len(unfinished_list)>0:
                        for w in unfinished_list:
                            w.run(True)
                            total = self.widgets.count()
Janez K's avatar
Janez K committed
125
                            completed = self.widgets.filter(finished=True).count()
Janez K's avatar
Janez K committed
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
                            self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total)))
                            self.widget.save()
                        unfinished_list = self.get_runnable_widgets()
                except:
                    raise
            else:
                unfinished_list = self.get_runnable_widgets()
                try:
                    statuses = {}
                    total = self.widgets.count()
                    completed = self.widgets.filter(finished=True).count()
                    while len(unfinished_list)>0:
                        for w in unfinished_list:
                            if statuses.has_key(w.pk):
                                if statuses[w.pk].failed():
                                    raise Exception(statuses[w.pk].info[0])
                            else:
                                statuses[w.pk]=runWidgetAsync.delay(w)
                            completed = self.widgets.filter(finished=True).count()
                            if self.widget:
                                self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total)))
                                self.widget.save()
                        is_running=True
                        unfinished_list = self.get_runnable_widgets()
                        while len(unfinished_list)==0 and is_running:
                            unfinished_list = self.get_runnable_widgets()
                            is_running = False
                            for st in statuses.values():
                                if st.status == 'PENDING':
                                    is_running = True
                                else:
                                    st.get()
                                    completed = self.widgets.filter(finished=True).count()
                                    if self.widget:
                                        self.widget.progress = (int)((current_iteration*100.0/progress_total)+(((completed*1.0)/total)*(100/progress_total)))
Janez K's avatar
Janez K committed
161
                                        self.widget.save()
Janez K's avatar
Janez K committed
162 163 164 165 166 167
                            unfinished_list = self.get_runnable_widgets()
                except:
                    raise
            current_iteration = current_iteration+1

    def run(self):
Janez K's avatar
Janez K committed
168
        if not USE_CONCURRENCY or not self.widget:
Janez K's avatar
Janez K committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
            unfinished_list = self.get_runnable_widgets()
            try:
                total = self.widgets.count()
                completed = self.widgets.filter(finished=True).count()
                while len(unfinished_list)>0:
                    for w in unfinished_list:
                        w.run(True)
                        #runWidgetAsync.delay(w)
                        completed = self.widgets.filter(finished=True).count()
                        if self.widget:
                            self.widget.progress = (int)(((completed*1.0)/total)*100)
                            self.widget.save()
                    unfinished_list = self.get_runnable_widgets()
            except:
                raise
        else:
            unfinished_list = self.get_runnable_widgets()
            try:
                statuses = {}
                total = self.widgets.count()
                completed = self.widgets.filter(finished=True).count()
                while len(unfinished_list)>0:
                    for w in unfinished_list:
                        if statuses.has_key(w.pk):
                            if statuses[w.pk].failed():
                                raise Exception(statuses[w.pk].info[0])
                        else:
                            statuses[w.pk]=runWidgetAsync.delay(w)
                        completed = self.widgets.filter(finished=True).count()
                        if self.widget:
                            self.widget.progress = (int)(((completed*1.0)/total)*100)
                            self.widget.save()
                    is_running=True
Janez K's avatar
Janez K committed
202
                    unfinished_list = self.get_runnable_widgets()
Janez K's avatar
Janez K committed
203 204 205 206 207 208 209 210 211 212 213
                    while len(unfinished_list)==0 and is_running:
                        unfinished_list = self.get_runnable_widgets()
                        is_running = False
                        for st in statuses.values():
                            if st.status == 'PENDING':
                                is_running = True
                            else:
                                st.get()
                                completed = self.widgets.filter(finished=True).count()
                                if self.widget:
                                    self.widget.progress = (int)(((completed*1.0)/total)*100)
Janez K's avatar
Janez K committed
214
                                    self.widget.save()
Janez K's avatar
Janez K committed
215
                        time.sleep(1)
Janez K's avatar
Janez K committed
216
                    unfinished_list = self.get_runnable_widgets()
Janez K's avatar
Janez K committed
217 218 219 220
            except:
                raise
    def rename(self,new_name):
        self.name = new_name
Janez K's avatar
Janez K committed
221 222
        self.save()

Janez K's avatar
Janez K committed
223 224
    @models.permalink
    def get_absolute_url(self):
Janez K's avatar
Janez K committed
225 226
        return ('open workflow', [str(self.id)])

Janez K's avatar
Janez K committed
227 228 229
    @models.permalink
    def get_copy_url(self):
        return ('copy workflow', [str(self.id)])
Janez K's avatar
Janez K committed
230

Janez K's avatar
Janez K committed
231 232 233
    @models.permalink
    def get_info_url(self):
        return ('workflow information', [str(self.id)])
Janez K's avatar
Janez K committed
234

Janez K's avatar
Janez K committed
235 236
    def __unicode__(self):
        return unicode(self.name)
Janez K's avatar
Janez K committed
237

Janez K's avatar
Janez K committed
238
    class Meta:
Janez K's avatar
Janez K committed
239
        ordering = ['name']
Janez K's avatar
Janez K committed
240 241

class AbstractWidget(models.Model):
Janez K's avatar
Janez K committed
242 243
    name = models.CharField(max_length=200,help_text='Name is the name that will be displayed in the widget repository and under the actual widget itself.')
    action = models.CharField(max_length=200,help_text='Action is the name of a python function that will be called when the widget is executed.')
Janez K's avatar
Janez K committed
244
    wsdl = models.URLField(max_length=200,blank=True,help_text='WSDL and WSDL method are used if the widget is a call of a Web Service. Web Service widgets are usually not entered in the admin panel, but in the application itself by importing a Web Service.')
Janez K's avatar
Janez K committed
245
    wsdl_method = models.CharField(max_length=200,blank=True,default='')
Janez K's avatar
Janez K committed
246 247 248
    description = models.TextField(blank=True,help_text='Description is used for a human readable description of what a widget does. A user will see this when he right clicks the widget and clicks help.')
    category = models.ForeignKey(Category,related_name="widgets",help_text='Category determines to which category this widget belongs. Categories can be nested.')
    visualization_view = models.CharField(max_length=200,blank=True,default='',help_text='Visualization view is (like the action) a python function that is a view that will render a template.')
249
    streaming_visualization_view = models.CharField(max_length=200,blank=True,default='',help_text='Visualization view is (like the action) a python function that is a view that will render a template.')
Janez K's avatar
Janez K committed
250 251
    user = models.ForeignKey(User,blank=True,null=True,related_name="widgets",help_text='If the User field is blank, everyone will see the widget, otherwise just this user. This is mainly used for Web Service imports as they are only visible to users that imported them.')
    interactive = models.BooleanField(default=False,help_text='The widget can be interactive. This means that when a user executes the widget, the action will perform, then the interaction view will be executed and finally the Post interact action will be executed.')
Janez K's avatar
Janez K committed
252 253
    interaction_view = models.CharField(max_length=200,blank=True,default='')
    post_interact_action = models.CharField(max_length=200,blank=True,default='')
Janez K's avatar
Janez K committed
254

Janez K's avatar
Janez K committed
255
    image = ThumbnailField(blank=True,null=True,upload_to="images",size=(34,34),help_text='Image and Treeview image are deprecated and will be phased out soon. Please use the static image field.')
Janez K's avatar
Janez K committed
256
    treeview_image = ThumbnailField(blank=True,null=True,upload_to="treeview",size=(16,16))
257

Janez K's avatar
Janez K committed
258
    static_image = models.CharField(max_length=250,blank=True,default='',help_text='In the static image field just enter the filename of the image (without the path). The path will be $package_name$/icons/widget/$filename$ and $package_name$/icons/treeview/$filename$ where the treeview image is the small image that appears in the treeview on the left side and the widget image is the actual normal sized icon for the widget. IMPORTANT: the static image field only works if the package is set.')
Janez K's avatar
Janez K committed
259

Janez K's avatar
Janez K committed
260 261
    has_progress_bar = models.BooleanField(default=False,help_text='The flag has progress bar determines if the widget implements a progress bar.')
    is_streaming = models.BooleanField(default=False,help_text='The is streaming flag is currently under construction, please do not use it yet.')
Janez K's avatar
Janez K committed
262

Janez K's avatar
Janez K committed
263
    order = models.PositiveIntegerField(default=1,help_text='The Order determines the order in which the widget will be displayed in the repository. This is set automatically when sorting widgets in a single category from the admin.')
Janez K's avatar
Janez K committed
264

Janez K's avatar
Janez K committed
265
    uid = models.CharField(max_length=250,blank=True,default='',help_text='UID is set automatically when you export a package with the -u switch.')
Janez K's avatar
Janez K committed
266

Janez K's avatar
Janez K committed
267
    package = models.CharField(max_length=150,blank=True,default='',help_text='Package is the package name. You are encouraged to use packages.')
Janez K's avatar
Janez K committed
268

269 270
    windows_queue = models.BooleanField(default=False,help_text="This is used for Matjaz Jursic's widgets.")

Janez K's avatar
Janez K committed
271 272
    class Meta:
        ordering = ('order','name',)
273

Janez K's avatar
Janez K committed
274 275 276 277 278 279 280 281 282
    def set_uid(self,commit=False):
        import uuid
        self.uid = uuid.uuid4()
        if commit:
            self.save()
        for i in self.inputs.all():
            i.uid = uuid.uuid4()
            if commit:
                i.save()
bogdan's avatar
bogdan committed
283
            for option in i.options.all():
284 285 286
                option.uid = uuid.uuid4()
                if commit:
                    option.save()
Janez K's avatar
Janez K committed
287 288 289 290 291
        for o in self.outputs.all():
            o.uid = uuid.uuid4()
            if commit:
                o.save()

Janez K's avatar
Janez K committed
292 293 294 295 296 297 298
    def __unicode__(self):
        return unicode(self.name)

class AbstractInput(models.Model):
    name = models.CharField(max_length=200)
    short_name = models.CharField(max_length=3)
    description = models.TextField(blank=True)
Janez K's avatar
Janez K committed
299
    variable = models.CharField(max_length=50,help_text='The variable attribute of both the input and the output are important because this is how the data will be accessed in the python function that is executed when the widget runs.')
Janez K's avatar
Janez K committed
300 301 302
    widget = models.ForeignKey(AbstractWidget,related_name="inputs")
    required = models.BooleanField()
    parameter = models.BooleanField()
Janez K's avatar
Janez K committed
303
    multi = models.BooleanField(default=False,help_text='Inputs with this flag set will behave like this: whenever a connection is added to this input another input will be created on the fly that accepts the same data. In the action function, this will be represented as a list.')
Janez K's avatar
Janez K committed
304 305 306
    default = models.TextField(blank=True)
    PARAMETER_CHOICES = (
        ('text','Single line'),
307
        ('password', 'Password'),
Janez K's avatar
Janez K committed
308 309 310 311 312 313
        ('textarea','Multi line text'),
        ('select', 'Select box'),
        ('checkbox', 'Checkbox'),
        ('file', 'File'),
    )
    parameter_type = models.CharField(max_length=50,choices=PARAMETER_CHOICES,blank=True,null=True)
Janez K's avatar
Janez K committed
314

Janez K's avatar
Janez K committed
315
    order = models.PositiveIntegerField(default=1)
Janez K's avatar
Janez K committed
316 317

    uid = models.CharField(max_length=250,blank=True,default='')
Janez K's avatar
Janez K committed
318

Janez K's avatar
Janez K committed
319 320
    def __unicode__(self):
        return unicode(self.name)
Janez K's avatar
Janez K committed
321

Janez K's avatar
Janez K committed
322 323
    class Meta:
        ordering = ('order',)
Janez K's avatar
Janez K committed
324

Janez K's avatar
Janez K committed
325 326 327 328
class AbstractOption(models.Model):
    abstract_input = models.ForeignKey(AbstractInput,related_name="options")
    name = models.CharField(max_length=200)
    value = models.TextField(blank=True)
Janez K's avatar
Janez K committed
329 330 331

    uid = models.CharField(max_length=250,blank=True,default='')

Janez K's avatar
Janez K committed
332 333
    def __unicode__(self):
        return unicode(self.name)
Janez K's avatar
Janez K committed
334

Janez K's avatar
Janez K committed
335
    class Meta:
Janez K's avatar
Janez K committed
336
        ordering = ['name']
Janez K's avatar
Janez K committed
337 338 339 340 341

class AbstractOutput(models.Model):
    name = models.CharField(max_length=200)
    short_name = models.CharField(max_length=3)
    description = models.TextField(blank=True)
Janez K's avatar
Janez K committed
342
    variable = models.CharField(max_length=50,help_text='The variable attribute of both the input and the output are important because this is how the data will be accessed in the python function that is executed when the widget runs.')
Janez K's avatar
Janez K committed
343
    widget = models.ForeignKey(AbstractWidget,related_name="outputs")
Janez K's avatar
Janez K committed
344

Janez K's avatar
Janez K committed
345
    order = models.PositiveIntegerField(default=1)
Janez K's avatar
Janez K committed
346 347

    uid = models.CharField(max_length=250,blank=True,default='')
Janez K's avatar
Janez K committed
348

Janez K's avatar
Janez K committed
349
    class Meta:
Janez K's avatar
Janez K committed
350 351
        ordering = ('order',)

Janez K's avatar
Janez K committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
    def __unicode__(self):
        return unicode(self.name)

class Widget(models.Model):
    workflow = models.ForeignKey(Workflow,related_name="widgets")
    x = models.IntegerField()
    y = models.IntegerField()
    name = models.CharField(max_length=200)
    abstract_widget = models.ForeignKey(AbstractWidget,related_name="instances",blank=True,null=True)
    finished = models.BooleanField(default=False)
    error = models.BooleanField(default=False)
    running = models.BooleanField(default=False)
    interaction_waiting = models.BooleanField(default=False)
    WIDGET_CHOICES = (
        ('regular','Regular widget'),
        ('subprocess','Subprocess widget'),
        ('input', 'Input widget'),
        ('output', 'Output widget'),
    )
    type = models.CharField(max_length=50,choices=WIDGET_CHOICES,default='regular')
Janez K's avatar
Janez K committed
372

Janez K's avatar
Janez K committed
373
    progress = models.IntegerField(default=0)
Janez K's avatar
Janez K committed
374

Janez K's avatar
Janez K committed
375 376 377 378 379 380
    def is_visualization(self):
        try:
            if self.abstract_widget.visualization_view != '':
                return True
        except:
            return False
Janez K's avatar
Janez K committed
381

Janez K's avatar
Janez K committed
382 383 384 385 386 387
    def ready_to_run(self):
        cons = Connection.objects.filter(input__widget=self)
        for c in cons:
            if not c.output.widget.finished:
                return False
        return True
Janez K's avatar
Janez K committed
388

Janez K's avatar
Janez K committed
389
    def unfinish(self):
Janez K's avatar
Janez K committed
390
        self.reset_descendants()
Janez K's avatar
Janez K committed
391

Janez K's avatar
Janez K committed
392 393 394 395 396 397 398 399
    def subunfinish(self):
        if self.type == 'subprocess':
            for w in self.workflow_link.widgets.all():
                w.finished=False
                w.error = False
                w.save()
                if w.type=='subprocess':
                    w.subunfinish()
Janez K's avatar
Janez K committed
400

Janez K's avatar
Janez K committed
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
    def rename(self,new_name):
        self.name = new_name
        self.save()
        if self.type=='input':
            inp = self.outputs.all()[0]
            inp.short_name = self.name[:3]
            inp.name = self.name
            inp.save()
            inp.outer_input.name = self.name
            inp.outer_input.short_name = self.name[:3]
            inp.outer_input.save()
        if self.type=='output':
            inp = self.inputs.all()[0]
            inp.short_name = self.name[:3]
            inp.name = self.name
            inp.save()
            inp.outer_output.name = self.name
            inp.outer_output.short_name = self.name[:3]
Janez K's avatar
Janez K committed
419
            inp.outer_output.save()
Janez K's avatar
Janez K committed
420 421 422 423 424 425
        try:
            w_link = self.workflow_link
            w_link.name=new_name
            w_link.save()
        except Workflow.DoesNotExist:
            pass
Janez K's avatar
Janez K committed
426

Janez K's avatar
Janez K committed
427 428
    def run(self,offline):
        if not self.ready_to_run():
Janez K's avatar
Janez K committed
429
            raise WidgetException("The prerequisites for running this widget have not been met.")
Janez K's avatar
Janez K committed
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
        self.running=True
        self.save()
        if self.type == 'regular' or self.type == 'subprocess':
            if not self.abstract_widget is None:
                function_to_call = getattr(workflows.library,self.abstract_widget.action)
            input_dict = {}
            outputs = {}
            for i in self.inputs.all():
                #gremo pogledat ce obstaja povezava in ce obstaja gremo value prebrat iz outputa
                if not i.parameter:
                    if i.connections.count() > 0:
                        i.value = i.connections.all()[0].output.value
                        i.save()
                    else:
                        i.value = None
                        i.save()
                if i.multi_id == 0:
                    input_dict[i.variable]=i.value
                else:
                    if not i.variable in input_dict:
                        input_dict[i.variable]=[]
                    if not i.value==None:
                        input_dict[i.variable].append(i.value)
Janez K's avatar
Janez K committed
453
            start = time.time()
Janez K's avatar
Janez K committed
454 455 456 457 458 459
            try:
                if not self.abstract_widget is None:
                    if self.abstract_widget.wsdl != '':
                        input_dict['wsdl']=self.abstract_widget.wsdl
                        input_dict['wsdl_method']=self.abstract_widget.wsdl_method
                    if self.abstract_widget.has_progress_bar:
460 461 462 463 464
                        if self.abstract_widget.windows_queue:
                            t = executeWidgetProgressBar.apply_async([self,input_dict],queue="windows")
                            outputs = t.wait()
                        else:
                            outputs = executeWidgetProgressBar(self,input_dict)
Janez K's avatar
Janez K committed
465
                    elif self.abstract_widget.is_streaming:
466 467 468 469 470
                        if self.abstract_widget.windows_queue:
                            t = executeWidgetStreaming.apply_async([self,input_dict],queue="windows")
                            outputs = t.wait()
                        else:
                            outputs = executeWidgetStreaming(self,input_dict)
Janez K's avatar
Janez K committed
471
                    else:
472 473 474 475 476
                        if self.abstract_widget.windows_queue:
                            t = executeWidgetFunction.apply_async([self,input_dict],queue="windows")
                            outputs = t.wait()
                        else:
                            outputs = executeWidgetFunction(self,input_dict)
Janez K's avatar
Janez K committed
477 478 479
                else:
                    if self.workflow_link.is_for_loop():
                        self.workflow_link.run_for_loop()
480
                        #print self.outputs.all()[0].value
Janez K's avatar
Janez K committed
481 482 483 484 485 486 487 488
                    else:
                        self.workflow_link.run()
            except:
                self.error=True
                self.running=False
                self.finished=False
                self.save()
                raise
Janez K's avatar
Janez K committed
489 490
            elapsed = (time.time()-start)
            outputs['clowdflows_elapsed']=elapsed
Janez K's avatar
Janez K committed
491 492
            for o in self.outputs.all():
                if not self.abstract_widget is None:
Janez K's avatar
Janez K committed
493 494 495 496
                    try:
                        o.value = outputs[o.variable]
                    except:
                        pass
Janez K's avatar
Janez K committed
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
                    o.save()
                else:
                    #gremo v outpute pogledat
                    if not self.workflow_link.is_for_loop():
                        o.value = o.inner_input.value
                        o.save()
            if self.abstract_widget is None:
                self.finished=True
                self.running=False
                self.error=False
                self.save()
            else:
                if not self.abstract_widget.interactive or offline:
                    self.finished=True
                    self.running=False
                    self.error=False
                    self.save()
            cons = Connection.objects.filter(output__widget=self)
            for c in cons:
                c.input.widget.unfinish()
            return outputs
        elif self.type == 'for_input':
            for o in self.outputs.all():
                o.value=o.outer_input.value
                o.save()
            self.finished=True
            self.running=False
            self.error=False
            self.save()
        elif self.type == 'for_output':
            for i in self.inputs.all():
                if not i.parameter:
                    if i.connections.count() > 0:
                        i.value = i.connections.all()[0].output.value
                        i.save()
                        i.outer_output.value.append(i.value)
                        i.outer_output.save()
                        self.finished=True
            self.finished=True
            self.running=False
            self.error=False
            self.save()
        elif self.type == 'input':
            for o in self.outputs.all():
                o.value=o.outer_input.value
                o.save()
            self.finished=True
            self.running=False
            self.error=False
            self.save()
        elif self.type == 'output':
            for i in self.inputs.all():
                if not i.parameter:
                    if i.connections.count() > 0:
                        i.value = i.connections.all()[0].output.value
                        i.save()
                        i.outer_output.value = i.value
                        i.outer_output.save()
                        self.finished=True
Janez K's avatar
Janez K committed
556
            self.finished=True
Janez K's avatar
Janez K committed
557 558 559 560
            self.running=False
            self.error=False
            self.save()
        return None
Janez K's avatar
Janez K committed
561

562
    def reset(self,offline):
Janez K's avatar
Janez K committed
563 564 565 566 567 568 569
        #for i in self.inputs.all():
        #    if not i.parameter:
        #        i.value = None
        #        i.save()
        #for i in self.outputs.all():
        #    i.value = None
        #    i.save()
570 571 572 573
        self.finished = False
        self.error = False
        self.running = False
        self.save()
Janez K's avatar
Janez K committed
574 575
        if self.type == 'subprocess':
            self.subunfinish()
576

Janez K's avatar
Janez K committed
577
    def reset_descendants(self):
Janez K's avatar
Janez K committed
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
        pairs = []
        for c in self.workflow.connections.select_related("output","input").defer("output__value","input__value").all():
            if not (c.output.widget_id,c.input.widget_id) in pairs:
                pairs.append((c.output.widget_id,c.input.widget_id))
        next = {}                
        for p in pairs:
            if not next.has_key(p[0]):
                next[p[0]]=set()
            next[p[0]].add(p[1])
        widgets_that_need_reset = set([self.pk,])
        current_widgets_that_need_reset = set([self.pk,])
        while len(current_widgets_that_need_reset)>0:
            new_widgets_that_need_reset = set()
            for w_id in current_widgets_that_need_reset:
                try:
                    for p in next.get(w_id):
                        new_widgets_that_need_reset.add(p)
                        widgets_that_need_reset.add(p)
                except:
                    pass
            current_widgets_that_need_reset = new_widgets_that_need_reset
        Widget.objects.filter(id__in=widgets_that_need_reset).update(finished=False,error=False,running=False)
        subprocesses = Widget.objects.filter(id__in=widgets_that_need_reset,type='subprocess')
        for w in subprocesses:
            w.subunfinish()
        return widgets_that_need_reset

    def reset_descendants_slow(self):
Janez K's avatar
Janez K committed
606
        #find all descendants and reset them as well
Janez K's avatar
Janez K committed
607
        widgets = list(self.workflow.widgets.prefetch_related('inputs','outputs','inputs__connections','outputs__connections','outputs__connections__input','inputs__connections__output'))
Janez K's avatar
Janez K committed
608 609 610 611 612 613 614 615 616 617 618 619 620
        widgets_dict = {}
        widgets_that_need_reset = set([self.pk,])
        current_widgets_that_need_reset = set([self.pk,])
        for w in widgets:
            widgets_dict[w.pk]=w
        while len(current_widgets_that_need_reset)>0:
            new_widgets_that_need_reset = set()
            for w_id in current_widgets_that_need_reset:
                for o in widgets_dict[w_id].outputs.all():
                    for c in o.connections.all():
                        new_widgets_that_need_reset.add(c.input.widget_id)
                        widgets_that_need_reset.add(c.input.widget_id)
            current_widgets_that_need_reset = new_widgets_that_need_reset
Janez K's avatar
Janez K committed
621
        Widget.objects.filter(id__in=widgets_that_need_reset).update(finished=False,error=False,running=False)
Janez K's avatar
Janez K committed
622
        for w in widgets_that_need_reset:
Janez K's avatar
Janez K committed
623 624 625
            if widgets_dict[w].type == 'subprocess':
                widgets_dict[w].subunfinish()
        #    widgets_dict[w].reset(False)
Janez K's avatar
Janez K committed
626 627
        return widgets_that_need_reset

Janez K's avatar
Janez K committed
628 629
    def run_post(self,request):
        if not self.ready_to_run():
Janez K's avatar
Janez K committed
630
            raise WidgetException("The prerequisites for running this widget have not been met.")
Janez K's avatar
Janez K committed
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
        self.running=True
        self.save()
        function_to_call = getattr(workflows.library,self.abstract_widget.post_interact_action)
        input_dict = {}
        outputs = {}
        output_dict = {}
        for o in self.outputs.all():
            output_dict[o.variable]=o.value
        for i in self.inputs.all():
            #gremo pogledat ce obstaja povezava in ce obstaja gremo value prebrat iz outputa
            if not i.parameter:
                if i.connections.count() > 0:
                    i.value = i.connections.all()[0].output.value
                    i.save()
                else:
                    i.value = None
                    i.save()
            if i.multi_id == 0:
                input_dict[i.variable]=i.value
            else:
                if not i.variable in input_dict:
                    input_dict[i.variable]=[]
                if not i.value==None:
                    input_dict[i.variable].append(i.value)
        try:
            if not self.abstract_widget is None:
657 658 659 660 661
                if self.abstract_widget.windows_queue:
                    t = executeWidgetWithRequest.apply_async([widget,input_dict,output_dict,request],queue="windows")
                    outputs = t.wait()
                else:
                    outputs = executeWidgetWithRequest(widget,input_dict,output_dict,request)
Janez K's avatar
Janez K committed
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
            else:
                self.workflow_link.run()
        except:
            self.error=True
            self.running=False
            self.finished=False
            self.save()
            raise
        for o in self.outputs.all():
            o.value = outputs[o.variable]
            o.save()
        self.finished=True
        self.running=False
        self.error=False
        self.interaction_waiting=False
        self.save()
        cons = Connection.objects.filter(output__widget=self)
        for c in cons:
            c.input.widget.unfinish()
Janez K's avatar
Janez K committed
681 682
        return outputs

Janez K's avatar
Janez K committed
683 684
    def __unicode__(self):
        return unicode(self.name)
Janez K's avatar
Janez K committed
685

Janez K's avatar
Janez K committed
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
class Input(models.Model):
    name = models.CharField(max_length=200)
    short_name = models.CharField(max_length=3)
    description = models.TextField(blank=True,null=True)
    variable = models.CharField(max_length=50)
    widget = models.ForeignKey(Widget,related_name="inputs")
    required = models.BooleanField()
    parameter = models.BooleanField()
    value = PickledObjectField(null=True)
    multi_id = models.IntegerField(default=0)
    inner_output = models.ForeignKey('Output',related_name="outer_input_rel",blank=True,null=True) #za subprocess
    outer_output = models.ForeignKey('Output',related_name="inner_input_rel",blank=True,null=True) #za subprocess
    PARAMETER_CHOICES = (
        ('text','Single line'),
        ('textarea','Multi line text'),
        ('select', 'Select box'),
    )
    parameter_type = models.CharField(max_length=50,choices=PARAMETER_CHOICES,blank=True,null=True)
704 705 706 707
    order = models.PositiveIntegerField(default=1)

    class Meta:
        ordering = ('order',)
Janez K's avatar
Janez K committed
708

Janez K's avatar
Janez K committed
709
    def __unicode__(self):
Janez K's avatar
Janez K committed
710 711
        return unicode(self.name)

Janez K's avatar
Janez K committed
712 713 714 715
class Option(models.Model):
    input = models.ForeignKey(Input,related_name="options")
    name = models.CharField(max_length=200)
    value = models.TextField(blank=True,null=True)
Janez K's avatar
Janez K committed
716

Janez K's avatar
Janez K committed
717
    class Meta:
Janez K's avatar
Janez K committed
718
        ordering = ['name']
Janez K's avatar
Janez K committed
719 720 721 722 723 724 725 726 727 728

class Output(models.Model):
    name = models.CharField(max_length=200)
    short_name = models.CharField(max_length=5)
    description = models.TextField(blank=True)
    variable = models.CharField(max_length=50)
    widget = models.ForeignKey(Widget,related_name="outputs")
    value = PickledObjectField(null=True)
    inner_input = models.ForeignKey(Input,related_name="outer_output_rel",blank=True,null=True) #za subprocess
    outer_input = models.ForeignKey(Input,related_name="inner_output_rel",blank=True,null=True) #za subprocess
729 730 731 732
    order = models.PositiveIntegerField(default=1)

    class Meta:
        ordering = ('order',)
Janez K's avatar
Janez K committed
733

Janez K's avatar
Janez K committed
734 735 736 737 738 739
    def __unicode__(self):
        return unicode(self.name)

class UserProfile(models.Model):
    user = models.OneToOneField(User,related_name="userprofile")
    active_workflow = models.ForeignKey(Workflow,related_name="users",null=True,blank=True,on_delete=models.SET_NULL)
Janez K's avatar
Janez K committed
740

Janez K's avatar
Janez K committed
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
    def __unicode__(self):
        return unicode(self.user)

def create_user_profile(sender, instance, created, **kwargs):
    profile_set = UserProfile.objects.filter(user__id = instance.id)
    if created and not profile_set.exists():
        UserProfile.objects.create(user=instance)

# nardi da k nardimo userja da se avtomatsko nardi se UserProfile
post_save.connect(create_user_profile, sender=User)

def copy_workflow(old, user, parent_widget_conversion={},parent_input_conversion={},parent_output_conversion={},parent_widget=None):
    w = Workflow()
    if parent_widget is None:
        w.name = old.name+" (copy)"
    else:
        w.name = old.name
    w.user = user
    w.public = False
    w.description = old.description
    w.template_parent = old
    if not parent_widget is None:
        w.widget = parent_widget
    w.save()
    widget_conversion = {}
    input_conversion = {}
    output_conversion = {}
    for widget in old.widgets.all():
        new_widget = Widget()
        new_widget.workflow = w
        new_widget.x = widget.x
        new_widget.y = widget.y
        new_widget.name = widget.name
        new_widget.abstract_widget = widget.abstract_widget
        new_widget.finished = widget.finished
        new_widget.error = widget.error
        new_widget.running = widget.running
        new_widget.interaction_waiting = widget.interaction_waiting
        new_widget.type = widget.type
        new_widget.progress = widget.progress
        new_widget.save()
        widget_conversion[widget.id]=new_widget.id
        for input in widget.inputs.all():
            new_input = Input()
            new_input.name = input.name
            new_input.short_name = input.short_name
            new_input.description = input.description
            new_input.variable = input.variable
            new_input.widget = new_widget
            new_input.required = input.required
            new_input.parameter = input.parameter
            new_input.value = input.value
            new_input.multi_id = input.multi_id
            #inner_output nikol ne nastavlamo
            #outer_output in njemu spremenimo inner input
            if not parent_widget is None:
                if not input.outer_output is None:
                    new_input.outer_output = Output.objects.get(pk=parent_output_conversion[input.outer_output.id])
            new_input.parameter_type = input.parameter_type
            new_input.save()
            for option in input.options.all():
                new_option = Option()
                new_option.input = new_input
                new_option.name = option.name
                new_option.value = option.value
                new_option.save()
            if not parent_widget is None:
                if not input.outer_output is None:
                    new_input.outer_output.inner_input = new_input
                    new_input.outer_output.save()
            input_conversion[input.id]=new_input.id
        for output in widget.outputs.all():
            new_output = Output()
            new_output.name = output.name
            new_output.short_name = output.short_name
            new_output.description = output.description
            new_output.variable = output.variable
            new_output.widget = new_widget
            new_output.value = output.value
            #inner input nikol ne nastavlamo
            #outer input in njemu spremenimo inner output
            if not parent_widget is None:
                if not output.outer_input is None:
                    new_output.outer_input = Input.objects.get(pk=parent_input_conversion[output.outer_input.id])
            new_output.save()
            if not parent_widget is None:
                if not output.outer_input is None:
                    new_output.outer_input.inner_output = new_output
                    new_output.outer_input.save()
            output_conversion[output.id]=new_output.id
    for connection in old.connections.all():
        new_connection = Connection()
        new_connection.workflow = w
        new_connection.output = Output.objects.get(pk=output_conversion[connection.output.id])
        new_connection.input = Input.objects.get(pk=input_conversion[connection.input.id])
        new_connection.save()
    for widget in old.widgets.filter(type='subprocess'):
        #tuki mormo vse subprocesse zrihtat
        copy_workflow(widget.workflow_link, user, widget_conversion, input_conversion, output_conversion, Widget.objects.get(pk=widget_conversion[widget.id]))
Janez K's avatar
Janez K committed
840
    return w