Skip to content

GitLab

C
clowdflows
Commit 9479ce38 authored by Janez K's avatar Janez K
Browse files
Options

naredu windows queue za matjazeve widgete

parent 0d4f1d26
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 12110 additions and 11615 deletions
apache/wsgi_app.py
View file @ 9479ce38
...@@ -17,6 +17,9 @@ import os ...@@ -17,6 +17,9 @@ import os
import sys import sys
import site import site
import djcelery
djcelery.setup_loader()
site.addsitedir('/srv/django-envs/mothra/lib/python2.7/site-packages') site.addsitedir('/srv/django-envs/mothra/lib/python2.7/site-packages')
project_path = '/srv/django-projects/mothra' project_path = '/srv/django-projects/mothra'
......
mothra/settings.py
View file @ 9479ce38
...@@ -171,15 +171,18 @@ INSTALLED_APPS_DEFAULT = ( ...@@ -171,15 +171,18 @@ INSTALLED_APPS_DEFAULT = (
'picklefield', 'picklefield',
'streams', 'streams',
#'rest_framework', #'rest_framework',
#'djcelery', 'djcelery',
#'kombu.transport.django', 'kombu.transport.django',
) )
INSTALLED_APPS_WORKFLOWS_SUB = () INSTALLED_APPS_WORKFLOWS_SUB = ()
CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend' CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend'
CELERY_TASK_SERIALIZER='json' import djcelery
djcelery.setup_loader()
BROKER_URL = 'django://'
try: try:
LOCAL_SETTINGS LOCAL_SETTINGS
......
mothra/wsgi.py
View file @ 9479ce38
...@@ -17,6 +17,9 @@ import os ...@@ -17,6 +17,9 @@ import os
import sys import sys
import site import site
import djcelery
djcelery.setup_loader()
project_path = '/srv/django-projects/mothra' project_path = '/srv/django-projects/mothra'
if project_path not in sys.path: if project_path not in sys.path:
sys.path.append(project_path) sys.path.append(project_path)
......
requirements.txt
View file @ 9479ce38
...@@ -10,5 +10,4 @@ httplib2==0.7.5 ...@@ -10,5 +10,4 @@ httplib2==0.7.5
wsgiref==0.1.2 wsgiref==0.1.2
unidecode==0.04.13 unidecode==0.04.13
django-jsonview==0.3.0 django-jsonview==0.3.0
feedparser==5.1.3 django-celery==3.1.9
pyteaser==1.0
workflows/crossbee/db/package_data.json
View file @ 9479ce38
This diff is collapsed.
workflows/latino/db/package_data.json
View file @ 9479ce38
This diff is collapsed.
workflows/lemmagen/db/package_data.json
View file @ 9479ce38
This diff is collapsed.
workflows/migrations/0039_auto__add_field_abstractwidget_windows_queue.py 0 → 100644
View file @ 9479ce38
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'AbstractWidget.windows_queue'
db.add_column('workflows_abstractwidget', 'windows_queue',
self.gf('django.db.models.fields.BooleanField')(default=False),
keep_default=False)
def backwards(self, orm):
# Deleting field 'AbstractWidget.windows_queue'
db.delete_column('workflows_abstractwidget', 'windows_queue')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'workflows.abstractinput': {
'Meta': {'ordering': "('order',)", 'object_name': 'AbstractInput'},
'default': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'multi': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'parameter': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'parameter_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'short_name': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
'uid': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'variable': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'widget': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'inputs'", 'to': "orm['workflows.AbstractWidget']"})
},
'workflows.abstractoption': {
'Meta': {'ordering': "['name']", 'object_name': 'AbstractOption'},
'abstract_input': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['workflows.AbstractInput']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'uid': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'workflows.abstractoutput': {
'Meta': {'ordering': "('order',)", 'object_name': 'AbstractOutput'},
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'short_name': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
'uid': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'variable': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'widget': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'outputs'", 'to': "orm['workflows.AbstractWidget']"})
},
'workflows.abstractwidget': {
'Meta': {'ordering': "('order', 'name')", 'object_name': 'AbstractWidget'},
'action': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'category': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'widgets'", 'to': "orm['workflows.Category']"}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'has_progress_bar': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image': ('workflows.thumbs.ThumbnailField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'interaction_view': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200', 'blank': 'True'}),
'interactive': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_streaming': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'package': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '150', 'blank': 'True'}),
'post_interact_action': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200', 'blank': 'True'}),
'static_image': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'streaming_visualization_view': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200', 'blank': 'True'}),
'treeview_image': ('workflows.thumbs.ThumbnailField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'uid': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'widgets'", 'null': 'True', 'to': "orm['auth.User']"}),
'visualization_view': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200', 'blank': 'True'}),
'windows_queue': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'wsdl': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'wsdl_method': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '200', 'blank': 'True'})
},
'workflows.category': {
'Meta': {'ordering': "('order', 'name')", 'object_name': 'Category'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'parent': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': "orm['workflows.Category']"}),
'uid': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '250', 'blank': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'categories'", 'null': 'True', 'to': "orm['auth.User']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'categories'", 'null': 'True', 'to': "orm['workflows.Workflow']"})
},
'workflows.connection': {
'Meta': {'object_name': 'Connection'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'input': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'connections'", 'to': "orm['workflows.Input']"}),
'output': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'connections'", 'to': "orm['workflows.Output']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'connections'", 'to': "orm['workflows.Workflow']"})
},
'workflows.input': {
'Meta': {'ordering': "('order',)", 'object_name': 'Input'},
'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'inner_output': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'outer_input_rel'", 'null': 'True', 'to': "orm['workflows.Output']"}),
'multi_id': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'outer_output': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'inner_input_rel'", 'null': 'True', 'to': "orm['workflows.Output']"}),
'parameter': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'parameter_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
'required': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'short_name': ('django.db.models.fields.CharField', [], {'max_length': '3'}),
'value': ('picklefield.fields.PickledObjectField', [], {'null': 'True'}),
'variable': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'widget': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'inputs'", 'to': "orm['workflows.Widget']"})
},
'workflows.option': {
'Meta': {'ordering': "['name']", 'object_name': 'Option'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'input': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['workflows.Input']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
},
'workflows.output': {
'Meta': {'ordering': "('order',)", 'object_name': 'Output'},
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'inner_input': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'outer_output_rel'", 'null': 'True', 'to': "orm['workflows.Input']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'order': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
'outer_input': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'inner_output_rel'", 'null': 'True', 'to': "orm['workflows.Input']"}),
'short_name': ('django.db.models.fields.CharField', [], {'max_length': '5'}),
'value': ('picklefield.fields.PickledObjectField', [], {'null': 'True'}),
'variable': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'widget': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'outputs'", 'to': "orm['workflows.Widget']"})
},
'workflows.userprofile': {
'Meta': {'object_name': 'UserProfile'},
'active_workflow': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'users'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['workflows.Workflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"})
},
'workflows.widget': {
'Meta': {'object_name': 'Widget'},
'abstract_widget': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'instances'", 'null': 'True', 'to': "orm['workflows.AbstractWidget']"}),
'error': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'finished': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'interaction_waiting': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'progress': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'running': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'type': ('django.db.models.fields.CharField', [], {'default': "'regular'", 'max_length': '50'}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'widgets'", 'to': "orm['workflows.Workflow']"}),
'x': ('django.db.models.fields.IntegerField', [], {}),
'y': ('django.db.models.fields.IntegerField', [], {})
},
'workflows.workflow': {
'Meta': {'ordering': "['name']", 'object_name': 'Workflow'},
'description': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'default': "'Untitled workflow'", 'max_length': '200'}),
'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'template_parent': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'to': "orm['workflows.Workflow']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'workflows'", 'to': "orm['auth.User']"}),
'widget': ('django.db.models.fields.related.OneToOneField', [], {'blank': 'True', 'related_name': "'workflow_link'", 'unique': 'True', 'null': 'True', 'to': "orm['workflows.Widget']"})
}
}
complete_apps = ['workflows']
\ No newline at end of file
workflows/models.py
View file @ 9479ce38
...@@ -15,6 +15,8 @@ from mothra.settings import USE_CONCURRENCY ...@@ -15,6 +15,8 @@ from mothra.settings import USE_CONCURRENCY
if USE_CONCURRENCY: if USE_CONCURRENCY:
from workflows.tasks import runWidgetAsync, runForLoopIteration from workflows.tasks import runWidgetAsync, runForLoopIteration
from workflows.tasks import executeWidgetFunction, executeWidgetProgressBar, executeWidgetStreaming, executeWidgetWithRequest
class WidgetException(Exception): class WidgetException(Exception):
pass pass
...@@ -264,6 +266,8 @@ class AbstractWidget(models.Model): ...@@ -264,6 +266,8 @@ class AbstractWidget(models.Model):
package = models.CharField(max_length=150,blank=True,default='',help_text='Package is the package name. You are encouraged to use packages.') package = models.CharField(max_length=150,blank=True,default='',help_text='Package is the package name. You are encouraged to use packages.')
windows_queue = models.BooleanField(default=False,help_text="This is used for Matjaz Jursic's widgets.")
class Meta: class Meta:
ordering = ('order','name',) ordering = ('order','name',)
...@@ -453,11 +457,23 @@ class Widget(models.Model): ...@@ -453,11 +457,23 @@ class Widget(models.Model):
input_dict['wsdl']=self.abstract_widget.wsdl input_dict['wsdl']=self.abstract_widget.wsdl
input_dict['wsdl_method']=self.abstract_widget.wsdl_method input_dict['wsdl_method']=self.abstract_widget.wsdl_method
if self.abstract_widget.has_progress_bar: if self.abstract_widget.has_progress_bar:
outputs = function_to_call(input_dict,self) if self.abstract_widget.windows_queue:
t = executeWidgetProgressBar.apply_async([self,input_dict],queue="windows")
outputs = t.wait()
else:
outputs = executeWidgetProgressBar(self,input_dict)
elif self.abstract_widget.is_streaming: elif self.abstract_widget.is_streaming:
outputs = function_to_call(input_dict,self,None) if self.abstract_widget.windows_queue:
t = executeWidgetStreaming.apply_async([self,input_dict],queue="windows")
outputs = t.wait()
else:
outputs = executeWidgetStreaming(self,input_dict)
else: else:
outputs = function_to_call(input_dict) if self.abstract_widget.windows_queue:
t = executeWidgetFunction.apply_async([self,input_dict],queue="windows")
outputs = t.wait()
else:
outputs = executeWidgetFunction(self,input_dict)
else: else:
if self.workflow_link.is_for_loop(): if self.workflow_link.is_for_loop():
self.workflow_link.run_for_loop() self.workflow_link.run_for_loop()
...@@ -638,7 +654,11 @@ class Widget(models.Model): ...@@ -638,7 +654,11 @@ class Widget(models.Model):
input_dict[i.variable].append(i.value) input_dict[i.variable].append(i.value)
try: try:
if not self.abstract_widget is None: if not self.abstract_widget is None:
outputs = function_to_call(request,input_dict, output_dict) if self.abstract_widget.windows_queue:
t = executeWidgetWithRequest.apply_async([widget,input_dict,output_dict,request],queue="windows")
outputs = t.wait()
else:
outputs = executeWidgetWithRequest(widget,input_dict,output_dict,request)
else: else:
self.workflow_link.run() self.workflow_link.run()
except: except:
......
workflows/tasks.py
View file @ 9479ce38
from celery.task import task from celery.task import task
import workflows.library
@task()
def add(a,b):
import time
time.sleep(10)
raise Exception("test")
return a+b
@task() @task()
def runForLoopIteration(workflow,iteration): def runForLoopIteration(workflow,iteration):
pass pass
@task()
def executeWidgetFunction(widget,input_dict):
function_to_call = getattr(workflows.library,widget.abstract_widget.action)
return function_to_call(input_dict)
@task() @task()
def runWidgetAsync(widget): def executeWidgetProgressBar(widget,input_dict):
widget.run(True) function_to_call = getattr(workflows.library,widget.abstract_widget.action)
return function_to_call(input_dict,widget)
@task() @task()
def add(x, y): def executeWidgetStreaming(widget,input_dict):
return x + y function_to_call = getattr(workflows.library,widget.abstract_widget.action)
\ No newline at end of file return function_to_call(input_dict,widget,None)
@task()
def executeWidgetWithRequest(widget,input_dict,output_dict,request):
function_to_call = getattr(workflows.library,widget.abstract_widget.action)
return function_to_call(request,input_dict,output_dict)
@task()
def runWidgetAsync(widget):
widget.run(True)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment