Commit 8f470499 authored by romanorac's avatar romanorac

bug fixes

parents f65999d7 c358ad19
from unicodedata import category
from django.core.management.base import BaseCommand, CommandError
from workflows.models import Category, AbstractWidget, AbstractInput, AbstractOutput, AbstractOption
from django.core import serializers
from optparse import make_option
import uuid
import os
import sys
from django.conf import settings
import json
def add_category(category,categories):
categories.add(category.pk)
if category.parent:
add_category(category.parent,categories)
def ensure_dir(directory):
if not os.path.exists(directory):
os.makedirs(directory)
def choice(choices,question="Your choice: "):
choice = None
while 1:
if not choice:
input_msg = ""
for i in range(0,len(choices)):
input_msg += "["+str(i)+"] "+str(choices[i])+"\n"
choice_number = raw_input(input_msg + question)
try:
choice = choices[int(choice_number)]
return choice
except:
sys.stderr.write("Error: Wrong choice.\n")
def serialize_widget(aw):
data = json.loads(serializers.serialize("json",[aw,]))[0]
if data.has_key('pk'):
data.pop('pk')
if data['fields'].has_key('user'):
data['fields'].pop('user')
if not data['fields']['category'] is None:
data['fields']['category'] = aw.category.uid
input_data = json.loads(serializers.serialize("json",aw.inputs.all()))
for i in input_data:
if i.has_key('pk'):
i.pop('pk')
i['fields']['widget']=aw.uid
output_data = json.loads(serializers.serialize("json",aw.outputs.all()))
for i in output_data:
if i.has_key('pk'):
i.pop('pk')
i['fields']['widget']=aw.uid
options_data = json.loads(serializers.serialize("json",AbstractOption.objects.filter(abstract_input__widget=aw)))
for o in options_data:
if o.has_key('pk'):
o.pop('pk')
o['fields']['abstract_input']=AbstractInput.objects.get(id=o['fields']['abstract_input']).uid
return [data,]+input_data+output_data+options_data
def serialize_category(c):
data = json.loads(serializers.serialize("json",[c,]))[0]
if data.has_key('pk'):
data.pop('pk')
if not data['fields']['parent'] is None:
c2 = Category.objects.get(id=data['fields']['parent'])
data['fields']['parent'] = c2.uid
if data['fields'].has_key('workflow'):
data['fields'].pop('workflow')
if data['fields'].has_key('user'):
data['fields'].pop('user')
return data
class Command(BaseCommand):
args = 'package_name'
help = 'Exports the package "package_name".'
def handle(self, *args, **options):
if (len(args) < 1):
raise CommandError('Argument "package_name" is required.')
package_name = args[0]
if 'workflows.'+package_name not in settings.INSTALLED_APPS:
raise CommandError("Package not found in INSTALLED_APPS.")
#here we check the integrity of the package
aws = AbstractWidget.objects.filter(package=package_name)
for aw in aws:
if aw.uid:
for bw in aws:
if bw.uid == aw.uid and bw.id != aw.id:
self.stdout.write("Found two widgets with the same UID. Please select a widget to assign new UID to.\n")
selected_widget = choice([aw,bw],"Select a widget: ")
selected_widget.set_uid(commit=True)
#first we check if package_data directory exists and make it if it doesn't
package_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../../'+package_name+"/package_data/")
ensure_dir(package_directory)
widgets_directory = os.path.join(package_directory,"widgets")
ensure_dir(widgets_directory)
categories_directory = os.path.join(package_directory,"categories")
ensure_dir(categories_directory)
self.stdout.write(" > Ensuring package directory for "+package_name+".\n")
categories = set()
self.stdout.write(" > Exporting widgets\n")
global_change = False
for aw in aws:
aw.update_uid()
add_category(aw.category,categories)
serialized_widget = serialize_widget(aw)
created = True
change = True
try:
widget_file = open(os.path.join(widgets_directory,aw.uid+'.json'),'r')
created = False
w_data = json.loads(widget_file.read())
widget_file.close()
if w_data == serialized_widget:
change = False
except:
created = True
change = True
if change:
global_change = True
if created:
self.stdout.write(" + Exporting widget "+str(aw)+"\n")
else:
self.stdout.write(" + Updating widget "+str(aw)+"\n")
widget_data = json.dumps(serialized_widget,indent=2)
widget_file = open(os.path.join(widgets_directory,aw.uid+'.json'),'w')
widget_file.write(widget_data)
widget_file.close()
if not global_change:
self.stdout.write(" No changes in the widgets detected!\n")
self.stdout.write(" > Exporting categories\n")
global_change = False
for category in categories:
c = Category.objects.get(id=category)
c.update_uid()
data = serialize_category(c)
created = True
change = True
try:
category_file = open(os.path.join(categories_directory,c.uid+'.json'),'r')
created = False
c_data = json.loads(category_file.read())
category_file.close()
if c_data == data:
change = False
except:
created = True
change = True
if change:
global_change = True
if created:
self.stdout.write(" + Exporting category "+str(c)+"\n")
else:
self.stdout.write(" + Updating category "+str(c)+"\n")
category_data = json.dumps(data,indent=2)
category_file = open(os.path.join(categories_directory,c.uid+'.json'),'w')
category_file.write(category_data)
category_file.close()
if not global_change:
self.stdout.write(" No changes in the categories detected!\n")
self.stdout.write('Thanks for using the new export command. You rock.\n')
from unicodedata import category
from django.core.management.base import BaseCommand, CommandError
from workflows.models import Category, AbstractWidget, AbstractInput, AbstractOutput, AbstractOption
from django.core import serializers
from optparse import make_option
import uuid
import os
import sys
from django.conf import settings
import json
from .export import serialize_category, serialize_widget
def parsewidgetdata(widget_data):
widget = None
inputs = []
outputs = []
options = []
for i in widget_data:
if i['model']=='workflows.abstractwidget':
widget = i
elif i['model']=='workflows.abstractinput':
inputs.append(i)
elif i['model']=='workflows.abstractoutput':
outputs.append(i)
elif i['model']=='workflows.abstractoption':
options.append(i)
else:
raise CommandError("Wrong data in widget files!")
return widget, inputs, outputs, options
class Command(BaseCommand):
args = 'package_name'
help = 'Imports the package "package_name".'
def handle(self, *args, **options):
if (len(args) < 1):
raise CommandError('Argument "package_name" is required.')
package_name = args[0]
if 'workflows.'+package_name not in settings.INSTALLED_APPS:
raise CommandError("Package not found in INSTALLED_APPS.")
package_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../../'+package_name+"/package_data/")
widgets_directory = os.path.join(package_directory,"widgets")
categories_directory = os.path.join(package_directory,"categories")
if not os.path.exists(package_directory) or not os.path.exists(widgets_directory) or not os.path.exists(categories_directory):
raise CommandError("Cannot find package data. Are you sure this package has been exported already?")
widget_files = os.listdir(widgets_directory)
category_files = os.listdir(categories_directory)
self.stdout.write(' > Importing categories\n')
global_change = False
for category_file in category_files:
cfilepath = os.path.join(categories_directory,category_file)
c_file = open(cfilepath,'r')
c_data = json.loads(c_file.read())
c_file.close()
uid = c_data['fields']['uid']
created = False
try:
c = Category.objects.get(uid=uid)
except Category.DoesNotExist:
created = True
c = Category(uid=uid)
old_c_data = serialize_category(c)
if old_c_data != c_data:
global_change = True
if created:
self.stdout.write(' + Creating category '+str(c_data['fields']['name'])+'\n')
else:
self.stdout.write(' + Updating category '+str(c_data['fields']['name'])+'\n')
for field in c_data['fields'].keys():
if field != 'parent':
setattr(c,field,c_data['fields'][field])
else:
parent = None
if c_data['fields']['parent'] != None:
try:
parent = Category.objects.get(uid=c_data['fields']['parent'])
except Category.DoesNotExist:
parent = Category(uid=c_data['fields']['parent'],name="Temporary category name")
parent.save()
c.parent = parent
c.save()
if not global_change:
self.stdout.write(" No changes detected in the categories.\n")
global_change = False
self.stdout.write(' > Importing widgets\n')
for widget_file in widget_files:
wfilepath = os.path.join(widgets_directory,widget_file)
w_file = open(wfilepath,'r')
w_data = json.loads(w_file.read())
w_file.close()
widget, inputs, outputs, options = parsewidgetdata(w_data)
created = False
try:
aw = AbstractWidget.objects.get(uid=widget['fields']['uid'],package=package_name)
except AbstractWidget.DoesNotExist:
aw = AbstractWidget(uid=widget['fields']['uid'],package=package_name)
created = True
if w_data != serialize_widget(aw):
global_change = True
if created:
self.stdout.write(' + Creating widget '+str(widget['fields']['name'])+'\n')
else:
self.stdout.write(' + Updating widget '+str(widget['fields']['name'])+'\n')
for field in widget['fields'].keys():
if field != 'category':
setattr(aw,field,widget['fields'][field])
else:
aw.category = Category.objects.get(uid=widget['fields']['category'])
aw.save()
for inp in inputs:
try:
i = AbstractInput.objects.get(uid=inp['fields']['uid'])
except AbstractInput.DoesNotExist:
i = AbstractInput(uid=inp['fields']['uid'])
for field in inp['fields'].keys():
if field != 'widget':
setattr(i,field,inp['fields'][field])
i.widget = aw
i.save()
for out in outputs:
try:
o = AbstractOutput.objects.get(uid=out['fields']['uid'])
except AbstractOutput.DoesNotExist:
o = AbstractOutput(uid=out['fields']['uid'])
for field in out['fields'].keys():
if field != 'widget':
setattr(o,field,out['fields'][field])
o.widget = aw
o.save()
for option in options:
try:
o = AbstractOption.objects.get(uid=option['fields']['uid'])
except AbstractOption.DoesNotExist:
o = AbstractOption(uid=option['fields']['uid'])
for field in option['fields'].keys():
if field != 'abstract_input':
setattr(o,field,option['fields'][field])
else:
o.abstract_input = AbstractInput.objects.get(uid=option['fields']['abstract_input'])
o.save()
if not global_change:
self.stdout.write(" No changes detected in the widgets.\n")
self.stdout.write('Thanks for using the new import command. You rock.\n')
......@@ -38,6 +38,14 @@ class Category(models.Model):
uid = models.CharField(max_length=250,blank=True,default='')
def update_uid(self):
import uuid
if self.uid == '' or self.uid is None:
self.uid = uuid.uuid4()
self.save()
if self.parent:
self.parent.update_uid()
class Meta:
verbose_name_plural = "categories"
ordering = ('order','name',)
......@@ -429,6 +437,22 @@ class AbstractWidget(models.Model):
if commit:
o.save()
def update_uid(self):
import uuid
if self.uid == '' or self.uid is None:
self.uid = uuid.uuid4()
self.save()
for i in self.inputs.filter(uid=''):
i.uid = uuid.uuid4()
i.save()
for option in i.options.filter(uid=''):
option.uid = uuid.uuid4()
option.save()
for o in self.outputs.filter(uid=''):
o.uid = uuid.uuid4()
o.save()
self.category.update_uid()
def __unicode__(self):
return unicode(self.name)
......
[
{
"pk": 5,
"model": "workflows.category",
"fields": {
"uid": "15acb469-c510-44f0-8330-60bfe11a463c",
"parent": null,
"workflow": null,
"user": null,
"order": 1,
"pk": 1,
"model": "workflows.category",
"fields": {
"uid": "15acb469-c510-44f0-8330-60bfe11a463c",
"parent": null,
"workflow": null,
"user": null,
"order": 1,
"name": "NLP"
}
},
{
"pk": 33,
"model": "workflows.abstractwidget",
"fields": {
"category": 5,
"treeview_image": "treeview/nlp_3.png",
"windows_queue": false,
"name": "Load corpus",
"is_streaming": false,
"uid": "ce3c4bc7-689c-4eff-8c5c-2aa1aaf82971",
"interaction_view": "",
"image": "images/nlp_9.png",
"package": "nlp",
"static_image": "nlp.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",
"streaming_visualization_view": "",
"action": "load_corpus",
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"has_progress_bar": false,
"order": 1,
},
{
"pk": 1,
"model": "workflows.abstractwidget",
"fields": {
"category": 1,
"treeview_image": "treeview/nlp_3.png",
"name": "Load corpus",
"is_streaming": false,
"uid": "ce3c4bc7-689c-4eff-8c5c-2aa1aaf82971",
"interaction_view": "",
"image": "images/nlp_9.png",
"package": "nlp",
"static_image": "nlp.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",
"streaming_visualization_view": "",
"action": "load_corpus",
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"has_progress_bar": false,
"order": 1,
"description": ""
}
},
{
"pk": 49,
"model": "workflows.abstractinput",
"fields": {
"widget": 33,
"name": "Web service address",
"short_name": "ws",
"uid": "907565a7-4336-48c3-8989-9024aae0d7b1",
"default": "http://vihar.ijs.si:8095/totale?wsdl",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "wsdl",
"parameter": true,
"order": 1,
},
{
"pk": 413,
"model": "workflows.abstractinput",
"fields": {
"widget": 1,
"name": "Web service address",
"short_name": "ws",
"uid": "907565a7-4336-48c3-8989-9024aae0d7b1",
"default": "http://vihar.ijs.si:8095/totale?wsdl",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "wsdl",
"parameter": true,
"order": 1,
"description": "Web service address"
}
},
{
"pk": 50,
"model": "workflows.abstractinput",
"fields": {
"widget": 33,
"name": "File",
"short_name": "fil",
"uid": "096159a3-ddc8-4a57-b769-dddfb2590065",
"default": "",
"required": true,
"multi": false,
"parameter_type": "file",
"variable": "file",
"parameter": true,
"order": 2,
},
{
"pk": 1,
"model": "workflows.abstractinput",
"fields": {
"widget": 1,
"name": "File",
"short_name": "fil",
"uid": "096159a3-ddc8-4a57-b769-dddfb2590065",
"default": "",
"required": true,
"multi": false,
"parameter_type": "file",
"variable": "file",
"parameter": true,
"order": 2,
"description": "File"
}
},
},
{
"pk": 30,
"model": "workflows.abstractoutput",
"pk": 1,
"model": "workflows.abstractoutput",
"fields": {
"widget": 33,
"name": "Corpus",
"short_name": "cor",
"variable": "corpus",
"uid": "edcc6852-ba3b-4fd7-a46a-6988a7ff61ad",
"order": 1,
"widget": 1,
"name": "Corpus",
"short_name": "cor",
"variable": "corpus",
"uid": "edcc6852-ba3b-4fd7-a46a-6988a7ff61ad",
"order": 1,
"description": "corpus"
}
<<<<<<< HEAD
},
{
"pk": 42,
......@@ -111,171 +111,362 @@
"interactive": false,
"has_progress_bar": false,
"order": 1,
=======
},
{
"pk": 4,
"model": "workflows.abstractwidget",
"fields": {
"category": 1,
"treeview_image": "treeview/nlp_2.png",
"name": "Term candidates viewer",
"is_streaming": false,
"uid": "af5ee96e-7307-46f9-8d8c-55cb7c36d111",
"interaction_view": "",
"image": "images/nlp_8.png",
"package": "nlp",
"static_image": "nlp.png",
"post_interact_action": "",
"user": null,
"visualization_view": "term_candidate_viewer",
"streaming_visualization_view": "",
"action": "term_candidates",
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"has_progress_bar": false,
"order": 1,
>>>>>>> c358ad193041132145a6e3752dff9b8bbf328620
"description": ""
}
},
},
{
"pk": 5,
"model": "workflows.abstractinput",
"fields": {
"widget": 4,
"name": "candidates",
"short_name": "can",
"uid": "32536744-964e-4206-b582-be92ab355e8e",
"default": "",
"required": true,
"multi": false,
"parameter_type": null,
"variable": "candidates",
"parameter": false,
"order": 1,
"description": ""
}
},
{
"pk": 176,
"model": "workflows.abstractwidget",
"fields": {
"category": 1,
"treeview_image": "",
"name": "Term extraction",
"is_streaming": false,
"uid": "8a8f2aee-dc2e-4e1e-85d5-c977dc5c2867",
"interaction_view": "",
"image": "",
"package": "nlp",
"static_image": "nlp.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",