Commit b244d7ad authored by matjaz's avatar matjaz

Merge remote-tracking branch 'remotes/origin/dev_package_independance' into dev

parents 107f04ed c523cbdb
This diff is collapsed.
import logging
import sys
from settings import *
#------------------------------------------------------------------------------
# prepare environment for loading latino (Python.net interpreter should be used)
# see: http://pythonnet.sourceforge.net/
#------------------------------------------------------------------------------
sys.path.append(package_bin)
try:
from LatinoCloudFlows import *
import System
import Latino
except Exception:
logging.warning("LatinoClowdFlows could not be imported! Either there are no Latino dll available or a "\
"wrong interpreter is used. See 'http://pythonnet.sourceforge.net' for interpreter details. "\
"Other functionality (besides latino) will work as .")
pass
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
from serialization_utils import *
#------------------------------------------------------------------------------
# VISUALISATIONS
#------------------------------------------------------------------------------
def show_adc(input_dict):
if input_dict["adc"] == None:
raise Exception("Input ADC is required for displaying Anotated Document Corpus!")
return {}
def show_clusters(input_dict):
return {}
def show_classifications(input_dict):
return {}
def advanced_object_viewer(input_dict):
return {}
def show_table(input_dict):
return {}
#------------------------------------------------------------------------------
# SUPPLEMENTARY FUNCTIONS
#------------------------------------------------------------------------------
def split_object(input_dict):
output_dict = {}
obj = input_dict['object']
output_dict['object'] = eval("obj"+input_dict['attribute'])
return output_dict
def python_snippet(input_dict):
output_dict = {}
input = input_dict['in']
for (i, val) in enumerate(input):
vars()["in"+str(i+1)] = val
out1 = None
exec(input_dict['pycode'])
output_dict['out'] = out1
return output_dict
def create_range(input_dict):
rng = range(ToInt(input_dict['start']), ToInt(input_dict['stop']), ToInt(input_dict['step']))
return {'range':rng }
def compare_lists(input_dict):
l1 = input_dict['list1']
l2 = input_dict['list2']
l = min(len(l1),len(l2))
cntEq = 0
for i in range(0,l):
if l1[i]==l2[i]:
cntEq += 1
return {
'accuracy':(0.0+cntEq)/l,
'statistics':{
'elements':l,
'equal':cntEq,
'different':l-cntEq,
'accuracy':(0.0+cntEq)/l,
'error':(0.0+l-cntEq)/l,
}
}
from django.core.management.base import BaseCommand, CommandError
from workflows.models import Category, AbstractWidget, AbstractInput, AbstractOutput, AbstractOption
from django.core import serializers
from collections import Counter
from optparse import make_option
from workflows.latino.latino import LatinoCF
from pprint import pprint
import sys
class Command(BaseCommand):
args = 'file_name'
help = 'Export all widgets marked in the c# dll package as widgets to the json file which can be imported to the mothra database'
def handle(self, *args, **options):
if (len(args)<1):
raise CommandError('Argument "file_name" is required!')
try:
f = open(args[0], 'w')
except:
raise CommandError('There was a problem with creating/overwriting given output file')
self.stdout.write('Creating json representations of widgets, inputs, outputs, options and categories ... ')
outstr = LatinoCF.GetJsonDbDefinitions()
self.stdout.write('done.\n')
try:
f.write(outstr)
except:
raise CommandError('There was a problem with writing to the given output file')
self.stdout.write('Json definitions successfully created. Results written to the file.\n')
\ No newline at end of file
from django.core.management.base import BaseCommand, CommandError
from django.core import serializers
from collections import Counter
from optparse import make_option
from workflows.latino.import_dotnet import LatinoCF
from workflows.latino.import_dotnet import ClowdFlowsDefinitions
from pprint import pprint
import sys
from workflows.latino.settings import create_backups
from workflows.latino.settings import package_root
import workflows.management.commands.import_package
import os
import shutil
from datetime import datetime
from workflows.latino import serialization_utils
class Command(BaseCommand):
args = 'file_name'
help = 'Export all widgets marked in the c# dll package as widgets and directly imports them to the mothra database.' \
'As if "latino.generate_package a.json" followed by "workflows.import_package a.json" would be called.' \
'!However!, the export file goes to the standard place (latino/db/package_data.json) and replaces the existing file.' \
'Variable "create_backups" in "latino/setings.py" controls whether backups of package_data.json are kept in "latino/backups".'
option_list = BaseCommand.option_list + (
make_option('-f', '--force',
action="store_true",
dest='force',
default=False,
help='By default, the old "package_data.json" and newly generated definition strings are compared. In case there are no differences'
'nothing happens. This switch forces (regardles of equality) the import of the newly generated data and creation of backups '
'(if latino/setings.py/create_backups == True).'
),
make_option('-r', '--replace',
action="store_true",
dest='replace',
default=False,
help='Completely replace whole widgets with the new one where UIDs match. Default behaviour merges widgets sub-models (AbstractInputs, AbstractOutputs and AbstratcOptions)'
'based on their sub-model\'s own UID. When using the option "--replace" then all widget\'s old submodels are deleted and completely replaced by new sub-models.)'
),
)
def handle(self, *args, **options):
timeStamp = datetime.now().strftime('.backup_%Y%m%d_%H%M%S.json')
backupDir = os.path.join(package_root,"backup")
dataFileName = "package_data.json"
dataFilePath = os.path.join(package_root,"db",dataFileName)
dataFileBackupPath = os.path.join(backupDir,dataFileName)+timeStamp
codeFileName = "library_gen.py"
codeFilePath = os.path.join(package_root, codeFileName)
codeFileBackupPath = os.path.join(backupDir,codeFileName)+timeStamp
self.stdout.write('Creating json and python code representations of widgets, inputs, outputs, options and categories ... ')
newDataCode = serialization_utils.ToPyObj(ClowdFlowsDefinitions.Get())
newDataDef = newDataCode['json']
newCodeDef = newDataCode['library']
self.stdout.write('done.\n')
self.stdout.write('Reading previous json definitions if existing and compare to the new ... ')
oldDataDef = ""
try:
oldDataDef = open(dataFilePath).read()
except:
pass
self.stdout.write('done.\n')
dataReplace = True
if oldDataDef == newDataDef:
self.stdout.write('The newly generated data is the same as the file "latino/db/package_data.json".\n')
if not options['force']:
self.stdout.write('Nothing will be done with the data definition (use "--force" option to force the import).\n')
dataReplace = False
else:
self.stdout.write('The newly generated data is new or different from the file "latino/db/package_data.json".\n')
self.stdout.write('Reading previous code definitions if existing and compare to the new ... ')
oldCodeDef = ""
try:
oldCodeDef = open(codeFilePath).read()
except:
pass
self.stdout.write('done.\n')
codeReplace = True
if oldCodeDef == newCodeDef:
self.stdout.write('The newly generated code is the same as the file "latino/library_gen.py".\n')
if not options['force']:
self.stdout.write('Nothing will be done with the code file (use "--force" option to force the code replace).\n')
dataReplace = False
else:
self.stdout.write('The newly generated definitions are new or different from the file "latino/library_gen.py".\n')
if (dataReplace or codeReplace) and create_backups:
if not os.path.exists(backupDir):
os.makedirs(backupDir)
if dataReplace and create_backups and os.path.exists(dataFilePath):
try:
shutil.move(dataFilePath, dataFileBackupPath)
self.stdout.write('Backup of database successfully created.\n')
except:
raise CommandError('There was a problem with the operation of backing up the data file. '
'Procedure update_package was terminated without any changes to the database.')
if dataReplace:
open(dataFilePath,"w").write(newDataDef)
self.stdout.write('New data file successfully written to its location.\n')
importString = workflows.management.commands.import_package.Command()
importString.import_package_string(self, newDataDef, options['replace'])
self.stdout.write('Widget definitions in the database were updated successfully from the dll library.\n')
if codeReplace and create_backups and os.path.exists(codeFilePath):
try:
shutil.move(codeFilePath, codeFileBackupPath)
self.stdout.write('Backup of code successfully created.\n')
except:
raise CommandError('There was a problem with the operation of backing up the code file. Procedure update_package terminated.')
if codeReplace:
open(codeFilePath,"w").write(newCodeDef)
self.stdout.write('New code file successfully written to its location.\n')
self.stdout.write('Procedure update_package successfully finished.\n')
import os
package_name = 'latino'
package_root = os.path.dirname(__file__)
package_statics = os.path.join(os.path.dirname(__file__), 'static', package_name)
package_bin = os.path.join(package_root, 'bin')
auto_update_db = True
create_backups = True
......@@ -4,32 +4,27 @@ from django.http import Http404, HttpResponse
# modeli
from workflows.models import *
# auth fore
from django.contrib.auth.decorators import login_required
# latino
import latino
import logging
@login_required
def get_adc_index(request, widget_id, narrow_doc = 'n', document_id_from=0, document_id_to=-1):
logging.info('__get_adc_index__')
w = get_object_or_404(Widget, pk=widget_id)
if w.workflow.user == request.user:
firstInput = w.inputs.all()[0]
adc = firstInput.value
return latino.makeAdcIndexPage(adc, document_id_from, document_id_to, narrow_doc=='n')
data = adc.netObj.MakeIndexPage(document_id_from, document_id_to, 100, narrow_doc)
return HttpResponse(data, mimetype='text/html')
else:
return HttpResponse(status=400)
@login_required
def get_adc_page(request, widget_id, document_id, narrow_doc = 'n'):
logging.info('__get_adc_page__')
w = get_object_or_404(Widget, pk=widget_id)
if w.workflow.user == request.user:
firstInput = w.inputs.all()[0]
adc = firstInput.value
return latino.makeAdcDocPage(adc, document_id, narrow_doc)
data = adc.netObj.MakeDocumentPage(document_id, narrow_doc)
return HttpResponse(data, mimetype='text/html')
else:
return HttpResponse(status=400)
\ No newline at end of file
return HttpResponse(status=400)
from django.shortcuts import render
#latino
import os
import latino
import logging
from serialization_utils import *
import django.template.loader
def show_adc(request,input_dict,output_dict,widget):
logging.info('__show_adc__VIEW__')
return latino.adcView(request,input_dict,output_dict,widget)
view = django.shortcuts.render(request, 'visualizations/adc.html', {'widget': widget,
'input_dict': input_dict,
'output_dict': output_dict})
return view
def show_clusters(request,input_dict,output_dict,widget):
return {}
def show_classifications(request,input_dict,output_dict,widget):
return {}
def advanced_object_viewer(request,input_dict,output_dict,widget):
logging.info('__advanced_object_viewer__VIEW__')
import objectPprint as opp
obj = input_dict['object']
maxStringLen = latino.ToInt(input_dict['maxStringLen'])
obj = input_dict['obj']
maxStringLen = ToInt(input_dict['maxStringLen'])
objstr = ""
if input_dict.has_key('attribute') and input_dict['attribute']!="":
try:
......@@ -29,5 +33,6 @@ def advanced_object_viewer(request,input_dict,output_dict,widget):
return render(request, 'visualizations/advanced_object_viewer.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
def show_table(request,input_dict,output_dict,widget):
logging.info('__show_table__VIEW__')
return latino.ShowTable(request,input_dict,output_dict,widget)
\ No newline at end of file
tbl= input_dict['tbl']
output_dict = {'attrs':list(tbl.items()[0][0]), 'metas':[], 'data':tbl.values()[0], 'title':'Vocabulary Table'}
return render(request, 'visualizations/table_viewer.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
\ No newline at end of file
......@@ -10,6 +10,8 @@ from workflows import packageLibImporter
def setattr_local(name, value, package):
setattr(sys.modules[__name__], name, value)
packageLibImporter.importAllPackagesLib("library",setattr_local)
from latino.library import *
def test_interaction(input_dict):
return input_dict
......
from unicodedata import category
from django.core.management.base import BaseCommand, CommandError
from workflows.models import Category, AbstractWidget, AbstractInput, AbstractOutput, AbstractOption
from django.core import serializers
......@@ -29,7 +30,7 @@ class Command(BaseCommand):
except:
raise CommandError('There was a problem with creating/overwriting given output file')
result = self.export_package_string(self.stdout.write, args[1:], options['newuid'], options['all'])
result = self.export_package_string(self.stdout.write, args[1:], options['newuid'], options['all'], int(options['verbosity']))
try:
f.write(result.encode('utf-8'))
......@@ -38,54 +39,101 @@ class Command(BaseCommand):
self.stdout.write('Export procedure successfully finished. Results written to the file.\n')
def export_package_string(self, write, packages, newuid, all):
result = []
def print_stataistics(self, objs, verbosity, write):
if verbosity > 0:
write('Selection contains:\n')
write(' % 4i AbstractWidget(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractWidget)]))
write(' % 4i AbstractInput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractInput)]))
write(' % 4i AbstractOutput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOutput)]))
write(' % 4i AbstractOption(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOption)]))
write(' % 4i Category(s)\n' % len([obj for obj in objs if isinstance(obj, Category)]))
if (verbosity == 1):
write('Exported categories:\n')
if (verbosity == 2):
write('Exported categories and widgets:\n')
if (verbosity == 2):
write('Exported categories, widgets, inputs, outputs and options:\n')
indent = 0
indentAddon = 0
for obj in objs:
s = ''
if isinstance(obj, Category):
indent = str(obj).count('::')
s = '% 3i. Category ===== %s =====' % (obj.order, obj)
if isinstance(obj, AbstractWidget):
s = ' % 3i. AbstractWidget: %s [%s]' % (obj.order, obj.name, obj.action)
if isinstance(obj, AbstractInput):
s = ' % 3i. AbstractInput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOutput):
s = ' % 3i. AbstractOutput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOption):
s = ' AbstractOption: %s | %s' % (obj.name, obj.value)
if isinstance(obj, Category) or (isinstance(obj, AbstractWidget) and verbosity > 1) or verbosity > 2:
write(' ' * indent + s + '\n')
self.stdout.write(
'Tip: use higher "verbosity" option numbers to se more detailed output of what is being exported.\n')
def export_package_string(self, write, packages, newuid, all, verbosity):
assert isinstance(packages, tuple)
assert isinstance(newuid, bool)
assert isinstance(all, bool)
objs = []
if all:
packages = ['all']
for package in packages:
if all:
wids = AbstractWidget.objects.all()
else:
wids = AbstractWidget.objects.filter(package=package)
inps = AbstractInput.objects.filter(widget__in=[x.id for x in wids])
outs = AbstractOutput.objects.filter(widget__in=[x.id for x in wids])
opts = AbstractOption.objects.filter(abstract_input__in=[x.id for x in inps])
cats = Category.objects.filter(id__in=[x.category.id for x in wids])
#retrieve all parents
catNum = len(cats)
while True:
cats = cats | Category.objects.filter(id__in=[x.parent.id for x in cats if x.parent != None])
if catNum == len(cats):
break
else:
catNum = len(cats)
objs.extend(cats)
objs.extend(wids)
objs.extend(outs)
objs.extend(inps)
objs.extend(opts)
if len(wids) > 0:
write('Package "%s" contains:\n' % package)
write(' % 4i AbstractWidget(s)\n' % len(wids))
write(' % 4i AbstractInput(s)\n' % len(inps))
write(' % 4i AbstractOutput(s)\n' % len(outs))
write(' % 4i AbstractOption(s)\n' % len(opts))
write(' % 4i Category(s)\n' % len(cats))
else:
write('Package "%s" was not found!\n' % package)
for topCat in Category.objects.filter(parent = None):
objs.extend(self.get_package_objs_in_category(topCat, packages, all))
if len(objs) == 0:
write('Selected package(s) were not found!\n')
return "[\\n]"
#be careful uid is only changed on these instances and is not written to the database
if newuid:
for a in objs:
a.uid = str(uuid.uuid4())
result = ""
if len(objs) > 0:
result = serializers.serialize("json", objs, indent=2, ensure_ascii=False)
self.print_stataistics(objs, verbosity, write)
result = serializers.serialize("json", objs, indent=2, ensure_ascii=False)
return result
def get_package_objs_in_category(self, cat, packages, all):
assert isinstance(cat, Category)
assert isinstance(packages, tuple)
assert isinstance(all, bool)
objs = []
objs.extend(self.get_package_wids_in_category(cat, packages, all))
for catChild in cat.children.all():
objs.extend(self.get_package_objs_in_category(catChild, packages, all))
if len(objs)>0:
objs.insert(0,cat)
return objs
def get_package_wids_in_category(self, cat, packages, all):
assert isinstance(cat, Category)
assert isinstance(packages, tuple)
assert isinstance(all, bool)
objs = []
if all:
wids = cat.widgets.all()
else:
wids = cat.widgets.filter(package__in=packages)
for wid in wids:
objs.append(wid)
for inp in wid.inputs.all():
objs.append(inp)
objs.extend(inp.options.all())
objs.extend(wid.outputs.all())
return result
\ No newline at end of file
return objs
......@@ -94,9 +94,9 @@ class Command(BaseCommand):
#prepare statistics
statDict = dict([('old:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]])
for objFile in objsFile:
for modelType in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]:
for operation in ['mod','add','del']:
statDict[operation+':'+str(type(objFile.object))]=0
statDict[operation+':'+str(modelType)]=0
#sort so that AbstractWidgets come in front (are processsed first in the following block)
#objsFileNew = []
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment