Commit 9902de94 authored by matjaz's avatar matjaz

Improved import/export functionality (packageLibImporter.py -> module_importer.py)

parent 72fbc6dc
......@@ -3,10 +3,10 @@ from django.shortcuts import render
from decision_support.interaction import *
from subgroup_discovery.interaction import *
from workflows import packageLibImporter
from workflows import module_importer
def setattr_local(name, value, package):
setattr(sys.modules[__name__], name, value)
packageLibImporter.importAllPackagesLib("interaction_views",setattr_local)
module_importer.import_all_packages_libs("interaction_views",setattr_local)
def test_interaction(request,input_dict,output_dict,widget):
return render(request, 'interactions/test_interaction.html',{'widget':widget})
......
import os
# === STANDARD PACKAGE SETTINGS ===
package_root = os.path.dirname(__file__)
PACKAGE_ROOT = os.path.dirname(__file__)
# === AUTO IMPORT OPTIONS ===
#If auto_import_package_data is true then package_data.json file is automatically imported when ClowdFlows project is newly deployed or refreshed from git
auto_import_package_data = True
#If auto_import_package_data is true then given data file is automatically imported when ClowdFlows project is newly deployed or refreshed from git
AUTO_IMPORT_DB = False
#For auto_import_package_data_replace_option description see the 'replace' option in workflows/import_package command
auto_import_package_data_replace_option = True
AUTO_IMPORT_DB_REPLACE_OPTION = True
#If file(s) other than ./db/package_data.json should be imported, auto_import_package_data_files should be corrected
auto_import_package_data_files = [os.path.join(package_root,'db/package_data.json')]
AUTO_IMPORT_DB_FILES = [os.path.join(PACKAGE_ROOT,'db/package_data.json')]
......@@ -6,10 +6,10 @@ import sys
from decision_support.library import *
from subgroup_discovery.library import *
from workflows import packageLibImporter
from workflows import module_importer
def setattr_local(name, value, package):
setattr(sys.modules[__name__], name, value)
packageLibImporter.importAllPackagesLib("library",setattr_local)
module_importer.import_all_packages_libs("library",setattr_local)
def test_interaction(input_dict):
return input_dict
......
......@@ -30,63 +30,29 @@ class Command(BaseCommand):
except:
raise CommandError('There was a problem with creating/overwriting given output file')
result = self.export_package_string(self.stdout.write, args[1:], options['newuid'], options['all'], int(options['verbosity']))
result = export_package_string(self.stdout.write, args[1:], options['newuid'], options['all'], int(options['verbosity']))
try:
f.write(result.encode('utf-8'))
except:
raise CommandError('There was a problem with writing to the given output file')
verbosity = int(options['verbosity'])
if verbosity>0 and verbosity<3:
self.stdout.write('Tip: use higher "verbosity" option numbers to se more detailed output of what is being exported.\n')
self.stdout.write('Export procedure successfully finished. Results written to the file.\n')
def print_stataistics(self, objs, verbosity, write):
if verbosity > 0:
write('Selection contains:\n')
write(' % 4i AbstractWidget(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractWidget)]))
write(' % 4i AbstractInput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractInput)]))
write(' % 4i AbstractOutput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOutput)]))
write(' % 4i AbstractOption(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOption)]))
write(' % 4i Category(s)\n' % len([obj for obj in objs if isinstance(obj, Category)]))
if (verbosity == 1):
write('Exported categories:\n')
if (verbosity == 2):
write('Exported categories and widgets:\n')
if (verbosity == 2):
write('Exported categories, widgets, inputs, outputs and options:\n')
indent = 0
indentAddon = 0
for obj in objs:
s = ''
if isinstance(obj, Category):
indent = str(obj).count('::')
s = '% 3i. Category ===== %s =====' % (obj.order, obj)
if isinstance(obj, AbstractWidget):
s = ' % 3i. AbstractWidget: %s [%s]' % (obj.order, obj.name, obj.action)
if isinstance(obj, AbstractInput):
s = ' % 3i. AbstractInput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOutput):
s = ' % 3i. AbstractOutput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOption):
s = ' AbstractOption: %s | %s' % (obj.name, obj.value)
if isinstance(obj, Category) or (isinstance(obj, AbstractWidget) and verbosity > 1) or verbosity > 2:
write(' ' * indent + s + '\n')
self.stdout.write(
'Tip: use higher "verbosity" option numbers to se more detailed output of what is being exported.\n')
def export_package_string(self, write, packages, newuid, all, verbosity):
def export_package_string(writeFunc, packages, newuid, all, verbosity):
assert isinstance(packages, tuple)
assert isinstance(newuid, bool)
assert isinstance(all, bool)
objs = []
for topCat in Category.objects.filter(parent = None):
objs.extend(self.get_package_objs_in_category(topCat, packages, all))
objs.extend(get_package_objs_in_category(topCat, packages, all))
if len(objs) == 0:
write('Selected package(s) were not found!\n')
writeFunc('Selected package(s) were not found!\n')
return "[\\n]"
#be careful uid is only changed on these instances and is not written to the database
......@@ -94,30 +60,30 @@ class Command(BaseCommand):
for a in objs:
a.uid = str(uuid.uuid4())
self.print_stataistics(objs, verbosity, write)
print_stataistics(objs, verbosity, writeFunc)
result = serializers.serialize("json", objs, indent=2, ensure_ascii=False)
return result
def get_package_objs_in_category(self, cat, packages, all):
def get_package_objs_in_category(cat, packages, all):
assert isinstance(cat, Category)
assert isinstance(packages, tuple)
assert isinstance(all, bool)
objs = []
objs.extend(self.get_package_wids_in_category(cat, packages, all))
objs.extend(get_package_wids_in_category(cat, packages, all))
for catChild in cat.children.all():
objs.extend(self.get_package_objs_in_category(catChild, packages, all))
objs.extend(get_package_objs_in_category(catChild, packages, all))
if len(objs)>0:
objs.insert(0,cat)
return objs
def get_package_wids_in_category(self, cat, packages, all):
def get_package_wids_in_category(cat, packages, all):
assert isinstance(cat, Category)
assert isinstance(packages, tuple)
assert isinstance(all, bool)
......@@ -137,3 +103,38 @@ class Command(BaseCommand):
objs.extend(wid.outputs.all())
return objs
def print_stataistics(objs, verbosity, writeFunc):
if verbosity > 0:
writeFunc('Selection contains:\n')
writeFunc(' % 4i AbstractWidget(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractWidget)]))
writeFunc(' % 4i AbstractInput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractInput)]))
writeFunc(' % 4i AbstractOutput(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOutput)]))
writeFunc(' % 4i AbstractOption(s)\n' % len([obj for obj in objs if isinstance(obj, AbstractOption)]))
writeFunc(' % 4i Category(s)\n' % len([obj for obj in objs if isinstance(obj, Category)]))
if (verbosity == 1):
writeFunc('Exported categories:\n')
if (verbosity == 2):
writeFunc('Exported categories and widgets:\n')
if (verbosity == 3):
writeFunc('Exported categories, widgets, inputs, outputs and options:\n')
indent = 0
for obj in objs:
s = ''
if isinstance(obj, Category):
indent = str(obj).count('::')
s = '% 3i. Category ===== %s =====' % (obj.order, obj)
if isinstance(obj, AbstractWidget):
s = ' % 3i. AbstractWidget: %s [%s]' % (obj.order, obj.name, obj.action)
if isinstance(obj, AbstractInput):
s = ' % 3i. AbstractInput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOutput):
s = ' % 3i. AbstractOutput: (%s) %s' % (obj.order, obj.short_name, obj.name)
if isinstance(obj, AbstractOption):
s = ' AbstractOption: %s | %s' % (obj.name, obj.value)
if isinstance(obj, Category) or (isinstance(obj, AbstractWidget) and verbosity > 1) or verbosity > 2:
writeFunc(' ' * indent + s + '\n')
......@@ -34,16 +34,16 @@ class Command(BaseCommand):
except:
raise CommandError('There was a problem with opening given input file')
self.import_package_string(self, string, options['replace'])
import_package_string(self.stdout.write, string, options['replace'])
self.stdout.write('Import procedure successfully finished.\n')
def import_package_string(self, outWriter, string, replace):
def import_package_string(writeFunc, string, replace):
#get all objects from file and eliminate empty UID and check for UID duplicates
objsFile = list(serializers.deserialize("json", string))
objsFileNoUid = [x for x in objsFile if len(x.object.uid) == 0]
objsFile = [x for x in objsFile if len(x.object.uid) != 0]
if len(objsFileNoUid)>0:
outWriter.stdout.write('File contains %i model(s) without UID field set. Those will not be imported! If you wish to'
writeFunc('File contains %i model(s) without UID field set. Those will not be imported! If you wish to'
' assign them random UIDs then use the "-n" option when exporting models with the "export_package"'
' command. Afterwards, you will be able to import them.\n' % len(objsFileNoUid))
if len(Counter([x.object.uid for x in objsFile])) != len(objsFile):
......@@ -59,12 +59,12 @@ class Command(BaseCommand):
cats = [x for x in objsFile if isinstance(x.object, Category)]
#ouput statistics about file
outWriter.stdout.write('Import contains:\n')
outWriter.stdout.write(' % 4i AbstractWidget(s)\n' % len(wids))
outWriter.stdout.write(' % 4i AbstractInput(s)\n' % len(inps))
outWriter.stdout.write(' % 4i AbstractOutput(s)\n' % len(outs))
outWriter.stdout.write(' % 4i AbstractOption(s)\n' % len(opts))
outWriter.stdout.write(' % 4i Category(s)\n' % len(cats))
writeFunc('Import contains:\n')
writeFunc(' % 4i AbstractWidget(s)\n' % len(wids))
writeFunc(' % 4i AbstractInput(s)\n' % len(inps))
writeFunc(' % 4i AbstractOutput(s)\n' % len(outs))
writeFunc(' % 4i AbstractOption(s)\n' % len(opts))
writeFunc(' % 4i Category(s)\n' % len(cats))
#get all objects from database
objsDb = []
......@@ -95,9 +95,9 @@ class Command(BaseCommand):
'by type:\n - from file: %s\n - from database: %s\n Please resolve manually!'% (objFileTypeId, objDbTypeId))
#ouput statistics about database
outWriter.stdout.write('Current database contains %i models,\n' % len(objsDb))
outWriter.stdout.write(' of which %i models have UID set,\n' % len(objsdbDict))
outWriter.stdout.write(' of which %i models match with the imported models and will be updated.\n' % len(idMappingDict))
writeFunc('Current database contains %i models,\n' % len(objsDb))
writeFunc(' of which %i models have UID set,\n' % len(objsdbDict))
writeFunc(' of which %i models match with the imported models and will be updated.\n' % len(idMappingDict))
#prepare statistics
statDict = dict([('old:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]])
......@@ -112,7 +112,7 @@ class Command(BaseCommand):
#objsFile = objsFileNew
#save models to the database - update the ids for the matching models and remove the ids (to get a new one) for the non matching models
outWriter.stdout.write('Merging file and database models ...')
writeFunc('Merging file and database models ...')
importedUids = dict()
for objFile in objsFile:
objFileTypeId = str(type(objFile.object))+':'+str(objFile.object.id)
......@@ -139,10 +139,10 @@ class Command(BaseCommand):
objFile.save()
idMappingDict[objFileTypeId] = objFile.object.id
importedUids[objFile.object.uid]=True
outWriter.stdout.write(' done.\n')
writeFunc(' done.\n')
#correct also the foreign keys
outWriter.stdout.write('Updating model\'s foreign keys ...')
writeFunc('Updating model\'s foreign keys ...')
for objFile in wids:
objFile.object.category = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_category_id)])
objFile.save()
......@@ -159,10 +159,10 @@ class Command(BaseCommand):
if not objFile.object.parent_id is None:
objFile.object.parent = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_parent_id)])
objFile.save()
outWriter.stdout.write(' done.\n')
writeFunc(' done.\n')
if replace:
outWriter.stdout.write('Removing unnecessary inputs/options/outputs...')
writeFunc('Removing unnecessary inputs/options/outputs...')
for wid in [wid for wid in objsFile if isinstance(wid.object, AbstractWidget)]:
for inp in AbstractInput.objects.filter(widget = wid.object.id):
for opt in AbstractOption.objects.filter(abstract_input = inp.id):
......@@ -176,13 +176,13 @@ class Command(BaseCommand):
if not importedUids.has_key(out.uid):
statDict['del:'+str(AbstractOutput)]+=1
out.delete()
outWriter.stdout.write(' done.\n')
writeFunc(' done.\n')
#update and output statistics
statDict = dict(statDict.items() + dict([('new:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]]).items())
outWriter.stdout.write('Database models count statistics: pre-import + ( added | modified | deleted ) = after-import\n')
writeFunc('Database models count statistics: pre-import + ( added | modified | deleted ) = after-import\n')
for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]:
outWriter.stdout.write(' % 15s: % 5i + (% 4i | % 4i | % 4i ) = % 5i\n' %
writeFunc(' % 15s: % 5i + (% 4i | % 4i | % 4i ) = % 5i\n' %
(t.__name__,
statDict['old:'+str(t)],
statDict['add:'+str(t)],
......
from mothra.settings import INSTALLED_APPS
appName = 'workflows'
def get_installed_apps():
return [name[len(appName)+1:] for name in INSTALLED_APPS if name.startswith(appName+'.') and len(name)>len(appName)+1]
#Following functions deal with imports of libraries as dicts
def import_all_packages_libs_as_dict(libName):
pckLibs = {}
for pck in get_installed_apps():
pckLibs[pck] = import_package_lib_as_dict(pck, libName)
return pckLibs
def import_package_lib_as_dict(packageName, libName):
return dynamic_import_globals_as_dict(appName+"."+packageName + "." + libName, packageName)
def dynamic_import_globals_as_dict(name, package):
try:
m = __import__(name, globals(), locals(), ['*'])
except:
return None
return m
#Following functions deal with imports of libraries as globals, hovever localSetAttrFunc must be provided - this function should set local global in file where we want import like:
#def setattr_local(name, value, package):
# setattr(sys.modules[__name__], name, value)
def import_all_packages_libs(libName, localSetAttrFunc):
for pck in get_installed_apps():
import_package_lib(pck, libName, localSetAttrFunc)
def import_package_lib(packageName, libName, localSetAttrFunc):
dynamic_import_globals(appName+"."+packageName + "." + libName, packageName, localSetAttrFunc)
def dynamic_import_globals(name, package, localSetAttrFunc):
try:
m = __import__(name, globals(), locals(), ['*'])
except:
return
all_names = [name for name in dir(m) if name[0]!='_']
g = globals()
for name in all_names:
#g[name] = m.__dict__.get(name)
#print name
localSetAttrFunc(name, m.__dict__.get(name), package)
from mothra.settings import INSTALLED_APPS
appName = 'workflows'
import workflows.library
def getAllInstalledAps():
return [name[len(appName)+1:] for name in INSTALLED_APPS if name.startswith(appName+'.') and len(name)>len(appName)+1]
def importAllPackagesLib(libName, localSetattr):
for pck in getAllInstalledAps():
importPackageLib(pck, libName, localSetattr)
def importPackageLib(packageName, libName, localSetattr):
dynamicImportAllGlobals(appName+"."+packageName + "." + libName, packageName, localSetattr)
def dynamicImportAllGlobals(name, package, localSetattr):
try:
m = __import__(name, globals(), locals(), ['*'])
except:
return
all_names = [name for name in dir(m) if name[0]!='_']
g = globals()
for name in all_names:
#g[name] = m.__dict__.get(name)
#print name
localSetattr(name, m.__dict__.get(name), package)
from django.conf.urls.defaults import patterns, include, url
packageUrls = {}
from workflows import packageLibImporter
from workflows import module_importer
def set_package_url(name, value, package):
if name == 'urlpatterns':
packageUrls[package] = value
packageLibImporter.importAllPackagesLib("urls",set_package_url)
module_importer.import_all_packages_libs("urls",set_package_url)
urlpatterns = patterns('',)
......
......@@ -5,10 +5,10 @@ import nlp
from decision_support.visualization import *
from subgroup_discovery.visualization import *
from workflows import packageLibImporter
from workflows import module_importer
def setattr_local(name, value, package):
setattr(sys.modules[__name__], name, value)
packageLibImporter.importAllPackagesLib("visualization_views",setattr_local)
module_importer.import_all_packages_libs("visualization_views",setattr_local)
def odt_to_tab(request,input_dict,output_dict,widget):
import Orange
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment