Skip to content
GitLab
Commit 36b971d2 authored by matjaz's avatar matjaz
Browse files

Improved import_package functionality - now import can remove widget's... 

Improved import_package functionality - now import can remove widget's outputs,inputs and options which are not beeing used.
parent 27094eb9
Hide whitespace changes
Inline Side-by-side
workflows/management/commands/import_package.py
View file @ 36b971d2
......@@ -8,6 +8,15 @@ from optparse import make_option
class Command(BaseCommand):
args = 'file_name'
help = 'Imports all models from the file named "file_name". All models in the database which have the same uuid as imported models are updated. The folowing models are included in inport: AbstractWidget, Category, AbstractInput, AbstractOutput, and AbstractOption.'
option_list = BaseCommand.option_list + (
make_option('-r', '--replace',
action="store_true",
dest='replace',
default=False,
help='Completely replace whole widgets with the new one where UIDs match. Default behaviour merges widgets submodels (AbstractInputs, AbstractOutputs and AbstratcOptions)'
'based on their submodel\'s own UID. When using this option all widget\'s old submodels are deleted and completely replaced by new submodels.)'
),
)
def handle(self, *args, **options):
if (len(args)<1):
......@@ -18,12 +27,16 @@ class Command(BaseCommand):
except:
raise CommandError('There was a problem with opening given input file')
self.importString(self, instr, options['replace'])
self.stdout.write('Import procedure successfully finished.\n')
def importString(self, obj, instr, replace):
#get all objects from file and eliminate empty UID and check for UID duplicates
objsFile = list(serializers.deserialize("json", instr))
objsFileNoUid = [x for x in objsFile if len(x.object.uid) == 0]
objsFile = [x for x in objsFile if len(x.object.uid) != 0]
if len(objsFileNoUid)>0:
self.stdout.write('File contains %i model(s) without UID field set. Those will not be imported! If you wish to'
obj.stdout.write('File contains %i model(s) without UID field set. Those will not be imported! If you wish to'
' assign them random UIDs then use the "-n" option when exporting models with the "export_package"'
' command. Afterwards, you will be able to import them.\n' % len(objsFileNoUid))
if len(Counter([x.object.uid for x in objsFile])) != len(objsFile):
......@@ -39,12 +52,12 @@ class Command(BaseCommand):
cats = [x for x in objsFile if isinstance(x.object, Category)]
#ouput statistics about file
self.stdout.write('File contains:\n')
self.stdout.write(' % 4i AbstractWidget(s)\n' % len(wids))
self.stdout.write(' % 4i AbstractInput(s)\n' % len(inps))
self.stdout.write(' % 4i AbstractOutput(s)\n' % len(outs))
self.stdout.write(' % 4i AbstractOption(s)\n' % len(opts))
self.stdout.write(' % 4i Category(s)\n' % len(cats))
obj.stdout.write('Import contains:\n')
obj.stdout.write(' % 4i AbstractWidget(s)\n' % len(wids))
obj.stdout.write(' % 4i AbstractInput(s)\n' % len(inps))
obj.stdout.write(' % 4i AbstractOutput(s)\n' % len(outs))
obj.stdout.write(' % 4i AbstractOption(s)\n' % len(opts))
obj.stdout.write(' % 4i Category(s)\n' % len(cats))
#get all objects from database
objsDb = []
......@@ -54,8 +67,6 @@ class Command(BaseCommand):
objsDb.extend(AbstractOption.objects.all())
objsDb.extend(Category.objects.all())
#check for DB UID duplicates
objsdbDict = dict((x.uid,x) for x in objsDb if len(x.uid) != 0)
if len([x for x in objsDb if len(x.uid) != 0]) != len(objsdbDict):
......@@ -77,20 +88,25 @@ class Command(BaseCommand):
'by type:\n - from file: %s\n - from database: %s\n Please resolve manually!'% (objFileTypeId, objDbTypeId))
#ouput statistics about database
self.stdout.write('Current database contains %i models,\n' % len(objsDb))
self.stdout.write(' of which %i models have UID set,\n' % len(objsdbDict))
self.stdout.write(' of which %i models match with the imported models and will be updated.\n' % len(idMappingDict))
obj.stdout.write('Current database contains %i models,\n' % len(objsDb))
obj.stdout.write(' of which %i models have UID set,\n' % len(objsdbDict))
obj.stdout.write(' of which %i models match with the imported models and will be updated.\n' % len(idMappingDict))
#prepare statistics
statDict = dict([('old:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]])
for objFile in objsFile:
statDict['mod:'+str(type(objFile.object))]=0
statDict['add:'+str(type(objFile.object))]=0
for operation in ['mod','add','del']:
statDict[operation+':'+str(type(objFile.object))]=0
#sort so that AbstractWidgets come in front (are processsed first in the following block)
#objsFileNew = []
#objsFileNew.extend([objFile for objFile in objsFile if isinstance(objFile.object, AbstractWidget)])
#objsFileNew.extend([objFile for objFile in objsFile if not isinstance(objFile.object, AbstractWidget)])
#objsFile = objsFileNew
#save models to the database - update the ids for the matching models and remove the ids (to get a new one) for the non matching models
self.stdout.write('Merging file and database models ...')
obj.stdout.write('Merging file and database models ...')
importedUids = dict()
for objFile in objsFile:
objFileTypeId = str(type(objFile.object))+':'+str(objFile.object.id)
if isinstance(objFile.object, AbstractWidget):
......@@ -106,17 +122,20 @@ class Command(BaseCommand):
objFile.old_parent_id = objFile.object.parent_id
if idMappingDict.has_key(objFileTypeId):
#there is already an existing model with same uid
statDict['mod:'+str(type(objFile.object))]+=1
objFile.object.id = idMappingDict[objFileTypeId]
else:
#there is no model jet, add it
statDict['add:'+str(type(objFile.object))]+=1
objFile.object.id = None
objFile.save()
idMappingDict[objFileTypeId] = objFile.object.id
self.stdout.write(' done.\n')
importedUids[objFile.object.uid]=True
obj.stdout.write(' done.\n')
#correct also the foreign keys
self.stdout.write('Updating model\'s foreign keys ...')
obj.stdout.write('Updating model\'s foreign keys ...')
for objFile in wids:
objFile.object.category = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_category_id)])
objFile.save()
......@@ -133,13 +152,33 @@ class Command(BaseCommand):
if not objFile.object.parent_id is None:
objFile.object.parent = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_parent_id)])
objFile.save()
self.stdout.write(' done.\n')
obj.stdout.write(' done.\n')
if replace:
obj.stdout.write('Removing unnecessary inputs/options/outputs...')
for wid in [wid for wid in objsFile if isinstance(wid.object, AbstractWidget)]:
for inp in AbstractInput.objects.filter(widget = wid.object.id):
for opt in AbstractOption.objects.filter(abstract_input = inp.id):
if not importedUids.has_key(opt.uid):
statDict['del:'+str(AbstractOption)]+=1
opt.delete()
if not importedUids.has_key(inp.uid):
statDict['del:'+str(AbstractInput)]+=1
inp.delete()
for out in AbstractOutput.objects.filter(widget = wid.object.id):
if not importedUids.has_key(out.uid):
statDict['del:'+str(AbstractOutput)]+=1
out.delete()
obj.stdout.write(' done.\n')
#update and output statistics
statDict = dict(statDict.items() + dict([('new:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]]).items())
self.stdout.write('Database models count statistics: pre-import + ( added | modified ) = after-import\n')
obj.stdout.write('Database models count statistics: pre-import + ( added | modified | deleted ) = after-import\n')
for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]:
self.stdout.write(' % 15s: % 5i + (% 4i | % 4i ) = % 5i\n' % (t.__name__, statDict['old:'+str(t)], statDict['add:'+str(t)], statDict['mod:'+str(t)], statDict['new:'+str(t)]))
self.stdout.write('Import procedure successfully finished.\n')
\ No newline at end of file
obj.stdout.write(' % 15s: % 5i + (% 4i | % 4i | % 4i ) = % 5i\n' %
(t.__name__,
statDict['old:'+str(t)],
statDict['add:'+str(t)],
statDict['mod:'+str(t)],
statDict['del:'+str(t)],
statDict['new:'+str(t)]))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment