import_package.py 10.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
from django.core.management.base import BaseCommand, CommandError
from workflows.models import Category, AbstractWidget, AbstractInput, AbstractOutput, AbstractOption
from django.core import serializers
from collections import Counter
from optparse import make_option


class Command(BaseCommand):
    args = 'file_name'
    help = 'Imports all models from the file named "file_name". All models in the database which have the same uuid as imported models are updated. The folowing models are included in inport: AbstractWidget, Category, AbstractInput, AbstractOutput, and AbstractOption.'
11
12
13
14
15
16
17
18
19
    option_list = BaseCommand.option_list + (
        make_option('-r', '--replace',
            action="store_true",
            dest='replace',
            default=False,
            help='Completely replace whole widgets with the new one where UIDs match. Default behaviour merges widgets submodels (AbstractInputs, AbstractOutputs and AbstratcOptions)'
                 'based on their submodel\'s own UID. When using this option all widget\'s old submodels are deleted and completely replaced by new submodels.)'
        ),
    )
20
21
22

    def handle(self, *args, **options):
        if (len(args)<1):
23
            raise CommandError('Arguments "file_name" is required!')
24
25

        try:
26
            string = open(args[0], 'r').read()
27
28
29
        except:
            raise CommandError('There was a problem with opening given input file')

30
        self.import_package_string(self, string, options['replace'])
31
32
        self.stdout.write('Import procedure successfully finished.\n')

33
    def import_package_string(self, outWriter, string, replace):
34
        #get all objects from file and eliminate empty UID and check for UID duplicates
35
        objsFile = list(serializers.deserialize("json", string))
36
37
38
        objsFileNoUid = [x for x in objsFile if len(x.object.uid) == 0]
        objsFile = [x for x in objsFile if len(x.object.uid) != 0]
        if len(objsFileNoUid)>0:
39
            outWriter.stdout.write('File contains %i model(s) without UID field set. Those will not be imported! If you wish to'
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
                              ' assign them random UIDs then use the "-n" option when exporting models with the "export_package"'
                              ' command. Afterwards, you will be able to import them.\n' % len(objsFileNoUid))
        if len(Counter([x.object.uid for x in objsFile])) != len(objsFile):
            raise CommandError('Input process terminated without any changes to the database. There were multiple equal '
                               'UIDs defined on different models in the given input file. The input procedure can not continue '
                               'from safety reasons. Please resolve manually!')

        #divide new objects by type
        wids = [x for x in objsFile if isinstance(x.object, AbstractWidget)]
        inps = [x for x in objsFile if isinstance(x.object, AbstractInput)]
        outs = [x for x in objsFile if isinstance(x.object, AbstractOutput)]
        opts = [x for x in objsFile if isinstance(x.object, AbstractOption)]
        cats = [x for x in objsFile if isinstance(x.object, Category)]

        #ouput statistics about file
55
56
57
58
59
60
        outWriter.stdout.write('Import contains:\n')
        outWriter.stdout.write('    % 4i AbstractWidget(s)\n' % len(wids))
        outWriter.stdout.write('    % 4i AbstractInput(s)\n' % len(inps))
        outWriter.stdout.write('    % 4i AbstractOutput(s)\n' % len(outs))
        outWriter.stdout.write('    % 4i AbstractOption(s)\n' % len(opts))
        outWriter.stdout.write('    % 4i Category(s)\n' % len(cats))
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

        #get all objects from database
        objsDb = []
        objsDb.extend(AbstractWidget.objects.all())
        objsDb.extend(AbstractInput.objects.all())
        objsDb.extend(AbstractOutput.objects.all())
        objsDb.extend(AbstractOption.objects.all())
        objsDb.extend(Category.objects.all())

        #check for DB UID duplicates
        objsdbDict = dict((x.uid,x) for x in objsDb if len(x.uid) != 0)
        if len([x for x in objsDb if len(x.uid) != 0]) != len(objsdbDict):
            raise CommandError('Input process terminated without any changes to the database. There were multiple equal '
                               'UIDs defined on different models in the database. The input procedure can not continue '
                               'from safety reasons. Please resolve manually!')

        #create new to existing id mapping and check for type match
        idMappingDict = dict()
        for objFile in objsFile:
            if objsdbDict.has_key(objFile.object.uid):
                objDb = objsdbDict[objFile.object.uid]
                objFileTypeId = str(type(objFile.object))+':'+str(objFile.object.id)
                objDbTypeId = str(type(objDb))+':'+str(objDb.id)
                if type(objFile.object) == type(objsdbDict[objFile.object.uid]):
                    idMappingDict[objFileTypeId] = objDb.id
                else:
                    raise CommandError('Input process terminated without any changes to the database. Two models match by uid but not '
                                       'by type:\n    - from file: %s\n    - from database: %s\n    Please resolve manually!'% (objFileTypeId, objDbTypeId))

        #ouput statistics about database
91
92
93
        outWriter.stdout.write('Current database contains %i models,\n' % len(objsDb))
        outWriter.stdout.write('    of which %i models have UID set,\n' % len(objsdbDict))
        outWriter.stdout.write('    of which %i models match with the imported models and will be updated.\n' % len(idMappingDict))
94
95
96

        #prepare statistics
        statDict = dict([('old:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]])
97
        for modelType in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]:
98
            for operation in ['mod','add','del']:
99
                statDict[operation+':'+str(modelType)]=0
100

101
102
103
104
105
        #sort so that AbstractWidgets come in front (are processsed first in the following block)
        #objsFileNew = []
        #objsFileNew.extend([objFile for objFile in objsFile if isinstance(objFile.object, AbstractWidget)])
        #objsFileNew.extend([objFile for objFile in objsFile if not isinstance(objFile.object, AbstractWidget)])
        #objsFile = objsFileNew
106
107

        #save models to the database - update the ids for the matching models and remove the ids (to get a new one) for the non matching models
108
        outWriter.stdout.write('Merging file and database models ...')
109
        importedUids = dict()
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        for objFile in objsFile:
            objFileTypeId = str(type(objFile.object))+':'+str(objFile.object.id)
            if isinstance(objFile.object, AbstractWidget):
                objFile.old_category_id = objFile.object.category_id
            if isinstance(objFile.object, AbstractInput):
                objFile.old_widget_id = objFile.object.widget_id
            if isinstance(objFile.object, AbstractOutput):
                objFile.old_widget_id = objFile.object.widget_id
            if isinstance(objFile.object, AbstractOption):
                objFile.old_abstract_input_id = objFile.object.abstract_input_id
            if isinstance(objFile.object, Category):
                if not objFile.object.parent_id is None:
                    objFile.old_parent_id = objFile.object.parent_id

            if idMappingDict.has_key(objFileTypeId):
125
                #there is already an existing model with same uid
126
127
128
                statDict['mod:'+str(type(objFile.object))]+=1
                objFile.object.id = idMappingDict[objFileTypeId]
            else:
129
                #there is no model jet, add it
130
131
132
133
                statDict['add:'+str(type(objFile.object))]+=1
                objFile.object.id = None
            objFile.save()
            idMappingDict[objFileTypeId] = objFile.object.id
134
            importedUids[objFile.object.uid]=True
135
        outWriter.stdout.write(' done.\n')
136
137

        #correct also the foreign keys
138
        outWriter.stdout.write('Updating model\'s foreign keys ...')
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
        for objFile in wids:
            objFile.object.category = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_category_id)])
            objFile.save()
        for objFile in inps:
            objFile.object.widget = AbstractWidget.objects.get(id=idMappingDict[str(AbstractWidget)+':'+str(objFile.old_widget_id)])
            objFile.save()
        for objFile in outs:
            objFile.object.widget = AbstractWidget.objects.get(id=idMappingDict[str(AbstractWidget)+':'+str(objFile.old_widget_id)])
            objFile.save()
        for objFile in opts:
            objFile.object.abstract_input = AbstractInput.objects.get(id=idMappingDict[str(AbstractInput)+':'+str(objFile.old_abstract_input_id)])
            objFile.save()
        for objFile in cats:
            if not objFile.object.parent_id is None:
                objFile.object.parent = Category.objects.get(id=idMappingDict[str(Category)+':'+str(objFile.old_parent_id)])
                objFile.save()
155
        outWriter.stdout.write(' done.\n')
156
157

        if replace:
158
            outWriter.stdout.write('Removing unnecessary inputs/options/outputs...')
159
160
161
162
163
164
165
166
167
168
169
170
171
            for wid in [wid for wid in objsFile if isinstance(wid.object, AbstractWidget)]:
                for inp in AbstractInput.objects.filter(widget = wid.object.id):
                    for opt in AbstractOption.objects.filter(abstract_input = inp.id):
                        if not importedUids.has_key(opt.uid):
                            statDict['del:'+str(AbstractOption)]+=1
                            opt.delete()
                    if not importedUids.has_key(inp.uid):
                        statDict['del:'+str(AbstractInput)]+=1
                        inp.delete()
                for out in AbstractOutput.objects.filter(widget = wid.object.id):
                    if not importedUids.has_key(out.uid):
                        statDict['del:'+str(AbstractOutput)]+=1
                        out.delete()
172
            outWriter.stdout.write(' done.\n')
173
174
175

        #update and output statistics
        statDict = dict(statDict.items() + dict([('new:'+str(t),len(t.objects.all())) for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]]).items())
176
        outWriter.stdout.write('Database models count statistics: pre-import + ( added | modified | deleted ) = after-import\n')
177
        for t in [AbstractWidget, AbstractInput, AbstractOutput, AbstractOption, Category]:
178
            outWriter.stdout.write('    % 15s: % 5i + (% 4i | % 4i | % 4i ) = % 5i\n' %
179
180
181
182
183
184
                             (t.__name__,
                              statDict['old:'+str(t)],
                              statDict['add:'+str(t)],
                              statDict['mod:'+str(t)],
                              statDict['del:'+str(t)],
                              statDict['new:'+str(t)]))