Merge branch 'dev' of workflow.ijs.si:mothra into dev

workflows/cforange/interaction_views.py

 from django.shortcuts import render
+import json
 
 def cforange_filter_integers(request,input_dict,output_dict,widget):
-    return render(request, 'interactions/cforange_filter_integers.html',{'widget':widget,'intList':input_dict['intList']})
\ No newline at end of file
+    return render(request, 'interactions/cforange_filter_integers.html',{'widget':widget,'intList':input_dict['intList']})
+
+def cforange_hierarchical_clustering(request,input_dict,output_dict,widget):
+    import orange
+    from library import Clustering
+    matrix = input_dict['dm']
+    linkage = int(input_dict['linkage'])
+    root = Clustering.hierarchical_clustering(linkage, matrix)
+    dm_examples = True
+    try:
+        attributes = [x.name for x in matrix.items.domain]
+    except:
+        attributes = ['attribute']
+        dm_examples = False
+    def build_hierarchy(node, root=False):
+        if dm_examples:
+            values_dict = dict([(x,matrix.items[node.first][x].value) for x in attributes]) if not node.branches else {}
+        else:
+            values_dict = dict([(x,matrix.items[node.first].name) for x in attributes]) if not node.branches else {}
+        for attribute in values_dict.keys():
+            if type(values_dict[attribute]) == float:
+                values_dict[attribute]="%.3f" % values_dict[attribute]
+        return {
+            'name' : 'root' if root else '',
+            'id' : node.first if not node.branches else -1,
+            'height' : node.height if node.branches else 0,
+            'children' : [build_hierarchy(node.left), build_hierarchy(node.right)] if node.branches else [],
+            'values' : values_dict,
+            'leaf' : True if not node.branches else False
+        }
+    hierarchy = json.dumps(build_hierarchy(root, root=True))
+    return render(request, 'interactions/cforange_hierarchical_clustering.html', {'widget' : widget, 'hierarchy' : hierarchy, 'attributes':attributes,'vizualization':input_dict['visualization']})

workflows/cforange/library.py

@@ -169,7 +169,6 @@ def cforange_confusion_matrix(input_dict):
         cm = orngStat.confusionMatrices(results,classIndex=classIndex)
     if len(cm)==1:
         cm = cm[0]
-    print cm
     output_dict = {}
     output_dict['cm']=cm
     return output_dict
@@ -282,4 +281,160 @@ def cforange_prepare_results(input_dict):
         newdict['fscore']=input_dict['f'][i]
         newlist.append(newdict)
     output_dict['alp']=newlist
-    return output_dict
\ No newline at end of file
+    return output_dict
+
+def cforange_example_distance(input_dict):
+    import orange
+    import random
+    import orngClustering
+    import orngMisc
+    inputdata = input_dict['dataset']
+    metricsIndex = int(input_dict['distanceMetrics'])
+    metrics = [
+        ("Euclidean", orange.ExamplesDistanceConstructor_Euclidean),
+        ("Pearson Correlation", orngClustering.ExamplesDistanceConstructor_PearsonR),
+        ("Spearman Rank Correlation", orngClustering.ExamplesDistanceConstructor_SpearmanR),
+        ("Manhattan", orange.ExamplesDistanceConstructor_Manhattan),
+        ("Hamming", orange.ExamplesDistanceConstructor_Hamming),
+        ("Relief", orange.ExamplesDistanceConstructor_Relief),
+        ]
+
+    normalize = input_dict['normalization']
+    if normalize=='true':
+        normalize = True
+    else:
+        normalize = False
+
+    data = inputdata
+    constructor = metrics[metricsIndex][1]()
+    constructor.normalize = normalize
+    dist = constructor(data)
+    matrix = orange.SymMatrix(len(data))
+    matrix.setattr('items', data)
+    for i in range(len(data)):
+        for j in range(i+1):
+            matrix[i, j] = dist(data[i], data[j])
+    output_dict = {}
+    output_dict['dm']=matrix
+    return output_dict
+
+def cforange_attribute_distance(input_dict):
+    import orange
+    import orngInteract
+    inputdata = input_dict['dataset']
+    discretizedData = None
+    classInteractions = int(input_dict['classInteractions'])
+    atts = inputdata.domain.attributes
+    if len(atts) < 2:
+        return None
+    matrix = orange.SymMatrix(len(atts))
+    matrix.setattr('items', atts)
+    if classInteractions < 3:
+        if inputdata.domain.hasContinuousAttributes():
+            if discretizedData is None:
+                try:
+                    discretizedData = orange.Preprocessor_discretize(inputdata, method=orange.EquiNDiscretization(numberOfIntervals=4))
+                except orange.KernelException, ex:
+                    return None
+            data = discretizedData
+        else:
+            data = inputdata
+
+        # This is ugly (no shit)
+        if not data.domain.classVar:
+            if classInteractions == 0:
+                classedDomain = orange.Domain(data.domain.attributes, orange.EnumVariable("foo", values=["0", "1"]))
+                data = orange.ExampleTable(classedDomain, data)
+            else:
+                return None
+
+        im = orngInteract.InteractionMatrix(data, dependencies_too=1)
+        off = 1
+        if classInteractions == 0:
+            diss,labels = im.exportChi2Matrix()
+            off = 0
+        elif classInteractions == 1:
+            (diss,labels) = im.depExportDissimilarityMatrix(jaccard=1)  # 2-interactions
+        else:
+            (diss,labels) = im.exportDissimilarityMatrix(jaccard=1)  # 3-interactions
+
+        for i in range(len(atts)-off):
+            for j in range(i+1):
+                matrix[i+off, j] = diss[i][j]
+
+    else:
+        if classInteractions == 3:
+            for a1 in range(len(atts)):
+                for a2 in range(a1):
+                    matrix[a1, a2] = (1.0 - orange.PearsonCorrelation(a1, a2, inputdata, 0).r) / 2.0
+        else:
+            if len(inputdata) < 3:
+                return None
+            import numpy, statc
+            m = inputdata.toNumpyMA("A")[0]
+            averages = numpy.ma.average(m, axis=0)
+            filleds = [list(numpy.ma.filled(m[:,i], averages[i])) for i in range(len(atts))]
+            for a1, f1 in enumerate(filleds):
+                for a2 in range(a1):
+                    matrix[a1, a2] = (1.0 - statc.spearmanr(f1, filleds[a2])[0]) / 2.0
+    output_dict = {}
+    output_dict['dm']=matrix        
+    return output_dict
+
+def cforange_hierarchical_clustering(input_dict):
+    return {'centroids' : None, 'selected_examples' : None, 'unselected_examples' : None}
+
+class Clustering:
+    @staticmethod
+    def hierarchical_clustering(linkage, distance_matrix):
+        import orange
+        linkages = [("Single linkage", orange.HierarchicalClustering.Single),
+                    ("Average linkage", orange.HierarchicalClustering.Average),
+                    ("Ward's linkage", orange.HierarchicalClustering.Ward),
+                    ("Complete linkage", orange.HierarchicalClustering.Complete)]
+        return orange.HierarchicalClustering(distance_matrix, linkage=linkages[linkage][1])
+
+def cforange_hierarchical_clustering_finished(postdata, input_dict, output_dict):
+    import json
+    import orange
+    matrix = input_dict['dm']
+    linkage = int(input_dict['linkage'])
+    widget_pk = postdata['widget_id'][0]
+    try:
+        selected_nodes = json.loads(postdata.get('selected_nodes')[0])
+    except:
+        raise Exception('Please select a threshold for determining clusters.')
+    if isinstance(matrix.items, orange.ExampleTable):
+        root = Clustering.hierarchical_clustering(linkage, matrix)
+        cluster_ids = set([cluster for _,_,cluster in selected_nodes])
+        selected_clusters = set([cluster for _,selected,cluster in selected_nodes if selected])
+        clustVar = orange.EnumVariable(str('Cluster'), values=["Cluster %d" % i for i in cluster_ids] + ["Other"])
+        origDomain = matrix.items.domain
+        domain = orange.Domain(origDomain.attributes, origDomain.classVar)
+        domain.addmeta(orange.newmetaid(), clustVar)
+        domain.addmetas(origDomain.getmetas())
+        # Build table with selected clusters
+        selected_table, unselected_table = orange.ExampleTable(domain), orange.ExampleTable(domain)
+        for id, selected, cluster in selected_nodes:
+            new_ex = orange.Example(domain, matrix.items[id])
+            if selected:
+                new_ex[clustVar] = clustVar("Cluster %d" % cluster)
+                selected_table.append(new_ex)
+            else:
+                new_ex[clustVar] = clustVar("Other")
+                unselected_table.append(new_ex)
+        # Build table of centroids
+        centroids = orange.ExampleTable(selected_table.domain)
+        if len(selected_table) > 0:
+            for cluster in sorted(selected_clusters):
+                clusterEx = orange.ExampleTable([ex for ex in selected_table if ex[clustVar] == "Cluster %d" % cluster])
+                # Attribute statistics
+                contstat = orange.DomainBasicAttrStat(clusterEx)
+                discstat = orange.DomainDistributions(clusterEx, 0, 0, 1)
+                ex = [cs.avg if cs else (ds.modus() if ds else "?") for cs, ds in zip(contstat, discstat)]
+                example = orange.Example(centroids.domain, ex)
+                example[clustVar] = clustVar("Cluster %d" % cluster)
+                centroids.append(example)
+    else: # Attribute distance
+        centroids, selected_table, unselected_table = None, None, None
+    return {'centroids' : centroids, 'selected_examples' : selected_table, 'unselected_examples' : unselected_table}

workflows/cforange/templates/interactions/cforange_hierarchical_clustering.html

+<div id="widgetinteract-{{widget.pk}}" rel="{{widget.pk}}" class="widgetinteractdialog" title="{{widget.name}} interaction" width="1050" height="800">
+        <div id="clustering_canvas{{widget.pk}}"></div>
+        <form> 
+            <input type="hidden" name="widget_id" value="{{widget.pk}}" style="display:none;" />
+            <input type="hidden" name="selected_height" value="0" style="display:none;"/>
+            <input type="hidden" class="selected_nodes{{widget.pk}}" name="selected_nodes" value="" style="display:none;"/>
+        </form>
+        <div>
+          Display label:
+          <select class="attributeselect">
+            <option value="-1">None</option>
+            {% for attribute in attributes %}
+            <option value="{{forloop.counter0}}">{{attribute}}</option>
+            {% endfor %}
+          </select>
+        </div>
+        <div id="legend{{widget.pk}}">
+        </div>
+
+</div>
+<style type="text/css">
+path.arc {
+ /* cursor: move;*/
+  fill: #fff;
+}
+
+path.selectedarc {
+  fill: red;
+}
+
+.node circle {
+  fill: #fff;
+  stroke: black;
+  stroke-width: 1.5px;
+}
+
+.node {
+  font-size: 10px;
+  pointer-events: none;
+}
+
+.link {
+  fill: none;
+  stroke: #ccc;
+  stroke-width: 1.5px;
+}
+
+
+</style>
+<script type="text/javascript">
+var hierarchy = {{hierarchy|safe}};
+draw(1000, 600, hierarchy, "#clustering_canvas{{widget.pk}}");
+function draw(w, h, hierarchy, target) {
+
+    var vizualization = '{{vizualization}}';
+
+    var colors = ['red','blue','yellow','green','magenta','cyan','black'];
+
+    var attributes = [];
+    {% for attribute in attributes %}
+    attributes.push('{{attribute}}');
+    {% endfor %}
+
+    var rx = w/2,
+      ry = h/2,
+      m0,
+      rotate = 0,
+      delta = 30;
+
+    var root = [rx, ry];
+    var width = ry - delta;
+    if (vizualization == "tree") {
+      width=w-100;
+      height=width*2;
+      ry=1000;
+    }
+
+    if (vizualization == "circle") {
+    var cluster = d3.layout.cluster()
+      .size([360, ry - delta])
+      .sort(null);
+    } else {
+    var cluster = d3.layout.cluster()
+      .size([h-50, ry - delta])
+      .sort(null);      
+    }
+
+
+if (vizualization=="circle") {
+
+    var diagonal = d3.svg.diagonal.radial()
+      .projection(function(d) { return [d.y, d.x / 180 * Math.PI]; });
+
+    } else {
+
+ var diagonal = d3.svg.diagonal()
+    .projection(function(d) { return [d.y, d.x]; });     
+    }
+
+    var svg = d3.select(target).append("div")
+    //  .style("width", w + "px")
+    //  .style("height", (h+50) + "px")
+      .on("click", select_clusters);
+
+      if (vizualization=="circle") {
+    var vis = svg.append("svg:svg")
+      .attr("width", w)
+      .attr("height", h+50)
+    .append("svg:g")
+      .attr("transform", "translate(" + rx + "," + ry + ")");
+
+    } else {
+     var vis = svg.append("svg:svg")
+      .attr("width", w)
+      .attr("height", h)
+    .append("svg:g")
+    .attr("transform", "translate(40,10)");   
+    }
+
+      if (vizualization=="circle") {
+      vis.append("svg:path")
+        .attr("class", "arc")
+        .attr("d", d3.svg.arc().innerRadius(ry - delta).outerRadius(ry).startAngle(0).endAngle(2 * Math.PI))
+        .on("mousedown", mousedown);
+    }
+
+    var nodes = cluster.nodes(hierarchy);
+    var max_height = 0;
+    for (node in nodes) {
+        if (nodes[node].height>max_height) {
+            max_height=nodes[node].height;
+        }
+    }
+
+    for (node in nodes) {
+        nodes[node].y = width-(nodes[node].height/max_height) * width;
+    }
+
+    var link = vis.selectAll("path.link")
+      .data(cluster.links(nodes))
+    .enter().append("svg:path")
+      .attr("class", "link")
+      .attr("d", diagonal);
+
+
+if (vizualization=="circle") {
+    var node = vis.selectAll("g.node")
+      .data(nodes)
+    .enter().append("svg:g")
+      .attr("class", "node")
+      .attr("transform", function(d) { return "rotate(" + (d.x - 90) + ")translate(" + d.y + ")"; });
+
+    node.append("svg:circle")
+      .attr("r", 3);
+
+    } else {
+    var node = vis.selectAll("g.node")
+      .data(nodes)
+    .enter().append("svg:g")
+      .attr("class", "node")
+      .attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; })
+
+    node.append("svg:circle")
+      .attr("r", 3);      
+    }
+
+    $(".node").mouseenter(function() {
+      alert("test");
+    });
+
+    if (vizualization=="circle") {
+      for (a in attributes) {
+        node.append("svg:text")
+          .attr("dx", function(d) { return d.x < 180 ? 8 : -8; })
+          .attr("dy", ".31em")
+          .attr("text-anchor", function(d) { return d.x < 180 ? "start" : "end"; })
+          .attr("transform", function(d) { return d.x < 180 ? null : "rotate(180)"; })
+          .attr("class", "attribute"+a+" attributes")
+          .text(function(d) { return d.values[attributes[a]]; });
+      }
+    } else {
+       for (a in attributes) {
+        node.append("svg:text")
+          .attr("dx", -10)
+          .attr("dy", 3)
+          .style("text-anchor", function(d) { return d.children ? "end" : "start"; })
+          .attr("class", "attribute"+a+" attributes")
+          .text(function(d) { return d.values[attributes[a]]; });
+      }     
+    }
+
+    $(".attributes").hide();
+
+    $(".attribute"+$(".attributeselect").val()).show();
+
+    $(".attributeselect").change(function () {
+      $(".attributes").hide();
+      $(".attribute"+$(".attributeselect").val()).show();      
+    })
+
+    d3.select(window)
+      .on("mousemove", mousemove)
+      .on("mouseup", mouseup);
+
+    var cluster_index=0;
+    var clusters = [];
+    var counters = [];
+
+    function color_nodes(node,treshold,cluster) {
+        if (node.height<treshold||node.height==0) {
+          node.cluster = cluster;
+          if (clusters.indexOf(cluster)==-1) {
+           clusters.push(cluster);
+           counters.push(0);
+           selected_clusters.push(true);
+          }
+          for (n in node.children) {
+            color_nodes(node.children[n],treshold,cluster);
+          }
+        } else {
+          node.cluster = -1;
+          for (n in node.children) {
+            color_nodes(node.children[n],treshold,++cluster_index);
+          }
+        }
+
+    }
+
+    hsv2rgb = function(h,s,v) {
+      var rgb, i, data = [];
+      if (s === 0) {
+        rgb = [v,v,v];
+      } else {
+        h = h / 60;
+        i = Math.floor(h);
+        data = [v*(1-s), v*(1-s*(h-i)), v*(1-s*(1-(h-i)))];
+        switch(i) {
+          case 0:
+            rgb = [v, data[2], data[0]];
+            break;
+          case 1:
+            rgb = [data[1], v, data[0]];
+            break;
+          case 2:
+            rgb = [data[0], v, data[2]];
+            break;
+          case 3:
+            rgb = [data[0], data[1], v];
+            break;
+          case 4:
+            rgb = [data[2], data[0], v];
+            break;
+          default:
+            rgb = [v, data[0], data[1]];
+            break;
+        }
+      }
+      return '#' + rgb.map(function(x){ 
+        return ("0" + Math.round(x*255).toString(16)).slice(-2);
+      }).join('');
+    };    
+
+    var threshold_arc = null;
+    var root_node = nodes[0];
+
+    function display_nodes() {
+      return_nodes = [];
+      node.selectAll("circle")
+          .style('fill', function(d) {
+            if (d.cluster==-1) {
+              return 'white';
+            } else {
+              if (d.children.length==0) {
+                counters[clusters.indexOf(d.cluster)]++;
+              } 
+              var sat = 1;               
+              if (!selected_clusters[clusters.indexOf(d.cluster)]) {
+                d.selected = false;
+                sat=0.15
+              } else {
+                d.selected = true;
+              }
+              return hsv2rgb(((clusters.indexOf(d.cluster)*1.0)/(clusters.length))*360,sat,1);
+            }
+          })
+          .style('stroke', function(d) {
+              if (d.cluster==-1) {
+                return 'black';
+              } else {
+              var sat = 1;               
+              if (!selected_clusters[clusters.indexOf(d.cluster)]) {
+                sat=0.15
+              }
+
+              if (d.leaf) {
+                return_nodes.push([d.id,d.selected,clusters.indexOf(d.cluster)]);
+              }
+
+              return hsv2rgb(((clusters.indexOf(d.cluster)*1.0)/(clusters.length))*360,sat,0.8);
+              }
+
+          });
+          $(".selected_nodes{{widget.pk}}").val(JSON.stringify(return_nodes));
+      }
+
+    function select_clusters() {
+        var click_pos = d3.mouse(this);
+        if (vizualization=="circle") {
+          var r = Math.min(distance(click_pos, root), ry-delta);
+          var r_dist = distance(click_pos,root)
+          if (r_dist > ry-delta) {
+            return false;
+          }
+          var r_height = max_height-((distance(click_pos,root)/(ry-delta))*max_height);
+
+          if (threshold_arc) {
+              threshold_arc.remove();
+          }
+          threshold_arc = vis.append("svg:path")
+            .attr("class", "selectedarc")
+            .attr("d", d3.svg.arc().innerRadius(r).outerRadius(r+1).startAngle(0).endAngle(2 * Math.PI));
+
+        } else {
+          var x = click_pos[0];
+          x = x-40;
+          if (x<0) {
+            x=0;
+          }
+          var r_height = max_height-(x/width)*max_height;
+
+          if (threshold_arc) {
+              threshold_arc.remove();
+          }
+          threshold_arc = vis.append("svg:line")
+            .attr("class", "selectedarc")
+            .attr("x1",x)
+            .attr("y1","0")
+            .attr("x2",x)
+            .attr("y2","1000")
+            .attr("stroke-width","2")
+            .attr("stroke","red");
+        }
+        cluster_index = 0;
+        clusters = [];
+        counters = [];
+        selected_clusters = [];
+        color_nodes(nodes[0],r_height,0);
+
+        //clusters.sort(function() { return 0.5 - Math.random();});
+