Commit c58fe07d authored by Janez's avatar Janez

Merge branch 'totrtale_multiproc' of /home/git/repositories/kt/mothra

parents 1ff85c2e 9b785640
...@@ -17,4 +17,3 @@ liac-arff==2.0.1 ...@@ -17,4 +17,3 @@ liac-arff==2.0.1
networkx==1.9.1 networkx==1.9.1
djangorestframework==3.0.3 djangorestframework==3.0.3
django-filter==0.9.1 django-filter==0.9.1
discomll
This diff is collapsed.
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73", "category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "", "treeview_image": "",
"uid": "4f2ce923-62e6-4be1-a394-72ac52988386", "uid": "4f2ce923-62e6-4be1-a394-72ac52988386",
"is_streaming": false, "windows_queue": false,
"package": "big_data", "package": "big_data",
"interaction_view": "", "interaction_view": "",
"has_progress_bar": false, "has_progress_bar": false,
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
"wsdl_method": "", "wsdl_method": "",
"wsdl": "", "wsdl": "",
"interactive": false, "interactive": false,
"windows_queue": false, "is_streaming": false,
"order": 1, "order": 1,
"name": "Distributed Random Forest" "name": "Distributed Random Forest"
} }
...@@ -171,7 +171,7 @@ ...@@ -171,7 +171,7 @@
"required": true, "required": true,
"multi": false, "multi": false,
"parameter_type": "text", "parameter_type": "text",
"variable": "leaf_min_inst", "variable": "min_samples_split",
"parameter": true, "parameter": true,
"order": 4, "order": 4,
"uid": "a2f366a9-af74-4b3f-90ed-33c1fcad4c3a" "uid": "a2f366a9-af74-4b3f-90ed-33c1fcad4c3a"
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73", "category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "", "treeview_image": "",
"uid": "72a39fab-5433-493f-ae22-12a264075356", "uid": "72a39fab-5433-493f-ae22-12a264075356",
"is_streaming": false, "windows_queue": false,
"package": "big_data", "package": "big_data",
"interaction_view": "", "interaction_view": "",
"has_progress_bar": false, "has_progress_bar": false,
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
"wsdl_method": "", "wsdl_method": "",
"wsdl": "", "wsdl": "",
"interactive": false, "interactive": false,
"windows_queue": false, "is_streaming": false,
"order": 1, "order": 1,
"name": "Distributed Weighted Forest" "name": "Distributed Weighted Forest"
} }
...@@ -205,7 +205,7 @@ ...@@ -205,7 +205,7 @@
"required": true, "required": true,
"multi": false, "multi": false,
"parameter_type": "text", "parameter_type": "text",
"variable": "leaf_min_inst", "variable": "min_samples_split",
"parameter": true, "parameter": true,
"order": 5, "order": 5,
"uid": "ac032f38-f4a4-44ea-8c02-96506d4f8e86" "uid": "ac032f38-f4a4-44ea-8c02-96506d4f8e86"
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
"category": "08c90036-fd13-4ff3-ac20-01982262a782", "category": "08c90036-fd13-4ff3-ac20-01982262a782",
"treeview_image": "", "treeview_image": "",
"uid": "d9de579e-fdd9-47da-948e-69183f24340f", "uid": "d9de579e-fdd9-47da-948e-69183f24340f",
"is_streaming": false, "windows_queue": false,
"package": "big_data", "package": "big_data",
"interaction_view": "", "interaction_view": "",
"has_progress_bar": false, "has_progress_bar": false,
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
"wsdl_method": "", "wsdl_method": "",
"wsdl": "", "wsdl": "",
"interactive": false, "interactive": false,
"windows_queue": false, "is_streaming": false,
"order": 1, "order": 1,
"name": "Apply Classifier" "name": "Apply Classifier"
} }
...@@ -37,7 +37,7 @@ ...@@ -37,7 +37,7 @@
"parameter_type": null, "parameter_type": null,
"variable": "dataset", "variable": "dataset",
"parameter": false, "parameter": false,
"order": 5, "order": 4,
"uid": "0c174cfd-0d61-44ff-8ce2-9c32f8791293" "uid": "0c174cfd-0d61-44ff-8ce2-9c32f8791293"
} }
}, },
...@@ -58,23 +58,6 @@ ...@@ -58,23 +58,6 @@
"uid": "46e9a02d-a613-4a01-a8ee-51d578a32fea" "uid": "46e9a02d-a613-4a01-a8ee-51d578a32fea"
} }
}, },
{
"model": "workflows.abstractinput",
"fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Random forest - random state",
"short_name": "rds",
"default": "None",
"description": "Define a random state for predict phase.",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "seed",
"parameter": true,
"order": 3,
"uid": "8ec2b906-2b9e-4cda-9455-09ccd7d134fb"
}
},
{ {
"model": "workflows.abstractinput", "model": "workflows.abstractinput",
"fields": { "fields": {
...@@ -96,7 +79,7 @@ ...@@ -96,7 +79,7 @@
"model": "workflows.abstractinput", "model": "workflows.abstractinput",
"fields": { "fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f", "widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Weighted forest - similarity coeff", "name": "Distributed Weighted forest - similarity coeff",
"short_name": "coe", "short_name": "coe",
"default": "0.5", "default": "0.5",
"description": "Percentage of most similar treees to include in prediction (0 - 1)", "description": "Percentage of most similar treees to include in prediction (0 - 1)",
...@@ -105,7 +88,7 @@ ...@@ -105,7 +88,7 @@
"parameter_type": "text", "parameter_type": "text",
"variable": "coeff", "variable": "coeff",
"parameter": true, "parameter": true,
"order": 4, "order": 3,
"uid": "d979ff23-eb11-40cf-9d81-2a71ddf5d790" "uid": "d979ff23-eb11-40cf-9d81-2a71ddf5d790"
} }
}, },
......
from django.shortcuts import render from django.shortcuts import render
def measure_distribution(request,input_dict,output_dict,widget):
def measure_distribution(request, input_dict, output_dict, widget):
from disco.core import result_iterator from disco.core import result_iterator
from discomll.utils import distribution from discomll.utils import distribution
...@@ -11,9 +12,11 @@ def measure_distribution(request,input_dict,output_dict,widget): ...@@ -11,9 +12,11 @@ def measure_distribution(request,input_dict,output_dict,widget):
dist += str(k.split("/")[-1][:-2]) + " " + str(v) + "\n" dist += str(k.split("/")[-1][:-2]) + " " + str(v) + "\n"
input_dict["string"] = dist input_dict["string"] = dist
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict}) return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_ca(request,input_dict,output_dict,widget): def bigdata_ca(request, input_dict, output_dict, widget):
from discomll.utils import accuracy from discomll.utils import accuracy
import os.path import os.path
from mothra.settings import MEDIA_ROOT from mothra.settings import MEDIA_ROOT
...@@ -21,70 +24,72 @@ def bigdata_ca(request,input_dict,output_dict,widget): ...@@ -21,70 +24,72 @@ def bigdata_ca(request,input_dict,output_dict,widget):
folder = 'discomll_measures' folder = 'discomll_measures'
tag = input_dict["predictions"] tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt' destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination) ensure_dir(destination)
if input_dict["dataset"].params["id_index"] == -1: if input_dict["dataset"].params["id_index"] == -1:
input_dict["string"] = "ID index should be defined." input_dict["string"] = "ID index should be defined."
elif not os.path.isfile(destination): #file doesnt exists elif not os.path.isfile(destination): # file doesnt exists
results = accuracy.measure(test_data = input_dict["dataset"], measure, acc = accuracy.measure(test_data=input_dict["dataset"],
predictions = input_dict["predictions"], predictions=input_dict["predictions"],
measure = "ca") measure="ca")
string = "Classification Accuracy \n" string = "Classification Accuracy \n"
for k, v in results: score = str(measure) + " " + str(acc) + "\n"
string += str(v) + "\n" string += score
input_dict["string"] = string input_dict["string"] = string
f = open(destination,'w') f = open(destination, 'w')
f.write(str(v)) f.write(score)
f.close() f.close()
else: else:
string = "Classification Accuracy \n" #ca results are cached
f = open(destination,'r') string = "Classification Accuracy \n"
f = open(destination, 'r')
input_dict["string"] = string + str(f.readlines()[0]) input_dict["string"] = string + str(f.readlines()[0])
f.close() f.close()
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
def bigdata_mse(request,input_dict,output_dict,widget): return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_mse(request, input_dict, output_dict, widget):
from discomll.utils import accuracy from discomll.utils import accuracy
from disco.core import result_iterator
import os.path import os.path
from mothra.settings import MEDIA_ROOT from mothra.settings import MEDIA_ROOT
from workflows.helpers import ensure_dir from workflows.helpers import ensure_dir
folder = 'discomll_measures' folder = 'discomll_measures'
tag = input_dict["predictions"] tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt' destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination) ensure_dir(destination)
if input_dict["dataset"].params["id_index"] == -1: if input_dict["dataset"].params["id_index"] == -1:
input_dict["string"] = "ID index should be defined." input_dict["string"] = "ID index should be defined."
elif not os.path.isfile(destination): #file doesnt exists elif not os.path.isfile(destination): # file doesnt exists
results = accuracy.measure(test_data = input_dict["dataset"], measure, acc = accuracy.measure(test_data=input_dict["dataset"],
predictions = input_dict["predictions"], predictions=input_dict["predictions"],
measure = "mse") measure="mse")
string = "Mean squared error\n" string = "Mean squared error\n"
for k, v in results: score = str(measure) + " " + str(acc) + "\n"
string += str(v) + "\n" string += score
input_dict["string"] = string input_dict["string"] = string
f = open(destination,'w') f = open(destination, 'w')
f.write(str(v)) f.write(score)
f.close() f.close()
else: else:
string = "Mean squared error\n" string = "Mean squared error\n"
f = open(destination,'r') f = open(destination, 'r')
input_dict["string"] = string + str(f.readlines()[0]) input_dict["string"] = string + str(f.readlines()[0])
f.close() f.close()
return render(request, 'visualizations/display_string.html',
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict}) {'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def results_to_file(request,input_dict,output_dict,widget): def results_to_file(request, input_dict, output_dict, widget):
from disco.core import result_iterator from disco.core import result_iterator
import os.path import os.path
from mothra.settings import MEDIA_ROOT from mothra.settings import MEDIA_ROOT
...@@ -93,14 +98,14 @@ def results_to_file(request,input_dict,output_dict,widget): ...@@ -93,14 +98,14 @@ def results_to_file(request,input_dict,output_dict,widget):
tag = input_dict["string"] tag = input_dict["string"]
folder = 'discomll_results' folder = 'discomll_results'
add = "add" if input_dict["add_params"] == "true" else "" add = "add" if input_dict["add_params"] == "true" else ""
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+add+'.txt' destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + add + '.txt'
ensure_dir(destination) ensure_dir(destination)
if not os.path.isfile(destination): #file doesnt exists if not os.path.isfile(destination): # file doesnt exists
f = open(destination,'w') f = open(destination, 'w')
if input_dict["add_params"] == "true": if input_dict["add_params"] == "true":
for k, v in result_iterator(tag): for k, v in result_iterator(tag):
f.writelines(str(k) + " " + str(v) + "\n") f.writelines(str(k) + " " + str(v) + "\n")
...@@ -108,12 +113,14 @@ def results_to_file(request,input_dict,output_dict,widget): ...@@ -108,12 +113,14 @@ def results_to_file(request,input_dict,output_dict,widget):
for k, v in result_iterator(tag): for k, v in result_iterator(tag):
f.writelines(str(k) + " " + str(v[0]) + "\n") f.writelines(str(k) + " " + str(v[0]) + "\n")
f.close() f.close()
filename = folder+"/"+tag[0][6:]+add+'.txt' filename = folder + "/" + tag[0][6:] + add + '.txt'
output_dict['filename'] = filename output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict}) return render(request, 'visualizations/string_to_file.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def model_view(request,input_dict,output_dict,widget): def model_view(request, input_dict, output_dict, widget):
from discomll.utils import model_view from discomll.utils import model_view
import os.path import os.path
from mothra.settings import MEDIA_ROOT from mothra.settings import MEDIA_ROOT
...@@ -122,23 +129,18 @@ def model_view(request,input_dict,output_dict,widget): ...@@ -122,23 +129,18 @@ def model_view(request,input_dict,output_dict,widget):
folder = 'discomll_models' folder = 'discomll_models'
tag_name = input_dict["fitmodel_url"] tag_name = input_dict["fitmodel_url"]
tag = input_dict["fitmodel_url"].values()[0] tag = input_dict["fitmodel_url"].values()[0]
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
ensure_dir(destination) ensure_dir(destination)
if not os.path.isfile(destination): #file doesnt exists if not os.path.isfile(destination): # file doesnt exists
model = model_view.output_model(tag_name) model = model_view.output_model(tag_name)
f = open(destination,'w') f = open(destination, 'w')
f.write(model) f.write(model)
f.close() f.close()
filename = folder+"/"+tag[0][6:]+'.txt' filename = folder + "/" + tag[0][6:] + '.txt'
output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment