Skip to content

GitLab

C
clowdflows
Commit c58fe07d authored by Janez's avatar Janez
Browse files
Options

Merge branch 'totrtale_multiproc' of /home/git/repositories/kt/mothra

parents 1ff85c2e 9b785640
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 198 additions and 198 deletions
requirements.txt
View file @ c58fe07d
......@@ -17,4 +17,3 @@ liac-arff==2.0.1
networkx==1.9.1
djangorestframework==3.0.3
django-filter==0.9.1
discomll
workflows/big_data/library.py
View file @ c58fe07d
This diff is collapsed.
workflows/big_data/package_data/widgets/4f2ce923-62e6-4be1-a394-72ac52988386.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "",
"uid": "4f2ce923-62e6-4be1-a394-72ac52988386",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Distributed Random Forest"
}
......@@ -171,7 +171,7 @@
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "leaf_min_inst",
"variable": "min_samples_split",
"parameter": true,
"order": 4,
"uid": "a2f366a9-af74-4b3f-90ed-33c1fcad4c3a"
......
workflows/big_data/package_data/widgets/72a39fab-5433-493f-ae22-12a264075356.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "",
"uid": "72a39fab-5433-493f-ae22-12a264075356",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Distributed Weighted Forest"
}
......@@ -205,7 +205,7 @@
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "leaf_min_inst",
"variable": "min_samples_split",
"parameter": true,
"order": 5,
"uid": "ac032f38-f4a4-44ea-8c02-96506d4f8e86"
......
workflows/big_data/package_data/widgets/d9de579e-fdd9-47da-948e-69183f24340f.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "08c90036-fd13-4ff3-ac20-01982262a782",
"treeview_image": "",
"uid": "d9de579e-fdd9-47da-948e-69183f24340f",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Apply Classifier"
}
......@@ -37,7 +37,7 @@
"parameter_type": null,
"variable": "dataset",
"parameter": false,
"order": 5,
"order": 4,
"uid": "0c174cfd-0d61-44ff-8ce2-9c32f8791293"
}
},
......@@ -58,23 +58,6 @@
"uid": "46e9a02d-a613-4a01-a8ee-51d578a32fea"
}
},
{
"model": "workflows.abstractinput",
"fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Random forest - random state",
"short_name": "rds",
"default": "None",
"description": "Define a random state for predict phase.",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "seed",
"parameter": true,
"order": 3,
"uid": "8ec2b906-2b9e-4cda-9455-09ccd7d134fb"
}
},
{
"model": "workflows.abstractinput",
"fields": {
......@@ -96,7 +79,7 @@
"model": "workflows.abstractinput",
"fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Weighted forest - similarity coeff",
"name": "Distributed Weighted forest - similarity coeff",
"short_name": "coe",
"default": "0.5",
"description": "Percentage of most similar treees to include in prediction (0 - 1)",
......@@ -105,7 +88,7 @@
"parameter_type": "text",
"variable": "coeff",
"parameter": true,
"order": 4,
"order": 3,
"uid": "d979ff23-eb11-40cf-9d81-2a71ddf5d790"
}
},
......
workflows/big_data/visualization_views.py
View file @ c58fe07d
from django.shortcuts import render
def measure_distribution(request,input_dict,output_dict,widget):
def measure_distribution(request, input_dict, output_dict, widget):
from disco.core import result_iterator
from discomll.utils import distribution
......@@ -11,9 +12,11 @@ def measure_distribution(request,input_dict,output_dict,widget):
dist += str(k.split("/")[-1][:-2]) + " " + str(v) + "\n"
input_dict["string"] = dist
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_ca(request,input_dict,output_dict,widget):
def bigdata_ca(request, input_dict, output_dict, widget):
from discomll.utils import accuracy
import os.path
from mothra.settings import MEDIA_ROOT
......@@ -21,70 +24,72 @@ def bigdata_ca(request,input_dict,output_dict,widget):
folder = 'discomll_measures'
tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination)
if input_dict["dataset"].params["id_index"] == -1:
input_dict["string"] = "ID index should be defined."
elif not os.path.isfile(destination): #file doesnt exists
results = accuracy.measure(test_data = input_dict["dataset"],
predictions = input_dict["predictions"],
measure = "ca")
string = "Classification Accuracy \n"
for k, v in results:
string += str(v) + "\n"
elif not os.path.isfile(destination): # file doesnt exists
measure, acc = accuracy.measure(test_data=input_dict["dataset"],
predictions=input_dict["predictions"],
measure="ca")
string = "Classification Accuracy \n"
score = str(measure) + " " + str(acc) + "\n"
string += score
input_dict["string"] = string
f = open(destination,'w')
f.write(str(v))
f = open(destination, 'w')
f.write(score)
f.close()
else:
string = "Classification Accuracy \n"
f = open(destination,'r')
#ca results are cached
string = "Classification Accuracy \n"
f = open(destination, 'r')
input_dict["string"] = string + str(f.readlines()[0])
f.close()
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
def bigdata_mse(request,input_dict,output_dict,widget):
return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_mse(request, input_dict, output_dict, widget):
from discomll.utils import accuracy
from disco.core import result_iterator
import os.path
from mothra.settings import MEDIA_ROOT
from workflows.helpers import ensure_dir
folder = 'discomll_measures'
tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination)
if input_dict["dataset"].params["id_index"] == -1:
input_dict["string"] = "ID index should be defined."
elif not os.path.isfile(destination): #file doesnt exists
results = accuracy.measure(test_data = input_dict["dataset"],
predictions = input_dict["predictions"],
measure = "mse")
elif not os.path.isfile(destination): # file doesnt exists
measure, acc = accuracy.measure(test_data=input_dict["dataset"],
predictions=input_dict["predictions"],
measure="mse")
string = "Mean squared error\n"
for k, v in results:
string += str(v) + "\n"
score = str(measure) + " " + str(acc) + "\n"
string += score
input_dict["string"] = string
f = open(destination,'w')
f.write(str(v))
f = open(destination, 'w')
f.write(score)
f.close()
else:
string = "Mean squared error\n"
f = open(destination,'r')
f = open(destination, 'r')
input_dict["string"] = string + str(f.readlines()[0])
f.close()
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def results_to_file(request,input_dict,output_dict,widget):
def results_to_file(request, input_dict, output_dict, widget):
from disco.core import result_iterator
import os.path
from mothra.settings import MEDIA_ROOT
......@@ -93,14 +98,14 @@ def results_to_file(request,input_dict,output_dict,widget):
tag = input_dict["string"]
folder = 'discomll_results'
add = "add" if input_dict["add_params"] == "true" else ""
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+add+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + add + '.txt'
ensure_dir(destination)
if not os.path.isfile(destination): #file doesnt exists
f = open(destination,'w')
if not os.path.isfile(destination): # file doesnt exists
f = open(destination, 'w')
if input_dict["add_params"] == "true":
for k, v in result_iterator(tag):
f.writelines(str(k) + " " + str(v) + "\n")
......@@ -108,12 +113,14 @@ def results_to_file(request,input_dict,output_dict,widget):
for k, v in result_iterator(tag):
f.writelines(str(k) + " " + str(v[0]) + "\n")
f.close()
filename = folder+"/"+tag[0][6:]+add+'.txt'
filename = folder + "/" + tag[0][6:] + add + '.txt'
output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
return render(request, 'visualizations/string_to_file.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def model_view(request,input_dict,output_dict,widget):
def model_view(request, input_dict, output_dict, widget):
from discomll.utils import model_view
import os.path
from mothra.settings import MEDIA_ROOT
......@@ -122,23 +129,18 @@ def model_view(request,input_dict,output_dict,widget):
folder = 'discomll_models'
tag_name = input_dict["fitmodel_url"]
tag = input_dict["fitmodel_url"].values()[0]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination)
if not os.path.isfile(destination): #file doesnt exists
if not os.path.isfile(destination): # file doesnt exists
model = model_view.output_model(tag_name)
f = open(destination,'w')
f = open(destination, 'w')
f.write(model)
f.close()
filename = folder+"/"+tag[0][6:]+'.txt'
output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
filename = folder + "/" + tag[0][6:] + '.txt'
output_dict['filename'] = filename
return render(request, 'visualizations/string_to_file.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment