Skip to content

GitLab

Commit c58fe07d authored by Janez's avatar Janez
Browse files

Merge branch 'totrtale_multiproc' of /home/git/repositories/kt/mothra

parents 1ff85c2e 9b785640
Hide whitespace changes
Inline Side-by-side
requirements.txt
View file @ c58fe07d
......@@ -17,4 +17,3 @@ liac-arff==2.0.1
networkx==1.9.1
djangorestframework==3.0.3
django-filter==0.9.1
discomll
workflows/big_data/library.py
View file @ c58fe07d
def file_url(input_dict):
from discomll import dataset
if input_dict["range"] == "true":
urls = [url.strip() for url in input_dict["url"].split("\n") if url != ""]
else:
urls = [[url.strip()] for url in input_dict["url"].split("\n") if url != ""]
urls = [[url.strip()] for url in input_dict["url"].split("\n") if url != ""]
for url in urls:
if url[0].split("://")[0] == "https":
raise Exception("Dataset should be accessible over HTTP.")
del(input_dict["url"])
del (input_dict["url"])
X_indices_splited = input_dict["X_indices"].replace(" ","").split("-")
X_indices_splited = input_dict["X_indices"].replace(" ", "").split("-")
if len(X_indices_splited) == 2:
a, b = X_indices_splited
if not a.isdigit() or not b.isdigit():
raise Exception("Feature indices should be integers. Example: 1-10")
X_indices = range(int(a), int(b))
else:
X_indices = [int(v) for v in input_dict["X_indices"].replace(" ","").split(",") if v != ""]
del(input_dict["X_indices"])
X_indices = [int(v) for v in input_dict["X_indices"].replace(" ", "").split(",") if v != ""]
del (input_dict["X_indices"])
input_dict["data_type"] = "gzip" if input_dict["data_type"] == "true" else ""
......@@ -29,20 +27,20 @@ def file_url(input_dict):
elif input_dict["atr_meta"] == "discrete":
X_meta = ["d" for i in range(len(X_indices))]
else:
X_meta = input_dict["custom"]
X_meta = input_dict["custom"]
data = dataset.Data(data_tag = urls,
X_indices = X_indices,
X_meta = X_meta,
generate_urls = True if input_dict["range"] == "true" else False,
**input_dict)
data = dataset.Data(data_tag=urls,
X_indices=X_indices,
X_meta=X_meta,
generate_urls=True if input_dict["range"] == "true" else False,
**input_dict)
print data.params
return {"dataset" : data}
return {"dataset": data}
def big_data_apply_classifier(input_dict):
if "naivebayes_fitmodel" in input_dict["fitmodel_url"]:
return naivebayes_predict(input_dict)
elif "logreg_fitmodel" in input_dict["fitmodel_url"]:
......@@ -60,206 +58,224 @@ def big_data_apply_classifier(input_dict):
elif "linreg_fitmodel" in input_dict["fitmodel_url"]:
return linreg_predict(input_dict)
def lwlr_fit_predict(input_dict):
from discomll.regression import locally_weighted_linear_regression
predictions_url = locally_weighted_linear_regression.fit_predict(
fitting_data = input_dict["fitting_dataset"],
training_data = input_dict["training_dataset"],
tau = input_dict["tau"],
save_results = True)
fitting_data=input_dict["fitting_dataset"],
training_data=input_dict["training_dataset"],
tau=input_dict["tau"],
save_results=True)
return {"string": predictions_url}
def dt_fit(input_dict):
from discomll.ensemble import forest_distributed_decision_trees
random_state = None if input_dict["seed"] == "None" else int(input_dict["seed"])
fitmodel_url = forest_distributed_decision_trees.fit(input_dict["dataset"],
trees_per_chunk = input_dict["trees_per_subset"],
max_tree_nodes = input_dict["tree_nodes"],
min_samples_leaf = input_dict["min_samples_leaf"],
min_samples_split = input_dict["min_samples_split"],
class_majority = input_dict["majority"],
bootstrap = input_dict["bootstrap"] == "true",
measure = input_dict["measure"],
accuracy = input_dict["accuracy"],
separate_max = input_dict["separate_max"],
random_state = random_state,
save_results = True)
return {"fitmodel_url" : fitmodel_url}
trees_per_chunk=input_dict["trees_per_subset"],
max_tree_nodes=input_dict["tree_nodes"],
min_samples_leaf=input_dict["min_samples_leaf"],
min_samples_split=input_dict["min_samples_split"],
class_majority=input_dict["majority"],
bootstrap=input_dict["bootstrap"] == "true",
measure=input_dict["measure"],
accuracy=input_dict["accuracy"],
separate_max=input_dict["separate_max"] == "true",
random_state=random_state,
save_results=True)
return {"fitmodel_url": fitmodel_url}
def dt_predict(input_dict):
from discomll.ensemble import forest_distributed_decision_trees
predictions_url = forest_distributed_decision_trees.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def rf_fit(input_dict):
from discomll.ensemble import distributed_random_forest
random_state = None if input_dict["seed"] == "None" else int(input_dict["seed"])
fitmodel_url = distributed_random_forest.fit(input_dict["dataset"],
trees_per_chunk = input_dict["trees_per_subset"],
max_tree_nodes = input_dict["tree_nodes"],
min_samples_leaf = input_dict["min_samples_leaf"],
min_samples_split = input_dict["min_samples_split"],
class_majority = input_dict["majority"],
measure = input_dict["measure"],
accuracy = input_dict["accuracy"],
separate_max = input_dict["separate_max"],
random_state = random_state,
save_results = True)
return {"fitmodel_url" : fitmodel_url}
trees_per_chunk=input_dict["trees_per_subset"],
max_tree_nodes=input_dict["tree_nodes"],
min_samples_leaf=input_dict["min_samples_leaf"],
min_samples_split=input_dict["min_samples_split"],
class_majority=input_dict["majority"],
measure=input_dict["measure"],
accuracy=input_dict["accuracy"],
separate_max=input_dict["separate_max"] == "true",
random_state=random_state,
save_results=True)
return {"fitmodel_url": fitmodel_url}
def rf_predict(input_dict):
from discomll.ensemble import distributed_random_forest
random_state = None if input_dict["seed"] == "None" else int(input_dict["seed"])
predictions_url = distributed_random_forest.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
random_state = random_state,
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def wrf_fit(input_dict):
from discomll.ensemble import distributed_weighted_forest_rand
random_state = None if input_dict["seed"] == "None" else int(input_dict["seed"])
fitmodel_url = distributed_weighted_forest_rand.fit(input_dict["dataset"],
trees_per_chunk = input_dict["trees_per_subset"],
max_tree_nodes = input_dict["tree_nodes"],
num_medoids = input_dict["num_medoids"],
min_samples_leaf = input_dict["min_samples_leaf"],
min_samples_split = input_dict["min_samples_split"],
class_majority = input_dict["majority"],
measure = input_dict["measure"],
accuracy = input_dict["accuracy"],
separate_max = input_dict["separate_max"],
random_state = random_state,
save_results = True)
return {"fitmodel_url" : fitmodel_url}
trees_per_chunk=input_dict["trees_per_subset"],
max_tree_nodes=input_dict["tree_nodes"],
num_medoids=input_dict["num_medoids"],
min_samples_leaf=input_dict["min_samples_leaf"],
min_samples_split=input_dict["min_samples_split"],
class_majority=input_dict["majority"],
measure=input_dict["measure"],
accuracy=input_dict["accuracy"],
separate_max=input_dict["separate_max"] == "true",
random_state=random_state,
save_results=True)
return {"fitmodel_url": fitmodel_url}
def wrf_predict(input_dict):
from discomll.ensemble import distributed_weighted_forest_rand
predictions_url = distributed_weighted_forest_rand.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
coeff = input_dict["coeff"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
coeff=input_dict["coeff"],
save_results=True)
return {"string": predictions_url}
def linsvm_fit(input_dict):
from discomll.classification import linear_svm
fitmodel_url = linear_svm.fit(input_dict["dataset"],
nu = input_dict["nu"],
save_results = True)
return {"fitmodel_url" : fitmodel_url}
nu=input_dict["nu"],
save_results=True)
return {"fitmodel_url": fitmodel_url}
def linsvm_predict(input_dict):
from discomll.classification import linear_svm
predictions_url = linear_svm.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def linreg_fit(input_dict):
from discomll.regression import linear_regression
fitmodel_url = linear_regression.fit(input_dict["dataset"],
save_results = True)
return {"fitmodel_url" : fitmodel_url}
save_results=True)
return {"fitmodel_url": fitmodel_url}
def linreg_predict(input_dict):
from discomll.regression import linear_regression
predictions_url = linear_regression.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def kmeans_fit(input_dict):
from discomll.clustering import kmeans
random_state = None if input_dict["seed"] == "None" else int(input_dict["seed"])
fitmodel_url = kmeans.fit(input_dict["dataset"],
n_clusters = input_dict["clusters"],
max_iterations = input_dict["itr"],
random_state = random_state,
save_results = True)
n_clusters=input_dict["clusters"],
max_iterations=input_dict["itr"],
random_state=random_state,
save_results=True)
return {"fitmodel_url": fitmodel_url}
return {"fitmodel_url" : fitmodel_url}
def kmeans_predict(input_dict):
from discomll.clustering import kmeans
predictions_url = kmeans.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def logreg_fit(input_dict):
from discomll.classification import logistic_regression
fitmodel_url = logistic_regression.fit(input_dict["dataset"],
alpha = input_dict["alpha"],
max_iterations = input_dict["itr"],
save_results = True)
return {"fitmodel_url" : fitmodel_url}
alpha=input_dict["alpha"],
max_iterations=input_dict["itr"],
save_results=True)
return {"fitmodel_url": fitmodel_url}
def logreg_predict(input_dict):
from discomll.classification import logistic_regression
predictions_url = logistic_regression.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
save_results = True)
fitmodel_url=input_dict["fitmodel_url"],
save_results=True)
return {"string": predictions_url}
def naivebayes_fit(input_dict):
from discomll.classification import naivebayes
fitmodel_url = naivebayes.fit(input_dict["dataset"], save_results = True)
return {"fitmodel_url" : fitmodel_url}
fitmodel_url = naivebayes.fit(input_dict["dataset"], save_results=True)
return {"fitmodel_url": fitmodel_url}
def naivebayes_predict(input_dict):
from discomll.classification import naivebayes
m = 1 if input_dict["m"] == "" else input_dict["m"]
predictions_url = naivebayes.predict(input_dict["dataset"],
fitmodel_url = input_dict["fitmodel_url"],
m = input_dict["m"],
save_results = True )
fitmodel_url=input_dict["fitmodel_url"],
m=input_dict["m"],
save_results=True)
return {"string": predictions_url}
def results_to_file(input_dict):
#implementation is in visualization_views.py
return {}
# implementation is in visualization_views.py
return {}
def measure_distribution(input_dict):
#implementation is in visualization_views.py
# implementation is in visualization_views.py
return {}
def model_view(input_dict):
#implementation is in visualization_views.py
# implementation is in visualization_views.py
return {}
def bigdata_ca(input_dict):
#implementation is in visualization_views.py
# implementation is in visualization_views.py
return {}
def bigdata_mse(input_dict):
#implementation is in visualization_views.py
# implementation is in visualization_views.py
return {}
workflows/big_data/package_data/widgets/4f2ce923-62e6-4be1-a394-72ac52988386.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "",
"uid": "4f2ce923-62e6-4be1-a394-72ac52988386",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Distributed Random Forest"
}
......@@ -171,7 +171,7 @@
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "leaf_min_inst",
"variable": "min_samples_split",
"parameter": true,
"order": 4,
"uid": "a2f366a9-af74-4b3f-90ed-33c1fcad4c3a"
......
workflows/big_data/package_data/widgets/72a39fab-5433-493f-ae22-12a264075356.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "1be7b5eb-c1b2-485a-8dbe-56abce63fc73",
"treeview_image": "",
"uid": "72a39fab-5433-493f-ae22-12a264075356",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Distributed Weighted Forest"
}
......@@ -205,7 +205,7 @@
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "leaf_min_inst",
"variable": "min_samples_split",
"parameter": true,
"order": 5,
"uid": "ac032f38-f4a4-44ea-8c02-96506d4f8e86"
......
workflows/big_data/package_data/widgets/d9de579e-fdd9-47da-948e-69183f24340f.json
View file @ c58fe07d
......@@ -5,7 +5,7 @@
"category": "08c90036-fd13-4ff3-ac20-01982262a782",
"treeview_image": "",
"uid": "d9de579e-fdd9-47da-948e-69183f24340f",
"is_streaming": false,
"windows_queue": false,
"package": "big_data",
"interaction_view": "",
"has_progress_bar": false,
......@@ -19,7 +19,7 @@
"wsdl_method": "",
"wsdl": "",
"interactive": false,
"windows_queue": false,
"is_streaming": false,
"order": 1,
"name": "Apply Classifier"
}
......@@ -37,7 +37,7 @@
"parameter_type": null,
"variable": "dataset",
"parameter": false,
"order": 5,
"order": 4,
"uid": "0c174cfd-0d61-44ff-8ce2-9c32f8791293"
}
},
......@@ -58,23 +58,6 @@
"uid": "46e9a02d-a613-4a01-a8ee-51d578a32fea"
}
},
{
"model": "workflows.abstractinput",
"fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Random forest - random state",
"short_name": "rds",
"default": "None",
"description": "Define a random state for predict phase.",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "seed",
"parameter": true,
"order": 3,
"uid": "8ec2b906-2b9e-4cda-9455-09ccd7d134fb"
}
},
{
"model": "workflows.abstractinput",
"fields": {
......@@ -96,7 +79,7 @@
"model": "workflows.abstractinput",
"fields": {
"widget": "d9de579e-fdd9-47da-948e-69183f24340f",
"name": "Weighted forest - similarity coeff",
"name": "Distributed Weighted forest - similarity coeff",
"short_name": "coe",
"default": "0.5",
"description": "Percentage of most similar treees to include in prediction (0 - 1)",
......@@ -105,7 +88,7 @@
"parameter_type": "text",
"variable": "coeff",
"parameter": true,
"order": 4,
"order": 3,
"uid": "d979ff23-eb11-40cf-9d81-2a71ddf5d790"
}
},
......
workflows/big_data/visualization_views.py
View file @ c58fe07d
from django.shortcuts import render
def measure_distribution(request,input_dict,output_dict,widget):
def measure_distribution(request, input_dict, output_dict, widget):
from disco.core import result_iterator
from discomll.utils import distribution
......@@ -11,9 +12,11 @@ def measure_distribution(request,input_dict,output_dict,widget):
dist += str(k.split("/")[-1][:-2]) + " " + str(v) + "\n"
input_dict["string"] = dist
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_ca(request,input_dict,output_dict,widget):
def bigdata_ca(request, input_dict, output_dict, widget):
from discomll.utils import accuracy
import os.path
from mothra.settings import MEDIA_ROOT
......@@ -21,70 +24,72 @@ def bigdata_ca(request,input_dict,output_dict,widget):
folder = 'discomll_measures'
tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination)
if input_dict["dataset"].params["id_index"] == -1:
input_dict["string"] = "ID index should be defined."
elif not os.path.isfile(destination): #file doesnt exists
results = accuracy.measure(test_data = input_dict["dataset"],
predictions = input_dict["predictions"],
measure = "ca")
string = "Classification Accuracy \n"
for k, v in results:
string += str(v) + "\n"
elif not os.path.isfile(destination): # file doesnt exists
measure, acc = accuracy.measure(test_data=input_dict["dataset"],
predictions=input_dict["predictions"],
measure="ca")
string = "Classification Accuracy \n"
score = str(measure) + " " + str(acc) + "\n"
string += score
input_dict["string"] = string
f = open(destination,'w')
f.write(str(v))
f = open(destination, 'w')
f.write(score)
f.close()
else:
string = "Classification Accuracy \n"
f = open(destination,'r')
#ca results are cached
string = "Classification Accuracy \n"
f = open(destination, 'r')
input_dict["string"] = string + str(f.readlines()[0])
f.close()
return render(request, 'visualizations/display_string.html',{'widget':widget,'input_dict':input_dict,'output_dict':output_dict})
def bigdata_mse(request,input_dict,output_dict,widget):
return render(request, 'visualizations/display_string.html',
{'widget': widget, 'input_dict': input_dict, 'output_dict': output_dict})
def bigdata_mse(request, input_dict, output_dict, widget):
from discomll.utils import accuracy
from disco.core import result_iterator
import os.path
from mothra.settings import MEDIA_ROOT
from workflows.helpers import ensure_dir
folder = 'discomll_measures'
tag = input_dict["predictions"]
destination = MEDIA_ROOT+'/'+folder+"/"+tag[0][6:]+'.txt'
destination = MEDIA_ROOT + '/' + folder + "/" + tag[0][6:] + '.txt'
ensure_dir(destination)