Skip to content

GitLab

C
clowdflows
Commit 9bb16e35 authored by Anze Vavpetic's avatar Anze Vavpetic
Browse files
Options

Merge branch 'mysql' into dev

parents d0746d3a 5abf512e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 939 additions and 116 deletions
workflows/ilp/db/package_data.json
View file @ 9bb16e35
This diff is collapsed.
workflows/mysql/context.py
View file @ 9bb16e35
......@@ -11,6 +11,14 @@ class DBConnection:
self.password = password
self.host = host
self.database = database
self.check_connection()
def check_connection(self):
try:
con = sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
con.close()
except Exception,e:
raise Exception('Problem connecting to the database. Please re-check your credentials.')
def connect(self):
return sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
......@@ -44,13 +52,14 @@ class DBContext:
for table, cols in self.cols.items():
self.col_vals[table] = {}
for col in cols:
cursor.execute("SELECT DISTINCT %s FROM %s" % (col, table))
cursor.execute("SELECT DISTINCT `%s` FROM `%s` LIMIT 51" % (col, table))
self.col_vals[table][col] = [val for (val,) in cursor]
print self.col_vals
self.connected = {}
cursor.execute(
"SELECT table_name, column_name, referenced_table_name, referenced_column_name \
FROM information_schema.KEY_COLUMN_USAGE \
WHERE referenced_table_name IS NOT NULL")
WHERE referenced_table_name IS NOT NULL AND table_schema='%s'" % connection.database)
self.fkeys = defaultdict(set)
for (table, col, ref_table, ref_col) in cursor:
self.connected[(table, ref_table)] = (col, ref_col)
......@@ -60,12 +69,12 @@ class DBContext:
cursor.execute(
"SELECT table_name, column_name \
FROM information_schema.KEY_COLUMN_USAGE \
WHERE constraint_name='PRIMARY' and table_schema='%s'" % connection.database)
WHERE constraint_name='PRIMARY' AND table_schema='%s'" % connection.database)
for (table, pk) in cursor:
self.pkeys[table] = pk
self.target_table = self.tables[0]
self.target_att = None
self.target_att_val = None
#self.target_att_val = None
con.close()
def update(self, postdata):
......@@ -75,7 +84,7 @@ class DBContext:
widget_id = postdata.get('widget_id')[0]
self.target_table = postdata.get('target_table%s' % widget_id)[0]
self.target_att = postdata.get('target_att%s' % widget_id)[0]
self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0]
#self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0]
self.tables = postdata.get('tables%s' % widget_id, [])
if self.target_table not in self.tables:
raise Exception('The selected target table "%s" is not among the selected tables.' % self.target_table)
......@@ -92,6 +101,30 @@ class DBContext:
if table == self.target_table and self.target_att not in self.cols[table]:
raise Exception('The selected target attribute ("%s") is not among the columns selected for the target table ("%s").' % (self.target_att, self.target_table))
def fmt_cols(self, cols):
return ','.join(["`%s`" % col for col in cols])
def rows(self, table, cols):
con = self.connection.connect()
cursor = con.cursor()
cursor.execute("SELECT %s FROM %s" % (self.fmt_cols(cols), table))
con.close()
return [cols for cols in cursor]
def fetch_types(self, table, cols):
'''
Returns a dictionary of field types for the given table and columns.
'''
con = self.connection.connect()
cursor = con.cursor()
cursor.execute("SELECT %s FROM `%s` LIMIT 1" % (self.fmt_cols(cols), table))
cursor.fetchall()
types = {}
for desc in cursor.description:
types[desc[0]] = sql.FieldType.get_info(desc[1])
con.close()
return types
def __repr__(self):
return str((self.target_table, self.target_att, self.tables, self.cols, self.connected))
workflows/mysql/converters.py
View file @ 9bb16e35
......@@ -16,23 +16,6 @@ class Converter:
def __del__(self):
self.connection.close()
def rows(self, table, cols):
self.cursor.execute("SELECT %s FROM %s" % (','.join(cols), table))
return [cols for cols in self.cursor]
def fetch_types(self, table, cols):
'''
Returns a dictionary of field types for the given table and columns.
'''
from mysql.connector import FieldType
c = self.cursor
c.execute('SELECT %s FROM %s LIMIT 1' % (','.join(cols), table))
c.fetchall()
types = {}
for desc in self.cursor.description:
types[desc[0]] = FieldType.get_info(desc[1])
return types
class ILP_Converter(Converter):
'''
Base class for converting between a given database context (selected tables, columns, etc)
......@@ -89,8 +72,8 @@ class RSD_Converter(ILP_Converter):
'''
def all_examples(self):
target = self.db.target_table
examples = self.rows(target, [self.db.target_att, self.db.pkeys[target]])
return '\n'.join(['%s(%s, %s).' % (target, cls, pk) for cls, pk in examples])
examples = self.db.rows(target, [self.db.target_att, self.db.pkeys[target]])
return '\n'.join(["%s('%s', %s)." % (target, cls, pk) for cls, pk in examples])
def background_knowledge(self):
modeslist, getters = [self.mode(self.db.target_table, [('+', self.db.target_table)], head=True)], []
......@@ -114,13 +97,14 @@ class Aleph_Converter(ILP_Converter):
Converts the database context to Aleph inputs.
'''
def __init__(self, *args, **kwargs):
self.target_att_val = kwargs.pop('target_att_val')
ILP_Converter.__init__(self, *args, **kwargs)
self.__pos_examples, self.__neg_examples = None, None
def __examples(self):
if not (self.__pos_examples and self.__neg_examples):
target, att, target_val = self.db.target_table, self.db.target_att, self.db.target_att_val
rows = self.rows(target, [att, self.db.pkeys[target]])
target, att, target_val = self.db.target_table, self.db.target_att, self.target_att_val
rows = self.db.rows(target, [att, self.db.pkeys[target]])
pos_rows, neg_rows = [], []
for row in rows:
if row[0] == target_val:
......@@ -138,13 +122,13 @@ class Aleph_Converter(ILP_Converter):
return self.__examples()[1]
def background_knowledge(self):
modeslist, getters = [self.mode(self.db.target_att_val, [('+', self.db.target_table)], head=True)], []
modeslist, getters = [self.mode(self.target_att_val, [('+', self.db.target_table)], head=True)], []
determinations, types = [], []
for (table, ref_table) in self.db.connected.keys():
if ref_table == self.db.target_table:
continue # Skip backward connections
modeslist.append(self.mode('has_%s' % ref_table, [('+', table), ('-', ref_table)], recall='*'))
determinations.append(':- determination(%s/1, has_%s/2).' % (self.db.target_att_val, ref_table))
determinations.append(':- determination(%s/1, has_%s/2).' % (self.target_att_val, ref_table))
types.extend(self.concept_type_def(table))
types.extend(self.concept_type_def(ref_table))
getters.extend(self.connecting_clause(table, ref_table))
......@@ -154,7 +138,7 @@ class Aleph_Converter(ILP_Converter):
att in self.db.fkeys[table] or att == self.db.pkeys[table]:
continue
modeslist.append(self.mode('%s_%s' % (table, att), [('+', table), ('#', att)], recall='*'))
determinations.append(':- determination(%s/1, %s_%s/2).' % (self.db.target_att_val, table, att))
determinations.append(':- determination(%s/1, %s_%s/2).' % (self.target_att_val, table, att))
types.extend(self.constant_type_def(table, att))
getters.extend(self.attribute_clause(table, att))
local_copies = [self.local_copy(table) for table in self.db.tables]
......@@ -189,11 +173,12 @@ class Orange_Converter(Converter):
Converts the target table selected in the given context as an orange example table.
'''
continuous_types = ('FLOAT','DOUBLE','DECIMAL','NEWDECIMAL')
discrete_types = ('TINY','SHORT','LONG','LONGLONG','INT24','YEAR','VARCHAR','BIT','SET','VAR_STRING','STRING')
integer_types = ('TINY','SHORT','LONG','LONGLONG','INT24')
ordinal_types = ('YEAR','VARCHAR','SET','VAR_STRING','STRING','BIT')
def __init__(self, *args, **kwargs):
Converter.__init__(self, *args, **kwargs)
self.types = self.fetch_types(self.db.target_table, self.db.cols[self.db.target_table])
self.types = self.db.fetch_types(self.db.target_table, self.db.cols[self.db.target_table])
def target_table(self):
'''
......@@ -206,8 +191,8 @@ class Orange_Converter(Converter):
attributes, metas, classVar = [], [], None
for col in cols:
att_type = self.orng_type(col)
att_vals = self.db.col_vals[table][col]
if att_type == 'd':
att_vals = self.db.col_vals[table][col]
att_var = orange.EnumVariable(str(col), values=[str(val) for val in att_vals])
elif att_type == 'c':
att_var = orange.FloatVariable(str(col))
......@@ -226,7 +211,7 @@ class Orange_Converter(Converter):
for meta in metas:
domain.addmeta(orange.newmetaid(), meta)
dataset = orange.ExampleTable(domain)
for row in self.rows(table, cols):
for row in self.db.rows(table, cols):
example = orange.Example(domain)
for col, val in zip(cols, row):
example[str(col)] = str(val)
......@@ -239,9 +224,9 @@ class Orange_Converter(Converter):
'''
mysql_type = self.types[col]
n_vals = len(self.db.col_vals[self.db.target_table][col])
if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.discrete_types):
if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.integer_types):
return 'c'
elif mysql_type in Orange_Converter.discrete_types:
elif mysql_type in Orange_Converter.ordinal_types:
return 'd'
else:
return 'string'
......@@ -252,14 +237,13 @@ if __name__ == '__main__':
context = DBContext(DBConnection('root','','localhost','test'))
context.target_table = 'trains'
context.target_att = 'direction'
context.target_att_val = 'east'
# rsd = RSD_Converter(context)
# ex, bk = rsd.all_examples(), rsd.background_knowledge()
rsd = RSD_Converter(context)
ex, bk = rsd.all_examples(), rsd.background_knowledge()
# aleph = Aleph_Converter(context)
# print aleph.positive_examples()
# print aleph.negative_examples()
# print aleph.background_knowledge()
aleph = Aleph_Converter(context, target_att_val='east')
print aleph.positive_examples()
print aleph.negative_examples()
print aleph.background_knowledge()
orange = Orange_Converter(context)
orange.target_table()
\ No newline at end of file
workflows/mysql/db/package_data.json
View file @ 9bb16e35
[
{
"pk": 75,
"pk": 39,
"model": "workflows.category",
"fields": {
"uid": "b66b71b1-99d6-4efc-b264-a914a3e42911",
......@@ -12,10 +12,10 @@
}
},
{
"pk": 309,
"pk": 190,
"model": "workflows.abstractwidget",
"fields": {
"category": 75,
"category": 39,
"treeview_image": "",
"name": "Database Context",
"is_streaming": false,
......@@ -23,7 +23,7 @@
"interaction_view": "mysql_db_context",
"image": "",
"package": "mysql",
"static_image": "",
"static_image": "mysql.png",
"post_interact_action": "mysql_db_context_finished",
"user": null,
"visualization_view": "",
......@@ -37,10 +37,10 @@
}
},
{
"pk": 1036,
"pk": 443,
"model": "workflows.abstractinput",
"fields": {
"widget": 309,
"widget": 190,
"name": "connection",
"short_name": "con",
"uid": "6c23b4a2-a18e-498e-a22c-4de86932da3e",
......@@ -55,10 +55,10 @@
}
},
{
"pk": 368,
"pk": 216,
"model": "workflows.abstractoutput",
"fields": {
"widget": 309,
"widget": 190,
"name": "context",
"short_name": "cxt",
"variable": "context",
......@@ -68,10 +68,10 @@
}
},
{
"pk": 311,
"pk": 191,
"model": "workflows.abstractwidget",
"fields": {
"category": 75,
"category": 39,
"treeview_image": "",
"name": "Database To Aleph",
"is_streaming": false,
......@@ -79,7 +79,7 @@
"interaction_view": "",
"image": "",
"package": "mysql",
"static_image": "",
"static_image": "mysql.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",
......@@ -93,10 +93,28 @@
}
},
{
"pk": 1052,
"pk": 451,
"model": "workflows.abstractinput",
"fields": {
"widget": 311,
"widget": 191,
"name": "Target attribute value",
"short_name": "tgt",
"uid": "5b7354f1-35d8-41c8-a0ae-c3848855f29d",
"default": "",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "target_att_val",
"parameter": true,
"order": 1,
"description": "Target attribute value to be used as the positive class"
}
},
{
"pk": 444,
"model": "workflows.abstractinput",
"fields": {
"widget": 191,
"name": "context",
"short_name": "cxt",
"uid": "474e9673-9c55-48c5-bae8-a7b986aa0287",
......@@ -106,54 +124,54 @@
"parameter_type": null,
"variable": "context",
"parameter": false,
"order": 1,
"order": 2,
"description": "Database context object"
}
},
{
"pk": 371,
"pk": 217,
"model": "workflows.abstractoutput",
"fields": {
"widget": 311,
"widget": 191,
"name": "pos_examples",
"short_name": "pex",
"variable": "pos_examples",
"uid": "48184527-16a5-446c-9181-48a6558430c2",
"order": 1,
"order": 2,
"description": "positive examples file"
}
},
{
"pk": 372,
"pk": 218,
"model": "workflows.abstractoutput",
"fields": {
"widget": 311,
"widget": 191,
"name": "neg_examples",
"short_name": "nex",
"variable": "neg_examples",
"uid": "2c623064-95cd-48ac-8bff-617edf2b9468",
"order": 1,
"order": 3,
"description": "negative examples file"
}
},
{
"pk": 373,
"pk": 219,
"model": "workflows.abstractoutput",
"fields": {
"widget": 311,
"widget": 191,
"name": "bk",
"short_name": "bk",
"variable": "bk",
"uid": "5e61e44e-cc52-4471-9a41-3afa52f58ab1",
"order": 1,
"order": 4,
"description": "background knowledge"
}
},
{
"pk": 414,
"pk": 192,
"model": "workflows.abstractwidget",
"fields": {
"category": 75,
"category": 39,
"treeview_image": "",
"name": "Database To Orange Table",
"is_streaming": false,
......@@ -161,7 +179,7 @@
"interaction_view": "mysql_orange_converter",
"image": "",
"package": "mysql",
"static_image": "",
"static_image": "mysql.png",
"post_interact_action": "mysql_orange_converter_finished",
"user": null,
"visualization_view": "",
......@@ -175,10 +193,10 @@
}
},
{
"pk": 1241,
"pk": 445,
"model": "workflows.abstractinput",
"fields": {
"widget": 414,
"widget": 192,
"name": "context",
"short_name": "cxt",
"uid": "1f9b5ccf-65c3-4ccc-818e-afb3a6ffee20",
......@@ -193,10 +211,10 @@
}
},
{
"pk": 489,
"pk": 220,
"model": "workflows.abstractoutput",
"fields": {
"widget": 414,
"widget": 192,
"name": "Data table",
"short_name": "odt",
"variable": "dataset",
......@@ -206,10 +224,10 @@
}
},
{
"pk": 310,
"pk": 193,
"model": "workflows.abstractwidget",
"fields": {
"category": 75,
"category": 39,
"treeview_image": "",
"name": "Database To RSD",
"is_streaming": false,
......@@ -217,7 +235,7 @@
"interaction_view": "",
"image": "",
"package": "mysql",
"static_image": "",
"static_image": "mysql.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",
......@@ -231,10 +249,10 @@
}
},
{
"pk": 1037,
"pk": 446,
"model": "workflows.abstractinput",
"fields": {
"widget": 310,
"widget": 193,
"name": "context",
"short_name": "cxt",
"uid": "4f1397a8-4e72-4b34-b31d-d09bf9a7e7d9",
......@@ -249,10 +267,10 @@
}
},
{
"pk": 369,
"pk": 221,
"model": "workflows.abstractoutput",
"fields": {
"widget": 310,
"widget": 193,
"name": "examples",
"short_name": "exm",
"variable": "examples",
......@@ -262,10 +280,10 @@
}
},
{
"pk": 370,
"pk": 222,
"model": "workflows.abstractoutput",
"fields": {
"widget": 310,
"widget": 193,
"name": "bk",
"short_name": "bk",
"variable": "bk",
......@@ -275,10 +293,10 @@
}
},
{
"pk": 308,
"pk": 194,
"model": "workflows.abstractwidget",
"fields": {
"category": 75,
"category": 39,
"treeview_image": "",
"name": "MySQL Connect",
"is_streaming": false,
......@@ -286,7 +304,7 @@
"interaction_view": "",
"image": "",
"package": "mysql",
"static_image": "",
"static_image": "mysql.png",
"post_interact_action": "",
"user": null,
"visualization_view": "",
......@@ -300,10 +318,10 @@
}
},
{
"pk": 1032,
"pk": 447,
"model": "workflows.abstractinput",
"fields": {
"widget": 308,
"widget": 194,
"name": "user",
"short_name": "usr",
"uid": "0f36f10b-066f-4ad3-9fa8-39205440076a",
......@@ -318,17 +336,17 @@
}
},
{
"pk": 1033,
"pk": 448,
"model": "workflows.abstractinput",
"fields": {
"widget": 308,
"widget": 194,
"name": "password",
"short_name": "pwd",
"uid": "8ead911a-3d1b-4e50-b56a-64873b85d3cf",
"default": "",
"required": false,
"multi": false,
"parameter_type": "text",
"parameter_type": "password",
"variable": "password",
"parameter": true,
"order": 2,
......@@ -336,10 +354,10 @@
}
},
{
"pk": 1034,
"pk": 449,
"model": "workflows.abstractinput",
"fields": {
"widget": 308,
"widget": 194,
"name": "host",
"short_name": "hst",
"uid": "9812cc11-b21e-4444-8dfe-bc0c68fe79e8",
......@@ -354,10 +372,10 @@
}
},
{
"pk": 1035,
"pk": 450,
"model": "workflows.abstractinput",
"fields": {
"widget": 308,
"widget": 194,
"name": "database",
"short_name": "db",
"uid": "d132adce-c3ec-411a-8ee7-906be4a99f61",
......@@ -372,10 +390,10 @@
}
},
{
"pk": 367,
"pk": 223,
"model": "workflows.abstractoutput",
"fields": {
"widget": 308,
"widget": 194,
"name": "connection",
"short_name": "con",
"variable": "connection",
......
workflows/mysql/library.py
View file @ 9bb16e35
......@@ -29,7 +29,7 @@ def mysql_rsd_converter(input_dict):
return {'examples' : rsd.all_examples(), 'bk' : rsd.background_knowledge()}
def mysql_aleph_converter(input_dict):
aleph = Aleph_Converter(input_dict['context'])
aleph = Aleph_Converter(input_dict['context'], target_att_val=input_dict['target_att_val'])
return {'pos_examples' : aleph.positive_examples(), 'neg_examples' : aleph.negative_examples(), 'bk' : aleph.background_knowledge()}
def mysql_query_to_odt(input_dict):
......
workflows/mysql/templates/interactions/db_context.html
View file @ 9bb16e35
......@@ -13,12 +13,6 @@
<option value="{{col}}" {% if forloop.first %}selected="selected"{% endif %}>{{col}}</option>
{% endfor %}
</select>
<label>Target attribute value:</label>
<select name="target_att_val{{widget.pk}}">
{% for val in target_col_vals %}
<option value="{{val}}" {% if forloop.first %}selected="selected"{% endif %}>{{val}}</option>
{% endfor %}
</select>
<label>Select tables to be used:</label>
<select multiple name="tables{{widget.pk}}">
{% for table in context.tables %}
......@@ -49,7 +43,8 @@
<script type="text/javascript">
// Columns data.
var cols = {{cols|safe}};
var col_vals = {{col_vals|safe}};
//var col_vals = {{col_vals|safe}};
console.log(col_vals);
$('select[name="target_table{{widget.pk}}"]').change(function () {
var selected_table = $(this).first('option:selected').val();
var available_cols = cols[selected_table];
......@@ -61,18 +56,6 @@
att_select.append($('<option></option>').attr('value', col).text(col));
}
});
$('select[name="target_att{{widget.pk}}"]').change(function () {
var selected_table = $('select[name="target_table{{widget.pk}}"]').first('option:selected').val();
var selected_att = $(this).first('option:selected').val();
var available_vals = col_vals[selected_table][selected_att];
// Change the list of options as the target attribute
var val_select = $('select[name="target_att_val{{widget.pk}}"]');
val_select.empty();
for (idx in available_vals) {
var col = available_vals[idx];
val_select.append($('<option></option>').attr('value', col).text(col));
}