Commit 9bb16e35 authored by Anze Vavpetic's avatar Anze Vavpetic
Browse files

Merge branch 'mysql' into dev

parents d0746d3a 5abf512e
This diff is collapsed.
...@@ -11,6 +11,14 @@ class DBConnection: ...@@ -11,6 +11,14 @@ class DBConnection:
self.password = password self.password = password
self.host = host self.host = host
self.database = database self.database = database
self.check_connection()
def check_connection(self):
try:
con = sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
con.close()
except Exception,e:
raise Exception('Problem connecting to the database. Please re-check your credentials.')
def connect(self): def connect(self):
return sql.connect(user=self.user, password=self.password, host=self.host, database=self.database) return sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
...@@ -44,13 +52,14 @@ class DBContext: ...@@ -44,13 +52,14 @@ class DBContext:
for table, cols in self.cols.items(): for table, cols in self.cols.items():
self.col_vals[table] = {} self.col_vals[table] = {}
for col in cols: for col in cols:
cursor.execute("SELECT DISTINCT %s FROM %s" % (col, table)) cursor.execute("SELECT DISTINCT `%s` FROM `%s` LIMIT 51" % (col, table))
self.col_vals[table][col] = [val for (val,) in cursor] self.col_vals[table][col] = [val for (val,) in cursor]
print self.col_vals
self.connected = {} self.connected = {}
cursor.execute( cursor.execute(
"SELECT table_name, column_name, referenced_table_name, referenced_column_name \ "SELECT table_name, column_name, referenced_table_name, referenced_column_name \
FROM information_schema.KEY_COLUMN_USAGE \ FROM information_schema.KEY_COLUMN_USAGE \
WHERE referenced_table_name IS NOT NULL") WHERE referenced_table_name IS NOT NULL AND table_schema='%s'" % connection.database)
self.fkeys = defaultdict(set) self.fkeys = defaultdict(set)
for (table, col, ref_table, ref_col) in cursor: for (table, col, ref_table, ref_col) in cursor:
self.connected[(table, ref_table)] = (col, ref_col) self.connected[(table, ref_table)] = (col, ref_col)
...@@ -60,12 +69,12 @@ class DBContext: ...@@ -60,12 +69,12 @@ class DBContext:
cursor.execute( cursor.execute(
"SELECT table_name, column_name \ "SELECT table_name, column_name \
FROM information_schema.KEY_COLUMN_USAGE \ FROM information_schema.KEY_COLUMN_USAGE \
WHERE constraint_name='PRIMARY' and table_schema='%s'" % connection.database) WHERE constraint_name='PRIMARY' AND table_schema='%s'" % connection.database)
for (table, pk) in cursor: for (table, pk) in cursor:
self.pkeys[table] = pk self.pkeys[table] = pk
self.target_table = self.tables[0] self.target_table = self.tables[0]
self.target_att = None self.target_att = None
self.target_att_val = None #self.target_att_val = None
con.close() con.close()
def update(self, postdata): def update(self, postdata):
...@@ -75,7 +84,7 @@ class DBContext: ...@@ -75,7 +84,7 @@ class DBContext:
widget_id = postdata.get('widget_id')[0] widget_id = postdata.get('widget_id')[0]
self.target_table = postdata.get('target_table%s' % widget_id)[0] self.target_table = postdata.get('target_table%s' % widget_id)[0]
self.target_att = postdata.get('target_att%s' % widget_id)[0] self.target_att = postdata.get('target_att%s' % widget_id)[0]
self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0] #self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0]
self.tables = postdata.get('tables%s' % widget_id, []) self.tables = postdata.get('tables%s' % widget_id, [])
if self.target_table not in self.tables: if self.target_table not in self.tables:
raise Exception('The selected target table "%s" is not among the selected tables.' % self.target_table) raise Exception('The selected target table "%s" is not among the selected tables.' % self.target_table)
...@@ -92,6 +101,30 @@ class DBContext: ...@@ -92,6 +101,30 @@ class DBContext:
if table == self.target_table and self.target_att not in self.cols[table]: if table == self.target_table and self.target_att not in self.cols[table]:
raise Exception('The selected target attribute ("%s") is not among the columns selected for the target table ("%s").' % (self.target_att, self.target_table)) raise Exception('The selected target attribute ("%s") is not among the columns selected for the target table ("%s").' % (self.target_att, self.target_table))
def fmt_cols(self, cols):
return ','.join(["`%s`" % col for col in cols])
def rows(self, table, cols):
con = self.connection.connect()
cursor = con.cursor()
cursor.execute("SELECT %s FROM %s" % (self.fmt_cols(cols), table))
con.close()
return [cols for cols in cursor]
def fetch_types(self, table, cols):
'''
Returns a dictionary of field types for the given table and columns.
'''
con = self.connection.connect()
cursor = con.cursor()
cursor.execute("SELECT %s FROM `%s` LIMIT 1" % (self.fmt_cols(cols), table))
cursor.fetchall()
types = {}
for desc in cursor.description:
types[desc[0]] = sql.FieldType.get_info(desc[1])
con.close()
return types
def __repr__(self): def __repr__(self):
return str((self.target_table, self.target_att, self.tables, self.cols, self.connected)) return str((self.target_table, self.target_att, self.tables, self.cols, self.connected))
...@@ -16,23 +16,6 @@ class Converter: ...@@ -16,23 +16,6 @@ class Converter:
def __del__(self): def __del__(self):
self.connection.close() self.connection.close()
def rows(self, table, cols):
self.cursor.execute("SELECT %s FROM %s" % (','.join(cols), table))
return [cols for cols in self.cursor]
def fetch_types(self, table, cols):
'''
Returns a dictionary of field types for the given table and columns.
'''
from mysql.connector import FieldType
c = self.cursor
c.execute('SELECT %s FROM %s LIMIT 1' % (','.join(cols), table))
c.fetchall()
types = {}
for desc in self.cursor.description:
types[desc[0]] = FieldType.get_info(desc[1])
return types
class ILP_Converter(Converter): class ILP_Converter(Converter):
''' '''
Base class for converting between a given database context (selected tables, columns, etc) Base class for converting between a given database context (selected tables, columns, etc)
...@@ -89,8 +72,8 @@ class RSD_Converter(ILP_Converter): ...@@ -89,8 +72,8 @@ class RSD_Converter(ILP_Converter):
''' '''
def all_examples(self): def all_examples(self):
target = self.db.target_table target = self.db.target_table
examples = self.rows(target, [self.db.target_att, self.db.pkeys[target]]) examples = self.db.rows(target, [self.db.target_att, self.db.pkeys[target]])
return '\n'.join(['%s(%s, %s).' % (target, cls, pk) for cls, pk in examples]) return '\n'.join(["%s('%s', %s)." % (target, cls, pk) for cls, pk in examples])
def background_knowledge(self): def background_knowledge(self):
modeslist, getters = [self.mode(self.db.target_table, [('+', self.db.target_table)], head=True)], [] modeslist, getters = [self.mode(self.db.target_table, [('+', self.db.target_table)], head=True)], []
...@@ -114,13 +97,14 @@ class Aleph_Converter(ILP_Converter): ...@@ -114,13 +97,14 @@ class Aleph_Converter(ILP_Converter):
Converts the database context to Aleph inputs. Converts the database context to Aleph inputs.
''' '''
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.target_att_val = kwargs.pop('target_att_val')
ILP_Converter.__init__(self, *args, **kwargs) ILP_Converter.__init__(self, *args, **kwargs)
self.__pos_examples, self.__neg_examples = None, None self.__pos_examples, self.__neg_examples = None, None
def __examples(self): def __examples(self):
if not (self.__pos_examples and self.__neg_examples): if not (self.__pos_examples and self.__neg_examples):
target, att, target_val = self.db.target_table, self.db.target_att, self.db.target_att_val target, att, target_val = self.db.target_table, self.db.target_att, self.target_att_val
rows = self.rows(target, [att, self.db.pkeys[target]]) rows = self.db.rows(target, [att, self.db.pkeys[target]])
pos_rows, neg_rows = [], [] pos_rows, neg_rows = [], []
for row in rows: for row in rows:
if row[0] == target_val: if row[0] == target_val:
...@@ -138,13 +122,13 @@ class Aleph_Converter(ILP_Converter): ...@@ -138,13 +122,13 @@ class Aleph_Converter(ILP_Converter):
return self.__examples()[1] return self.__examples()[1]
def background_knowledge(self): def background_knowledge(self):
modeslist, getters = [self.mode(self.db.target_att_val, [('+', self.db.target_table)], head=True)], [] modeslist, getters = [self.mode(self.target_att_val, [('+', self.db.target_table)], head=True)], []
determinations, types = [], [] determinations, types = [], []
for (table, ref_table) in self.db.connected.keys(): for (table, ref_table) in self.db.connected.keys():
if ref_table == self.db.target_table: if ref_table == self.db.target_table:
continue # Skip backward connections continue # Skip backward connections
modeslist.append(self.mode('has_%s' % ref_table, [('+', table), ('-', ref_table)], recall='*')) modeslist.append(self.mode('has_%s' % ref_table, [('+', table), ('-', ref_table)], recall='*'))
determinations.append(':- determination(%s/1, has_%s/2).' % (self.db.target_att_val, ref_table)) determinations.append(':- determination(%s/1, has_%s/2).' % (self.target_att_val, ref_table))
types.extend(self.concept_type_def(table)) types.extend(self.concept_type_def(table))
types.extend(self.concept_type_def(ref_table)) types.extend(self.concept_type_def(ref_table))
getters.extend(self.connecting_clause(table, ref_table)) getters.extend(self.connecting_clause(table, ref_table))
...@@ -154,7 +138,7 @@ class Aleph_Converter(ILP_Converter): ...@@ -154,7 +138,7 @@ class Aleph_Converter(ILP_Converter):
att in self.db.fkeys[table] or att == self.db.pkeys[table]: att in self.db.fkeys[table] or att == self.db.pkeys[table]:
continue continue
modeslist.append(self.mode('%s_%s' % (table, att), [('+', table), ('#', att)], recall='*')) modeslist.append(self.mode('%s_%s' % (table, att), [('+', table), ('#', att)], recall='*'))
determinations.append(':- determination(%s/1, %s_%s/2).' % (self.db.target_att_val, table, att)) determinations.append(':- determination(%s/1, %s_%s/2).' % (self.target_att_val, table, att))
types.extend(self.constant_type_def(table, att)) types.extend(self.constant_type_def(table, att))
getters.extend(self.attribute_clause(table, att)) getters.extend(self.attribute_clause(table, att))
local_copies = [self.local_copy(table) for table in self.db.tables] local_copies = [self.local_copy(table) for table in self.db.tables]
...@@ -189,11 +173,12 @@ class Orange_Converter(Converter): ...@@ -189,11 +173,12 @@ class Orange_Converter(Converter):
Converts the target table selected in the given context as an orange example table. Converts the target table selected in the given context as an orange example table.
''' '''
continuous_types = ('FLOAT','DOUBLE','DECIMAL','NEWDECIMAL') continuous_types = ('FLOAT','DOUBLE','DECIMAL','NEWDECIMAL')
discrete_types = ('TINY','SHORT','LONG','LONGLONG','INT24','YEAR','VARCHAR','BIT','SET','VAR_STRING','STRING') integer_types = ('TINY','SHORT','LONG','LONGLONG','INT24')
ordinal_types = ('YEAR','VARCHAR','SET','VAR_STRING','STRING','BIT')
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
Converter.__init__(self, *args, **kwargs) Converter.__init__(self, *args, **kwargs)
self.types = self.fetch_types(self.db.target_table, self.db.cols[self.db.target_table]) self.types = self.db.fetch_types(self.db.target_table, self.db.cols[self.db.target_table])
def target_table(self): def target_table(self):
''' '''
...@@ -206,8 +191,8 @@ class Orange_Converter(Converter): ...@@ -206,8 +191,8 @@ class Orange_Converter(Converter):
attributes, metas, classVar = [], [], None attributes, metas, classVar = [], [], None
for col in cols: for col in cols:
att_type = self.orng_type(col) att_type = self.orng_type(col)
att_vals = self.db.col_vals[table][col]
if att_type == 'd': if att_type == 'd':
att_vals = self.db.col_vals[table][col]
att_var = orange.EnumVariable(str(col), values=[str(val) for val in att_vals]) att_var = orange.EnumVariable(str(col), values=[str(val) for val in att_vals])
elif att_type == 'c': elif att_type == 'c':
att_var = orange.FloatVariable(str(col)) att_var = orange.FloatVariable(str(col))
...@@ -226,7 +211,7 @@ class Orange_Converter(Converter): ...@@ -226,7 +211,7 @@ class Orange_Converter(Converter):
for meta in metas: for meta in metas:
domain.addmeta(orange.newmetaid(), meta) domain.addmeta(orange.newmetaid(), meta)
dataset = orange.ExampleTable(domain) dataset = orange.ExampleTable(domain)
for row in self.rows(table, cols): for row in self.db.rows(table, cols):
example = orange.Example(domain) example = orange.Example(domain)
for col, val in zip(cols, row): for col, val in zip(cols, row):
example[str(col)] = str(val) example[str(col)] = str(val)
...@@ -239,9 +224,9 @@ class Orange_Converter(Converter): ...@@ -239,9 +224,9 @@ class Orange_Converter(Converter):
''' '''
mysql_type = self.types[col] mysql_type = self.types[col]
n_vals = len(self.db.col_vals[self.db.target_table][col]) n_vals = len(self.db.col_vals[self.db.target_table][col])
if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.discrete_types): if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.integer_types):
return 'c' return 'c'
elif mysql_type in Orange_Converter.discrete_types: elif mysql_type in Orange_Converter.ordinal_types:
return 'd' return 'd'
else: else:
return 'string' return 'string'
...@@ -252,14 +237,13 @@ if __name__ == '__main__': ...@@ -252,14 +237,13 @@ if __name__ == '__main__':
context = DBContext(DBConnection('root','','localhost','test')) context = DBContext(DBConnection('root','','localhost','test'))
context.target_table = 'trains' context.target_table = 'trains'
context.target_att = 'direction' context.target_att = 'direction'
context.target_att_val = 'east'
# rsd = RSD_Converter(context) rsd = RSD_Converter(context)
# ex, bk = rsd.all_examples(), rsd.background_knowledge() ex, bk = rsd.all_examples(), rsd.background_knowledge()
# aleph = Aleph_Converter(context) aleph = Aleph_Converter(context, target_att_val='east')
# print aleph.positive_examples() print aleph.positive_examples()
# print aleph.negative_examples() print aleph.negative_examples()
# print aleph.background_knowledge() print aleph.background_knowledge()
orange = Orange_Converter(context) orange = Orange_Converter(context)
orange.target_table() orange.target_table()
\ No newline at end of file
[ [
{ {
"pk": 75, "pk": 39,
"model": "workflows.category", "model": "workflows.category",
"fields": { "fields": {
"uid": "b66b71b1-99d6-4efc-b264-a914a3e42911", "uid": "b66b71b1-99d6-4efc-b264-a914a3e42911",
...@@ -12,10 +12,10 @@ ...@@ -12,10 +12,10 @@
} }
}, },
{ {
"pk": 309, "pk": 190,
"model": "workflows.abstractwidget", "model": "workflows.abstractwidget",
"fields": { "fields": {
"category": 75, "category": 39,
"treeview_image": "", "treeview_image": "",
"name": "Database Context", "name": "Database Context",
"is_streaming": false, "is_streaming": false,
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
"interaction_view": "mysql_db_context", "interaction_view": "mysql_db_context",
"image": "", "image": "",
"package": "mysql", "package": "mysql",
"static_image": "", "static_image": "mysql.png",
"post_interact_action": "mysql_db_context_finished", "post_interact_action": "mysql_db_context_finished",
"user": null, "user": null,
"visualization_view": "", "visualization_view": "",
...@@ -37,10 +37,10 @@ ...@@ -37,10 +37,10 @@
} }
}, },
{ {
"pk": 1036, "pk": 443,
"model": "workflows.abstractinput", "model": "workflows.abstractinput",
"fields": { "fields": {
"widget": 309, "widget": 190,
"name": "connection", "name": "connection",
"short_name": "con", "short_name": "con",
"uid": "6c23b4a2-a18e-498e-a22c-4de86932da3e", "uid": "6c23b4a2-a18e-498e-a22c-4de86932da3e",
...@@ -55,10 +55,10 @@ ...@@ -55,10 +55,10 @@
} }
}, },
{ {
"pk": 368, "pk": 216,
"model": "workflows.abstractoutput", "model": "workflows.abstractoutput",
"fields": { "fields": {
"widget": 309, "widget": 190,
"name": "context", "name": "context",
"short_name": "cxt", "short_name": "cxt",
"variable": "context", "variable": "context",
...@@ -68,10 +68,10 @@ ...@@ -68,10 +68,10 @@
} }
}, },
{ {
"pk": 311, "pk": 191,
"model": "workflows.abstractwidget", "model": "workflows.abstractwidget",
"fields": { "fields": {
"category": 75, "category": 39,
"treeview_image": "", "treeview_image": "",
"name": "Database To Aleph", "name": "Database To Aleph",
"is_streaming": false, "is_streaming": false,
...@@ -79,7 +79,7 @@ ...@@ -79,7 +79,7 @@
"interaction_view": "", "interaction_view": "",
"image": "", "image": "",
"package": "mysql", "package": "mysql",
"static_image": "", "static_image": "mysql.png",
"post_interact_action": "", "post_interact_action": "",
"user": null, "user": null,
"visualization_view": "", "visualization_view": "",
...@@ -93,10 +93,28 @@ ...@@ -93,10 +93,28 @@
} }
}, },
{ {
"pk": 1052, "pk": 451,
"model": "workflows.abstractinput", "model": "workflows.abstractinput",
"fields": { "fields": {
"widget": 311, "widget": 191,
"name": "Target attribute value",
"short_name": "tgt",
"uid": "5b7354f1-35d8-41c8-a0ae-c3848855f29d",
"default": "",
"required": true,
"multi": false,
"parameter_type": "text",
"variable": "target_att_val",
"parameter": true,
"order": 1,
"description": "Target attribute value to be used as the positive class"
}
},
{
"pk": 444,
"model": "workflows.abstractinput",
"fields": {
"widget": 191,
"name": "context", "name": "context",
"short_name": "cxt", "short_name": "cxt",
"uid": "474e9673-9c55-48c5-bae8-a7b986aa0287", "uid": "474e9673-9c55-48c5-bae8-a7b986aa0287",
...@@ -106,54 +124,54 @@ ...@@ -106,54 +124,54 @@
"parameter_type": null, "parameter_type": null,
"variable": "context", "variable": "context",
"parameter": false, "parameter": false,
"order": 1, "order": 2,
"description": "Database context object" "description": "Database context object"
} }
}, },
{ {
"pk": 371, "pk": 217,
"model": "workflows.abstractoutput", "model": "workflows.abstractoutput",
"fields": { "fields": {
"widget": 311, "widget": 191,
"name": "pos_examples", "name": "pos_examples",
"short_name": "pex", "short_name": "pex",
"variable": "pos_examples", "variable": "pos_examples",
"uid": "48184527-16a5-446c-9181-48a6558430c2", "uid": "48184527-16a5-446c-9181-48a6558430c2",
"order": 1, "order": 2,
"description": "positive examples file" "description": "positive examples file"
} }
}, },
{ {
"pk": 372, "pk": 218,
"model": "workflows.abstractoutput", "model": "workflows.abstractoutput",
"fields": { "fields": {
"widget": 311, "widget": 191,
"name": "neg_examples", "name": "neg_examples",
"short_name": "nex", "short_name": "nex",
"variable": "neg_examples", "variable": "neg_examples",
"uid": "2c623064-95cd-48ac-8bff-617edf2b9468", "uid": "2c623064-95cd-48ac-8bff-617edf2b9468",
"order": 1, "order": 3,
"description": "negative examples file" "description": "negative examples file"
} }
}, },
{ {
"pk": 373, "pk": 219,
"model": "workflows.abstractoutput", "model": "workflows.abstractoutput",
"fields": { "fields": {
"widget": 311, "widget": 191,
"name": "bk", "name": "bk",
"short_name": "bk", "short_name": "bk",
"variable": "bk", "variable": "bk",
"uid": "5e61e44e-cc52-4471-9a41-3afa52f58ab1", "uid": "5e61e44e-cc52-4471-9a41-3afa52f58ab1",
"order": 1, "order": 4,
"description": "background knowledge" "description": "background knowledge"
} }
}, },
{ {
"pk": 414, "pk": 192,
"model": "workflows.abstractwidget", "model": "workflows.abstractwidget",
"fields": { "fields": {
"category": 75, "category": 39,
"treeview_image": "", "treeview_image": "",
"name": "Database To Orange Table", "name": "Database To Orange Table",
"is_streaming": false, "is_streaming": false,
...@@ -161,7 +179,7 @@ ...@@ -161,7 +179,7 @@
"interaction_view": "mysql_orange_converter", "interaction_view": "mysql_orange_converter",
"image": "", "image": "",
"package": "mysql", "package": "mysql",
"static_image": "", "static_image": "mysql.png",
"post_interact_action": "mysql_orange_converter_finished", "post_interact_action": "mysql_orange_converter_finished",
"user": null, "user": null,
"visualization_view": "", "visualization_view": "",
...@@ -175,10 +193,10 @@ ...@@ -175,10 +193,10 @@
} }
}, },
{ {
"pk": 1241, "pk": 445,
"model": "workflows.abstractinput", "model": "workflows.abstractinput",
"fields": { "fields": {
"widget": 414, "widget": 192,
"name": "context", "name": "context",