Merge branch 'mysql' into dev

workflows/mysql/context.py

@@ -11,6 +11,14 @@ class DBConnection:
         self.password = password
         self.host = host
         self.database = database
+        self.check_connection()
+
+    def check_connection(self):
+        try:
+            con = sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
+            con.close()
+        except Exception,e:
+            raise Exception('Problem connecting to the database. Please re-check your credentials.')
 
     def connect(self):
         return sql.connect(user=self.user, password=self.password, host=self.host, database=self.database)
@@ -44,13 +52,14 @@ class DBContext:
         for table, cols in self.cols.items():
             self.col_vals[table] = {}
             for col in cols:
-                cursor.execute("SELECT DISTINCT %s FROM %s" % (col, table))
+                cursor.execute("SELECT DISTINCT `%s` FROM `%s` LIMIT 51" % (col, table))
                 self.col_vals[table][col] = [val for (val,) in cursor]
+        print self.col_vals
         self.connected = {}
         cursor.execute(
            "SELECT table_name, column_name, referenced_table_name, referenced_column_name \
             FROM information_schema.KEY_COLUMN_USAGE \
-            WHERE referenced_table_name IS NOT NULL")
+            WHERE referenced_table_name IS NOT NULL AND table_schema='%s'" % connection.database)
         self.fkeys = defaultdict(set)
         for (table, col, ref_table, ref_col) in cursor:
             self.connected[(table, ref_table)] = (col, ref_col)
@@ -60,12 +69,12 @@ class DBContext:
         cursor.execute(
             "SELECT table_name, column_name \
              FROM information_schema.KEY_COLUMN_USAGE \
-             WHERE constraint_name='PRIMARY' and table_schema='%s'" % connection.database)
+             WHERE constraint_name='PRIMARY' AND table_schema='%s'" % connection.database)
         for (table, pk) in cursor:
             self.pkeys[table] = pk
         self.target_table = self.tables[0]
         self.target_att = None
-        self.target_att_val = None
+        #self.target_att_val = None
         con.close()
 
     def update(self, postdata):
@@ -75,7 +84,7 @@ class DBContext:
         widget_id = postdata.get('widget_id')[0]
         self.target_table = postdata.get('target_table%s' % widget_id)[0]
         self.target_att = postdata.get('target_att%s' % widget_id)[0]
-        self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0]
+        #self.target_att_val = postdata.get('target_att_val%s' % widget_id)[0]
         self.tables = postdata.get('tables%s' % widget_id, [])
         if self.target_table not in self.tables:
             raise Exception('The selected target table "%s" is not among the selected tables.' % self.target_table)
@@ -92,6 +101,30 @@ class DBContext:
             if table == self.target_table and self.target_att not in self.cols[table]:
                 raise Exception('The selected target attribute ("%s") is not among the columns selected for the target table ("%s").' % (self.target_att, self.target_table))
 
+    def fmt_cols(self, cols):
+        return ','.join(["`%s`" % col for col in cols])
+
+    def rows(self, table, cols):
+        con = self.connection.connect()
+        cursor = con.cursor() 
+        cursor.execute("SELECT %s FROM %s" % (self.fmt_cols(cols), table))
+        con.close()
+        return [cols for cols in cursor]
+
+    def fetch_types(self, table, cols):
+        '''
+        Returns a dictionary of field types for the given table and columns.
+        '''
+        con = self.connection.connect()
+        cursor = con.cursor() 
+        cursor.execute("SELECT %s FROM `%s` LIMIT 1" % (self.fmt_cols(cols), table))
+        cursor.fetchall()
+        types = {}
+        for desc in cursor.description:
+            types[desc[0]] = sql.FieldType.get_info(desc[1])
+        con.close()
+        return types
+
     def __repr__(self):
         return str((self.target_table, self.target_att, self.tables, self.cols, self.connected))
 

workflows/mysql/converters.py

@@ -16,23 +16,6 @@ class Converter:
     def __del__(self):  
         self.connection.close()
 
-    def rows(self, table, cols):    
-        self.cursor.execute("SELECT %s FROM %s" % (','.join(cols), table))
-        return [cols for cols in self.cursor]
-
-    def fetch_types(self, table, cols):
-        '''
-        Returns a dictionary of field types for the given table and columns.
-        '''
-        from mysql.connector import FieldType
-        c = self.cursor
-        c.execute('SELECT %s FROM %s LIMIT 1' % (','.join(cols), table))
-        c.fetchall()
-        types = {}
-        for desc in self.cursor.description:
-            types[desc[0]] = FieldType.get_info(desc[1])
-        return types
-
 class ILP_Converter(Converter):
     '''
     Base class for converting between a given database context (selected tables, columns, etc)
@@ -89,8 +72,8 @@ class RSD_Converter(ILP_Converter):
     '''
     def all_examples(self):
         target = self.db.target_table
-        examples = self.rows(target, [self.db.target_att, self.db.pkeys[target]])
-        return '\n'.join(['%s(%s, %s).' % (target, cls, pk) for cls, pk in examples])
+        examples = self.db.rows(target, [self.db.target_att, self.db.pkeys[target]])
+        return '\n'.join(["%s('%s', %s)." % (target, cls, pk) for cls, pk in examples])
 
     def background_knowledge(self):
         modeslist, getters = [self.mode(self.db.target_table, [('+', self.db.target_table)], head=True)], []
@@ -114,13 +97,14 @@ class Aleph_Converter(ILP_Converter):
     Converts the database context to Aleph inputs.
     '''
     def __init__(self, *args, **kwargs):
+        self.target_att_val = kwargs.pop('target_att_val')
         ILP_Converter.__init__(self, *args, **kwargs)
         self.__pos_examples, self.__neg_examples = None, None
 
     def __examples(self):
         if not (self.__pos_examples and self.__neg_examples):
-            target, att, target_val = self.db.target_table, self.db.target_att, self.db.target_att_val
-            rows = self.rows(target, [att, self.db.pkeys[target]])
+            target, att, target_val = self.db.target_table, self.db.target_att, self.target_att_val
+            rows = self.db.rows(target, [att, self.db.pkeys[target]])
             pos_rows, neg_rows = [], []
             for row in rows:
                 if row[0] == target_val:
@@ -138,13 +122,13 @@ class Aleph_Converter(ILP_Converter):
         return self.__examples()[1]
 
     def background_knowledge(self):
-        modeslist, getters = [self.mode(self.db.target_att_val, [('+', self.db.target_table)], head=True)], []
+        modeslist, getters = [self.mode(self.target_att_val, [('+', self.db.target_table)], head=True)], []
         determinations, types = [], []
         for (table, ref_table) in self.db.connected.keys():
             if ref_table == self.db.target_table:
                 continue # Skip backward connections
             modeslist.append(self.mode('has_%s' % ref_table, [('+', table), ('-', ref_table)], recall='*'))
-            determinations.append(':- determination(%s/1, has_%s/2).' % (self.db.target_att_val, ref_table))
+            determinations.append(':- determination(%s/1, has_%s/2).' % (self.target_att_val, ref_table))
             types.extend(self.concept_type_def(table))
             types.extend(self.concept_type_def(ref_table))
             getters.extend(self.connecting_clause(table, ref_table))
@@ -154,7 +138,7 @@ class Aleph_Converter(ILP_Converter):
                    att in self.db.fkeys[table] or att == self.db.pkeys[table]:
                     continue
                 modeslist.append(self.mode('%s_%s' % (table, att), [('+', table), ('#', att)], recall='*'))
-                determinations.append(':- determination(%s/1, %s_%s/2).' % (self.db.target_att_val, table, att))
+                determinations.append(':- determination(%s/1, %s_%s/2).' % (self.target_att_val, table, att))
                 types.extend(self.constant_type_def(table, att))
                 getters.extend(self.attribute_clause(table, att))
         local_copies = [self.local_copy(table) for table in self.db.tables]
@@ -189,11 +173,12 @@ class Orange_Converter(Converter):
     Converts the target table selected in the given context as an orange example table.
     '''
     continuous_types = ('FLOAT','DOUBLE','DECIMAL','NEWDECIMAL')
-    discrete_types = ('TINY','SHORT','LONG','LONGLONG','INT24','YEAR','VARCHAR','BIT','SET','VAR_STRING','STRING')
+    integer_types = ('TINY','SHORT','LONG','LONGLONG','INT24')
+    ordinal_types = ('YEAR','VARCHAR','SET','VAR_STRING','STRING','BIT')
     
     def __init__(self, *args, **kwargs):
         Converter.__init__(self, *args, **kwargs)
-        self.types = self.fetch_types(self.db.target_table, self.db.cols[self.db.target_table])
+        self.types = self.db.fetch_types(self.db.target_table, self.db.cols[self.db.target_table])
 
     def target_table(self):
         '''
@@ -206,8 +191,8 @@ class Orange_Converter(Converter):
         attributes, metas, classVar = [], [], None
         for col in cols:
             att_type = self.orng_type(col)
-            att_vals = self.db.col_vals[table][col]
             if att_type == 'd':
+                att_vals = self.db.col_vals[table][col]
                 att_var = orange.EnumVariable(str(col), values=[str(val) for val in att_vals])
             elif att_type == 'c':
                 att_var = orange.FloatVariable(str(col))
@@ -226,7 +211,7 @@ class Orange_Converter(Converter):
         for meta in metas:
             domain.addmeta(orange.newmetaid(), meta)
         dataset = orange.ExampleTable(domain)
-        for row in self.rows(table, cols):
+        for row in self.db.rows(table, cols):
             example = orange.Example(domain)
             for col, val in zip(cols, row):
                 example[str(col)] = str(val)
@@ -239,9 +224,9 @@ class Orange_Converter(Converter):
         '''
         mysql_type = self.types[col]
         n_vals = len(self.db.col_vals[self.db.target_table][col])
-        if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.discrete_types):
+        if mysql_type in Orange_Converter.continuous_types or (n_vals >= 50 and mysql_type in Orange_Converter.integer_types):
             return 'c'
-        elif mysql_type in Orange_Converter.discrete_types:
+        elif mysql_type in Orange_Converter.ordinal_types:
             return 'd'
         else:
             return 'string'
@@ -252,14 +237,13 @@ if __name__ == '__main__':
     context = DBContext(DBConnection('root','','localhost','test'))
     context.target_table = 'trains'
     context.target_att = 'direction'
-    context.target_att_val = 'east'
 
-    # rsd = RSD_Converter(context)
-    # ex, bk = rsd.all_examples(), rsd.background_knowledge()
+    rsd = RSD_Converter(context)
+    ex, bk = rsd.all_examples(), rsd.background_knowledge()
 
-    # aleph = Aleph_Converter(context)
-    # print aleph.positive_examples()
-    # print aleph.negative_examples()
-    # print aleph.background_knowledge()
+    aleph = Aleph_Converter(context, target_att_val='east')
+    print aleph.positive_examples()
+    print aleph.negative_examples()
+    print aleph.background_knowledge()
     orange = Orange_Converter(context)
     orange.target_table()
\ No newline at end of file

workflows/mysql/db/package_data.json

 [
   {
-    "pk": 75,
+    "pk": 39,
     "model": "workflows.category",
     "fields": {
       "uid": "b66b71b1-99d6-4efc-b264-a914a3e42911",
@@ -12,10 +12,10 @@
     }
   },
   {
-    "pk": 309,
+    "pk": 190,
     "model": "workflows.abstractwidget",
     "fields": {
-      "category": 75,
+      "category": 39,
       "treeview_image": "",
       "name": "Database Context",
       "is_streaming": false,
@@ -23,7 +23,7 @@
       "interaction_view": "mysql_db_context",
       "image": "",
       "package": "mysql",
-      "static_image": "",
+      "static_image": "mysql.png",
       "post_interact_action": "mysql_db_context_finished",
       "user": null,
       "visualization_view": "",
@@ -37,10 +37,10 @@
     }
   },
   {
-    "pk": 1036,
+    "pk": 443,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 309,
+      "widget": 190,
       "name": "connection",
       "short_name": "con",
       "uid": "6c23b4a2-a18e-498e-a22c-4de86932da3e",
@@ -55,10 +55,10 @@
     }
   },
   {
-    "pk": 368,
+    "pk": 216,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 309,
+      "widget": 190,
       "name": "context",
       "short_name": "cxt",
       "variable": "context",
@@ -68,10 +68,10 @@
     }
   },
   {
-    "pk": 311,
+    "pk": 191,
     "model": "workflows.abstractwidget",
     "fields": {
-      "category": 75,
+      "category": 39,
       "treeview_image": "",
       "name": "Database To Aleph",
       "is_streaming": false,
@@ -79,7 +79,7 @@
       "interaction_view": "",
       "image": "",
       "package": "mysql",
-      "static_image": "",
+      "static_image": "mysql.png",
       "post_interact_action": "",
       "user": null,
       "visualization_view": "",
@@ -93,10 +93,28 @@
     }
   },
   {
-    "pk": 1052,
+    "pk": 451,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 311,
+      "widget": 191,
+      "name": "Target attribute value",
+      "short_name": "tgt",
+      "uid": "5b7354f1-35d8-41c8-a0ae-c3848855f29d",
+      "default": "",
+      "required": true,
+      "multi": false,
+      "parameter_type": "text",
+      "variable": "target_att_val",
+      "parameter": true,
+      "order": 1,
+      "description": "Target attribute value to be used as the positive class"
+    }
+  },
+  {
+    "pk": 444,
+    "model": "workflows.abstractinput",
+    "fields": {
+      "widget": 191,
       "name": "context",
       "short_name": "cxt",
       "uid": "474e9673-9c55-48c5-bae8-a7b986aa0287",
@@ -106,54 +124,54 @@
       "parameter_type": null,
       "variable": "context",
       "parameter": false,
-      "order": 1,
+      "order": 2,
       "description": "Database context object"
     }
   },
   {
-    "pk": 371,
+    "pk": 217,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 311,
+      "widget": 191,
       "name": "pos_examples",
       "short_name": "pex",
       "variable": "pos_examples",
       "uid": "48184527-16a5-446c-9181-48a6558430c2",
-      "order": 1,
+      "order": 2,
       "description": "positive examples file"
     }
   },
   {
-    "pk": 372,
+    "pk": 218,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 311,
+      "widget": 191,
       "name": "neg_examples",
       "short_name": "nex",
       "variable": "neg_examples",
       "uid": "2c623064-95cd-48ac-8bff-617edf2b9468",
-      "order": 1,
+      "order": 3,
       "description": "negative examples file"
     }
   },
   {
-    "pk": 373,
+    "pk": 219,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 311,
+      "widget": 191,
       "name": "bk",
       "short_name": "bk",
       "variable": "bk",
       "uid": "5e61e44e-cc52-4471-9a41-3afa52f58ab1",
-      "order": 1,
+      "order": 4,
       "description": "background knowledge"
     }
   },
   {
-    "pk": 414,
+    "pk": 192,
     "model": "workflows.abstractwidget",
     "fields": {
-      "category": 75,
+      "category": 39,
       "treeview_image": "",
       "name": "Database To Orange Table",
       "is_streaming": false,
@@ -161,7 +179,7 @@
       "interaction_view": "mysql_orange_converter",
       "image": "",
       "package": "mysql",
-      "static_image": "",
+      "static_image": "mysql.png",
       "post_interact_action": "mysql_orange_converter_finished",
       "user": null,
       "visualization_view": "",
@@ -175,10 +193,10 @@
     }
   },
   {
-    "pk": 1241,
+    "pk": 445,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 414,
+      "widget": 192,
       "name": "context",
       "short_name": "cxt",
       "uid": "1f9b5ccf-65c3-4ccc-818e-afb3a6ffee20",
@@ -193,10 +211,10 @@
     }
   },
   {
-    "pk": 489,
+    "pk": 220,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 414,
+      "widget": 192,
       "name": "Data table",
       "short_name": "odt",
       "variable": "dataset",
@@ -206,10 +224,10 @@
     }
   },
   {
-    "pk": 310,
+    "pk": 193,
     "model": "workflows.abstractwidget",
     "fields": {
-      "category": 75,
+      "category": 39,
       "treeview_image": "",
       "name": "Database To RSD",
       "is_streaming": false,
@@ -217,7 +235,7 @@
       "interaction_view": "",
       "image": "",
       "package": "mysql",
-      "static_image": "",
+      "static_image": "mysql.png",
       "post_interact_action": "",
       "user": null,
       "visualization_view": "",
@@ -231,10 +249,10 @@
     }
   },
   {
-    "pk": 1037,
+    "pk": 446,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 310,
+      "widget": 193,
       "name": "context",
       "short_name": "cxt",
       "uid": "4f1397a8-4e72-4b34-b31d-d09bf9a7e7d9",
@@ -249,10 +267,10 @@
     }
   },
   {
-    "pk": 369,
+    "pk": 221,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 310,
+      "widget": 193,
       "name": "examples",
       "short_name": "exm",
       "variable": "examples",
@@ -262,10 +280,10 @@
     }
   },
   {
-    "pk": 370,
+    "pk": 222,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 310,
+      "widget": 193,
       "name": "bk",
       "short_name": "bk",
       "variable": "bk",
@@ -275,10 +293,10 @@
     }
   },
   {
-    "pk": 308,
+    "pk": 194,
     "model": "workflows.abstractwidget",
     "fields": {
-      "category": 75,
+      "category": 39,
       "treeview_image": "",
       "name": "MySQL Connect",
       "is_streaming": false,
@@ -286,7 +304,7 @@
       "interaction_view": "",
       "image": "",
       "package": "mysql",
-      "static_image": "",
+      "static_image": "mysql.png",
       "post_interact_action": "",
       "user": null,
       "visualization_view": "",
@@ -300,10 +318,10 @@
     }
   },
   {
-    "pk": 1032,
+    "pk": 447,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 308,
+      "widget": 194,
       "name": "user",
       "short_name": "usr",
       "uid": "0f36f10b-066f-4ad3-9fa8-39205440076a",
@@ -318,17 +336,17 @@
     }
   },
   {
-    "pk": 1033,
+    "pk": 448,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 308,
+      "widget": 194,
       "name": "password",
       "short_name": "pwd",
       "uid": "8ead911a-3d1b-4e50-b56a-64873b85d3cf",
       "default": "",
       "required": false,
       "multi": false,
-      "parameter_type": "text",
+      "parameter_type": "password",
       "variable": "password",
       "parameter": true,
       "order": 2,
@@ -336,10 +354,10 @@
     }
   },
   {
-    "pk": 1034,
+    "pk": 449,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 308,
+      "widget": 194,
       "name": "host",
       "short_name": "hst",
       "uid": "9812cc11-b21e-4444-8dfe-bc0c68fe79e8",
@@ -354,10 +372,10 @@
     }
   },
   {
-    "pk": 1035,
+    "pk": 450,
     "model": "workflows.abstractinput",
     "fields": {
-      "widget": 308,
+      "widget": 194,
       "name": "database",
       "short_name": "db",
       "uid": "d132adce-c3ec-411a-8ee7-906be4a99f61",
@@ -372,10 +390,10 @@
     }
   },
   {
-    "pk": 367,
+    "pk": 223,
     "model": "workflows.abstractoutput",
     "fields": {
-      "widget": 308,
+      "widget": 194,
       "name": "connection",
       "short_name": "con",
       "variable": "connection",

workflows/mysql/library.py

@@ -29,7 +29,7 @@ def mysql_rsd_converter(input_dict):
     return {'examples' : rsd.all_examples(), 'bk' : rsd.background_knowledge()}
 
 def mysql_aleph_converter(input_dict):
-    aleph = Aleph_Converter(input_dict['context'])
+    aleph = Aleph_Converter(input_dict['context'], target_att_val=input_dict['target_att_val'])
     return {'pos_examples' : aleph.positive_examples(), 'neg_examples' : aleph.negative_examples(), 'bk' : aleph.background_knowledge()}
 
 def mysql_query_to_odt(input_dict):

workflows/mysql/templates/interactions/db_context.html

@@ -13,12 +13,6 @@
 				<option value="{{col}}" {% if forloop.first %}selected="selected"{% endif %}>{{col}}</option>
 				{% endfor %}
 			</select>
-			<label>Target attribute value:</label> 
-			<select name="target_att_val{{widget.pk}}">
-				{% for val in target_col_vals %}
-				<option value="{{val}}" {% if forloop.first %}selected="selected"{% endif %}>{{val}}</option>
-				{% endfor %}
-			</select>
 			<label>Select tables to be used:</label> 
 			<select multiple name="tables{{widget.pk}}">
 				{% for table in context.tables %}
@@ -49,7 +43,8 @@
 		<script type="text/javascript">