Commit 8b64c610 authored by Anze Vavpetic's avatar Anze Vavpetic

merge ilp security fix from dev

parent 2c9c152b
......@@ -9,7 +9,15 @@ import logging
import re
import tempfile
from stat import S_IREAD, S_IEXEC
from subprocess import Popen, PIPE
from subprocess import PIPE
if __name__ != '__main__':
from ..security import SafePopen
else:
import os
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.sys.path.append(parent_dir)
from security import SafePopen
DEBUG = True
......@@ -25,7 +33,6 @@ class Aleph(object):
# The aleph source file is presumed to be in the same dir as this file.
THIS_DIR = os.path.dirname(__file__) if os.path.dirname(__file__) else '.'
ALEPH_FN = 'aleph.pl'
YAP = '/usr/local/bin/yap'
RULES_SUFFIX = 'Rules'
SCRIPT = 'run_aleph.pl'
......@@ -97,7 +104,7 @@ class Aleph(object):
logger.info("Running aleph...")
# Run the aleph script.
p = Popen(['./' + Aleph.SCRIPT], cwd=self.tmpdir, stdout=PIPE)
p = SafePopen(['yap', '-s50000', '-h200000', '-L', Aleph.SCRIPT], cwd=self.tmpdir, stdout=PIPE).safe_run()
stdout_str, stderr_str = p.communicate()
logger.debug(stdout_str)
......@@ -146,14 +153,11 @@ class Aleph(object):
"""
scriptPath = '%s/%s' % (self.tmpdir, Aleph.SCRIPT)
script = open(scriptPath, 'w')
#print scriptPath
# Permit the owner to execute and read this script
os.chmod(scriptPath, S_IREAD | S_IEXEC)
cat = lambda x: script.write(x + '\n')
cat("#!%s -L -s50000 -h200000" % Aleph.YAP)
cat(":- initialization(run_aleph).")
cat("run_aleph :- ")
cat("consult(aleph),")
......
......@@ -4,6 +4,8 @@ from random import choice
from aleph import Aleph
from rsd import RSD
from wordification import Wordification
from security import check_input
from services.webservice import WebService
......@@ -20,6 +22,9 @@ def ilp_aleph(input_dict):
# Parse settings provided as parameters (these have higher priority)
for setting, def_val in Aleph.ESSENTIAL_PARAMS.items():
aleph.set(setting, input_dict.get(setting, def_val))
# Check for illegal predicates
for pl_script in [b, pos, neg]:
check_input(pl_script)
# Run aleph
theory = aleph.induce(mode, pos, neg, b)
return {'theory' : theory}
......@@ -38,6 +43,9 @@ def ilp_rsd(input_dict):
# Parse settings provided as parameters (these have higher priority)
for setting, def_val in RSD.ESSENTIAL_PARAMS.items():
rsd.set(setting, input_dict.get(setting, def_val))
# Check for illegal predicates
for pl_script in [b, pos, neg, examples]:
check_input(pl_script)
# Run rsd
features, arff, rules = rsd.induce(b, examples=examples, pos=pos, neg=neg, cn2sd=subgroups)
return {'features' : features, 'arff' : arff, 'rules' : rules}
......@@ -60,4 +68,12 @@ def ilp_sdmaleph(input_dict):
clauseLen=input_dict.get('clauseLen') if input_dict.get('clauseLen') != '' else None,
dataFormat=input_dict.get('dataFormat') if input_dict.get('dataFormat') != '' else None
)
return {'theory' : response['theory']}
\ No newline at end of file
return {'theory' : response['theory']}
def ilp_wordification(input_dict):
target_table = input_dict.get('target_table',None)
other_tables = input_dict.get('other_tables', None)
context = input_dict.get('context', None)
wordification = Wordification(target_table,other_tables,context)
return {'corpus' : wordification.wordify()}
\ No newline at end of file
import pwd
import os
import shutil
import tempfile
import stat
import subprocess
jail_dir = tempfile.mkdtemp()
os.chmod(jail_dir, stat.S_IRUSR | stat.S_IWUSR)#etc
#we use several other permission bits here.
#c.f. best practices article
jail_etc = os.path.abspath(os.path.join(jail_dir, "etc"))
os.mkdir(jail_etc)
shutil.copy('/etc/resolv.conf', jail_etc)
for required_dir in ['/usr/bin/']:
#use mount -r --bind as a convenient way of giving read-only access
#to other folders (libraries, etc) required by the jailed process
mount_point = "%s%s" % (jail_dir, required_dir)
os.makedirs(mount_point)
subprocess.call(["sudo", "mount", "-r", "--bind", required_dir, mount_point])
nobody_user = pwd.getpwnam("nobody").pw_uid
os.chroot(jail_dir)
os.chdir('/')
os.seteuid(nobody_user)
p = subprocess.Popen(['python', 'rsd.py'])
stdout_str, stderr_str = p.communicate()
print stdout_str
print stderr_str
\ No newline at end of file
......@@ -8,7 +8,15 @@ import logging
import re
import tempfile
from stat import S_IREAD, S_IEXEC
from subprocess import Popen, PIPE
from subprocess import PIPE
if __name__ != '__main__':
from ..security import SafePopen
else:
import os
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.sys.path.append(parent_dir)
from security import SafePopen
DEBUG = True
......@@ -23,7 +31,6 @@ logger.addHandler(ch)
class RSD(object):
THIS_DIR = os.path.dirname(__file__) if os.path.dirname(__file__) else '.'
RSD_FILES = ['featurize.pl', 'process.pl', 'rules.pl']
YAP = '/usr/local/bin/yap'
# Generated scripts filenames
CONSTRUCT = '_construct.pl'
......@@ -92,23 +99,26 @@ class RSD(object):
# Run the script
logger.info("Running RSD...")
for script in RSD.SCRIPTS:
# Skip subgroup discovery part?
if script == RSD.SUBGROUPS and not cn2sd:
continue
p = Popen(['./' + script], cwd=self.tmpdir, stdout=PIPE)
stdout_str, stderr_str = p.communicate()
logger.debug(stdout_str)
logger.debug(stderr_str)
logger.info("Done.")
# Return the rules written in the output file.
features = open('%s/%s' % (self.tmpdir, filestem + '_frs.pl')).read()
weka = open('%s/%s' % (self.tmpdir, filestem + '.arff')).read()
rules = open('%s/%s' % (self.tmpdir, filestem + '.rules')).read() if cn2sd else ''
self.__cleanup()
return (features, weka, rules)
try:
for script in RSD.SCRIPTS:
# Skip subgroup discovery part?
if script == RSD.SUBGROUPS and not cn2sd:
continue
p = SafePopen(['yap', '-s50000', '-h200000', '-L', script], cwd=self.tmpdir, stdout=PIPE).safe_run()
stdout_str, stderr_str = p.communicate()
logger.debug(stdout_str)
logger.debug(stderr_str)
logger.info("Done.")
# Return the rules written in the output file.
features = open('%s/%s' % (self.tmpdir, filestem + '_frs.pl')).read()
weka = open('%s/%s' % (self.tmpdir, filestem + '.arff')).read()
rules = open('%s/%s' % (self.tmpdir, filestem + '.rules')).read() if cn2sd else ''
self.__cleanup()
return (features, weka, rules)
except OSError:
raise RuntimeError("Yap compiler could not be loaded! (see http://www.dcc.fc.up.pt/~vsc/Yap/).")
def __prepare(self, filestem, b, examples=None, pos=None, neg=None):
"""
......@@ -163,7 +173,6 @@ class RSD(object):
# 'Construction' script
#
w = new_script(script_construct)
w("#!%s -L -s50000 -h200000\n#." % RSD.YAP)
w(':- initialization(main).')
w('main :-')
w('[featurize],')
......@@ -175,7 +184,6 @@ class RSD(object):
# 'Saving' script
#
w = new_script(script_save)
w("#!%s -L -s50000 -h200000\n#." % RSD.YAP)
w(':- initialization(main).')
w('main :-')
w('[process],')
......@@ -189,7 +197,6 @@ class RSD(object):
# 'Subgroups' script
#
w = new_script(script_subgroups)
w("#!%s -L -s50000 -h200000\n#." % RSD.YAP)
w(':- initialization(main).')
w('main :-')
w('[rules],')
......
import time
import re
import pwd
import os
import sys
from subprocess import Popen, PIPE
import threading
illegal_predicates = [
('library(?!\(myddas\))', 'library'), ('use_module(?!\(library\(myddas\)\))', 'use_module'), ('\[.+\]', '[]'), \
'abort', 'absolute_file_name', 'access', 'access_file', 'add_to_path', 'alarm', 'alias', 'always_prompt_user', 'at_end_of_stream', \
'autoload', 'b_getval', 'b_setval', 'bb_delete', 'bb_get', 'bb_put', 'bb_update', 'call', 'catch', 'cd', 'clause', 'close', 'codes', \
'compilation_mode', 'compile', 'consult', 'create_prolog_flag', 'current_atom', 'current_char_conversion', 'current_host', 'current_input', \
'current_key', 'current_line_number', 'current_op', 'current_output', 'current_predicate', 'current_stream', 'cycles', 'db_assert', \
'db_close', 'db_datalog_describe', 'db_datalog_show_tables', 'db_describe', 'db_get_attributes_types', 'db_insert', 'db_module', \
'db_my_result_set', 'db_my_sql_mode', 'db_number_of_fields', 'db_show_tables', 'db_sql', 'db_top_level', 'db_verbose', 'db_view', \
'derived_from', 'direct_cycle', 'discontiguous', 'display', 'dynamic', 'end_of_stream', 'ensure_loaded', 'environ', 'eof_action', \
'erase', 'eraseall', 'erased', 'exception', 'exists', 'exited', 'expand', 'expand_exprs', 'expand_filename', 'extensions', 'file_base_name', \
'file_errors', 'file_name', 'file_name_extension', 'file_search_path', 'file_type', 'flush_output', 'force', 'format', 'garbage_collect', \
'garbage_collect_atoms', 'gc', 'get_code', 'get_value', 'getcwd', 'grow_heap', 'grow_stack', 'halt', 'hide', 'hide_predicate', \
'hostname_address', 'idb', 'ignore_ops', 'imports', 'include', 'initialization', 'insert', 'instance', 'is_stream', 'key_statistics', \
'library_directory', 'line_count', 'line_position', 'listing', 'load_files', 'max_depth', 'message_hook', 'message_to_string', \
'must_be_module', 'nb_current', 'nb_delete', 'nb_getval', 'nb_linkarg', 'nb_linkval', 'nb_set_shared_arg', 'nb_set_shared_val', \
'nb_setarg', 'nb_setval', 'no_style_check', 'nogc', 'nth_instance', 'null', 'numbervars', 'on_signal', 'open', 'path', 'peek_byte', \
'peek_char', 'peek_code', 'phrase', 'portray', 'portray_clause', 'portrayed', 'predicate_property', 'print', 'print_message', \
'print_message_lines', 'printf', 'priority', 'prolog_file_name', 'prolog_flag', 'prolog_initialization', 'prolog_load_context', 'prompt', \
'pthread_setconcurrency', 'put', 'put_byte', 'put_char', 'put_code', 'putenv', 'quoted', 'read', 'read_and_increment_counter', 'read_term', \
'reconsult', 'recorda', 'recorda_at', 'recordaifnot', 'recorded', 'recordz', 'recordz_at', 'recordzifnot', 'reexport', 'relative_to', \
'remove_from_path', 'rename', 'reposition', 'representation_errors', 'restore', 'retract', 'save', 'save_program', 'see', 'seeing', \
'set_input', 'set_output', 'set_prolog_flag', 'set_stream_position', 'set_value', 'sh', 'silent', 'singletons', 'skip', 'socket', \
'socket_accept', 'socket_bind', 'socket_buffering', 'socket_close', 'socket_connect', 'socket_listen', 'socket_select', 'solutions', \
'source_file', 'source_location', 'source_mode', 'stream', 'stream_position', 'stream_position_data', 'stream_property', 'stream_select', \
'style_check', 'syntax_errors', 'system', 'system_predicate', 'tableInfo', 'tell', 'telling', 'term_position', 'thread_at_exit', \
'thread_create', 'thread_detach', 'thread_exit', 'thread_join', 'thread_self', 'thread_setconcurrency', 'thread_sleep', 'throw', 'time_file', \
'ttyget', 'ttyget0', 'ttyput', 'ttyskip', 'ttytab', 'unhide', 'unix', 'var', 'variable_names', 'variables', 'version', 'view', 'with_output_to', \
'write', 'write_canonical', 'write_depth', 'write_many_as', 'write_term', 'writeln', 'writeq', 'yap_flag'
]
allowed_mysql_predicates = [
'use_module(library(myddas))',
'db_import',
'db_open',
]
default_timeout = 15 * 60
class SafePopen(threading.Thread):
'''
Executes a given command and kills it if it is not done in 'timeout' seconds.
TODO: run child in some sort of jail.
'''
def __init__(self, args, **kwargs):
threading.Thread.__init__(self)
self.timeout = kwargs.pop('timeout') if kwargs.has_key('timeout') else default_timeout
self.args = args
self.kwargs = kwargs
def run(self):
self.p = Popen(self.args, **self.kwargs)
self.p.wait()
def safe_run(self):
self.start()
self.join(self.timeout)
if self.is_alive():
self.p.terminate()
self.join()
raise Exception('Your computation has exceeded the maximum available time of %.2f minutes.' % (self.timeout / 60.))
return self.p
def check_input(prolog_file):
'''
Check for illegal predicates (like reading/writing, opening sockets, etc).
'''
if prolog_file == None:
return
for pred in illegal_predicates:
if type(pred) == tuple:
print_name = pred[1]
pred = pred[0]
else:
print_name = pred
if re.search(r'[^\w]' + pred + r'\s*[\(\)\:\.\,\;]+', prolog_file):
raise Exception('Illegal predicate "%s" used in your input, aborting. If your own predicate clashes with a predefined YAP predicate, you must rename it.' % print_name)
if __name__ == '__main__':
nasty_scripts = [
":- foo,socket_connect('sock',83,stream),foo.", \
":- foo, sh, foo.", \
"do_nasty(_) :- open('fn').",\
":- true;system('ls -l').", \
"foo:-p,['another_nasty_script'],q.",
":-use_module(library(assoc)).",
]
errors = 0
for script in nasty_scripts:
try:
check_input(script)
except:
errors += 1
assert errors == len(nasty_scripts)
print 'All illegal predicates found.'
try:
p = SafePopen(['find', '/', '-name', '"a"'], stdout=PIPE, timeout=0.5).safe_run()
except:
print 'Timeout exceeded.'
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment