From dc1fb7000adb77c647f70428fd6ce224e3938220 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Wed, 20 Aug 2014 11:16:11 -0700 Subject: [PATCH] Clean up some of our python scripts. Change-Id: Ifa75345db43434298cfb6113fbe2f7a33b88c79d --- libc/kernel/tools/clean_header.py | 3 +- libc/kernel/tools/cpp.py | 17 ++- libc/kernel/tools/utils.py | 89 ++-------------- libc/tools/bionic_utils.py | 165 ------------------------------ libc/tools/gensyscalls.py | 148 ++++++++++++++++++++++++--- 5 files changed, 151 insertions(+), 271 deletions(-) delete mode 100644 libc/tools/bionic_utils.py diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py index 660181772..ebebe80f9 100755 --- a/libc/kernel/tools/clean_header.py +++ b/libc/kernel/tools/clean_header.py @@ -199,8 +199,7 @@ if __name__ == "__main__": if opt == '-u': noUpdate = 0 elif opt == '-v': - verbose = 1 - D_setlevel(1) + logging.basicConfig(level=logging.DEBUG) elif opt == '-k': kernel_original_path = arg elif opt == '-d': diff --git a/libc/kernel/tools/cpp.py b/libc/kernel/tools/cpp.py index 2be95320c..0c098de26 100644 --- a/libc/kernel/tools/cpp.py +++ b/libc/kernel/tools/cpp.py @@ -1711,7 +1711,7 @@ def optimize_if01( blocks ): while j < n and not blocks[j].isIf(): j += 1 if j > i: - D2("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno)) + logging.debug("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno)) result += blocks[i:j] if j >= n: break @@ -1730,17 +1730,17 @@ def optimize_if01( blocks ): break dir = blocks[j].directive if dir == "endif": - D2("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno)) + logging.debug("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno)) i = j + 1 elif dir == "else": # convert 'else' into 'if 1' - D2("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno)) + logging.debug("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno)) blocks[j].directive = "if" blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() ) i = j elif dir == "elif": # convert 'elif' into 'if' - D2("convert 'if 0' .. 'elif' into 'if'") + logging.debug("convert 'if 0' .. 'elif' into 'if'") blocks[j].directive = "if" i = j continue @@ -1749,25 +1749,25 @@ def optimize_if01( blocks ): k = find_matching_endif( blocks, j+1 ) if k >= n: # unterminated #if 1, finish here - D2("unterminated 'if 1'") + logging.debug("unterminated 'if 1'") result += blocks[j+1:k] break dir = blocks[k].directive if dir == "endif": - D2("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) + logging.debug("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) result += optimize_if01(blocks[j+1:k]) i = k+1 elif dir == "else": # convert 'else' into 'if 0' - D2("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) + logging.debug("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) result += optimize_if01(blocks[j+1:k]) blocks[k].directive = "if" blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) i = k elif dir == "elif": # convert 'elif' into 'if 0' - D2("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) + logging.debug("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) result += optimize_if01(blocks[j+1:k]) blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) i = k @@ -1835,7 +1835,6 @@ def test_optimizeAll(): out = StringOutput() lines = string.split(text, '\n') list = BlockParser().parse( CppLinesTokenizer(lines) ) - #D_setlevel(2) list.replaceTokens( kernel_token_replacements ) list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} ) list.write(out) diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py index 0478e93e6..e5a310e03 100644 --- a/libc/kernel/tools/utils.py +++ b/libc/kernel/tools/utils.py @@ -1,59 +1,29 @@ # common python utility routines for the Bionic tool scripts -import sys, os, commands, string, commands +import commands +import logging +import os +import string +import sys -# basic debugging trace support -# call D_setlevel to set the verbosity level -# and D(), D2(), D3(), D4() to add traces -# -verbose = 0 def panic(msg): - sys.stderr.write( find_program_name() + ": error: " ) - sys.stderr.write( msg ) + sys.stderr.write(os.path.basename(sys.argv[0]) + ": error: ") + sys.stderr.write(msg) sys.exit(1) -def D(msg): - global verbose - if verbose > 0: - print msg - -def D2(msg): - global verbose - if verbose >= 2: - print msg - -def D3(msg): - global verbose - if verbose >= 3: - print msg - -def D4(msg): - global verbose - if verbose >= 4: - print msg - -def D_setlevel(level): - global verbose - verbose = level - - -# other stuff -# -# -def find_program_name(): - return os.path.basename(sys.argv[0]) def find_program_dir(): return os.path.dirname(sys.argv[0]) + class StringOutput: def __init__(self): self.line = "" def write(self,msg): self.line += msg - D2("write '%s'" % msg) + logging.debug("write '%s'" % msg) def get(self): return self.line @@ -76,47 +46,6 @@ def create_file_path(path): continue os.mkdir(dir) -def walk_source_files(paths,callback,args,excludes=[]): - """recursively walk a list of paths and files, only keeping the source files in directories""" - for path in paths: - if len(path) > 0 and path[0] == '@': - # this is the name of another file, include it and parse it - path = path[1:] - if os.path.exists(path): - for line in open(path): - if len(line) > 0 and line[-1] == '\n': - line = line[:-1] - walk_source_files([line],callback,args,excludes) - continue - if not os.path.isdir(path): - callback(path,args) - else: - for root, dirs, files in os.walk(path): - #print "w-- %s (ex: %s)" % (repr((root,dirs)), repr(excludes)) - if len(excludes): - for d in dirs[:]: - if os.path.join(root,d) in excludes: - dirs.remove(d) - for f in files: - r, ext = os.path.splitext(f) - if ext in [ ".h", ".c", ".cpp", ".S" ]: - callback( "%s/%s" % (root,f), args ) - -def cleanup_dir(path): - """create a directory if needed, and ensure that it is totally empty - by removing any existing content in it""" - if not os.path.exists(path): - os.mkdir(path) - else: - for root, dirs, files in os.walk(path, topdown=False): - if root.endswith("kernel_headers/"): - # skip 'kernel_headers' - continue - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - class BatchFileUpdater: """a class used to edit several files at once""" diff --git a/libc/tools/bionic_utils.py b/libc/tools/bionic_utils.py deleted file mode 100644 index c38efb503..000000000 --- a/libc/tools/bionic_utils.py +++ /dev/null @@ -1,165 +0,0 @@ -# common python utility routines for the Bionic tool scripts - -import sys, os, commands, string - -all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ] - -# basic debugging trace support -# call D_setlevel to set the verbosity level -# and D(), D2(), D3(), D4() to add traces -# -verbose = 0 - -def D(msg): - global verbose - if verbose > 0: - print msg - -def D2(msg): - global verbose - if verbose >= 2: - print msg - -def D3(msg): - global verbose - if verbose >= 3: - print msg - -def D4(msg): - global verbose - if verbose >= 4: - print msg - -def D_setlevel(level): - global verbose - verbose = level - - -# parser for the SYSCALLS.TXT file -# -class SysCallsTxtParser: - def __init__(self): - self.syscalls = [] - self.lineno = 0 - - def E(self, msg): - print "%d: %s" % (self.lineno, msg) - - def parse_line(self, line): - """ parse a syscall spec line. - - line processing, format is - return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list - """ - pos_lparen = line.find('(') - E = self.E - if pos_lparen < 0: - E("missing left parenthesis in '%s'" % line) - return - - pos_rparen = line.rfind(')') - if pos_rparen < 0 or pos_rparen <= pos_lparen: - E("missing or misplaced right parenthesis in '%s'" % line) - return - - return_type = line[:pos_lparen].strip().split() - if len(return_type) < 2: - E("missing return type in '%s'" % line) - return - - syscall_func = return_type[-1] - return_type = string.join(return_type[:-1],' ') - socketcall_id = -1 - - pos_colon = syscall_func.find(':') - if pos_colon < 0: - syscall_name = syscall_func - else: - if pos_colon == 0 or pos_colon+1 >= len(syscall_func): - E("misplaced colon in '%s'" % line) - return - - # now find if there is a socketcall_id for a dispatch-type syscall - # after the optional 2nd colon - pos_colon2 = syscall_func.find(':', pos_colon + 1) - if pos_colon2 < 0: - syscall_name = syscall_func[pos_colon+1:] - syscall_func = syscall_func[:pos_colon] - else: - if pos_colon2+1 >= len(syscall_func): - E("misplaced colon2 in '%s'" % line) - return - syscall_name = syscall_func[(pos_colon+1):pos_colon2] - socketcall_id = int(syscall_func[pos_colon2+1:]) - syscall_func = syscall_func[:pos_colon] - - alias_delim = syscall_func.find('|') - if alias_delim > 0: - alias_list = syscall_func[alias_delim+1:].strip() - syscall_func = syscall_func[:alias_delim] - alias_delim = syscall_name.find('|') - if alias_delim > 0: - syscall_name = syscall_name[:alias_delim] - syscall_aliases = string.split(alias_list, ',') - else: - syscall_aliases = [] - - if pos_rparen > pos_lparen+1: - syscall_params = line[pos_lparen+1:pos_rparen].split(',') - params = string.join(syscall_params,',') - else: - syscall_params = [] - params = "void" - - t = { - "name" : syscall_name, - "func" : syscall_func, - "aliases" : syscall_aliases, - "params" : syscall_params, - "decl" : "%-15s %s (%s);" % (return_type, syscall_func, params), - "socketcall_id" : socketcall_id - } - - # Parse the architecture list. - arch_list = line[pos_rparen+1:].strip() - if arch_list == "all": - for arch in all_arches: - t[arch] = True - else: - for arch in string.split(arch_list, ','): - if arch in all_arches: - t[arch] = True - else: - E("invalid syscall architecture '%s' in '%s'" % (arch, line)) - return - - self.syscalls.append(t) - - global verbose - if verbose >= 2: - print t - - - def parse_file(self, file_path): - D2("parse_file: %s" % file_path) - fp = open(file_path) - for line in fp.xreadlines(): - self.lineno += 1 - line = line.strip() - if not line: continue - if line[0] == '#': continue - self.parse_line(line) - - fp.close() - - -class StringOutput: - def __init__(self): - self.line = "" - - def write(self,msg): - self.line += msg - D2("write '%s'" % msg) - - def get(self): - return self.line diff --git a/libc/tools/gensyscalls.py b/libc/tools/gensyscalls.py index 4619ec690..929bec47e 100755 --- a/libc/tools/gensyscalls.py +++ b/libc/tools/gensyscalls.py @@ -8,14 +8,18 @@ import atexit import commands import filecmp import glob +import logging import os.path import re import shutil import stat +import string import sys import tempfile -from bionic_utils import * + +all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ] + # temp directory where we store all intermediate files bionic_temp = tempfile.mkdtemp(prefix="bionic_gensyscalls"); @@ -387,6 +391,120 @@ def x86_64_genstub(syscall): return result +class SysCallsTxtParser: + def __init__(self): + self.syscalls = [] + self.lineno = 0 + + def E(self, msg): + print "%d: %s" % (self.lineno, msg) + + def parse_line(self, line): + """ parse a syscall spec line. + + line processing, format is + return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list + """ + pos_lparen = line.find('(') + E = self.E + if pos_lparen < 0: + E("missing left parenthesis in '%s'" % line) + return + + pos_rparen = line.rfind(')') + if pos_rparen < 0 or pos_rparen <= pos_lparen: + E("missing or misplaced right parenthesis in '%s'" % line) + return + + return_type = line[:pos_lparen].strip().split() + if len(return_type) < 2: + E("missing return type in '%s'" % line) + return + + syscall_func = return_type[-1] + return_type = string.join(return_type[:-1],' ') + socketcall_id = -1 + + pos_colon = syscall_func.find(':') + if pos_colon < 0: + syscall_name = syscall_func + else: + if pos_colon == 0 or pos_colon+1 >= len(syscall_func): + E("misplaced colon in '%s'" % line) + return + + # now find if there is a socketcall_id for a dispatch-type syscall + # after the optional 2nd colon + pos_colon2 = syscall_func.find(':', pos_colon + 1) + if pos_colon2 < 0: + syscall_name = syscall_func[pos_colon+1:] + syscall_func = syscall_func[:pos_colon] + else: + if pos_colon2+1 >= len(syscall_func): + E("misplaced colon2 in '%s'" % line) + return + syscall_name = syscall_func[(pos_colon+1):pos_colon2] + socketcall_id = int(syscall_func[pos_colon2+1:]) + syscall_func = syscall_func[:pos_colon] + + alias_delim = syscall_func.find('|') + if alias_delim > 0: + alias_list = syscall_func[alias_delim+1:].strip() + syscall_func = syscall_func[:alias_delim] + alias_delim = syscall_name.find('|') + if alias_delim > 0: + syscall_name = syscall_name[:alias_delim] + syscall_aliases = string.split(alias_list, ',') + else: + syscall_aliases = [] + + if pos_rparen > pos_lparen+1: + syscall_params = line[pos_lparen+1:pos_rparen].split(',') + params = string.join(syscall_params,',') + else: + syscall_params = [] + params = "void" + + t = { + "name" : syscall_name, + "func" : syscall_func, + "aliases" : syscall_aliases, + "params" : syscall_params, + "decl" : "%-15s %s (%s);" % (return_type, syscall_func, params), + "socketcall_id" : socketcall_id + } + + # Parse the architecture list. + arch_list = line[pos_rparen+1:].strip() + if arch_list == "all": + for arch in all_arches: + t[arch] = True + else: + for arch in string.split(arch_list, ','): + if arch in all_arches: + t[arch] = True + else: + E("invalid syscall architecture '%s' in '%s'" % (arch, line)) + return + + self.syscalls.append(t) + + logging.debug(t) + + + def parse_file(self, file_path): + logging.debug("parse_file: %s" % file_path) + fp = open(file_path) + for line in fp.xreadlines(): + self.lineno += 1 + line = line.strip() + if not line: continue + if line[0] == '#': continue + self.parse_line(line) + + fp.close() + + class State: def __init__(self): self.old_stubs = [] @@ -444,7 +562,7 @@ class State: def gen_glibc_syscalls_h(self): # TODO: generate a separate file for each architecture, like glibc's bits/syscall.h. glibc_syscalls_h_path = "include/sys/glibc-syscalls.h" - D("generating " + glibc_syscalls_h_path) + logging.info("generating " + glibc_syscalls_h_path) glibc_fp = create_file(glibc_syscalls_h_path) glibc_fp.write("/* %s */\n" % warning) glibc_fp.write("#ifndef _BIONIC_GLIBC_SYSCALLS_H_\n") @@ -473,7 +591,7 @@ class State: for arch in all_arches: if syscall.has_key("asm-%s" % arch): filename = "arch-%s/syscalls/%s.S" % (arch, syscall["func"]) - D2(">>> generating " + filename) + logging.info(">>> generating " + filename) fp = create_file(filename) fp.write(syscall["asm-%s" % arch]) fp.close() @@ -481,28 +599,28 @@ class State: def regenerate(self): - D("scanning for existing architecture-specific stub files...") + logging.info("scanning for existing architecture-specific stub files...") for arch in all_arches: arch_dir = "arch-" + arch - D("scanning " + os.path.join(bionic_libc_root, arch_dir)) + logging.info("scanning " + os.path.join(bionic_libc_root, arch_dir)) rel_path = os.path.join(arch_dir, "syscalls") for file in os.listdir(os.path.join(bionic_libc_root, rel_path)): if file.endswith(".S"): self.old_stubs.append(os.path.join(rel_path, file)) - D("found %d stub files" % len(self.old_stubs)) + logging.info("found %d stub files" % len(self.old_stubs)) if not os.path.exists(bionic_temp): - D("creating %s..." % bionic_temp) + logging.info("creating %s..." % bionic_temp) make_dir(bionic_temp) - D("re-generating stubs and support files...") + logging.info("re-generating stubs and support files...") self.gen_glibc_syscalls_h() self.gen_syscall_stubs() - D("comparing files...") + logging.info("comparing files...") adds = [] edits = [] @@ -511,18 +629,18 @@ class State: libc_file = os.path.join(bionic_libc_root, stub) if not os.path.exists(libc_file): # new file, git add it - D("new file: " + stub) + logging.info("new file: " + stub) adds.append(libc_file) shutil.copyfile(tmp_file, libc_file) elif not filecmp.cmp(tmp_file, libc_file): - D("changed file: " + stub) + logging.info("changed file: " + stub) edits.append(stub) deletes = [] for stub in self.old_stubs: if not stub in self.new_stubs: - D("deleted file: " + stub) + logging.info("deleted file: " + stub) deletes.append(os.path.join(bionic_libc_root, stub)) if not DRY_RUN: @@ -539,11 +657,11 @@ class State: commands.getoutput("git add %s" % (os.path.join(bionic_libc_root, "SYSCALLS.TXT"))) if (not adds) and (not deletes) and (not edits): - D("no changes detected!") + logging.info("no changes detected!") else: - D("ready to go!!") + logging.info("ready to go!!") -D_setlevel(1) +logging.basicConfig(level=logging.INFO) state = State() state.process_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT"))