| #! /usr/bin/python2 |
| import os.path |
| import sys |
| import shlex |
| import re |
| import tempfile |
| import copy |
| |
| from headerutils import * |
| |
| requires = { } |
| provides = { } |
| |
| no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ] |
| |
| # These targets are the ones which provide "coverage". Typically, if any |
| # target is going to fail compilation, it's one of these. This was determined |
| # during the initial runs of reduce-headers... On a full set of target builds, |
| # every failure which occured was triggered by one of these. |
| # This list is used during target-list construction simply to put any of these |
| # *first* in the candidate list, increasing the probability that a failure is |
| # found quickly. |
| target_priority = [ |
| "aarch64-linux-gnu", |
| "arm-netbsdelf", |
| "c6x-elf", |
| "epiphany-elf", |
| "i686-mingw32crt", |
| "i686-pc-msdosdjgpp", |
| "mipsel-elf", |
| "powerpc-eabisimaltivec", |
| "rs6000-ibm-aix5.1.0", |
| "sh-superh-elf", |
| "sparc64-elf" |
| ] |
| |
| |
| target_dir = "" |
| build_dir = "" |
| ignore_list = list() |
| target_builds = list() |
| |
| target_dict = { } |
| header_dict = { } |
| search_path = [ ".", "../include", "../libcpp/include" ] |
| |
| remove_count = { } |
| |
| |
| # Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while |
| # the same header could be referenced from within the cp subdirectory as |
| # just cp-tree.h |
| # for now, just assume basenames are unique |
| |
| def normalize_header (header): |
| return os.path.basename (header) |
| |
| |
| # Adds a header file and its sub-includes to the global dictionary if they |
| # aren't already there. Specify s_path since different build directories may |
| # append themselves on demand to the global list. |
| # return entry for the specified header, knowing all sub entries are completed |
| |
| def get_header_info (header, s_path): |
| global header_dict |
| global empty_iinfo |
| process_list = list () |
| location = "" |
| bname = "" |
| bname_iinfo = empty_iinfo |
| for path in s_path: |
| if os.path.exists (path + "/" + header): |
| location = path + "/" + header |
| break |
| |
| if location: |
| bname = normalize_header (location) |
| if header_dict.get (bname): |
| bname_iinfo = header_dict[bname] |
| loc2 = ii_path (bname_iinfo)+ "/" + bname |
| if loc2[:2] == "./": |
| loc2 = loc2[2:] |
| if location[:2] == "./": |
| location = location[2:] |
| if loc2 != location: |
| # Don't use the cache if it isnt the right one. |
| bname_iinfo = process_ii_macro (location) |
| return bname_iinfo |
| |
| bname_iinfo = process_ii_macro (location) |
| header_dict[bname] = bname_iinfo |
| # now decend into the include tree |
| for i in ii_include_list (bname_iinfo): |
| get_header_info (i, s_path) |
| else: |
| # if the file isnt in the source directories, look in the build and target |
| # directories. If it is here, then aggregate all the versions. |
| location = build_dir + "/gcc/" + header |
| build_inc = target_inc = False |
| if os.path.exists (location): |
| build_inc = True |
| for x in target_dict: |
| location = target_dict[x] + "/gcc/" + header |
| if os.path.exists (location): |
| target_inc = True |
| break |
| |
| if (build_inc or target_inc): |
| bname = normalize_header(header) |
| defines = set() |
| consumes = set() |
| incl = set() |
| if build_inc: |
| iinfo = process_ii_macro (build_dir + "/gcc/" + header) |
| defines = set (ii_macro_define (iinfo)) |
| consumes = set (ii_macro_consume (iinfo)) |
| incl = set (ii_include_list (iinfo)) |
| |
| if (target_inc): |
| for x in target_dict: |
| location = target_dict[x] + "/gcc/" + header |
| if os.path.exists (location): |
| iinfo = process_ii_macro (location) |
| defines.update (ii_macro_define (iinfo)) |
| consumes.update (ii_macro_consume (iinfo)) |
| incl.update (ii_include_list (iinfo)) |
| |
| bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list()) |
| |
| header_dict[bname] = bname_iinfo |
| for i in incl: |
| get_header_info (i, s_path) |
| |
| return bname_iinfo |
| |
| |
| # return a list of all headers brought in by this header |
| def all_headers (fname): |
| global header_dict |
| headers_stack = list() |
| headers_list = list() |
| if header_dict.get (fname) == None: |
| return list () |
| for y in ii_include_list (header_dict[fname]): |
| headers_stack.append (y) |
| |
| while headers_stack: |
| h = headers_stack.pop () |
| hn = normalize_header (h) |
| if hn not in headers_list: |
| headers_list.append (hn) |
| if header_dict.get(hn): |
| for y in ii_include_list (header_dict[hn]): |
| if normalize_header (y) not in headers_list: |
| headers_stack.append (y) |
| |
| return headers_list |
| |
| |
| |
| |
| # Search bld_dir for all target tuples, confirm that they have a build path with |
| # bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by |
| # target tuple.. |
| |
| def build_target_dict (bld_dir, just_these): |
| global target_dict |
| target_doct = { } |
| error = False |
| if os.path.exists (bld_dir): |
| if just_these: |
| ls = just_these |
| else: |
| ls = os.listdir(bld_dir) |
| for t in ls: |
| if t.find("-") != -1: |
| target = t.strip() |
| tpath = bld_dir + "/" + target |
| if not os.path.exists (tpath + "/gcc"): |
| print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc" |
| error = True |
| else: |
| target_dict[target] = tpath |
| |
| if error: |
| target_dict = { } |
| |
| def get_obj_name (src_file): |
| if src_file[-2:] == ".c": |
| return src_file.replace (".c", ".o") |
| elif src_file[-3:] == ".cc": |
| return src_file.replace (".cc", ".o") |
| return "" |
| |
| def target_obj_exists (target, obj_name): |
| global target_dict |
| # look in a subdir if src has a subdir, then check gcc base directory. |
| if target_dict.get(target): |
| obj = target_dict[target] + "/gcc/" + obj_name |
| if not os.path.exists (obj): |
| obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name) |
| if os.path.exists (obj): |
| return True |
| return False |
| |
| # Given a src file, return a list of targets which may build this file. |
| def find_targets (src_file): |
| global target_dict |
| targ_list = list() |
| obj_name = get_obj_name (src_file) |
| if not obj_name: |
| print "Error: " + src_file + " - Cannot determine object name." |
| return list() |
| |
| # Put the high priority targets which tend to trigger failures first |
| for target in target_priority: |
| if target_obj_exists (target, obj_name): |
| targ_list.append ((target, target_dict[target])) |
| |
| for target in target_dict: |
| if target not in target_priority and target_obj_exists (target, obj_name): |
| targ_list.append ((target, target_dict[target])) |
| |
| return targ_list |
| |
| |
| def try_to_remove (src_file, h_list, verbose): |
| global target_dict |
| global header_dict |
| global build_dir |
| |
| # build from scratch each time |
| header_dict = { } |
| summary = "" |
| rmcount = 0 |
| |
| because = { } |
| src_info = process_ii_macro_src (src_file) |
| src_data = ii_src (src_info) |
| if src_data: |
| inclist = ii_include_list_non_cond (src_info) |
| # work is done if there are no includes to check |
| if not inclist: |
| return src_file + ": No include files to attempt to remove" |
| |
| # work on the include list in reverse. |
| inclist.reverse() |
| |
| # Get the target list |
| targ_list = list() |
| targ_list = find_targets (src_file) |
| |
| spath = search_path |
| if os.path.dirname (src_file): |
| spath.append (os.path.dirname (src_file)) |
| |
| hostbuild = True |
| if src_file.find("config/") != -1: |
| # config files dont usually build on the host |
| hostbuild = False |
| obn = get_obj_name (os.path.basename (src_file)) |
| if obn and os.path.exists (build_dir + "/gcc/" + obn): |
| hostbuild = True |
| if not target_dict: |
| summary = src_file + ": Target builds are required for config files. None found." |
| print summary |
| return summary |
| if not targ_list: |
| summary =src_file + ": Cannot find any targets which build this file." |
| print summary |
| return summary |
| |
| if hostbuild: |
| # confirm it actually builds before we do anything |
| print "Confirming source file builds" |
| res = get_make_output (build_dir + "/gcc", "all") |
| if res[0] != 0: |
| message = "Error: " + src_file + " does not build currently." |
| summary = src_file + " does not build on host." |
| print message |
| print res[1] |
| if verbose: |
| verbose.write (message + "\n") |
| verbose.write (res[1]+ "\n") |
| return summary |
| |
| src_requires = set (ii_macro_consume (src_info)) |
| for macro in src_requires: |
| because[macro] = src_file |
| header_seen = list () |
| |
| os.rename (src_file, src_file + ".bak") |
| src_orig = copy.deepcopy (src_data) |
| src_tmp = copy.deepcopy (src_data) |
| |
| try: |
| # process the includes from bottom to top. This is because we know that |
| # later includes have are known to be needed, so any dependency from this |
| # header is a true dependency |
| for inc_file in inclist: |
| inc_file_norm = normalize_header (inc_file) |
| |
| if inc_file in no_remove: |
| continue |
| if len (h_list) != 0 and inc_file_norm not in h_list: |
| continue |
| if inc_file_norm[0:3] == "gt-": |
| continue |
| if inc_file_norm[0:6] == "gtype-": |
| continue |
| if inc_file_norm.replace(".h",".c") == os.path.basename(src_file): |
| continue |
| |
| lookfor = ii_src_line(src_info)[inc_file] |
| src_tmp.remove (lookfor) |
| message = "Trying " + src_file + " without " + inc_file |
| print message |
| if verbose: |
| verbose.write (message + "\n") |
| out = open(src_file, "w") |
| for line in src_tmp: |
| out.write (line) |
| out.close() |
| |
| keep = False |
| if hostbuild: |
| res = get_make_output (build_dir + "/gcc", "all") |
| else: |
| res = (0, "") |
| |
| rc = res[0] |
| message = "Passed Host build" |
| if (rc != 0): |
| # host build failed |
| message = "Compilation failed:\n"; |
| keep = True |
| else: |
| if targ_list: |
| objfile = get_obj_name (src_file) |
| t1 = targ_list[0] |
| if objfile and os.path.exists(t1[1] +"/gcc/"+objfile): |
| res = get_make_output_parallel (targ_list, objfile, 0) |
| else: |
| res = get_make_output_parallel (targ_list, "all-gcc", 0) |
| rc = res[0] |
| if rc != 0: |
| message = "Compilation failed on TARGET : " + res[2] |
| keep = True |
| else: |
| message = "Passed host and target builds" |
| |
| if keep: |
| print message + "\n" |
| |
| if (rc != 0): |
| if verbose: |
| verbose.write (message + "\n"); |
| verbose.write (res[1]) |
| verbose.write ("\n"); |
| if os.path.exists (inc_file): |
| ilog = open(inc_file+".log","a") |
| ilog.write (message + " for " + src_file + ":\n\n"); |
| ilog.write ("============================================\n"); |
| ilog.write (res[1]) |
| ilog.write ("\n"); |
| ilog.close() |
| if os.path.exists (src_file): |
| ilog = open(src_file+".log","a") |
| ilog.write (message + " for " +inc_file + ":\n\n"); |
| ilog.write ("============================================\n"); |
| ilog.write (res[1]) |
| ilog.write ("\n"); |
| ilog.close() |
| |
| # Given a sequence where : |
| # #include "tm.h" |
| # #include "target.h" // includes tm.h |
| |
| # target.h was required, and when attempting to remove tm.h we'd see that |
| # all the macro defintions are "required" since they all look like: |
| # #ifndef HAVE_blah |
| # #define HAVE_blah |
| # endif |
| |
| # when target.h was found to be required, tm.h will be tagged as included. |
| # so when we get this far, we know we dont have to check the macros for |
| # tm.h since we know it is already been included. |
| |
| if inc_file_norm not in header_seen: |
| iinfo = get_header_info (inc_file, spath) |
| newlist = all_headers (inc_file_norm) |
| if ii_path(iinfo) == "build" and not target_dict: |
| keep = True |
| text = message + " : Will not remove a build file without some targets." |
| print text |
| ilog = open(src_file+".log","a") |
| ilog.write (text +"\n") |
| ilog.write ("============================================\n"); |
| ilog.close() |
| ilog = open("reduce-headers-kept.log","a") |
| ilog.write (src_file + " " + text +"\n") |
| ilog.close() |
| else: |
| newlist = list() |
| if not keep and inc_file_norm not in header_seen: |
| # now look for any macro requirements. |
| for h in newlist: |
| if not h in header_seen: |
| if header_dict.get(h): |
| defined = ii_macro_define (header_dict[h]) |
| for dep in defined: |
| if dep in src_requires and dep not in ignore_list: |
| keep = True; |
| text = message + ", but must keep " + inc_file + " because it provides " + dep |
| if because.get(dep) != None: |
| text = text + " Possibly required by " + because[dep] |
| print text |
| ilog = open(inc_file+".log","a") |
| ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n") |
| ilog.write ("============================================\n"); |
| ilog.close() |
| ilog = open(src_file+".log","a") |
| ilog.write (text +"\n") |
| ilog.write ("============================================\n"); |
| ilog.close() |
| ilog = open("reduce-headers-kept.log","a") |
| ilog.write (src_file + " " + text +"\n") |
| ilog.close() |
| if verbose: |
| verbose.write (text + "\n") |
| |
| if keep: |
| # add all headers 'consumes' to src_requires list, and mark as seen |
| for h in newlist: |
| if not h in header_seen: |
| header_seen.append (h) |
| if header_dict.get(h): |
| consume = ii_macro_consume (header_dict[h]) |
| for dep in consume: |
| if dep not in src_requires: |
| src_requires.add (dep) |
| if because.get(dep) == None: |
| because[dep] = inc_file |
| |
| src_tmp = copy.deepcopy (src_data) |
| else: |
| print message + " --> removing " + inc_file + "\n" |
| rmcount += 1 |
| if verbose: |
| verbose.write (message + " --> removing " + inc_file + "\n") |
| if remove_count.get(inc_file) == None: |
| remove_count[inc_file] = 1 |
| else: |
| remove_count[inc_file] += 1 |
| src_data = copy.deepcopy (src_tmp) |
| except: |
| print "Interuption: restoring original file" |
| out = open(src_file, "w") |
| for line in src_orig: |
| out.write (line) |
| out.close() |
| raise |
| |
| # copy current version, since it is the "right" one now. |
| out = open(src_file, "w") |
| for line in src_data: |
| out.write (line) |
| out.close() |
| |
| # Try a final host bootstrap build to make sure everything is kosher. |
| if hostbuild: |
| res = get_make_output (build_dir, "all") |
| rc = res[0] |
| if (rc != 0): |
| # host build failed! return to original version |
| print "Error: " + src_file + " Failed to bootstrap at end!!! restoring." |
| print " Bad version at " + src_file + ".bad" |
| os.rename (src_file, src_file + ".bad") |
| out = open(src_file, "w") |
| for line in src_orig: |
| out.write (line) |
| out.close() |
| return src_file + ": failed to build after reduction. Restored original" |
| |
| if src_data == src_orig: |
| summary = src_file + ": No change." |
| else: |
| summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed." |
| print summary |
| return summary |
| |
| only_h = list () |
| ignore_cond = False |
| |
| usage = False |
| src = list() |
| only_targs = list () |
| for x in sys.argv[1:]: |
| if x[0:2] == "-b": |
| build_dir = x[2:] |
| elif x[0:2] == "-f": |
| fn = normalize_header (x[2:]) |
| if fn not in only_h: |
| only_h.append (fn) |
| elif x[0:2] == "-h": |
| usage = True |
| elif x[0:2] == "-d": |
| ignore_cond = True |
| elif x[0:2] == "-D": |
| ignore_list.append(x[2:]) |
| elif x[0:2] == "-T": |
| only_targs.append(x[2:]) |
| elif x[0:2] == "-t": |
| target_dir = x[2:] |
| elif x[0] == "-": |
| print "Error: Unrecognized option " + x |
| usgae = True |
| else: |
| if not os.path.exists (x): |
| print "Error: specified file " + x + " does not exist." |
| usage = True |
| else: |
| src.append (x) |
| |
| if target_dir: |
| build_target_dict (target_dir, only_targs) |
| |
| if build_dir == "" and target_dir == "": |
| print "Error: Must specify a build directory, and/or a target directory." |
| usage = True |
| |
| if build_dir and not os.path.exists (build_dir): |
| print "Error: specified build directory does not exist : " + build_dir |
| usage = True |
| |
| if target_dir and not os.path.exists (target_dir): |
| print "Error: specified target directory does not exist : " + target_dir |
| usage = True |
| |
| if usage: |
| print "Attempts to remove extraneous include files from source files." |
| print " " |
| print "Should be run from the main gcc source directory, and works on a target" |
| print "directory, as we attempt to make the 'all' target." |
| print " " |
| print "By default, gcc-reorder-includes is run on each file before attempting" |
| print "to remove includes. this removes duplicates and puts some headers in a" |
| print "canonical ordering" |
| print " " |
| print "The build directory should be ready to compile via make. Time is saved" |
| print "if the build is already complete, so that only changes need to be built." |
| print " " |
| print "Usage: [options] file1.c [file2.c] ... [filen.c]" |
| print " -bdir : the root build directory to attempt buiding .o files." |
| print " -tdir : the target build directory" |
| print " -d : Ignore conditional macro dependencies." |
| print " " |
| print " -Dmacro : Ignore a specific macro for dependencies" |
| print " -Ttarget : Only consider target in target directory." |
| print " -fheader : Specifies a specific .h file to be considered." |
| print " " |
| print " -D, -T, and -f can be specified mulitple times and are aggregated." |
| print " " |
| print " The original file will be in filen.bak" |
| print " " |
| sys.exit (0) |
| |
| if only_h: |
| print "Attempting to remove only these files:" |
| for x in only_h: |
| print x |
| print " " |
| |
| logfile = open("reduce-headers.log","w") |
| |
| for x in src: |
| msg = try_to_remove (x, only_h, logfile) |
| ilog = open("reduce-headers.sum","a") |
| ilog.write (msg + "\n") |
| ilog.close() |
| |
| ilog = open("reduce-headers.sum","a") |
| ilog.write ("===============================================================\n") |
| for x in remove_count: |
| msg = x + ": Removed " + str(remove_count[x]) + " times." |
| print msg |
| logfile.write (msg + "\n") |
| ilog.write (msg + "\n") |
| |
| |
| |
| |
| |
