contrib/header-tools/reduce-headers

#! /usr/bin/python2
import os.path
import sys
import shlex
import re
import tempfile
import copy

from headerutils import *

requires = { }
provides = { }

no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]

# These targets are the ones which provide "coverage".  Typically, if any
# target is going to fail compilation, it's one of these.  This was determined
# during the initial runs of reduce-headers... On a full set of target builds,
# every failure which occured was triggered by one of these.  
# This list is used during target-list construction simply to put any of these
# *first* in the candidate list, increasing the probability that a failure is 
# found quickly.
target_priority = [
    "aarch64-linux-gnu",
    "arm-netbsdelf",
    "c6x-elf",
    "epiphany-elf",
    "hppa2.0-hpux10.1",
    "i686-mingw32crt",
    "i686-pc-msdosdjgpp",
    "mipsel-elf",
    "powerpc-eabisimaltivec",
    "rs6000-ibm-aix5.1.0",
    "sh-superh-elf",
    "sparc64-elf"
]


target_dir = ""
build_dir = ""
ignore_list = list()
target_builds = list()

target_dict = { }
header_dict = { }
search_path = [ ".", "../include", "../libcpp/include" ]

remove_count = { }


# Given a header name, normalize it.  ie.  cp/cp-tree.h could be in gcc, while
# the same header could be referenced from within the cp subdirectory as
# just cp-tree.h
# for now, just assume basenames are unique

def normalize_header (header):
  return os.path.basename (header)


# Adds a header file and its sub-includes to the global dictionary if they
# aren't already there.  Specify s_path since different build directories may
# append themselves on demand to the global list.
# return entry for the specified header, knowing all sub entries are completed

def get_header_info (header, s_path):
  global header_dict
  global empty_iinfo
  process_list = list ()
  location = ""
  bname = ""
  bname_iinfo = empty_iinfo
  for path in s_path:
    if os.path.exists (path + "/" + header):
      location = path + "/" + header
      break

  if location:
    bname = normalize_header (location)
    if header_dict.get (bname):
      bname_iinfo = header_dict[bname]
      loc2 = ii_path (bname_iinfo)+ "/" + bname
      if loc2[:2] == "./":
        loc2 = loc2[2:]
      if location[:2] == "./":
        location = location[2:]
      if loc2 != location:
        # Don't use the cache if it isnt the right one.
        bname_iinfo = process_ii_macro (location)
      return bname_iinfo

    bname_iinfo = process_ii_macro (location)
    header_dict[bname] = bname_iinfo
    # now decend into the include tree
    for i in ii_include_list (bname_iinfo):
      get_header_info (i, s_path)
  else:
    # if the file isnt in the source directories, look in the build and target
    # directories. If it is here, then aggregate all the versions.
    location = build_dir + "/gcc/" + header
    build_inc = target_inc = False
    if os.path.exists (location):
      build_inc = True
    for x in target_dict:
      location = target_dict[x] + "/gcc/" + header
      if os.path.exists (location):
        target_inc = True
        break

    if (build_inc or target_inc):
      bname = normalize_header(header)
      defines = set()
      consumes = set()
      incl = set()
      if build_inc:
        iinfo = process_ii_macro (build_dir + "/gcc/" + header)
        defines = set (ii_macro_define (iinfo))
        consumes = set (ii_macro_consume (iinfo))
        incl = set (ii_include_list (iinfo))

      if (target_inc):
        for x in target_dict:
          location = target_dict[x] + "/gcc/" + header
          if os.path.exists (location):
            iinfo = process_ii_macro (location)
            defines.update (ii_macro_define (iinfo))
            consumes.update (ii_macro_consume (iinfo))
            incl.update (ii_include_list (iinfo))

      bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())

      header_dict[bname] = bname_iinfo
      for i in incl:
        get_header_info (i, s_path)

  return bname_iinfo


# return a list of all headers brought in by this header
def all_headers (fname):
  global header_dict
  headers_stack = list()
  headers_list = list()
  if header_dict.get (fname) == None:
    return list ()
  for y in ii_include_list (header_dict[fname]):
    headers_stack.append (y)

  while headers_stack:
    h = headers_stack.pop ()
    hn = normalize_header (h)
    if hn not in headers_list:
      headers_list.append (hn)
      if header_dict.get(hn):
        for y in ii_include_list (header_dict[hn]):
          if normalize_header (y) not in headers_list:
            headers_stack.append (y)

  return headers_list




# Search bld_dir for all target tuples, confirm that they have a build path with
# bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
# target tuple..

def build_target_dict (bld_dir, just_these):
  global target_dict
  target_doct = { }
  error = False
  if os.path.exists (bld_dir):
    if just_these:
      ls = just_these
    else:
      ls = os.listdir(bld_dir)
    for t in ls:
      if t.find("-") != -1:
        target = t.strip()
        tpath = bld_dir + "/" + target
        if not os.path.exists (tpath + "/gcc"):
          print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc"
          error = True
        else:
          target_dict[target] = tpath

  if error:
    target_dict = { }

def get_obj_name (src_file):
  if src_file[-2:] == ".c":
    return src_file.replace (".c", ".o")
  elif src_file[-3:] == ".cc":
    return src_file.replace (".cc", ".o")
  return ""

def target_obj_exists (target, obj_name):
  global target_dict
  # look in a subdir if src has a subdir, then check gcc base directory.
  if target_dict.get(target):
    obj = target_dict[target] + "/gcc/" + obj_name
    if not os.path.exists (obj):
      obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
    if os.path.exists (obj):
      return True
  return False
 
# Given a src file, return a list of targets which may build this file.
def find_targets (src_file):
  global target_dict
  targ_list = list()
  obj_name = get_obj_name (src_file)
  if not obj_name:
    print "Error: " + src_file + " - Cannot determine object name."
    return list()

  # Put the high priority targets which tend to trigger failures first
  for target in target_priority:
    if target_obj_exists (target, obj_name):
      targ_list.append ((target, target_dict[target]))

  for target in target_dict:
    if target not in target_priority and target_obj_exists (target, obj_name):
      targ_list.append ((target, target_dict[target]))
        
  return targ_list


def try_to_remove (src_file, h_list, verbose):
  global target_dict
  global header_dict
  global build_dir

  # build from scratch each time
  header_dict = { }
  summary = ""
  rmcount = 0

  because = { }
  src_info = process_ii_macro_src (src_file)
  src_data = ii_src (src_info)
  if src_data:
    inclist = ii_include_list_non_cond (src_info)
    # work is done if there are no includes to check
    if not inclist:
      return src_file + ": No include files to attempt to remove"

    # work on the include list in reverse.
    inclist.reverse()

    # Get the target list 
    targ_list = list()
    targ_list = find_targets (src_file)

    spath = search_path
    if os.path.dirname (src_file):
      spath.append (os.path.dirname (src_file))

    hostbuild = True
    if src_file.find("config/") != -1:
      # config files dont usually build on the host
      hostbuild = False
      obn = get_obj_name (os.path.basename (src_file))
      if obn and os.path.exists (build_dir + "/gcc/" + obn):
        hostbuild = True
      if not target_dict:
        summary = src_file + ": Target builds are required for config files.  None found."
        print summary
        return summary
      if not targ_list:
        summary =src_file + ": Cannot find any targets which build this file."
        print summary
        return summary

    if hostbuild:
      # confirm it actually builds before we do anything
      print "Confirming source file builds"
      res = get_make_output (build_dir + "/gcc", "all")
      if res[0] != 0:
        message = "Error: " + src_file + " does not build currently."
        summary = src_file + " does not build on host."
        print message
        print res[1]
        if verbose:
          verbose.write (message + "\n")
          verbose.write (res[1]+ "\n")
        return summary

    src_requires = set (ii_macro_consume (src_info))
    for macro in src_requires:
      because[macro] = src_file
    header_seen = list ()

    os.rename (src_file, src_file + ".bak")
    src_orig = copy.deepcopy (src_data)
    src_tmp = copy.deepcopy (src_data)

    try:
      # process the includes from bottom to top.  This is because we know that
      # later includes have are known to be needed, so any dependency from this 
      # header is a true dependency
      for inc_file in inclist:
        inc_file_norm = normalize_header (inc_file)
        
        if inc_file in no_remove:
          continue
        if len (h_list) != 0 and inc_file_norm not in h_list:
          continue
        if inc_file_norm[0:3] == "gt-":
          continue
        if inc_file_norm[0:6] == "gtype-":
          continue
        if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
          continue
             
        lookfor = ii_src_line(src_info)[inc_file]
        src_tmp.remove (lookfor)
        message = "Trying " + src_file + " without " + inc_file
        print message
        if verbose:
          verbose.write (message + "\n")
        out = open(src_file, "w")
        for line in src_tmp:
          out.write (line)
        out.close()
          
        keep = False
        if hostbuild:
          res = get_make_output (build_dir + "/gcc", "all")
        else:
          res = (0, "")

        rc = res[0]
        message = "Passed Host build"
        if (rc != 0):
          # host build failed
          message  = "Compilation failed:\n";
          keep = True
        else:
          if targ_list:
            objfile = get_obj_name (src_file)
            t1 = targ_list[0]
            if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
              res = get_make_output_parallel (targ_list, objfile, 0)
            else:
              res = get_make_output_parallel (targ_list, "all-gcc", 0)
            rc = res[0]
            if rc != 0:
              message = "Compilation failed on TARGET : " + res[2]
              keep = True
            else:
              message = "Passed host and target builds"

        if keep:
          print message + "\n"

        if (rc != 0):
          if verbose:
            verbose.write (message + "\n");
            verbose.write (res[1])
            verbose.write ("\n");
            if os.path.exists (inc_file):
              ilog = open(inc_file+".log","a")
              ilog.write (message + " for " + src_file + ":\n\n");
              ilog.write ("============================================\n");
              ilog.write (res[1])
              ilog.write ("\n");
              ilog.close()
            if os.path.exists (src_file):
              ilog = open(src_file+".log","a")
              ilog.write (message + " for " +inc_file + ":\n\n");
              ilog.write ("============================================\n");
              ilog.write (res[1])
              ilog.write ("\n");
              ilog.close()

        # Given a sequence where :
        # #include "tm.h"
        # #include "target.h"  // includes tm.h

        # target.h was required, and when attempting to remove tm.h we'd see that
        # all the macro defintions are "required" since they all look like:
        # #ifndef HAVE_blah
        # #define HAVE_blah
        # endif

        # when target.h was found to be required, tm.h will be tagged as included.
        # so when we get this far, we know we dont have to check the macros for
        # tm.h since we know it is already been included.

        if inc_file_norm not in header_seen:
          iinfo = get_header_info (inc_file, spath)
          newlist = all_headers (inc_file_norm)
          if ii_path(iinfo) == "build" and not target_dict:
            keep = True
            text = message + " : Will not remove a build file without some targets."
            print text
            ilog = open(src_file+".log","a")
            ilog.write (text +"\n")
            ilog.write ("============================================\n");
            ilog.close()
            ilog = open("reduce-headers-kept.log","a")
            ilog.write (src_file + " " + text +"\n")
            ilog.close()
        else:
          newlist = list()
        if not keep and inc_file_norm not in header_seen:
          # now look for any macro requirements.
          for h in newlist:
            if not h in header_seen:
              if header_dict.get(h):
                defined = ii_macro_define (header_dict[h])
                for dep in defined:
                  if dep in src_requires and dep not in ignore_list:
                    keep = True;
                    text = message + ", but must keep " + inc_file + " because it provides " + dep 
                    if because.get(dep) != None:
                      text = text + " Possibly required by " + because[dep]
                    print text
                    ilog = open(inc_file+".log","a")
                    ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
                    ilog.write ("============================================\n");
                    ilog.close()
                    ilog = open(src_file+".log","a")
                    ilog.write (text +"\n")
                    ilog.write ("============================================\n");
                    ilog.close()
                    ilog = open("reduce-headers-kept.log","a")
                    ilog.write (src_file + " " + text +"\n")
                    ilog.close()
                    if verbose:
                      verbose.write (text + "\n")

        if keep:
          # add all headers 'consumes' to src_requires list, and mark as seen
          for h in newlist:
            if not h in header_seen:
              header_seen.append (h)
              if header_dict.get(h):
                consume = ii_macro_consume (header_dict[h])
                for dep in consume:
                  if dep not in src_requires:
                    src_requires.add (dep)
                    if because.get(dep) == None:
                      because[dep] = inc_file

          src_tmp = copy.deepcopy (src_data)
        else:
          print message + "  --> removing " + inc_file + "\n"
          rmcount += 1
          if verbose:
            verbose.write (message + "  --> removing " + inc_file + "\n")
          if remove_count.get(inc_file) == None:
            remove_count[inc_file] = 1
          else:
            remove_count[inc_file] += 1
          src_data = copy.deepcopy (src_tmp)
    except:
      print "Interuption: restoring original file"
      out = open(src_file, "w")
      for line in src_orig:
        out.write (line)
      out.close()
      raise

    # copy current version, since it is the "right" one now.
    out = open(src_file, "w")
    for line in src_data:
      out.write (line)
    out.close()
    
    # Try a final host bootstrap build to make sure everything is kosher.
    if hostbuild:
      res = get_make_output (build_dir, "all")
      rc = res[0]
      if (rc != 0):
        # host build failed! return to original version
        print "Error: " + src_file + " Failed to bootstrap at end!!! restoring."
        print "        Bad version at " + src_file + ".bad"
        os.rename (src_file, src_file + ".bad")
        out = open(src_file, "w")
        for line in src_orig:
          out.write (line)
        out.close()
        return src_file + ": failed to build after reduction.  Restored original"

    if src_data == src_orig:
      summary = src_file + ": No change."
    else:
      summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
  print summary
  return summary

only_h = list ()
ignore_cond = False

usage = False
src = list()
only_targs = list ()
for x in sys.argv[1:]:
  if x[0:2] == "-b":
    build_dir = x[2:]
  elif x[0:2] == "-f":
    fn = normalize_header (x[2:])
    if fn not in only_h:
      only_h.append (fn)
  elif x[0:2] == "-h":
    usage = True
  elif x[0:2] == "-d":
    ignore_cond = True
  elif x[0:2] == "-D":
    ignore_list.append(x[2:])
  elif x[0:2] == "-T":
    only_targs.append(x[2:])
  elif x[0:2] == "-t":
    target_dir = x[2:]
  elif x[0] == "-":
    print "Error:  Unrecognized option " + x
    usgae = True
  else:
    if not os.path.exists (x):
      print "Error: specified file " + x + " does not exist."
      usage = True
    else:
      src.append (x)

if target_dir:
  build_target_dict (target_dir, only_targs)

if build_dir == "" and target_dir == "":
  print "Error: Must specify a build directory, and/or a target directory."
  usage = True

if build_dir and not os.path.exists (build_dir):
    print "Error: specified build directory does not exist : " + build_dir
    usage = True

if target_dir and not os.path.exists (target_dir):
    print "Error: specified target directory does not exist : " + target_dir
    usage = True

if usage:
  print "Attempts to remove extraneous include files from source files."
  print " "
  print "Should be run from the main gcc source directory, and works on a target"
  print "directory, as we attempt to make the 'all' target."
  print " "
  print "By default, gcc-reorder-includes is run on each file before attempting"
  print "to remove includes. this removes duplicates and puts some headers in a"
  print "canonical ordering"
  print " "
  print "The build directory should be ready to compile via make. Time is saved"
  print "if the build is already complete, so that only changes need to be built."
  print " "
  print "Usage: [options] file1.c [file2.c] ... [filen.c]"
  print "      -bdir    : the root build directory to attempt buiding .o files."
  print "      -tdir    : the target build directory"
  print "      -d       : Ignore conditional macro dependencies."
  print " "
  print "      -Dmacro  : Ignore a specific macro for dependencies"
  print "      -Ttarget : Only consider target in target directory."
  print "      -fheader : Specifies a specific .h file to be considered."
  print " "
  print "      -D, -T, and -f can be specified mulitple times and are aggregated."
  print " "
  print "  The original file will be in filen.bak"
  print " "
  sys.exit (0)
 
if only_h:
  print "Attempting to remove only these files:"
  for x in only_h:
    print x
  print " "

logfile = open("reduce-headers.log","w")

for x in src:
  msg = try_to_remove (x, only_h, logfile)
  ilog = open("reduce-headers.sum","a")
  ilog.write (msg + "\n")
  ilog.close()

ilog = open("reduce-headers.sum","a")
ilog.write ("===============================================================\n")
for x in remove_count:
  msg = x + ": Removed " + str(remove_count[x]) + " times."
  print msg
  logfile.write (msg + "\n")
  ilog.write (msg + "\n")