From 5a33d6ab06fa2cc94cdd864ff238d2c58d6b7e4e Mon Sep 17 00:00:00 2001 From: Chinmay Garde Date: Fri, 16 Aug 2019 13:01:07 -0700 Subject: [PATCH] Track toolchain headers to detect toolchain updates instead of macro. (#302) To detect and rebuild all TUs on toolchain updates, we inherited a rule from Chromium whereby the Git SHA of the //buildtools repo would be used to insert a preprocessor define in each TUs build invocation. But we don't use a //buildtools as a repo anymore. Instead buildtools comes from CIPD. So the script now picks the //buildroot revision instead. But we don't specify the toolchain revision in that repo either though. It is instead specified in the engine. So if the engine updates the toolchain, only the next (unrelated) //buildroot rev will ensure all old TUs are rebuilt unless the `out` directory is cleared on each invocation. This patch replaces the old buggy technique with one where the system headers (that include headers in buildtools) are tracked in addition to user headers. When the toolchain updates, these "system" header timestamps will be invalidated and the old artifacts invalidate. The simple change of just tracking the //flutter (where the toolchain SHA is present) would make it so that each engine change (frequent) would rebuild all TUs instead of just each buildtools change (rarer). Also, this technique would leave us susceptible to the kind of issue that led us in this situation in the first place. I also just searched and replaced all calls to track user dependencies with ones to track system + user dependencies. Not all toolchain are actually being used. We should clean these up in a separate patch. --- build/config/compiler/BUILD.gn | 14 -------------- build/toolchain/custom/BUILD.gn | 6 +++--- build/toolchain/fuchsia/BUILD.gn | 6 +++--- build/toolchain/gcc_toolchain.gni | 6 +++--- build/toolchain/mac/BUILD.gn | 10 +++++----- build/toolchain/nacl/BUILD.gn | 4 ++-- 6 files changed, 16 insertions(+), 30 deletions(-) diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index 5419349c5e2..a2367e9d80e 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn @@ -66,20 +66,6 @@ config("compiler") { ldflags = [] defines = [] - # We want to force a recompile and relink of the world whenever our toolchain - # changes since artifacts from an older version of the toolchain may or may - # not be compatible with newer ones. To achieve this, we insert a synthetic - # define into the compile line. We only do this for toolchains that we use - # from buildtools. - toolchain_version = - exec_script("//build/util/lastchange.py", - [ - "--source-dir=" + rebase_path("//buildtools"), - "--revision-only", - ], - "trim string") - defines = [ "TOOLCHAIN_VERSION=$toolchain_version" ] - # In general, Windows is totally different, but all the other builds share # some common GCC configuration. This section sets up Windows and the common # GCC flags, and then we handle the other non-Windows platforms specifically diff --git a/build/toolchain/custom/BUILD.gn b/build/toolchain/custom/BUILD.gn index 79d3df5d69a..a90afa31bcf 100644 --- a/build/toolchain/custom/BUILD.gn +++ b/build/toolchain/custom/BUILD.gn @@ -29,7 +29,7 @@ toolchain("custom") { tool("cc") { depfile = "{{output}}.d" - command = "$cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" + command = "$cc -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "CC {{output}}" outputs = [ @@ -39,7 +39,7 @@ toolchain("custom") { tool("cxx") { depfile = "{{output}}.d" - command = "$cxx -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}" + command = "$cxx -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "CXX {{output}}" outputs = [ @@ -49,7 +49,7 @@ toolchain("custom") { tool("asm") { depfile = "{{output}}.d" - command = "$cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" + command = "$cc -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "ASM {{output}}" outputs = [ diff --git a/build/toolchain/fuchsia/BUILD.gn b/build/toolchain/fuchsia/BUILD.gn index 7e95237f57b..3753a0d35d9 100644 --- a/build/toolchain/fuchsia/BUILD.gn +++ b/build/toolchain/fuchsia/BUILD.gn @@ -41,7 +41,7 @@ toolchain("fuchsia") { tool("cc") { depfile = "{{output}}.d" - command = "$goma_prefix $cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" + command = "$goma_prefix $cc -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "CC {{output}}" outputs = [ @@ -51,7 +51,7 @@ toolchain("fuchsia") { tool("cxx") { depfile = "{{output}}.d" - command = "$goma_prefix $cxx -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}" + command = "$goma_prefix $cxx -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "CXX {{output}}" outputs = [ @@ -61,7 +61,7 @@ toolchain("fuchsia") { tool("asm") { depfile = "{{output}}.d" - command = "$goma_prefix $cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" + command = "$goma_prefix $cc -MD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "ASM {{output}}" outputs = [ diff --git a/build/toolchain/gcc_toolchain.gni b/build/toolchain/gcc_toolchain.gni index 337a5445e61..6ba99ec6bc8 100644 --- a/build/toolchain/gcc_toolchain.gni +++ b/build/toolchain/gcc_toolchain.gni @@ -119,7 +119,7 @@ template("gcc_toolchain") { tool("cc") { depfile = "{{output}}.d" - command = "$cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" + command = "$cc -MD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "CC {{output}}" outputs = [ @@ -129,7 +129,7 @@ template("gcc_toolchain") { tool("cxx") { depfile = "{{output}}.d" - command = "$cxx -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} $coverage_flags -c {{source}} -o {{output}}" + command = "$cxx -MD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "CXX {{output}}" outputs = [ @@ -140,7 +140,7 @@ template("gcc_toolchain") { tool("asm") { # For GCC we can just use the C compiler to compile assembly. depfile = "{{output}}.d" - command = "$cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" + command = "$cc -MD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "ASM {{output}}" outputs = [ diff --git a/build/toolchain/mac/BUILD.gn b/build/toolchain/mac/BUILD.gn index b4ed4d52cb3..c3209878a5a 100644 --- a/build/toolchain/mac/BUILD.gn +++ b/build/toolchain/mac/BUILD.gn @@ -82,7 +82,7 @@ template("mac_toolchain") { tool("cc") { depfile = "{{output}}.d" - command = "$goma_prefix $cc -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" + command = "$goma_prefix $cc -MD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "CC {{output}}" outputs = [ @@ -92,7 +92,7 @@ template("mac_toolchain") { tool("cxx") { depfile = "{{output}}.d" - command = "$goma_prefix $cxx -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_cc}} $coverage_flags -c {{source}} -o {{output}}" + command = "$goma_prefix $cxx -MD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_cc}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "CXX {{output}}" outputs = [ @@ -103,7 +103,7 @@ template("mac_toolchain") { tool("asm") { # For GCC we can just use the C compiler to compile assembly. depfile = "{{output}}.d" - command = "$goma_prefix $cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" + command = "$goma_prefix $cc -MD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}" depsformat = "gcc" description = "ASM {{output}}" outputs = [ @@ -113,7 +113,7 @@ template("mac_toolchain") { tool("objc") { depfile = "{{output}}.d" - command = "$goma_prefix $cxx -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} {{cflags_objc}} $coverage_flags -c {{source}} -o {{output}}" + command = "$goma_prefix $cxx -MD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_c}} {{cflags_objc}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "OBJC {{output}}" outputs = [ @@ -123,7 +123,7 @@ template("mac_toolchain") { tool("objcxx") { depfile = "{{output}}.d" - command = "$goma_prefix $cxx -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_cc}} {{cflags_objcc}} $coverage_flags -c {{source}} -o {{output}}" + command = "$goma_prefix $cxx -MD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $lto_flags {{cflags}} {{cflags_cc}} {{cflags_objcc}} $coverage_flags -c {{source}} -o {{output}}" depsformat = "gcc" description = "OBJCXX {{output}}" outputs = [ diff --git a/build/toolchain/nacl/BUILD.gn b/build/toolchain/nacl/BUILD.gn index 5fa637cb9f9..b290cf38e7d 100644 --- a/build/toolchain/nacl/BUILD.gn +++ b/build/toolchain/nacl/BUILD.gn @@ -9,14 +9,14 @@ toolchain("x86_newlib") { ld = toolprefix + "g++" tool("cc") { - command = "$cc -MMD -MF \$out.d \$defines \$includes \$cflags \$cflags_c -c \$in -o \$out" + command = "$cc -MD -MF \$out.d \$defines \$includes \$cflags \$cflags_c -c \$in -o \$out" description = "CC(NaCl x86 Newlib) \$out" depfile = "\$out.d" depsformat = "gcc" } tool("cxx") { # cflags_pch_cc - command = "$cxx -MMD -MF \$out.d \$defines \$includes \$cflags \$cflags_cc -c \$in -o \$out" + command = "$cxx -MD -MF \$out.d \$defines \$includes \$cflags \$cflags_cc -c \$in -o \$out" description = "CXX(NaCl x86 Newlib) \$out" depfile = "\$out.d" depsformat = "gcc"