Prepare buildroot for working with Fuchsia SDK defined targets. (#239)

This prepares the buildroot to stamp GN targets for Fuchsia SDK defined targets.

GN templates have been created in //build/fuchsia/fuchsia.gni that
can read the JSON metadata that describes the vended SDK.

GN targets that describe all known SDK parts (as described in the SDK
manifest) can be instantiated in one shot by defining a top level fuchsia_sdk
target. This stamping needs to happen once and can be done anywhere (currenly
in //build/fuchsia/BUILD.gn). Once stamped, targets that depend on
specific Fuchsia SDK parts need to explicitly depend on them by using the
name of that part. For example, depending on the :fuchsia.images part of the
SDK will generate headers for all fidl files in that part, compile any source
libraries and link the required dynamic libraries into the executable. Unlike regular
targets, this target does not exist in a GN file. The name must be looked up
from JSON manifest. This scheme seems to work well but is not resilient
to part name collision.

So that SDK and non-SDK Fuchsia builds may co-exist till the migration is done,
the is_fuchsia_sdk GN variable and FUCHSIA_SDK preprocessor defines are
set in all Fuchsia SDK builds. This allows code to target both variants. It is
the hope that the non SDK variant will be turned down and the Flutter team
assumes ownership of this component.

.gitignore

@@ -24,3 +24,6 @@ Thumbs.db
 /flutter/
 /ios_tools/
 /out/
+
+# This is where the gclient hook downloads the Fuchsia SDK and toolchain.
+/fuchsia/

build/config/BUILD.gn

@@ -221,3 +221,8 @@ config("precompiled_headers") {
     cflags = [ "/FI$precompiled_header" ]
   }
 }
+
+config("symbol_visibility_hidden") {
+  # Empty but present because this is also present in other buildroots.
+  # The same config from gcc/BUILD.gn is used instead.
+}

build/config/BUILDCONFIG.gn

@@ -259,21 +259,24 @@ if (current_os == "win") {
   is_mac = false
   is_posix = true
   is_win = false
-} else if (current_os == "fuchsia") {
+} else if (current_os == "fuchsia" || target_os == "fuchsia") {
   is_android = false
   is_chromeos = false
+  is_fnl = false
   is_fuchsia = true
   is_fuchsia_host = false
   is_ios = false
-  is_linux = true
+  is_linux = false
   is_mac = false
   is_posix = true
   is_win = false
 }
 
+import("//build/fuchsia/fuchsia.gni")
+
 # Some library targets may be built as different type depending on the target
 # platform. This variable specifies the default library type for each target.
-if (is_fuchsia) {
+if (is_fuchsia && !is_fuchsia_sdk) {
   default_library_type = "shared_library"
 } else {
   default_library_type = "static_library"
@@ -435,6 +438,7 @@ if (is_posix) {
   _native_compiler_configs += [
     "//build/config/gcc:no_exceptions",
     "//build/config/gcc:symbol_visibility_hidden",
+    "//build/config:symbol_visibility_hidden",
   ]
 }
 
@@ -628,6 +632,15 @@ if (custom_toolchain != "") {
   }
 }
 
+if (is_fuchsia_sdk) {
+  if (host_os == "mac") {
+    host_toolchain = "//build/toolchain/mac:clang_$host_cpu"
+  } else {
+    host_toolchain = "//build/toolchain/linux:clang_$host_cpu"
+  }
+  set_default_toolchain("//build/toolchain/fuchsia")
+}
+
 # Sets default dependencies for executable and shared_library targets.
 #
 # Variables

build/config/compiler/BUILD.gn

@@ -16,10 +16,10 @@ if (is_win) {
   import("//build/config/win/visual_studio_version.gni")
 }
 
-import("//build/toolchain/ccache.gni")
-import("//build/toolchain/clang.gni")
 import("//build/config/profiler.gni")
 import("//build/config/sanitizers/sanitizers.gni")
+import("//build/toolchain/ccache.gni")
+import("//build/toolchain/clang.gni")
 
 declare_args() {
   # Normally, Android builds are lightly optimized, even for debug builds, to
@@ -398,6 +398,8 @@ config("compiler") {
     cc_std = [ "-std=c++11" ]
   } else if (is_win) {
     cc_std = [ "/std:c++14" ]
+  } else if (is_fuchsia) {
+    cc_std = [ "-std=c++17" ]
   } else {
     cc_std = [ "-std=c++14" ]
   }
@@ -463,6 +465,12 @@ config("compiler") {
     }
   }
 
+  # Fuchsia-specific flags setup.
+  # -----------------------------
+  if (is_fuchsia_sdk) {
+    defines += [ "FUCHSIA_SDK" ]
+  }
+
   asmflags = cflags
 }
 

build/config/gcc/BUILD.gn

@@ -18,6 +18,7 @@ config("symbol_visibility_hidden") {
   # Note that -fvisibility-inlines-hidden is set globally in the compiler
   # config since that can almost always be applied.
   if (!enable_profiling) {
+    defines = [ "_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS" ]
     cflags = [ "-fvisibility=hidden" ]
   }
 }

build/fuchsia/BUILD.gn

@@ -0,0 +1,11 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/fuchsia/fuchsia.gni")
+
+if (is_fuchsia_sdk) {
+  fuchsia_sdk("sdk") {
+    meta = "$fuchsia_sdk_path/meta/manifest.json"
+  }
+}

build/fuchsia/fidl_gen_cpp.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import collections
+import json
+import os
+import subprocess
+import sys
+
+def GetFIDLFilesRecursive(libraries, sdk_base, path):
+  with open(path) as json_file:
+    parsed = json.load(json_file)
+    result = []
+    deps =  parsed['deps']
+    for dep in deps:
+      dep_meta_json = os.path.abspath('%s/fidl/%s/meta.json' % (sdk_base, dep))
+      GetFIDLFilesRecursive(libraries, sdk_base, dep_meta_json)
+    libraries[parsed['name']] = result + parsed['sources']
+
+def GetFIDLFilesByLibraryName(sdk_base, root):
+  libraries = collections.OrderedDict()
+  GetFIDLFilesRecursive(libraries, sdk_base, root)
+  return libraries
+
+def main():
+  parser = argparse.ArgumentParser();
+
+  parser.add_argument('--fidlc-bin', dest='fidlc_bin', action='store', required=True)
+  parser.add_argument('--fidlgen-bin', dest='fidlgen_bin', action='store', required=True)
+  
+  parser.add_argument('--sdk-base', dest='sdk_base', action='store', required=True)
+  parser.add_argument('--root', dest='root', action='store', required=True)
+  parser.add_argument('--json', dest='json', action='store', required=True)
+  parser.add_argument('--include-base', dest='include_base', action='store', required=True)
+  parser.add_argument('--output-base-cc', dest='output_base_cc', action='store', required=True)
+  parser.add_argument('--output-c-header', dest='output_header_c', action='store', required=True)
+  parser.add_argument('--output-c-tables', dest='output_c_tables', action='store', required=True)
+
+  args = parser.parse_args()
+
+  assert os.path.exists(args.fidlc_bin)
+  assert os.path.exists(args.fidlgen_bin)
+
+  fidl_files_by_name = GetFIDLFilesByLibraryName(args.sdk_base, args.root)
+
+  fidlc_command = [
+    args.fidlc_bin,
+    '--c-header',
+    args.output_header_c,
+    '--tables',
+    args.output_c_tables,
+    '--json',
+    args.json
+  ]
+
+  for _, fidl_files in fidl_files_by_name.iteritems():
+    fidlc_command.append('--files')
+    for fidl_file in fidl_files:
+      fidl_abspath = os.path.abspath('%s/%s' % (args.sdk_base, fidl_file))
+      fidlc_command.append(fidl_abspath)
+
+  subprocess.check_call(fidlc_command);
+
+  assert os.path.exists(args.json)
+
+  fidlgen_command = [
+    args.fidlgen_bin,
+    '-generators',
+    'cpp',
+    '-include-base',
+    args.include_base,
+    '-json',
+    args.json,
+    '-output-base',
+    args.output_base_cc
+  ]
+
+  subprocess.check_call(fidlgen_command)
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())

build/fuchsia/fuchsia.gni

@@ -0,0 +1,237 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+declare_args() {
+  fuchsia_sdk_path = "//fuchsia/sdk"
+  fuchsia_sdk = "//build/fuchsia"
+  is_fuchsia_sdk = false
+}
+
+template("fuchsia_sysroot") {
+  assert(defined(invoker.meta), "The meta.json file path must be specified.")
+
+  meta_json = read_file(invoker.meta, "json")
+
+  assert(meta_json.type == "sysroot")
+
+  meta_json_versions = meta_json.versions
+  defs = meta_json_versions.x64
+
+  _libs = []
+  _lib_dirs = []
+  _include_dirs = []
+
+  foreach(link_lib, defs.link_libs) {
+    if (link_lib != "arch/x64/sysroot/lib/Scrt1.o") {
+      _libs += [ "$fuchsia_sdk_path/$link_lib" ]
+    }
+  }
+
+  defs_include_dir = defs.include_dir
+  _include_dirs += [ "$fuchsia_sdk_path/$defs_include_dir" ]
+
+  config_name = "config_$target_name"
+  config(config_name) {
+    lib_dirs = _lib_dirs
+    libs = _libs
+    include_dirs = _include_dirs
+  }
+
+  group(target_name) {
+    public_configs = [ ":$config_name" ]
+  }
+}
+
+template("fuchsia_fidl_library") {
+  assert(defined(invoker.meta), "The meta.json file path must be specified.")
+
+  meta_json = read_file(invoker.meta, "json")
+
+  assert(meta_json.type == "fidl_library")
+
+  _deps = [ ":fidl_cpp" ]
+
+  library_name = string_replace(meta_json.name, "fuchsia.", "")
+  library_name_json = "$library_name.json"
+
+  foreach(dep, meta_json.deps) {
+    _deps += [ ":$dep" ]
+  }
+
+  config_name = "config_$target_name"
+  config(config_name) {
+    include_dirs = [ target_gen_dir ]
+  }
+
+  fidl_gen_target_name = "fidlgen_$target_name"
+  action(fidl_gen_target_name) {
+    script = "//build/fuchsia/fidl_gen_cpp.py"
+
+    library_name_slashes = string_replace(library_name, ".", "/")
+
+    inputs = [
+      invoker.meta,
+    ]
+
+    outputs = [
+      "$target_gen_dir/fuchsia/$library_name_slashes/c/fidl.h",
+      "$target_gen_dir/fuchsia/$library_name_slashes/cpp/fidl.h",
+      "$target_gen_dir/fuchsia/$library_name_slashes/cpp/fidl.cc",
+      "$target_gen_dir/fuchsia/$library_name_slashes/cpp/tables.cc",
+      "$target_gen_dir/fuchsia/$library_name_slashes/cpp/fidl_test_base.h",
+    ]
+
+    args = [
+      "--fidlc-bin",
+      rebase_path("$fuchsia_sdk_path/tools/fidlc"),
+      "--fidlgen-bin",
+      rebase_path("$fuchsia_sdk_path/tools/fidlgen"),
+      "--sdk-base",
+      rebase_path(fuchsia_sdk_path),
+      "--root",
+      rebase_path(invoker.meta),
+      "--json",
+      rebase_path("$target_gen_dir/$library_name_json"),
+      "--include-base",
+      rebase_path("$target_gen_dir"),
+      "--output-base-cc",
+      rebase_path("$target_gen_dir/fuchsia/$library_name_slashes/cpp/fidl"),
+      "--output-c-header",
+      rebase_path("$target_gen_dir/fuchsia/$library_name_slashes/c/fidl.h"),
+      "--output-c-tables",
+      rebase_path(
+          "$target_gen_dir/fuchsia/$library_name_slashes/cpp/tables.cc"),
+    ]
+  }
+
+  source_set(target_name) {
+    public_configs = [ ":$config_name" ]
+
+    sources = get_target_outputs(":$fidl_gen_target_name")
+
+    deps = [
+      ":$fidl_gen_target_name",
+    ]
+
+    public_deps = _deps
+  }
+}
+
+template("fuchsia_cc_source_library") {
+  assert(defined(invoker.meta), "The meta.json file path must be specified.")
+
+  meta_json = read_file(invoker.meta, "json")
+
+  assert(meta_json.type == "cc_source_library")
+
+  _include_dirs = []
+  _sources = []
+  _output_name = meta_json.name
+  _deps = []
+
+  meta_json_include_dir = meta_json.include_dir
+  _include_dirs += [ "$fuchsia_sdk_path/$meta_json_include_dir" ]
+
+  foreach(header, meta_json.headers) {
+    _sources += [ "$fuchsia_sdk_path/$header" ]
+  }
+
+  foreach(source, meta_json.sources) {
+    _sources += [ "$fuchsia_sdk_path/$source" ]
+  }
+
+  config_name = "config_$target_name"
+  config(config_name) {
+    include_dirs = _include_dirs
+  }
+
+  foreach(dep, meta_json.deps) {
+    _deps += [ ":$dep" ]
+  }
+
+  foreach(dep, meta_json.fidl_deps) {
+    _deps += [ ":$dep" ]
+  }
+
+  source_set(target_name) {
+    include_dirs = _include_dirs
+    sources = _sources
+    public_configs = [ ":$config_name" ]
+    output_name = _output_name
+    public_deps = _deps
+  }
+}
+
+template("fuchsia_cc_prebuilt_library") {
+  assert(defined(invoker.meta), "The meta.json file path must be specified.")
+  meta_json = read_file(invoker.meta, "json")
+
+  _include_dirs = []
+  _deps = []
+  _libs = []
+
+  meta_json_include_dir = meta_json.include_dir
+  _include_dirs += [ "$fuchsia_sdk_path/$meta_json_include_dir" ]
+
+  if (target_name == "vulkan") {
+    _include_dirs += [ "$fuchsia_sdk/vulkan/include" ]
+  }
+
+  foreach(dep, meta_json.deps) {
+    _deps += [ ":$dep" ]
+  }
+
+  meta_json_binaries = meta_json.binaries
+  meta_json_binaries_x64 = meta_json_binaries.x64
+  prebuilt_lib = meta_json_binaries_x64.link
+  _libs = [ "$fuchsia_sdk_path/$prebuilt_lib" ]
+
+  config_name = "config_$target_name"
+  config(config_name) {
+    include_dirs = _include_dirs
+    libs = _libs
+  }
+
+  group(target_name) {
+    public_configs = [ ":$config_name" ]
+    public_deps = _deps
+  }
+}
+
+template("fuchsia_sdk") {
+  assert(defined(invoker.meta), "The meta.json file path must be specified.")
+
+  meta_json = read_file(invoker.meta, "json")
+
+  foreach(part, meta_json.parts) {
+    part_meta_json = {
+    }
+
+    part_meta = part.meta
+    part_meta_rebased = "$fuchsia_sdk_path/$part_meta"
+
+    part_meta_json = read_file(part_meta_rebased, "json")
+    subtarget_name = part_meta_json.name
+    if (part.type == "cc_source_library") {
+      fuchsia_cc_source_library(subtarget_name) {
+        meta = part_meta_rebased
+      }
+    } else if (part.type == "sysroot") {
+      fuchsia_sysroot(subtarget_name) {
+        meta = part_meta_rebased
+      }
+    } else if (part.type == "fidl_library") {
+      fuchsia_fidl_library(subtarget_name) {
+        meta = part_meta_rebased
+      }
+    } else if (part.type == "cc_prebuilt_library") {
+      fuchsia_cc_prebuilt_library(subtarget_name) {
+        meta = part_meta_rebased
+      }
+    }
+  }
+
+  group(target_name) {
+  }
+}

build/toolchain/fuchsia/BUILD.gn

@@ -0,0 +1,158 @@
+# Copyright 2018 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/fuchsia/fuchsia.gni")
+import("//build/toolchain/goma.gni")
+
+if (use_goma) {
+  goma_prefix = "$goma_dir/gomacc "
+} else {
+  goma_prefix = ""
+}
+
+toolchain("fuchsia") {
+  toolchain_bin = rebase_path("//fuchsia/toolchain/bin", root_out_dir)
+  fuchsia_sdk = rebase_path("//fuchsia/sdk", root_out_dir)
+
+  # We can't do string interpolation ($ in strings) on things with dots in
+  # them. To allow us to use $cc below, for example, we create copies of
+  # these values in our scope.
+  cc = "${toolchain_bin}/clang"
+  cxx = "${toolchain_bin}/clang++"
+  ar = "${toolchain_bin}/llvm-ar"
+  ld = "${toolchain_bin}/clang++"
+  readelf = "${toolchain_bin}/llvm-readelf"
+  nm = "${toolchain_bin}/llvm-nm"
+  strip = "${toolchain_bin}/llvm-strip"
+
+  target_triple_flags = "--target=x86_64-fuchsia"
+  sysroot_flags = "--sysroot ${fuchsia_sdk}/arch/x64/sysroot"
+
+  # These library switches can apply to all tools below.
+  lib_switch = "-l"
+  lib_dir_switch = "-L"
+
+  tool("cc") {
+    depfile = "{{output}}.d"
+    command = "$goma_prefix $cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
+    depsformat = "gcc"
+    description = "CC {{output}}"
+    outputs = [
+      "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+    ]
+  }
+
+  tool("cxx") {
+    depfile = "{{output}}.d"
+    command = "$goma_prefix $cxx -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}"
+    depsformat = "gcc"
+    description = "CXX {{output}}"
+    outputs = [
+      "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+    ]
+  }
+
+  tool("asm") {
+    depfile = "{{output}}.d"
+    command = "$goma_prefix $cc -MMD -MF $depfile $target_triple_flags $sysroot_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
+    depsformat = "gcc"
+    description = "ASM {{output}}"
+    outputs = [
+      "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+    ]
+  }
+
+  tool("alink") {
+    rspfile = "{{output}}.rsp"
+    command = "rm -f {{output}} && $ar rcs {{output}} @$rspfile"
+    description = "AR {{output}}"
+    rspfile_content = "{{inputs}}"
+    outputs = [
+      "{{target_out_dir}}/{{target_output_name}}{{output_extension}}",
+    ]
+    default_output_extension = ".a"
+    output_prefix = "lib"
+  }
+
+  tool("solink") {
+    soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
+    sofile = "{{root_out_dir}}/$soname"  # Possibly including toolchain dir.
+    unstripped_sofile = "{{root_out_dir}}/so.unstripped/$soname"  # Possibly including toolchain dir.
+    rspfile = sofile + ".rsp"
+
+    # These variables are not built into GN but are helpers that implement
+    # (1) linking to produce a .so, (2) extracting the symbols from that file
+    # to a temporary file, (3) if the temporary file has differences from the
+    # existing .TOC file, overwrite it, otherwise, don't change it.
+    tocfile = sofile + ".TOC"
+    temporary_tocname = sofile + ".tmp"
+    link_command = "$goma_prefix $ld $target_triple_flags $sysroot_flags -shared {{ldflags}} -o $unstripped_sofile -Wl,--build-id -Wl,-soname=$soname @$rspfile"
+    toc_command = "{ $readelf -d $unstripped_sofile | grep SONAME ; $nm -gD -f p $unstripped_sofile | cut -f1-2 -d' '; } > $temporary_tocname"
+    replace_command = "if ! cmp -s $temporary_tocname $tocfile; then mv $temporary_tocname $tocfile; fi"
+    strip_command = "$strip -o $sofile $unstripped_sofile"
+
+    command =
+        "$link_command && $toc_command && $replace_command && $strip_command"
+    rspfile_content = "-Wl,--whole-archive {{inputs}} {{solibs}} -Wl,--no-whole-archive {{libs}}"
+
+    description = "SOLINK $sofile"
+
+    default_output_extension = ".so"
+
+    output_prefix = "lib"
+
+    # Since the above commands only updates the .TOC file when it changes, ask
+    # Ninja to check if the timestamp actually changed to know if downstream
+    # dependencies should be recompiled.
+    restat = true
+
+    # Tell GN about the output files. It will link to the sofile but use the
+    # tocfile for dependency management.
+    outputs = [
+      sofile,
+      unstripped_sofile,
+      tocfile,
+    ]
+
+    link_output = sofile
+    depend_output = tocfile
+  }
+
+  tool("link") {
+    exename = "{{target_output_name}}{{output_extension}}"
+    outfile = "{{root_out_dir}}/$exename"
+    rspfile = "$outfile.rsp"
+    unstripped_outfile = "{{root_out_dir}}/exe.unstripped/$exename"
+    command = "$goma_prefix $ld $target_triple_flags $sysroot_flags {{ldflags}} -o $unstripped_outfile -Wl,--build-id -Wl,--start-group @$rspfile {{solibs}} -Wl,--end-group {{libs}} && ${strip} -o $outfile $unstripped_outfile"
+    description = "LINK $outfile"
+    rspfile_content = "{{inputs}}"
+    outputs = [
+      unstripped_outfile,
+      outfile,
+    ]
+  }
+
+  tool("stamp") {
+    command = "touch {{output}}"
+    description = "STAMP {{output}}"
+  }
+
+  tool("copy") {
+    command = "ln -f {{source}} {{output}} 2>/dev/null || (rm -rf {{output}} && cp -af {{source}} {{output}})"
+    description = "COPY {{source}} {{output}}"
+  }
+
+  # When invoking this toolchain not as the default one, these args will be
+  # passed to the build. They are ignored when this is the default toolchain.
+  toolchain_args = {
+    current_cpu = target_cpu
+    current_os = target_os
+
+    # These values need to be passed through unchanged.
+    target_os = target_os
+    target_cpu = target_cpu
+
+    is_clang = true
+  }
+}

third_party/zlib/BUILD.gn

@@ -115,7 +115,7 @@ static_library("minizip") {
       "contrib/minizip/iowin32.h",
     ]
   }
-  if (is_mac || is_ios || is_android) {
+  if (is_mac || is_ios || is_android || is_fuchsia) {
     # Mac, Android and the BSDs don't have fopen64, ftello64, or fseeko64. We
     # use fopen, ftell, and fseek instead on these systems.
     defines = [ "USE_FILE32API" ]