flutter/tools/dart/dart_roll_helper.py

#!/usr/bin/python -u
# Copyright 2018 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script automates the Dart SDK roll steps, including:
#   - Updating the Dart revision in DEPS
#   - Updating the Dart dependencies in DEPS
#   - Syncing dependencies with 'gclient sync'
#   - Generating GN files for relevant engine configurations
#   - Building relevant engine configurations
#   - Running tests in 'example/flutter_gallery' and 'packages/flutter'
#   - Launching flutter_gallery in release and debug mode
#   - Running license.sh and updating license files in
#     'flutter/ci/licenses_golden'
#   - Generating a commit with relevant Dart SDK commit logs (optional)
#
# The following environment variables can be set instead of being passed as
# arguments:
#   - FLUTTER_HOME: the absolute path to the 'flutter' directory
#   - ENGINE_HOME: the absolute path to the 'engine/src' directory
#   - DART_SDK_HOME: the absolute path to the root of a Dart SDK project

from dart_roll_utils import *
import argparse
import atexit
import datetime
import fileinput
import os
import platform
import signal
import shutil
import subprocess
import sys

DART_REVISION_ENTRY = 'dart_revision'
FLUTTER = '{}/bin/flutter'.format(flutter_home())
FLUTTER_DOCTOR = [FLUTTER, 'doctor']
FLUTTER_RUN    = [FLUTTER, 'run']
FLUTTER_TEST   = [FLUTTER, 'test']

MAX_GCLIENT_RETRIES = 3

# Returned when licenses do not require updating.
LICENSE_SCRIPT_OKAY       = 0
# Returned when licenses require updating.
LICENSE_SCRIPT_UPDATES    = 1
# Returned when either 'pub' or 'dart' isn't in the path.
LICENSE_SCRIPT_EXIT_ERROR = 127

CURRENT_SUBPROCESS = None

def update_dart_revision(dart_revision):
  original_revision = ''
  print_status('Updating Dart revision to {}'.format(dart_revision))
  content = get_deps()
  for idx, line in enumerate(content):
    if DART_REVISION_ENTRY in line:
      original_revision = line.strip().split(' ')[1][1:-2]
      if not is_ancestor_commit(original_revision,
                                dart_revision,
                                dart_sdk_home()):
        print_error('Dart revision {} is older than existing revision, {}.' +
                    ' Aborting roll.'.format(dart_revision, original_revision))
        sys.exit(ERROR_OLD_COMMIT_PROVIDED)

      content[idx] = "  'dart_revision': '" + dart_revision + "',\n"
      break
  write_deps(content)
  return original_revision

def run_process(args, cwd=None, stdout=None):
  global CURRENT_SUBPROCESS
  CURRENT_SUBPROCESS = subprocess.Popen(args, cwd=cwd, stdout=stdout)
  exit_code = CURRENT_SUBPROCESS.wait()
  CURRENT_SUBPROCESS = None
  return exit_code

def gclient_sync():
  global CURRENT_SUBPROCESS
  exit_code = None
  num_retries = 0

  while ((exit_code != 0) and not (num_retries >= MAX_GCLIENT_RETRIES)):
    print_status('Running gclient sync (Attempt {}/{})'
                 .format(num_retries + 1, MAX_GCLIENT_RETRIES))
    exit_code = run_process(['gclient', 'sync', '--delete_unversioned_trees'],
                            cwd=engine_home())
    if exit_code != 0:
      num_retries += 1
  if num_retries == MAX_GCLIENT_RETRIES:
    print_error('Max number of gclient sync retries attempted. Aborting roll.')
    sys.exit(ERROR_GCLIENT_SYNC_FAILED)


def get_deps():
  with open(flutter_deps_path(), 'r') as f:
    content = f.readlines()
  return content


def update_deps():
  print_status('Updating Dart dependencies')
  run_process([update_dart_deps_path()], cwd=engine_home())


def write_deps(newdeps):
  with open(flutter_deps_path(), 'w') as f:
    f.write(''.join(newdeps))


def run_gn():
  print_status('Generating build files')
  common = [os.path.join('flutter', 'tools', 'gn'), '--goma', '--full-dart-sdk']
  debug = ['--runtime-mode=debug']
  profile = ['--runtime-mode=profile']
  release = ['--runtime-mode=release']
  runtime_modes = [debug, profile, release]
  unopt = ['--unoptimized']
  android = ['--android']
  fuchsia = ['--fuchsia']

  for mode in runtime_modes:
    if set(mode) != set(release):
      run_process(common + android + unopt + mode, cwd=engine_home())
    run_process(common + android + mode, cwd=engine_home())
    host = common[:]
    if set(mode) == set(debug):
      host += unopt
    run_process(host + mode, cwd=engine_home())
  run_process(common + fuchsia + debug + unopt, cwd=engine_home())


def build():
  print_status('Building Flutter engine')
  command = ['ninja', '-j1000']
  configs = [
    'host_debug_unopt',
    'host_release',
    'host_profile',
    'android_debug_unopt',
    'android_profile',
    'android_release',
    'fuchsia_debug_unopt_x64',
  ]

  build_dir = 'out'
  if platform.system() == 'Darwin':
    build_dir = 'xcodebuild'

  for config in configs:
    error_code = run_process(command + ['-C', os.path.join(build_dir, config)],
                             cwd=engine_home())
    if error_code != 0:
      print_error('Build failure for configuration "' +
                  config +
                  '". Aborting roll.')
      sys.exit(ERROR_BUILD_FAILED)


def run_flutter_doctor():
  print_status('Running flutter doctor')
  engine_src_path = '--local-engine-src-path={}'.format(engine_home())
  result = run_process(FLUTTER_DOCTOR + ['--local-engine=host_debug_unopt',
                                    engine_src_path],
                                    cwd=package_flutter_path())
  if result != 0:
    print_error('flutter doctor failed. Aborting roll.')
    sys.exit(ERROR_FLUTTER_DOCTOR_FAILED)


def run_tests():
  print_status('Running tests in packages/flutter')
  engine_src_path = '--local-engine-src-path={}'.format(engine_home())
  result = run_process(FLUTTER_TEST + ['--local-engine=host_debug_unopt',
                                       engine_src_path],
                                       cwd=package_flutter_path())
  if result != 0:
    print_error('package/flutter tests failed. Aborting roll.')
    sys.exit(ERROR_PKG_FLUTTER_FAILED)

  print_status('Running tests in examples/flutter_gallery')
  result = run_process(FLUTTER_TEST + ['--local-engine=host_debug_unopt',
                                       engine_src_path,
                                       '--disable-service-auth-codes'],
                                       cwd=flutter_gallery_path());
  if result != 0:
    print_error('flutter_gallery tests failed. Aborting roll.')
    sys.exit(ERROR_FLUTTER_GALLERY_FAILED)


def run_hot_reload_configurations():
  print_status('Running flutter gallery release')
  engine_src_path = '--local-engine-src-path={}'.format(engine_home())
  run_process(FLUTTER_RUN + ['--release',
                             '--local-engine=android_release',
                             engine_src_path],
                             cwd=flutter_gallery_path())
  print_status('Running flutter gallery debug')
  run_process(FLUTTER_RUN + ['--local-engine=android_debug_unopt',
                             engine_src_path],
                             cwd=flutter_gallery_path())


def update_licenses():
  print_status('Updating Flutter licenses')
  result = run_process([engine_license_script_path()], cwd=engine_home())
  if result == LICENSE_SCRIPT_EXIT_ERROR:
    print_error('License script failed to run. Is the Dart SDK (specifically' +
                ' dart and pub) in your path? Aborting roll.')
    sys.exit(ERROR_LICENSE_SCRIPT_FAILED)
  elif (result != LICENSE_SCRIPT_OKAY) and (result != LICENSE_SCRIPT_UPDATES):
    print_error('Unknown license script error: {}. Aborting roll.'
                .format(result))
    sys.exit(ERROR_LICENSE_SCRIPT_FAILED)

  # Ignore 'licenses_skia' as they shouldn't change during a Dart SDK roll.
  src_files = ['licenses_flutter', 'licenses_third_party', 'tool_signature']
  for f in src_files:
    path = os.path.join(license_script_output_path(), f)
    if os.path.isfile(path):
      shutil.copy(path, engine_golden_licenses_path())
  run_process(['pub', 'get'], cwd=engine_license_script_package_path())
  gclient_sync()

  # Update the LICENSE file.
  with open(sky_license_file_path(), 'w') as sky_license:
    run_process(['dart', os.path.join('lib', 'main.dart'),
                 '--release', '--src', engine_home(),
                 '--out', engine_license_script_output_path()],
                 cwd=engine_license_script_package_path(),
                 stdout=sky_license)


def get_commit_range(start, finish):
  range_str = '{}..{}'.format(start, finish)
  command = ['git', 'log', '--oneline', range_str]
  orig_dir = os.getcwd()
  os.chdir(dart_sdk_home())
  result = subprocess.check_output(command)
  result = '\n'.join(['dart-lang/sdk@' + l for l in result.splitlines()])
  os.chdir(orig_dir)
  return result


def get_short_rev(rev):
  command = ['git', 'rev-parse', '--short', rev]
  orig_dir = os.getcwd()
  os.chdir(dart_sdk_home())
  result = subprocess.check_output(command).rstrip()
  os.chdir(orig_dir)
  return result


def git_commit(original_revision, updated_revision):
  print_status('Committing Dart SDK roll')
  current_date = datetime.date.today()
  sdk_log = get_commit_range(original_revision, updated_revision)
  num_commits = len(sdk_log.splitlines())
  commit_msg = ('Roll src/third_party/dart {}..{} ({} commits)'
                .format(get_short_rev(original_revision),
                        get_short_rev(updated_revision), num_commits))
  commit_msg += '\n\n' + sdk_log
  commit_cmd = ['git', 'commit', '-a', '-m', commit_msg]
  run_process(commit_cmd, cwd=engine_flutter_path())


def update_roots(args):
  if args.flutter_home:
    set_flutter_home(args.flutter_home)

  if args.engine_home:
    set_engine_home(args.engine_home)

  if args.dart_sdk_home:
    set_dart_sdk_home(args.dart_sdk_home)

  if flutter_home() == '':
    print_error('Either "--flutter-home" must be provided or FLUTTER_HOME must' +
                ' be set. Aborting roll.')
    sys.exit(ERROR_MISSING_ROOTS)

  if engine_home() == '':
    print_error('Either "--engine-home" must be provided or ENGINE_HOME must' +
                ' be set. Aborting roll.')
    sys.exit(ERROR_MISSING_ROOTS)

  if dart_sdk_home() == '':
    print_error('Either "--dart-sdk-home" must be provided or DART_SDK_HOME ' +
                'must be set. Aborting roll.')
    sys.exit(ERROR_MISSING_ROOTS)


def sys_exit(signal, frame):
  sys.exit()

def cleanup_children():
  if CURRENT_SUBPROCESS != None:
    CURRENT_SUBPROCESS.terminate()

def main():
  parser = argparse.ArgumentParser(description='Automate most Dart SDK roll tasks.')
  parser.add_argument('--dart-sdk-home', help='Path to the Dart SDK ' +
                      'repository. Overrides DART_SDK_HOME environment variable')
  parser.add_argument('dart_sdk_revision', help='Target Dart SDK revision')
  parser.add_argument('--create-commit', action='store_true',
                      help='Create the engine commit with Dart SDK commit log')
  parser.add_argument('--engine-home', help='Path to the Flutter engine ' +
                      'repository. Overrides ENGINE_HOME environment variable')
  parser.add_argument('--flutter-home', help='Path to the Flutter framework ' +
                      'repository. Overrides FLUTTER_HOME environment variable')
  parser.add_argument('--no-build', action='store_true',
                      help='Skip rebuilding the Flutter engine')
  parser.add_argument('--no-hot-reload', action='store_true',
                      help="Skip hot reload testing")
  parser.add_argument('--no-test', action='store_true',
                      help='Skip running host tests for package/flutter and ' +
                      'flutter_gallery')
  parser.add_argument('--no-update-deps', action='store_true',
                      help='Skip updating DEPS file')
  parser.add_argument('--no-update-licenses', action='store_true',
                      help='Skip updating licenses')

  args = parser.parse_args()

  atexit.register(cleanup_children)
  signal.signal(signal.SIGTERM, sys_exit)

  original_revision = None
  updated_revision = args.dart_sdk_revision

  # Disable buffering of log output
  os.environ["PYTHONUNBUFFERED"] = "1"

  update_roots(args)

  print_status('Starting Dart SDK roll')
  if not args.no_update_deps:
    original_revision = update_dart_revision(updated_revision)
    gclient_sync()
    update_deps()
    gclient_sync()
  if not args.no_build:
    run_gn()
    build()
  if ((not args.no_test) or (not args.no_hot_reload)):
    run_flutter_doctor()
  if not args.no_test:
    run_tests()
  if not args.no_hot_reload:
    run_hot_reload_configurations()
  if not args.no_update_licenses:
    update_licenses()
  if args.create_commit:
    if original_revision == None:
      print_warning('"original_revision" not specified. Skipping commit.')
      print_warning('This happens when the "--no_update_deps" argument is ' +
                    'provided')
    else:
      git_commit(original_revision, updated_revision)
  print_status('Dart SDK roll complete!')


if __name__ == '__main__':
  main()