#!/usr/bin/python -u # Copyright 2019 The Dart project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # This script automates the steps required to fully roll a recent version of the # Dart SDK into the Flutter engine (and can easily be extended to roll into the # Flutter framework as well). # # The following steps are completed as part of the roll: # - The Dart buildbots are queried to determine which SDK revision should be # used. Only revisions which have finished all VM and Flutter builds will be # considered, and revisions with < 90% of the VM bots green are ignored. See # dart_buildbot_helper.py for more details. # - dart_roll_helper.py is run with the chosen revision. This performs all the # interesting parts of the roll including running tests. If the steps all # complete successfully, dart_roll_helper.py exits with code 0. Other # possible statuses are listed in dart_roll_utils.py, and will cause this # script to exit with a non-zero exit code. dart_roll_helper.py can also be # run manually to perform a Dart SDK roll with specific parameters. # - The commit created by dart_roll_helper.py is pushed to a branch in # flutter/engine and a pull request is created. Once all PR checks are # complete, the PR is either merged or closed and all state created by this # script is cleaned up. # # In order for this script to work, the following environment variables much be # set: # - GITHUB_API_KEY: A GitHub personal access token for the GitHub account to # be used for uploading the SDK roll changes (see # https://github.com/settings/tokens) # - FLUTTER_HOME: the absolute path to the 'flutter' directory # - ENGINE_HOME: the absolute path to the 'engine/src' directory # - DART_SDK_HOME: the absolute path to the root of a Dart SDK project # - ENGINE_FORK: the name of the GitHub fork to use (e.g., bkonyi/engine or # flutter/engine) # # Finally, the following pip commands need to be run: # - `pip install gitpython` for GitPython (git) # - `pip install PyGithub` for PyGithub (github) from dart_buildbot_helper import get_most_recent_green_build from dart_roll_utils import * from git import Repo from github import Github, GithubException import argparse import atexit import datetime import os import signal import shutil import subprocess import sys import time GITHUB_STATUS_FAILURE = 'failure' GITHUB_STATUS_PENDING = 'pending' GITHUB_STATUS_SUCCESS = 'success' PULL_REQUEST_DESCRIPTION = ( 'This is an automated pull request which will automatically merge once ' 'checks pass.' ) FLAG_skip_wait_for_artifacts = False CURRENT_SUBPROCESS = None CURRENT_PR = None GITHUB_ENGINE_REPO = None GITHUB_ENGINE_FORK = None def run_dart_roll_helper(most_recent_commit, extra_args): global CURRENT_SUBPROCESS os.environ["PYTHONUNBUFFERED"] = "1" args = ['python', os.path.join(os.path.dirname(__file__), 'dart_roll_helper.py'), '--no-hot-reload', most_recent_commit] + extra_args CURRENT_SUBPROCESS = subprocess.Popen(args) result = CURRENT_SUBPROCESS.wait() CURRENT_SUBPROCESS = None return result # TODO(bkonyi): uncomment if we decide to roll into the framework. # def get_engine_version_path(flutter_repo_path): # return os.path.join(flutter_repo_path, # 'bin', # 'internal', # 'engine.version') # # # def get_current_engine_version(flutter_repo_path): # with open(get_engine_version_path(flutter_repo_path), 'r') as f: # return f.readline().strip() # # # def update_engine_version(flutter_repo_path, sha): # with open(get_engine_version_path(flutter_repo_path), 'w') as f: # f.write(sha) # # # def run_engine_roll_helper(engine_local_repo, # flutter_repo_path, # flutter_local_repo, # flutter_github_repo): # clean_and_update_repo(engine_local_repo) # engine_commits = list(engine_local_repo.iter_commits())[:2] # pre_roll_commit = engine_commits[1].hexsha # roll_commit = engine_commits[0].hexsha # current_engine_version = get_current_engine_version(flutter_repo_path) # # # Run `flutter doctor` until artifacts are uploaded to cloud. # wait_for_engine_artifacts(flutter_repo_path, roll_commit) # # # Update the engine repo again to get any changes that may have gone in while # # waiting for the artifacts to build. # clean_and_update_repo(engine_local_repo) # # if not is_ancestor_commit(current_engine_version, # roll_commit, # engine_flutter_path()): # print_status(('Existing revision {} already contains the Dart SDK roll. ' # 'No more work to do!').format(current_engine_version)) # sys.exit(ERROR_ROLL_SUCCESS) # # current_date = datetime.datetime.today().strftime('%Y-%m-%d') # branch_name = 'dart-sdk-roll-{}'.format(current_date) # engine_version_path = get_engine_version_path(flutter_repo_path) # pr_name = 'Dart SDK roll for {}'.format(current_date) # # if pre_roll_commit != current_engine_version: # # Update the engine version to the commit before the Dart SDK roll. # # This ensures that the Dart SDK version bump is the only change in # # the engine roll. # update_engine_version(flutter_repo_path, pre_roll_commit) # create_commit(flutter_local_repo, # branch_name, # 'Roll engine ahead of Dart SDK roll', # [engine_version_path]) # # # Actually update the engine version to include the Dart SDK version bump. # update_engine_version(flutter_repo_path, roll_commit) # create_commit(flutter_local_repo, # branch_name, # 'Roll engine with Dart SDK roll', # [engine_version_path]) # # pull_request = create_pull_request(flutter_github_repo, # flutter_local_repo, # pr_name, # branch_name) # # merge_on_success(flutter_github_repo, pull_request) # # # def wait_for_engine_artifacts(flutter_repo_path, engine_revision): # if FLAG_skip_wait_for_artifacts: # print_warning('Skipping wait for Flutter engine artifacts.') # return # # flutter_tools = os.path.join(flutter_repo_path, 'bin', 'flutter') # cache_path = os.path.join(flutter_repo_path, 'bin', 'cache') # # # Run `flutter doctor` until it can successfully find the engine artifacts # args = [flutter_tools, # 'doctor', # '--check-for-remote-artifacts', # engine_revision] # while True: # result = subprocess.Popen(args, stdout=subprocess.DEVNULL).wait() # if result == 0: # break # time.sleep(15) # # # def create_commit(local_repo, branch, message, files): # local_repo.create_head(branch) # local_repo.git.checkout(branch) # index = local_repo.index # index.add(files) # index.commit(message) def clean_and_update_repo(local_repo): local_repo.git.checkout('.') local_repo.git.clean('-xdf') local_repo.git.checkout('master') local_repo.git.pull() def clean_and_update_forked_repo(local_repo): local_repo.git.checkout('.') local_repo.git.clean('-xdf') local_repo.git.fetch('upstream') local_repo.git.checkout('master') local_repo.git.merge('upstream/master') def clean_build_outputs(): print_status('Cleaning build directory...') args = ['rm', '-rf', os.path.join(ENGINE_HOME, 'out')] CURRENT_SUBPROCESS = subprocess.Popen(args) CURRENT_SUBPROCESS.wait() CURRENT_SUBPROCESS = None def delete_local_branch(local_repo, branch): print_status('Deleting local branch {} in: {}'.format( branch, local_repo.working_tree_dir)) local_repo.git.checkout('master') local_repo.delete_head(branch, '-D') def delete_remote_branch(github_repo, branch): print_status('Deleting remote branch on {}: {}'.format(github_repo.full_name, branch)) github_repo.get_git_ref('heads/{}'.format(branch)).delete() def get_most_recent_commit(local_repo): commits = list(local_repo.iter_commits())[:1] return commits[0] def get_pr_title(local_repo): commit = get_most_recent_commit(local_repo) return commit.message.splitlines()[0].rstrip() def create_pull_request(github_repo, local_repo, title, branch): local_repo.create_head(branch) local_repo.git.checkout(branch) local_repo.git.push('origin', branch) commit = get_most_recent_commit(local_repo) description = PULL_REQUEST_DESCRIPTION + '\n\n' + commit.message try: return github_repo.create_pull(title, description, 'master', '{}:{}'.format('bkonyi', branch)) except GithubException as e: delete_remote_branch(GITHUB_ENGINE_FORK, branch) raise DartAutorollerException(e.data['errors'][0]['message']) finally: print_status('Cleaning up local branch: {}'.format(branch)) delete_local_branch(local_repo, branch) # Remove the commit from the local master branch. local_repo.git.reset('--hard','origin/master') def cleanup_pr(github_repo, pull_request, reason): msg = '{}, abandoning roll.'.format(reason) pull_request.create_issue_comment(msg) pull_request.edit(state='closed') print_error(msg) delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref) def merge_on_success(github_repo, local_repo, pull_request): sha = pull_request.head.sha commit = github_repo.get_commit(sha=sha) # TODO(bkonyi): Handle case where Flutter tree is red and we're trying to # merge into flutter/flutter. should_merge = wait_for_status(pull_request, commit) if should_merge: if not pull_request.is_merged(): pull_request.create_issue_comment('Checks successful, automatically merging.') merge_status = pull_request.merge(merge_method='squash').merged if not merge_status: print_error('Merge failed! Aborting roll.') sys.exit(1) print_status('Merge was successful!') else: print_status('Manual merge was performed.') else: cleanup_pr(github_repo, pull_request, 'Checks failed') sys.exit(1) delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref) # TODO(bkonyi): Check to see if the Flutter build is green for flutter/flutter # if we decide to roll the engine into the framework. # def flutter_build_passing(commit): # FLUTTER_BUILD = 'flutter-build' # statuses = commit.get_statuses() # for status in statuses: # if status.context == FLUTTER_BUILD: # return (status.state == GITHUB_STATUS_SUCCESS) # If flutter-build isn't a valid status, the PR checks don't require the # Flutter framework to be green to submit. # return True # Determines if any failures are actual failures from the PR or are just # failures from the engine builders. def is_only_engine_build_failing(commit): BUILD = '-build' LUCI_ENGINE = 'luci-engine' statuses = commit.get_statuses() for status in statuses: if (not ((BUILD in status.context) or (LUCI_ENGINE == status.context)) and (status.state == GITHUB_STATUS_FAILURE)): return False print_status("An engine builder is still failing...") return True def wait_for_status(pull_request, commit): if FLAG_skip_wait_for_artifacts: return True print_status('Sleeping for 120 seconds to allow for Cirrus to start...') # Give Cirrus a chance to start. The GitHub statuses posted by Cirrus go # through some weird states when the PR is created and can be marked as # failing temporarily, causing this check to return False if we don't wait. time.sleep(120) print_status('Starting PR status checks (this may take awhile).') # Ensure all checks pass. while True: # Check to see if the PR was manually merged. if pull_request.is_merged(): break status = commit.get_combined_status().state if status == GITHUB_STATUS_SUCCESS: break # If the only the engine builders are red, keep trying. elif ((status == GITHUB_STATUS_FAILURE) and (not is_only_engine_build_failing(commit))): return False time.sleep(5) # TODO(bkonyi): Re-enable this check if we decide to roll the engine into the # framework. # Once all checks are passing, wait for the Flutter build to be green. # while not flutter_build_passing(commit): # print_status('Waiting for Flutter build to pass...') # time.sleep(60) # print_status('Flutter build passing!') return True def sys_exit(signal, frame): sys.exit() def cleanup_children(): print_error('Roll canceled! Shutting down dart_autoroller.py.') if CURRENT_SUBPROCESS != None: CURRENT_SUBPROCESS.terminate() if CURRENT_PR != None: cleanup_pr(GITHUB_ENGINE_REPO, CURRENT_PR, 'Canceled by roller') def main(): global CURRENT_PR global FLAG_skip_wait_for_artifacts global GITHUB_ENGINE_REPO global GITHUB_ENGINE_FORK parser = argparse.ArgumentParser(description='Dart SDK autoroller for Flutter.') parser.add_argument('--dart-sdk-revision', help='Provide a Dart SDK revision to roll instead of ' 'choosing one automatically') parser.add_argument('--no-update-repos', help='Skip cleaning and updating local repositories', action='store_true') parser.add_argument('--skip-roll', help='Skip running dart_roll_helper.py', action='store_true') parser.add_argument('--skip-tests', help='Skip running Flutter tests', action='store_true') parser.add_argument('--skip-build', help='Skip building all configurations', action='store_true') parser.add_argument('--skip-update-deps', help='Skip updating the Dart SDK dependencies', action='store_true') parser.add_argument('--skip-wait-for-artifacts', help="Don't wait for PR statuses to pass or for engine" + " artifacts to be uploaded to the cloud", action='store_true', default=False) parser.add_argument('--skip-update-licenses', help='Skip updating the licenses for the Flutter engine', action='store_true') parser.add_argument('--skip-pull-request', help="Skip creating a pull request and don't commit", action='store_true') args = parser.parse_args() FLAG_skip_wait_for_artifacts = args.skip_wait_for_artifacts github_api_key = os.getenv('GITHUB_API_KEY') dart_sdk_path = os.getenv('DART_SDK_HOME') flutter_path = os.getenv('FLUTTER_HOME') engine_path = os.getenv('ENGINE_HOME') engine_fork = os.getenv('ENGINE_FORK') local_dart_sdk_repo = Repo(dart_sdk_path) local_flutter_repo = Repo(flutter_path) local_engine_flutter_repo = Repo(os.path.join(engine_path, 'flutter')) assert(not local_dart_sdk_repo.bare) assert(not local_flutter_repo.bare) assert(not local_engine_flutter_repo.bare) github = Github(github_api_key) GITHUB_ENGINE_REPO = github.get_repo('flutter/engine') GITHUB_ENGINE_FORK = github.get_repo(engine_fork) github_flutter_repo = github.get_repo('flutter/flutter') atexit.register(cleanup_children) signal.signal(signal.SIGTERM, sys_exit) if not args.no_update_repos: print_status('Cleaning and updating local trees...') clean_build_outputs() clean_and_update_repo(local_dart_sdk_repo) clean_and_update_repo(local_flutter_repo) clean_and_update_forked_repo(local_engine_flutter_repo) else: print_warning('Skipping cleaning and updating of local trees') # Use the most recent Dart SDK commit for the roll. if not args.skip_roll: print_status('Starting Dart roll helper') most_recent_commit = '' dart_roll_helper_args = [] if args.skip_update_deps: dart_roll_helper_args.append('--no-update-deps') elif args.dart_sdk_revision != None: most_recent_commit = args.dart_sdk_revision else: # Get the most recent commit that is a reasonable candidate. most_recent_commit = get_most_recent_green_build() if args.skip_tests: dart_roll_helper_args.append('--no-test') if args.skip_build: dart_roll_helper_args.append('--no-build') if args.skip_update_licenses: dart_roll_helper_args.append('--no-update-licenses') if not args.skip_pull_request: dart_roll_helper_args.append('--create-commit') # Will exit with code ERROR_OLD_COMMIT_PROVIDED if `most_recent_commit` is # older than the current revision of the SDK used by Flutter. result = run_dart_roll_helper(most_recent_commit, dart_roll_helper_args) if result != 0: sys.exit(result) else: print_warning('Skipping roll step!') if not args.skip_pull_request: # If the local roll was successful, try to merge into the engine. print_status('Creating flutter/engine pull request') current_date = datetime.datetime.today().strftime('%Y-%m-%d') try: CURRENT_PR = create_pull_request(GITHUB_ENGINE_REPO, local_engine_flutter_repo, get_pr_title(local_engine_flutter_repo), 'dart-sdk-roll-{}'.format(current_date)) except DartAutorollerException as e: print_error(('Error while creating flutter/engine pull request: {}.' ' Aborting roll.').format(e)) sys.exit(1) print_status('Waiting for PR checks to complete...') merge_on_success(GITHUB_ENGINE_REPO, local_engine_flutter_repo, CURRENT_PR) print_status('PR checks complete!') CURRENT_PR = None else: print_warning('Not creating flutter/engine PR!') # TODO(bkonyi): uncomment if we decide to roll the engine into the framework. # print_status('Starting roll of flutter/engine into flutter/flutter') # If the roll into the engine succeeded, prep the roll into the framework. # run_engine_roll_helper(local_engine_flutter_repo, # flutter_path, # local_flutter_repo, # github_flutter_repo) # Status code should be 0 anyway, but let's make sure our exit status is # consistent throughout the tool on a successful roll. sys.exit(ERROR_ROLL_SUCCESS) if __name__ == '__main__': main()