diff --git a/automate-git.py b/automate-git.py deleted file mode 100644 index 7fe89ef18..000000000 --- a/automate-git.py +++ /dev/null @@ -1,1494 +0,0 @@ -# Copyright (c) 2014 The Chromium Embedded Framework Authors. All rights -# reserved. Use of this source code is governed by a BSD-style license that -# can be found in the LICENSE file. - -from __future__ import absolute_import -from __future__ import print_function -from datetime import datetime -from io import open -from optparse import OptionParser -import os -import re -import shlex -import shutil -import subprocess -import sys -import tempfile -import zipfile - -is_python2 = sys.version_info.major == 2 - -if is_python2: - from urllib import FancyURLopener - from urllib2 import urlopen -else: - from urllib.request import FancyURLopener, urlopen - -## -# Default URLs. -## - -depot_tools_url = 'https://chromium.googlesource.com/chromium/tools/depot_tools.git' -depot_tools_archive_url = 'https://storage.googleapis.com/chrome-infra/depot_tools.zip' - -cef_git_url = 'https://bitbucket.org/chromiumembedded/cef.git' - -## -# Global system variables. -## - -# Script directory. -script_dir = os.path.dirname(__file__) - -## -# Helper functions. -## - - -def msg(message): - """ Output a message. """ - sys.stdout.write('--> ' + message + "\n") - - -def run(command_line, working_dir, depot_tools_dir=None, output_file=None): - """ Runs the specified command. """ - # add depot_tools to the path - env = os.environ - if not depot_tools_dir is None: - env['PATH'] = depot_tools_dir + os.pathsep + env['PATH'] - - sys.stdout.write('-------- Running "'+command_line+'" in "'+\ - working_dir+'"...'+"\n") - if not options.dryrun: - args = shlex.split(command_line.replace('\\', '\\\\')) - - if not output_file: - return subprocess.check_call( - args, cwd=working_dir, env=env, shell=(sys.platform == 'win32')) - try: - msg('Writing %s' % output_file) - with open(output_file, 'w', encoding='utf-8') as fp: - return subprocess.check_call( - args, - cwd=working_dir, - env=env, - shell=(sys.platform == 'win32'), - stderr=subprocess.STDOUT, - stdout=fp) - except subprocess.CalledProcessError: - msg('ERROR Run failed. See %s for output.' % output_file) - raise - - -def create_directory(path): - """ Creates a directory if it doesn't already exist. """ - if not os.path.exists(path): - msg("Creating directory %s" % (path)) - if not options.dryrun: - os.makedirs(path) - - -def delete_directory(path): - """ Removes an existing directory. """ - if os.path.exists(path): - msg("Removing directory %s" % (path)) - if not options.dryrun: - shutil.rmtree(path, onerror=onerror) - - -def copy_directory(source, target, allow_overwrite=False): - """ Copies a directory from source to target. """ - if not options.dryrun and os.path.exists(target): - if not allow_overwrite: - raise Exception("Directory %s already exists" % (target)) - remove_directory(target) - if os.path.exists(source): - msg("Copying directory %s to %s" % (source, target)) - if not options.dryrun: - shutil.copytree(source, target) - - -def move_directory(source, target, allow_overwrite=False): - """ Copies a directory from source to target. """ - if not options.dryrun and os.path.exists(target): - if not allow_overwrite: - raise Exception("Directory %s already exists" % (target)) - remove_directory(target) - if os.path.exists(source): - msg("Moving directory %s to %s" % (source, target)) - if not options.dryrun: - shutil.move(source, target) - - -def is_git_checkout(path): - """ Returns true if the path represents a git checkout. """ - return os.path.exists(os.path.join(path, '.git')) - - -def exec_cmd(cmd, path): - """ Execute the specified command and return the result. """ - out = '' - err = '' - sys.stdout.write("-------- Running \"%s\" in \"%s\"...\n" % (cmd, path)) - parts = cmd.split() - try: - process = subprocess.Popen( - parts, - cwd=path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=(sys.platform == 'win32')) - out, err = process.communicate() - except IOError as e: - (errno, strerror) = e.args - raise - except: - raise - return {'out': out.decode('utf-8'), 'err': err.decode('utf-8')} - - -def get_git_hash(path, branch): - """ Returns the git hash for the specified branch/tag/hash. """ - cmd = "%s rev-parse %s" % (git_exe, branch) - result = exec_cmd(cmd, path) - if result['out'] != '': - return result['out'].strip() - return 'Unknown' - - -def get_git_url(path): - """ Returns the origin url for the specified path. """ - cmd = "%s config --get remote.origin.url" % (git_exe) - result = exec_cmd(cmd, path) - if result['out'] != '': - return result['out'].strip() - return 'Unknown' - - -def download_and_extract(src, target): - """ Extracts the contents of src, which may be a URL or local file, to the - target directory. """ - temporary = False - - if src[:4] == 'http': - # Attempt to download a URL. - opener = FancyURLopener({}) - response = opener.open(src) - - temporary = True - handle, archive_path = tempfile.mkstemp(suffix='.zip') - os.write(handle, response.read()) - os.close(handle) - elif os.path.exists(src): - # Use a local file. - archive_path = src - else: - raise Exception('Path type is unsupported or does not exist: ' + src) - - if not zipfile.is_zipfile(archive_path): - raise Exception('Not a valid zip archive: ' + src) - - # Attempt to extract the archive file. - try: - os.makedirs(target) - zf = zipfile.ZipFile(archive_path, 'r') - zf.extractall(target) - except: - shutil.rmtree(target, onerror=onerror) - raise - zf.close() - - # Delete the archive file if temporary. - if temporary and os.path.exists(archive_path): - os.remove(archive_path) - - -def read_file(path): - """ Read a file. """ - if os.path.exists(path): - with open(path, 'r', encoding='utf-8') as fp: - return fp.read() - else: - raise Exception("Path does not exist: %s" % (path)) - - -def write_fp(fp, data): - if is_python2: - fp.write(data.decode('utf-8')) - else: - fp.write(data) - - -def write_file(path, data): - """ Write a file. """ - msg('Writing %s' % path) - if not options.dryrun: - with open(path, 'w', encoding='utf-8') as fp: - write_fp(fp, data) - - -def read_config_file(path): - """ Read a configuration file. """ - # Parse the contents. - return eval(read_file(path), {'__builtins__': None}, None) - - -def write_config_file(path, contents): - """ Write a configuration file. """ - data = "{\n" - for key in sorted(contents.keys()): - data += " '%s': '%s',\n" % (key, contents[key]) - data += "}\n" - write_file(path, data) - - -def read_branch_config_file(path): - """ Read the CEF branch from the specified path. """ - config_file = os.path.join(path, 'cef.branch') - if os.path.isfile(config_file): - contents = read_config_file(config_file) - if 'branch' in contents: - return contents['branch'] - return '' - - -def write_branch_config_file(path, branch): - """ Write the CEF branch to the specified path. """ - config_file = os.path.join(path, 'cef.branch') - if not os.path.isfile(config_file): - write_config_file(config_file, {'branch': branch}) - - -def apply_patch(name): - patch_file = os.path.join(cef_dir, 'patch', 'patches', name) - if os.path.exists(patch_file + ".patch"): - # Attempt to apply the patch file. - patch_tool = os.path.join(cef_dir, 'tools', 'patcher.py') - run('%s %s --patch-file "%s" --patch-dir "%s"' % - (python_exe, patch_tool, patch_file, - chromium_src_dir), chromium_src_dir, depot_tools_dir) - - -def apply_deps_patch(): - """ Patch the Chromium DEPS file before `gclient sync` if necessary. """ - deps_path = os.path.join(chromium_src_dir, deps_file) - if os.path.isfile(deps_path): - msg("Chromium DEPS file: %s" % (deps_path)) - apply_patch(deps_file) - else: - raise Exception("Path does not exist: %s" % (deps_path)) - - -def apply_runhooks_patch(): - """ Patch the Chromium runhooks files before `gclient runhooks` if necessary. """ - apply_patch('runhooks') - - -def run_patch_updater(args='', output_file=None): - """ Run the patch updater script. """ - tool = os.path.join(cef_src_dir, 'tools', 'patch_updater.py') - if len(args) > 0: - args = ' ' + args - run('%s %s%s' % (python_exe, tool, args), cef_src_dir, depot_tools_dir, - output_file) - - -def onerror(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - import stat - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - - -def get_chromium_main_position(commit): - """ Returns the closest main position for the specified Chromium commit. """ - # Using -2 because a "Publish DEPS" commit which does not have a master - # position may be first. - cmd = "%s log -2 %s" % (git_exe, commit) - result = exec_cmd(cmd, chromium_src_dir) - if result['out'] != '': - match = re.search(r'refs/heads/(?:master|main)@{#([\d]+)}', result['out']) - assert match != None, 'Failed to find position' - return int(match.groups()[0]) - return None - - -def get_chromium_main_commit(position): - """ Returns the main commit for the specified Chromium commit position. """ - cmd = '%s log -1 --grep=refs/heads/master@{#%s} --grep=refs/heads/main@{#%s} origin/main' % ( - git_exe, str(position), str(position)) - result = exec_cmd(cmd, chromium_src_dir) - if result['out'] != '': - match = re.search(r'^commit ([a-f0-9]+)', result['out']) - assert match != None, 'Failed to find commit' - return match.groups()[0] - return None - - -def get_build_compat_versions(): - """ Returns the compatible Chromium and (optionally) depot_tools versions - specified by the CEF checkout. """ - compat_path = os.path.join(cef_dir, 'CHROMIUM_BUILD_COMPATIBILITY.txt') - msg("Reading %s" % compat_path) - config = read_config_file(compat_path) - - if not 'chromium_checkout' in config: - raise Exception("Missing chromium_checkout value in %s" % (compat_path)) - return config - - -def get_build_directory_name(is_debug): - build_dir = ('Debug' if is_debug else 'Release') + '_' - - # CEF uses a consistent directory naming scheme for GN via - # GetAllPlatformConfigs in tools/gn_args.py. - if options.x64build: - build_dir += 'GN_x64' - elif options.armbuild: - build_dir += 'GN_arm' - elif options.arm64build: - build_dir += 'GN_arm64' - else: - build_dir += 'GN_x86' - return build_dir - - -def read_update_file(): - update_path = os.path.join(cef_src_dir, 'CHROMIUM_UPDATE.txt') - if not os.path.exists(update_path): - msg("Missing file: %s" % update_path) - return None - - msg("Reading %s" % update_path) - return read_config_file(update_path) - - -def log_chromium_changes(): - """ Evaluate the Chromium checkout for changes. """ - config = read_update_file() - if config is None: - msg("Skipping Chromium changes log.") - return - - if 'files' in config: - out_file = os.path.join(download_dir, 'chromium_update_changes.diff') - if os.path.exists(out_file): - os.remove(out_file) - - old_commit = get_chromium_main_commit( - get_chromium_main_position(chromium_compat_version)) - new_commit = get_chromium_main_commit( - get_chromium_main_position(chromium_checkout)) - - cmd = '%s diff --relative --no-prefix %s..%s -- %s' % ( - git_exe, old_commit, new_commit, ' '.join(config['files'])) - result = exec_cmd(cmd, chromium_src_dir) - if result['out'] != '': - write_file(out_file, result['out']) - - -def check_pattern_matches(output_file=None): - """ Evaluate the Chromium checkout for pattern matches. """ - config = read_update_file() - if config is None: - msg("Skipping Chromium pattern matching.") - return - - if 'patterns' in config: - if output_file is None: - fp = sys.stdout - else: - msg('Writing %s' % output_file) - fp = open(output_file, 'w', encoding='utf-8') - - has_output = False - for entry in config['patterns']: - msg("Evaluating pattern: %s" % entry['pattern']) - - # Read patterns from a file to avoid formatting problems. - pattern_handle, pattern_file = tempfile.mkstemp() - os.write(pattern_handle, entry['pattern']) - os.close(pattern_handle) - - cmd = '%s grep -n -f %s' % (git_exe, pattern_file) - result = exec_cmd(cmd, chromium_src_dir) - os.remove(pattern_file) - - if result['out'] != '': - write_msg = True - re_exclude = re.compile( - entry['exclude_matches']) if 'exclude_matches' in entry else None - - for line in result['out'].split('\n'): - line = line.strip() - if len(line) == 0: - continue - skip = not re_exclude is None and re_exclude.match(line) != None - if not skip: - if write_msg: - if has_output: - write_fp(fp, '\n') - write_fp(fp, - '!!!! WARNING: FOUND PATTERN: %s\n' % entry['pattern']) - if 'message' in entry: - write_fp(fp, entry['message'] + '\n') - write_fp(fp, '\n') - write_msg = False - write_fp(fp, line + '\n') - has_output = True - - if not output_file is None: - if has_output: - msg('ERROR Matches found. See %s for output.' % out_file) - else: - write_fp(fp, 'Good news! No matches.\n') - fp.close() - - if has_output: - # Don't continue when we know the build will be wrong. - sys.exit(1) - - -## -# Program entry point. -## - -# Cannot be loaded as a module. -if __name__ != "__main__": - sys.stderr.write('This file cannot be loaded as a module!') - sys.exit(1) - -# Parse command-line options. -disc = """ -This utility implements automation for the download, update, build and -distribution of CEF. -""" - -parser = OptionParser(description=disc) - -# Setup options. -parser.add_option( - '--download-dir', - dest='downloaddir', - metavar='DIR', - help='Download directory with no spaces [required].') -parser.add_option( - '--depot-tools-dir', - dest='depottoolsdir', - metavar='DIR', - help='Download directory for depot_tools.', - default='') -parser.add_option('--depot-tools-archive', dest='depottoolsarchive', - help='Zip archive file that contains a single top-level '+\ - 'depot_tools directory.', default='') -parser.add_option('--branch', dest='branch', - help='Branch of CEF to build (master, 3987, ...). This '+\ - 'will be used to name the CEF download directory and '+\ - 'to identify the correct URL if --url is not '+\ - 'specified. The default value is master.', - default='master') -parser.add_option('--url', dest='url', - help='CEF download URL. If not specified the default URL '+\ - 'will be used.', - default='') -parser.add_option('--chromium-url', dest='chromiumurl', - help='Chromium download URL. If not specified the default '+\ - 'URL will be used.', - default='') -parser.add_option('--checkout', dest='checkout', - help='Version of CEF to checkout. If not specified the '+\ - 'most recent remote version of the branch will be used.', - default='') -parser.add_option('--chromium-checkout', dest='chromiumcheckout', - help='Version of Chromium to checkout (Git '+\ - 'branch/hash/tag). This overrides the value specified '+\ - 'by CEF in CHROMIUM_BUILD_COMPATIBILITY.txt.', - default='') - -# Miscellaneous options. -parser.add_option( - '--force-config', - action='store_true', - dest='forceconfig', - default=False, - help='Force creation of a new gclient config file.') -parser.add_option('--force-clean', - action='store_true', dest='forceclean', default=False, - help='Force a clean checkout of Chromium and CEF. This will'+\ - ' trigger a new update, build and distribution.') -parser.add_option('--force-clean-deps', - action='store_true', dest='forcecleandeps', default=False, - help='Force a clean checkout of Chromium dependencies. Used'+\ - ' in combination with --force-clean.') -parser.add_option( - '--dry-run', - action='store_true', - dest='dryrun', - default=False, - help="Output commands without executing them.") -parser.add_option('--dry-run-platform', dest='dryrunplatform', default=None, - help='Simulate a dry run on the specified platform '+\ - '(windows, mac, linux). Must be used in combination'+\ - ' with the --dry-run flag.') - -# Update-related options. -parser.add_option('--force-update', - action='store_true', dest='forceupdate', default=False, - help='Force a Chromium and CEF update. This will trigger a '+\ - 'new build and distribution.') -parser.add_option('--no-update', - action='store_true', dest='noupdate', default=False, - help='Do not update Chromium or CEF. Pass --force-build or '+\ - '--force-distrib if you desire a new build or '+\ - 'distribution.') -parser.add_option('--no-cef-update', - action='store_true', dest='nocefupdate', default=False, - help='Do not update CEF. Pass --force-build or '+\ - '--force-distrib if you desire a new build or '+\ - 'distribution.') -parser.add_option('--force-cef-update', - action='store_true', dest='forcecefupdate', default=False, - help='Force a CEF update. This will cause local changes in '+\ - 'the CEF checkout to be discarded and patch files to '+\ - 'be reapplied.') -parser.add_option( - '--no-chromium-update', - action='store_true', - dest='nochromiumupdate', - default=False, - help='Do not update Chromium.') -parser.add_option( - '--no-depot-tools-update', - action='store_true', - dest='nodepottoolsupdate', - default=False, - help='Do not update depot_tools.') -parser.add_option('--fast-update', - action='store_true', dest='fastupdate', default=False, - help='Update existing Chromium/CEF checkouts for fast incremental '+\ - 'builds by attempting to minimize the number of modified files. '+\ - 'The update will fail if there are unstaged CEF changes or if '+\ - 'Chromium changes are not included in a patch file.') -parser.add_option( - '--force-patch-update', - action='store_true', - dest='forcepatchupdate', - default=False, - help='Force update of patch files.') -parser.add_option( - '--resave', - action='store_true', - dest='resave', - default=False, - help='Resave patch files.') -parser.add_option( - '--log-chromium-changes', - action='store_true', - dest='logchromiumchanges', - default=False, - help='Create a log of the Chromium changes.') - -# Build-related options. -parser.add_option('--force-build', - action='store_true', dest='forcebuild', default=False, - help='Force CEF debug and release builds. This builds '+\ - '[build-target] on all platforms and chrome_sandbox '+\ - 'on Linux.') -parser.add_option( - '--no-build', - action='store_true', - dest='nobuild', - default=False, - help='Do not build CEF.') -parser.add_option( - '--build-target', - dest='buildtarget', - default='cefclient', - help='Target name(s) to build (defaults to "cefclient").') -parser.add_option( - '--build-tests', - action='store_true', - dest='buildtests', - default=False, - help='Also build the test target specified via --test-target.') -parser.add_option( - '--no-debug-build', - action='store_true', - dest='nodebugbuild', - default=False, - help="Don't perform the CEF debug build.") -parser.add_option( - '--no-release-build', - action='store_true', - dest='noreleasebuild', - default=False, - help="Don't perform the CEF release build.") -parser.add_option( - '--verbose-build', - action='store_true', - dest='verbosebuild', - default=False, - help='Show all command lines while building.') -parser.add_option( - '--build-failure-limit', - dest='buildfailurelimit', - default=1, - type="int", - help='Keep going until N jobs fail.') -parser.add_option('--build-log-file', - action='store_true', dest='buildlogfile', default=False, - help='Write build logs to file. The file will be named '+\ - '"build-[branch]-[debug|release].log" in the download '+\ - 'directory.') -parser.add_option( - '--x64-build', - action='store_true', - dest='x64build', - default=False, - help='Create a 64-bit build.') -parser.add_option( - '--arm-build', - action='store_true', - dest='armbuild', - default=False, - help='Create an ARM build.') -parser.add_option( - '--arm64-build', - action='store_true', - dest='arm64build', - default=False, - help='Create an ARM64 build.') -parser.add_option( - '--with-pgo-profiles', - action='store_true', - dest='withpgoprofiles', - default=False, - help='Download PGO profiles for the build.') - -# Test-related options. -parser.add_option( - '--run-tests', - action='store_true', - dest='runtests', - default=False, - help='Run the ceftests target.') -parser.add_option( - '--no-debug-tests', - action='store_true', - dest='nodebugtests', - default=False, - help="Don't run debug build tests.") -parser.add_option( - '--no-release-tests', - action='store_true', - dest='noreleasetests', - default=False, - help="Don't run release build tests.") -parser.add_option( - '--test-target', - dest='testtarget', - default='ceftests', - help='Test target name to build (defaults to "ceftests").') -parser.add_option( - '--test-prefix', - dest='testprefix', - default='', - help='Prefix for running the test executable (e.g. `xvfb-run` on Linux).') -parser.add_option( - '--test-args', - dest='testargs', - default='', - help='Arguments that will be passed to the test executable.') - -# Distribution-related options. -parser.add_option( - '--force-distrib', - action='store_true', - dest='forcedistrib', - default=False, - help='Force creation of a CEF binary distribution.') -parser.add_option( - '--no-distrib', - action='store_true', - dest='nodistrib', - default=False, - help="Don't create a CEF binary distribution.") -parser.add_option( - '--minimal-distrib', - action='store_true', - dest='minimaldistrib', - default=False, - help='Create a minimal CEF binary distribution.') -parser.add_option( - '--minimal-distrib-only', - action='store_true', - dest='minimaldistribonly', - default=False, - help='Create a minimal CEF binary distribution only.') -parser.add_option( - '--client-distrib', - action='store_true', - dest='clientdistrib', - default=False, - help='Create a client CEF binary distribution.') -parser.add_option( - '--client-distrib-only', - action='store_true', - dest='clientdistribonly', - default=False, - help='Create a client CEF binary distribution only.') -parser.add_option( - '--sandbox-distrib', - action='store_true', - dest='sandboxdistrib', - default=False, - help='Create a cef_sandbox static library distribution.') -parser.add_option( - '--sandbox-distrib-only', - action='store_true', - dest='sandboxdistribonly', - default=False, - help='Create a cef_sandbox static library distribution only.') -parser.add_option( - '--no-distrib-docs', - action='store_true', - dest='nodistribdocs', - default=False, - help="Don't create CEF documentation.") -parser.add_option( - '--no-distrib-archive', - action='store_true', - dest='nodistribarchive', - default=False, - help="Don't create archives for output directories.") -parser.add_option( - '--clean-artifacts', - action='store_true', - dest='cleanartifacts', - default=False, - help='Clean the artifacts output directory.') -parser.add_option( - '--distrib-subdir', - dest='distribsubdir', - default='', - help='CEF distrib dir name, child of chromium/src/cef/binary_distrib') -parser.add_option( - '--distrib-subdir-suffix', - dest='distribsubdirsuffix', - default='', - help='CEF distrib dir name suffix, child of chromium/src/cef/binary_distrib' -) - -(options, args) = parser.parse_args() - -if options.downloaddir is None: - print("The --download-dir option is required.") - parser.print_help(sys.stderr) - sys.exit(1) - -# Opt into component-specific flags for later use. -if options.noupdate: - options.nocefupdate = True - options.nochromiumupdate = True - options.nodepottoolsupdate = True - -if options.runtests: - options.buildtests = True - -if (options.nochromiumupdate and options.forceupdate) or \ - (options.nocefupdate and options.forceupdate) or \ - (options.nobuild and options.forcebuild) or \ - (options.nodistrib and options.forcedistrib) or \ - ((options.forceclean or options.forcecleandeps) and options.fastupdate): - print("Invalid combination of options.") - parser.print_help(sys.stderr) - sys.exit(1) - -if (options.noreleasebuild and \ - (options.minimaldistrib or options.minimaldistribonly or \ - options.clientdistrib or options.clientdistribonly)) or \ - (options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly > 1): - print('Invalid combination of options.') - parser.print_help(sys.stderr) - sys.exit(1) - -if options.x64build + options.armbuild + options.arm64build > 1: - print('Invalid combination of options.') - parser.print_help(sys.stderr) - sys.exit(1) - -if (options.buildtests or options.runtests) and len(options.testtarget) == 0: - print("A test target must be specified via --test-target.") - parser.print_help(sys.stderr) - sys.exit(1) - -# Operating system. -if options.dryrun and options.dryrunplatform is not None: - platform = options.dryrunplatform - if not platform in ['windows', 'mac', 'linux']: - print('Invalid dry-run-platform value: %s' % (platform)) - sys.exit(1) -elif sys.platform == 'win32': - platform = 'windows' -elif sys.platform == 'darwin': - platform = 'mac' -elif sys.platform.startswith('linux'): - platform = 'linux' -else: - print('Unknown operating system platform') - sys.exit(1) - -if options.clientdistrib or options.clientdistribonly: - if platform == 'linux' or (platform == 'windows' and options.arm64build): - client_app = 'cefsimple' - else: - client_app = 'cefclient' - if options.buildtarget.find(client_app) == -1: - print('A client distribution cannot be generated if --build-target ' + - 'excludes %s.' % client_app) - parser.print_help(sys.stderr) - sys.exit(1) - -# CEF branch. -cef_branch = options.branch - -branch_is_master = (cef_branch == 'master' or cef_branch == 'trunk') -if not branch_is_master: - # Verify that the branch value is numeric. - if not cef_branch.isdigit(): - print('Invalid branch value: %s' % cef_branch) - sys.exit(1) - - # Verify the minimum supported branch number. - if int(cef_branch) < 3071: - print('The requested branch (%s) is too old to build using this tool. ' + - 'The minimum supported branch is 3071.' % cef_branch) - sys.exit(1) - -# True if the requested branch is 3538 or newer. -branch_is_3538_or_newer = (branch_is_master or int(cef_branch) >= 3538) - -# True if the requested branch is 3945 or newer. -branch_is_3945_or_newer = (branch_is_master or int(cef_branch) >= 3945) - -# Enable Python 3 usage in Chromium for branches 3945 and newer. -if branch_is_3945_or_newer and not is_python2 and \ - not 'GCLIENT_PY3' in os.environ.keys(): - os.environ['GCLIENT_PY3'] = '1' - -if not branch_is_3945_or_newer and \ - (not is_python2 or bool(int(os.environ.get('GCLIENT_PY3', '0')))): - print('Python 3 is not supported with branch 3904 and older ' + - '(set GCLIENT_PY3=0 and run with Python 2 executable).') - sys.exit(1) - -if options.armbuild: - if platform != 'linux': - print('The ARM build option is only supported on Linux.') - sys.exit(1) - -deps_file = 'DEPS' - -if platform == 'mac' and not (options.x64build or options.arm64build): - print('32-bit MacOS builds are not supported. ' + - 'Add --x64-build or --arm64-build flag to generate a 64-bit build.') - sys.exit(1) - -# Platforms that build a cef_sandbox library. -sandbox_lib_platforms = ['windows'] -if branch_is_3538_or_newer: - sandbox_lib_platforms.append('mac') - -if not platform in sandbox_lib_platforms and (options.sandboxdistrib or - options.sandboxdistribonly): - print('The sandbox distribution is not supported on this platform.') - sys.exit(1) - -# Options that force the sources to change. -force_change = options.forceclean or options.forceupdate - -# Options that cause local changes to be discarded. -discard_local_changes = force_change or options.forcecefupdate - -if options.resave and (options.forcepatchupdate or discard_local_changes): - print('--resave cannot be combined with options that modify or discard ' + - 'patches.') - parser.print_help(sys.stderr) - sys.exit(1) - -if platform == 'windows': - # Avoid errors when the "vs_toolchain.py update" Chromium hook runs. - os.environ['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' - -download_dir = os.path.abspath(options.downloaddir) -chromium_dir = os.path.join(download_dir, 'chromium') -chromium_src_dir = os.path.join(chromium_dir, 'src') -out_src_dir = os.path.join(chromium_src_dir, 'out') -cef_src_dir = os.path.join(chromium_src_dir, 'cef') - -if options.fastupdate and os.path.exists(cef_src_dir): - cef_dir = cef_src_dir -else: - cef_dir = os.path.join(download_dir, 'cef') - -## -# Manage the download directory. -## - -# Create the download directory if necessary. -create_directory(download_dir) - -msg("Download Directory: %s" % (download_dir)) - -## -# Manage the depot_tools directory. -## - -# Check if the depot_tools directory exists. -if options.depottoolsdir != '': - depot_tools_dir = os.path.abspath(options.depottoolsdir) -else: - depot_tools_dir = os.path.join(download_dir, 'depot_tools') - -msg("Depot Tools Directory: %s" % (depot_tools_dir)) - -if not os.path.exists(depot_tools_dir): - if platform == 'windows' and options.depottoolsarchive == '': - # On Windows download depot_tools as an archive file since we can't assume - # that git is already installed. - options.depottoolsarchive = depot_tools_archive_url - - if options.depottoolsarchive != '': - # Extract depot_tools from an archive file. - msg('Extracting %s to %s.' % \ - (options.depottoolsarchive, depot_tools_dir)) - if not options.dryrun: - download_and_extract(options.depottoolsarchive, depot_tools_dir) - else: - # On Linux and OS X check out depot_tools using Git. - run('git clone ' + depot_tools_url + ' ' + depot_tools_dir, download_dir) - -if not options.nodepottoolsupdate: - # Update depot_tools. - # On Windows this will download required python and git binaries. - msg('Updating depot_tools') - if platform == 'windows': - run('update_depot_tools.bat', depot_tools_dir, depot_tools_dir) - else: - run('update_depot_tools', depot_tools_dir, depot_tools_dir) - -# Determine the executables to use. -if platform == 'windows': - # Force use of the version bundled with depot_tools. - git_exe = os.path.join(depot_tools_dir, 'git.bat') - python_bat = 'python.bat' if is_python2 else 'python3.bat' - python_exe = os.path.join(depot_tools_dir, python_bat) - if options.dryrun and not os.path.exists(git_exe): - sys.stdout.write("WARNING: --dry-run assumes that depot_tools" \ - " is already in your PATH. If it isn't\nplease" \ - " specify a --depot-tools-dir value.\n") - git_exe = 'git.bat' - python_exe = python_bat -else: - git_exe = 'git' - python_exe = sys.executable - -## -# Manage the cef directory. -## - -# Delete the existing CEF directory if requested. -if options.forceclean and os.path.exists(cef_dir): - delete_directory(cef_dir) - -# Determine the type of CEF checkout to use. -if os.path.exists(cef_dir) and not is_git_checkout(cef_dir): - raise Exception("Not a valid CEF Git checkout: %s" % (cef_dir)) - -# Determine the CEF download URL to use. -cef_url = options.url.strip() -if cef_url == '': - cef_url = cef_git_url - -# Verify that the requested CEF URL matches the existing checkout. -if not options.nocefupdate and os.path.exists(cef_dir): - cef_existing_url = get_git_url(cef_dir) - if cef_url != cef_existing_url: - raise Exception( - 'Requested CEF checkout URL %s does not match existing URL %s' % - (cef_url, cef_existing_url)) - -msg("CEF Branch: %s" % (cef_branch)) -msg("CEF URL: %s" % (cef_url)) -msg("CEF Source Directory: %s" % (cef_dir)) - -# Determine the CEF Git branch to use. -if options.checkout == '': - # Target the most recent branch commit from the remote repo. - if branch_is_master: - cef_checkout = 'origin/master' - else: - cef_checkout = 'origin/' + cef_branch -else: - cef_checkout = options.checkout - -# Create the CEF checkout if necessary. -if not options.nocefupdate and not os.path.exists(cef_dir): - cef_checkout_new = True - run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir, - depot_tools_dir) -else: - cef_checkout_new = False - -# Determine if the CEF checkout needs to change. -if not options.nocefupdate and os.path.exists(cef_dir): - cef_current_hash = get_git_hash(cef_dir, 'HEAD') - - if not cef_checkout_new: - # Fetch updated sources. - run('%s fetch' % (git_exe), cef_dir, depot_tools_dir) - - cef_desired_hash = get_git_hash(cef_dir, cef_checkout) - cef_checkout_changed = cef_checkout_new or force_change or \ - options.forcecefupdate or \ - cef_current_hash != cef_desired_hash - - msg("CEF Current Checkout: %s" % (cef_current_hash)) - msg("CEF Desired Checkout: %s (%s)" % (cef_desired_hash, cef_checkout)) - - if cef_checkout_changed: - if cef_dir == cef_src_dir: - # Running in fast update mode. Backup and revert the patched files before - # changing the CEF checkout. - run_patch_updater("--backup --revert") - - # Update the CEF checkout. - run('%s checkout %s%s' % - (git_exe, '--force ' if discard_local_changes else '', cef_checkout), \ - cef_dir, depot_tools_dir) -else: - cef_checkout_changed = False - -build_compat_versions = get_build_compat_versions() - -if not options.nodepottoolsupdate and \ - 'depot_tools_checkout' in build_compat_versions: - # Update the depot_tools checkout. - depot_tools_compat_version = build_compat_versions['depot_tools_checkout'] - run('%s checkout %s%s' % - (git_exe, '--force ' if discard_local_changes else '', depot_tools_compat_version), \ - depot_tools_dir, depot_tools_dir) - -# Disable further depot_tools updates. -os.environ['DEPOT_TOOLS_UPDATE'] = '0' - -## -# Manage the out directory. -## - -out_dir = os.path.join(download_dir, 'out_' + cef_branch) - -# Delete the existing out directory if requested. -if options.forceclean and os.path.exists(out_dir): - delete_directory(out_dir) - -msg("CEF Output Directory: %s" % (out_dir)) - -## -# Manage the chromium directory. -## - -# Create the chromium directory if necessary. -create_directory(chromium_dir) - -if options.chromiumurl != '': - chromium_url = options.chromiumurl -else: - chromium_url = 'https://chromium.googlesource.com/chromium/src.git' - -# Create gclient configuration file. -gclient_file = os.path.join(chromium_dir, '.gclient') -if not os.path.exists(gclient_file) or options.forceconfig: - # Exclude unnecessary directories. Intentionally written without newlines. - gclient_spec = \ - "solutions = [{"+\ - "'managed': False,"+\ - "'name': 'src', "+\ - "'url': '" + chromium_url + "', "+\ - "'custom_vars': {"+\ - "'checkout_pgo_profiles': " + ('True' if options.withpgoprofiles else 'False') + ", "+\ - "}, "+\ - "'custom_deps': {"+\ - "'build': None, "+\ - "'build/scripts/command_wrapper/bin': None, "+\ - "'build/scripts/gsd_generate_index': None, "+\ - "'build/scripts/private/data/reliability': None, "+\ - "'build/scripts/tools/deps2git': None, "+\ - "'build/third_party/lighttpd': None, "+\ - "'commit-queue': None, "+\ - "'depot_tools': None, "+\ - "'src/chrome_frame/tools/test/reference_build/chrome': None, "+\ - "'src/chrome/tools/test/reference_build/chrome_linux': None, "+\ - "'src/chrome/tools/test/reference_build/chrome_mac': None, "+\ - "'src/chrome/tools/test/reference_build/chrome_win': None, "+\ - "}, "+\ - "'deps_file': '" + deps_file + "', "+\ - "'safesync_url': ''"+\ - "}]" - - msg('Writing %s' % gclient_file) - if not options.dryrun: - with open(gclient_file, 'w', encoding='utf-8') as fp: - write_fp(fp, gclient_spec) - -# Initial Chromium checkout. -if not options.nochromiumupdate and not os.path.exists(chromium_src_dir): - chromium_checkout_new = True - run("gclient sync --nohooks --with_branch_heads --jobs 16", \ - chromium_dir, depot_tools_dir) -else: - chromium_checkout_new = False - -# Verify the Chromium checkout. -if not options.dryrun and not is_git_checkout(chromium_src_dir): - raise Exception('Not a valid git checkout: %s' % (chromium_src_dir)) - -if os.path.exists(chromium_src_dir): - msg("Chromium URL: %s" % (get_git_url(chromium_src_dir))) - -# Fetch Chromium changes so that we can perform the necessary calculations using -# local history. -if not options.nochromiumupdate and os.path.exists(chromium_src_dir): - # Fetch updated sources. - run("%s fetch" % (git_exe), chromium_src_dir, depot_tools_dir) - # Also fetch tags, which are required for release branch builds. - run("%s fetch --tags" % (git_exe), chromium_src_dir, depot_tools_dir) - -# Determine the Chromium checkout options required by CEF. -chromium_compat_version = build_compat_versions['chromium_checkout'] -if len(options.chromiumcheckout) > 0: - chromium_checkout = options.chromiumcheckout -else: - chromium_checkout = chromium_compat_version - -# Determine if the Chromium checkout needs to change. -if not options.nochromiumupdate and os.path.exists(chromium_src_dir): - chromium_current_hash = get_git_hash(chromium_src_dir, 'HEAD') - chromium_desired_hash = get_git_hash(chromium_src_dir, chromium_checkout) - chromium_checkout_changed = chromium_checkout_new or force_change or \ - chromium_current_hash != chromium_desired_hash - - msg("Chromium Current Checkout: %s" % (chromium_current_hash)) - msg("Chromium Desired Checkout: %s (%s)" % \ - (chromium_desired_hash, chromium_checkout)) -else: - chromium_checkout_changed = options.dryrun - -if cef_checkout_changed: - if cef_dir != cef_src_dir and os.path.exists(cef_src_dir): - # Delete the existing src/cef directory. It will be re-copied from the - # download directory later. - delete_directory(cef_src_dir) -elif chromium_checkout_changed and cef_dir == cef_src_dir: - # Running in fast update mode. Backup and revert the patched files before - # changing the Chromium checkout. - run_patch_updater("--backup --revert") - -# Delete the existing src/out directory if requested. -if options.forceclean and os.path.exists(out_src_dir): - delete_directory(out_src_dir) - -# Move the existing src/out directory to the correct location in the download -# directory. It will be moved back from the download directory later. -if os.path.exists(out_src_dir): - old_branch = read_branch_config_file(out_src_dir) - if old_branch != '' and (chromium_checkout_changed or - old_branch != cef_branch): - old_out_dir = os.path.join(download_dir, 'out_' + old_branch) - move_directory(out_src_dir, old_out_dir) - -# Update the Chromium checkout. -if chromium_checkout_changed: - if not chromium_checkout_new and not options.fastupdate: - if options.forceclean and options.forcecleandeps: - # Remove all local changes including third-party git checkouts managed by - # gclient. - run("%s clean -dffx" % (git_exe), chromium_src_dir, depot_tools_dir) - else: - # Revert all changes in the Chromium checkout. - run("gclient revert --nohooks", chromium_dir, depot_tools_dir) - - # Checkout the requested branch. - run("%s checkout %s%s" % \ - (git_exe, '--force ' if discard_local_changes else '', chromium_checkout), \ - chromium_src_dir, depot_tools_dir) - - # Patch the Chromium DEPS file if necessary. - apply_deps_patch() - - # Update third-party dependencies including branch/tag information. - run("gclient sync %s--nohooks --with_branch_heads --jobs 16" % \ - ('--reset ' if discard_local_changes else ''), chromium_dir, depot_tools_dir) - - # Patch the Chromium runhooks scripts if necessary. - apply_runhooks_patch() - - # Runs hooks for files that have been modified in the local working copy. - run("gclient runhooks --jobs 16", chromium_dir, depot_tools_dir) - - # Delete the src/out directory created by `gclient sync`. - delete_directory(out_src_dir) - -if cef_dir == cef_src_dir: - # Running in fast update mode. - if cef_checkout_changed or chromium_checkout_changed: - # Check and restore the patched files. - run_patch_updater("--reapply --restore") -elif os.path.exists(cef_dir) and not os.path.exists(cef_src_dir): - # Restore the src/cef directory. - copy_directory(cef_dir, cef_src_dir) - -# Restore the src/out directory. -out_src_dir_exists = os.path.exists(out_src_dir) -if os.path.exists(out_dir) and not out_src_dir_exists: - move_directory(out_dir, out_src_dir) - out_src_dir_exists = True -elif not out_src_dir_exists: - create_directory(out_src_dir) - -# Write the config file for identifying the branch. -write_branch_config_file(out_src_dir, cef_branch) - -if options.logchromiumchanges and chromium_checkout != chromium_compat_version: - log_chromium_changes() - -if options.forcepatchupdate or ((chromium_checkout_new or not options.fastupdate) and \ - chromium_checkout_changed and \ - chromium_checkout != chromium_compat_version): - # Not using the known-compatible Chromium version. Try to update patch files. - if options.logchromiumchanges: - out_file = os.path.join(download_dir, 'chromium_update_patches.txt') - if os.path.exists(out_file): - os.remove(out_file) - else: - out_file = None - run_patch_updater(output_file=out_file) -elif options.resave: - # Resave patch files. - run_patch_updater("--resave") - -if chromium_checkout != chromium_compat_version: - if options.logchromiumchanges: - out_file = os.path.join(download_dir, 'chromium_update_patterns.txt') - if os.path.exists(out_file): - os.remove(out_file) - else: - out_file = None - check_pattern_matches(output_file=out_file) - -## -# Build CEF. -## - -if not options.nobuild and (chromium_checkout_changed or \ - cef_checkout_changed or options.forcebuild or \ - not out_src_dir_exists): - # Building should also force a distribution. - options.forcedistrib = True - - # Make sure the GN configuration exists. - if not options.dryrun and \ - not os.path.exists(os.path.join(cef_src_dir, 'BUILD.gn')): - raise Exception('GN configuration does not exist.') - - # Print all build-related environment variables including any that were set - # previously. - for key in os.environ.keys(): - if key.startswith('CEF_') or key.startswith('GCLIENT_') or \ - key.startswith('GN_') or key.startswith('GYP_') or \ - key.startswith('DEPOT_TOOLS_'): - msg('%s=%s' % (key, os.environ[key])) - - # Generate project files. - tool = os.path.join(cef_src_dir, 'tools', 'gclient_hook.py') - run('%s %s' % (python_exe, tool), cef_src_dir, depot_tools_dir) - - # Build using Ninja. - command = 'ninja ' - if options.verbosebuild: - command += '-v ' - if options.buildfailurelimit != 1: - command += '-k %d ' % options.buildfailurelimit - command += '-C ' - target = ' ' + options.buildtarget - if options.buildtests: - target += ' ' + options.testtarget - if platform == 'linux': - target += ' chrome_sandbox' - - # Make a CEF Debug build. - if not options.nodebugbuild: - build_path = os.path.join('out', get_build_directory_name(True)) - args_path = os.path.join(chromium_src_dir, build_path, 'args.gn') - msg(args_path + ' contents:\n' + read_file(args_path)) - - run(command + build_path + target, chromium_src_dir, depot_tools_dir, - os.path.join(download_dir, 'build-%s-debug.log' % (cef_branch)) \ - if options.buildlogfile else None) - - if platform in sandbox_lib_platforms: - # Make the separate cef_sandbox build when GN is_official_build=true. - build_path += '_sandbox' - if os.path.exists(os.path.join(chromium_src_dir, build_path)): - args_path = os.path.join(chromium_src_dir, build_path, 'args.gn') - msg(args_path + ' contents:\n' + read_file(args_path)) - - run(command + build_path + ' cef_sandbox', chromium_src_dir, depot_tools_dir, - os.path.join(download_dir, 'build-%s-debug-sandbox.log' % (cef_branch)) \ - if options.buildlogfile else None) - - # Make a CEF Release build. - if not options.noreleasebuild: - build_path = os.path.join('out', get_build_directory_name(False)) - args_path = os.path.join(chromium_src_dir, build_path, 'args.gn') - msg(args_path + ' contents:\n' + read_file(args_path)) - - run(command + build_path + target, chromium_src_dir, depot_tools_dir, - os.path.join(download_dir, 'build-%s-release.log' % (cef_branch)) \ - if options.buildlogfile else None) - - if platform in sandbox_lib_platforms: - # Make the separate cef_sandbox build when GN is_official_build=true. - build_path += '_sandbox' - if os.path.exists(os.path.join(chromium_src_dir, build_path)): - args_path = os.path.join(chromium_src_dir, build_path, 'args.gn') - msg(args_path + ' contents:\n' + read_file(args_path)) - - run(command + build_path + ' cef_sandbox', chromium_src_dir, depot_tools_dir, - os.path.join(download_dir, 'build-%s-release-sandbox.log' % (cef_branch)) \ - if options.buildlogfile else None) - -elif not options.nobuild: - msg('Not building. The source hashes have not changed and ' + - 'the output folder "%s" already exists' % (out_src_dir)) - -## -# Run CEF tests. -## - -if options.runtests: - if platform == 'windows': - test_exe = '%s.exe' % options.testtarget - elif platform == 'mac': - test_exe = '%s.app/Contents/MacOS/%s' % (options.testtarget, - options.testtarget) - elif platform == 'linux': - test_exe = options.testtarget - - test_prefix = options.testprefix - if len(test_prefix) > 0: - test_prefix += ' ' - - test_args = options.testargs - if len(test_args) > 0: - test_args = ' ' + test_args - - if not options.nodebugtests: - build_path = os.path.join(out_src_dir, get_build_directory_name(True)) - test_path = os.path.join(build_path, test_exe) - if os.path.exists(test_path): - run(test_prefix + test_path + test_args, build_path, depot_tools_dir) - else: - msg('Not running debug tests. Missing executable: %s' % test_path) - - if not options.noreleasetests: - build_path = os.path.join(out_src_dir, get_build_directory_name(False)) - test_path = os.path.join(build_path, test_exe) - if os.path.exists(test_path): - run(test_prefix + test_path + test_args, build_path, depot_tools_dir) - else: - msg('Not running release tests. Missing executable: %s' % test_path) - -## -# Create the CEF binary distribution. -## - -if not options.nodistrib and (chromium_checkout_changed or \ - cef_checkout_changed or options.forcedistrib): - if not options.forceclean and options.cleanartifacts: - # Clean the artifacts output directory. - artifacts_path = os.path.join(cef_src_dir, 'binary_distrib') - delete_directory(artifacts_path) - - # Determine the requested distribution types. - distrib_types = [] - if options.minimaldistribonly: - distrib_types.append('minimal') - elif options.clientdistribonly: - distrib_types.append('client') - elif options.sandboxdistribonly: - distrib_types.append('sandbox') - else: - distrib_types.append('standard') - if options.minimaldistrib: - distrib_types.append('minimal') - if options.clientdistrib: - distrib_types.append('client') - if options.sandboxdistrib: - distrib_types.append('sandbox') - - cef_tools_dir = os.path.join(cef_src_dir, 'tools') - - # Create the requested distribution types. - first_type = True - for type in distrib_types: - path = '%s make_distrib.py --output-dir=../binary_distrib/' % python_exe - - if options.nodebugbuild or options.noreleasebuild or type != 'standard': - path += ' --allow-partial' - path = path + ' --ninja-build' - if options.x64build: - path += ' --x64-build' - elif options.armbuild: - path += ' --arm-build' - elif options.arm64build: - path += ' --arm64-build' - - if type == 'minimal': - path += ' --minimal' - elif type == 'client': - path += ' --client' - elif type == 'sandbox': - path += ' --sandbox' - - if first_type: - if options.nodistribdocs: - path += ' --no-docs' - if options.nodistribarchive: - path += ' --no-archive' - first_type = False - else: - # Don't create the symbol archives or documentation more than once. - path += ' --no-symbols --no-docs' - - # Override the subdirectory name of binary_distrib if the caller requested. - if options.distribsubdir != '': - path += ' --distrib-subdir=' + options.distribsubdir - if options.distribsubdirsuffix != '': - path += ' --distrib-subdir-suffix=' + options.distribsubdirsuffix - - # Create the distribution. - run(path, cef_tools_dir, depot_tools_dir) \ No newline at end of file