diff --git a/.flake8 b/.flake8 --- a/.flake8 +++ b/.flake8 @@ -1,25 +1,24 @@ [flake8] # See http://pep8.readthedocs.io/en/latest/intro.html#configuration ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402, E741 max-line-length = 99 exclude = browser/extensions/mortar/ppapi/, browser/moz.configure, build/moz.configure/*.configure, build/pymake/, dom/canvas/test/webgl-conf/checkout/closure-library/, editor/libeditor/tests/browserscope/, intl/icu/, ipc/chromium/, gfx/angle/, gfx/harfbuzz, glx/skia/, js/*.configure, memory/moz.configure, mobile/android/*.configure, node_modules, security/nss/, testing/mochitest/pywebsocket, tools/lint/test/files, build/build-infer/build-infer.py, - tools/infer/test/*.configure, diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,191 +1,190 @@ # .hgignore - List of filenames hg should ignore # Filenames that should be ignored wherever they appear ~$ \.py(c|o)$ (?i)(^|/)TAGS$ (^|/)ID$ (^|/)\.DS_Store$ \.pdb \.egg-info \.gcda \.gcno \.gcov compile_commands\.json # Vim swap files. ^\.sw.$ .[^/]*\.sw.$ # Emacs directory variable files. \.dir-locals\.el # User files that may appear at the root ^\.mozconfig ^mozconfig* ^configure$ ^old-configure$ ^config\.cache$ ^config\.log$ ^\.clang_complete ^\.?machrc$ # Empty marker file that's generated when we check out NSS ^security/manager/\.nss\.checkout$ # Build directories ^obj # gecko.log is generated by various test harnesses ^gecko\.log # Build directories for js shell _DBG\.OBJ/ _OPT\.OBJ/ ^js/src/.*-obj/ ^js/src/obj-.*/ # SpiderMonkey configury ^js/src/configure$ ^js/src/old-configure$ ^js/src/autom4te.cache$ # SpiderMonkey test result logs ^js/src/tests/results-.*\.(html|txt)$ ^js/src/devtools/rootAnalysis/t/out # SpiderMonkey clone of the webassembly spec repository ^js/src/wasm/spec # Java HTML5 parser classes ^parser/html/java/(html|java)parser/ # SVN directories \.svn/ # Ignore the files and directory that Eclipse IDE creates \.project$ \.cproject$ \.settings/ # Ignore the files and directory that JetBrains IDEs create. \.idea/ \.iml$ # Android Monitor in Android Studio creates a captures/ directory. ^captures/ # Gradle cache. ^.gradle/ -^tools/infer/test/.gradle/ # Local Gradle configuration properties. ^local.properties$ # Python stuff installed at build time. ^third_party/python/psutil/.*\.so ^third_party/python/psutil/.*\.pyd ^third_party/python/psutil/build/ ^third_party/python/psutil/tmp/ # Git repositories .git/ # Ignore chrome.manifest files from the devtools loader ^devtools/client/chrome.manifest$ ^devtools/shared/chrome.manifest$ # Ignore node_modules directories in devtools ^devtools/.*/node_modules/ # git checkout of libstagefright ^media/libstagefright/android$ # Tag files generated by GNU Global GTAGS GRTAGS GSYMS GPATH # Git clone directory for updating web-platform-tests ^testing/web-platform/sync/ # Third party metadata for web-platform-tests ^testing/web-platform/products/ # Android Gradle artifacts. ^mobile/android/gradle/.gradle # XCode project cruft ^[^/]*\.xcodeproj/ ^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/project.xcworkspace/xcuserdata ^embedding/ios/GeckoEmbed/GeckoEmbed.xcodeproj/xcuserdata # Rust/Cargo output from running `cargo` directly ^target/ # Ignore mozharness execution files ^testing/mozharness/.tox/ ^testing/mozharness/build/ ^testing/mozharness/logs/ ^testing/mozharness/.coverage ^testing/mozharness/nosetests.xml # Ignore tox generated dir .tox/ # Ignore ESLint node_modules ^node_modules/ ^tools/lint/eslint/eslint-plugin-mozilla/node_modules/ # Ignore talos virtualenv and tp5n files. # The tp5n set is supposed to be decompressed at # testing/talos/talos/tests/tp5n in order to run tests like tps # locally. Similarly, running talos requires a Python package virtual # environment. Both the virtual environment and tp5n files end up littering # the status command, so we ignore them. ^testing/talos/.Python ^testing/talos/bin/ ^testing/talos/include/ ^testing/talos/lib/ ^testing/talos/talos/tests/tp5n.zip ^testing/talos/talos/tests/tp5n.tar.gz ^testing/talos/talos/tests/tp5n ^testing/talos/talos/tests/devtools/damp.manifest.develop ^talos-venv ^py3venv ^testing/talos/talos/mitmproxy/mitmdump ^testing/talos/talos/mitmproxy/mitmproxy ^testing/talos/talos/mitmproxy/mitmweb # Ignore talos webkit benchmark files; source is copied from in-tree /third_party # into testing/talos/talos/tests/webkit/PerformanceTests/ when run locally # i.e. speedometer, motionmark, stylebench ^testing/talos/talos/tests/webkit/PerformanceTests # Ignore toolchains.json created by tooltool. ^toolchains\.json # Ignore files created when running a reftest. ^lextab.py$ # tup database ^\.tup # Ignore sync tps logs and reports tps\.log tps_result\.json # Ignore Visual Studio Code workspace files. \.vscode/(?!extensions\.json|tasks\.json) subinclude:servo/.hgignore # Ignore Infer output ^infer-out/ # https://bz.mercurial-scm.org/show_bug.cgi?id=5322 ^comm/ # Ignore various raptor performance framework files ^testing/raptor/.raptor-venv ^testing/raptor/raptor-venv ^testing/raptor/raptor/tests/.*.json ^testing/raptor/webext/raptor/auto_gen_test_config.js diff --git a/moz.configure b/moz.configure --- a/moz.configure +++ b/moz.configure @@ -1,620 +1,610 @@ # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. include('build/moz.configure/init.configure') # Note: # - Gecko-specific options and rules should go in toolkit/moz.configure. # - Firefox-specific options and rules should go in browser/moz.configure. # - Fennec-specific options and rules should go in # mobile/android/moz.configure. # - Spidermonkey-specific options and rules should go in js/moz.configure. # - etc. option('--enable-artifact-builds', env='MOZ_ARTIFACT_BUILDS', help='Download and use prebuilt binary artifacts.') @depends('--enable-artifact-builds') def artifact_builds(value): if value: return True set_config('MOZ_ARTIFACT_BUILDS', artifact_builds) imply_option('--enable-artifact-build-symbols', depends(artifact_builds)(lambda v: False if v is None else None), reason='--disable-artifact-builds') option('--enable-artifact-build-symbols', help='Download symbols when artifact builds are enabled.') set_config('MOZ_ARTIFACT_BUILD_SYMBOLS', depends_if('--enable-artifact-build-symbols')(lambda _: True)) @depends('--enable-artifact-builds') def imply_disable_compile_environment(value): if value: return False imply_option('--enable-compile-environment', imply_disable_compile_environment) option('--disable-compile-environment', help='Disable compiler/library checks') @depends('--disable-compile-environment') def compile_environment(compile_env): if compile_env: return True set_config('COMPILE_ENVIRONMENT', compile_environment) add_old_configure_assignment('COMPILE_ENVIRONMENT', compile_environment) js_option('--disable-tests', help='Do not build test libraries & programs') @depends('--disable-tests') def enable_tests(value): if value: return True set_config('ENABLE_TESTS', enable_tests) set_define('ENABLE_TESTS', enable_tests) js_option(env='MOZILLA_OFFICIAL', help='Build an official release') @depends('MOZILLA_OFFICIAL') def mozilla_official(official): if official: return True set_config('MOZILLA_OFFICIAL', mozilla_official) set_define('MOZILLA_OFFICIAL', mozilla_official) add_old_configure_assignment('MOZILLA_OFFICIAL', mozilla_official) @depends(enable_tests) def gtest_has_rtti(value): if value: return '0' set_define('GTEST_HAS_RTTI', gtest_has_rtti) @depends(target, enable_tests) def linux_gtest_defines(target, enable_tests): if enable_tests and target.os == 'Android': return namespace(os_linux_android=True, use_own_tr1_tuple=True, has_clone='0') set_define('GTEST_OS_LINUX_ANDROID', linux_gtest_defines.os_linux_android) set_define('GTEST_USE_OWN_TR1_TUPLE', linux_gtest_defines.use_own_tr1_tuple) set_define('GTEST_HAS_CLONE', linux_gtest_defines.has_clone) js_option('--enable-debug', nargs='?', help='Enable building with developer debug info ' '(using the given compiler flags).') @depends('--enable-debug') def moz_debug(debug): if debug: return bool(debug) set_config('MOZ_DEBUG', moz_debug) set_define('MOZ_DEBUG', moz_debug) # Override any value MOZ_DEBUG may have from the environment when passing it # down to old-configure. add_old_configure_assignment('MOZ_DEBUG', depends('--enable-debug')(lambda x: bool(x))) js_option('--enable-rust-debug', help='Build Rust code with debug assertions turned on.') @depends('--enable-rust-debug', '--enable-debug') def debug_rust(value, debug): if value.origin == 'default': return bool(debug) or None elif bool(value): return True set_config('MOZ_DEBUG_RUST', debug_rust) set_define('MOZ_DEBUG_RUST', debug_rust) include('build/moz.configure/pkg.configure') # Make this assignment here rather than in pkg.configure to avoid # requiring this file in unit tests. add_old_configure_assignment('PKG_CONFIG', pkg_config) include('build/moz.configure/toolchain.configure', when='--enable-compile-environment') include('build/moz.configure/memory.configure', when='--enable-compile-environment') include('build/moz.configure/headers.configure', when='--enable-compile-environment') include('build/moz.configure/warnings.configure', when='--enable-compile-environment') include('build/moz.configure/flags.configure', when='--enable-compile-environment') # rust.configure is included by js/moz.configure. js_option('--enable-valgrind', help='Enable Valgrind integration hooks') valgrind_h = check_header('valgrind/valgrind.h', when='--enable-valgrind') @depends('--enable-valgrind', valgrind_h) def check_valgrind(valgrind, valgrind_h): if valgrind: if not valgrind_h: die('--enable-valgrind specified but Valgrind is not installed') return True set_define('MOZ_VALGRIND', check_valgrind) set_config('MOZ_VALGRIND', check_valgrind) @depends(target, host) def is_openbsd(target, host): return target.kernel == 'OpenBSD' or host.kernel == 'OpenBSD' option(env='SO_VERSION', nargs=1, default='1.0', when=is_openbsd, help='Shared library version for OpenBSD systems') @depends('SO_VERSION', when=is_openbsd) def so_version(value): return value @template def library_name_info_template(host_or_target): assert host_or_target in (host, target) compiler = { host: host_c_compiler, target: c_compiler, }[host_or_target] @depends(host_or_target, compiler, so_version) def library_name_info_impl(host_or_target, compiler, so_version): if host_or_target.kernel == 'WINNT': # There aren't artifacts for mingw builds, so it's OK that the # results are inaccurate in that case. if compiler and compiler.type not in ('msvc', 'clang-cl'): return namespace( dll=namespace(prefix='', suffix='.dll'), lib=namespace(prefix='lib', suffix='a'), import_lib=namespace(prefix='lib', suffix='a'), rust_lib=namespace(prefix='', suffix='lib'), obj=namespace(prefix='', suffix='o'), ) return namespace( dll=namespace(prefix='', suffix='.dll'), lib=namespace(prefix='', suffix='lib'), import_lib=namespace(prefix='', suffix='lib'), rust_lib=namespace(prefix='', suffix='lib'), obj=namespace(prefix='', suffix='obj'), ) elif host_or_target.kernel == 'Darwin': return namespace( dll=namespace(prefix='lib', suffix='.dylib'), lib=namespace(prefix='lib', suffix='a'), import_lib=namespace(prefix=None, suffix=''), rust_lib=namespace(prefix='lib', suffix='a'), obj=namespace(prefix='', suffix='o'), ) elif so_version: so = '.so.%s' % so_version else: so = '.so' return namespace( dll=namespace(prefix='lib', suffix=so), lib=namespace(prefix='lib', suffix='a'), import_lib=namespace(prefix=None, suffix=''), rust_lib=namespace(prefix='lib', suffix='a'), obj=namespace(prefix='', suffix='o'), ) return library_name_info_impl host_library_name_info = library_name_info_template(host) library_name_info = library_name_info_template(target) set_config('DLL_PREFIX', library_name_info.dll.prefix) set_config('DLL_SUFFIX', library_name_info.dll.suffix) set_config('HOST_DLL_PREFIX', host_library_name_info.dll.prefix) set_config('HOST_DLL_SUFFIX', host_library_name_info.dll.suffix) set_config('LIB_PREFIX', library_name_info.lib.prefix) set_config('LIB_SUFFIX', library_name_info.lib.suffix) set_config('RUST_LIB_PREFIX', library_name_info.rust_lib.prefix) set_config('RUST_LIB_SUFFIX', library_name_info.rust_lib.suffix) set_config('OBJ_SUFFIX', library_name_info.obj.suffix) # Lots of compilation tests depend on this variable being present. add_old_configure_assignment('OBJ_SUFFIX', library_name_info.obj.suffix) set_config('IMPORT_LIB_SUFFIX', library_name_info.import_lib.suffix) set_define('MOZ_DLL_PREFIX', depends(library_name_info.dll.prefix)(lambda s: '"%s"' % s)) set_define('MOZ_DLL_SUFFIX', depends(library_name_info.dll.suffix)(lambda s: '"%s"' % s)) # Depends on host_library_name_info, so needs to go here. include('build/moz.configure/bindgen.configure', when='--enable-compile-environment') include(include_project_configure) @depends('--help') @imports(_from='mozbuild.backend', _import='backends') def build_backends_choices(_): return tuple(backends) @deprecated_option('--enable-build-backend', nargs='+', choices=build_backends_choices) def build_backend(backends): if backends: return tuple('+%s' % b for b in backends) imply_option('--build-backends', build_backend) @depends('--enable-artifact-builds', '--disable-compile-environment', '--enable-build-backend', '--help') @imports('sys') def build_backend_defaults(artifact_builds, compile_environment, requested_backends, _): if 'Tup' in requested_backends: # As a special case, if Tup was requested, do not combine it with any # Make based backend by default. all_backends = [] elif artifact_builds: all_backends = ['FasterMake+RecursiveMake'] else: all_backends = ['RecursiveMake', 'FasterMake'] # Normally, we'd use target.os == 'WINNT', but a dependency on target # would require target to depend on --help, as well as host and shell, # and this is not a can of worms we can open at the moment. if sys.platform == 'win32' and compile_environment: all_backends.append('VisualStudio') return tuple(all_backends) or None option('--build-backends', nargs='+', default=build_backend_defaults, choices=build_backends_choices, help='Build backends to generate') @depends('--build-backends') def build_backends(backends): return backends set_config('BUILD_BACKENDS', build_backends) option('--disable-gtest-in-build', help='Force disable building the gtest libxul during the build.', when='--enable-compile-environment') # Determine whether to build the gtest xul. This happens in automation # on Desktop platforms with the exception of Windows PGO, where linking # xul-gtest.dll takes too long. @depends('MOZ_PGO', build_project, target, 'MOZ_AUTOMATION', '--disable-gtest-in-build', enable_tests, when='--enable-compile-environment') def build_gtest(pgo, build_project, target, automation, enabled, enable_tests): if not enable_tests or not enabled: return None if (automation and build_project == 'browser' and not (pgo and target.os == 'WINNT')): return True set_config('LINK_GTEST_DURING_COMPILE', build_gtest) # Localization # ============================================================== option('--enable-ui-locale', default='en-US', help='Select the user interface locale (default: en-US)') set_config('MOZ_UI_LOCALE', depends('--enable-ui-locale')(lambda x: x)) # clang-plugin location # ============================================================== @depends(host_library_name_info, check_build_environment, when='--enable-clang-plugin') def clang_plugin_path(library_name_info, build_env): topobjdir = build_env.topobjdir if topobjdir.endswith('/js/src'): topobjdir = topobjdir[:-7] return os.path.abspath( os.path.join(topobjdir, 'build', 'clang-plugin', '%sclang-plugin%s' % (library_name_info.dll.prefix, library_name_info.dll.suffix)) ) add_old_configure_assignment('CLANG_PLUGIN', clang_plugin_path) # Awk detection # ============================================================== awk = check_prog('AWK', ('gawk', 'mawk', 'nawk', 'awk')) # Until the AWK variable is not necessary in old-configure @depends(awk) def awk_for_old_configure(value): return value add_old_configure_assignment('AWK', awk_for_old_configure) # Perl detection # ============================================================== perl = check_prog('PERL', ('perl5', 'perl')) # Until the PERL variable is not necessary in old-configure @depends(perl) def perl_for_old_configure(value): return value add_old_configure_assignment('PERL', perl_for_old_configure) @template def perl_version_check(min_version): @depends(perl) @checking('for minimum required perl version >= %s' % min_version) def get_perl_version(perl): return Version(check_cmd_output( perl, '-e', 'print $]', onerror=lambda: die('Failed to get perl version.') )) @depends(get_perl_version) def check_perl_version(version): if version < min_version: die('Perl %s or higher is required.', min_version) @depends(perl) @checking('for full perl installation') @imports('subprocess') def has_full_perl_installation(perl): ret = subprocess.call( [perl, '-e', 'use Config; exit(!-d $Config{archlib})']) return ret == 0 @depends(has_full_perl_installation) def require_full_perl_installation(has_full_perl_installation): if not has_full_perl_installation: die('Cannot find Config.pm or $Config{archlib}. ' 'A full perl installation is required.') perl_version_check('5.006') # GNU make detection # ============================================================== option(env='MAKE', nargs=1, help='Path to GNU make') @depends('MAKE', host) def possible_makes(make, host): candidates = [] if host.kernel == 'WINNT': candidates.append('mingw32-make') if make: candidates.append(make[0]) if host.kernel == 'WINNT': candidates.extend(('make', 'gmake')) else: candidates.extend(('gmake', 'make')) return candidates check_prog('GMAKE', possible_makes) @depends(build_backends, build_project) def tup_include(build_backends, build_project): # We need to check the rustc version when building with tup, but # rustc_info isn't available when configuring js (and build_backends isn't # available from project-specific configure), so as a workaround we only # include the file when we know we'll need it. This can be removed when # we globally require a rustc recent enough to build with tup. if build_project not in ('browser', 'mobile/android'): return None for backend in build_backends: if 'Tup' in backend: return 'build/moz.configure/tup.configure' include(tup_include) # watchman detection # ============================================================== option(env='WATCHMAN', nargs=1, help='Path to the watchman program') @depends('WATCHMAN') @checking('for watchman', callback=lambda w: w.path if w else 'not found') def watchman(prog): if not prog: prog = find_program('watchman') if not prog: return # `watchman version` will talk to the Watchman daemon service. # This can hang due to permissions problems. e.g. # https://github.com/facebook/watchman/issues/376. So use # `watchman --version` to prevent a class of failures. out = check_cmd_output(prog, '--version', onerror=lambda: None) if out is None: return return namespace(path=prog, version=Version(out.strip())) @depends_if(watchman) @checking('for watchman version') def watchman_version(w): return w.version set_config('WATCHMAN', watchman.path) @depends_all(hg_version, hg_config, watchman) @checking('for watchman Mercurial integration') @imports('os') def watchman_hg(hg_version, hg_config, watchman): if hg_version < Version('3.8'): return 'no (Mercurial 3.8+ required)' ext_enabled = False mode_disabled = False for k in ('extensions.fsmonitor', 'extensions.hgext.fsmonitor'): if k in hg_config and hg_config[k] != '!': ext_enabled = True mode_disabled = hg_config.get('fsmonitor.mode') == 'off' if not ext_enabled: return 'no (fsmonitor extension not enabled)' if mode_disabled: return 'no (fsmonitor.mode=off disables fsmonitor)' return True # Miscellaneous programs # ============================================================== check_prog('XARGS', ('xargs',)) @depends(target) def extra_programs(target): if target.kernel == 'Darwin': return namespace( DSYMUTIL=('dsymutil', 'llvm-dsymutil'), MKFSHFS=('newfs_hfs', 'mkfs.hfsplus'), HFS_TOOL=('hfsplus',) ) if target.os == 'GNU' and target.kernel == 'Linux': return namespace(RPMBUILD=('rpmbuild',)) check_prog('DSYMUTIL', extra_programs.DSYMUTIL, allow_missing=True) check_prog('MKFSHFS', extra_programs.MKFSHFS, allow_missing=True) check_prog('HFS_TOOL', extra_programs.HFS_TOOL, allow_missing=True) check_prog('RPMBUILD', extra_programs.RPMBUILD, allow_missing=True) @depends(target) @imports('os') def makensis_progs(target): if target.kernel != 'WINNT': return candidates = [ 'makensis-3.01', 'makensis-3.0b3', 'makensis-3.0b1', 'makensis', ] # Look for nsis installed by msys environment. But only the 32-bit version. # We use an absolute path and insert as the first entry so it is preferred # over a 64-bit exe that may be in PATH. if 'MSYSTEM_PREFIX' in os.environ: prefix = os.path.dirname(os.environ['MSYSTEM_PREFIX']) candidates.insert(0, os.path.join(prefix, 'mingw32', 'bin', 'makensis.exe')) return tuple(candidates) nsis = check_prog('MAKENSISU', makensis_progs, allow_missing=True) # Make sure the version of makensis is up to date. @depends_if(nsis) @checking('for NSIS version') @imports('re') def nsis_version(nsis): nsis_min_version = '3.0b1' out = check_cmd_output(nsis, '-version', onerror=lambda: die('Failed to get nsis version.')) m = re.search(r'(?<=v)[0-9]+\.[0-9]+((a|b|rc)[0-9]+)?', out) if not m: raise FatalCheckError('Unknown version of makensis') ver = Version(m.group(0)) # Versions comparisons don't quite work well with beta versions, so ensure # it works for the non-beta version. if ver < nsis_min_version and (ver >= '3.0a' or ver < '3'): raise FatalCheckError('To build the installer you must have NSIS' ' version %s or greater in your path' % nsis_min_version) return ver # And that makensis is 32-bit (but only on Windows). @depends_if(nsis, when=depends(host)(lambda h: h.kernel == 'WINNT')) @checking('for 32-bit NSIS') def nsis_binary_type(nsis): bin_type = windows_binary_type(nsis) if bin_type != 'win32': raise FatalCheckError('%s is not a 32-bit Windows application' % nsis) return 'yes' # And any flags we have to give to makensis @depends(host) def nsis_flags(host): if host.kernel != 'WINNT': return '-nocd' return '' set_config('MAKENSISU_FLAGS', nsis_flags) check_prog('7Z', ('7z', '7za'), allow_missing=True, when=target_is_windows) -@depends(host) -def include_infer_autotest(host): - # Do not include autotest configuration on Windows or macos - # because infer is not available there - if host.os in ['WINNT', 'OSX']: - return None - return 'tools/infer/test/moz.configure' - -include(include_infer_autotest) - # Fallthrough to autoconf-based configure include('build/moz.configure/old.configure') @depends(check_build_environment, build_project) @imports('__sandbox__') @imports('glob') def config_status_deps(build_env, build_project): topsrcdir = build_env.topsrcdir topobjdir = build_env.topobjdir if not build_env.topobjdir.endswith('js/src'): extra_deps = [ os.path.join(topsrcdir, build_project, 'confvars.sh'), os.path.join(topobjdir, '.mozconfig.json'), ] else: # mozconfig changes may impact js configure. extra_deps = [os.path.join(topobjdir[:-7], '.mozconfig.json')] return list(__sandbox__._all_paths) + extra_deps + [ os.path.join(topsrcdir, 'CLOBBER'), os.path.join(topsrcdir, 'configure'), os.path.join(topsrcdir, 'js', 'src', 'configure'), os.path.join(topsrcdir, 'configure.in'), os.path.join(topsrcdir, 'js', 'src', 'configure.in'), os.path.join(topsrcdir, 'nsprpub', 'configure'), os.path.join(topsrcdir, 'config', 'milestone.txt'), os.path.join(topsrcdir, 'browser', 'config', 'version.txt'), os.path.join(topsrcdir, 'browser', 'config', 'version_display.txt'), os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'), os.path.join(topsrcdir, 'python', 'mozbuild', 'mozbuild', 'virtualenv.py'), os.path.join(topsrcdir, 'testing', 'mozbase', 'packages.txt'), os.path.join(topsrcdir, 'aclocal.m4'), os.path.join(topsrcdir, 'old-configure.in'), os.path.join(topsrcdir, 'js', 'src', 'aclocal.m4'), os.path.join(topsrcdir, 'js', 'src', 'old-configure.in'), ] + glob.glob(os.path.join(topsrcdir, 'build', 'autoconf', '*.m4')) set_config('CONFIG_STATUS_DEPS', config_status_deps) # Please do not add anything after setting config_dep_paths. diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1,2921 +1,2803 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import argparse import hashlib import itertools import json import logging import operator import os import re import subprocess import sys import tempfile from collections import OrderedDict import mozpack.path as mozpath from mach.decorators import ( CommandArgument, CommandArgumentGroup, CommandProvider, Command, SettingsProvider, SubCommand, ) from mach.main import Mach from mozbuild.base import ( BuildEnvironmentNotFoundException, MachCommandBase, MachCommandConditions as conditions, MozbuildObject, ) from mozbuild.util import ensureParentDir from mozbuild.backend import ( backends, ) BUILD_WHAT_HELP = ''' What to build. Can be a top-level make target or a relative directory. If multiple options are provided, they will be built serially. Takes dependency information from `topsrcdir/build/dumbmake-dependencies` to build additional targets as needed. BUILDING ONLY PARTS OF THE TREE CAN RESULT IN BAD TREE STATE. USE AT YOUR OWN RISK. '''.strip() EXCESSIVE_SWAP_MESSAGE = ''' =================== PERFORMANCE WARNING Your machine experienced a lot of swap activity during the build. This is possibly a sign that your machine doesn't have enough physical memory or not enough available memory to perform the build. It's also possible some other system activity during the build is to blame. If you feel this message is not appropriate for your machine configuration, please file a Firefox Build System :: General bug at https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=General and tell us about your machine and build configuration so we can adjust the warning heuristic. =================== ''' class StoreDebugParamsAndWarnAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): sys.stderr.write('The --debugparams argument is deprecated. Please ' + 'use --debugger-args instead.\n\n') setattr(namespace, self.dest, values) @CommandProvider class Watch(MachCommandBase): """Interface to watch and re-build the tree.""" @Command('watch', category='post-build', description='Watch and re-build the tree.', conditions=[conditions.is_firefox]) @CommandArgument('-v', '--verbose', action='store_true', help='Verbose output for what commands the watcher is running.') def watch(self, verbose=False): """Watch and re-build the source tree.""" if not conditions.is_artifact_build(self): print('mach watch requires an artifact build. See ' 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build') return 1 if not self.substs.get('WATCHMAN', None): print('mach watch requires watchman to be installed. See ' 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') return 1 self._activate_virtualenv() try: self.virtualenv_manager.install_pip_package('pywatchman==1.3.0') except Exception: print('Could not install pywatchman from pip. See ' 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') return 1 from mozbuild.faster_daemon import Daemon daemon = Daemon(self.config_environment) try: return daemon.watch() except KeyboardInterrupt: # Suppress ugly stack trace when user hits Ctrl-C. sys.exit(3) @CommandProvider class Build(MachCommandBase): """Interface to build the tree.""" @Command('build', category='build', description='Build the tree.') @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, help='Number of concurrent jobs to run. Default is the number of CPUs.') @CommandArgument('-C', '--directory', default=None, help='Change to a subdirectory of the build directory first.') @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) @CommandArgument('-X', '--disable-extra-make-dependencies', default=False, action='store_true', help='Do not add extra make dependencies.') @CommandArgument('-v', '--verbose', action='store_true', help='Verbose output for what commands the build is running.') @CommandArgument('--keep-going', action='store_true', help='Keep building after an error has occurred') def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False, keep_going=False): """Build the source tree. With no arguments, this will perform a full build. Positional arguments define targets to build. These can be make targets or patterns like "/" to indicate a make target within a directory. There are a few special targets that can be used to perform a partial build faster than what `mach build` would perform: * binaries - compiles and links all C/C++ sources and produces shared libraries and executables (binaries). * faster - builds JavaScript, XUL, CSS, etc files. "binaries" and "faster" almost fully complement each other. However, there are build actions not captured by either. If things don't appear to be rebuilding, perform a vanilla `mach build` to rebuild the world. """ from mozbuild.controller.building import ( BuildDriver, ) self.log_manager.enable_all_structured_loggers() driver = self._spawn(BuildDriver) return driver.build( what=what, disable_extra_make_dependencies=disable_extra_make_dependencies, jobs=jobs, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=self._mach_context) @Command('configure', category='build', description='Configure the tree (run configure and config.status).') @CommandArgument('options', default=None, nargs=argparse.REMAINDER, help='Configure options') def configure(self, options=None, buildstatus_messages=False, line_handler=None): from mozbuild.controller.building import ( BuildDriver, ) self.log_manager.enable_all_structured_loggers() driver = self._spawn(BuildDriver) return driver.configure( options=options, buildstatus_messages=buildstatus_messages, line_handler=line_handler) @Command('resource-usage', category='post-build', description='Show information about system resource usage for a build.') @CommandArgument('--address', default='localhost', help='Address the HTTP server should listen on.') @CommandArgument('--port', type=int, default=0, help='Port number the HTTP server should listen on.') @CommandArgument('--browser', default='firefox', help='Web browser to automatically open. See webbrowser Python module.') @CommandArgument('--url', help='URL of JSON document to display') def resource_usage(self, address=None, port=None, browser=None, url=None): import webbrowser from mozbuild.html_build_viewer import BuildViewerServer server = BuildViewerServer(address, port) if url: server.add_resource_json_url('url', url) else: last = self._get_state_filename('build_resources.json') if not os.path.exists(last): print('Build resources not available. If you have performed a ' 'build and receive this message, the psutil Python package ' 'likely failed to initialize properly.') return 1 server.add_resource_json_file('last', last) try: webbrowser.get(browser).open_new_tab(server.url) except Exception: print('Cannot get browser specified, trying the default instead.') try: browser = webbrowser.get().open_new_tab(server.url) except Exception: print('Please open %s in a browser.' % server.url) print('Hit CTRL+c to stop server.') server.run() @Command('build-backend', category='build', description='Generate a backend used to build the tree.') @CommandArgument('-d', '--diff', action='store_true', help='Show a diff of changes.') # It would be nice to filter the choices below based on # conditions, but that is for another day. @CommandArgument('-b', '--backend', nargs='+', choices=sorted(backends), help='Which backend to build.') @CommandArgument('-v', '--verbose', action='store_true', help='Verbose output.') @CommandArgument('-n', '--dry-run', action='store_true', help='Do everything except writing files out.') def build_backend(self, backend, diff=False, verbose=False, dry_run=False): python = self.virtualenv_manager.python_path config_status = os.path.join(self.topobjdir, 'config.status') if not os.path.exists(config_status): print('config.status not found. Please run |mach configure| ' 'or |mach build| prior to building the %s build backend.' % backend) return 1 args = [python, config_status] if backend: args.append('--backend') args.extend(backend) if diff: args.append('--diff') if verbose: args.append('--verbose') if dry_run: args.append('--dry-run') return self._run_command_in_objdir(args=args, pass_thru=True, ensure_exit_code=False) @CommandProvider class CargoProvider(MachCommandBase): """Invoke cargo in useful ways.""" @Command('cargo', category='build', description='Invoke cargo in useful ways.') def cargo(self): self.parser.print_usage() return 1 @SubCommand('cargo', 'check', description='Run `cargo check` on a given crate. Defaults to gkrust.') @CommandArgument('--all-crates', default=None, action='store_true', help='Check all of the crates in the tree.') @CommandArgument('crates', default=None, nargs='*', help='The crate name(s) to check.') def check(self, all_crates=None, crates=None): # XXX duplication with `mach vendor rust` crates_and_roots = { 'gkrust': 'toolkit/library/rust', 'gkrust-gtest': 'toolkit/library/gtest/rust', 'js': 'js/rust', 'mozjs_sys': 'js/src', 'geckodriver': 'testing/geckodriver', } if all_crates: crates = crates_and_roots.keys() elif crates == None or crates == []: crates = ['gkrust'] for crate in crates: root = crates_and_roots.get(crate, None) if not root: print('Cannot locate crate %s. Please check your spelling or ' 'add the crate information to the list.' % crate) return 1 check_targets = [ 'force-cargo-library-check', 'force-cargo-host-library-check', 'force-cargo-program-check', 'force-cargo-host-program-check', ] ret = self._run_make(srcdir=False, directory=root, ensure_exit_code=0, silent=True, print_directory=False, target=check_targets) if ret != 0: return ret return 0 @CommandProvider class Doctor(MachCommandBase): """Provide commands for diagnosing common build environment problems""" @Command('doctor', category='devenv', description='') @CommandArgument('--fix', default=None, action='store_true', help='Attempt to fix found problems.') def doctor(self, fix=None): self._activate_virtualenv() from mozbuild.doctor import Doctor doctor = Doctor(self.topsrcdir, self.topobjdir, fix) return doctor.check_all() @CommandProvider class Clobber(MachCommandBase): NO_AUTO_LOG = True CLOBBER_CHOICES = ['objdir', 'python'] @Command('clobber', category='build', description='Clobber the tree (delete the object directory).') @CommandArgument('what', default=['objdir'], nargs='*', help='Target to clobber, must be one of {{{}}} (default objdir).'.format( ', '.join(CLOBBER_CHOICES))) @CommandArgument('--full', action='store_true', help='Perform a full clobber') def clobber(self, what, full=False): """Clean up the source and object directories. Performing builds and running various commands generate various files. Sometimes it is necessary to clean up these files in order to make things work again. This command can be used to perform that cleanup. By default, this command removes most files in the current object directory (where build output is stored). Some files (like Visual Studio project files) are not removed by default. If you would like to remove the object directory in its entirety, run with `--full`. The `python` target will clean up various generated Python files from the source directory and will remove untracked files from well-known directories containing Python packages. Run this to remove .pyc files, compiled C extensions, etc. Note: all files not tracked or ignored by version control in well-known Python package directories will be deleted. Run the `status` command of your VCS to see if any untracked files you haven't committed yet will be deleted. """ invalid = set(what) - set(self.CLOBBER_CHOICES) if invalid: print('Unknown clobber target(s): {}'.format(', '.join(invalid))) return 1 ret = 0 if 'objdir' in what: from mozbuild.controller.clobber import Clobberer try: Clobberer(self.topsrcdir, self.topobjdir).remove_objdir(full) except OSError as e: if sys.platform.startswith('win'): if isinstance(e, WindowsError) and e.winerror in (5,32): self.log(logging.ERROR, 'file_access_error', {'error': e}, "Could not clobber because a file was in use. If the " "application is running, try closing it. {error}") return 1 raise if 'python' in what: if conditions.is_hg(self): cmd = ['hg', 'purge', '--all', '-I', 'glob:**.py[cdo]', '-I', 'path:python/', '-I', 'path:third_party/python/'] elif conditions.is_git(self): cmd = ['git', 'clean', '-f', '-x', '*.py[cdo]', 'python/', 'third_party/python/'] else: # We don't know what is tracked/untracked if we don't have VCS. # So we can't clean python/ and third_party/python/. cmd = ['find', '.', '-type', 'f', '-name', '*.py[cdo]', '-delete'] ret = subprocess.call(cmd, cwd=self.topsrcdir) return ret @property def substs(self): try: return super(Clobber, self).substs except BuildEnvironmentNotFoundException: return {} @CommandProvider class Logs(MachCommandBase): """Provide commands to read mach logs.""" NO_AUTO_LOG = True @Command('show-log', category='post-build', description='Display mach logs') @CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'), help='Filename to read log data from. Defaults to the log of the last ' 'mach command.') def show_log(self, log_file=None): if not log_file: path = self._get_state_filename('last_log.json') log_file = open(path, 'rb') if os.isatty(sys.stdout.fileno()): env = dict(os.environ) if 'LESS' not in env: # Sensible default flags if none have been set in the user # environment. env[b'LESS'] = b'FRX' less = subprocess.Popen(['less'], stdin=subprocess.PIPE, env=env) # Various objects already have a reference to sys.stdout, so we # can't just change it, we need to change the file descriptor under # it to redirect to less's input. # First keep a copy of the sys.stdout file descriptor. output_fd = os.dup(sys.stdout.fileno()) os.dup2(less.stdin.fileno(), sys.stdout.fileno()) startTime = 0 for line in log_file: created, action, params = json.loads(line) if not startTime: startTime = created self.log_manager.terminal_handler.formatter.start_time = \ created if 'line' in params: record = logging.makeLogRecord({ 'created': created, 'name': self._logger.name, 'levelno': logging.INFO, 'msg': '{line}', 'params': params, 'action': action, }) self._logger.handle(record) if self.log_manager.terminal: # Close less's input so that it knows that we're done sending data. less.stdin.close() # Since the less's input file descriptor is now also the stdout # file descriptor, we still actually have a non-closed system file # descriptor for less's input. Replacing sys.stdout's file # descriptor with what it was before we replaced it will properly # close less's input. os.dup2(output_fd, sys.stdout.fileno()) less.wait() @CommandProvider class Warnings(MachCommandBase): """Provide commands for inspecting warnings.""" @property def database_path(self): return self._get_state_filename('warnings.json') @property def database(self): from mozbuild.compilation.warnings import WarningsDatabase path = self.database_path database = WarningsDatabase() if os.path.exists(path): database.load_from_file(path) return database @Command('warnings-summary', category='post-build', description='Show a summary of compiler warnings.') @CommandArgument('-C', '--directory', default=None, help='Change to a subdirectory of the build directory first.') @CommandArgument('report', default=None, nargs='?', help='Warnings report to display. If not defined, show the most ' 'recent report.') def summary(self, directory=None, report=None): database = self.database if directory: dirpath = self.join_ensure_dir(self.topsrcdir, directory) if not dirpath: return 1 else: dirpath = None type_counts = database.type_counts(dirpath) sorted_counts = sorted(type_counts.iteritems(), key=operator.itemgetter(1)) total = 0 for k, v in sorted_counts: print('%d\t%s' % (v, k)) total += v print('%d\tTotal' % total) @Command('warnings-list', category='post-build', description='Show a list of compiler warnings.') @CommandArgument('-C', '--directory', default=None, help='Change to a subdirectory of the build directory first.') @CommandArgument('--flags', default=None, nargs='+', help='Which warnings flags to match.') @CommandArgument('report', default=None, nargs='?', help='Warnings report to display. If not defined, show the most ' 'recent report.') def list(self, directory=None, flags=None, report=None): database = self.database by_name = sorted(database.warnings) topsrcdir = mozpath.normpath(self.topsrcdir) if directory: directory = mozpath.normsep(directory) dirpath = self.join_ensure_dir(topsrcdir, directory) if not dirpath: return 1 if flags: # Flatten lists of flags. flags = set(itertools.chain(*[flaglist.split(',') for flaglist in flags])) for warning in by_name: filename = mozpath.normsep(warning['filename']) if filename.startswith(topsrcdir): filename = filename[len(topsrcdir) + 1:] if directory and not filename.startswith(directory): continue if flags and warning['flag'] not in flags: continue if warning['column'] is not None: print('%s:%d:%d [%s] %s' % (filename, warning['line'], warning['column'], warning['flag'], warning['message'])) else: print('%s:%d [%s] %s' % (filename, warning['line'], warning['flag'], warning['message'])) def join_ensure_dir(self, dir1, dir2): dir1 = mozpath.normpath(dir1) dir2 = mozpath.normsep(dir2) joined_path = mozpath.join(dir1, dir2) if os.path.isdir(joined_path): return joined_path else: print('Specified directory not found.') return None @CommandProvider class GTestCommands(MachCommandBase): @Command('gtest', category='testing', description='Run GTest unit tests (C++ tests).') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int, help='Run the tests in parallel using multiple processes.') @CommandArgument('--tbpl-parser', '-t', action='store_true', help='Output test results in a format that can be parsed by TBPL.') @CommandArgument('--shuffle', '-s', action='store_true', help='Randomize the execution order of tests.') @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, group='debugging', help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, debug, debugger, debugger_args): # We lazy build gtest because it's slow to link try: config = self.config_environment except Exception: print("Please run |./mach build| before |./mach gtest|.") return 1 active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if 'Tup' in active_backend: gtest_build_target = mozpath.join(self.topobjdir, '') else: gtest_build_target = 'recurse_gtest' res = self._mach_context.commands.dispatch('build', self._mach_context, what=[gtest_build_target]) if res: print("Could not build xul-gtest") return res if self.substs.get('MOZ_WIDGET_TOOLKIT') == 'cocoa': self._run_make(directory='browser/app', target='repackage', ensure_exit_code=True) app_path = self.get_binary_path('app') args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe']; if debug or debugger or debugger_args: args = self.prepend_debugger_args(args, debugger, debugger_args) cwd = os.path.join(self.topobjdir, '_tests', 'gtest') if not os.path.isdir(cwd): os.makedirs(cwd) # Use GTest environment variable to control test execution # For details see: # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options gtest_env = {b'GTEST_FILTER': gtest_filter} # Note: we must normalize the path here so that gtest on Windows sees # a MOZ_GMP_PATH which has only Windows dir seperators, because # nsIFile cannot open the paths with non-Windows dir seperators. xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin") gtest_env["MOZ_XRE_DIR"] = xre_path gtest_env["MOZ_GMP_PATH"] = os.pathsep.join( os.path.join(xre_path, p, "1.0") for p in ('gmp-fake', 'gmp-fakeopenh264') ) gtest_env[b"MOZ_RUN_GTEST"] = b"True" if shuffle: gtest_env[b"GTEST_SHUFFLE"] = b"True" if tbpl_parser: gtest_env[b"MOZ_TBPL_PARSER"] = b"True" if jobs == 1: return self.run_process(args=args, append_env=gtest_env, cwd=cwd, ensure_exit_code=False, pass_thru=True) from mozprocess import ProcessHandlerMixin import functools def handle_line(job_id, line): # Prepend the jobId line = '[%d] %s' % (job_id + 1, line.strip()) self.log(logging.INFO, "GTest", {'line': line}, '{line}') gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs) processes = {} for i in range(0, jobs): gtest_env["GTEST_SHARD_INDEX"] = str(i) processes[i] = ProcessHandlerMixin([app_path, "-unittest"], cwd=cwd, env=gtest_env, processOutputLine=[functools.partial(handle_line, i)], universal_newlines=True) processes[i].run() exit_code = 0 for process in processes.values(): status = process.wait() if status: exit_code = status # Clamp error code to 255 to prevent overflowing multiple of # 256 into 0 if exit_code > 255: exit_code = 255 return exit_code def prepend_debugger_args(self, args, debugger, debugger_args): ''' Given an array with program arguments, prepend arguments to run it under a debugger. :param args: The executable and arguments used to run the process normally. :param debugger: The debugger to use, or empty to use the default debugger. :param debugger_args: Any additional parameters to pass to the debugger. ''' import mozdebug if not debugger: # No debugger name was provided. Look for the default ones on # current OS. debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) if debugger: debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) if not debuggerInfo: print("Could not find a suitable debugger in your PATH.") return 1 # Parameters come from the CLI. We need to convert them before # their use. if debugger_args: from mozbuild import shellutil try: debugger_args = shellutil.split(debugger_args) except shellutil.MetaCharacterException as e: print("The --debugger_args you passed require a real shell to parse them.") print("(We can't handle the %r character.)" % e.char) return 1 # Prepend the debugger args. args = [debuggerInfo.path] + debuggerInfo.args + args return args @CommandProvider class ClangCommands(MachCommandBase): @Command('clang-complete', category='devenv', description='Generate a .clang_complete file.') def clang_complete(self): import shlex build_vars = {} def on_line(line): elements = [s.strip() for s in line.split('=', 1)] if len(elements) != 2: return build_vars[elements[0]] = elements[1] try: old_logger = self.log_manager.replace_terminal_handler(None) self._run_make(target='showbuild', log=False, line_handler=on_line) finally: self.log_manager.replace_terminal_handler(old_logger) def print_from_variable(name): if name not in build_vars: return value = build_vars[name] value = value.replace('-I.', '-I%s' % self.topobjdir) value = value.replace(' .', ' %s' % self.topobjdir) value = value.replace('-I..', '-I%s/..' % self.topobjdir) value = value.replace(' ..', ' %s/..' % self.topobjdir) args = shlex.split(value) for i in range(0, len(args) - 1): arg = args[i] if arg.startswith(('-I', '-D')): print(arg) continue if arg.startswith('-include'): print(arg + ' ' + args[i + 1]) continue print_from_variable('COMPILE_CXXFLAGS') print('-I%s/ipc/chromium/src' % self.topsrcdir) print('-I%s/ipc/glue' % self.topsrcdir) print('-I%s/ipc/ipdl/_ipdlheaders' % self.topobjdir) @CommandProvider class Package(MachCommandBase): """Package the built product for distribution.""" @Command('package', category='post-build', description='Package the built product for distribution as an APK, DMG, etc.') @CommandArgument('-v', '--verbose', action='store_true', help='Verbose output for what commands the packaging process is running.') def package(self, verbose=False): ret = self._run_make(directory=".", target='package', silent=not verbose, ensure_exit_code=False) if ret == 0: self.notify('Packaging complete') return ret @CommandProvider class Install(MachCommandBase): """Install a package.""" @Command('install', category='post-build', description='Install the package on the machine, or on a device.') @CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output when installing to an Android emulator.') def install(self, verbose=False): if conditions.is_android(self): from mozrunner.devices.android_device import verify_android_device verify_android_device(self, verbose=verbose) ret = self._run_make(directory=".", target='install', ensure_exit_code=False) if ret == 0: self.notify('Install complete') return ret @SettingsProvider class RunSettings(): config_settings = [ ('runprefs.*', 'string', """ Pass a pref into Firefox when using `mach run`, of the form `foo.bar=value`. Prefs will automatically be cast into the appropriate type. Integers can be single quoted to force them to be strings. """.strip()), ] @CommandProvider class RunProgram(MachCommandBase): """Run the compiled program.""" prog_group = 'the compiled program' @Command('run', category='post-build', description='Run the compiled program, possibly under a debugger or DMD.') @CommandArgument('params', nargs='...', group=prog_group, help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') @CommandArgumentGroup(prog_group) @CommandArgument('--remote', '-r', action='store_true', group=prog_group, help='Do not pass the --no-remote argument by default.') @CommandArgument('--background', '-b', action='store_true', group=prog_group, help='Do not pass the --foreground argument by default on Mac.') @CommandArgument('--noprofile', '-n', action='store_true', group=prog_group, help='Do not pass the --profile argument by default.') @CommandArgument('--disable-e10s', action='store_true', group=prog_group, help='Run the program with electrolysis disabled.') @CommandArgument('--enable-crash-reporter', action='store_true', group=prog_group, help='Run the program with the crash reporter enabled.') @CommandArgument('--setpref', action='append', default=[], group=prog_group, help='Set the specified pref before starting the program. Can be set multiple times. Prefs can also be set in ~/.mozbuild/machrc in the [runprefs] section - see `./mach settings` for more information.') @CommandArgument('--temp-profile', action='store_true', group=prog_group, help='Run the program using a new temporary profile created inside the objdir.') @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, group='debugging', help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') @CommandArgument('--debugparams', action=StoreDebugParamsAndWarnAction, default=None, type=str, dest='debugger_args', group='debugging', help=argparse.SUPPRESS) @CommandArgumentGroup('DMD') @CommandArgument('--dmd', action='store_true', group='DMD', help='Enable DMD. The following arguments have no effect without this.') @CommandArgument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'], group='DMD', help='Profiling mode. The default is \'dark-matter\'.') @CommandArgument('--stacks', choices=['partial', 'full'], group='DMD', help='Allocation stack trace coverage. The default is \'partial\'.') @CommandArgument('--show-dump-stats', action='store_true', group='DMD', help='Show stats when doing dumps.') def run(self, params, remote, background, noprofile, disable_e10s, enable_crash_reporter, setpref, temp_profile, debug, debugger, debugger_args, dmd, mode, stacks, show_dump_stats): if conditions.is_android(self): # Running Firefox for Android is completely different if dmd: print("DMD is not supported for Firefox for Android") return 1 from mozrunner.devices.android_device import verify_android_device, run_firefox_for_android if not (debug or debugger or debugger_args): verify_android_device(self, install=True) return run_firefox_for_android(self, params) verify_android_device(self, install=True, debugger=True) args = [''] else: from mozprofile import Profile, Preferences try: binpath = self.get_binary_path('app') except Exception as e: print("It looks like your program isn't built.", "You can run |mach build| to build it.") print(e) return 1 args = [binpath] if params: args.extend(params) if not remote: args.append('-no-remote') if not background and sys.platform == 'darwin': args.append('-foreground') if sys.platform.startswith('win') and \ 'MOZ_LAUNCHER_PROCESS' in self.defines: args.append('-wait-for-browser') no_profile_option_given = \ all(p not in params for p in ['-profile', '--profile', '-P']) if no_profile_option_given and not noprofile: prefs = { 'browser.shell.checkDefaultBrowser': False, 'general.warnOnAboutConfig': False, } prefs.update(self._mach_context.settings.runprefs) prefs.update([p.split('=', 1) for p in setpref]) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) tmpdir = os.path.join(self.topobjdir, 'tmp') if not os.path.exists(tmpdir): os.makedirs(tmpdir) if (temp_profile): path = tempfile.mkdtemp(dir=tmpdir, prefix='profile-') else: path = os.path.join(tmpdir, 'profile-default') profile = Profile(path, preferences=prefs) args.append('-profile') args.append(profile.profile) if not no_profile_option_given and setpref: print("setpref is only supported if a profile is not specified") return 1 extra_env = { 'MOZ_DEVELOPER_REPO_DIR': self.topsrcdir, 'MOZ_DEVELOPER_OBJ_DIR': self.topobjdir, 'RUST_BACKTRACE': 'full', } if not enable_crash_reporter: extra_env['MOZ_CRASHREPORTER_DISABLE'] = '1' else: extra_env['MOZ_CRASHREPORTER'] = '1' if disable_e10s: extra_env['MOZ_FORCE_DISABLE_E10S'] = '1' if debug or debugger or debugger_args: if 'INSIDE_EMACS' in os.environ: self.log_manager.terminal_handler.setLevel(logging.WARNING) import mozdebug if not debugger: # No debugger name was provided. Look for the default ones on # current OS. debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) if debugger: self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) if not self.debuggerInfo: print("Could not find a suitable debugger in your PATH.") return 1 # Parameters come from the CLI. We need to convert them before # their use. if debugger_args: from mozbuild import shellutil try: debugger_args = shellutil.split(debugger_args) except shellutil.MetaCharacterException as e: print("The --debugger-args you passed require a real shell to parse them.") print("(We can't handle the %r character.)" % e.char) return 1 # Prepend the debugger args. args = [self.debuggerInfo.path] + self.debuggerInfo.args + args if dmd: dmd_params = [] if mode: dmd_params.append('--mode=' + mode) if stacks: dmd_params.append('--stacks=' + stacks) if show_dump_stats: dmd_params.append('--show-dump-stats=yes') if dmd_params: extra_env['DMD'] = ' '.join(dmd_params) else: extra_env['DMD'] = '1' return self.run_process(args=args, ensure_exit_code=False, pass_thru=True, append_env=extra_env) @CommandProvider class Buildsymbols(MachCommandBase): """Produce a package of debug symbols suitable for use with Breakpad.""" @Command('buildsymbols', category='post-build', description='Produce a package of Breakpad-format symbols.') def buildsymbols(self): return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False) @CommandProvider class Makefiles(MachCommandBase): @Command('empty-makefiles', category='build-dev', description='Find empty Makefile.in in the tree.') def empty(self): import pymake.parser import pymake.parserdata IGNORE_VARIABLES = { 'DEPTH': ('@DEPTH@',), 'topsrcdir': ('@top_srcdir@',), 'srcdir': ('@srcdir@',), 'relativesrcdir': ('@relativesrcdir@',), 'VPATH': ('@srcdir@',), } IGNORE_INCLUDES = [ 'include $(DEPTH)/config/autoconf.mk', 'include $(topsrcdir)/config/config.mk', 'include $(topsrcdir)/config/rules.mk', ] def is_statement_relevant(s): if isinstance(s, pymake.parserdata.SetVariable): exp = s.vnameexp if not exp.is_static_string: return True if exp.s not in IGNORE_VARIABLES: return True return s.value not in IGNORE_VARIABLES[exp.s] if isinstance(s, pymake.parserdata.Include): if s.to_source() in IGNORE_INCLUDES: return False return True for path in self._makefile_ins(): relpath = os.path.relpath(path, self.topsrcdir) try: statements = [s for s in pymake.parser.parsefile(path) if is_statement_relevant(s)] if not statements: print(relpath) except pymake.parser.SyntaxError: print('Warning: Could not parse %s' % relpath, file=sys.stderr) def _makefile_ins(self): for root, dirs, files in os.walk(self.topsrcdir): for f in files: if f == 'Makefile.in': yield os.path.join(root, f) @CommandProvider class MachDebug(MachCommandBase): @Command('environment', category='build-dev', description='Show info about the mach and build environment.') @CommandArgument('--format', default='pretty', choices=['pretty', 'configure', 'json'], help='Print data in the given format.') @CommandArgument('--output', '-o', type=str, help='Output to the given file.') @CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.') def environment(self, format, output=None, verbose=False): func = getattr(self, '_environment_%s' % format.replace('.', '_')) if output: # We want to preserve mtimes if the output file already exists # and the content hasn't changed. from mozbuild.util import FileAvoidWrite with FileAvoidWrite(output) as out: return func(out, verbose) return func(sys.stdout, verbose) def _environment_pretty(self, out, verbose): state_dir = self._mach_context.state_dir import platform print('platform:\n\t%s' % platform.platform(), file=out) print('python version:\n\t%s' % sys.version, file=out) print('python prefix:\n\t%s' % sys.prefix, file=out) print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out) print('os cwd:\n\t%s' % os.getcwd(), file=out) print('mach directory:\n\t%s' % self._mach_context.topdir, file=out) print('state directory:\n\t%s' % state_dir, file=out) print('object directory:\n\t%s' % self.topobjdir, file=out) if self.mozconfig['path']: print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out) if self.mozconfig['configure_args']: print('mozconfig configure args:', file=out) for arg in self.mozconfig['configure_args']: print('\t%s' % arg, file=out) if self.mozconfig['make_extra']: print('mozconfig extra make args:', file=out) for arg in self.mozconfig['make_extra']: print('\t%s' % arg, file=out) if self.mozconfig['make_flags']: print('mozconfig make flags:', file=out) for arg in self.mozconfig['make_flags']: print('\t%s' % arg, file=out) config = None try: config = self.config_environment except Exception: pass if config: print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out) print('config topobjdir:\n\t%s' % config.topobjdir, file=out) if verbose: print('config substitutions:', file=out) for k in sorted(config.substs): print('\t%s: %s' % (k, config.substs[k]), file=out) print('config defines:', file=out) for k in sorted(config.defines): print('\t%s' % k, file=out) def _environment_json(self, out, verbose): import json class EnvironmentEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, MozbuildObject): result = { 'topsrcdir': obj.topsrcdir, 'topobjdir': obj.topobjdir, 'mozconfig': obj.mozconfig, } if verbose: result['substs'] = obj.substs result['defines'] = obj.defines return result elif isinstance(obj, set): return list(obj) return json.JSONEncoder.default(self, obj) json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out) class ArtifactSubCommand(SubCommand): def __call__(self, func): after = SubCommand.__call__(self, func) jobchoices = { 'android-api-16', 'android-x86', 'linux', 'linux64', 'macosx64', 'win32', 'win64' } args = [ CommandArgument('--tree', metavar='TREE', type=str, help='Firefox tree.'), CommandArgument('--job', metavar='JOB', choices=jobchoices, help='Build job.'), CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.'), ] for arg in args: after = arg(after) return after @CommandProvider class PackageFrontend(MachCommandBase): """Fetch and install binary artifacts from Mozilla automation.""" @Command('artifact', category='post-build', description='Use pre-built artifacts to build Firefox.') def artifact(self): '''Download, cache, and install pre-built binary artifacts to build Firefox. Use |mach build| as normal to freshen your installed binary libraries: artifact builds automatically download, cache, and install binary artifacts from Mozilla automation, replacing whatever may be in your object directory. Use |mach artifact last| to see what binary artifacts were last used. Never build libxul again! ''' pass def _make_artifacts(self, tree=None, job=None, skip_cache=False): state_dir = self._mach_context.state_dir cache_dir = os.path.join(state_dir, 'package-frontend') hg = None if conditions.is_hg(self): hg = self.substs['HG'] git = None if conditions.is_git(self): git = self.substs['GIT'] from mozbuild.artifacts import Artifacts artifacts = Artifacts(tree, self.substs, self.defines, job, log=self.log, cache_dir=cache_dir, skip_cache=skip_cache, hg=hg, git=git, topsrcdir=self.topsrcdir) return artifacts @ArtifactSubCommand('artifact', 'install', 'Install a good pre-built artifact.') @CommandArgument('source', metavar='SRC', nargs='?', type=str, help='Where to fetch and install artifacts from. Can be omitted, in ' 'which case the current hg repository is inspected; an hg revision; ' 'a remote URL; or a local file.', default=None) @CommandArgument('--skip-cache', action='store_true', help='Skip all local caches to force re-fetching remote artifacts.', default=False) def artifact_install(self, source=None, skip_cache=False, tree=None, job=None, verbose=False): self._set_log_level(verbose) artifacts = self._make_artifacts(tree=tree, job=job, skip_cache=skip_cache) return artifacts.install_from(source, self.distdir) @ArtifactSubCommand('artifact', 'clear-cache', 'Delete local artifacts and reset local artifact cache.') def artifact_clear_cache(self, tree=None, job=None, verbose=False): self._set_log_level(verbose) artifacts = self._make_artifacts(tree=tree, job=job) artifacts.clear_cache() return 0 @SubCommand('artifact', 'toolchain') @CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.') @CommandArgument('--cache-dir', metavar='DIR', help='Directory where to store the artifacts cache') @CommandArgument('--skip-cache', action='store_true', help='Skip all local caches to force re-fetching remote artifacts.', default=False) @CommandArgument('--from-build', metavar='BUILD', nargs='+', help='Download toolchains resulting from the given build(s); ' 'BUILD is a name of a toolchain task, e.g. linux64-clang') @CommandArgument('--tooltool-manifest', metavar='MANIFEST', help='Explicit tooltool manifest to process') @CommandArgument('--authentication-file', metavar='FILE', help='Use the RelengAPI token found in the given file to authenticate') @CommandArgument('--tooltool-url', metavar='URL', help='Use the given url as tooltool server') @CommandArgument('--no-unpack', action='store_true', help='Do not unpack any downloaded file') @CommandArgument('--retry', type=int, default=4, help='Number of times to retry failed downloads') @CommandArgument('--artifact-manifest', metavar='FILE', help='Store a manifest about the downloaded taskcluster artifacts') @CommandArgument('files', nargs='*', help='A list of files to download, in the form path@task-id, in ' 'addition to the files listed in the tooltool manifest.') def artifact_toolchain(self, verbose=False, cache_dir=None, skip_cache=False, from_build=(), tooltool_manifest=None, authentication_file=None, tooltool_url=None, no_unpack=False, retry=None, artifact_manifest=None, files=()): '''Download, cache and install pre-built toolchains. ''' from mozbuild.artifacts import ArtifactCache from mozbuild.action.tooltool import ( FileRecord, open_manifest, unpack_file, ) from requests.adapters import HTTPAdapter import redo import requests import shutil from taskgraph.config import load_graph_config from taskgraph.generator import Kind from taskgraph.util.taskcluster import ( get_artifact_url, ) self._set_log_level(verbose) # Normally, we'd use self.log_manager.enable_unstructured(), # but that enables all logging, while we only really want tooltool's # and it also makes structured log output twice. # So we manually do what it does, and limit that to the tooltool # logger. if self.log_manager.terminal_handler: logging.getLogger('mozbuild.action.tooltool').addHandler( self.log_manager.terminal_handler) logging.getLogger('redo').addHandler( self.log_manager.terminal_handler) self.log_manager.terminal_handler.addFilter( self.log_manager.structured_filter) if not cache_dir: cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains') tooltool_url = (tooltool_url or 'https://tooltool.mozilla-releng.net').rstrip('/') cache = ArtifactCache(cache_dir=cache_dir, log=self.log, skip_cache=skip_cache) if authentication_file: with open(authentication_file, 'rb') as f: token = f.read().strip() class TooltoolAuthenticator(HTTPAdapter): def send(self, request, *args, **kwargs): request.headers['Authorization'] = \ 'Bearer {}'.format(token) return super(TooltoolAuthenticator, self).send( request, *args, **kwargs) cache._download_manager.session.mount( tooltool_url, TooltoolAuthenticator()) class DownloadRecord(FileRecord): def __init__(self, url, *args, **kwargs): super(DownloadRecord, self).__init__(*args, **kwargs) self.url = url self.basename = self.filename def fetch_with(self, cache): self.filename = cache.fetch(self.url) return self.filename def validate(self): if self.size is None and self.digest is None: return True return super(DownloadRecord, self).validate() class ArtifactRecord(DownloadRecord): def __init__(self, task_id, artifact_name): for _ in redo.retrier(attempts=retry+1, sleeptime=60): cot = cache._download_manager.session.get( get_artifact_url(task_id, 'public/chainOfTrust.json.asc')) if cot.status_code >= 500: continue cot.raise_for_status() break else: cot.raise_for_status() digest = algorithm = None data = {} # The file is GPG-signed, but we don't care about validating # that. Instead of parsing the PGP signature, we just take # the one line we're interested in, which starts with a `{`. for l in cot.content.splitlines(): if l.startswith('{'): try: data = json.loads(l) break except Exception: pass for algorithm, digest in (data.get('artifacts', {}) .get(artifact_name, {}).items()): pass name = os.path.basename(artifact_name) artifact_url = get_artifact_url(task_id, artifact_name, use_proxy=not artifact_name.startswith('public/')) super(ArtifactRecord, self).__init__( artifact_url, name, None, digest, algorithm, unpack=True) records = OrderedDict() downloaded = [] if tooltool_manifest: manifest = open_manifest(tooltool_manifest) for record in manifest.file_records: url = '{}/{}/{}'.format(tooltool_url, record.algorithm, record.digest) records[record.filename] = DownloadRecord( url, record.filename, record.size, record.digest, record.algorithm, unpack=record.unpack, version=record.version, visibility=record.visibility, setup=record.setup) if from_build: if 'TASK_ID' in os.environ: self.log(logging.ERROR, 'artifact', {}, 'Do not use --from-build in automation; all dependencies ' 'should be determined in the decision task.') return 1 from taskgraph.optimize import IndexSearch from taskgraph.parameters import Parameters params = Parameters( level=os.environ.get('MOZ_SCM_LEVEL', '3'), strict=False, ignore_fetches=True, ) # TODO: move to the taskcluster package def tasks(kind_name): root_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci') graph_config = load_graph_config(root_path) tasks = Kind.load(root_path, graph_config, kind_name).load_tasks(params, {}) return { task.task['metadata']['name']: task for task in tasks } toolchains = tasks('toolchain') aliases = {} for t in toolchains.values(): alias = t.attributes.get('toolchain-alias') if alias: aliases['toolchain-{}'.format(alias)] = \ t.task['metadata']['name'] for b in from_build: user_value = b if not b.startswith('toolchain-'): b = 'toolchain-{}'.format(b) task = toolchains.get(aliases.get(b, b)) if not task: self.log(logging.ERROR, 'artifact', {'build': user_value}, 'Could not find a toolchain build named `{build}`') return 1 task_id = IndexSearch().should_replace_task( task, {}, task.optimization.get('index-search', [])) artifact_name = task.attributes.get('toolchain-artifact') if task_id in (True, False) or not artifact_name: self.log(logging.ERROR, 'artifact', {'build': user_value}, 'Could not find artifacts for a toolchain build ' 'named `{build}`. Local commits and other changes ' 'in your checkout may cause this error. Try ' 'updating to a fresh checkout of mozilla-central ' 'to use artifact builds.') return 1 record = ArtifactRecord(task_id, artifact_name) records[record.filename] = record # Handle the list of files of the form path@task-id on the command # line. Each of those give a path to an artifact to download. for f in files: if '@' not in f: self.log(logging.ERROR, 'artifact', {}, 'Expected a list of files of the form path@task-id') return 1 name, task_id = f.rsplit('@', 1) record = ArtifactRecord(task_id, name) records[record.filename] = record for record in records.itervalues(): self.log(logging.INFO, 'artifact', {'name': record.basename}, 'Downloading {name}') valid = False # sleeptime is 60 per retry.py, used by tooltool_wrapper.sh for attempt, _ in enumerate(redo.retrier(attempts=retry+1, sleeptime=60)): try: record.fetch_with(cache) except (requests.exceptions.HTTPError, requests.exceptions.ChunkedEncodingError, requests.exceptions.ConnectionError) as e: if isinstance(e, requests.exceptions.HTTPError): # The relengapi proxy likes to return error 400 bad request # which seems improbably to be due to our (simple) GET # being borked. status = e.response.status_code should_retry = status >= 500 or status == 400 else: should_retry = True if should_retry or attempt < retry: level = logging.WARN else: level = logging.ERROR # e.message is not always a string, so convert it first. self.log(level, 'artifact', {}, str(e.message)) if not should_retry: break if attempt < retry: self.log(logging.INFO, 'artifact', {}, 'Will retry in a moment...') continue try: valid = record.validate() except Exception: pass if not valid: os.unlink(record.filename) if attempt < retry: self.log(logging.INFO, 'artifact', {}, 'Will retry in a moment...') continue downloaded.append(record) break if not valid: self.log(logging.ERROR, 'artifact', {'name': record.basename}, 'Failed to download {name}') return 1 artifacts = {} if artifact_manifest else None for record in downloaded: local = os.path.join(os.getcwd(), record.basename) if os.path.exists(local): os.unlink(local) # unpack_file needs the file with its final name to work # (https://github.com/mozilla/build-tooltool/issues/38), so we # need to copy it, even though we remove it later. Use hard links # when possible. try: os.link(record.filename, local) except Exception: shutil.copy(record.filename, local) # Keep a sha256 of each downloaded file, for the chain-of-trust # validation. if artifact_manifest is not None: with open(local) as fh: h = hashlib.sha256() while True: data = fh.read(1024 * 1024) if not data: break h.update(data) artifacts[record.url] = { 'sha256': h.hexdigest(), } if record.unpack and not no_unpack: unpack_file(local, record.setup) os.unlink(local) if not downloaded: self.log(logging.ERROR, 'artifact', {}, 'Nothing to download') if files: return 1 if artifacts: ensureParentDir(artifact_manifest) with open(artifact_manifest, 'w') as fh: json.dump(artifacts, fh, indent=4, sort_keys=True) return 0 class StaticAnalysisSubCommand(SubCommand): def __call__(self, func): after = SubCommand.__call__(self, func) args = [ CommandArgument('--verbose', '-v', action='store_true', help='Print verbose output.'), ] for arg in args: after = arg(after) return after class StaticAnalysisMonitor(object): def __init__(self, srcdir, objdir, total): self._total = total self._processed = 0 self._current = None self._srcdir = srcdir from mozbuild.compilation.warnings import ( WarningsCollector, WarningsDatabase, ) self._warnings_database = WarningsDatabase() def on_warning(warning): filename = warning['filename'] self._warnings_database.insert(warning) self._warnings_collector = WarningsCollector(on_warning, objdir=objdir) @property def num_files(self): return self._total @property def num_files_processed(self): return self._processed @property def current_file(self): return self._current @property def warnings_db(self): return self._warnings_database def on_line(self, line): warning = None try: warning = self._warnings_collector.process_line(line) except: pass if line.find('clang-tidy') != -1: filename = line.split(' ')[-1] if os.path.isfile(filename): self._current = os.path.relpath(filename, self._srcdir) else: self._current = None self._processed = self._processed + 1 return (warning, False) return (warning, True) @CommandProvider class StaticAnalysis(MachCommandBase): """Utilities for running C++ static analysis checks and format.""" # List of file extension to consider (should start with dot) _format_include_extensions = ('.cpp', '.c', '.h', '.java') # File contaning all paths to exclude from formatting _format_ignore_file = '.clang-format-ignore' @Command('static-analysis', category='testing', description='Run C++ static analysis checks') def static_analysis(self): # If not arguments are provided, just print a help message. mach = Mach(os.getcwd()) mach.run(['static-analysis', '--help']) @StaticAnalysisSubCommand('static-analysis', 'check', 'Run the checks using the helper tool') @CommandArgument('source', nargs='*', default=['.*'], help='Source files to be analyzed (regex on path). ' 'Can be omitted, in which case the entire code base ' 'is analyzed. The source argument is ignored if ' 'there is anything fed through stdin, in which case ' 'the analysis is only performed on the files changed ' 'in the patch streamed through stdin. This is called ' 'the diff mode.') @CommandArgument('--checks', '-c', default='-*', metavar='checks', help='Static analysis checks to enable. By default, this enables only ' 'checks that are published here: https://mzl.la/2DRHeTh, but can be any ' 'clang-tidy checks syntax.') @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, help='Number of concurrent jobs to run. Default is the number of CPUs.') @CommandArgument('--strip', '-p', default='1', metavar='NUM', help='Strip NUM leading components from file names in diff mode.') @CommandArgument('--fix', '-f', default=False, action='store_true', help='Try to autofix errors detected by clang-tidy checkers.') @CommandArgument('--header-filter', '-h-f', default='', metavar='header_filter', help='Regular expression matching the names of the headers to ' 'output diagnostics from. Diagnostics from the main file ' 'of each translation unit are always displayed') def check(self, source=None, jobs=2, strip=1, verbose=False, checks='-*', fix=False, header_filter=''): from mozbuild.controller.building import ( StaticAnalysisFooter, StaticAnalysisOutputManager, ) self._set_log_level(verbose) self.log_manager.enable_all_structured_loggers() rc = self._build_compile_db(verbose=verbose) - rc = rc or self._build_export(jobs=jobs, verbose=verbose) - rc = rc or self._get_clang_tools(verbose=verbose) + if rc != 0: + return rc + + rc = self._build_export(jobs=jobs, verbose=verbose) + if rc != 0: + return rc + + rc = self._get_clang_tools(verbose=verbose) if rc != 0: return rc compile_db = json.loads(open(self._compile_db, 'r').read()) total = 0 import re name_re = re.compile('(' + ')|('.join(source) + ')') for f in compile_db: if name_re.search(f['file']): total = total + 1 if not total: return 0 cwd = self.topobjdir self._compilation_commands_path = self.topobjdir args = self._get_clang_tidy_command( checks=checks, header_filter=header_filter, sources=source, jobs=jobs, fix=fix) monitor = StaticAnalysisMonitor(self.topsrcdir, self.topobjdir, total) footer = StaticAnalysisFooter(self.log_manager.terminal, monitor) with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output: rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_db)}, '{count} warnings present.') if rc != 0: return rc # if we are building firefox for android it might be nice to # also analyze the java code base if self.substs['MOZ_BUILD_APP'] == 'mobile/android': rc = self.check_java(source, jobs, strip, verbose, skip_export=True) return rc @StaticAnalysisSubCommand('static-analysis', 'check-java', 'Run infer on the java codebase.') @CommandArgument('source', nargs='*', default=['mobile'], help='Source files to be analyzed. ' 'Can be omitted, in which case the entire code base ' 'is analyzed. The source argument is ignored if ' 'there is anything fed through stdin, in which case ' 'the analysis is only performed on the files changed ' 'in the patch streamed through stdin. This is called ' 'the diff mode.') @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*', help='Static analysis checks to enable.') @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, help='Number of concurrent jobs to run.' ' Default is the number of CPUs.') @CommandArgument('--task', '-t', type=str, default='compileLocalWithGeckoBinariesNoMinApiDebugSources', help='Which gradle tasks to use to compile the java codebase.') def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[], task='compileLocalWithGeckoBinariesNoMinApiDebugSources', skip_export=False): self._set_log_level(verbose) self.log_manager.enable_all_structured_loggers() if self.substs['MOZ_BUILD_APP'] != 'mobile/android': self.log(logging.WARNING, 'static-analysis', {}, 'Cannot check java source code unless you are building for android!') return 1 # if source contains the whole mobile folder, then we just have to # analyze everything check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source) # gather all java sources from the source variable java_sources = [] if not check_all: java_sources = self._get_java_files(source) if not java_sources: return 0 if not skip_export: rc = self._build_export(jobs=jobs, verbose=verbose) if rc != 0: return rc rc = self._get_infer(verbose=verbose) if rc != 0: self.log(logging.WARNING, 'static-analysis', {}, 'This command is only available for linux64!') return rc # which checkers to use, and which folders to exclude all_checkers, third_party_path = self._get_infer_config() checkers, excludes = self._get_infer_args( checks=checks or all_checkers, third_party_path=third_party_path ) - rc = rc or self._gradle(['clean']) # clean so that we can recompile + gradlew = mozpath.join(self.topsrcdir, 'gradlew') # infer capture command - capture_cmd = [self._infer_path, 'capture'] + excludes + ['--'] - rc = rc or self._gradle([task], infer_args=capture_cmd, verbose=verbose) + capture_cmd = [self._infer_path, 'capture'] + excludes + \ + ['--', gradlew, task] tmp_file, args = self._get_infer_source_args(java_sources) # infer analyze command analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] + \ checkers + args - rc = rc or self.run_process(args=analysis_cmd, cwd=self.topsrcdir, pass_thru=True) + # capture, then analyze the sources + for args in [[gradlew, 'clean'], capture_cmd, analysis_cmd]: + rc = self.run_process(args=args, cwd=self.topsrcdir, + pass_thru=True) + # if a command fails, break and close the tmp file before returning + if rc != 0: + break if tmp_file: tmp_file.close() return rc def _get_java_files(self, sources): java_sources = [] for i in sources: f = mozpath.join(self.topsrcdir, i) if os.path.isdir(f): for root, dirs, files in os.walk(f): dirs.sort() for file in sorted(files): if file.endswith('.java'): java_sources.append(mozpath.join(root, file)) elif f.endswith('.java'): java_sources.append(f) return java_sources def _get_infer_source_args(self, sources): '''Return the arguments to only analyze ''' if not sources: return (None, []) # create a temporary file in which we place all sources # this is used by the analysis command to only analyze certain files f = tempfile.NamedTemporaryFile() for source in sources: f.write(source+'\n') f.flush() return (f, ['--changed-files-index', f.name]) def _get_infer_config(self): '''Load the infer config file.''' import yaml checkers = [] tp_path = '' with open(mozpath.join(self.topsrcdir, 'tools', 'infer', 'config.yaml')) as f: try: config = yaml.safe_load(f) for item in config['infer_checkers']: if item['publish']: checkers.append(item['name']) tp_path = mozpath.join(self.topsrcdir, config['third_party']) except Exception as e: print('Looks like config.yaml is not valid, so we are unable ' 'to determine default checkers, and which folder to ' 'exclude, using defaults provided by infer') return checkers, tp_path def _get_infer_args(self, checks, third_party_path): '''Return the arguments which include the checkers , and excludes all folder in .''' checkers = ['-a', 'checkers'] excludes = [] for checker in checks: checkers.append('--' + checker) with open(third_party_path) as f: for line in f: excludes.append('--skip-analysis-in-path') excludes.append(line.strip('\n')) return checkers, excludes def _get_clang_tidy_command(self, checks, header_filter, sources, jobs, fix): if checks == '-*': checks = self._get_checks() common_args = ['-clang-tidy-binary', self._clang_tidy_path, '-clang-apply-replacements-binary', self._clang_apply_replacements, '-checks=%s' % checks, '-extra-arg=-DMOZ_CLANG_PLUGIN'] # Flag header-filter is passed in order to limit the diagnostic messages only # to the specified header files. When no value is specified the default value # is considered to be the source in order to limit the diagnostic message to # the source files or folders. common_args += ['-header-filter=%s' % (header_filter if len(header_filter) else '|'.join(sources))] if fix: common_args += '-fix' return [ self.virtualenv_manager.python_path, self._run_clang_tidy_path, '-j', str(jobs), '-p', self._compilation_commands_path ] + common_args + sources - def _gradle(self, args, infer_args=None, verbose=False, autotest=False, - suppress_output=True): - infer_args = infer_args or [] - java_home = os.path.dirname(os.path.dirname(self.substs['JAVA'])) - if autotest: - cwd = mozpath.join(self.topsrcdir, 'tools', 'infer', 'test') - gradle = mozpath.join(cwd, 'gradlew') - else: - gradle = self.substs['GRADLE'] - cwd = self.topsrcdir - extra_env = { - 'GRADLE_OPTS': '-Dfile.encoding=utf-8', # see mobile/android/mach_commands.py - 'JAVA_HOME': java_home, - 'JAVA_TOOL_OPTIONS': '-Dfile.encoding=utf-8', - } - if suppress_output: - devnull = open(os.devnull, 'w') - return subprocess.call( - infer_args + [gradle] + args, - env=dict(os.environ, **extra_env), - cwd=cwd, stdout=devnull, stderr=subprocess.STDOUT, close_fds=True) - else: - return self.run_process( - infer_args + [gradle] + args, - append_env=extra_env, - pass_thru=True, # Allow user to run gradle interactively. - ensure_exit_code=False, # Don't throw on non-zero exit code. - cwd=cwd) - @StaticAnalysisSubCommand('static-analysis', 'autotest', 'Run the auto-test suite in order to determine that' ' the analysis did not regress.') @CommandArgument('--dump-results', '-d', default=False, action='store_true', help='Generate the baseline for the regression test. Based on' ' this baseline we will test future results.') @CommandArgument('--intree-tool', '-i', default=False, action='store_true', help='Use a pre-aquired in-tree clang-tidy package.') @CommandArgument('checker_names', nargs='*', default=[], help='Checkers that are going to be auto-tested.') def autotest(self, verbose=False, dump_results=False, intree_tool=False, checker_names=[]): # If 'dump_results' is True than we just want to generate the issues files for each # checker in particulat and thus 'force_download' becomes 'False' since we want to # do this on a local trusted clang-tidy package. self._set_log_level(verbose) self._dump_results = dump_results - force_download = not self._dump_results + + force_download = True + + if self._dump_results: + force_download = False # Function return codes self.TOOLS_SUCCESS = 0 self.TOOLS_FAILED_DOWNLOAD = 1 self.TOOLS_UNSUPORTED_PLATFORM = 2 self.TOOLS_CHECKER_NO_TEST_FILE = 3 self.TOOLS_CHECKER_RETURNED_NO_ISSUES = 4 self.TOOLS_CHECKER_RESULT_FILE_NOT_FOUND = 5 self.TOOLS_CHECKER_DIFF_FAILED = 6 self.TOOLS_CHECKER_NOT_FOUND = 7 self.TOOLS_CHECKER_FAILED_FILE = 8 - self.TOOLS_GRADLE_FAILED = 9 - - rc = self._get_clang_tools(force=force_download, verbose=verbose, intree_tool=intree_tool) + + # Configure the tree or download clang-tidy package, depending on the option that we choose + if intree_tool: + _, config, _ = self._get_config_environment() + clang_tools_path = self.topsrcdir + self._clang_tidy_path = mozpath.join( + clang_tools_path, "clang", "bin", + "clang-tidy" + config.substs.get('BIN_SUFFIX', '')) + self._clang_format_path = mozpath.join( + clang_tools_path, "clang", "bin", + "clang-format" + config.substs.get('BIN_SUFFIX', '')) + self._clang_apply_replacements = mozpath.join( + clang_tools_path, "clang", "bin", + "clang-apply-replacements" + config.substs.get('BIN_SUFFIX', '')) + self._run_clang_tidy_path = mozpath.join(clang_tools_path, "clang", "share", + "clang", "run-clang-tidy.py") + self._clang_format_diff = mozpath.join(clang_tools_path, "clang", "share", + "clang", "clang-format-diff.py") + + # Ensure that clang-tidy is present + rc = not os.path.exists(self._clang_tidy_path) + else: + rc = self._get_clang_tools(force=force_download, verbose=verbose) + if rc != 0: self.log(logging.ERROR, 'ERROR: static-analysis', {}, 'clang-tidy unable to locate package.') return self.TOOLS_FAILED_DOWNLOAD self._clang_tidy_base_path = mozpath.join(self.topsrcdir, "tools", "clang-tidy") # For each checker run it f = open(mozpath.join(self._clang_tidy_base_path, "config.yaml")) import yaml config = yaml.safe_load(f) platform, _ = self.platform if platform not in config['platforms']: self.log(logging.ERROR, 'static-analysis', {}, - "RUNNING: clang-tidy autotest for platform {} not supported." - .format(platform)) - return self.TOOLS_UNSUPORTED_PLATFORM + "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform)) + return TOOLS_UNSUPORTED_PLATFORM import concurrent.futures import multiprocessing import shutil max_workers = multiprocessing.cpu_count() self.log(logging.INFO, 'static-analysis', {}, "RUNNING: clang-tidy autotest for platform {0} with {1} workers.".format( platform, max_workers)) # List all available checkers cmd = [self._clang_tidy_path, '-list-checks', '-checks=*'] clang_output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('utf-8') available_checks = clang_output.split('\n')[1:] self._clang_tidy_checks = [c.strip() for c in available_checks if c] # Build the dummy compile_commands.json self._compilation_commands_path = self._create_temp_compilation_db(config) with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] for item in config['clang_checkers']: # Skip if any of the following statements is true: # 1. Checker attribute 'publish' is False. not_published = not bool(item.get('publish', True)) # 2. Checker has restricted-platforms and current platform is not of them. ignored_platform = 'restricted-platforms' in item and platform not in item['restricted-platforms'] # 3. Checker name is mozilla-* or -*. ignored_checker = item['name'] in ['mozilla-*', '-*'] # 4. List checker_names is passed and the current checker is not part of the # list or 'publish' is False checker_not_in_list = checker_names and (item['name'] not in checker_names or not_published) if not_published or \ ignored_platform or \ ignored_checker or \ checker_not_in_list: continue futures.append(executor.submit(self._verify_checker, item)) error_code = self.TOOLS_SUCCESS for future in concurrent.futures.as_completed(futures): # Wait for every task to finish ret_val = future.result() if ret_val != self.TOOLS_SUCCESS: # We are interested only in one error and we don't break # the execution of for loop since we want to make sure that all # tasks finished. error_code = ret_val if error_code != self.TOOLS_SUCCESS: self.log(logging.INFO, 'static-analysis', {}, "FAIL: clang-tidy some tests failed.") # Also delete the tmp folder shutil.rmtree(self._compilation_commands_path) return error_code self.log(logging.INFO, 'static-analysis', {}, "SUCCESS: clang-tidy all tests passed.") # Also delete the tmp folder shutil.rmtree(self._compilation_commands_path) - return self._autotest_infer(intree_tool, force_download, verbose) + return self.TOOLS_SUCCESS def _create_temp_compilation_db(self, config): directory = tempfile.mkdtemp(prefix='cc') with open(mozpath.join(directory, "compile_commands.json"), "wb") as file_handler: compile_commands = [] director = mozpath.join(self.topsrcdir, 'tools', 'clang-tidy', 'test') for item in config['clang_checkers']: if item['name'] in ['-*', 'mozilla-*']: continue file = item['name'] + '.cpp' element = {} element["directory"] = director - element["command"] = 'cpp ' + file + element["command"] = 'cpp '+ file element["file"] = mozpath.join(director, file) compile_commands.append(element) json.dump(compile_commands, file_handler) file_handler.flush() return directory - def _autotest_infer(self, intree_tool, force_download, verbose): - # infer is not available on other platforms, but autotest should work even without - # it being installed - if self.platform[0] == 'linux64': - rc = self._get_infer(force=force_download, verbose=verbose, intree_tool=intree_tool) - if rc != 0: - self.log(logging.ERROR, 'ERROR: static-analysis', {}, - 'infer unable to locate package.') - return self.TOOLS_FAILED_DOWNLOAD - self.__infer_tool = mozpath.join(self.topsrcdir, 'tools', 'infer') - self.__infer_test_folder = mozpath.join(self.__infer_tool, 'test') - - import concurrent.futures - import multiprocessing - max_workers = multiprocessing.cpu_count() - self.log(logging.INFO, 'static-analysis', {}, - "RUNNING: infer autotest for platform {0} with {1} workers.".format( - self.platform[0], max_workers)) - # clean previous autotest if it exists - rc = self._gradle(['autotest:clean'], autotest=True) - if rc != 0: - return rc - import yaml - with open(mozpath.join(self.__infer_tool, 'config.yaml')) as f: - config = yaml.safe_load(f) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [] - for item in config['infer_checkers']: - if item['publish']: - futures.append(executor.submit(self._verify_infer_checker, item)) - # this is always included in check-java, but not in config.yaml - futures.append(executor.submit(self._verify_infer_checker, - {'name': 'checkers'})) - for future in concurrent.futures.as_completed(futures): - ret_val = future.result() - if ret_val != self.TOOLS_SUCCESS: - return ret_val - self.log(logging.INFO, 'static-analysis', {}, "SUCCESS: infer all tests passed.") - else: - self.log(logging.WARNING, 'static-analysis', {}, - "Skipping infer autotest, because it is only available on linux64!") - return self.TOOLS_SUCCESS - - def _verify_infer_checker(self, item): - '''Given a checker, this method verifies the following: - 1. if there is a `checker`.json and `checker`.java file in - `tools/infer/test/autotest/src` - 2. if running infer on `checker`.java yields the same result as `checker`.json - An `item` is simply a dictionary, which needs to have a `name` field set, which is the - name of the checker. - ''' - def to_camelcase(str): - return ''.join([s.capitalize() for s in str.split('-')]) - check = item['name'] - test_file_path = mozpath.join(self.__infer_tool, 'test', 'autotest', 'src', - 'main', 'java', to_camelcase(check)) - test_file_path_java = test_file_path + '.java' - test_file_path_json = test_file_path + '.json' - self.log(logging.INFO, 'static-analysis', {}, "RUNNING: infer check {}.".format(check)) - # Verify if the test file exists for this checker - if not os.path.exists(test_file_path_java): - self.log(logging.ERROR, 'static-analysis', {}, - "ERROR: infer check {} doesn't have a test file.".format(check)) - return self.TOOLS_CHECKER_NO_TEST_FILE - # run infer on a particular test file - out_folder = mozpath.join(self.__infer_test_folder, 'test-infer-{}'.format(check)) - if check == 'checkers': - check_arg = ['-a', 'checkers'] - else: - check_arg = ['--{}-only'.format(check)] - infer_args = [self._infer_path, 'run'] + check_arg + ['-o', out_folder, '--'] - gradle_args = ['autotest:compileInferTest{}'.format(to_camelcase(check))] - rc = self._gradle(gradle_args, infer_args=infer_args, autotest=True) - if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, - "ERROR: infer failed to execute gradle {}.".format(gradle_args)) - return self.TOOLS_GRADLE_FAILED - issues = json.load(open(mozpath.join(out_folder, 'report.json'))) - # remove folder that infer creates because the issues are loaded into memory - import shutil - shutil.rmtree(out_folder) - # Verify to see if we got any issues, if not raise exception - if not issues: - self.log( - logging.ERROR, 'static-analysis', {}, - "ERROR: infer check {0} did not find any issues in its associated test suite." - .format(check) - ) - return self.TOOLS_CHECKER_RETURNED_NO_ISSUES - if self._dump_results: - self._build_autotest_result(test_file_path_json, issues) - else: - if not os.path.exists(test_file_path_json): - # Result file for test not found maybe regenerate it? - self.log( - logging.ERROR, 'static-analysis', {}, - "ERROR: infer result file not found for check {0}".format(check) - ) - return self.TOOLS_CHECKER_RESULT_FILE_NOT_FOUND - # Read the pre-determined issues - baseline_issues = self._get_autotest_stored_issues(test_file_path_json) - - def ordered(obj): - if isinstance(obj, dict): - return sorted((k, ordered(v)) for k, v in obj.items()) - if isinstance(obj, list): - return sorted(ordered(x) for x in obj) - return obj - # Compare the two lists - if ordered(issues) != ordered(baseline_issues): - error_str = "ERROR: in check {} Expected: ".format(check) - error_str += '\n' + json.dumps(baseline_issues, indent=2) - error_str += '\n Got:\n' + json.dumps(issues, indent=2) - self.log(logging.ERROR, 'static-analysis', {}, - 'ERROR: infer autotest for check {} failed, check stdout for more details' - .format(check)) - print(error_str) - return self.TOOLS_CHECKER_DIFF_FAILED - return self.TOOLS_SUCCESS - @StaticAnalysisSubCommand('static-analysis', 'install', 'Install the static analysis helper tool') @CommandArgument('source', nargs='?', type=str, help='Where to fetch a local archive containing the static-analysis and ' 'format helper tool.' 'It will be installed in ~/.mozbuild/clang-tools and ~/.mozbuild/infer.' 'Can be omitted, in which case the latest clang-tools and infer ' 'helper for the platform would be automatically detected and installed.') @CommandArgument('--skip-cache', action='store_true', help='Skip all local caches to force re-fetching the helper tool.', default=False) def install(self, source=None, skip_cache=False, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(force=True, skip_cache=skip_cache, source=source, verbose=verbose) if rc == 0: # XXX ignore the return code because if it fails or not, infer is # not mandatory, but clang-tidy is self._get_infer(force=True, skip_cache=skip_cache, verbose=verbose) return rc @StaticAnalysisSubCommand('static-analysis', 'clear-cache', 'Delete local helpers and reset static analysis helper tool cache') def clear_cache(self, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(force=True, download_if_needed=True, skip_cache=True, verbose=verbose) if rc == 0: self._get_infer(force=True, download_if_needed=True, skip_cache=True, verbose=verbose) if rc != 0: return rc return self._artifact_manager.artifact_clear_cache() @StaticAnalysisSubCommand('static-analysis', 'print-checks', 'Print a list of the static analysis checks performed by default') def print_checks(self, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(verbose=verbose) if rc == 0: rc = self._get_infer(verbose=verbose) if rc != 0: return rc args = [self._clang_tidy_path, '-list-checks', '-checks=%s' % self._get_checks()] rc = self._run_command_in_objdir(args=args, pass_thru=True) if rc != 0: return rc checkers, _ = self._get_infer_config() print('Infer checks:') for checker in checkers: print(' '*4 + checker) return 0 @Command('clang-format', category='misc', description='Run clang-format on current changes') @CommandArgument('--show', '-s', action='store_true', default=False, help='Show diff output on instead of applying changes') @CommandArgument('--path', '-p', nargs='+', default=None, help='Specify the path(s) to reformat') def clang_format(self, show, path, verbose=False): # Run clang-format or clang-format-diff on the local changes # or files/directories if path is not None: path = self._conv_to_abspath(path) os.chdir(self.topsrcdir) rc = self._get_clang_tools(verbose=verbose) if rc != 0: return rc if path is None: return self._run_clang_format_diff(self._clang_format_diff, self._clang_format_path, show) else: return self._run_clang_format_path(self._clang_format_path, show, path) def _verify_checker(self, item): check = item['name'] test_file_path = mozpath.join(self._clang_tidy_base_path, "test", check) test_file_path_cpp = test_file_path + '.cpp' test_file_path_json = test_file_path + '.json' self.log(logging.INFO, 'static-analysis', {}, "RUNNING: clang-tidy checker {}.".format(check)) # Verify if this checker actually exists if not check in self._clang_tidy_checks: self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't exist in this clang-tidy version.".format(check)) return self.TOOLS_CHECKER_NOT_FOUND # Verify if the test file exists for this checker if not os.path.exists(test_file_path_cpp): self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't have a test file.".format(check)) return self.TOOLS_CHECKER_NO_TEST_FILE cmd = self._get_clang_tidy_command( checks='-*,' + check, header_filter='', sources=[test_file_path_cpp], jobs=1, fix=False) try: clang_output = subprocess.check_output( cmd, stderr=subprocess.STDOUT).decode('utf-8') except subprocess.CalledProcessError as e: print(e.output) return self.TOOLS_CHECKER_FAILED_FILE else: issues = self._parse_issues(clang_output) # Verify to see if we got any issues, if not raise exception if not issues: self.log( logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {0} did not find any issues in its associated test file.". format(check)) return self.CHECKER_RETURNED_NO_ISSUES if self._dump_results: self._build_autotest_result(test_file_path_json, issues) else: if not os.path.exists(test_file_path_json): # Result file for test not found maybe regenerate it? self.log( logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy result file not found for checker {0}".format( check)) return self.TOOLS_CHECKER_RESULT_FILE_NOT_FOUND # Read the pre-determined issues baseline_issues = self._get_autotest_stored_issues(test_file_path_json) # Compare the two lists if issues != baseline_issues: print("Clang output: {}".format(clang_output)) self.log( logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy auto-test failed for checker {0} Expected: {1} Got: {2}". format(check, baseline_issues, issues)) return self.TOOLS_CHECKER_DIFF_FAILED return self.TOOLS_SUCCESS def _build_autotest_result(self, file, issues): with open(file, 'w') as f: json.dump(issues, f, indent=4, sort_keys=True) def _get_autotest_stored_issues(self, file): with open(file) as f: return json.load(f) def _parse_issues(self, clang_output): ''' Parse clang-tidy output into structured issues ''' # Limit clang output parsing to 'Enabled checks:' end = re.search(r'^Enabled checks:\n', clang_output, re.MULTILINE) if end is not None: clang_output = clang_output[:end.start()-1] # Sort headers by positions regex_header = re.compile( r'(.+):(\d+):(\d+): (warning|error): ([^\[\]\n]+)(?: \[([\.\w-]+)\])?$', re.MULTILINE) something = regex_header.finditer(clang_output) headers = sorted( regex_header.finditer(clang_output), key=lambda h: h.start() ) issues = [] for _, header in enumerate(headers): header_group = header.groups() element = [header_group[3], header_group[4], header_group[5]] issues.append(element) return json.dumps(issues) def _get_checks(self): checks = '-*' import yaml with open(mozpath.join(self.topsrcdir, "tools", "clang-tidy", "config.yaml")) as f: try: config = yaml.safe_load(f) for item in config['clang_checkers']: if item.get('publish', True): checks += ',' + item['name'] except Exception: print('Looks like config.yaml is not valid, so we are unable to ' 'determine default checkers, using \'-checks=-*,mozilla-*\'') checks += ',mozilla-*' return checks def _get_config_environment(self): ran_configure = False config = None builder = Build(self._mach_context) try: config = self.config_environment except Exception: print('Looks like configure has not run yet, running it now...') rc = builder.configure() if rc != 0: return (rc, config, ran_configure) ran_configure = True try: config = self.config_environment except Exception as e: pass return (0, config, ran_configure) def _build_compile_db(self, verbose=False): self._compile_db = mozpath.join(self.topobjdir, 'compile_commands.json') if os.path.exists(self._compile_db): return 0 else: rc, config, ran_configure = self._get_config_environment() if rc != 0: return rc if ran_configure: # Configure may have created the compilation database if the # mozconfig enables building the CompileDB backend by default, # So we recurse to see if the file exists once again. return self._build_compile_db(verbose=verbose) if config: print('Looks like a clang compilation database has not been ' 'created yet, creating it now...') builder = Build(self._mach_context) rc = builder.build_backend(['CompileDB'], verbose=verbose) if rc != 0: return rc assert os.path.exists(self._compile_db) return 0 def _build_export(self, jobs, verbose=False): def on_line(line): self.log(logging.INFO, 'build_output', {'line': line}, '{line}') builder = Build(self._mach_context) # First install what we can through install manifests. rc = builder._run_make(directory=self.topobjdir, target='pre-export', line_handler=None, silent=not verbose) if rc != 0: return rc # Then build the rest of the build dependencies by running the full # export target, because we can't do anything better. return builder._run_make(directory=self.topobjdir, target='export', line_handler=None, silent=not verbose, num_jobs=jobs) def _conv_to_abspath(self, paths): # Converts all the paths to absolute pathnames tmp_path = [] for f in paths: tmp_path.append(os.path.abspath(f)) return tmp_path def _get_clang_tools(self, force=False, skip_cache=False, source=None, download_if_needed=True, - verbose=False, intree_tool=False): + verbose=False): rc, config, _ = self._get_config_environment() if rc != 0: return rc - clang_tools_path = self.topsrcdir if intree_tool else \ - mozpath.join(self._mach_context.state_dir, "clang-tools") + clang_tools_path = mozpath.join(self._mach_context.state_dir, "clang-tools") self._clang_tidy_path = mozpath.join(clang_tools_path, "clang", "bin", "clang-tidy" + config.substs.get('BIN_SUFFIX', '')) self._clang_format_path = mozpath.join( clang_tools_path, "clang", "bin", "clang-format" + config.substs.get('BIN_SUFFIX', '')) self._clang_apply_replacements = mozpath.join( clang_tools_path, "clang", "bin", "clang-apply-replacements" + config.substs.get('BIN_SUFFIX', '')) self._run_clang_tidy_path = mozpath.join(clang_tools_path, "clang", "share", "clang", "run-clang-tidy.py") self._clang_format_diff = mozpath.join(clang_tools_path, "clang", "share", "clang", "clang-format-diff.py") - if intree_tool: - # Ensure that clang-tidy is present - return not os.path.exists(self._clang_tidy_path) + if os.path.exists(self._clang_tidy_path) and \ os.path.exists(self._clang_format_path) and \ os.path.exists(self._clang_apply_replacements) and \ os.path.exists(self._run_clang_tidy_path) and \ not force: return 0 else: if os.path.isdir(clang_tools_path) and download_if_needed: # The directory exists, perhaps it's corrupted? Delete it # and start from scratch. import shutil shutil.rmtree(clang_tools_path) return self._get_clang_tools(force=force, skip_cache=skip_cache, source=source, verbose=verbose, download_if_needed=download_if_needed) # Create base directory where we store clang binary os.mkdir(clang_tools_path) if source: return self._get_clang_tools_from_source(source) self._artifact_manager = PackageFrontend(self._mach_context) if not download_if_needed: return 0 job, _ = self.platform if job is None: raise Exception('The current platform isn\'t supported. ' 'Currently only the following platforms are ' 'supported: win32/win64, linux64 and macosx64.') else: job += '-clang-tidy' # We want to unpack data in the clang-tidy mozbuild folder currentWorkingDir = os.getcwd() os.chdir(clang_tools_path) rc = self._artifact_manager.artifact_toolchain(verbose=verbose, skip_cache=skip_cache, from_build=[job], no_unpack=False, retry=0) # Change back the cwd os.chdir(currentWorkingDir) return rc def _get_clang_tools_from_source(self, filename): from mozbuild.action.tooltool import unpack_file clang_tidy_path = mozpath.join(self._mach_context.state_dir, "clang-tools") currentWorkingDir = os.getcwd() os.chdir(clang_tidy_path) unpack_file(filename) # Change back the cwd os.chdir(currentWorkingDir) clang_path = mozpath.join(clang_tidy_path, 'clang') if not os.path.isdir(clang_path): raise Exception('Extracted the archive but didn\'t find ' 'the expected output') assert os.path.exists(self._clang_tidy_path) assert os.path.exists(self._clang_format_path) assert os.path.exists(self._clang_apply_replacements) assert os.path.exists(self._run_clang_tidy_path) return 0 def _get_clang_format_diff_command(self): if self.repository.name == 'hg': args = ["hg", "diff", "-U0", "-r" ".^"] for dot_extension in self._format_include_extensions: args += ['--include', 'glob:**{0}'.format(dot_extension)] args += ['--exclude', 'listfile:{0}'.format(self._format_ignore_file)] else: args = ["git", "diff", "--no-color", "-U0", "HEAD", "--"] for dot_extension in self._format_include_extensions: args += ['*{0}'.format(dot_extension)] # git-diff doesn't support an 'exclude-from-files' param, but # allow to add individual exclude pattern since v1.9, see # https://git-scm.com/docs/gitglossary#gitglossary-aiddefpathspecapathspec with open(self._format_ignore_file, 'rb') as exclude_pattern_file: for pattern in exclude_pattern_file.readlines(): pattern = pattern.rstrip() pattern = pattern.replace('.*', '**') if not pattern or pattern.startswith('#'): continue # empty or comment magics = ['exclude'] if pattern.startswith('^'): magics += ['top'] pattern = pattern[1:] args += [':({0}){1}'.format(','.join(magics), pattern)] return args - def _get_infer(self, force=False, skip_cache=False, download_if_needed=True, - verbose=False, intree_tool=False): + def _get_infer(self, force=False, skip_cache=False, + download_if_needed=True, verbose=False): rc, config, _ = self._get_config_environment() if rc != 0: return rc - infer_path = self.topsrcdir if intree_tool else \ - mozpath.join(self._mach_context.state_dir, 'infer') - self._infer_path = mozpath.join(infer_path, 'infer', 'bin', 'infer' + + infer_path = mozpath.join(self._mach_context.state_dir, 'infer') + self._infer_path = mozpath.join(infer_path, 'infer', 'bin', + 'infer' + config.substs.get('BIN_SUFFIX', '')) - if intree_tool: - return not os.path.exists(self._infer_path) if os.path.exists(self._infer_path) and not force: return 0 else: if os.path.isdir(infer_path) and download_if_needed: # The directory exists, perhaps it's corrupted? Delete it # and start from scratch. import shutil shutil.rmtree(infer_path) return self._get_infer(force=force, skip_cache=skip_cache, verbose=verbose, download_if_needed=download_if_needed) os.mkdir(infer_path) self._artifact_manager = PackageFrontend(self._mach_context) if not download_if_needed: return 0 job, _ = self.platform if job != 'linux64': return -1 else: job += '-infer' # We want to unpack data in the infer mozbuild folder currentWorkingDir = os.getcwd() os.chdir(infer_path) rc = self._artifact_manager.artifact_toolchain(verbose=verbose, skip_cache=skip_cache, from_build=[job], no_unpack=False, retry=0) # Change back the cwd os.chdir(currentWorkingDir) return rc def _run_clang_format_diff(self, clang_format_diff, clang_format, show): # Run clang-format on the diff # Note that this will potentially miss a lot things from subprocess import Popen, PIPE, check_output, CalledProcessError diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE) args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format] if not show: args.append("-i") try: output = check_output(args, stdin=diff_process.stdout) if show: # We want to print the diffs print(output) return 0 except CalledProcessError as e: # Something wrong happend print("clang-format: An error occured while running clang-format-diff.") return e.returncode def _is_ignored_path(self, ignored_dir_re, f): # Remove upto topsrcdir in pathname and match if f.startswith(self.topsrcdir + '/'): match_f = f[len(self.topsrcdir + '/'):] else: match_f = f return re.match(ignored_dir_re, match_f) def _generate_path_list(self, paths): path_to_third_party = os.path.join(self.topsrcdir, self._format_ignore_file) ignored_dir = [] with open(path_to_third_party, 'r') as fh: for line in fh: # Remove comments and empty lines if line.startswith('#') or len(line.strip()) == 0: continue # The regexp is to make sure we are managing relative paths ignored_dir.append(r"^[\./]*" + line.rstrip()) # Generates the list of regexp ignored_dir_re = '(%s)' % '|'.join(ignored_dir) extensions = self._format_include_extensions path_list = [] for f in paths: if self._is_ignored_path(ignored_dir_re, f): # Early exit if we have provided an ignored directory print("clang-format: Ignored third party code '{0}'".format(f)) continue if os.path.isdir(f): # Processing a directory, generate the file list for folder, subs, files in os.walk(f): subs.sort() for filename in sorted(files): f_in_dir = os.path.join(folder, filename) if (f_in_dir.endswith(extensions) and not self._is_ignored_path(ignored_dir_re, f_in_dir)): # Supported extension and accepted path path_list.append(f_in_dir) else: if f.endswith(extensions): path_list.append(f) return path_list def _run_clang_format_path(self, clang_format, show, paths): # Run clang-format on files or directories directly from subprocess import check_output, CalledProcessError args = [clang_format, "-i"] path_list = self._generate_path_list(paths) if path_list == []: return print("Processing %d file(s)..." % len(path_list)) batchsize = 200 for i in range(0, len(path_list), batchsize): l = path_list[i: (i + batchsize)] # Run clang-format on the list try: check_output(args + l) except CalledProcessError as e: # Something wrong happend print("clang-format: An error occured while running clang-format.") return e.returncode if show: # show the diff if self.repository.name == 'hg': diff_command = ["hg", "diff"] + paths else: assert self.repository.name == 'git' diff_command = ["git", "diff"] + paths try: output = check_output(diff_command) print(output) except CalledProcessError as e: # Something wrong happend print("clang-format: Unable to run the diff command.") return e.returncode return 0 @CommandProvider class Vendor(MachCommandBase): """Vendor third-party dependencies into the source repository.""" @Command('vendor', category='misc', description='Vendor third-party dependencies into the source repository.') def vendor(self): self.parser.print_usage() sys.exit(1) @SubCommand('vendor', 'rust', description='Vendor rust crates from crates.io into third_party/rust') @CommandArgument('--ignore-modified', action='store_true', help='Ignore modified files in current checkout', default=False) @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true', help='Permit overly-large files to be added to the repository', default=False) def vendor_rust(self, **kwargs): from mozbuild.vendor_rust import VendorRust vendor_command = self._spawn(VendorRust) vendor_command.vendor(**kwargs) @SubCommand('vendor', 'aom', description='Vendor av1 video codec reference implementation into the source repository.') @CommandArgument('-r', '--revision', help='Repository tag or commit to update to.') @CommandArgument('--repo', help='Repository url to pull a snapshot from. Supports github and googlesource.') @CommandArgument('--ignore-modified', action='store_true', help='Ignore modified files in current checkout', default=False) def vendor_aom(self, **kwargs): from mozbuild.vendor_aom import VendorAOM vendor_command = self._spawn(VendorAOM) vendor_command.vendor(**kwargs) @SubCommand('vendor', 'python', description='Vendor Python packages from pypi.org into third_party/python') @CommandArgument('packages', default=None, nargs='*', help='Packages to vendor. If omitted, packages and their dependencies defined in Pipfile.lock will be vendored. If Pipfile has been modified, then Pipfile.lock will be regenerated. Note that transient dependencies may be updated when running this command.') def vendor_python(self, **kwargs): from mozbuild.vendor_python import VendorPython vendor_command = self._spawn(VendorPython) vendor_command.vendor(**kwargs) @SubCommand('vendor', 'manifest', description='Vendor externally hosted repositories into this ' 'repository.') @CommandArgument('files', nargs='+', help='Manifest files to work on') @CommandArgumentGroup('verify') @CommandArgument('--verify', '-v', action='store_true', group='verify', required=True, help='Verify manifest') def vendor_manifest(self, files, verify): from mozbuild.vendor_manifest import verify_manifests verify_manifests(files) @CommandProvider class WebRTCGTestCommands(GTestCommands): @Command('webrtc-gtest', category='testing', description='Run WebRTC.org GTest unit tests.') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, group='debugging', help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') def gtest(self, gtest_filter, debug, debugger, debugger_args): app_path = self.get_binary_path('webrtc-gtest') args = [app_path] if debug or debugger or debugger_args: args = self.prepend_debugger_args(args, debugger, debugger_args) # Used to locate resources used by tests cwd = os.path.join(self.topsrcdir, 'media', 'webrtc', 'trunk') if not os.path.isdir(cwd): print('Unable to find working directory for tests: %s' % cwd) return 1 gtest_env = { # These tests are not run under ASAN upstream, so we need to # disable some checks. b'ASAN_OPTIONS': 'alloc_dealloc_mismatch=0', # Use GTest environment variable to control test execution # For details see: # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options b'GTEST_FILTER': gtest_filter } return self.run_process(args=args, append_env=gtest_env, cwd=cwd, ensure_exit_code=False, pass_thru=True) @CommandProvider class Repackage(MachCommandBase): '''Repackages artifacts into different formats. This is generally used after packages are signed by the signing scriptworkers in order to bundle things up into shippable formats, such as a .dmg on OSX or an installer exe on Windows. ''' @Command('repackage', category='misc', description='Repackage artifacts into different formats.') def repackage(self): print("Usage: ./mach repackage [dmg|installer|mar] [args...]") @SubCommand('repackage', 'dmg', description='Repackage a tar file into a .dmg for OSX') @CommandArgument('--input', '-i', type=str, required=True, help='Input filename') @CommandArgument('--output', '-o', type=str, required=True, help='Output filename') def repackage_dmg(self, input, output): if not os.path.exists(input): print('Input file does not exist: %s' % input) return 1 if not os.path.exists(os.path.join(self.topobjdir, 'config.status')): print('config.status not found. Please run |mach configure| ' 'prior to |mach repackage|.') return 1 from mozbuild.repackaging.dmg import repackage_dmg repackage_dmg(input, output) @SubCommand('repackage', 'installer', description='Repackage into a Windows installer exe') @CommandArgument('--tag', type=str, required=True, help='The .tag file used to build the installer') @CommandArgument('--setupexe', type=str, required=True, help='setup.exe file inside the installer') @CommandArgument('--package', type=str, required=False, help='Optional package .zip for building a full installer') @CommandArgument('--output', '-o', type=str, required=True, help='Output filename') @CommandArgument('--package-name', type=str, required=False, help='Name of the package being rebuilt') @CommandArgument('--sfx-stub', type=str, required=True, help='Path to the self-extraction stub.') def repackage_installer(self, tag, setupexe, package, output, package_name, sfx_stub): from mozbuild.repackaging.installer import repackage_installer repackage_installer( topsrcdir=self.topsrcdir, tag=tag, setupexe=setupexe, package=package, output=output, package_name=package_name, sfx_stub=sfx_stub, ) @SubCommand('repackage', 'mar', description='Repackage into complete MAR file') @CommandArgument('--input', '-i', type=str, required=True, help='Input filename') @CommandArgument('--mar', type=str, required=True, help='Mar binary path') @CommandArgument('--output', '-o', type=str, required=True, help='Output filename') @CommandArgument('--format', type=str, default='lzma', choices=('lzma', 'bz2'), help='Mar format') def repackage_mar(self, input, mar, output, format): from mozbuild.repackaging.mar import repackage_mar repackage_mar(self.topsrcdir, input, mar, output, format) @CommandProvider class Analyze(MachCommandBase): """ Get information about a file in the build graph """ @Command('analyze', category='misc', description='Analyze the build graph.') def analyze(self): print("Usage: ./mach analyze [files|report] [args...]") @SubCommand('analyze', 'files', description='Get incremental build cost for file(s) from the tup database.') @CommandArgument('--path', help='Path to tup db', default=None) @CommandArgument('files', nargs='*', help='Files to analyze') def analyze_files(self, path, files): from mozbuild.analyze.graph import Graph if path is None: path = mozpath.join(self.topsrcdir, '.tup', 'db') if os.path.isfile(path): g = Graph(path) g.file_summaries(files) g.close() else: res = 'Please make sure you have a local tup db *or* specify the location with --path.' print ('Could not find a valid tup db in ' + path, res, sep='\n') return 1 @SubCommand('analyze', 'all', description='Get a report of files changed within the last n days and their corresponding build cost.') @CommandArgument('--days', '-d', type=int, default=14, help='Number of days to include in the report.') @CommandArgument('--format', default='pretty', choices=['pretty', 'csv', 'json', 'html'], help='Print or export data in the given format.') @CommandArgument('--limit', type=int, default=None, help='Get the top n most expensive files from the report.') @CommandArgument('--path', help='Path to cost_dict.gz', default=None) def analyze_report(self, days, format, limit, path): from mozbuild.analyze.hg import Report self._activate_virtualenv() try: self.virtualenv_manager.install_pip_package('tablib==0.12.1') except Exception: print ('Could not install tablib via pip.') return 1 if path is None: # go find tup db and make a cost_dict from mozbuild.analyze.graph import Graph db_path = mozpath.join(self.topsrcdir, '.tup', 'db') if os.path.isfile(db_path): g = Graph(db_path) r = Report(days, cost_dict=g.get_cost_dict()) g.close() r.generate_output(format, limit, self.topobjdir) else: res = 'Please specify the location of cost_dict.gz with --path.' print ('Could not find %s to make a cost dictionary.' % db_path, res, sep='\n') return 1 else: # path to cost_dict.gz was specified if os.path.isfile(path): r = Report(days, path) r.generate_output(format, limit, self.topobjdir) else: res = 'Please specify the location of cost_dict.gz with --path.' print ('Could not find cost_dict.gz at %s' % path, res, sep='\n') return 1 diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -1,114 +1,114 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. loader: taskgraph.loader.transform:loader kind-dependencies: - packages transforms: - taskgraph.transforms.docker_image:transforms - taskgraph.transforms.task:transforms # make a task for each docker-image we might want. For the moment, since we # write artifacts for each, these are whitelisted, but ideally that will change # (to use subdirectory clones of the proper directory), at which point we can # generate tasks for every docker image in the directory, secure in the # knowledge that unnecessary images will be omitted from the target task graph jobs: image_builder: symbol: I(ib) desktop1604-test: symbol: I(dt16t) debian7-base: symbol: I(deb7-base) definition: debian-base args: DIST: wheezy BASE_TAG: '20171210' SNAPSHOT: '20171210T214726Z' packages: - deb7-gdb - deb7-git - deb7-make - deb7-mercurial - deb7-python - deb7-python3.5 - deb7-python3-defaults - deb7-python-zstandard - deb7-xz-utils toolchain-build: symbol: I(toolchain) parent: debian7-base packages: - deb7-cmake - deb7-ninja debian7-amd64-build: symbol: I(deb7) parent: debian7-base definition: debian7-build packages: - deb7-valgrind args: ARCH: amd64 debian7-i386-build: symbol: I(deb7-32) parent: debian7-base definition: debian7-build packages: - deb7-valgrind args: ARCH: i386 debian7-mozjs-rust-build: symbol: I(deb7jsrs) parent: debian7-amd64-build packages: - deb7-cmake valgrind-build: symbol: I(vb) parent: debian7-amd64-build lint: symbol: I(lnt) debian9-base: symbol: I(deb9-base) definition: debian-base args: DIST: stretch BASE_TAG: '20170620' SNAPSHOT: '20170830T000511Z' packages: - deb9-mercurial - deb9-python-zstandard android-build: symbol: I(agb) parent: debian9-base fetch: symbol: I(fetch) parent: debian9-base - static-analysis-build: - symbol: I(static-analysis-build) - parent: android-build + infer-build: + symbol: I(infer) + parent: debian9-base mingw32-build: symbol: I(mingw) parent: debian9-base index-task: symbol: I(idx) funsize-update-generator: symbol: I(pg) google-play-strings: symbol: I(gps) update-verify: symbol: I(uv) diffoscope: symbol: I(diff) partner-repack: symbol: I(PR) parent: debian9-base definition: partner-repack periodic-updates: symbol: I(file) pipfile-updates: symbol: I(pip) firefox-snap: symbol: I(snap) diff --git a/taskcluster/ci/static-analysis-autotest/kind.yml b/taskcluster/ci/static-analysis-autotest/kind.yml --- a/taskcluster/ci/static-analysis-autotest/kind.yml +++ b/taskcluster/ci/static-analysis-autotest/kind.yml @@ -1,84 +1,80 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. loader: taskgraph.loader.transform:loader kind-dependencies: - toolchain transforms: - taskgraph.transforms.build_attrs:transforms - taskgraph.transforms.build_lints:transforms - taskgraph.transforms.use_toolchains:transforms - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms job-defaults: index: product: firefox worker: skip-artifacts: true max-run-time: 3600 env: PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest run: using: mozharness actions: [static-analysis-autotest] script: mozharness/scripts/fx_desktop_build.py treeherder: symbol: Sa kind: build tier: 1 jobs: linux64-st-autotest/debug: description: "Linux64 Debug Static Analysis Autotest" index: job-name: linux64-st-autotest-debug - worker: - docker-image: {in-tree: static-analysis-build} - env: - PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest treeherder: platform: linux64/debug worker-type: aws-provisioner-v1/gecko-t-linux-large run: config: - builds/releng_base_firefox.py - builds/releng_sub_linux_configs/64_stat_and_debug.py tooltool-downloads: public keep-artifacts: false toolchains: - linux64-clang - linux64-clang-tidy - linux64-infer - linux64-rust - linux64-sccache - linux64-cbindgen - linux64-node win64-st-autotest/debug: description: "Win64 Debug Static Analysis Autotest" index: job-name: win64-st-autotest-debug treeherder: platform: windows2012-64/debug tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 worker: env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest" run: config: - builds/releng_base_firefox.py - builds/taskcluster_base_windows.py - builds/taskcluster_base_win64.py - builds/taskcluster_sub_win64/clang_debug.py toolchains: - win64-clang-cl-st-an - win64-rust - win64-sccache - win64-cbindgen - win64-clang-tidy - win64-node diff --git a/taskcluster/ci/toolchain/linux.yml b/taskcluster/ci/toolchain/linux.yml --- a/taskcluster/ci/toolchain/linux.yml +++ b/taskcluster/ci/toolchain/linux.yml @@ -1,761 +1,761 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. job-defaults: worker: docker-image: {in-tree: toolchain-build} linux64-clang-3.9: description: "Clang 3.9 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(clang3.9) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large worker: max-run-time: 7200 run: using: toolchain-script script: build-clang-3.9-linux.sh resources: - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-3.9-linux64.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/clang.tar.xz toolchains: - linux64-gcc-4.9 linux64-clang-6: description: "Clang 6 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(clang6) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge worker: max-run-time: 7200 run: using: toolchain-script script: build-clang-6-linux.sh resources: - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-6-linux64.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-alias: linux64-clang toolchain-artifact: public/build/clang.tar.xz toolchains: - linux64-gcc-4.9 linux64-clang-7: description: "Clang 7 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(clang7) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge worker: max-run-time: 7200 run: using: toolchain-script script: build-clang-7-linux.sh resources: - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-7-pre-linux64.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/clang.tar.xz toolchains: - linux64-gcc-4.9 linux64-clang-7-mingw: description: "MinGW-Clang 7 Pre toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TMW(clang7p) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge worker: max-run-time: 7200 run: using: toolchain-script script: build-clang-7-pre-mingw.sh resources: - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-7-pre-mingw.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/clangmingw.tar.xz toolchains: - linux64-gcc-4.9 linux64-clang-6-macosx-cross: description: "Clang 6 toolchain build with MacOS Compiler RT libs" treeherder: kind: build platform: toolchains/opt symbol: TL(clang6-macosx-cross) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest" run: using: toolchain-script script: build-clang-6-linux-macosx-cross.sh resources: - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-6-macosx64.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-alias: linux64-clang-macosx-cross toolchain-artifact: public/build/clang.tar.xz tooltool-downloads: internal toolchains: - linux64-cctools-port - linux64-clang-6 - linux64-gcc-4.9 linux64-clang-tidy: description: "Clang-tidy build" index: product: static-analysis job-name: linux64-clang-tidy treeherder: kind: build platform: toolchains/opt symbol: TL(clang-tidy) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large worker: max-run-time: 3600 run: using: toolchain-script script: build-clang-tidy-linux.sh resources: - 'build/clang-plugin/**' - 'build/build-clang/build-clang.py' - 'build/build-clang/clang-tidy-linux64.json' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/clang-tidy.tar.xz run-on-projects: - trunk - try toolchains: - linux64-gcc-4.9 linux64-infer: description: "infer build" index: product: static-analysis job-name: linux64-infer treeherder: kind: build platform: toolchains/opt symbol: TL(infer) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: - docker-image: {in-tree: static-analysis-build} + docker-image: {in-tree: infer-build} max-run-time: 3600 run: using: toolchain-script script: build-infer-linux.sh resources: - 'build/build-infer/build-infer.py' - 'build/build-infer/infer-linux64.json' toolchain-artifact: public/build/infer.tar.xz linux64-gcc-4.9: description: "GCC 4.9 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(gcc4.9) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-gcc-4.9-linux.sh resources: - 'build/unix/build-gcc/build-gcc.sh' toolchain-artifact: public/build/gcc.tar.xz fetches: fetch: - binutils-2.25.1 - cloog-0.18.1 - gcc-4.9.4 - gmp-5.1.3 - isl-0.12.2 - mpc-0.8.2 - mpfr-3.1.5 linux64-gcc-6: description: "GCC 6 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(gcc6) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-gcc-6-linux.sh resources: - 'build/unix/build-gcc/build-gcc.sh' toolchain-alias: linux64-gcc toolchain-artifact: public/build/gcc.tar.xz fetches: fetch: - binutils-2.28.1 - gcc-6.4.0 - gmp-5.1.3 - isl-0.15 - mpc-0.8.2 - mpfr-3.1.5 linux64-gcc-7: description: "GCC 7 toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(gcc7) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-gcc-7-linux.sh resources: - 'build/unix/build-gcc/build-gcc.sh' toolchain-artifact: public/build/gcc.tar.xz fetches: fetch: - binutils-2.28.1 - gcc-7.3.0 - gmp-6.1.0 - isl-0.16.1 - mpc-1.0.3 - mpfr-3.1.4 linux64-gcc-sixgill: description: "sixgill GCC plugin build" treeherder: kind: build platform: toolchains/opt symbol: TL(sixgill) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-gcc-sixgill-plugin-linux.sh resources: - 'build/unix/build-gcc/build-gcc.sh' - 'taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh' toolchain-artifact: public/build/sixgill.tar.xz toolchains: - linux64-gcc-6 fetches: fetch: - binutils-2.28.1 - isl-0.15 - gcc-6.4.0 - gmp-5.1.3 - mpc-0.8.2 - mpfr-3.1.5 linux64-llvm-dsymutil: description: "llvm-dsymutil toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(dsymutil) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-llvm-dsymutil.sh toolchain-artifact: public/build/llvm-dsymutil.tar.xz toolchains: - linux64-gcc-4.9 linux64-binutils: description: "Binutils toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(binutil) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-binutils-linux.sh resources: - 'build/unix/build-binutils/build-binutils.sh' toolchain-artifact: public/build/binutils.tar.xz fetches: fetch: - binutils-2.31.1 linux64-cctools-port: description: "cctools-port toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(cctools) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-cctools-port.sh resources: - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/cctools.tar.xz toolchains: - linux64-clang-6 linux64-hfsplus: description: "hfsplus toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(hfs+) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-hfsplus-linux.sh resources: - 'build/unix/build-hfsplus/build-hfsplus.sh' - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/hfsplus-tools.tar.xz toolchains: - linux64-clang-6 linux64-libdmg: description: "libdmg-hfsplus toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(libdmg-hfs+) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-libdmg-hfsplus.sh toolchain-artifact: public/build/dmg.tar.xz linux64-node: description: "Node repack toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(node) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: toolchain-build} max-run-time: 1800 run: using: toolchain-script script: repack-node.sh arguments: ['linux64'] toolchain-artifact: public/build/node.tar.xz linux64-android-sdk-linux-repack: description: "Android SDK (Linux) repack toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(android-sdk-linux) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: android-build} max-run-time: 1800 artifacts: - name: project/gecko/android-sdk path: /builds/worker/project/gecko/android-sdk/ type: directory run: using: toolchain-script script: repack-android-sdk-linux.sh resources: - 'python/mozboot/**/*android*' toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz toolchain-alias: android-sdk-linux linux64-android-ndk-linux-repack: description: "Android NDK (Linux) repack toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(android-ndk-linux) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: android-build} max-run-time: 1800 artifacts: - name: project/gecko/android-ndk path: /builds/worker/project/gecko/android-ndk/ type: directory run: using: toolchain-script script: repack-android-ndk-linux.sh resources: - 'python/mozboot/**/*android*' toolchain-artifact: project/gecko/android-ndk/android-ndk.tar.xz toolchain-alias: android-ndk-linux linux64-android-gradle-dependencies: description: "Android Gradle dependencies toolchain task" treeherder: kind: build platform: toolchains/opt symbol: TL(gradle-dependencies) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: android-build} env: GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online" max-run-time: 1800 run: using: toolchain-script script: android-gradle-dependencies.sh sparse-profile: null resources: - 'taskcluster/scripts/misc/tooltool-download.sh' - 'taskcluster/scripts/misc/android-gradle-dependencies/**' - '*.gradle' - 'mobile/android/**/*.gradle' - 'mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/**' - 'mobile/android/config/mozconfigs/common*' - 'mobile/android/gradle.configure' toolchain-artifact: public/build/android-gradle-dependencies.tar.xz toolchain-alias: android-gradle-dependencies toolchains: # Aliases aren't allowed for toolchains depending on toolchains. - linux64-android-sdk-linux-repack - linux64-node linux64-rust-1.28: description: "rust repack" treeherder: kind: build platform: toolchains/opt symbol: TL(rust) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 7200 env: UPLOAD_DIR: artifacts run: using: toolchain-script script: repack_rust.py arguments: [ '--channel', '1.28.0', '--host', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-unknown-linux-gnu', '--target', 'i686-unknown-linux-gnu', ] toolchain-alias: linux64-rust toolchain-artifact: public/build/rustc.tar.xz linux64-rust-nightly: description: "rust nightly repack" treeherder: kind: build platform: toolchains/opt symbol: TL(rust-nightly) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 7200 env: UPLOAD_DIR: artifacts run: using: toolchain-script script: repack_rust.py arguments: [ '--channel', 'nightly-2018-08-18', '--host', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-unknown-linux-gnu', '--target', 'i686-unknown-linux-gnu', ] toolchain-artifact: public/build/rustc.tar.xz linux64-rust-macos-1.28: description: "rust repack with macos-cross support" treeherder: kind: build platform: toolchains/opt symbol: TL(rust-macos) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 7200 env: UPLOAD_DIR: artifacts run: using: toolchain-script script: repack_rust.py arguments: [ '--channel', '1.28.0', '--host', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-apple-darwin', ] toolchain-alias: linux64-rust-macos toolchain-artifact: public/build/rustc.tar.xz linux64-rust-android-1.28: description: "rust repack with android-cross support" treeherder: kind: build platform: toolchains/opt symbol: TL(rust-android) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 7200 env: UPLOAD_DIR: artifacts run: using: toolchain-script script: repack_rust.py arguments: [ '--channel', '1.28.0', '--host', 'x86_64-unknown-linux-gnu', '--target', 'x86_64-unknown-linux-gnu', '--target', 'armv7-linux-androideabi', '--target', 'aarch64-linux-android', '--target', 'i686-linux-android', ] toolchain-alias: linux64-rust-android toolchain-artifact: public/build/rustc.tar.xz linux64-sccache: description: "sccache toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(sccache) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-sccache.sh resources: - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/sccache2.tar.xz toolchains: - linux64-rust-1.28 linux64-cbindgen: description: "cbindgen toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(cbindgen) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-cbindgen.sh arguments: ['x86_64-unknown-linux-gnu'] resources: - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/cbindgen.tar.xz toolchains: - linux64-rust-1.28 linux64-rust-size: description: "rust-size toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(rust-size) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-rust-size.sh resources: - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/rust-size.tar.xz toolchains: - linux64-rust-1.28 linux64-gn: description: "gn toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(gn) tier: 1 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 1800 run: using: toolchain-script script: build-gn-linux.sh tooltool-downloads: public resources: - 'taskcluster/scripts/misc/tooltool-download.sh' - 'taskcluster/scripts/misc/build-gn-common.sh' toolchain-artifact: public/build/gn.tar.xz run-on-projects: - trunk - try toolchains: - linux64-gcc-4.9 linux64-tup: description: "tup toolchain build" treeherder: kind: build platform: toolchains/opt symbol: TL(tup) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: max-run-time: 3600 run: using: toolchain-script script: build-tup-linux.sh resources: - 'taskcluster/scripts/misc/tooltool-download.sh' toolchain-artifact: public/build/tup.tar.xz run-on-projects: - trunk - try toolchains: - linux64-gcc-4.9 linux64-upx: description: "UPX build for MinGW32 Cross Compile" treeherder: kind: build platform: toolchains/opt symbol: TMW(upx) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: mingw32-build} max-run-time: 3600 run: using: toolchain-script script: build-upx.sh toolchain-artifact: public/build/upx.tar.xz linux64-wine: description: "Wine build for MinGW32 Cross Compile" treeherder: kind: build platform: toolchains/opt symbol: TMW(wine) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: mingw32-build} max-run-time: 10800 run: using: toolchain-script script: build-wine.sh toolchain-artifact: public/build/wine.tar.xz linux64-mingw32-gcc: description: "GCC toolchain build for MinGW32 Cross Compile" treeherder: kind: build platform: toolchains/opt symbol: TMW(mingw32-gcc) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: mingw32-build} max-run-time: 10800 run: using: toolchain-script script: build-gcc-mingw32.sh resources: - 'build/unix/build-gcc/build-gcc.sh' toolchain-artifact: public/build/mingw32.tar.xz fetches: fetch: - binutils-2.27 - gcc-6.4.0 - gmp-5.1.3 - isl-0.15 - mpc-0.8.2 - mpfr-3.1.5 linux64-mingw32-nsis: description: "NSIS build for MinGW32 Cross Compile" treeherder: kind: build platform: toolchains/opt symbol: TMW(mingw32-nsis) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: mingw32-build} max-run-time: 3600 run: using: toolchain-script script: build-mingw32-nsis.sh resources: - 'build/unix/build-gcc/build-gcc.sh' - 'taskcluster/scripts/misc/build-gcc-mingw32.sh' toolchain-artifact: public/build/nsis.tar.xz toolchains: - linux64-mingw32-gcc linux64-mingw32-fxc2: description: "fxc2.exe build for MinGW32 Cross Compile" treeherder: kind: build platform: toolchains/opt symbol: TMW(mingw32-fxc2) tier: 2 worker-type: aws-provisioner-v1/gecko-{level}-b-linux worker: docker-image: {in-tree: mingw32-build} max-run-time: 1800 run: using: toolchain-script script: build-mingw32-fxc2.sh resources: - 'build/unix/build-gcc/build-gcc.sh' - 'taskcluster/scripts/misc/build-gcc-mingw32.sh' toolchain-artifact: public/build/fxc2.tar.xz toolchains: - linux64-mingw32-gcc diff --git a/taskcluster/docker/static-analysis-build/Dockerfile b/taskcluster/docker/infer-build/Dockerfile rename from taskcluster/docker/static-analysis-build/Dockerfile rename to taskcluster/docker/infer-build/Dockerfile --- a/taskcluster/docker/static-analysis-build/Dockerfile +++ b/taskcluster/docker/infer-build/Dockerfile @@ -1,58 +1,30 @@ # %ARG DOCKER_IMAGE_PARENT FROM $DOCKER_IMAGE_PARENT MAINTAINER Robert Bartlensky VOLUME /builds/worker/checkouts VOLUME /builds/worker/workspace VOLUME /builds/worker/tooltool-cache ENV XZ_OPT=-T0 RUN apt-get update && \ apt-get install \ - autoconf2.13 \ - automake \ + autoconf \ bison \ bzip2 \ - cmake \ flex \ curl \ + git \ opam \ libsqlite3-dev \ - file \ - gawk \ - gcc-multilib \ - gnupg \ + autoconf \ + automake \ + cmake \ libc6-dev \ openjdk-8-jdk-headless \ pkg-config \ patch \ - p7zip-full \ - procps \ - python-pip \ - python-setuptools \ - python-virtualenv \ - rsync \ - screen \ tar \ unzip \ - uuid \ - valgrind \ - wget \ - yasm \ - zip \ - zlib1g-dev \ - x11-utils \ - xvfb \ - linux-libc-dev \ - libdbus-glib-1-dev \ - libfontconfig1-dev \ - libfreetype6-dev \ - libgconf2-dev \ - libgtk-3-dev \ - libgtk2.0-dev \ - libpango1.0-dev \ - libpulse-dev \ - libx11-xcb-dev \ - libxt-dev \ - lib32z1 + zlib1g-dev diff --git a/testing/mozharness/mozharness/base/script.py b/testing/mozharness/mozharness/base/script.py --- a/testing/mozharness/mozharness/base/script.py +++ b/testing/mozharness/mozharness/base/script.py @@ -1,2327 +1,2328 @@ + #!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """Generic script objects. script.py, along with config.py and log.py, represents the core of mozharness. """ import codecs from contextlib import contextmanager import datetime import errno import fnmatch import functools import gzip import inspect import itertools import os import platform import pprint import re import shutil import socket import subprocess import sys import tarfile import time import traceback import urllib2 import zipfile import httplib import urlparse import hashlib import zlib if os.name == 'nt': import locale try: import win32file import win32api PYWIN32 = True except ImportError: PYWIN32 = False try: import simplejson as json assert json except ImportError: import json from io import BytesIO import mozinfo from mozprocess import ProcessHandler from mozharness.base.config import BaseConfig from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \ LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL class ContentLengthMismatch(Exception): pass def platform_name(): pm = PlatformMixin() if pm._is_linux() and pm._is_64_bit(): return 'linux64' elif pm._is_linux() and not pm._is_64_bit(): return 'linux' elif pm._is_darwin(): return 'macosx' elif pm._is_windows() and pm._is_64_bit(): return 'win64' elif pm._is_windows() and not pm._is_64_bit(): return 'win32' else: return None class PlatformMixin(object): def _is_windows(self): """ check if the current operating system is Windows. Returns: bool: True if the current platform is Windows, False otherwise """ system = platform.system() if system in ("Windows", "Microsoft"): return True if system.startswith("CYGWIN"): return True if os.name == 'nt': return True def _is_darwin(self): """ check if the current operating system is Darwin. Returns: bool: True if the current platform is Darwin, False otherwise """ if platform.system() in ("Darwin"): return True if sys.platform.startswith("darwin"): return True def _is_linux(self): """ check if the current operating system is a Linux distribution. Returns: bool: True if the current platform is a Linux distro, False otherwise """ if platform.system() in ("Linux"): return True if sys.platform.startswith("linux"): return True def _is_redhat(self): """ check if the current operating system is a Redhat derived Linux distribution. Returns: bool: True if the current platform is a Redhat Linux distro, False otherwise """ if not self._is_linux(): return False re_redhat_distro = re.compile('Redhat|Fedora|CentOS|Oracle') return re_redhat_distro.match(mozinfo.linux_distro) is not None def _is_64_bit(self): if self._is_darwin(): # osx is a special snowflake and to ensure the arch, it is better to use the following return sys.maxsize > 2**32 # context: https://docs.python.org/2/library/platform.html else: # Using machine() gives you the architecture of the host rather # than the build type of the Python binary return '64' in platform.machine() # ScriptMixin {{{1 class ScriptMixin(PlatformMixin): """This mixin contains simple filesystem commands and the like. It also contains some very special but very complex methods that, together with logging and config, provide the base for all scripts in this harness. WARNING !!! This class depends entirely on `LogMixin` methods in such a way that it will only works if a class inherits from both `ScriptMixin` and `LogMixin` simultaneously. Depends on self.config of some sort. Attributes: env (dict): a mapping object representing the string environment. script_obj (ScriptMixin): reference to a ScriptMixin instance. """ env = None script_obj = None def query_filesize(self, file_path): self.info("Determining filesize for %s" % file_path) length = os.path.getsize(file_path) self.info(" %s" % str(length)) return length # TODO this should be parallelized with the to-be-written BaseHelper! def query_sha512sum(self, file_path): self.info("Determining sha512sum for %s" % file_path) m = hashlib.sha512() contents = self.read_from_file(file_path, verbose=False, open_mode='rb') m.update(contents) sha512 = m.hexdigest() self.info(" %s" % sha512) return sha512 def platform_name(self): """ Return the platform name on which the script is running on. Returns: None: for failure to determine the platform. str: The name of the platform (e.g. linux64) """ return platform_name() # Simple filesystem commands {{{2 def mkdir_p(self, path, error_level=ERROR): """ Create a directory if it doesn't exists. This method also logs the creation, error or current existence of the directory to be created. Args: path (str): path of the directory to be created. error_level (str): log level name to be used in case of error. Returns: None: for sucess. int: -1 on error """ if not os.path.exists(path): self.info("mkdir: %s" % path) try: os.makedirs(path) except OSError: self.log("Can't create directory %s!" % path, level=error_level) return -1 else: self.debug("mkdir_p: %s Already exists." % path) def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1): """ Delete an entire directory tree and log its result. This method also logs the platform rmtree function, its retries, errors, and current existence of the directory. Args: path (str): path to the directory tree root to remove. log_level (str, optional): log level name to for this operation. Defaults to `INFO`. error_level (str, optional): log level name to use in case of error. Defaults to `ERROR`. exit_code (int, optional): useless parameter, not use here. Defaults to -1 Returns: None: for success """ self.log("rmtree: %s" % path, level=log_level) error_message = "Unable to remove %s!" % path if self._is_windows(): # Call _rmtree_windows() directly, since even checking # os.path.exists(path) will hang if path is longer than MAX_PATH. self.info("Using _rmtree_windows ...") return self.retry( self._rmtree_windows, error_level=error_level, error_message=error_message, args=(path, ), log_level=log_level, ) if os.path.exists(path): if os.path.isdir(path): return self.retry( shutil.rmtree, error_level=error_level, error_message=error_message, retry_exceptions=(OSError, ), args=(path, ), log_level=log_level, ) else: return self.retry( os.remove, error_level=error_level, error_message=error_message, retry_exceptions=(OSError, ), args=(path, ), log_level=log_level, ) else: self.debug("%s doesn't exist." % path) def query_msys_path(self, path): """ replaces the Windows harddrive letter path style with a linux path style, e.g. C:// --> /C/ Note: method, not used in any script. Args: path (str?): path to convert to the linux path style. Returns: str: in case `path` is a string. The result is the path with the new notation. type(path): `path` itself is returned in case `path` is not str type. """ if not isinstance(path, basestring): return path path = path.replace("\\", "/") def repl(m): return '/%s/' % m.group(1) path = re.sub(r'''^([a-zA-Z]):/''', repl, path) return path def _rmtree_windows(self, path): """ Windows-specific rmtree that handles path lengths longer than MAX_PATH. Ported from clobberer.py. Args: path (str): directory path to remove. Returns: None: if the path doesn't exists. int: the return number of calling `self.run_command` int: in case the path specified is not a directory but a file. 0 on success, non-zero on error. Note: The returned value is the result of calling `win32file.DeleteFile` """ assert self._is_windows() path = os.path.realpath(path) full_path = '\\\\?\\' + path if not os.path.exists(full_path): return if not PYWIN32: if not os.path.isdir(path): return self.run_command('del /F /Q "%s"' % path) else: return self.run_command('rmdir /S /Q "%s"' % path) # Make sure directory is writable win32file.SetFileAttributesW('\\\\?\\' + path, win32file.FILE_ATTRIBUTE_NORMAL) # Since we call rmtree() with a file, sometimes if not os.path.isdir('\\\\?\\' + path): return win32file.DeleteFile('\\\\?\\' + path) for ffrec in win32api.FindFiles('\\\\?\\' + path + '\\*.*'): file_attr = ffrec[0] name = ffrec[8] if name == '.' or name == '..': continue full_name = os.path.join(path, name) if file_attr & win32file.FILE_ATTRIBUTE_DIRECTORY: self._rmtree_windows(full_name) else: try: win32file.SetFileAttributesW('\\\\?\\' + full_name, win32file.FILE_ATTRIBUTE_NORMAL) win32file.DeleteFile('\\\\?\\' + full_name) except: # DeleteFile fails on long paths, del /f /q works just fine self.run_command('del /F /Q "%s"' % full_name) win32file.RemoveDirectory('\\\\?\\' + path) def get_filename_from_url(self, url): """ parse a filename base on an url. Args: url (str): url to parse for the filename Returns: str: filename parsed from the url, or `netloc` network location part of the url. """ parsed = urlparse.urlsplit(url.rstrip('/')) if parsed.path != '': return parsed.path.rsplit('/', 1)[-1] else: return parsed.netloc def _urlopen(self, url, **kwargs): """ open the url `url` using `urllib2`. This method can be overwritten to extend its complexity Args: url (str | urllib2.Request): url to open kwargs: Arbitrary keyword arguments passed to the `urllib2.urlopen` function. Returns: file-like: file-like object with additional methods as defined in `urllib2.urlopen`_. None: None may be returned if no handler handles the request. Raises: urllib2.URLError: on errors .. _urllib2.urlopen: https://docs.python.org/2/library/urllib2.html#urllib2.urlopen """ # http://bugs.python.org/issue13359 - urllib2 does not automatically quote the URL url_quoted = urllib2.quote(url, safe='%/:=&?~#+!$,;\'@()*[]|') return urllib2.urlopen(url_quoted, **kwargs) def fetch_url_into_memory(self, url): ''' Downloads a file from a url into memory instead of disk. Args: url (str): URL path where the file to be downloaded is located. Raises: IOError: When the url points to a file on disk and cannot be found ContentLengthMismatch: When the length of the retrieved content does not match the Content-Length response header. ValueError: When the scheme of a url is not what is expected. Returns: BytesIO: contents of url ''' self.info('Fetch {} into memory'.format(url)) parsed_url = urlparse.urlparse(url) if parsed_url.scheme in ('', 'file'): path = parsed_url.path if not os.path.isfile(path): raise IOError('Could not find file to extract: {}'.format(url)) content_length = os.stat(path).st_size # In case we're referrencing a file without file:// if parsed_url.scheme == '': url = 'file://%s' % os.path.abspath(url) parsed_url = urlparse.urlparse(url) request = urllib2.Request(url) # When calling fetch_url_into_memory() you should retry when we raise one of these exceptions: # * Bug 1300663 - HTTPError: HTTP Error 404: Not Found # * Bug 1300413 - HTTPError: HTTP Error 500: Internal Server Error # * Bug 1300943 - HTTPError: HTTP Error 503: Service Unavailable # * Bug 1300953 - URLError: # * Bug 1301594 - URLError: # * Bug 1302237 - URLError: # * Bug 1301807 - BadStatusLine: '' # # Bug 1309912 - Adding timeout in hopes to solve blocking on response.read() (bug 1300413) response = urllib2.urlopen(request, timeout=30) if parsed_url.scheme in ('http', 'https'): content_length = int(response.headers.get('Content-Length')) response_body = response.read() response_body_size = len(response_body) self.info('Content-Length response header: {}'.format(content_length)) self.info('Bytes received: {}'.format(response_body_size)) if response_body_size != content_length: raise ContentLengthMismatch( 'The retrieved Content-Length header declares a body length of {} bytes, while we actually retrieved {} bytes'.format( content_length, response_body_size) ) if response.info().get('Content-Encoding') == 'gzip': self.info('Content-Encoding is "gzip", so decompressing response body') # See http://www.zlib.net/manual.html#Advanced # section "ZEXTERN int ZEXPORT inflateInit2 OF....": # Add 32 to windowBits to enable zlib and gzip decoding with automatic # header detection, or add 16 to decode only the gzip format (the zlib # format will return a Z_DATA_ERROR). # Adding 16 since we only wish to support gzip encoding. file_contents = zlib.decompress(response_body, zlib.MAX_WBITS|16) else: file_contents = response_body # Use BytesIO instead of StringIO # http://stackoverflow.com/questions/34162017/unzip-buffer-with-python/34162395#34162395 return BytesIO(file_contents) def _download_file(self, url, file_name): """ Helper function for download_file() Additionaly this function logs all exceptions as warnings before re-raising them Args: url (str): string containing the URL with the file location file_name (str): name of the file where the downloaded file is written. Returns: str: filename of the written file on disk Raises: urllib2.URLError: on incomplete download. urllib2.HTTPError: on Http error code socket.timeout: on connection timeout socket.error: on socket error """ # If our URLs look like files, prefix them with file:// so they can # be loaded like URLs. if not (url.startswith("http") or url.startswith("file://")): if not os.path.isfile(url): self.fatal("The file %s does not exist" % url) url = 'file://%s' % os.path.abspath(url) try: f_length = None f = self._urlopen(url, timeout=30) if f.info().get('content-length') is not None: f_length = int(f.info()['content-length']) got_length = 0 if f.info().get('Content-Encoding') == 'gzip': # Note, we'll download the full compressed content into its own # file, since that allows the gzip library to seek through it. # Once downloaded, we'll decompress it into the real target # file, and delete the compressed version. local_file = open(file_name + '.gz', 'wb') else: local_file = open(file_name, 'wb') while True: block = f.read(1024 ** 2) if not block: if f_length is not None and got_length != f_length: raise urllib2.URLError("Download incomplete; content-length was %d, but only received %d" % (f_length, got_length)) break local_file.write(block) if f_length is not None: got_length += len(block) local_file.close() if f.info().get('Content-Encoding') == 'gzip': # Decompress file into target location, then remove compressed version with open(file_name, 'wb') as f_out: # On some execution paths, this could be called with python 2.6 # whereby gzip.open(...) cannot be used with a 'with' statement. # So let's do this the python 2.6 way... try: f_in = gzip.open(file_name + '.gz', 'rb') shutil.copyfileobj(f_in, f_out) finally: f_in.close() os.remove(file_name + '.gz') return file_name except urllib2.HTTPError, e: self.warning("Server returned status %s %s for %s" % (str(e.code), str(e), url)) raise except urllib2.URLError, e: self.warning("URL Error: %s" % url) # Failures due to missing local files won't benefit from retry. # Raise the original OSError. if isinstance(e.args[0], OSError) and e.args[0].errno == errno.ENOENT: raise e.args[0] raise except socket.timeout, e: self.warning("Timed out accessing %s: %s" % (url, str(e))) raise except socket.error, e: self.warning("Socket error when accessing %s: %s" % (url, str(e))) raise def _retry_download(self, url, error_level, file_name=None, retry_config=None): """ Helper method to retry download methods. This method calls `self.retry` on `self._download_file` using the passed parameters if a file_name is specified. If no file is specified, we will instead call `self._urlopen`, which grabs the contents of a url but does not create a file on disk. Args: url (str): URL path where the file is located. file_name (str): file_name where the file will be written to. error_level (str): log level to use in case an error occurs. retry_config (dict, optional): key-value pairs to be passed to `self.retry`. Defaults to `None` Returns: str: `self._download_file` return value is returned unknown: `self.retry` `failure_status` is returned on failure, which defaults to -1 """ retry_args = dict( failure_status=None, retry_exceptions=(urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine, socket.timeout, socket.error), error_message="Can't download from %s to %s!" % (url, file_name), error_level=error_level, ) if retry_config: retry_args.update(retry_config) download_func = self._urlopen kwargs = {"url": url} if file_name: download_func = self._download_file kwargs = {"url": url, "file_name": file_name} return self.retry( download_func, kwargs=kwargs, **retry_args ) def _filter_entries(self, namelist, extract_dirs): """Filter entries of the archive based on the specified list of to extract dirs.""" filter_partial = functools.partial(fnmatch.filter, namelist) entries = itertools.chain(*map(filter_partial, extract_dirs or ['*'])) for entry in entries: yield entry def unzip(self, compressed_file, extract_to, extract_dirs='*', verbose=False): """This method allows to extract a zip file without writing to disk first. Args: compressed_file (object): File-like object with the contents of a compressed zip file. extract_to (str): where to extract the compressed file. extract_dirs (list, optional): directories inside the archive file to extract. Defaults to '*'. verbose (bool, optional): whether or not extracted content should be displayed. Defaults to False. Raises: zipfile.BadZipfile: on contents of zipfile being invalid """ with zipfile.ZipFile(compressed_file) as bundle: entries = self._filter_entries(bundle.namelist(), extract_dirs) for entry in entries: if verbose: self.info(' {}'.format(entry)) # Exception to be retried: # Bug 1301645 - BadZipfile: Bad CRC-32 for file ... # http://stackoverflow.com/questions/5624669/strange-badzipfile-bad-crc-32-problem/5626098#5626098 # Bug 1301802 - error: Error -3 while decompressing: invalid stored block lengths bundle.extract(entry, path=extract_to) # ZipFile doesn't preserve permissions during extraction: # http://bugs.python.org/issue15795 fname = os.path.realpath(os.path.join(extract_to, entry)) try: # getinfo() can raise KeyError mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF # Only set permissions if attributes are available. Otherwise all # permissions will be removed eg. on Windows. if mode: os.chmod(fname, mode) except KeyError: self.warning('{} was not found in the zip file'.format(entry)) def deflate(self, compressed_file, mode, extract_to='.', *args, **kwargs): """This method allows to extract a compressed file from a tar, tar.bz2 and tar.gz files. Args: compressed_file (object): File-like object with the contents of a compressed file. mode (str): string of the form 'filemode[:compression]' (e.g. 'r:gz' or 'r:bz2') extract_to (str, optional): where to extract the compressed file. """ t = tarfile.open(fileobj=compressed_file, mode=mode) t.extractall(path=extract_to) def download_unpack(self, url, extract_to='.', extract_dirs='*', verbose=False): """Generic method to download and extract a compressed file without writing it to disk first. Args: url (str): URL where the file to be downloaded is located. extract_to (str, optional): directory where the downloaded file will be extracted to. extract_dirs (list, optional): directories inside the archive to extract. Defaults to `*`. It currently only applies to zip files. verbose (bool, optional): whether or not extracted content should be displayed. Defaults to False. """ def _determine_extraction_method_and_kwargs(url): EXTENSION_TO_MIMETYPE = { 'bz2': 'application/x-bzip2', 'gz': 'application/x-gzip', 'tar': 'application/x-tar', 'zip': 'application/zip', } MIMETYPES = { 'application/x-bzip2': { 'function': self.deflate, 'kwargs': {'mode': 'r:bz2'}, }, 'application/x-gzip': { 'function': self.deflate, 'kwargs': {'mode': 'r:gz'}, }, 'application/x-tar': { 'function': self.deflate, 'kwargs': {'mode': 'r'}, }, 'application/zip': { 'function': self.unzip, }, 'application/x-zip-compressed': { 'function': self.unzip, }, } filename = url.split('/')[-1] # XXX: bz2/gz instead of tar.{bz2/gz} extension = filename[filename.rfind('.')+1:] mimetype = EXTENSION_TO_MIMETYPE[extension] self.debug('Mimetype: {}'.format(mimetype)) function = MIMETYPES[mimetype]['function'] kwargs = { 'compressed_file': compressed_file, 'extract_to': extract_to, 'extract_dirs': extract_dirs, 'verbose': verbose, } kwargs.update(MIMETYPES[mimetype].get('kwargs', {})) return function, kwargs # Many scripts overwrite this method and set extract_dirs to None extract_dirs = '*' if extract_dirs is None else extract_dirs self.info('Downloading and extracting to {} these dirs {} from {}'.format( extract_to, ', '.join(extract_dirs), url, )) # 1) Let's fetch the file retry_args = dict( retry_exceptions=( urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine, socket.timeout, socket.error, ContentLengthMismatch, ), sleeptime=30, attempts=5, error_message="Can't download from {}".format(url), error_level=FATAL, ) compressed_file = self.retry( self.fetch_url_into_memory, kwargs={'url': url}, **retry_args ) # 2) We're guaranteed to have download the file with error_level=FATAL # Let's unpack the file function, kwargs = _determine_extraction_method_and_kwargs(url) try: function(**kwargs) except zipfile.BadZipfile: # Dump the exception and exit self.exception(level=FATAL) def load_json_url(self, url, error_level=None, *args, **kwargs): """ Returns a json object from a url (it retries). """ contents = self._retry_download( url=url, error_level=error_level, *args, **kwargs ) return json.loads(contents.read()) # http://www.techniqal.com/blog/2008/07/31/python-file-read-write-with-urllib2/ # TODO thinking about creating a transfer object. def download_file(self, url, file_name=None, parent_dir=None, create_parent_dir=True, error_level=ERROR, exit_code=3, retry_config=None): """ Python wget. Download the filename at `url` into `file_name` and put it on `parent_dir`. On error log with the specified `error_level`, on fatal exit with `exit_code`. Execute all the above based on `retry_config` parameter. Args: url (str): URL path where the file to be downloaded is located. file_name (str, optional): file_name where the file will be written to. Defaults to urls' filename. parent_dir (str, optional): directory where the downloaded file will be written to. Defaults to current working directory create_parent_dir (bool, optional): create the parent directory if it doesn't exist. Defaults to `True` error_level (str, optional): log level to use in case an error occurs. Defaults to `ERROR` retry_config (dict, optional): key-value pairs to be passed to `self.retry`. Defaults to `None` Returns: str: filename where the downloaded file was written to. unknown: on failure, `failure_status` is returned. """ if not file_name: try: file_name = self.get_filename_from_url(url) except AttributeError: self.log("Unable to get filename from %s; bad url?" % url, level=error_level, exit_code=exit_code) return if parent_dir: file_name = os.path.join(parent_dir, file_name) if create_parent_dir: self.mkdir_p(parent_dir, error_level=error_level) self.info("Downloading %s to %s" % (url, file_name)) status = self._retry_download( url=url, error_level=error_level, file_name=file_name, retry_config=retry_config ) if status == file_name: self.info("Downloaded %d bytes." % os.path.getsize(file_name)) return status def move(self, src, dest, log_level=INFO, error_level=ERROR, exit_code=-1): """ recursively move a file or directory (src) to another location (dest). Args: src (str): file or directory path to move. dest (str): file or directory path where to move the content to. log_level (str): log level to use for normal operation. Defaults to `INFO` error_level (str): log level to use on error. Defaults to `ERROR` Returns: int: 0 on success. -1 on error. """ self.log("Moving %s to %s" % (src, dest), level=log_level) try: shutil.move(src, dest) # http://docs.python.org/tutorial/errors.html except IOError, e: self.log("IO error: %s" % str(e), level=error_level, exit_code=exit_code) return -1 except shutil.Error, e: self.log("shutil error: %s" % str(e), level=error_level, exit_code=exit_code) return -1 return 0 def chmod(self, path, mode): """ change `path` mode to `mode`. Args: path (str): path whose mode will be modified. mode (hex): one of the values defined at `stat`_ .. _stat: https://docs.python.org/2/library/os.html#os.chmod """ self.info("Chmoding %s to %s" % (path, str(oct(mode)))) os.chmod(path, mode) def copyfile(self, src, dest, log_level=INFO, error_level=ERROR, copystat=False, compress=False): """ copy or compress `src` into `dest`. Args: src (str): filepath to copy. dest (str): filepath where to move the content to. log_level (str, optional): log level to use for normal operation. Defaults to `INFO` error_level (str, optional): log level to use on error. Defaults to `ERROR` copystat (bool, optional): whether or not to copy the files metadata. Defaults to `False`. compress (bool, optional): whether or not to compress the destination file. Defaults to `False`. Returns: int: -1 on error None: on success """ if compress: self.log("Compressing %s to %s" % (src, dest), level=log_level) try: infile = open(src, "rb") outfile = gzip.open(dest, "wb") outfile.writelines(infile) outfile.close() infile.close() except IOError, e: self.log("Can't compress %s to %s: %s!" % (src, dest, str(e)), level=error_level) return -1 else: self.log("Copying %s to %s" % (src, dest), level=log_level) try: shutil.copyfile(src, dest) except (IOError, shutil.Error), e: self.log("Can't copy %s to %s: %s!" % (src, dest, str(e)), level=error_level) return -1 if copystat: try: shutil.copystat(src, dest) except (IOError, shutil.Error), e: self.log("Can't copy attributes of %s to %s: %s!" % (src, dest, str(e)), level=error_level) return -1 def copytree(self, src, dest, overwrite='no_overwrite', log_level=INFO, error_level=ERROR): """ An implementation of `shutil.copytree` that allows for `dest` to exist and implements different overwrite levels: - 'no_overwrite' will keep all(any) existing files in destination tree - 'overwrite_if_exists' will only overwrite destination paths that have the same path names relative to the root of the src and destination tree - 'clobber' will replace the whole destination tree(clobber) if it exists Args: src (str): directory path to move. dest (str): directory path where to move the content to. overwrite (str): string specifying the overwrite level. log_level (str, optional): log level to use for normal operation. Defaults to `INFO` error_level (str, optional): log level to use on error. Defaults to `ERROR` Returns: int: -1 on error None: on success """ self.info('copying tree: %s to %s' % (src, dest)) try: if overwrite == 'clobber' or not os.path.exists(dest): self.rmtree(dest) shutil.copytree(src, dest) elif overwrite == 'no_overwrite' or overwrite == 'overwrite_if_exists': files = os.listdir(src) for f in files: abs_src_f = os.path.join(src, f) abs_dest_f = os.path.join(dest, f) if not os.path.exists(abs_dest_f): if os.path.isdir(abs_src_f): self.mkdir_p(abs_dest_f) self.copytree(abs_src_f, abs_dest_f, overwrite='clobber') else: shutil.copy2(abs_src_f, abs_dest_f) elif overwrite == 'no_overwrite': # destination path exists if os.path.isdir(abs_src_f) and os.path.isdir(abs_dest_f): self.copytree(abs_src_f, abs_dest_f, overwrite='no_overwrite') else: self.debug('ignoring path: %s as destination: \ %s exists' % (abs_src_f, abs_dest_f)) else: # overwrite == 'overwrite_if_exists' and destination exists self.debug('overwriting: %s with: %s' % (abs_dest_f, abs_src_f)) self.rmtree(abs_dest_f) if os.path.isdir(abs_src_f): self.mkdir_p(abs_dest_f) self.copytree(abs_src_f, abs_dest_f, overwrite='overwrite_if_exists') else: shutil.copy2(abs_src_f, abs_dest_f) else: self.fatal("%s is not a valid argument for param overwrite" % (overwrite)) except (IOError, shutil.Error): self.exception("There was an error while copying %s to %s!" % (src, dest), level=error_level) return -1 def write_to_file(self, file_path, contents, verbose=True, open_mode='w', create_parent_dir=False, error_level=ERROR): """ Write `contents` to `file_path`, according to `open_mode`. Args: file_path (str): filepath where the content will be written to. contents (str): content to write to the filepath. verbose (bool, optional): whether or not to log `contents` value. Defaults to `True` open_mode (str, optional): open mode to use for openning the file. Defaults to `w` create_parent_dir (bool, optional): whether or not to create the parent directory of `file_path` error_level (str, optional): log level to use on error. Defaults to `ERROR` Returns: str: `file_path` on success None: on error. """ self.info("Writing to file %s" % file_path) if verbose: self.info("Contents:") for line in contents.splitlines(): self.info(" %s" % line) if create_parent_dir: parent_dir = os.path.dirname(file_path) self.mkdir_p(parent_dir, error_level=error_level) try: fh = open(file_path, open_mode) try: fh.write(contents) except UnicodeEncodeError: fh.write(contents.encode('utf-8', 'replace')) fh.close() return file_path except IOError: self.log("%s can't be opened for writing!" % file_path, level=error_level) @contextmanager def opened(self, file_path, verbose=True, open_mode='r', error_level=ERROR): """ Create a context manager to use on a with statement. Args: file_path (str): filepath of the file to open. verbose (bool, optional): useless parameter, not used here. Defaults to True. open_mode (str, optional): open mode to use for openning the file. Defaults to `r` error_level (str, optional): log level name to use on error. Defaults to `ERROR` Yields: tuple: (file object, error) pair. In case of error `None` is yielded as file object, together with the corresponding error. If there is no error, `None` is returned as the error. """ # See opened_w_error in http://www.python.org/dev/peps/pep-0343/ self.info("Reading from file %s" % file_path) try: fh = open(file_path, open_mode) except IOError, err: self.log("unable to open %s: %s" % (file_path, err.strerror), level=error_level) yield None, err else: try: yield fh, None finally: fh.close() def read_from_file(self, file_path, verbose=True, open_mode='r', error_level=ERROR): """ Use `self.opened` context manager to open a file and read its content. Args: file_path (str): filepath of the file to read. verbose (bool, optional): whether or not to log the file content. Defaults to True. open_mode (str, optional): open mode to use for openning the file. Defaults to `r` error_level (str, optional): log level name to use on error. Defaults to `ERROR` Returns: None: on error. str: file content on success. """ with self.opened(file_path, verbose, open_mode, error_level) as (fh, err): if err: return None contents = fh.read() if verbose: self.info("Contents:") for line in contents.splitlines(): self.info(" %s" % line) return contents def chdir(self, dir_name): self.log("Changing directory to %s." % dir_name) os.chdir(dir_name) def is_exe(self, fpath): """ Determine if fpath is a file and if it is executable. """ return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def which(self, program): """ OS independent implementation of Unix's which command Args: program (str): name or path to the program whose executable is being searched. Returns: None: if the executable was not found. str: filepath of the executable file. """ if self._is_windows() and not program.endswith(".exe"): program += ".exe" fpath, fname = os.path.split(program) if fpath: if self.is_exe(program): return program else: # If the exe file is defined in the configs let's use that exe = self.query_exe(program) if self.is_exe(exe): return exe # If not defined, let's look for it in the $PATH env = self.query_env() for path in env["PATH"].split(os.pathsep): exe_file = os.path.join(path, program) if self.is_exe(exe_file): return exe_file return None # More complex commands {{{2 def retry(self, action, attempts=None, sleeptime=60, max_sleeptime=5 * 60, retry_exceptions=(Exception, ), good_statuses=None, cleanup=None, error_level=ERROR, error_message="%(action)s failed after %(attempts)d tries!", failure_status=-1, log_level=INFO, args=(), kwargs={}): """ generic retry command. Ported from `util.retry`_ Args: action (func): callable object to retry. attempts (int, optinal): maximum number of times to call actions. Defaults to `self.config.get('global_retries', 5)` sleeptime (int, optional): number of seconds to wait between attempts. Defaults to 60 and doubles each retry attempt, to a maximum of `max_sleeptime' max_sleeptime (int, optional): maximum value of sleeptime. Defaults to 5 minutes retry_exceptions (tuple, optional): Exceptions that should be caught. If exceptions other than those listed in `retry_exceptions' are raised from `action', they will be raised immediately. Defaults to (Exception) good_statuses (object, optional): return values which, if specified, will result in retrying if the return value isn't listed. Defaults to `None`. cleanup (func, optional): If `cleanup' is provided and callable it will be called immediately after an Exception is caught. No arguments will be passed to it. If your cleanup function requires arguments it is recommended that you wrap it in an argumentless function. Defaults to `None`. error_level (str, optional): log level name in case of error. Defaults to `ERROR`. error_message (str, optional): string format to use in case none of the attempts success. Defaults to '%(action)s failed after %(attempts)d tries!' failure_status (int, optional): flag to return in case the retries were not successfull. Defaults to -1. log_level (str, optional): log level name to use for normal activity. Defaults to `INFO`. args (tuple, optional): positional arguments to pass onto `action`. kwargs (dict, optional): key-value arguments to pass onto `action`. Returns: object: return value of `action`. int: failure status in case of failure retries. """ if not callable(action): self.fatal("retry() called with an uncallable method %s!" % action) if cleanup and not callable(cleanup): self.fatal("retry() called with an uncallable cleanup method %s!" % cleanup) if not attempts: attempts = self.config.get("global_retries", 5) if max_sleeptime < sleeptime: self.debug("max_sleeptime %d less than sleeptime %d" % ( max_sleeptime, sleeptime)) n = 0 while n <= attempts: retry = False n += 1 try: self.log("retry: Calling %s with args: %s, kwargs: %s, attempt #%d" % (action.__name__, str(args), str(kwargs), n), level=log_level) status = action(*args, **kwargs) if good_statuses and status not in good_statuses: retry = True except retry_exceptions, e: retry = True error_message = "%s\nCaught exception: %s" % (error_message, str(e)) self.log('retry: attempt #%d caught %s exception: %s' % (n, type(e).__name__, str(e)), level=INFO) if not retry: return status else: if cleanup: cleanup() if n == attempts: self.log(error_message % {'action': action, 'attempts': n}, level=error_level) return failure_status if sleeptime > 0: self.log("retry: Failed, sleeping %d seconds before retrying" % sleeptime, level=log_level) time.sleep(sleeptime) sleeptime = sleeptime * 2 if sleeptime > max_sleeptime: sleeptime = max_sleeptime def query_env(self, partial_env=None, replace_dict=None, purge_env=(), set_self_env=None, log_level=DEBUG, avoid_host_env=False): """ Environment query/generation method. The default, self.query_env(), will look for self.config['env'] and replace any special strings in there ( %(PATH)s ). It will then store it as self.env for speeding things up later. If you specify partial_env, partial_env will be used instead of self.config['env'], and we don't save self.env as it's a one-off. Args: partial_env (dict, optional): key-value pairs of the name and value of different environment variables. Defaults to an empty dictionary. replace_dict (dict, optional): key-value pairs to replace the old environment variables. purge_env (list): environment names to delete from the final environment dictionary. set_self_env (boolean, optional): whether or not the environment variables dictionary should be copied to `self`. Defaults to True. log_level (str, optional): log level name to use on normal operation. Defaults to `DEBUG`. avoid_host_env (boolean, optional): if set to True, we will not use any environment variables set on the host except PATH. Defaults to False. Returns: dict: environment variables names with their values. """ if partial_env is None: if self.env is not None: return self.env partial_env = self.config.get('env', None) if partial_env is None: partial_env = {} if set_self_env is None: set_self_env = True env = {'PATH': os.environ['PATH']} if avoid_host_env else os.environ.copy() default_replace_dict = self.query_abs_dirs() default_replace_dict['PATH'] = os.environ['PATH'] if not replace_dict: replace_dict = default_replace_dict else: for key in default_replace_dict: if key not in replace_dict: replace_dict[key] = default_replace_dict[key] for key in partial_env.keys(): env[key] = partial_env[key] % replace_dict self.log("ENV: %s is now %s" % (key, env[key]), level=log_level) for k in purge_env: if k in env: del env[k] if os.name == 'nt': pref_encoding = locale.getpreferredencoding() for k, v in env.iteritems(): # When run locally on Windows machines, some environment # variables may be unicode. if isinstance(v, unicode): env[k] = v.encode(pref_encoding) if set_self_env: self.env = env return env def query_exe(self, exe_name, exe_dict='exes', default=None, return_type=None, error_level=FATAL): """One way to work around PATH rewrites. By default, return exe_name, and we'll fall through to searching os.environ["PATH"]. However, if self.config[exe_dict][exe_name] exists, return that. This lets us override exe paths via config file. If we need runtime setting, we can build in self.exes support later. Args: exe_name (str): name of the executable to search for. exe_dict(str, optional): name of the dictionary of executables present in `self.config`. Defaults to `exes`. default (str, optional): default name of the executable to search for. Defaults to `exe_name`. return_type (str, optional): type to which the original return value will be turn into. Only 'list', 'string' and `None` are supported. Defaults to `None`. error_level (str, optional): log level name to use on error. Returns: list: in case return_type is 'list' str: in case return_type is 'string' None: in case return_type is `None` Any: if the found executable is not of type list, tuple nor str. """ if default is None: default = exe_name exe = self.config.get(exe_dict, {}).get(exe_name, default) repl_dict = {} if hasattr(self.script_obj, 'query_abs_dirs'): # allow for 'make': '%(abs_work_dir)s/...' etc. dirs = self.script_obj.query_abs_dirs() repl_dict.update(dirs) if isinstance(exe, dict): found = False # allow for searchable paths of the exe for name, path in exe.iteritems(): if isinstance(path, list) or isinstance(path, tuple): path = [x % repl_dict for x in path] if all([os.path.exists(section) for section in path]): found = True elif isinstance(path, str): path = path % repl_dict if os.path.exists(path): found = True else: self.log("a exes %s dict's value is not a string, list, or tuple. Got key " "%s and value %s" % (exe_name, name, str(path)), level=error_level) if found: exe = path break else: self.log("query_exe was a searchable dict but an existing path could not be " "determined. Tried searching in paths: %s" % (str(exe)), level=error_level) return None elif isinstance(exe, list) or isinstance(exe, tuple): exe = [x % repl_dict for x in exe] elif isinstance(exe, str): exe = exe % repl_dict else: self.log("query_exe: %s is not a list, tuple, dict, or string: " "%s!" % (exe_name, str(exe)), level=error_level) return exe if return_type == "list": if isinstance(exe, str): exe = [exe] elif return_type == "string": if isinstance(exe, list): exe = subprocess.list2cmdline(exe) elif return_type is not None: self.log("Unknown return_type type %s requested in query_exe!" % return_type, level=error_level) return exe def run_command(self, command, cwd=None, error_list=None, halt_on_failure=False, success_codes=None, env=None, partial_env=None, return_type='status', throw_exception=False, output_parser=None, output_timeout=None, fatal_exit_code=2, error_level=ERROR, **kwargs): """Run a command, with logging and error parsing. TODO: context_lines error_list example: [{'regex': re.compile('^Error: LOL J/K'), level=IGNORE}, {'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'}, {'substr': 'THE WORLD IS ENDING', level=FATAL, contextLines='20:'} ] (context_lines isn't written yet) Args: command (str | list | tuple): command or sequence of commands to execute and log. cwd (str, optional): directory path from where to execute the command. Defaults to `None`. error_list (list, optional): list of errors to pass to `mozharness.base.log.OutputParser`. Defaults to `None`. halt_on_failure (bool, optional): whether or not to redefine the log level as `FATAL` on errors. Defaults to False. success_codes (int, optional): numeric value to compare against the command return value. env (dict, optional): key-value of environment values to use to run the command. Defaults to None. partial_env (dict, optional): key-value of environment values to replace from the current environment values. Defaults to None. return_type (str, optional): if equal to 'num_errors' then the amount of errors matched by `error_list` is returned. Defaults to 'status'. throw_exception (bool, optional): whether or not to raise an exception if the return value of the command doesn't match any of the `success_codes`. Defaults to False. output_parser (OutputParser, optional): lets you provide an instance of your own OutputParser subclass. Defaults to `OutputParser`. output_timeout (int): amount of seconds to wait for output before the process is killed. fatal_exit_code (int, optional): call `self.fatal` if the return value of the command is not in `success_codes`. Defaults to 2. error_level (str, optional): log level name to use on error. Defaults to `ERROR`. **kwargs: Arbitrary keyword arguments. Returns: int: -1 on error. Any: `command` return value is returned otherwise. """ if success_codes is None: success_codes = [0] if cwd is not None: if not os.path.isdir(cwd): level = error_level if halt_on_failure: level = FATAL self.log("Can't run command %s in non-existent directory '%s'!" % (command, cwd), level=level) return -1 self.info("Running command: %s in %s" % (command, cwd)) else: self.info("Running command: %s" % command) if isinstance(command, list) or isinstance(command, tuple): self.info("Copy/paste: %s" % subprocess.list2cmdline(command)) shell = True if isinstance(command, list) or isinstance(command, tuple): shell = False if env is None: if partial_env: self.info("Using partial env: %s" % pprint.pformat(partial_env)) env = self.query_env(partial_env=partial_env) else: if hasattr(self, 'previous_env') and env == self.previous_env: self.info("Using env: (same as previous command)") else: self.info("Using env: %s" % pprint.pformat(env)) self.previous_env = env if output_parser is None: parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) else: parser = output_parser try: if output_timeout: def processOutput(line): parser.add_lines(line) def onTimeout(): self.info("Automation Error: mozprocess timed out after %s seconds running %s" % (str(output_timeout), str(command))) p = ProcessHandler(command, shell=shell, env=env, cwd=cwd, storeOutput=False, onTimeout=(onTimeout,), processOutputLine=[processOutput]) self.info("Calling %s with output_timeout %d" % (command, output_timeout)) p.run(outputTimeout=output_timeout) p.wait() if p.timedOut: self.log( 'timed out after %s seconds of no output' % output_timeout, level=error_level ) returncode = int(p.proc.returncode) else: p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, cwd=cwd, stderr=subprocess.STDOUT, env=env, bufsize=0) loop = True while loop: if p.poll() is not None: """Avoid losing the final lines of the log?""" loop = False while True: line = p.stdout.readline() if not line: break parser.add_lines(line) returncode = p.returncode except OSError, e: level = error_level if halt_on_failure: level = FATAL self.log('caught OS error %s: %s while running %s' % (e.errno, e.strerror, command), level=level) return -1 return_level = INFO if returncode not in success_codes: return_level = error_level if throw_exception: raise subprocess.CalledProcessError(returncode, command) self.log("Return code: %d" % returncode, level=return_level) if halt_on_failure: _fail = False if returncode not in success_codes: self.log( "%s not in success codes: %s" % (returncode, success_codes), level=error_level ) _fail = True if parser.num_errors: self.log("failures found while parsing output", level=error_level) _fail = True if _fail: self.return_code = fatal_exit_code self.fatal("Halting on failure while running %s" % command, exit_code=fatal_exit_code) if return_type == 'num_errors': return parser.num_errors return returncode def get_output_from_command(self, command, cwd=None, halt_on_failure=False, env=None, silent=False, log_level=INFO, tmpfile_base_path='tmpfile', return_type='output', save_tmpfiles=False, throw_exception=False, fatal_exit_code=2, ignore_errors=False, success_codes=None): """Similar to run_command, but where run_command is an os.system(command) analog, get_output_from_command is a `command` analog. Less error checking by design, though if we figure out how to do it without borking the output, great. TODO: binary mode? silent is kinda like that. TODO: since p.wait() can take a long time, optionally log something every N seconds? TODO: optionally only keep the first or last (N) line(s) of output? TODO: optionally only return the tmp_stdout_filename? ignore_errors=True is for the case where a command might produce standard error output, but you don't particularly care; setting to True will cause standard error to be logged at DEBUG rather than ERROR Args: command (str | list): command or list of commands to execute and log. cwd (str, optional): directory path from where to execute the command. Defaults to `None`. halt_on_failure (bool, optional): whether or not to redefine the log level as `FATAL` on error. Defaults to False. env (dict, optional): key-value of environment values to use to run the command. Defaults to None. silent (bool, optional): whether or not to output the stdout of executing the command. Defaults to False. log_level (str, optional): log level name to use on normal execution. Defaults to `INFO`. tmpfile_base_path (str, optional): base path of the file to which the output will be writen to. Defaults to 'tmpfile'. return_type (str, optional): if equal to 'output' then the complete output of the executed command is returned, otherwise the written filenames are returned. Defaults to 'output'. save_tmpfiles (bool, optional): whether or not to save the temporary files created from the command output. Defaults to False. throw_exception (bool, optional): whether or not to raise an exception if the return value of the command is not zero. Defaults to False. fatal_exit_code (int, optional): call self.fatal if the return value of the command match this value. ignore_errors (bool, optional): whether or not to change the log level to `ERROR` for the output of stderr. Defaults to False. success_codes (int, optional): numeric value to compare against the command return value. Returns: None: if the cwd is not a directory. None: on IOError. tuple: stdout and stderr filenames. str: stdout output. """ if cwd: if not os.path.isdir(cwd): level = ERROR if halt_on_failure: level = FATAL self.log("Can't run command %s in non-existent directory %s!" % (command, cwd), level=level) return None self.info("Getting output from command: %s in %s" % (command, cwd)) else: self.info("Getting output from command: %s" % command) if isinstance(command, list): self.info("Copy/paste: %s" % subprocess.list2cmdline(command)) # This could potentially return something? tmp_stdout = None tmp_stderr = None tmp_stdout_filename = '%s_stdout' % tmpfile_base_path tmp_stderr_filename = '%s_stderr' % tmpfile_base_path if success_codes is None: success_codes = [0] # TODO probably some more elegant solution than 2 similar passes try: tmp_stdout = open(tmp_stdout_filename, 'w') except IOError: level = ERROR if halt_on_failure: level = FATAL self.log("Can't open %s for writing!" % tmp_stdout_filename + self.exception(), level=level) return None try: tmp_stderr = open(tmp_stderr_filename, 'w') except IOError: level = ERROR if halt_on_failure: level = FATAL self.log("Can't open %s for writing!" % tmp_stderr_filename + self.exception(), level=level) return None shell = True if isinstance(command, list): shell = False p = subprocess.Popen(command, shell=shell, stdout=tmp_stdout, cwd=cwd, stderr=tmp_stderr, env=env, bufsize=0) # XXX: changed from self.debug to self.log due to this error: # TypeError: debug() takes exactly 1 argument (2 given) self.log("Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), level=DEBUG) p.wait() tmp_stdout.close() tmp_stderr.close() return_level = DEBUG output = None if return_type == 'output' or not silent: if os.path.exists(tmp_stdout_filename) and os.path.getsize(tmp_stdout_filename): output = self.read_from_file(tmp_stdout_filename, verbose=False) if not silent: self.log("Output received:", level=log_level) output_lines = output.rstrip().splitlines() for line in output_lines: if not line or line.isspace(): continue line = line.decode("utf-8") self.log(' %s' % line, level=log_level) output = '\n'.join(output_lines) if os.path.exists(tmp_stderr_filename) and os.path.getsize(tmp_stderr_filename): if not ignore_errors: return_level = ERROR self.log("Errors received:", level=return_level) errors = self.read_from_file(tmp_stderr_filename, verbose=False) for line in errors.rstrip().splitlines(): if not line or line.isspace(): continue line = line.decode("utf-8") self.log(' %s' % line, level=return_level) elif p.returncode not in success_codes and not ignore_errors: return_level = ERROR # Clean up. if not save_tmpfiles: self.rmtree(tmp_stderr_filename, log_level=DEBUG) self.rmtree(tmp_stdout_filename, log_level=DEBUG) if p.returncode and throw_exception: raise subprocess.CalledProcessError(p.returncode, command) self.log("Return code: %d" % p.returncode, level=return_level) if halt_on_failure and return_level == ERROR: self.return_code = fatal_exit_code self.fatal("Halting on failure while running %s" % command, exit_code=fatal_exit_code) # Hm, options on how to return this? I bet often we'll want # output_lines[0] with no newline. if return_type != 'output': return (tmp_stdout_filename, tmp_stderr_filename) else: return output def _touch_file(self, file_name, times=None, error_level=FATAL): """touch a file. Args: file_name (str): name of the file to touch. times (tuple, optional): 2-tuple as specified by `os.utime`_ Defaults to None. error_level (str, optional): log level name in case of error. Defaults to `FATAL`. .. _`os.utime`: https://docs.python.org/3.4/library/os.html?highlight=os.utime#os.utime """ self.info("Touching: %s" % file_name) try: os.utime(file_name, times) except OSError: try: open(file_name, 'w').close() except IOError as e: msg = "I/O error(%s): %s" % (e.errno, e.strerror) self.log(msg, error_level=error_level) os.utime(file_name, times) def unpack(self, filename, extract_to, extract_dirs=None, error_level=ERROR, fatal_exit_code=2, verbose=False): """The method allows to extract a file regardless of its extension. Args: filename (str): filename of the compressed file. extract_to (str): where to extract the compressed file. extract_dirs (list, optional): directories inside the archive file to extract. Defaults to `None`. fatal_exit_code (int, optional): call `self.fatal` if the return value of the command is not in `success_codes`. Defaults to 2. verbose (bool, optional): whether or not extracted content should be displayed. Defaults to False. Raises: IOError: on `filename` file not found. """ if not os.path.isfile(filename): raise IOError('Could not find file to extract: %s' % filename) if zipfile.is_zipfile(filename): try: self.info('Using ZipFile to extract {} to {}'.format(filename, extract_to)) with zipfile.ZipFile(filename) as bundle: for entry in self._filter_entries(bundle.namelist(), extract_dirs): if verbose: self.info(' %s' % entry) bundle.extract(entry, path=extract_to) # ZipFile doesn't preserve permissions during extraction: # http://bugs.python.org/issue15795 fname = os.path.realpath(os.path.join(extract_to, entry)) mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF # Only set permissions if attributes are available. Otherwise all # permissions will be removed eg. on Windows. if mode: os.chmod(fname, mode) except zipfile.BadZipfile as e: self.log('%s (%s)' % (e.message, filename), level=error_level, exit_code=fatal_exit_code) # Bug 1211882 - is_tarfile cannot be trusted for dmg files elif tarfile.is_tarfile(filename) and not filename.lower().endswith('.dmg'): try: self.info('Using TarFile to extract {} to {}'.format(filename, extract_to)) with tarfile.open(filename) as bundle: for entry in self._filter_entries(bundle.getnames(), extract_dirs): if verbose: self.info(' %s' % entry) bundle.extract(entry, path=extract_to) except tarfile.TarError as e: self.log('%s (%s)' % (e.message, filename), level=error_level, exit_code=fatal_exit_code) else: self.log('No extraction method found for: %s' % filename, level=error_level, exit_code=fatal_exit_code) def is_taskcluster(self): """Returns boolean indicating if we're running in TaskCluster.""" # This may need expanding in the future to work on return 'TASKCLUSTER_WORKER_TYPE' in os.environ def PreScriptRun(func): """Decorator for methods that will be called before script execution. Each method on a BaseScript having this decorator will be called at the beginning of BaseScript.run(). The return value is ignored. Exceptions will abort execution. """ func._pre_run_listener = True return func def PostScriptRun(func): """Decorator for methods that will be called after script execution. This is similar to PreScriptRun except it is called at the end of execution. The method will always be fired, even if execution fails. """ func._post_run_listener = True return func def PreScriptAction(action=None): """Decorator for methods that will be called at the beginning of each action. Each method on a BaseScript having this decorator will be called during BaseScript.run() before an individual action is executed. The method will receive the action's name as an argument. If no values are passed to the decorator, it will be applied to every action. If a string is passed, the decorated function will only be called for the action of that name. The return value of the method is ignored. Exceptions will abort execution. """ def _wrapped(func): func._pre_action_listener = action return func def _wrapped_none(func): func._pre_action_listener = None return func if type(action) == type(_wrapped): return _wrapped_none(action) return _wrapped def PostScriptAction(action=None): """Decorator for methods that will be called at the end of each action. This behaves similarly to PreScriptAction. It varies in that it is called after execution of the action. The decorated method will receive the action name as a positional argument. It will then receive the following named arguments: success - Bool indicating whether the action finished successfully. The decorated method will always be called, even if the action threw an exception. The return value is ignored. """ def _wrapped(func): func._post_action_listener = action return func def _wrapped_none(func): func._post_action_listener = None return func if type(action) == type(_wrapped): return _wrapped_none(action) return _wrapped # BaseScript {{{1 class BaseScript(ScriptMixin, LogMixin, object): def __init__(self, config_options=None, ConfigClass=BaseConfig, default_log_level="info", **kwargs): self._return_code = 0 super(BaseScript, self).__init__() # Collect decorated methods. We simply iterate over the attributes of # the current class instance and look for signatures deposited by # the decorators. self._listeners = dict( pre_run=[], pre_action=[], post_action=[], post_run=[], ) for k in dir(self): item = getattr(self, k) # We only decorate methods, so ignore other types. if not inspect.ismethod(item): continue if hasattr(item, '_pre_run_listener'): self._listeners['pre_run'].append(k) if hasattr(item, '_pre_action_listener'): self._listeners['pre_action'].append(( k, item._pre_action_listener)) if hasattr(item, '_post_action_listener'): self._listeners['post_action'].append(( k, item._post_action_listener)) if hasattr(item, '_post_run_listener'): self._listeners['post_run'].append(k) self.log_obj = None self.abs_dirs = None if config_options is None: config_options = [] self.summary_list = [] self.failures = [] rw_config = ConfigClass(config_options=config_options, **kwargs) self.config = rw_config.get_read_only_config() self.actions = tuple(rw_config.actions) self.all_actions = tuple(rw_config.all_actions) self.env = None self.new_log_obj(default_log_level=default_log_level) self.script_obj = self # Indicate we're a source checkout if VCS directory is present at the # appropriate place. This code will break if this file is ever moved # to another directory. self.topsrcdir = None srcreldir = 'testing/mozharness/mozharness/base' here = os.path.normpath(os.path.dirname(__file__)) if here.replace('\\', '/').endswith(srcreldir): topsrcdir = os.path.normpath(os.path.join(here, '..', '..', '..', '..')) hg_dir = os.path.join(topsrcdir, '.hg') git_dir = os.path.join(topsrcdir, '.git') if os.path.isdir(hg_dir) or os.path.isdir(git_dir): self.topsrcdir = topsrcdir # Set self.config to read-only. # # We can create intermediate config info programmatically from # this in a repeatable way, with logs; this is how we straddle the # ideal-but-not-user-friendly static config and the # easy-to-write-hard-to-debug writable config. # # To allow for other, script-specific configurations # (e.g., props json parsing), before locking, # call self._pre_config_lock(). If needed, this method can # alter self.config. self._pre_config_lock(rw_config) self._config_lock() self.info("Run as %s" % rw_config.command_line) if self.config.get("dump_config_hierarchy"): # we only wish to dump and display what self.config is made up of, # against the current script + args, without actually running any # actions self._dump_config_hierarchy(rw_config.all_cfg_files_and_dicts) if self.config.get("dump_config"): self.dump_config(exit_on_finish=True) def _dump_config_hierarchy(self, cfg_files): """ interpret each config file used. This will show which keys/values are being added or overwritten by other config files depending on their hierarchy (when they were added). """ # go through each config_file. We will start with the lowest and # print its keys/values that are being used in self.config. If any # keys/values are present in a config file with a higher precedence, # ignore those. dirs = self.query_abs_dirs() cfg_files_dump_config = {} # we will dump this to file # keep track of keys that did not come from a config file keys_not_from_file = set(self.config.keys()) if not cfg_files: cfg_files = [] self.info("Total config files: %d" % (len(cfg_files))) if len(cfg_files): self.info("cfg files used from lowest precedence to highest:") for i, (target_file, target_dict) in enumerate(cfg_files): unique_keys = set(target_dict.keys()) unique_dict = {} # iterate through the target_dicts remaining 'higher' cfg_files remaining_cfgs = cfg_files[slice(i + 1, len(cfg_files))] # where higher == more precedent for ii, (higher_file, higher_dict) in enumerate(remaining_cfgs): # now only keep keys/values that are not overwritten by a # higher config unique_keys = unique_keys.difference(set(higher_dict.keys())) # unique_dict we know now has only keys/values that are unique to # this config file. unique_dict = dict( (key, target_dict.get(key)) for key in unique_keys ) cfg_files_dump_config[target_file] = unique_dict self.action_message("Config File %d: %s" % (i + 1, target_file)) self.info(pprint.pformat(unique_dict)) # let's also find out which keys/values from self.config are not # from each target config file dict keys_not_from_file = keys_not_from_file.difference( set(target_dict.keys()) ) not_from_file_dict = dict( (key, self.config.get(key)) for key in keys_not_from_file ) cfg_files_dump_config["not_from_cfg_file"] = not_from_file_dict self.action_message("Not from any config file (default_config, " "cmd line options, etc)") self.info(pprint.pformat(not_from_file_dict)) # finally, let's dump this output as JSON and exit early self.dump_config( os.path.join(dirs['abs_log_dir'], "localconfigfiles.json"), cfg_files_dump_config, console_output=False, exit_on_finish=True ) def _pre_config_lock(self, rw_config): """This empty method can allow for config checking and manipulation before the config lock, when overridden in scripts. """ pass def _config_lock(self): """After this point, the config is locked and should not be manipulated (based on mozharness.base.config.ReadOnlyDict) """ self.config.lock() def _possibly_run_method(self, method_name, error_if_missing=False): """This is here for run(). """ if hasattr(self, method_name) and callable(getattr(self, method_name)): return getattr(self, method_name)() elif error_if_missing: self.error("No such method %s!" % method_name) @PostScriptRun def copy_logs_to_upload_dir(self): """Copies logs to the upload directory""" self.info("Copying logs to upload dir...") log_files = ['localconfig.json'] for log_name in self.log_obj.log_files.keys(): log_files.append(self.log_obj.log_files[log_name]) dirs = self.query_abs_dirs() for log_file in log_files: self.copy_to_upload_dir(os.path.join(dirs['abs_log_dir'], log_file), dest=os.path.join('logs', log_file), short_desc='%s log' % log_name, long_desc='%s log' % log_name, max_backups=self.config.get("log_max_rotate", 0)) def run_action(self, action): if action not in self.actions: self.action_message("Skipping %s step." % action) return method_name = action.replace("-", "_") self.action_message("Running %s step." % action) # An exception during a pre action listener should abort execution. for fn, target in self._listeners['pre_action']: if target is not None and target != action: continue try: self.info("Running pre-action listener: %s" % fn) method = getattr(self, fn) method(action) except Exception: self.error("Exception during pre-action for %s: %s" % ( action, traceback.format_exc())) for fn, target in self._listeners['post_action']: if target is not None and target != action: continue try: self.info("Running post-action listener: %s" % fn) method = getattr(self, fn) method(action, success=False) except Exception: self.error("An additional exception occurred during " "post-action for %s: %s" % (action, traceback.format_exc())) self.fatal("Aborting due to exception in pre-action listener.") # We always run post action listeners, even if the main routine failed. success = False try: self.info("Running main action method: %s" % method_name) self._possibly_run_method("preflight_%s" % method_name) self._possibly_run_method(method_name, error_if_missing=True) self._possibly_run_method("postflight_%s" % method_name) success = True finally: post_success = True for fn, target in self._listeners['post_action']: if target is not None and target != action: continue try: self.info("Running post-action listener: %s" % fn) method = getattr(self, fn) method(action, success=success and self.return_code == 0) except Exception: post_success = False self.error("Exception during post-action for %s: %s" % ( action, traceback.format_exc())) step_result = 'success' if success else 'failed' self.action_message("Finished %s step (%s)" % (action, step_result)) if not post_success: self.fatal("Aborting due to failure in post-action listener.") def run(self): """Default run method. This is the "do everything" method, based on actions and all_actions. First run self.dump_config() if it exists. Second, go through the list of all_actions. If they're in the list of self.actions, try to run self.preflight_ACTION(), self.ACTION(), and self.postflight_ACTION(). Preflight is sanity checking before doing anything time consuming or destructive. Postflight is quick testing for success after an action. """ for fn in self._listeners['pre_run']: try: self.info("Running pre-run listener: %s" % fn) method = getattr(self, fn) method() except Exception: self.error("Exception during pre-run listener: %s" % traceback.format_exc()) for fn in self._listeners['post_run']: try: method = getattr(self, fn) method() except Exception: self.error("An additional exception occurred during a " "post-run listener: %s" % traceback.format_exc()) self.fatal("Aborting due to failure in pre-run listener.") self.dump_config() try: for action in self.all_actions: self.run_action(action) except Exception: self.fatal("Uncaught exception: %s" % traceback.format_exc()) finally: post_success = True for fn in self._listeners['post_run']: try: self.info("Running post-run listener: %s" % fn) method = getattr(self, fn) method() except Exception: post_success = False self.error("Exception during post-run listener: %s" % traceback.format_exc()) if not post_success: self.fatal("Aborting due to failure in post-run listener.") if self.config.get("copy_logs_post_run", True): self.copy_logs_to_upload_dir() return self.return_code def run_and_exit(self): """Runs the script and exits the current interpreter.""" rc = self.run() if rc != 0: self.warning("returning nonzero exit status %d" % rc) sys.exit(rc) def clobber(self): """ Delete the working directory """ dirs = self.query_abs_dirs() self.rmtree(dirs['abs_work_dir'], error_level=FATAL) def query_abs_dirs(self): """We want to be able to determine where all the important things are. Absolute paths lend themselves well to this, though I wouldn't be surprised if this causes some issues somewhere. This should be overridden in any script that has additional dirs to query. The query_* methods tend to set self.VAR variables as their runtime cache. """ if self.abs_dirs: return self.abs_dirs c = self.config dirs = {} dirs['base_work_dir'] = c['base_work_dir'] dirs['abs_work_dir'] = os.path.join(c['base_work_dir'], c['work_dir']) dirs['abs_upload_dir'] = os.path.join(dirs['abs_work_dir'], 'upload') dirs['abs_log_dir'] = os.path.join(c['base_work_dir'], c.get('log_dir', 'logs')) self.abs_dirs = dirs return self.abs_dirs def dump_config(self, file_path=None, config=None, console_output=True, exit_on_finish=False): """Dump self.config to localconfig.json """ config = config or self.config dirs = self.query_abs_dirs() if not file_path: file_path = os.path.join(dirs['abs_log_dir'], "localconfig.json") self.info("Dumping config to %s." % file_path) self.mkdir_p(os.path.dirname(file_path)) json_config = json.dumps(config, sort_keys=True, indent=4) fh = codecs.open(file_path, encoding='utf-8', mode='w+') fh.write(json_config) fh.close() if console_output: self.info(pprint.pformat(config)) if exit_on_finish: sys.exit() # logging {{{2 def new_log_obj(self, default_log_level="info"): c = self.config log_dir = os.path.join(c['base_work_dir'], c.get('log_dir', 'logs')) log_config = { "logger_name": 'Simple', "log_name": 'log', "log_dir": log_dir, "log_level": default_log_level, "log_format": '%(asctime)s %(levelname)8s - %(message)s', "log_to_console": True, "append_to_log": False, } log_type = self.config.get("log_type", "multi") for key in log_config.keys(): value = self.config.get(key, None) if value is not None: log_config[key] = value if log_type == "multi": self.log_obj = MultiFileLogger(**log_config) else: self.log_obj = SimpleFileLogger(**log_config) def action_message(self, message): self.info("[mozharness: %sZ] %s" % ( datetime.datetime.utcnow().isoformat(' '), message)) def summary(self): """Print out all the summary lines added via add_summary() throughout the script. I'd like to revisit how to do this in a prettier fashion. """ self.action_message("%s summary:" % self.__class__.__name__) if self.summary_list: for item in self.summary_list: try: self.log(item['message'], level=item['level']) except ValueError: """log is closed; print as a default. Ran into this when calling from __del__()""" print "### Log is closed! (%s)" % item['message'] def add_summary(self, message, level=INFO): self.summary_list.append({'message': message, 'level': level}) # TODO write to a summary-only log? # Summaries need a lot more love. self.log(message, level=level) def summarize_success_count(self, success_count, total_count, message="%d of %d successful.", level=None): if level is None: level = INFO if success_count < total_count: level = ERROR self.add_summary(message % (success_count, total_count), level=level) def copy_to_upload_dir(self, target, dest=None, short_desc="unknown", long_desc="unknown", log_level=DEBUG, error_level=ERROR, max_backups=None, compress=False, upload_dir=None): """Copy target file to upload_dir/dest. Potentially update a manifest in the future if we go that route. Currently only copies a single file; would be nice to allow for recursive copying; that would probably done by creating a helper _copy_file_to_upload_dir(). short_desc and long_desc are placeholders for if/when we add upload_dir manifests. """ dest_filename_given = dest is not None if upload_dir is None: upload_dir = self.query_abs_dirs()['abs_upload_dir'] if dest is None: dest = os.path.basename(target) if dest.endswith('/'): dest_file = os.path.basename(target) dest_dir = os.path.join(upload_dir, dest) dest_filename_given = False else: dest_file = os.path.basename(dest) dest_dir = os.path.join(upload_dir, os.path.dirname(dest)) if compress and not dest_filename_given: dest_file += ".gz" dest = os.path.join(dest_dir, dest_file) if not os.path.exists(target): self.log("%s doesn't exist!" % target, level=error_level) return None self.mkdir_p(dest_dir) if os.path.exists(dest): if os.path.isdir(dest): self.log("%s exists and is a directory!" % dest, level=error_level) return -1 if max_backups: # Probably a better way to do this oldest_backup = 0 backup_regex = re.compile("^%s\.(\d+)$" % dest_file) for filename in os.listdir(dest_dir): r = backup_regex.match(filename) if r and int(r.groups()[0]) > oldest_backup: oldest_backup = int(r.groups()[0]) for backup_num in range(oldest_backup, 0, -1): # TODO more error checking? if backup_num >= max_backups: self.rmtree(os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num)), log_level=log_level) else: self.move(os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num)), os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num + 1)), log_level=log_level) if self.move(dest, "%s.1" % dest, log_level=log_level): self.log("Unable to move %s!" % dest, level=error_level) return -1 else: if self.rmtree(dest, log_level=log_level): self.log("Unable to remove %s!" % dest, level=error_level) return -1 self.copyfile(target, dest, log_level=log_level, compress=compress) if os.path.exists(dest): return dest else: self.log("%s doesn't exist after copy!" % dest, level=error_level) return None def get_hash_for_file(self, file_path, hash_type="sha512"): bs = 65536 hasher = hashlib.new(hash_type) with open(file_path, 'rb') as fh: buf = fh.read(bs) while len(buf) > 0: hasher.update(buf) buf = fh.read(bs) return hasher.hexdigest() @property def return_code(self): return self._return_code @return_code.setter def return_code(self, code): old_return_code, self._return_code = self._return_code, code if old_return_code != code: self.warning("setting return code to %d" % code) # __main__ {{{1 if __name__ == '__main__': """ Useless comparison, due to the `pass` keyword on its body""" pass diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -1,1853 +1,1843 @@ #!/usr/bin/env python # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """ buildbase.py. provides a base class for fx desktop builds author: Jordan Lund """ import json import os import pprint import time import uuid import copy import glob # import the power of mozharness ;) import sys from datetime import datetime import re from mozharness.base.config import BaseConfig, parse_config_file, DEFAULT_CONFIG_PATH from mozharness.base.log import ERROR, OutputParser, FATAL from mozharness.base.script import PostScriptRun from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.automation import ( AutomationMixin, EXIT_STATUS_DICT, TBPL_STATUS_DICT, TBPL_EXCEPTION, TBPL_FAILURE, TBPL_RETRY, TBPL_WARNING, TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE, ) from mozharness.mozilla.secrets import SecretsMixin from mozharness.mozilla.testing.errors import TinderBoxPrintRe from mozharness.mozilla.testing.unittest import tbox_print_summary from mozharness.mozilla.updates.balrog import BalrogMixin from mozharness.base.python import ( PerfherderResourceOptionsMixin, VirtualenvMixin, ) AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values() AUTOMATION_EXIT_CODES.sort() MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \ Please add this to your config." ERROR_MSGS = { 'undetermined_repo_path': 'The repo could not be determined. \ Please make sure that "repo" is in your config.', 'comments_undetermined': '"comments" could not be determined. This may be \ because it was a forced build.', 'tooltool_manifest_undetermined': '"tooltool_manifest_src" not set, \ Skipping run_tooltool...', } ### Output Parsers TBPL_UPLOAD_ERRORS = [ { 'regex': re.compile("Connection timed out"), 'level': TBPL_RETRY, }, { 'regex': re.compile("Connection reset by peer"), 'level': TBPL_RETRY, }, { 'regex': re.compile("Connection refused"), 'level': TBPL_RETRY, } ] class MakeUploadOutputParser(OutputParser): tbpl_error_list = TBPL_UPLOAD_ERRORS # let's create a switch case using name-spaces/dict # rather than a long if/else with duplicate code property_conditions = [ # key: property name, value: condition ('symbolsUrl', "m.endswith('crashreporter-symbols.zip') or " "m.endswith('crashreporter-symbols-full.zip')"), ('testsUrl', "m.endswith(('tests.tar.bz2', 'tests.zip', 'tests.tar.gz'))"), ('robocopApkUrl', "m.endswith('apk') and 'robocop' in m"), ('jsshellUrl', "'jsshell-' in m and m.endswith('.zip')"), ('partialMarUrl', "m.endswith('.mar') and '.partial.' in m"), ('completeMarUrl', "m.endswith('.mar')"), ('codeCoverageUrl', "m.endswith('code-coverage-gcno.zip')"), ] def __init__(self, use_package_as_marfile=False, package_filename=None, **kwargs): super(MakeUploadOutputParser, self).__init__(**kwargs) self.matches = {} self.tbpl_status = TBPL_SUCCESS self.use_package_as_marfile = use_package_as_marfile self.package_filename = package_filename def parse_single_line(self, line): prop_assigned = False pat = r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''' m = re.compile(pat).match(line) if m: m = m.group(1) for prop, condition in self.property_conditions: if eval(condition): self.matches[prop] = m prop_assigned = True break if not prop_assigned: # if we found a match but haven't identified the prop then this # is the packageURL. Alternatively, if we already know the # package filename, then use that explicitly so we don't pick up # just any random file and assume it's the package. if not self.package_filename or m.endswith(self.package_filename): self.matches['packageUrl'] = m # For android builds, the package is also used as the mar file. # Grab the first one, since that is the one in the # nightly/YYYY/MM directory if self.use_package_as_marfile: if 'tinderbox-builds' in m or 'nightly/latest-' in m: self.info("Skipping wrong packageUrl: %s" % m) else: if 'completeMarUrl' in self.matches: self.fatal("Found multiple package URLs. Please update buildbase.py") self.info("Using package as mar file: %s" % m) self.matches['completeMarUrl'] = m u, self.package_filename = os.path.split(m) if self.use_package_as_marfile and self.package_filename: # The checksum file is also dumped during 'make upload'. Look # through here to get the hash and filesize of the android package # for balrog submission. pat = r'''^([^ ]*) sha512 ([0-9]*) %s$''' % self.package_filename m = re.compile(pat).match(line) if m: self.matches['completeMarHash'] = m.group(1) self.matches['completeMarSize'] = m.group(2) self.info("Using package as mar file and found package hash=%s size=%s" % (m.group(1), m.group(2))) # now let's check for retry errors which will give log levels: # tbpl status as RETRY and mozharness status as WARNING for error_check in self.tbpl_error_list: if error_check['regex'].search(line): self.num_warnings += 1 self.warning(line) self.tbpl_status = self.worst_level( error_check['level'], self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE ) break else: self.info(line) class CheckTestCompleteParser(OutputParser): tbpl_error_list = TBPL_UPLOAD_ERRORS def __init__(self, **kwargs): self.matches = {} super(CheckTestCompleteParser, self).__init__(**kwargs) self.pass_count = 0 self.fail_count = 0 self.leaked = False self.harness_err_re = TinderBoxPrintRe['harness_error']['full_regex'] self.tbpl_status = TBPL_SUCCESS def parse_single_line(self, line): # Counts and flags. # Regular expression for crash and leak detections. if "TEST-PASS" in line: self.pass_count += 1 return self.info(line) if "TEST-UNEXPECTED-" in line: # Set the error flags. # Or set the failure count. m = self.harness_err_re.match(line) if m: r = m.group(1) if r == "missing output line for total leaks!": self.leaked = None else: self.leaked = True self.fail_count += 1 return self.warning(line) self.info(line) # else def evaluate_parser(self, return_code, success_codes=None): success_codes = success_codes or [0] if self.num_errors: # ran into a script error self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) if self.fail_count > 0: self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) # Account for the possibility that no test summary was output. if self.pass_count == 0 and self.fail_count == 0: self.error('No tests run or test summary not found') self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) if return_code not in success_codes: self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) # Print the summary. summary = tbox_print_summary(self.pass_count, self.fail_count, self.leaked) self.info("TinderboxPrint: check
%s\n" % summary) return self.tbpl_status class MozconfigPathError(Exception): """ There was an error getting a mozconfig path from a mozharness config. """ def get_mozconfig_path(script, config, dirs): """ Get the path to the mozconfig file to use from a mozharness config. :param script: The object to interact with the filesystem through. :type script: ScriptMixin: :param config: The mozharness config to inspect. :type config: dict :param dirs: The directories specified for this build. :type dirs: dict """ COMPOSITE_KEYS = {'mozconfig_variant', 'app_name', 'mozconfig_platform'} have_composite_mozconfig = COMPOSITE_KEYS <= set(config.keys()) have_partial_composite_mozconfig = len(COMPOSITE_KEYS & set(config.keys())) > 0 have_src_mozconfig = 'src_mozconfig' in config have_src_mozconfig_manifest = 'src_mozconfig_manifest' in config # first determine the mozconfig path if have_partial_composite_mozconfig and not have_composite_mozconfig: raise MozconfigPathError( "All or none of 'app_name', 'mozconfig_platform' and `mozconfig_variant' must be " "in the config in order to determine the mozconfig.") elif have_composite_mozconfig and have_src_mozconfig: raise MozconfigPathError( "'src_mozconfig' or 'mozconfig_variant' must be " "in the config but not both in order to determine the mozconfig.") elif have_composite_mozconfig and have_src_mozconfig_manifest: raise MozconfigPathError( "'src_mozconfig_manifest' or 'mozconfig_variant' must be " "in the config but not both in order to determine the mozconfig.") elif have_src_mozconfig and have_src_mozconfig_manifest: raise MozconfigPathError( "'src_mozconfig' or 'src_mozconfig_manifest' must be " "in the config but not both in order to determine the mozconfig.") elif have_composite_mozconfig: src_mozconfig = '%(app_name)s/config/mozconfigs/%(platform)s/%(variant)s' % { 'app_name': config['app_name'], 'platform': config['mozconfig_platform'], 'variant': config['mozconfig_variant'], } abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], src_mozconfig) elif have_src_mozconfig: abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], config.get('src_mozconfig')) elif have_src_mozconfig_manifest: manifest = os.path.join(dirs['abs_work_dir'], config['src_mozconfig_manifest']) if not os.path.exists(manifest): raise MozconfigPathError( 'src_mozconfig_manifest: "%s" not found. Does it exist?' % (manifest,)) else: with script.opened(manifest, error_level=ERROR) as (fh, err): if err: raise MozconfigPathError("%s exists but coud not read properties" % manifest) abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], json.load(fh)['gecko_path']) else: raise MozconfigPathError( "Must provide 'app_name', 'mozconfig_platform' and 'mozconfig_variant'; " "or one of 'src_mozconfig' or 'src_mozconfig_manifest' in the config " "in order to determine the mozconfig.") return abs_mozconfig_path class BuildingConfig(BaseConfig): # TODO add nosetests for this class def get_cfgs_from_files(self, all_config_files, options): """ Determine the configuration from the normal options and from `--branch`, `--build-pool`, and `--custom-build-variant-cfg`. If the files for any of the latter options are also given with `--config-file` or `--opt-config-file`, they are only parsed once. The build pool has highest precedence, followed by branch, build variant, and any normally-specified configuration files. """ # override from BaseConfig # this is what we will return. It will represent each config # file name and its associated dict # eg ('builds/branch_specifics.py', {'foo': 'bar'}) all_config_dicts = [] # important config files variant_cfg_file = branch_cfg_file = pool_cfg_file = '' # we want to make the order in which the options were given # not matter. ie: you can supply --branch before --build-pool # or vice versa and the hierarchy will not be different #### The order from highest precedence to lowest is: ## There can only be one of these... # 1) build_pool: this can be either staging, pre-prod, and prod cfgs # 2) branch: eg: mozilla-central, cedar, cypress, etc # 3) build_variant: these could be known like asan and debug # or a custom config ## ## There can be many of these # 4) all other configs: these are any configs that are passed with # --cfg and --opt-cfg. There order is kept in # which they were passed on the cmd line. This # behaviour is maintains what happens by default # in mozharness ## #### # so, let's first assign the configs that hold a known position of # importance (1 through 3) for i, cf in enumerate(all_config_files): if options.build_pool: if cf == BuildOptionParser.build_pool_cfg_file: pool_cfg_file = all_config_files[i] if cf == BuildOptionParser.branch_cfg_file: branch_cfg_file = all_config_files[i] if cf == options.build_variant: variant_cfg_file = all_config_files[i] # now remove these from the list if there was any. # we couldn't pop() these in the above loop as mutating a list while # iterating through it causes spurious results :) for cf in [pool_cfg_file, branch_cfg_file, variant_cfg_file]: if cf: all_config_files.remove(cf) # now let's update config with the remaining config files. # this functionality is the same as the base class all_config_dicts.extend( super(BuildingConfig, self).get_cfgs_from_files(all_config_files, options) ) # stack variant, branch, and pool cfg files on top of that, # if they are present, in that order if variant_cfg_file: # take the whole config all_config_dicts.append( (variant_cfg_file, parse_config_file(variant_cfg_file)) ) config_paths = options.config_paths or ['.'] if branch_cfg_file: # take only the specific branch, if present branch_configs = parse_config_file(branch_cfg_file, search_path=config_paths + [DEFAULT_CONFIG_PATH]) if branch_configs.get(options.branch or ""): all_config_dicts.append( (branch_cfg_file, branch_configs[options.branch]) ) if pool_cfg_file: # take only the specific pool. If we are here, the pool # must be present build_pool_configs = parse_config_file(pool_cfg_file, search_path=config_paths + [DEFAULT_CONFIG_PATH]) all_config_dicts.append( (pool_cfg_file, build_pool_configs[options.build_pool]) ) return all_config_dicts # noinspection PyUnusedLocal class BuildOptionParser(object): # TODO add nosetests for this class platform = None bits = None # add to this list and you can automagically do things like # --custom-build-variant-cfg asan # and the script will pull up the appropriate path for the config # against the current platform and bits. # *It will warn and fail if there is not a config for the current # platform/bits build_variants = { 'add-on-devel': 'builds/releng_sub_%s_configs/%s_add-on-devel.py', 'asan': 'builds/releng_sub_%s_configs/%s_asan.py', 'asan-tc': 'builds/releng_sub_%s_configs/%s_asan_tc.py', 'asan-reporter-tc': 'builds/releng_sub_%s_configs/%s_asan_reporter_tc.py', 'fuzzing-asan-tc': 'builds/releng_sub_%s_configs/%s_fuzzing_asan_tc.py', 'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py', 'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py', 'cross-debug-searchfox': 'builds/releng_sub_%s_configs/%s_cross_debug_searchfox.py', 'cross-debug-artifact': 'builds/releng_sub_%s_configs/%s_cross_debug_artifact.py', 'cross-noopt-debug': 'builds/releng_sub_%s_configs/%s_cross_noopt_debug.py', 'cross-fuzzing-asan': 'builds/releng_sub_%s_configs/%s_cross_fuzzing_asan.py', 'cross-artifact': 'builds/releng_sub_%s_configs/%s_cross_artifact.py', 'debug': 'builds/releng_sub_%s_configs/%s_debug.py', 'fuzzing-debug': 'builds/releng_sub_%s_configs/%s_fuzzing_debug.py', 'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py', 'asan-tc-and-debug': 'builds/releng_sub_%s_configs/%s_asan_tc_and_debug.py', 'lto-tc': 'builds/releng_sub_%s_configs/%s_lto_tc.py', 'lto-tc-and-debug': 'builds/releng_sub_%s_configs/%s_lto_tc_and_debug.py', 'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py', 'code-coverage-debug': 'builds/releng_sub_%s_configs/%s_code_coverage_debug.py', 'code-coverage-opt': 'builds/releng_sub_%s_configs/%s_code_coverage_opt.py', 'source': 'builds/releng_sub_%s_configs/%s_source.py', 'noopt-debug': 'builds/releng_sub_%s_configs/%s_noopt_debug.py', 'api-16-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_16_gradle_dependencies.py', 'api-16': 'builds/releng_sub_%s_configs/%s_api_16.py', 'api-16-artifact': 'builds/releng_sub_%s_configs/%s_api_16_artifact.py', 'api-16-debug': 'builds/releng_sub_%s_configs/%s_api_16_debug.py', 'api-16-debug-ccov': 'builds/releng_sub_%s_configs/%s_api_16_debug_ccov.py', 'api-16-debug-artifact': 'builds/releng_sub_%s_configs/%s_api_16_debug_artifact.py', 'api-16-gradle': 'builds/releng_sub_%s_configs/%s_api_16_gradle.py', 'api-16-gradle-artifact': 'builds/releng_sub_%s_configs/%s_api_16_gradle_artifact.py', 'api-16-without-google-play-services': 'builds/releng_sub_%s_configs/%s_api_16_without_google_play_services.py', 'rusttests': 'builds/releng_sub_%s_configs/%s_rusttests.py', 'rusttests-debug': 'builds/releng_sub_%s_configs/%s_rusttests_debug.py', 'x86': 'builds/releng_sub_%s_configs/%s_x86.py', 'x86-artifact': 'builds/releng_sub_%s_configs/%s_x86_artifact.py', 'x86-fuzzing-debug': 'builds/releng_sub_%s_configs/%s_x86_fuzzing_debug.py', 'api-16-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_16_partner_sample1.py', 'aarch64': 'builds/releng_sub_%s_configs/%s_aarch64.py', 'android-test': 'builds/releng_sub_%s_configs/%s_test.py', 'android-test-ccov': 'builds/releng_sub_%s_configs/%s_test_ccov.py', 'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py', 'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py', 'android-findbugs': 'builds/releng_sub_%s_configs/%s_findbugs.py', 'android-geckoview-docs': 'builds/releng_sub_%s_configs/%s_geckoview_docs.py', 'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py', 'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py', 'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py', 'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py', 'dmd': 'builds/releng_sub_%s_configs/%s_dmd.py', 'tup': 'builds/releng_sub_%s_configs/%s_tup.py', } build_pool_cfg_file = 'builds/build_pool_specifics.py' branch_cfg_file = 'builds/branch_specifics.py' @classmethod def _query_pltfrm_and_bits(cls, target_option, options): """ determine platform and bits This can be from either from a supplied --platform and --bits or parsed from given config file names. """ error_msg = ( 'Whoops!\nYou are trying to pass a shortname for ' '%s. \nHowever, I need to know the %s to find the appropriate ' 'filename. You can tell me by passing:\n\t"%s" or a config ' 'filename via "--config" with %s in it. \nIn either case, these ' 'option arguments must come before --custom-build-variant.' ) current_config_files = options.config_files or [] if not cls.bits: # --bits has not been supplied # lets parse given config file names for 32 or 64 for cfg_file_name in current_config_files: if '32' in cfg_file_name: cls.bits = '32' break if '64' in cfg_file_name: cls.bits = '64' break else: sys.exit(error_msg % (target_option, 'bits', '--bits', '"32" or "64"')) if not cls.platform: # --platform has not been supplied # lets parse given config file names for platform for cfg_file_name in current_config_files: if 'windows' in cfg_file_name: cls.platform = 'windows' break if 'mac' in cfg_file_name: cls.platform = 'mac' break if 'linux' in cfg_file_name: cls.platform = 'linux' break if 'android' in cfg_file_name: cls.platform = 'android' break else: sys.exit(error_msg % (target_option, 'platform', '--platform', '"linux", "windows", "mac", or "android"')) return cls.bits, cls.platform @classmethod def find_variant_cfg_path(cls, opt, value, parser): valid_variant_cfg_path = None # first let's see if we were given a valid short-name if cls.build_variants.get(value): bits, pltfrm = cls._query_pltfrm_and_bits(opt, parser.values) prospective_cfg_path = cls.build_variants[value] % (pltfrm, bits) else: # this is either an incomplete path or an invalid key in # build_variants prospective_cfg_path = value if os.path.exists(prospective_cfg_path): # now let's see if we were given a valid pathname valid_variant_cfg_path = value else: # FIXME: We should actually wait until we have parsed all arguments # before looking at this, otherwise the behavior will depend on the # order of arguments. But that isn't a problem as long as --extra-config-path # is always passed first. extra_config_paths = parser.values.config_paths or [] config_paths = extra_config_paths + [DEFAULT_CONFIG_PATH] # let's take our prospective_cfg_path and see if we can # determine an existing file for path in config_paths: if os.path.exists(os.path.join(path, prospective_cfg_path)): # success! we found a config file valid_variant_cfg_path = os.path.join(path, prospective_cfg_path) break return valid_variant_cfg_path, prospective_cfg_path @classmethod def set_build_variant(cls, option, opt, value, parser): """ sets an extra config file. This is done by either taking an existing filepath or by taking a valid shortname coupled with known platform/bits. """ valid_variant_cfg_path, prospective_cfg_path = cls.find_variant_cfg_path( '--custom-build-variant-cfg', value, parser) if not valid_variant_cfg_path: # either the value was an indeterminable path or an invalid short # name sys.exit("Whoops!\n'--custom-build-variant' was passed but an " "appropriate config file could not be determined. Tried " "using: '%s' but it was not:" "\n\t-- a valid shortname: %s " "\n\t-- a valid variant for the given platform and bits." % ( prospective_cfg_path, str(cls.build_variants.keys()))) parser.values.config_files.append(valid_variant_cfg_path) setattr(parser.values, option.dest, value) # the pool @classmethod def set_build_pool(cls, option, opt, value, parser): # first let's add the build pool file where there may be pool # specific keys/values. Then let's store the pool name parser.values.config_files.append(cls.build_pool_cfg_file) setattr(parser.values, option.dest, value) # the pool @classmethod def set_build_branch(cls, option, opt, value, parser): # first let's add the branch_specific file where there may be branch # specific keys/values. Then let's store the branch name we are using parser.values.config_files.append(cls.branch_cfg_file) setattr(parser.values, option.dest, value) # the branch name @classmethod def set_platform(cls, option, opt, value, parser): cls.platform = value setattr(parser.values, option.dest, value) @classmethod def set_bits(cls, option, opt, value, parser): cls.bits = value setattr(parser.values, option.dest, value) # this global depends on BuildOptionParser and therefore can not go at the # top of the file BUILD_BASE_CONFIG_OPTIONS = [ [['--developer-run'], { "action": "store_false", "dest": "is_automation", "default": True, "help": "If this is running outside of Mozilla's build" "infrastructure, use this option. It ignores actions" "that are not needed and adds config checks."}], [['--platform'], { "action": "callback", "callback": BuildOptionParser.set_platform, "type": "string", "dest": "platform", "help": "Sets the platform we are running this against" " valid values: 'windows', 'mac', 'linux'"}], [['--bits'], { "action": "callback", "callback": BuildOptionParser.set_bits, "type": "string", "dest": "bits", "help": "Sets which bits we are building this against" " valid values: '32', '64'"}], [['--custom-build-variant-cfg'], { "action": "callback", "callback": BuildOptionParser.set_build_variant, "type": "string", "dest": "build_variant", "help": "Sets the build type and will determine appropriate" " additional config to use. Either pass a config path" " or use a valid shortname from: " "%s" % (BuildOptionParser.build_variants.keys(),)}], [['--build-pool'], { "action": "callback", "callback": BuildOptionParser.set_build_pool, "type": "string", "dest": "build_pool", "help": "This will update the config with specific pool" " environment keys/values. The dicts for this are" " in %s\nValid values: staging or" " production" % ('builds/build_pool_specifics.py',)}], [['--branch'], { "action": "callback", "callback": BuildOptionParser.set_build_branch, "type": "string", "dest": "branch", "help": "This sets the branch we will be building this for." " If this branch is in branch_specifics.py, update our" " config with specific keys/values from that. See" " %s for possibilites" % ( BuildOptionParser.branch_cfg_file, )}], [['--scm-level'], { "action": "store", "type": "int", "dest": "scm_level", "default": 1, "help": "This sets the SCM level for the branch being built." " See https://www.mozilla.org/en-US/about/" "governance/policies/commit/access-policy/"}], [['--enable-pgo'], { "action": "store_true", "dest": "pgo_build", "default": False, "help": "Sets the build to run in PGO mode"}], [['--enable-nightly'], { "action": "store_true", "dest": "nightly_build", "default": False, "help": "Sets the build to run in nightly mode"}], [['--who'], { "dest": "who", "default": '', "help": "stores who made the created the change."}], ] def generate_build_ID(): return time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) def generate_build_UID(): return uuid.uuid4().hex class BuildScript(AutomationMixin, BalrogMixin, VirtualenvMixin, MercurialScript, SecretsMixin, PerfherderResourceOptionsMixin): def __init__(self, **kwargs): # objdir is referenced in _query_abs_dirs() so let's make sure we # have that attribute before calling BaseScript.__init__ self.objdir = None super(BuildScript, self).__init__(**kwargs) # epoch is only here to represent the start of the build # that this mozharn script came from. until I can grab bbot's # status.build.gettime()[0] this will have to do as a rough estimate # although it is about 4s off from the time it would be if it was # done through MBF. # TODO find out if that time diff matters or if we just use it to # separate each build self.epoch_timestamp = int(time.mktime(datetime.now().timetuple())) self.branch = self.config.get('branch') self.stage_platform = self.config.get('stage_platform') if not self.branch or not self.stage_platform: if not self.branch: self.error("'branch' not determined and is required") if not self.stage_platform: self.error("'stage_platform' not determined and is required") self.fatal("Please add missing items to your config") self.repo_path = None self.buildid = None self.query_buildid() # sets self.buildid self.generated_build_props = False self.client_id = None self.access_token = None # Call this before creating the virtualenv so that we can support # substituting config values with other config values. self.query_build_env() # We need to create the virtualenv directly (without using an action) in # order to use python modules in PreScriptRun/Action listeners self.create_virtualenv() def _pre_config_lock(self, rw_config): c = self.config cfg_files_and_dicts = rw_config.all_cfg_files_and_dicts build_pool = c.get('build_pool', '') build_variant = c.get('build_variant', '') variant_cfg = '' if build_variant: variant_cfg = BuildOptionParser.build_variants[build_variant] % ( BuildOptionParser.platform, BuildOptionParser.bits ) build_pool_cfg = BuildOptionParser.build_pool_cfg_file branch_cfg = BuildOptionParser.branch_cfg_file cfg_match_msg = "Script was run with '%(option)s %(type)s' and \ '%(type)s' matches a key in '%(type_config_file)s'. Updating self.config with \ items from that key's value." pf_override_msg = "The branch '%(branch)s' has custom behavior for the \ platform '%(platform)s'. Updating self.config with the following from \ 'platform_overrides' found in '%(pf_cfg_file)s':" for i, (target_file, target_dict) in enumerate(cfg_files_and_dicts): if branch_cfg and branch_cfg in target_file: self.info( cfg_match_msg % { 'option': '--branch', 'type': c['branch'], 'type_config_file': BuildOptionParser.branch_cfg_file } ) if build_pool_cfg and build_pool_cfg in target_file: self.info( cfg_match_msg % { 'option': '--build-pool', 'type': build_pool, 'type_config_file': build_pool_cfg, } ) if variant_cfg and variant_cfg in target_file: self.info( cfg_match_msg % { 'option': '--custom-build-variant-cfg', 'type': build_variant, 'type_config_file': variant_cfg, } ) if c.get("platform_overrides"): if c['stage_platform'] in c['platform_overrides'].keys(): self.info( pf_override_msg % { 'branch': c['branch'], 'platform': c['stage_platform'], 'pf_cfg_file': BuildOptionParser.branch_cfg_file } ) branch_pf_overrides = c['platform_overrides'][ c['stage_platform'] ] self.info(pprint.pformat(branch_pf_overrides)) c.update(branch_pf_overrides) self.info('To generate a config file based upon options passed and ' 'config files used, run script as before but extend options ' 'with "--dump-config"') self.info('For a diff of where self.config got its items, ' 'run the script again as before but extend options with: ' '"--dump-config-hierarchy"') self.info("Both --dump-config and --dump-config-hierarchy don't " "actually run any actions.") def _assert_cfg_valid_for_action(self, dependencies, action): """ assert dependency keys are in config for given action. Takes a list of dependencies and ensures that each have an assoctiated key in the config. Displays error messages as appropriate. """ # TODO add type and value checking, not just keys # TODO solution should adhere to: bug 699343 # TODO add this to BaseScript when the above is done # for now, let's just use this as a way to save typing... c = self.config undetermined_keys = [] err_template = "The key '%s' could not be determined \ and is needed for the action '%s'. Please add this to your config \ or run without that action (ie: --no-{action})" for dep in dependencies: if dep not in c: undetermined_keys.append(dep) if undetermined_keys: fatal_msgs = [err_template % (key, action) for key in undetermined_keys] self.fatal("".join(fatal_msgs)) # otherwise: return # all good def _query_build_prop_from_app_ini(self, prop, app_ini_path=None): dirs = self.query_abs_dirs() print_conf_setting_path = os.path.join(dirs['abs_src_dir'], 'config', 'printconfigsetting.py') if not app_ini_path: # set the default app_ini_path = dirs['abs_app_ini_path'] if (os.path.exists(print_conf_setting_path) and os.path.exists(app_ini_path)): cmd = [ sys.executable, os.path.join(dirs['abs_src_dir'], 'mach'), 'python', print_conf_setting_path, app_ini_path, 'App', prop ] env = self.query_build_env() # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on # mac, and that confuses mach. del env['MOZ_OBJDIR'] return self.get_output_from_command(cmd, cwd=dirs['abs_obj_dir'], env=env) else: return None def query_buildid(self): c = self.config if self.buildid: return self.buildid # for taskcluster, we pass MOZ_BUILD_DATE into mozharness as an # environment variable, only to have it pass the same value out with # the same name. buildid = os.environ.get('MOZ_BUILD_DATE') if not buildid: self.info("Creating buildid through current time") buildid = generate_build_ID() if c.get('is_automation') or os.environ.get("TASK_ID"): self.set_property('buildid', buildid) self.buildid = buildid return self.buildid def _query_objdir(self): if self.objdir: return self.objdir if not self.config.get('objdir'): return self.fatal(MISSING_CFG_KEY_MSG % ('objdir',)) self.objdir = self.config['objdir'] return self.objdir def _query_repo(self): if self.repo_path: return self.repo_path c = self.config # we actually supply the repo in mozharness so if it's in # the config, we use that (automation does not require it in # props) if not c.get('repo_path'): repo_path = 'projects/%s' % (self.branch,) self.info( "repo_path not in config. Using '%s' instead" % (repo_path,) ) else: repo_path = c['repo_path'] self.repo_path = '%s/%s' % (c['repo_base'], repo_path,) return self.repo_path def query_is_nightly_promotion(self): platform_enabled = self.config.get('enable_nightly_promotion') branch_enabled = self.branch in self.config.get('nightly_promotion_branches') return platform_enabled and branch_enabled def query_build_env(self, **kwargs): c = self.config # let's evoke the base query_env and make a copy of it # as we don't always want every key below added to the same dict env = copy.deepcopy( super(BuildScript, self).query_env(**kwargs) ) # first grab the buildid env['MOZ_BUILD_DATE'] = self.query_buildid() # Set the source repository to what we're building from since # the default is to query `hg paths` which isn't reliable with pooled # storage repo_path = self._query_repo() assert repo_path env['MOZ_SOURCE_REPO'] = repo_path if self.query_is_nightly() or self.query_is_nightly_promotion(): if self.query_is_nightly(): # nightly promotion needs to set update_channel but not do all the 'IS_NIGHTLY' # automation parts like uploading symbols for now env["IS_NIGHTLY"] = "yes" # in branch_specifics.py we might set update_channel explicitly if c.get('update_channel'): env["MOZ_UPDATE_CHANNEL"] = c['update_channel'] else: # let's just give the generic channel based on branch env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,) if self.config.get('pgo_build') or self._compile_against_pgo(): env['MOZ_PGO'] = '1' # to activate the right behaviour in mozonfigs while we transition if c.get('enable_release_promotion'): env['ENABLE_RELEASE_PROMOTION'] = "1" update_channel = c.get('update_channel', self.branch) self.info("Release promotion update channel: %s" % (update_channel,)) env["MOZ_UPDATE_CHANNEL"] = update_channel return env def query_mach_build_env(self, multiLocale=None): c = self.config if multiLocale is None and self.query_is_nightly(): multiLocale = c.get('multi_locale', False) mach_env = {} if c.get('upload_env'): mach_env.update(c['upload_env']) # this prevents taskcluster from overwriting the target files with # the multilocale files. Put everything from the en-US build in a # separate folder. if multiLocale and self.config.get('taskcluster_nightly'): if 'UPLOAD_PATH' in mach_env: mach_env['UPLOAD_PATH'] = os.path.join(mach_env['UPLOAD_PATH'], 'en-US') return mach_env def _compile_against_pgo(self): """determines whether a build should be run with pgo even if it is not a classified as a 'pgo build'. requirements: 1) must be a platform that can run against pgo 2) either: a) must be a nightly build b) must be on a branch that runs pgo if it can everytime """ c = self.config if self.stage_platform in c['pgo_platforms']: if c.get('branch_uses_per_checkin_strategy') or self.query_is_nightly(): return True return False def query_check_test_env(self): c = self.config dirs = self.query_abs_dirs() check_test_env = {} if c.get('check_test_env'): for env_var, env_value in c['check_test_env'].iteritems(): check_test_env[env_var] = env_value % dirs return check_test_env def _rm_old_package(self): """rm the old package.""" c = self.config dirs = self.query_abs_dirs() old_package_paths = [] old_package_patterns = c.get('old_packages') self.info("removing old packages...") if os.path.exists(dirs['abs_obj_dir']): for product in old_package_patterns: old_package_paths.extend( glob.glob(product % {"objdir": dirs['abs_obj_dir']}) ) if old_package_paths: for package_path in old_package_paths: self.rmtree(package_path) else: self.info("There wasn't any old packages to remove.") def _get_mozconfig(self): """assign mozconfig.""" dirs = self.query_abs_dirs() try: abs_mozconfig_path = get_mozconfig_path( script=self, config=self.config, dirs=dirs) except MozconfigPathError as e: self.fatal(e.message) self.info("Use mozconfig: {}".format(abs_mozconfig_path)) # print its contents content = self.read_from_file(abs_mozconfig_path, error_level=FATAL) self.info("mozconfig content:") self.info(content) # finally, copy the mozconfig to a path that 'mach build' expects it to be self.copyfile(abs_mozconfig_path, os.path.join(dirs['abs_src_dir'], '.mozconfig')) # TODO: replace with ToolToolMixin def _get_tooltool_auth_file(self): # set the default authentication file based on platform; this # corresponds to where puppet puts the token if 'tooltool_authentication_file' in self.config: fn = self.config['tooltool_authentication_file'] elif self._is_windows(): fn = r'c:\builds\relengapi.tok' else: fn = '/builds/relengapi.tok' # if the file doesn't exist, don't pass it to tooltool (it will just # fail). In taskcluster, this will work OK as the relengapi-proxy will # take care of auth. Everywhere else, we'll get auth failures if # necessary. if os.path.exists(fn): return fn def _run_tooltool(self): env = self.query_build_env() env.update(self.query_mach_build_env()) self._assert_cfg_valid_for_action( ['tooltool_url'], 'build' ) c = self.config dirs = self.query_abs_dirs() toolchains = os.environ.get('MOZ_TOOLCHAINS') manifest_src = os.environ.get('TOOLTOOL_MANIFEST') if not manifest_src: manifest_src = c.get('tooltool_manifest_src') if not manifest_src and not toolchains: return self.warning(ERROR_MSGS['tooltool_manifest_undetermined']) cmd = [ sys.executable, '-u', os.path.join(dirs['abs_src_dir'], 'mach'), 'artifact', 'toolchain', '-v', '--retry', '4', '--artifact-manifest', os.path.join(dirs['abs_src_dir'], 'toolchains.json'), ] if manifest_src: cmd.extend([ '--tooltool-manifest', os.path.join(dirs['abs_src_dir'], manifest_src), '--tooltool-url', c['tooltool_url'], ]) auth_file = self._get_tooltool_auth_file() if auth_file: cmd.extend(['--authentication-file', auth_file]) cache = c['env'].get('TOOLTOOL_CACHE') if cache: cmd.extend(['--cache-dir', cache]) if toolchains: cmd.extend(toolchains.split()) self.info(str(cmd)) self.run_command(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True, env=env) def query_revision(self, source_path=None): """ returns the revision of the build This method is used both to figure out what revision to check out and to figure out what revision *was* checked out. """ revision = None if not source_path: dirs = self.query_abs_dirs() source_path = dirs['abs_src_dir'] # let's take the default # Look at what we have checked out if os.path.exists(source_path): hg = self.query_exe('hg', return_type='list') revision = self.get_output_from_command( hg + ['parent', '--template', '{node}'], cwd=source_path ) return revision.encode('ascii', 'replace') if revision else None def _query_props_set_by_mach(self, console_output=True, error_level=FATAL): mach_properties_path = os.path.join( self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json' ) self.info("setting properties set by mach build. Looking in path: %s" % mach_properties_path) if os.path.exists(mach_properties_path): with self.opened(mach_properties_path, error_level=error_level) as (fh, err): build_props = json.load(fh) if err: self.log("%s exists but there was an error reading the " "properties. props: `%s` - error: " "`%s`" % (mach_properties_path, build_props or 'None', err or 'No error'), error_level) if console_output: self.info("Properties set from 'mach build'") self.info(pprint.pformat(build_props)) for key, prop in build_props.iteritems(): if prop != 'UNKNOWN': self.set_property(key, prop) else: self.info("No mach_build_properties.json found - not importing properties.") def generate_build_props(self, console_output=True, halt_on_failure=False): """sets props found from mach build and, in addition, buildid, sourcestamp, appVersion, and appName.""" error_level = ERROR if halt_on_failure: error_level = FATAL if self.generated_build_props: return # grab props set by mach if any self._query_props_set_by_mach(console_output=console_output, error_level=error_level) dirs = self.query_abs_dirs() print_conf_setting_path = os.path.join(dirs['abs_src_dir'], 'config', 'printconfigsetting.py') if (not os.path.exists(print_conf_setting_path) or not os.path.exists(dirs['abs_app_ini_path'])): self.log("Can't set the following properties: " "buildid, sourcestamp, appVersion, and appName. " "Required paths missing. Verify both %s and %s " "exist. These paths require the 'build' action to be " "run prior to this" % (print_conf_setting_path, dirs['abs_app_ini_path']), level=error_level) self.info("Setting properties found in: %s" % dirs['abs_app_ini_path']) base_cmd = [ sys.executable, os.path.join(dirs['abs_src_dir'], 'mach'), 'python', print_conf_setting_path, dirs['abs_app_ini_path'], 'App' ] properties_needed = [ {'ini_name': 'SourceStamp', 'prop_name': 'sourcestamp'}, {'ini_name': 'Version', 'prop_name': 'appVersion'}, {'ini_name': 'Name', 'prop_name': 'appName'} ] env = self.query_build_env() # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on # mac, and that confuses mach. del env['MOZ_OBJDIR'] for prop in properties_needed: prop_val = self.get_output_from_command( base_cmd + [prop['ini_name']], cwd=dirs['abs_obj_dir'], halt_on_failure=halt_on_failure, env=env ) self.set_property(prop['prop_name'], prop_val) if self.config.get('is_automation'): self.info("Verifying buildid from application.ini matches buildid " "from automation") app_ini_buildid = self._query_build_prop_from_app_ini('BuildID') # it would be hard to imagine query_buildid evaluating to a falsey # value (e.g. 0), but incase it does, force it to None automation_buildid = self.query_buildid() or None self.info( 'buildid from application.ini: "%s". buildid from automation ' 'properties: "%s"' % (app_ini_buildid, automation_buildid) ) if app_ini_buildid == automation_buildid is not None: self.info('buildids match.') else: self.error( 'buildids do not match or values could not be determined' ) # set the build to orange if not already worse self.return_code = self.worst_level( EXIT_STATUS_DICT[TBPL_WARNING], self.return_code, AUTOMATION_EXIT_CODES[::-1] ) self.generated_build_props = True def _create_mozbuild_dir(self, mozbuild_path=None): if not mozbuild_path: env = self.query_build_env() mozbuild_path = env.get('MOZBUILD_STATE_PATH') if mozbuild_path: self.mkdir_p(mozbuild_path) else: self.warning("mozbuild_path could not be determined. skipping " "creating it.") def preflight_build(self): """set up machine state for a complete build.""" if not self.query_is_nightly(): # the old package should live in source dir so we don't need to do # this for nighties since we clobber the whole work_dir in # clobber() self._rm_old_package() self._get_mozconfig() self._run_tooltool() self._create_mozbuild_dir() mach_props = os.path.join( self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json' ) if os.path.exists(mach_props): self.info("Removing previous mach property file: %s" % mach_props) self.rmtree(mach_props) def build(self): """builds application.""" # This will error on non-0 exit code. self._run_mach_command_in_build_env(['build', '-v']) self.generate_build_props(console_output=True, halt_on_failure=True) self._generate_build_stats() def static_analysis_autotest(self): """Run mach static-analysis autotest, in order to make sure we dont regress""" self.preflight_build() - self._run_mach_command_in_build_env(['configure']) - self._run_mach_command_in_build_env(['static-analysis', 'autotest', - '--intree-tool'], - use_subprocess=True) + self._run_mach_command_in_build_env(['static-analysis', 'autotest', '--intree-tool']) - def _run_mach_command_in_build_env(self, args, use_subprocess=False): + def _run_mach_command_in_build_env(self, args): """Run a mach command in a build context.""" env = self.query_build_env() env.update(self.query_mach_build_env()) + dirs = self.query_abs_dirs() if 'MOZILLABUILD' in os.environ: # We found many issues with intermittent build failures when not invoking mach via bash. # See bug 1364651 before considering changing. mach = [ os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'), os.path.join(dirs['abs_src_dir'], 'mach') ] else: mach = [sys.executable, 'mach'] - # XXX See bug 1483883 - # Work around an interaction between Gradle and mozharness - # Not using `subprocess` causes gradle to hang - if use_subprocess: - import subprocess - return_code = subprocess.call(mach + ['--log-no-times'] + args, - env=env, cwd=dirs['abs_src_dir']) - else: - return_code = self.run_command( - command=mach + ['--log-no-times'] + args, - cwd=dirs['abs_src_dir'], - env=env, - output_timeout=self.config.get('max_build_output_timeout', - 60 * 40) - ) + return_code = self.run_command( + command=mach + ['--log-no-times'] + args, + cwd=dirs['abs_src_dir'], + env=env, + output_timeout=self.config.get('max_build_output_timeout', 60 * 40) + ) + if return_code: self.return_code = self.worst_level( EXIT_STATUS_DICT[TBPL_FAILURE], self.return_code, AUTOMATION_EXIT_CODES[::-1] ) self.fatal("'mach %s' did not run successfully. Please check " "log for errors." % ' '.join(args)) def multi_l10n(self): if not self.query_is_nightly(): self.info("Not a nightly build, skipping multi l10n.") return dirs = self.query_abs_dirs() base_work_dir = dirs['base_work_dir'] objdir = dirs['abs_obj_dir'] branch = self.branch # Building a nightly with the try repository fails because a # config-file does not exist for try. Default to mozilla-central # settings (arbitrarily). if branch == 'try': branch = 'mozilla-central' multi_config_pf = self.config.get('multi_locale_config_platform', 'android') multil10n_path = 'build/src/testing/mozharness/scripts/multil10n.py' base_work_dir = os.path.join(base_work_dir, 'workspace') cmd = [ sys.executable, multil10n_path, '--config-file', 'multi_locale/%s_%s.json' % (branch, multi_config_pf), '--config-file', 'multi_locale/android-mozharness-build.json', '--pull-locale-source', '--add-locales', '--android-assemble-app', '--package-multi', '--summary', ] self.run_command(cmd, env=self.query_build_env(), cwd=base_work_dir, halt_on_failure=True) package_cmd = [ 'make', 'echo-variable-PACKAGE', 'AB_CD=multi', ] package_filename = self.get_output_from_command( package_cmd, cwd=objdir, ) if not package_filename: self.fatal("Unable to determine the package filename for the multi-l10n build. Was trying to run: %s" % package_cmd) self.info('Multi-l10n package filename is: %s' % package_filename) parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj, use_package_as_marfile=True, package_filename=package_filename, ) upload_cmd = ['make', 'upload', 'AB_CD=multi'] self.run_command(upload_cmd, env=self.query_mach_build_env(multiLocale=False), cwd=objdir, halt_on_failure=True, output_parser=parser) for prop in parser.matches: self.set_property(prop, parser.matches[prop]) upload_files_cmd = [ 'make', 'echo-variable-UPLOAD_FILES', 'AB_CD=multi', ] output = self.get_output_from_command( upload_files_cmd, cwd=objdir, ) def postflight_build(self): """grabs properties from post build and calls ccache -s""" # A list of argument lists. Better names gratefully accepted! mach_commands = self.config.get('postflight_build_mach_commands', []) for mach_command in mach_commands: self._execute_postflight_build_mach_command(mach_command) def _execute_postflight_build_mach_command(self, mach_command_args): env = self.query_build_env() env.update(self.query_mach_build_env()) command = [sys.executable, 'mach', '--log-no-times'] command.extend(mach_command_args) self.run_command( command=command, cwd=self.query_abs_dirs()['abs_src_dir'], env=env, output_timeout=self.config.get('max_build_output_timeout', 60 * 20), halt_on_failure=True, ) def preflight_package_source(self): self._get_mozconfig() def package_source(self): """generates source archives and uploads them""" env = self.query_build_env() env.update(self.query_mach_build_env()) dirs = self.query_abs_dirs() self.run_command( command=[sys.executable, 'mach', '--log-no-times', 'configure'], cwd=dirs['abs_src_dir'], env=env, output_timeout=60*3, halt_on_failure=True, ) self.run_command( command=[ 'make', 'source-package', 'source-upload', ], cwd=dirs['abs_obj_dir'], env=env, output_timeout=60*45, halt_on_failure=True, ) def check_test(self): if self.config.get('forced_artifact_build'): self.info('Skipping due to forced artifact build.') return c = self.config dirs = self.query_abs_dirs() env = self.query_build_env() env.update(self.query_check_test_env()) cmd = [ sys.executable, 'mach', '--log-no-times', 'build', '-v', '--keep-going', 'check', ] parser = CheckTestCompleteParser(config=c, log_obj=self.log_obj) return_code = self.run_command(command=cmd, cwd=dirs['abs_src_dir'], env=env, output_parser=parser) tbpl_status = parser.evaluate_parser(return_code) return_code = EXIT_STATUS_DICT[tbpl_status] if return_code: self.return_code = self.worst_level( return_code, self.return_code, AUTOMATION_EXIT_CODES[::-1] ) self.error("'mach build check' did not run successfully. Please " "check log for errors.") def _is_configuration_shipped(self): """Determine if the current build configuration is shipped to users. This is used to drive alerting so we don't see alerts for build configurations we care less about. """ # Ideally this would be driven by a config option. However, our # current inheritance mechanism of using a base config and then # one-off configs for variants isn't conducive to this since derived # configs we need to be reset and we don't like requiring boilerplate # in derived configs. # All PGO builds are shipped. This takes care of Linux and Windows. if self.config.get('pgo_build'): return True # Debug builds are never shipped. if self.config.get('debug_build'): return False # OS X opt builds without a variant are shipped. if self.config.get('platform') == 'macosx64': if not self.config.get('build_variant'): return True # Android opt builds without a variant are shipped. if self.config.get('platform') == 'android': if not self.config.get('build_variant'): return True return False def _load_build_resources(self): p = self.config.get('build_resources_path') % self.query_abs_dirs() if not os.path.exists(p): self.info('%s does not exist; not loading build resources' % p) return None with open(p, 'rb') as fh: resources = json.load(fh) if 'duration' not in resources: self.info('resource usage lacks duration; ignoring') return None data = { 'name': 'build times', 'value': resources['duration'], 'extraOptions': self.perfherder_resource_options(), 'subtests': [], } for phase in resources['phases']: if 'duration' not in phase: continue data['subtests'].append({ 'name': phase['name'], 'value': phase['duration'], }) return data def _load_sccache_stats(self): stats_file = os.path.join( self.query_abs_dirs()['abs_obj_dir'], 'sccache-stats.json' ) if not os.path.exists(stats_file): self.info('%s does not exist; not loading sccache stats' % stats_file) return with open(stats_file, 'rb') as fh: stats = json.load(fh) total = stats['stats']['requests_executed'] hits = stats['stats']['cache_hits'] if total > 0: hits /= float(total) yield { 'name': 'sccache hit rate', 'value': hits, 'extraOptions': self.perfherder_resource_options(), 'subtests': [], 'lowerIsBetter': False } yield { 'name': 'sccache cache_write_errors', 'value': stats['stats']['cache_write_errors'], 'extraOptions': self.perfherder_resource_options(), 'alertThreshold': 50.0, 'subtests': [], } yield { 'name': 'sccache requests_not_cacheable', 'value': stats['stats']['requests_not_cacheable'], 'extraOptions': self.perfherder_resource_options(), 'alertThreshold': 50.0, 'subtests': [], } def _get_package_metrics(self): import tarfile import zipfile dirs = self.query_abs_dirs() dist_dir = os.path.join(dirs['abs_obj_dir'], 'dist') for ext in ['apk', 'dmg', 'tar.bz2', 'zip']: name = 'target.' + ext if os.path.exists(os.path.join(dist_dir, name)): packageName = name break else: self.fatal("could not determine packageName") interests = ['libxul.so', 'classes.dex', 'omni.ja', 'xul.dll'] installer = os.path.join(dist_dir, packageName) installer_size = 0 size_measurements = [] def paths_with_sizes(installer): if zipfile.is_zipfile(installer): with zipfile.ZipFile(installer, 'r') as zf: for zi in zf.infolist(): yield zi.filename, zi.file_size elif tarfile.is_tarfile(installer): with tarfile.open(installer, 'r:*') as tf: for ti in tf: yield ti.name, ti.size if os.path.exists(installer): installer_size = self.query_filesize(installer) self.info('Size of %s: %s bytes' % (packageName, installer_size)) try: subtests = {} for path, size in paths_with_sizes(installer): name = os.path.basename(path) if name in interests: # We have to be careful here: desktop Firefox installers # contain two omni.ja files: one for the general runtime, # and one for the browser proper. if name == 'omni.ja': containing_dir = os.path.basename(os.path.dirname(path)) if containing_dir == 'browser': name = 'browser-omni.ja' if name in subtests: self.fatal('should not see %s (%s) multiple times!' % (name, path)) subtests[name] = size for name in subtests: self.info('Size of %s: %s bytes' % (name, subtests[name])) size_measurements.append( {'name': name, 'value': subtests[name]}) except: self.info('Unable to search %s for component sizes.' % installer) size_measurements = [] if not installer_size and not size_measurements: return # We want to always collect metrics. But alerts for installer size are # only use for builds with ship. So nix the alerts for builds we don't # ship. def filter_alert(alert): if not self._is_configuration_shipped(): alert['shouldAlert'] = False return alert if installer.endswith('.apk'): # Android yield filter_alert({ "name": "installer size", "value": installer_size, "alertChangeType": "absolute", "alertThreshold": (200 * 1024), "subtests": size_measurements }) else: yield filter_alert({ "name": "installer size", "value": installer_size, "alertChangeType": "absolute", "alertThreshold": (100 * 1024), "subtests": size_measurements }) def _get_sections(self, file, filter=None): """ Returns a dictionary of sections and their sizes. """ from StringIO import StringIO # Check for `rust_size`, our cross platform version of size. It should # be installed by tooltool in $abs_src_dir/rust-size/rust-size rust_size = os.path.join(self.query_abs_dirs()['abs_src_dir'], 'rust-size', 'rust-size') size_prog = self.which(rust_size) if not size_prog: self.info("Couldn't find `rust-size` program") return {} self.info("Using %s" % size_prog) cmd = [size_prog, file] output = self.get_output_from_command(cmd) if not output: self.info("`rust-size` failed") return {} # Format is JSON: # { # "section_type": { # "section_name": size, .... # }, # ... # } try: parsed = json.loads(output) except ValueError: self.info("`rust-size` failed: %s" % output) return {} sections = {} for sec_type in parsed.itervalues(): for name, size in sec_type.iteritems(): if not filter or name in filter: sections[name] = size return sections def _get_binary_metrics(self): """ Provides metrics on interesting compenents of the built binaries. Currently just the sizes of interesting sections. """ lib_interests = { 'XUL': ('libxul.so', 'xul.dll', 'XUL'), 'NSS': ('libnss3.so', 'nss3.dll', 'libnss3.dylib'), 'NSPR': ('libnspr4.so', 'nspr4.dll', 'libnspr4.dylib'), 'avcodec': ('libmozavcodec.so', 'mozavcodec.dll', 'libmozavcodec.dylib'), 'avutil': ('libmozavutil.so', 'mozavutil.dll', 'libmozavutil.dylib') } section_interests = ('.text', '.data', '.rodata', '.rdata', '.cstring', '.data.rel.ro', '.bss') lib_details = [] dirs = self.query_abs_dirs() dist_dir = os.path.join(dirs['abs_obj_dir'], 'dist') bin_dir = os.path.join(dist_dir, 'bin') for lib_type, lib_names in lib_interests.iteritems(): for lib_name in lib_names: lib = os.path.join(bin_dir, lib_name) if os.path.exists(lib): lib_size = 0 section_details = self._get_sections(lib, section_interests) section_measurements = [] # Build up the subtests # Lump rodata sections together # - Mach-O separates out read-only string data as .cstring # - PE really uses .rdata, but XUL at least has a .rodata as well for ro_alias in ('.cstring', '.rdata'): if ro_alias in section_details: if '.rodata' in section_details: section_details['.rodata'] += section_details[ro_alias] else: section_details['.rodata'] = section_details[ro_alias] del section_details[ro_alias] for k, v in section_details.iteritems(): section_measurements.append({'name': k, 'value': v}) lib_size += v lib_details.append({ 'name': lib_type, 'size': lib_size, 'sections': section_measurements }) for lib_detail in lib_details: yield { "name": "%s section sizes" % lib_detail['name'], "value": lib_detail['size'], "shouldAlert": False, "subtests": lib_detail['sections'] } def _generate_build_stats(self): """grab build stats following a compile. This action handles all statistics from a build: 'count_ctors' and then posts to graph server the results. We only post to graph server for non nightly build """ self.info('Collecting build metrics') if self.config.get('forced_artifact_build'): self.info('Skipping due to forced artifact build.') return c = self.config # Report some important file sizes for display in treeherder perfherder_data = { "framework": { "name": "build_metrics" }, "suites": [], } if not c.get('debug_build') and not c.get('disable_package_metrics'): perfherder_data['suites'].extend(self._get_package_metrics()) perfherder_data['suites'].extend(self._get_binary_metrics()) # Extract compiler warnings count. warnings = self.get_output_from_command( command=[sys.executable, 'mach', 'warnings-list'], cwd=self.query_abs_dirs()['abs_src_dir'], env=self.query_build_env(), # No need to pollute the log. silent=True, # Fail fast. halt_on_failure=True) if warnings is not None: perfherder_data['suites'].append({ 'name': 'compiler warnings', 'value': len(warnings.strip().splitlines()), 'alertThreshold': 100.0, 'subtests': [], }) build_metrics = self._load_build_resources() if build_metrics: perfherder_data['suites'].append(build_metrics) perfherder_data['suites'].extend(self._load_sccache_stats()) # Ensure all extra options for this configuration are present. for opt in os.environ.get('PERFHERDER_EXTRA_OPTIONS', '').split(): for suite in perfherder_data['suites']: if opt not in suite.get('extraOptions', []): suite.setdefault('extraOptions', []).append(opt) if self.query_is_nightly(): for suite in perfherder_data['suites']: suite.setdefault('extraOptions', []).insert(0, 'nightly') if perfherder_data["suites"]: self.info('PERFHERDER_DATA: %s' % json.dumps(perfherder_data)) def update(self): """ submit balrog update steps. """ if self.config.get('forced_artifact_build'): self.info('Skipping due to forced artifact build.') return if not self.query_is_nightly(): self.info("Not a nightly build, skipping balrog submission.") return # grab any props available from this or previous unclobbered runs self.generate_build_props(console_output=False, halt_on_failure=False) # generate balrog props as artifacts if self.config.get('taskcluster_nightly'): env = self.query_mach_build_env(multiLocale=False) props_path = os.path.join(env["UPLOAD_PATH"], 'balrog_props.json') self.generate_balrog_props(props_path) return def valgrind_test(self): '''Execute mach's valgrind-test for memory leaks''' env = self.query_build_env() env.update(self.query_mach_build_env()) return_code = self.run_command( command=[sys.executable, 'mach', 'valgrind-test'], cwd=self.query_abs_dirs()['abs_src_dir'], env=env, output_timeout=self.config.get('max_build_output_timeout', 60 * 40) ) if return_code: self.return_code = self.worst_level( EXIT_STATUS_DICT[TBPL_FAILURE], self.return_code, AUTOMATION_EXIT_CODES[::-1] ) self.fatal("'mach valgrind-test' did not run successfully. Please check " "log for errors.") def ensure_upload_path(self): env = self.query_mach_build_env() # Some Taskcluster workers don't like it if an artifacts directory # is defined but no artifacts are uploaded. Guard against this by always # ensuring the artifacts directory exists. if 'UPLOAD_PATH' in env and not os.path.exists(env['UPLOAD_PATH']): os.makedirs(env['UPLOAD_PATH']) def _post_fatal(self, message=None, exit_code=None): if not self.return_code: # only overwrite return_code if it's 0 self.error('setting return code to 2 because fatal was called') self.return_code = 2 @PostScriptRun def _summarize(self): """ If this is run in automation, ensure the return code is valid and set it to one if it's not. Finally, log any summaries we collected from the script run. """ if self.config.get("is_automation"): # let's ignore all mention of tbpl status until this # point so it will be easier to manage if self.return_code not in AUTOMATION_EXIT_CODES: self.error("Return code is set to: %s and is outside of " "automation's known values. Setting to 2(failure). " "Valid return codes %s" % (self.return_code, AUTOMATION_EXIT_CODES)) self.return_code = 2 for status, return_code in EXIT_STATUS_DICT.iteritems(): if return_code == self.return_code: self.record_status(status, TBPL_STATUS_DICT[status]) self.summary() diff --git a/tools/infer/config.yaml b/tools/infer/config.yaml --- a/tools/infer/config.yaml +++ b/tools/infer/config.yaml @@ -1,31 +1,23 @@ --- target: obj-x86_64-pc-linux-gnu # It is used by 'mach static-analysis' and 'mozreview static-analysis bot' # in order to have consistency across the used checkers. platforms: - linux64 infer_checkers: - # no issues were ever trigger by this - name: check-nullable - publish: !!bool no - - name: biabduction publish: !!bool yes - # very very noisy - # it could be useful, but it won't be part of the default enabled checkers - name: eradicate publish: !!bool no - # hard to use, not useful - name: quandary - publish: !!bool no + publish: !!bool yes - name: starvation publish: !!bool yes - # experimental - name: litho - publish: !!bool no + publish: !!bool yes - name: racerd publish: !!bool yes - # I think this is only for c++, can't trigger these errors in Java - name: liveness - publish: !!bool no + publish: !!bool yes # Third party files from mozilla-central third_party: tools/rewriting/ThirdPartyPaths.txt diff --git a/tools/infer/test/autotest/build.gradle b/tools/infer/test/autotest/build.gradle deleted file mode 100644 --- a/tools/infer/test/autotest/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -apply plugin: 'java' - -repositories { - mavenCentral() -} - -dependencies { - compile "com.google.code.findbugs:jsr305:3.0.2" -} - -def createSingleTask = { name -> - task("compileInferTest${name}", type: JavaCompile) { - source = fileTree(dir: '.', include: "src/main/java/${name}.java") - classpath = project.configurations.compileClasspath - destinationDir = file("${topobjdir}/gradle/build/tools/infer/test/autotest") - } -} - -createSingleTask('Biabduction') -createSingleTask('Checkers') -createSingleTask('Eradicate') -createSingleTask('Racerd') -createSingleTask('Starvation') \ No newline at end of file diff --git a/tools/infer/test/autotest/src/main/java/Biabduction.java b/tools/infer/test/autotest/src/main/java/Biabduction.java deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Biabduction.java +++ /dev/null @@ -1,22 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import javax.annotation.Nullable; -import java.util.List; - -public class Biabduction { - private String get() { return null; } - - public void f1() { - get().length(); // error - } - - public void f2() { - try { - get().length(); // error - } catch (NullPointerException e) { - - } - } -} diff --git a/tools/infer/test/autotest/src/main/java/Biabduction.json b/tools/infer/test/autotest/src/main/java/Biabduction.json deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Biabduction.json +++ /dev/null @@ -1,114 +0,0 @@ -[ - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "start of procedure f1()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 11 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 12 - }, - { - "column_number": -1, - "description": "start of procedure get()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 1, - "line_number": 9 - }, - { - "column_number": -1, - "description": "return from a call to String Biabduction.get()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 1, - "line_number": 9 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 12 - } - ], - "bug_type": "NULL_DEREFERENCE", - "bug_type_hum": "Null Dereference", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Biabduction.java", - "hash": "918d7eaedf45f651f04c55554c72478c", - "key": "Biabduction.java|f1|NULL_DEREFERENCE", - "kind": "ERROR", - "line": 12, - "node_key": "9afcdcc9d4253c36267a0d34b98c337d", - "procedure": "void Biabduction.f1()", - "procedure_id": "Biabduction.f1():void.4b49520e7621606a0d5661886ff0b098", - "procedure_start_line": 11, - "qualifier": "object returned by `get(this)` could be null and is dereferenced at line 12.", - "severity": "HIGH", - "visibility": "user" - }, - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "start of procedure f2()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 15 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 17 - }, - { - "column_number": -1, - "description": "start of procedure get()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 1, - "line_number": 9 - }, - { - "column_number": -1, - "description": "return from a call to String Biabduction.get()", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 1, - "line_number": 9 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Biabduction.java", - "level": 0, - "line_number": 17 - } - ], - "bug_type": "NULL_DEREFERENCE", - "bug_type_hum": "Null Dereference", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Biabduction.java", - "hash": "bc952ce8bad58dac5cb6672dc3150524", - "key": "Biabduction.java|f2|NULL_DEREFERENCE", - "kind": "ERROR", - "line": 17, - "node_key": "9afcdcc9d4253c36267a0d34b98c337d", - "procedure": "void Biabduction.f2()", - "procedure_id": "Biabduction.f2():void.41c05a632eb912a458482c1e2e4dcbb4", - "procedure_start_line": 15, - "qualifier": "object returned by `get(this)` could be null and is dereferenced at line 17.", - "severity": "HIGH", - "visibility": "user" - } -] diff --git a/tools/infer/test/autotest/src/main/java/Checkers.java b/tools/infer/test/autotest/src/main/java/Checkers.java deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Checkers.java +++ /dev/null @@ -1,34 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; - -public class Checkers { - public static void leak() { - try { - BufferedReader br = new BufferedReader( - new FileReader(new File("some.txt")) - ); - } catch (Exception e) { - - } - } - - public static void error1() { - String str = null; - try { - int x = str.length(); // Error: even if exception is caught - } catch (NullPointerException e) { - - } - } - - public static void error2() { - String str = null; - int x = str.length(); // Error: not checking for null - } - -} diff --git a/tools/infer/test/autotest/src/main/java/Checkers.json b/tools/infer/test/autotest/src/main/java/Checkers.json deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Checkers.json +++ /dev/null @@ -1,121 +0,0 @@ -[ - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "start of procedure leak()", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 10 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 12 - } - ], - "bug_type": "RESOURCE_LEAK", - "bug_type_hum": "Resource Leak", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Checkers.java", - "hash": "56806153823413731f2e2166ed8d30a0", - "key": "Checkers.java|leak|RESOURCE_LEAK", - "kind": "ERROR", - "line": 12, - "node_key": "3a2af627d5d1f10e1994f6259cf18e4c", - "procedure": "void Checkers.leak()", - "procedure_id": "Checkers.leak():void.e21648e10d3037f4559cdb7c08642c84", - "procedure_start_line": 10, - "qualifier": "resource of type `java.io.FileReader` acquired by call to `new()` at line 12 is not released after line 12.", - "severity": "HIGH", - "visibility": "user" - }, - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "start of procedure error1()", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 20 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 21 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 23 - } - ], - "bug_type": "NULL_DEREFERENCE", - "bug_type_hum": "Null Dereference", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Checkers.java", - "hash": "6de26e7c66c71b1114ad233679d55640", - "key": "Checkers.java|error1|NULL_DEREFERENCE", - "kind": "ERROR", - "line": 23, - "node_key": "c281f77c6dae544ee5fb7d5e2bb35118", - "procedure": "void Checkers.error1()", - "procedure_id": "Checkers.error1():void.59417424d80960700a32012973e98db5", - "procedure_start_line": 20, - "qualifier": "object `str` last assigned on line 21 could be null and is dereferenced at line 23.", - "severity": "HIGH", - "visibility": "user" - }, - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "start of procedure error2()", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 29 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 30 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Checkers.java", - "level": 0, - "line_number": 31 - } - ], - "bug_type": "NULL_DEREFERENCE", - "bug_type_hum": "Null Dereference", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Checkers.java", - "hash": "39e021b634ab428af7be2034688491a7", - "key": "Checkers.java|error2|NULL_DEREFERENCE", - "kind": "ERROR", - "line": 31, - "node_key": "c281f77c6dae544ee5fb7d5e2bb35118", - "procedure": "void Checkers.error2()", - "procedure_id": "Checkers.error2():void.e9146d80ba20c908c11d08947cd89d06", - "procedure_start_line": 29, - "qualifier": "object `str` last assigned on line 30 could be null and is dereferenced at line 31.", - "severity": "HIGH", - "visibility": "user" - } -] diff --git a/tools/infer/test/autotest/src/main/java/Eradicate.java b/tools/infer/test/autotest/src/main/java/Eradicate.java deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Eradicate.java +++ /dev/null @@ -1,57 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import javax.annotation.Nullable; - -// Examples taken from the infer website. -public class Eradicate { - - public String f; // Because it is not annoted with nullable -> can never be null! - - public void field(@Nullable Eradicate x) { - x.f = "3"; // Error: Eradicate null field access - } - - public void method(@Nullable Object x) { - String s = x.toString(); // Error: Eradicate null method call - } - - public void filedNotNull(@Nullable String s) { - f = s; // Error: Eradicate field not nullable - } - - public Eradicate() {} // Error: Eradicate field not initialized - - public void str(Eradicate x) { - String s = x.toString(); - } - - public void callStr(@Nullable Eradicate x) { - str(x); // Error: Eradicate parameter not nullable - } - - public String shouldNotReturnNullBecauseNotAnnotated() { - return null; // Error: Eradicate return not nullable - } - - public void redundant() { - String s = new String("abc"); - if (s != null) { // Error: Eradicate condition redundant - int n = s.length(); - } - } - - @Nullable - public static String someMethod() { - return ""; // Error: Eradicate return overannotated - } -} - -class B extends Eradicate { - @Nullable public String shouldNotReturnNullBecauseNotAnnotated() { - return null; // Error: Eradicate inconsistent subclass return annotation - } - - public void field(Eradicate x) {} // Error: Inconsistent subclass parameter annotation -} diff --git a/tools/infer/test/autotest/src/main/java/Racerd.java b/tools/infer/test/autotest/src/main/java/Racerd.java deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Racerd.java +++ /dev/null @@ -1,40 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import javax.annotation.concurrent.ThreadSafe; - -// Examples taken from the infer website. -@ThreadSafe -public class Racerd { - private int mTemperature; - - public void makeDinner() { - boilWater(); - } - - private void boilWater() { - mTemperature = 100; //Error: unprotected write. - } -} - -@ThreadSafe -class Account { - - int mBalance = 0; - - public void deposit(int amount) { - if (amount > 0) { - mBalance += amount; // Error: unsynchronized write - } - } - - public int withdraw(int amount){ - if (amount >= 0 && mBalance - amount >= 0) { - mBalance -= amount; // Error: unsynchronized write - return mBalance; // Error: unsynchronized read - } else { - return 0; - } - } -} diff --git a/tools/infer/test/autotest/src/main/java/Racerd.json b/tools/infer/test/autotest/src/main/java/Racerd.json deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Racerd.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "access": "hJWmvgAAAJMAAAApAAAAfgAAAHSwkNAnZGVwb3NpdKCgQCNpbnRAk6AnQWNjb3VudECQoEAkdm9pZECgoJKgIECwXAD/kgkiYXV0b3Rlc3Qvc3JjL21haW4vamF2YS9SYWNlcmQuamF2YZGgoJGwAjafTBegJHRoaXNAkAQcoKOglJOgBBpAsEBAQEAEAaCRkTBBY2NvdW50Lm1CYWxhbmNlQKAEF0A=", - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "access to `this.Account.mBalance`", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 28 - } - ], - "bug_type": "THREAD_SAFETY_VIOLATION", - "bug_type_hum": "Thread Safety Violation", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Racerd.java", - "hash": "6b62cb17008a3135d218108fa3123402", - "key": "Racerd.java|deposit|THREAD_SAFETY_VIOLATION", - "kind": "ERROR", - "line": 28, - "node_key": "9c5d6d9028928346cc4fb44cced5dea1", - "procedure": "void Account.deposit(int)", - "procedure_id": "Account.deposit(int):void.a9cc1805c1e3652887a5ee12b55803af", - "procedure_start_line": 0, - "qualifier": "Unprotected write. Non-private method `void Account.deposit(int)` writes to field `this.Account.mBalance` outside of synchronization.\n Reporting because the current class is annotated `@ThreadSafe`, so we assume that this method can run in parallel with other non-private methods in the class (including itself).", - "severity": "HIGH", - "visibility": "user" - }, - { - "access": "hJWmvgAAAKYAAAApAAAAhQAAAHqwkNAqbWFrZURpbm5lckCToCZSYWNlcmRAkKBAJHZvaWRAoKCQ0Clib2lsV2F0ZXJAk6AEC0AECkCwTQD/kgkiYXV0b3Rlc3Qvc3JjL21haW4vamF2YS9SYWNlcmQuamF2YZGgoJGwAjafTBegJHRoaXNAkAQboKOglJOgBBxAsEBAQEAEAaCRkTNSYWNlcmQubVRlbXBlcmF0dXJlQKCwUQD/BBdA", - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "call to void Racerd.boilWater()", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 13 - }, - { - "column_number": -1, - "description": "access to `this.Racerd.mTemperature`", - "filename": "autotest/src/main/java/Racerd.java", - "level": 1, - "line_number": 17 - } - ], - "bug_type": "THREAD_SAFETY_VIOLATION", - "bug_type_hum": "Thread Safety Violation", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Racerd.java", - "hash": "2882383086ab102a88144ae3c2cc4701", - "key": "Racerd.java|makeDinner|THREAD_SAFETY_VIOLATION", - "kind": "ERROR", - "line": 13, - "node_key": "9c5d6d9028928346cc4fb44cced5dea1", - "procedure": "void Racerd.makeDinner()", - "procedure_id": "Racerd.makeDinner():void.2796f75396b30d2d49b24ddfab722306", - "procedure_start_line": 0, - "qualifier": "Unprotected write. Non-private method `void Racerd.makeDinner()` indirectly writes to field `this.Racerd.mTemperature` outside of synchronization.\n Reporting because the current class is annotated `@ThreadSafe`, so we assume that this method can run in parallel with other non-private methods in the class (including itself).", - "severity": "HIGH", - "visibility": "user" - }, - { - "access": "hJWmvgAAAJgAAAAqAAAAgwAAAHqwkNAod2l0aGRyYXegoEAjaW50QJOgJ0FjY291bnRAkKBABAZAoKCSoCBAsGMA/5IJImF1dG90ZXN0L3NyYy9tYWluL2phdmEvUmFjZXJkLmphdmGQoKCRsAI2n0wXoCR0aGlzQJAEG6CjoJSToAQZQLBAQEBABAGgkZEwQWNjb3VudC5tQmFsYW5jZUCgBBegsGIA/wQYQA==", - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 35 - }, - { - "column_number": -1, - "description": "access to `this.Account.mBalance`", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 35 - }, - { - "column_number": -1, - "description": "", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 34 - }, - { - "column_number": -1, - "description": "access to `this.Account.mBalance`", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 34 - } - ], - "bug_type": "THREAD_SAFETY_VIOLATION", - "bug_type_hum": "Thread Safety Violation", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Racerd.java", - "hash": "5665f12d2392f93f11f556cd1b1e238a", - "key": "Racerd.java|withdraw|THREAD_SAFETY_VIOLATION", - "kind": "ERROR", - "line": 35, - "node_key": "9c5d6d9028928346cc4fb44cced5dea1", - "procedure": "int Account.withdraw(int)", - "procedure_id": "Account.withdraw(int):int.038de5054c5c25e60d169e42e0177a16", - "procedure_start_line": 0, - "qualifier": "Read/Write race. Non-private method `int Account.withdraw(int)` reads without synchronization from `this.Account.mBalance`. Potentially races with write in method `Account.withdraw(...)`.\n Reporting because the current class is annotated `@ThreadSafe`, so we assume that this method can run in parallel with other non-private methods in the class (including itself).", - "severity": "HIGH", - "visibility": "user" - }, - { - "access": "hJWmvgAAAJEAAAAoAAAAfAAAAHOwkNAod2l0aGRyYXegoEAjaW50QJOgJ0FjY291bnRAkKBABAZAoKCSoCBAsGIA/5IJImF1dG90ZXN0L3NyYy9tYWluL2phdmEvUmFjZXJkLmphdmGRoKCRsAI2n0wXoCR0aGlzQJAEG6CjoJSToAQZQLBAQEBABAGgkZEwQWNjb3VudC5tQmFsYW5jZUCgBBdA", - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "access to `this.Account.mBalance`", - "filename": "autotest/src/main/java/Racerd.java", - "level": 0, - "line_number": 34 - } - ], - "bug_type": "THREAD_SAFETY_VIOLATION", - "bug_type_hum": "Thread Safety Violation", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Racerd.java", - "hash": "a7c30fd1b251d9e16750fc7e5913b885", - "key": "Racerd.java|withdraw|THREAD_SAFETY_VIOLATION", - "kind": "ERROR", - "line": 34, - "node_key": "9c5d6d9028928346cc4fb44cced5dea1", - "procedure": "int Account.withdraw(int)", - "procedure_id": "Account.withdraw(int):int.038de5054c5c25e60d169e42e0177a16", - "procedure_start_line": 0, - "qualifier": "Unprotected write. Non-private method `int Account.withdraw(int)` writes to field `this.Account.mBalance` outside of synchronization.\n Reporting because the current class is annotated `@ThreadSafe`, so we assume that this method can run in parallel with other non-private methods in the class (including itself).", - "severity": "HIGH", - "visibility": "user" - } -] diff --git a/tools/infer/test/autotest/src/main/java/Starvation.java b/tools/infer/test/autotest/src/main/java/Starvation.java deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Starvation.java +++ /dev/null @@ -1,25 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -// Examples taken from the infer website. -public class Starvation { - - String lockA, lockB; - - public void lockAThenB() { - synchronized(lockA) { - synchronized(lockB) { - // do something with both resources - } - } - } - - public void lockBThenA() { - synchronized(lockB) { - synchronized(lockA) { - // do something with both resources - } - } - } -} diff --git a/tools/infer/test/autotest/src/main/java/Starvation.json b/tools/infer/test/autotest/src/main/java/Starvation.json deleted file mode 100644 --- a/tools/infer/test/autotest/src/main/java/Starvation.json +++ /dev/null @@ -1,65 +0,0 @@ -[ - { - "bug_class": "PROVER", - "bug_trace": [ - { - "column_number": -1, - "description": "[Trace 1] `void Starvation.lockAThenB()`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 0, - "line_number": 11 - }, - { - "column_number": -1, - "description": "locks `this.Starvation.lockA` in class `Starvation*`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 0, - "line_number": 11 - }, - { - "column_number": -1, - "description": "locks `this.Starvation.lockB` in class `Starvation*`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 1, - "line_number": 12 - }, - { - "column_number": -1, - "description": "[Trace 2] `void Starvation.lockBThenA()`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 0, - "line_number": 19 - }, - { - "column_number": -1, - "description": "locks `this.Starvation.lockB` in class `Starvation*`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 0, - "line_number": 19 - }, - { - "column_number": -1, - "description": "locks `this.Starvation.lockA` in class `Starvation*`", - "filename": "autotest/src/main/java/Starvation.java", - "level": 1, - "line_number": 20 - } - ], - "bug_type": "DEADLOCK", - "bug_type_hum": "Deadlock", - "censored_reason": "", - "column": -1, - "file": "autotest/src/main/java/Starvation.java", - "hash": "043d28a94431b4c573b949b8570fb318", - "key": "Starvation.java|lockAThenB|DEADLOCK", - "kind": "ERROR", - "line": 11, - "node_key": "9c5d6d9028928346cc4fb44cced5dea1", - "procedure": "void Starvation.lockAThenB()", - "procedure_id": "Starvation.lockAThenB():void.b7eb3955306c498af42d6336f52a796f", - "procedure_start_line": 0, - "qualifier": "Potential deadlock.\nTrace 1 (starts at `void Starvation.lockAThenB()`) first locks `this.Starvation.lockA` in class `Starvation*` (line 11 in `void Starvation.lockAThenB()`) and then locks `this.Starvation.lockB` in class `Starvation*` (line 12 in `void Starvation.lockAThenB()`).\nTrace 2 (starts at `void Starvation.lockBThenA()`), first locks `this.Starvation.lockB` in class `Starvation*` (line 19 in `void Starvation.lockBThenA()`) and then locks `this.Starvation.lockA` in class `Starvation*` (line 20 in `void Starvation.lockBThenA()`).", - "severity": "HIGH", - "visibility": "user" - } -] diff --git a/tools/infer/test/build.gradle b/tools/infer/test/build.gradle deleted file mode 100644 --- a/tools/infer/test/build.gradle +++ /dev/null @@ -1,6 +0,0 @@ -allprojects { - // Expose the per-object-directory configuration to all projects. - ext { - topobjdir = gradle.mozconfig.topobjdir - } -} diff --git a/tools/infer/test/gradle.properties b/tools/infer/test/gradle.properties deleted file mode 100644 --- a/tools/infer/test/gradle.properties +++ /dev/null @@ -1,3 +0,0 @@ -org.gradle.parallel=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2560M diff --git a/tools/infer/test/gradle/wrapper/gradle-wrapper.jar b/tools/infer/test/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 01b8bf6b1f99cad9213fc495b33ad5bbab8efd20..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@ \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/tools/infer/test/gradlew.bat b/tools/infer/test/gradlew.bat deleted file mode 100644 --- a/tools/infer/test/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/tools/infer/test/moz.configure b/tools/infer/test/moz.configure deleted file mode 100644 --- a/tools/infer/test/moz.configure +++ /dev/null @@ -1,15 +0,0 @@ -# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -@depends(build_project) -def java_include(build_project): - # if mobile/android is enabled then the moz.configure from ./mobile - # will include java - if build_project == 'mobile/android': - return None - return '../../../build/moz.configure/java.configure' - -include(java_include) diff --git a/tools/infer/test/settings.gradle b/tools/infer/test/settings.gradle deleted file mode 100644 --- a/tools/infer/test/settings.gradle +++ /dev/null @@ -1,24 +0,0 @@ -// Check out root/settings.gradle for more information - -rootProject.name = 'infer' - -def topsrcdir = rootProject.projectDir.absolutePath + '/../../..' - -def commandLine = ["${topsrcdir}/mach", "environment", "--format", "json", "--verbose"] -def proc = commandLine.execute(null, new File(topsrcdir)) -def standardOutput = new ByteArrayOutputStream() -proc.consumeProcessOutput(standardOutput, standardOutput) -proc.waitFor() - -if (proc.exitValue() != 0) { - throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${proc.exitValue()}:\n\n${standardOutput.toString()}") -} - -import groovy.json.JsonSlurper -def slurper = new JsonSlurper() -def json = slurper.parseText(standardOutput.toString()) - -gradle.ext.mozconfig = json - -include 'autotest' -project(':autotest').projectDir = new File("${json.topsrcdir}/tools/infer/test/autotest")