diff --git a/.taskcluster.yml b/.taskcluster.yml --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -8,11 +8,11 @@ # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering # takes place. - $let: taskgraph: branch: taskgraph - revision: d302e95cc988706ba746bfecbf1dae31090d212f + revision: 2b32db7d303dd175ed4c4abf5fcfe00c6c593acd trustDomain: ci in: $if: 'tasks_for in ["hg-push", "action", "cron"]' then: $let: @@ -48,11 +48,11 @@ description: '${action.description}' else: name: "Decision Task for cron job ${cron.job_name}" description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})' - provisionerId: "ci-${repository.level}" + provisionerId: "${trustDomain}-${repository.level}" workerType: "decision" tags: $if: 'tasks_for == "hg-push"' then: @@ -119,17 +119,30 @@ payload: env: # run-task uses these to check out the source; the inputs # to `mach taskgraph decision` are all on the command line. $merge: - - CI_BASE_REPOSITORY: '${repoUrl}' - CI_HEAD_REPOSITORY: '${repoUrl}' - CI_HEAD_REF: '${push.revision}' - CI_HEAD_REV: '${push.revision}' - CI_REPOSITORY_TYPE: hg - REPOSITORIES: {$json: {ci: ci-admin}} - HG_STORE_PATH: /builds/worker/checkouts/hg-store + - $if: 'repository.project[:8] == "ci-admin"' + then: + ADMIN_BASE_REPOSITORY: '${repoUrl}' + ADMIN_HEAD_REPOSITORY: '${repoUrl}' + ADMIN_HEAD_REF: '${push.revision}' + ADMIN_HEAD_REV: '${push.revision}' + ADMIN_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {admin: ci-admin}} + else: + CONFIG_BASE_REPOSITORY: '${repoUrl}' + CONFIG_HEAD_REPOSITORY: '${repoUrl}' + CONFIG_HEAD_REF: '${push.revision}' + CONFIG_HEAD_REV: '${push.revision}' + CONFIG_REPOSITORY_TYPE: hg + ADMIN_BASE_REPOSITORY: 'https://hg.mozilla.org/ci/ci-admin' + ADMIN_HEAD_REPOSITORY: 'https://hg.mozilla.org/ci/ci-admin-try' + ADMIN_HEAD_REF: 'default' + ADMIN_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {admin: ci-admin}} + - HG_STORE_PATH: /builds/worker/checkouts/hg-store - $if: 'tasks_for == "action"' then: ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) ACTION_INPUT: {$json: {$eval: 'input'}} @@ -154,11 +167,11 @@ maxRunTime: 1800 command: - /usr/local/bin/run-task - - '--ci-checkout=/builds/worker/checkouts/src' + - '--admin-checkout=/builds/worker/checkouts/src' - '--' - bash - -cx - $let: extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} diff --git a/src/ciadmin/boot.py b/src/ciadmin/boot.py --- a/src/ciadmin/boot.py +++ b/src/ciadmin/boot.py @@ -4,10 +4,12 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at http://mozilla.org/MPL/2.0/. import os +import click + from ciadmin import modify from ciadmin.generate import ( cron_tasks, grants, hg_pushes, @@ -41,10 +43,11 @@ appconfig.options.add( "--ci-configuration-directory", help="local directory containing ci-configuration repository " "(overrides repository/revision)", + type=click.Path(resolve_path=True), ) appconfig.check_path = os.path.join(os.path.dirname(__file__), "check") appconfig.generators.register(scm_group_roles.update_resources) diff --git a/src/ciadmin/check/check_hg_push_scopes.py b/src/ciadmin/check/check_hg_push_scopes.py --- a/src/ciadmin/check/check_hg_push_scopes.py +++ b/src/ciadmin/check/check_hg_push_scopes.py @@ -8,10 +8,14 @@ from tcadmin.resources import Hook, Role from tcadmin.util.scopes import satisfies +def test_fail(self): + assert False + + @pytest.fixture(scope="module") def ci_group_roles(generated, generated_resolver): """ A dictionary mapping ci-group roles (project:releng:ci-group:..) to their expanded scopes diff --git a/src/ciadmin/generate/ciconfig/projects.py b/src/ciadmin/generate/ciconfig/projects.py --- a/src/ciadmin/generate/ciconfig/projects.py +++ b/src/ciadmin/generate/ciconfig/projects.py @@ -49,10 +49,11 @@ trust_project = attr.ib(type=str, default=None) parent_repo = attr.ib(type=str, default=None) is_try = attr.ib(type=bool, default=False) features = attr.ib(type=dict, factory=lambda: {}) cron = attr.ib(type=dict, factory=lambda: {}) + taskcluster_yml_project = attr.ib(type=str, default=None) def __attrs_post_init__(self): """ Once the object is initialised, perform more sanity checks to ensure the values received are sane together @@ -93,10 +94,20 @@ async def fetch_all(): """Load project metadata from projects.yml in ci-configuration""" projects = await get_ciconfig_file("projects.yml") return [Project(alias, **info) for alias, info in projects.items()] + @staticmethod + async def get(alias): + projects = await Project.fetch_all() + + for project in projects: + if project.alias == alias: + return project + else: + raise KeyError("Project {} is not defined".format(alias)) + # The `features` property is designed for ease of use in yaml, with true and false # values for each feature; the `feature()` and `enabled_features` attributes provide # easier access for Python uses. def feature(self, feature): diff --git a/src/ciadmin/generate/hg_pushes.py b/src/ciadmin/generate/hg_pushes.py --- a/src/ciadmin/generate/hg_pushes.py +++ b/src/ciadmin/generate/hg_pushes.py @@ -29,10 +29,15 @@ "hookGroupId": hookGroupId, "hookId": hookId, "project_repo": project.repo, "project_role_prefix": project.role_prefix, "alias": project.alias, + "taskcluster_yml_repo": ( + (await Project.get(project.taskcluster_yml_project)).repo + if project.taskcluster_yml_project + else None + ), }, ) return Hook( hookGroupId=hookGroupId, diff --git a/src/tcadmin/appconfig.py b/src/tcadmin/appconfig.py --- a/src/tcadmin/appconfig.py +++ b/src/tcadmin/appconfig.py @@ -43,11 +43,11 @@ self.option_args = {} # mapping of `--foo` to the name click selected for it self.option_names = {} self.name = name - def add(self, name, *, required=False, help=None, default=None): + def add(self, name, *, required=False, help=None, default=None, type=None): """ Register a command-line option. Once options are parsed, the value will be available from the `get` method. """ @@ -57,10 +57,12 @@ kwargs["required"] = required if help: kwargs["help"] = help if default: kwargs["default"] = default + if type: + kwargs["type"] = type self.option_args[name] = kwargs # get click to generate a name for this opt = click.Option((name,), **kwargs) self.option_names[name] = opt.name diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -7,12 +7,22 @@ task-priority: low taskgraph: repositories: - ci: + admin: name: ci-admin + project-regex: ci-admin.* + default-repository: https://hg.mozilla.org/ci/ci-admin-try + default-ref: default + type: hg + config: + name: ci-configuration + project-regex: ci-configuration.* + default-repository: https://hg.mozilla.org/ci/ci-configuration + default-ref: default + type: hg workers: aliases: images: provisioner: 'ci-{level}' diff --git a/taskcluster/ci/tests/kind.yml b/taskcluster/ci/tests/kind.yml --- a/taskcluster/ci/tests/kind.yml +++ b/taskcluster/ci/tests/kind.yml @@ -8,44 +8,64 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms job-defaults: - attributes: {} + attributes: + code-review: true worker-type: t-linux + run-on-projects: + - ci-admin + - ci-admin-try worker: docker-image: {in-tree: python3.7} max-run-time: 3600 treeherder: kind: test platform: tests/opt tier: 1 run: cwd: '{checkout}' using: run-task + checkout: + admin: {} jobs: check-staging: description: "Run `ci-admin check` against the staging cluster to validate the latest changes" treeherder: symbol: check(staging) + run-on-projects: + - ci-configuration + - ci-configuration-try run: + checkout: + admin: {} + config: + path: config command: >- pip install --user -r requirements/base.txt && pip install --user . && TASKCLUSTER_ROOT_URL=https://stage.taskcluster.nonprod.cloudops.mozgcp.net - ci-admin check --environment staging + ci-admin check --environment staging --ci-configuration-directory config check-firefoxci: description: "Run `ci-admin check` against the firefoxci cluster to validate the latest changes" treeherder: symbol: check(firefoxci) + run-on-projects: + - ci-configuration + - ci-configuration-try run: + checkout: + admin: {} + config: + path: config command: >- pip install --user -r requirements/base.txt && pip install --user . && TASKCLUSTER_ROOT_URL=https://firefox-ci-tc.services.mozilla.com - ci-admin check --environment firefoxci + ci-admin check --environment firefoxci --ci-configuration-directory config unit: description: "Run `unit tests` to validate the latest changes" treeherder: symbol: unit worker: