diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml --- a/taskcluster/ci/source-test/kind.yml +++ b/taskcluster/ci/source-test/kind.yml @@ -29,6 +29,7 @@ - python.yml - remote.yml - shadow-scheduler.yml + - structdoc.yml - webidl.yml - wpt-manifest.yml - wpt-metadata.yml diff --git a/taskcluster/ci/source-test/node.yml b/taskcluster/ci/source-test/node.yml --- a/taskcluster/ci/source-test/node.yml +++ b/taskcluster/ci/source-test/node.yml @@ -29,8 +29,6 @@ cd /builds/worker/checkouts/gecko/browser/components/newtab && npm install && node bin/try-runner.js - attributes: - code-review: true when: files-changed: - "browser/components/newtab/**" @@ -55,8 +53,6 @@ cd devtools/client/debugger/ && yarn && node bin/try-runner.js - attributes: - code-review: true when: files-changed: - 'devtools/client/debugger/**' @@ -82,7 +78,6 @@ node devtools-node-test-runner.js --suite=aboutdebugging && node devtools-node-test-runner.js --suite=accessibility && node devtools-node-test-runner.js --suite=application && - node devtools-node-test-runner.js --suite=compatibility && node devtools-node-test-runner.js --suite=framework && node devtools-node-test-runner.js --suite=netmonitor && node devtools-node-test-runner.js --suite=webconsole diff --git a/taskcluster/ci/source-test/structdoc.yml b/taskcluster/ci/source-test/structdoc.yml new file mode 100644 --- /dev/null +++ b/taskcluster/ci/source-test/structdoc.yml @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +structdocs-verify: + description: Validates Performance Testing documentation + attributes: + code-review: true + platform: lint/opt + treeherder: + symbol: text(structdocs-verify) + kind: test + tier: 2 + worker-type: t-linux-xlarge + run-on-projects: ['try'] + run: + using: run-task + cwd: '{checkout}' + command: >- + ./mach python -m pip install jsonschema && + ./mach python testing/structdocs/structdocs/structdocs.py --verify + worker: + docker-image: {in-tree: "lint"} + env: + MOZ_UPLOAD_DIR: /builds/worker/ + WORKSPACE: /builds/worker/checkouts/gecko/ + max-run-time: 1800 + artifacts: + - type: file + name: public/code-review/issues.json + path: /builds/worker/issues.json + when: + files-changed: + - 'testing/raptor/**/*.ini' diff --git a/testing/raptor/raptor/structdoc/config.yml b/testing/raptor/raptor/structdoc/config.yml new file mode 100755 --- /dev/null +++ b/testing/raptor/raptor/structdoc/config.yml @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +name: raptor +manifest: testing/raptor/raptor/raptor.ini +tests: + desktop: "Used for testing page load performance." + mobile: "Used for testing page load performance on Android." + raptor-webaudio: "Web audio tests." + raptor-not-a-test: "Hello" + wasm: "Wasm tests..." diff --git a/testing/raptor/raptor/structdoc/index.rst b/testing/raptor/raptor/structdoc/index.rst new file mode 100755 --- /dev/null +++ b/testing/raptor/raptor/structdoc/index.rst @@ -0,0 +1,9 @@ +###### +Raptor +###### + +The following documents all testing we have for raptor. + +{documentation} + +The methods for calling the tests can be found below. diff --git a/testing/structdocs/structdocs/__init__.py b/testing/structdocs/structdocs/__init__.py new file mode 100644 diff --git a/testing/structdocs/structdocs/gatherer.py b/testing/structdocs/structdocs/gatherer.py new file mode 100644 --- /dev/null +++ b/testing/structdocs/structdocs/gatherer.py @@ -0,0 +1,164 @@ +from __future__ import absolute_import + +import glob +import os +import re +import yaml + +from manifestparser import TestManifest +from utils import StructDocLogger + +logger = StructDocLogger() + +frameworks = {} + + +class RaptorGatherer(object): + def __init__(self, yaml_path, workspace_dir): + self._yaml_path = yaml_path + self.workspace_dir = workspace_dir + self._suite_list = {} + self._manifest_path = '' + self._manifest = None + + def get_manifest_path(self): + if self._manifest_path: + return self._manifest_path + else: + yaml_content = open_yaml(self._yaml_path) + self._manifest_path = os.path.join( + self.workspace_dir, yaml_content["manifest"] + ) + return self._manifest_path + + def get_suite_list(self): + if self._suite_list: + return self._suite_list + else: + manifest_path = self.get_manifest_path() + test_manifest = TestManifest([manifest_path], strict=False) + test_list = test_manifest.active_tests(exists=False, disabled=False) + for test in test_list: + # Get the top-level suite + s = os.path.basename(test["here"]) + if s not in self._suite_list: + self._suite_list[s] = [] + + # Get the individual test + mpath = test['manifest'] + fpath = mpath.split('testing')[1:] + fpath = ['testing'] + [os.path.sep.join(fpath).lstrip('/')] + fpath = os.path.sep.join(fpath).lstrip('/') + + if fpath not in self._suite_list[s]: + self._suite_list[s].append(fpath) + + return self._suite_list + + +frameworks["raptor"] = RaptorGatherer + + +def open_file(path): + with open(path, 'r') as f: + content = f.read() + return content + + +def open_yaml(yaml_path): + '''Opens a YAML file and returns the contents.''' + contents = "" + try: + with open(yaml_path, 'r') as f: + logger.log("yaml_path: {}".format(yaml_path)) + contents = yaml.safe_load(f) + except Exception as e: + logger.warning("Error opening file {}: {}".format(yaml_path, str(e)), yaml_path) + + return contents + + +def get_rst_content(full_path): + content = [] + with open(full_path, 'r') as f: + content = f.readlines() + return content + + +class Gatherer(object): + def __init__(self, root_dir, workspace_dir): + self.root_dir = root_dir # os.path + self.workspace_dir = workspace_dir + # structdocs_tree [{ + # "path": os.path, + # "yml": os.path, + # "rst": os.path + # }] + self._structdocs_tree = [] + self._test_list = [] + + @property + def structdocs_tree(self): + if self._structdocs_tree: + return self._structdocs_tree + else: + self.fetch_structdocs_tree() + return self._structdocs_tree + + def fetch_structdocs_tree(self): + yml_match = re.compile('^config.y(a)?ml$') + rst_match = re.compile('^index.rst$') + for dirpath, dirname, files in os.walk(self.root_dir): + # Walk through the testing directory tree + if dirpath.endswith('/structdoc'): + logger.log("dirpath ends with /structdoc") + matched = {"path": dirpath, "yml": "", "rst": ""} + for file in files: + # Add the yml/rst file to its key if re finds the searched file + if re.search(yml_match, file): + matched["yml"] = re.search(yml_match, file).string + if re.search(rst_match, file): + matched["rst"] = re.search(rst_match, file).string + # Append to structdocs if all the searched files were found + if all(matched.values()): + self._structdocs_tree.append(matched) + + logger.log("Found {} structdocs directories in {}" + .format(len(self._structdocs_tree), self.root_dir)) + + def get_test_list(self, sdt_entry): + ''' + Use a structdocs_tree entry to find the test list for + the framework that was found. + + :return: A framework info dictionary with fields: { + 'yml_path': Path to YAML, + 'yml_content': Content of YAML, + 'name': Name of framework, + 'test_list': Test list found for the framework + } + ''' + + # If it was computed before, return it + yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml']) + for entry in self._test_list: + if entry['yml_path'] == yaml_path: + return entry + + # Set up framework entry with meta data + yaml_content = open_yaml(yaml_path) + framework = { + 'yml_content': yaml_content, + 'yml_path': yaml_path, + 'name': yaml_content["name"], + } + + # Get and then store the frameworks tests + framework_gatherer = frameworks[framework["name"]]( + framework["yml_path"], + self.workspace_dir + ) + framework["test_list"] = framework_gatherer.get_suite_list() + + self._test_list.append(framework) + return framework diff --git a/testing/structdocs/structdocs/structdocs.py b/testing/structdocs/structdocs/structdocs.py new file mode 100644 --- /dev/null +++ b/testing/structdocs/structdocs/structdocs.py @@ -0,0 +1,107 @@ +#!/usr/bin/python3 -u +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' +This script downloads the latest chromium build (or a manually +defined version) for a given platform. It then uploads the build, +with the revision of the build stored in a REVISION file. +''' + +from __future__ import absolute_import, print_function + +import argparse +import glob +import sys +import os + +from verifier import Verifier + +from utils import StructDocLogger + +logger = StructDocLogger() + + +def create_parser(): + parser = argparse.ArgumentParser() + add_arg = parser.add_argument + + add_arg('-v', '--verify', action='store_true', dest="verify", + required=False, default=True, + help="Run verifier") + add_arg('-g', '--generate', action='store_true', dest="generate", + required=False, default=False, + help="Run the generator") + + return parser + + +def main(argv=sys.argv[1:]): + ''' + Build up performance testing documentation dynamically by combining + text data from YAML files that reside in `structdoc` folders + across the `testing` directory. Each directory is expected to have + an index.rst file along with YAMLs defining what needs to be added + to it. + + The YAML must also define the name of the "feature" that should be + used in the main index.rst for the performance testing documentation. + + The testing documentation list will be ordered alphabetically once + it's produced (to avoid unwanted shifts because of unordered dicts + and path searching). + + Note that the test name headings will be given the H4 (---) style so it + is suggested that you use H3 (===) style as the heading for your + test section. + + Usage: python testing/structdocs/structdocs/main.py -v V + ''' + parser = create_parser() + args = parser.parse_args(argv) + logger.log("processing args '{}'".format([arg for arg in args.__dict__]), level="Debug") + + UPLOAD_DIR = os.environ.get('MOZ_UPLOAD_DIR', None) + if not UPLOAD_DIR: + UPLOAD_DIR = os.environ.get('UPLOAD_DIR', None) + if not UPLOAD_DIR: + raise Exception( + "Cannot find MOZ_UPLOAD_DIR and UPLOAD_DIR environment variables." + ) + if not os.path.exists(UPLOAD_DIR): + try: + os.makedirs(UPLOAD_DIR, exist_ok=True) + except Exception as e: + raise Exception("Cannot create upload directory here: %s" % UPLOAD_DIR) + + # Set the directory for where the validation artifact + # will go to + StructDocLogger.UPLOAD_DIR = UPLOAD_DIR + + WORKSPACE_DIR = os.environ.get('WORKSPACE', None) + if not WORKSPACE_DIR: + raise Exception("Cannot find WORKSPACE environment variable.") + + testing_dir = os.path.join(WORKSPACE_DIR, 'testing') + if not os.path.exists(testing_dir): + raise Exception("Cannot locate testing directory at %s" % testing_dir) + + if args.verify: + verifier = Verifier(testing_dir, WORKSPACE_DIR) + verifier.validate_tree() + if args.generate: + raise NotImplementedError + + # Fail the run if the validation log is not + # empty + if StructDocLogger.VALIDATION_LOG: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/testing/structdocs/structdocs/utils.py b/testing/structdocs/structdocs/utils.py new file mode 100644 --- /dev/null +++ b/testing/structdocs/structdocs/utils.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import + +import glob +import json +import os +import sys +from mozlog.commandline import setup_logging + + +def save_file(path, data, extension="rst"): + with open("{}.{}".format(path, extension), 'w') as f: + f.write(data) + + +class StructDocLogger(object): + UPLOAD_DIR = None + VALIDATION_LOG = {} + + def __init__(self): + self.logger = setup_logging('struct-docs', {}, {'tbpl': sys.stdout}) + self.validation_file = 'issues.json' + + def log(self, message, level='Info'): + self.logger.info('dyna-docs {}: {}'.format(level.upper(), message)) + + def warning(self, msg, files): + if type(files) != list: + files = [files] + + for file in files: + fpath = file.split('testing')[1:] + fpath = ['testing'] + [os.path.sep.join(fpath).lstrip('/')] + fpath = os.path.sep.join(fpath).lstrip('/') + + if fpath not in StructDocLogger.VALIDATION_LOG: + StructDocLogger.VALIDATION_LOG[fpath] = [] + + StructDocLogger.VALIDATION_LOG[fpath].append({ + "path": fpath, + "level": "error", + "message": msg, + "analyzer": "structdocs-verify", + "linter": "structdocs-verify", + "line": -1, + "column": -1, + "rule": "Flawless docs." + }) + + # Save after each change + issue_path = os.path.join(StructDocLogger.UPLOAD_DIR, self.validation_file) + with open(issue_path, 'w') as f: + json.dump(StructDocLogger.VALIDATION_LOG, f, indent=4) + self.log("Wrote out {} to: {}".format(self.validation_file, issue_path)) + self.log("JSON contents: {}".format(str(StructDocLogger.VALIDATION_LOG))) + + self.log(msg, level='warning') diff --git a/testing/structdocs/structdocs/verifier.py b/testing/structdocs/structdocs/verifier.py new file mode 100644 --- /dev/null +++ b/testing/structdocs/structdocs/verifier.py @@ -0,0 +1,221 @@ +from __future__ import absolute_import + +import glob +import json +import jsonschema +import os +import re +from jsonschema import ValidationError + +from utils import StructDocLogger +from gatherer import Gatherer, get_rst_content, open_yaml + +logger = StructDocLogger() + +CONFIG_SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "manifest": {"type": "string"}, + "tests": { + "type": "object", + "properties": { + "suite_name": {"type": "string"}, + } + } + } +} + + +class Verifier(object): + def __init__(self, root_dir, workspace_dir): + self.workspace_dir = workspace_dir + self.test_descriptions = {} + self._test_list = None + self._gatherer = Gatherer(root_dir, workspace_dir) + self._gatherer.fetch_structdocs_tree() + self._structdocs_tree = self._gatherer.structdocs_tree + + @property + def suite_list(self): + return self._test_list + + def validate_descriptions(self, framework_info): + yaml_content = framework_info['yml_content'] + + # Check for any bad test names in the yaml config file + global_descriptions = [] + for test in yaml_content['tests']: + found = False + for suite, tests in framework_info["test_list"].items(): + if suite == test: + found = True + break + for t in tests: + tb = os.path.basename(t) + tb = '.'.join(tb.split('.')[:-1]) + if test == tb: + found = True + break + if test in tb: + global_descriptions.append(test) + found = True + break + if found: + break + if not found: + logger.warning( + "Could not find an existing test for {} - bad test name?".format(test), + framework_info["yml_path"] + ) + + # Check for any missing tests + missing = { + framework_info["name"]: { + "missing": [], + "present": [], + "yml_path": framework_info["yml_path"], + } + } + for suite, tests in framework_info["test_list"].items(): + if yaml_content["tests"].get(suite): + # Description exists for the full suite, ignore individual tests + continue + + tests_found = 0 + missing_tests = [] + found_tests = [] + for test in tests: + tb = os.path.basename(test) + tb = '.'.join(tb.split('.')[:-1]) + if yaml_content['tests'].get(tb) or yaml_content['tests'].get(test): + # Test description exists, continue with the next test + tests_found += 1 + found_tests.append(tb) + continue + missing_tests.append(tb) + + # If no tests have a description, set the error to be a missing + # suite. Otherwise, set the error to be the missing tests. + if not tests_found: + missing[framework_info["name"]]["missing"].append(suite) + continue + + # Check if global test descriptions exist (i.e. + # ones that cover all of tp6) for the missing tests + new_mtests = [] + for mt in missing_tests: + found = False + for test in global_descriptions: + if test in mt: + # Global test exists for this missing test + found = True + break + if not found: + new_mtests.append(mt) + + if len(new_mtests) > 0: + missing[framework_info["name"]]["missing"].extend(new_mtests) + continue + + # XXX: Not sure why we would want missing stuff in the test descriptions?? + self.test_descriptions.update(missing) + + if missing[framework_info["name"]]['missing']: + logger.warning("For framework {} we found {} tests/suites with missing descriptions: {}".format( + framework_info["name"], + len(missing[framework_info["name"]]["missing"]), + missing[framework_info["name"]]["missing"] + ), framework_info["yml_path"]) + + def validate_yaml(self, yaml_path): + ''' + Validate that the YAML file has all the fields that are + required. + ''' + valid = False + yaml_content = open_yaml(yaml_path) + + try: + jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) + + for test, desc in yaml_content['tests'].items(): + desc_path = os.path.join(self.workspace_dir, desc) + if not os.path.exists(desc_path): + logger.log( + "Description is not a file, using '%s' as a description for test %s" % + (desc, test) + ) + else: + logger.log( + "Description is a file, using it's contects " + "as a description for test %s" % test + ) + with open(desc_path, 'r') as f: + yaml_content['tests'][test] = f.readlines() + + valid = True + except Exception as e: + logger.warning( + "YAML ValidationError: {}".format(str(e)), yaml_path + ) + + return valid + + def validate_rst_content(self, rst_path): + ''' + Validate that the index file given has a {tests} entry + so that the documentation can be inserted there. + ''' + rst_content = get_rst_content(rst_path) + # Check for a {documentation} entry in some line + valid = False + docs_match = re.compile('.*{documentation}.*') + for line in rst_content: + if docs_match.search(line): + valid = True + break + + if not valid: + logger.warning( + "Cannot find a '{documentation}' entry in the given index file", + rst_path + ) + + return valid + + def validate_tree(self): + ''' + Validate the `structdoc` directory that was found. + Returns True if it is good, false otherwise. + ''' + found_good = 0 + + # For each framework, check their definitions and validate + # descriptions + test_list = None + for matched in self._gatherer.structdocs_tree: + matched_yml = os.path.join(matched['path'], matched['yml']) + matched_rst = os.path.join(matched['path'], matched['rst']) + + valid_yml = self.validate_yaml(matched_yml) + valid_rst = self.validate_rst_content(matched_rst) + + if not (valid_yml and valid_rst): + logger.log("Bad structdocs directory found in {}".format(matched['path'])) + continue + found_good += 1 + + framework_info = self._gatherer.get_test_list(matched) + self.validate_descriptions(framework_info) + + if not found_good: + raise Exception("No valid structdocs directories found") + + @property + def structdocs_tree(self): + return self._structdocs_tree + + @structdocs_tree.setter + def structdocs_tree(self, value): + self._structdocs_tree = value