flutter_flutter/tools/webkitpy/tool/commands/flakytests.py

# Copyright (c) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import optparse
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
from webkitpy.layout_tests.models.test_expectations import TestExpectationParser, TestExpectationsModel, TestExpectations
from webkitpy.layout_tests.port import builders
from webkitpy.common.net import sheriff_calendar


class FlakyTests(AbstractDeclarativeCommand):
    name = "update-flaky-tests"
    help_text = "Update FlakyTests file from the flakiness dashboard"
    show_in_main_help = True

    ALWAYS_CC = [
        'ojan@chromium.org',
        'dpranke@chromium.org',
        'eseidel@chromium.org',
    ]

    COMMIT_MESSAGE = (
        'Update FlakyTests to match current flakiness dashboard results\n\n'
        'Automatically generated using:\n'
        'webkit-patch update-flaky-tests\n\n'
        'R=%s\n')

    FLAKY_TEST_CONTENTS = (
        '# This file is generated by webkit-patch update-flaky-tests from the flakiness dashboard data.\n'
        '# Manual changes will be overwritten.\n\n'
        '%s\n')

    def __init__(self):
        options = [
            optparse.make_option('--upload', action='store_true',
                help='upload the changed FlakyTest file for review'),
            optparse.make_option('--reviewers', action='store',
                help='comma-separated list of reviewers, defaults to blink gardeners'),
        ]
        AbstractDeclarativeCommand.__init__(self, options=options)
        # This is sorta silly, but allows for unit testing:
        self.expectations_factory = BotTestExpectationsFactory

    def _collect_expectation_lines(self, builder_names, factory):
        all_lines = []
        for builder_name in builder_names:
            model = TestExpectationsModel()
            expectations = factory.expectations_for_builder(builder_name)
            for line in expectations.expectation_lines(only_ignore_very_flaky=True):
                model.add_expectation_line(line)
            # FIXME: We need an official API to get all the test names or all test lines.
            all_lines.extend(model._test_to_expectation_line.values())
        return all_lines

    def _commit_and_upload(self, tool, options):
        files = tool.scm().changed_files()
        flaky_tests_path = 'tests/FlakyTests'
        if flaky_tests_path not in files:
            print "%s is not changed, not uploading." % flaky_tests_path
            return 0

        if options.reviewers:
            # FIXME: Could validate these as emails. sheriff_calendar has some code for that.
            reviewer_emails = options.reviewers.split(',')
        else:
            reviewer_emails = sheriff_calendar.current_gardener_emails()
            if not reviewer_emails:
                print "No gardener, and --reviewers not specified, not bothering."
                return 1

        commit_message = self.COMMIT_MESSAGE % ','.join(reviewer_emails)
        git_cmd = ['git', 'commit', '-m', commit_message,
            tool.filesystem.join(tool.scm().checkout_root, flaky_tests_path)]
        tool.executive.run_and_throw_if_fail(git_cmd)

        git_cmd = ['git', 'cl', 'upload', '--send-mail', '-f',
            '--cc', ','.join(self.ALWAYS_CC)]
        tool.executive.run_and_throw_if_fail(git_cmd)

    def execute(self, options, args, tool):
        factory = self.expectations_factory()

        # FIXME: WebKit Linux 32 and WebKit Linux have the same specifiers;
        # if we include both of them, we'll get duplicate lines. Ideally
        # Linux 32 would have unique speicifiers.
        most_builders = builders.all_builder_names()
        if 'WebKit Linux 32' in most_builders:
            most_builders.remove('WebKit Linux 32')

        lines = self._collect_expectation_lines(most_builders, factory)
        lines.sort(key=lambda line: line.path)

        port = tool.port_factory.get()
        # Skip any tests which are mentioned in the dashboard but not in our checkout:
        fs = tool.filesystem
        lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines)

        # Note: This includes all flaky tests from the dashboard, even ones mentioned
        # in existing TestExpectations. We could certainly load existing TestExpecations
        # and filter accordingly, or update existing TestExpectations instead of FlakyTests.
        flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests')
        flaky_tests_contents = self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines)
        fs.write_text_file(flaky_tests_path, flaky_tests_contents)
        print "Updated %s" % flaky_tests_path

        if options.upload:
            return self._commit_and_upload(tool, options)