From 8971e980cd0a9b24de59f15f647705e527a719df Mon Sep 17 00:00:00 2001 From: Ojan Vafai Date: Thu, 11 Dec 2014 15:24:02 -0800 Subject: [PATCH] Prepare python code for reftests. -s/-expected.html/-expected.sky/ -s/-expected-mismatch.html/-expected-mismatch.sky/ -In single_test_runner.py, get rid of the image hash checking for reftests. It doesn't provide any value and just makes getting reftests working more complicated. R=eseidel@chromium.org Review URL: https://codereview.chromium.org/802573002 --- tests/resources/results.html | 52 +++++++++---------- .../controllers/single_test_runner.py | 19 +++---- .../test_result_writer_unittest.py | 12 ++--- .../models/test_expectations_unittest.py | 4 +- .../models/test_run_results_unittest.py | 2 +- .../layout_tests/port/base_unittest.py | 6 +-- tools/webkitpy/layout_tests/port/test.py | 22 ++++---- .../layout_tests/run_webkit_tests_unittest.py | 8 +-- .../tool/commands/rebaseline_unittest.py | 2 +- 9 files changed, 61 insertions(+), 66 deletions(-) diff --git a/tests/resources/results.html b/tests/resources/results.html index db1f73207ff..6797a8185a2 100644 --- a/tests/resources/results.html +++ b/tests/resources/results.html @@ -331,14 +331,14 @@ function expandExpectations(expandLink) var row = parentOfType(expandLink, 'tr'); var parentTbody = row.parentNode; var existingResultsRow = parentTbody.querySelector('.results-row'); - + var enDash = '\u2013'; expandLink.textContent = enDash; if (existingResultsRow) { updateExpandedState(existingResultsRow, true); return; } - + var newRow = document.createElement('tr'); newRow.className = 'results-row'; var newCell = document.createElement('td'); @@ -355,7 +355,7 @@ function expandExpectations(expandLink) } else result = resultIframe(link.href); - appendHTML(newCell, result); + appendHTML(newCell, result); } newRow.appendChild(newCell); @@ -466,7 +466,7 @@ function processGlobalStateFor(testObject) globalState().hasHttpTests = globalState().hasHttpTests || test.indexOf('http/') == 0; - var actual = testObject.actual; + var actual = testObject.actual; var expected = testObject.expected || 'PASS'; if (actual == 'MISSING') { @@ -504,7 +504,7 @@ function processGlobalStateFor(testObject) globalState().timeoutTests.push(testObject); return; } - + globalState().failingTests.push(testObject); } @@ -547,11 +547,11 @@ function imageResultsCell(testObject, testPrefix, actual) { globalState().hasImageFailures = true; if (testObject.reftest_type && testObject.reftest_type.indexOf('!=') != -1) { - row += resultLink(testPrefix, '-expected-mismatch.html', 'ref mismatch html'); + row += resultLink(testPrefix, '-expected-mismatch.sky', 'ref mismatch html'); row += resultLink(testPrefix, '-actual.png', 'actual'); } else { if (testObject.reftest_type && testObject.reftest_type.indexOf('==') != -1) { - row += resultLink(testPrefix, '-expected.html', 'ref html'); + row += resultLink(testPrefix, '-expected.sky', 'ref html'); } if (globalState().shouldToggleImages) { row += resultLink(testPrefix, '-diffs.html', 'images'); @@ -571,7 +571,7 @@ function imageResultsCell(testObject, testPrefix, actual) { } function tableRow(testObject) -{ +{ var row = ' ref test hashes matched but diff failed" % self._test_name) + diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) + if not diff: + failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) + elif err_str: + _log.error(err_str) - elif reference_driver_output.image_hash != actual_driver_output.image_hash: + else: diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) if diff: failures.append(test_failures.FailureReftestMismatch(reference_filename)) elif err_str: _log.error(err_str) - else: - _log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name) return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid) diff --git a/tools/webkitpy/layout_tests/controllers/test_result_writer_unittest.py b/tools/webkitpy/layout_tests/controllers/test_result_writer_unittest.py index 213b6779fde..5b9d85ed156 100644 --- a/tools/webkitpy/layout_tests/controllers/test_result_writer_unittest.py +++ b/tools/webkitpy/layout_tests/controllers/test_result_writer_unittest.py @@ -54,16 +54,16 @@ class TestResultWriterTests(unittest.TestCase): def test_reference_exists(self): failure = test_failures.FailureReftestMismatch() - failure.reference_filename = '/src/exists-expected.html' - files = {'/src/exists-expected.html': 'yup'} + failure.reference_filename = '/src/exists-expected.sky' + files = {'/src/exists-expected.sky': 'yup'} written_files = self.run_test(failures=[failure], files=files) - self.assertEqual(written_files, {'/tmp/exists-expected.html': 'yup'}) + self.assertEqual(written_files, {'/tmp/exists-expected.sky': 'yup'}) failure = test_failures.FailureReftestMismatchDidNotOccur() - failure.reference_filename = '/src/exists-expected-mismatch.html' - files = {'/src/exists-expected-mismatch.html': 'yup'} + failure.reference_filename = '/src/exists-expected-mismatch.sky' + files = {'/src/exists-expected-mismatch.sky': 'yup'} written_files = self.run_test(failures=[failure], files=files) - self.assertEqual(written_files, {'/tmp/exists-expected-mismatch.html': 'yup'}) + self.assertEqual(written_files, {'/tmp/exists-expected-mismatch.sky': 'yup'}) def test_reference_is_missing(self): failure = test_failures.FailureReftestMismatch() diff --git a/tools/webkitpy/layout_tests/models/test_expectations_unittest.py b/tools/webkitpy/layout_tests/models/test_expectations_unittest.py index 5971a284c5e..2847de0ce57 100644 --- a/tools/webkitpy/layout_tests/models/test_expectations_unittest.py +++ b/tools/webkitpy/layout_tests/models/test_expectations_unittest.py @@ -185,9 +185,9 @@ class MiscTests(Base): try: filesystem = self._port.host.filesystem filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline.html'), 'content') - filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.html'), 'content') + filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.sky'), 'content') filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline.html'), 'content') - filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content') + filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.sky'), 'content') self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [ NeedsRebaseline ] Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]""", is_lint_mode=True) self.assertFalse(True, "ParseError wasn't raised") diff --git a/tools/webkitpy/layout_tests/models/test_run_results_unittest.py b/tools/webkitpy/layout_tests/models/test_run_results_unittest.py index 99d22462ab7..300bdbfa7e3 100644 --- a/tools/webkitpy/layout_tests/models/test_run_results_unittest.py +++ b/tools/webkitpy/layout_tests/models/test_run_results_unittest.py @@ -105,7 +105,7 @@ class InterpretTestFailuresTest(unittest.TestCase): self.port = host.port_factory.get(port_name='test') def test_interpret_test_failures(self): - test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))]) + test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.sky'))]) self.assertEqual(len(test_dict), 0) test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()]) diff --git a/tools/webkitpy/layout_tests/port/base_unittest.py b/tools/webkitpy/layout_tests/port/base_unittest.py index 494830d3d11..c1f77ebec51 100644 --- a/tools/webkitpy/layout_tests/port/base_unittest.py +++ b/tools/webkitpy/layout_tests/port/base_unittest.py @@ -287,10 +287,10 @@ class PortTest(unittest.TestCase): self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg')) self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png')) - self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html')) + self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.sky')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht')) - self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html')) + self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.sky')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml')) self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html')) @@ -321,7 +321,7 @@ class PortTest(unittest.TestCase): def test_reference_files(self): port = self.make_port(with_tests=True) self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')]) - self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')]) + self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.sky')]) self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')]) def test_operating_system(self): diff --git a/tools/webkitpy/layout_tests/port/test.py b/tools/webkitpy/layout_tests/port/test.py index 99f6874e33b..2afbe10e634 100644 --- a/tools/webkitpy/layout_tests/port/test.py +++ b/tools/webkitpy/layout_tests/port/test.py @@ -206,22 +206,22 @@ layer at (0,0) size 800x34 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') # For reftests. - tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True) + tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.sky', same_image=True) # This adds a different virtual reference to ensure that that also works. - tests.add('virtual/passes/reftest-expected.html', actual_checksum='xxx', actual_image='XXX', is_reftest=True) + tests.add('virtual/passes/reftest-expected.sky', actual_checksum='xxx', actual_image='XXX', is_reftest=True) - tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False) + tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.sky', same_image=False) tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) - tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True) + tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.sky', same_image=True) tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False) - tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False) - tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True) - tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpected/crash-reftest-expected.html', same_image=True, crash=True) - tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False) - tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True) + tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.sky', same_image=False) + tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.sky', same_image=True) + tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpected/crash-reftest-expected.sky', same_image=True, crash=True) + tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.sky', same_image=False) + tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.sky', same_image=True) tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) - tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True) + tests.add('failures/unexpected/reftest-nopixel-expected.sky', actual_checksum=None, actual_image=None, is_reftest=True) tests.add('reftests/foo/test.html') tests.add('reftests/foo/test-ref.html') @@ -237,7 +237,7 @@ layer at (0,0) size 800x34 tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi') # The following files shouldn't be treated as reftests - tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True) + tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.sky', same_image=True) tests.add('reftests/foo/reference/bar/common.html') tests.add('reftests/foo/reftest/bar/shared.html') diff --git a/tools/webkitpy/layout_tests/run_webkit_tests_unittest.py b/tools/webkitpy/layout_tests/run_webkit_tests_unittest.py index eb6bcf5a876..32cc45ecf42 100644 --- a/tools/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/tools/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -750,16 +750,16 @@ class RunTest(unittest.TestCase, StreamTestingMixin): self.assertEqual(['passes/reftest.html'], tests_run) def test_reftest_expected_html_should_be_ignored(self): - tests_run = get_tests_run(['passes/reftest-expected.html']) + tests_run = get_tests_run(['passes/reftest-expected.sky']) self.assertEqual([], tests_run) def test_reftest_driver_should_run_expected_html(self): tests_run = get_test_results(['passes/reftest.html']) - self.assertEqual(tests_run[0].references, ['passes/reftest-expected.html']) + self.assertEqual(tests_run[0].references, ['passes/reftest-expected.sky']) def test_reftest_driver_should_run_expected_mismatch_html(self): tests_run = get_test_results(['passes/mismatch.html']) - self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html']) + self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.sky']) def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self): host = MockHost() @@ -777,7 +777,7 @@ class RunTest(unittest.TestCase, StreamTestingMixin): def test_reftest_with_virtual_reference(self): _, err, _ = logging_run(['--details', 'virtual/passes/reftest.html'], tests_included=True) - self.assertTrue('ref: virtual/passes/reftest-expected.html' in err.getvalue()) + self.assertTrue('ref: virtual/passes/reftest-expected.sky' in err.getvalue()) def test_additional_platform_directory(self): self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo'])) diff --git a/tools/webkitpy/tool/commands/rebaseline_unittest.py b/tools/webkitpy/tool/commands/rebaseline_unittest.py index ab1ea087c0c..992d170fc75 100644 --- a/tools/webkitpy/tool/commands/rebaseline_unittest.py +++ b/tools/webkitpy/tool/commands/rebaseline_unittest.py @@ -263,7 +263,7 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] def test_rebaseline_reftest(self): self._write("userscripts/another-test.html", "test data") - self._write("userscripts/another-test-expected.html", "generic result") + self._write("userscripts/another-test-expected.sky", "generic result") OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options], expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n") self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': [], "delete": []})