Prepare python code for reftests.

-s/-expected.html/-expected.sky/
-s/-expected-mismatch.html/-expected-mismatch.sky/
-In single_test_runner.py, get rid of the image hash checking
for reftests. It doesn't provide any value and just makes
getting reftests working more complicated.

R=eseidel@chromium.org

Review URL: https://codereview.chromium.org/802573002
This commit is contained in:
Ojan Vafai 2014-12-11 15:24:02 -08:00
parent 63a8891e8d
commit 8971e980cd
9 changed files with 61 additions and 66 deletions

View File

@ -331,14 +331,14 @@ function expandExpectations(expandLink)
var row = parentOfType(expandLink, 'tr');
var parentTbody = row.parentNode;
var existingResultsRow = parentTbody.querySelector('.results-row');
var enDash = '\u2013';
expandLink.textContent = enDash;
if (existingResultsRow) {
updateExpandedState(existingResultsRow, true);
return;
}
var newRow = document.createElement('tr');
newRow.className = 'results-row';
var newCell = document.createElement('td');
@ -355,7 +355,7 @@ function expandExpectations(expandLink)
} else
result = resultIframe(link.href);
appendHTML(newCell, result);
appendHTML(newCell, result);
}
newRow.appendChild(newCell);
@ -466,7 +466,7 @@ function processGlobalStateFor(testObject)
globalState().hasHttpTests = globalState().hasHttpTests || test.indexOf('http/') == 0;
var actual = testObject.actual;
var actual = testObject.actual;
var expected = testObject.expected || 'PASS';
if (actual == 'MISSING') {
@ -504,7 +504,7 @@ function processGlobalStateFor(testObject)
globalState().timeoutTests.push(testObject);
return;
}
globalState().failingTests.push(testObject);
}
@ -547,11 +547,11 @@ function imageResultsCell(testObject, testPrefix, actual) {
globalState().hasImageFailures = true;
if (testObject.reftest_type && testObject.reftest_type.indexOf('!=') != -1) {
row += resultLink(testPrefix, '-expected-mismatch.html', 'ref mismatch html');
row += resultLink(testPrefix, '-expected-mismatch.sky', 'ref mismatch html');
row += resultLink(testPrefix, '-actual.png', 'actual');
} else {
if (testObject.reftest_type && testObject.reftest_type.indexOf('==') != -1) {
row += resultLink(testPrefix, '-expected.html', 'ref html');
row += resultLink(testPrefix, '-expected.sky', 'ref html');
}
if (globalState().shouldToggleImages) {
row += resultLink(testPrefix, '-diffs.html', 'images');
@ -571,7 +571,7 @@ function imageResultsCell(testObject, testPrefix, actual) {
}
function tableRow(testObject)
{
{
var row = '<tbody class="' + (testObject.is_unexpected ? '' : 'expected') + '"';
if (testObject.reftest_type && testObject.reftest_type.indexOf('!=') != -1)
row += ' mismatchreftest=true';
@ -581,7 +581,7 @@ function tableRow(testObject)
var testPrefix = stripExtension(testObject.name);
row += '<td>';
var actual = testObject.actual;
if (actual.indexOf('TEXT') != -1) {
globalState().hasTextFailures = true;
@ -591,7 +591,7 @@ function tableRow(testObject)
row += textResultLinks(testObject.name, testPrefix);
}
}
if (actual.indexOf('AUDIO') != -1) {
row += resultLink(testPrefix, '-expected.wav', 'expected audio');
row += resultLink(testPrefix, '-actual.wav', 'actual audio');
@ -708,7 +708,7 @@ function testList(tests, header, tableId)
// FIXME: only include timeout actual/diff results here if we actually spit out results for timeout tests.
html += textResultLinks(test, stripExtension(test));
}
if (testObject.has_repaint_overlay)
html += resultLink(stripExtension(test), '-overlay.html?' + encodeURIComponent(testLinkTarget(test)), 'overlay');
@ -777,7 +777,7 @@ TableSorter._sortRows = function(newHeader, reversed)
// Only need to support lexicographic sort for now.
var aText = TableSorter._textContent(a, sortColumn);
var bText = TableSorter._textContent(b, sortColumn);
// Forward sort equal values by test name.
if (sortColumn && aText == bText) {
var aTestName = TableSorter._textContent(a, 0);
@ -814,11 +814,11 @@ TableSorter.handleClick = function(e)
TableSorter._sort = function(newHeader)
{
TableSorter._updateHeaderClassNames(newHeader);
var reversed = newHeader.classList.contains('reversed');
var sortArrow = reversed ? TableSorter._backwardArrow : TableSorter._forwardArrow;
newHeader.innerHTML = TableSorter._sortedContents(newHeader, sortArrow);
TableSorter._sortRows(newHeader, reversed);
}
@ -839,15 +839,15 @@ PixelZoomer._zoomImageContainer = function(url)
container.className = 'zoom-image-container';
var title = url.match(/\-([^\-]*)\.png/)[1];
var label = document.createElement('div');
label.className = 'label';
label.appendChild(document.createTextNode(title));
container.appendChild(label);
var imageContainer = document.createElement('div');
imageContainer.className = 'scaled-image-container';
var image = new Image();
image.src = url;
image.style.display = 'none';
@ -857,7 +857,7 @@ PixelZoomer._zoomImageContainer = function(url)
imageContainer.appendChild(image);
imageContainer.appendChild(canvas);
container.appendChild(imageContainer);
return container;
}
@ -866,19 +866,19 @@ PixelZoomer._createContainer = function(e)
var tbody = parentOfType(e.target, 'tbody');
var row = tbody.querySelector('tr');
var imageDiffLinks = row.querySelectorAll('a[href$=".png"]');
var container = document.createElement('div');
container.className = 'pixel-zoom-container';
var html = '';
var togglingImageLink = row.querySelector('a[href$="-diffs.html"]');
if (togglingImageLink) {
var prefix = togglingImageLink.getAttribute('data-prefix');
container.appendChild(PixelZoomer._zoomImageContainer(prefix + '-expected.png'));
container.appendChild(PixelZoomer._zoomImageContainer(prefix + '-actual.png'));
}
for (var i = 0; i < imageDiffLinks.length; i++)
container.appendChild(PixelZoomer._zoomImageContainer(imageDiffLinks[i].href));
@ -935,7 +935,7 @@ PixelZoomer.handleMouseMove = function(e) {
return;
var container = document.querySelector('.pixel-zoom-container');
var resultContainer = (e.target.className == 'result-container') ?
e.target : parentOfType(e.target, '.result-container');
if (!resultContainer || !resultContainer.querySelector('img')) {
@ -959,7 +959,7 @@ PixelZoomer.handleMouseMove = function(e) {
PixelZoomer._createContainer(e);
return;
}
PixelZoomer._drawAll();
}
@ -1072,7 +1072,7 @@ TestNavigator._toggleCurrentTestFlagged = function()
TestNavigator.flagTest = function(testTbody, shouldFlag)
{
var testName = testTbody.querySelector('.test-link').innerText;
if (shouldFlag) {
testTbody.classList.add('flagged');
TestNavigator.flaggedTests[testName] = 1;
@ -1250,7 +1250,7 @@ function updateTogglingImages()
{
var shouldToggle = document.getElementById('toggle-images').checked;
globalState().shouldToggleImages = shouldToggle;
if (shouldToggle) {
forEach(document.querySelectorAll('table:not(#missing-table) tbody:not([mismatchreftest]) a[href$=".png"]'), convertToTogglingHandler(function(prefix) {
return resultLink(prefix, '-diffs.html', 'images');

View File

@ -71,7 +71,7 @@ class SingleTestRunner(object):
if self._reference_files:
# Detect and report a test which has a wrong combination of expectation files.
# For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
# For example, if 'foo.html' has two expectation files, 'foo-expected.sky' and
# 'foo-expected.txt', we should warn users. One test file must be used exclusively
# in either layout tests or reftests, but not in both.
for suffix in ('.txt', '.png', '.wav'):
@ -402,22 +402,17 @@ class SingleTestRunner(object):
if not reference_driver_output.image_hash and not actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
elif mismatch:
if reference_driver_output.image_hash == actual_driver_output.image_hash:
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if not diff:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif err_str:
_log.error(err_str)
else:
_log.warning(" %s -> ref test hashes matched but diff failed" % self._test_name)
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if not diff:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif err_str:
_log.error(err_str)
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
else:
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if diff:
failures.append(test_failures.FailureReftestMismatch(reference_filename))
elif err_str:
_log.error(err_str)
else:
_log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name)
return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)

View File

@ -54,16 +54,16 @@ class TestResultWriterTests(unittest.TestCase):
def test_reference_exists(self):
failure = test_failures.FailureReftestMismatch()
failure.reference_filename = '/src/exists-expected.html'
files = {'/src/exists-expected.html': 'yup'}
failure.reference_filename = '/src/exists-expected.sky'
files = {'/src/exists-expected.sky': 'yup'}
written_files = self.run_test(failures=[failure], files=files)
self.assertEqual(written_files, {'/tmp/exists-expected.html': 'yup'})
self.assertEqual(written_files, {'/tmp/exists-expected.sky': 'yup'})
failure = test_failures.FailureReftestMismatchDidNotOccur()
failure.reference_filename = '/src/exists-expected-mismatch.html'
files = {'/src/exists-expected-mismatch.html': 'yup'}
failure.reference_filename = '/src/exists-expected-mismatch.sky'
files = {'/src/exists-expected-mismatch.sky': 'yup'}
written_files = self.run_test(failures=[failure], files=files)
self.assertEqual(written_files, {'/tmp/exists-expected-mismatch.html': 'yup'})
self.assertEqual(written_files, {'/tmp/exists-expected-mismatch.sky': 'yup'})
def test_reference_is_missing(self):
failure = test_failures.FailureReftestMismatch()

View File

@ -185,9 +185,9 @@ class MiscTests(Base):
try:
filesystem = self._port.host.filesystem
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline.html'), 'content')
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.html'), 'content')
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.sky'), 'content')
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline.html'), 'content')
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content')
filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.sky'), 'content')
self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]""", is_lint_mode=True)
self.assertFalse(True, "ParseError wasn't raised")

View File

@ -105,7 +105,7 @@ class InterpretTestFailuresTest(unittest.TestCase):
self.port = host.port_factory.get(port_name='test')
def test_interpret_test_failures(self):
test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.sky'))])
self.assertEqual(len(test_dict), 0)
test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()])

View File

@ -287,10 +287,10 @@ class PortTest(unittest.TestCase):
self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg'))
self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.sky'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.sky'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml'))
self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html'))
@ -321,7 +321,7 @@ class PortTest(unittest.TestCase):
def test_reference_files(self):
port = self.make_port(with_tests=True)
self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])
self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')])
self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.sky')])
self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')])
def test_operating_system(self):

View File

@ -206,22 +206,22 @@ layer at (0,0) size 800x34
expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
# For reftests.
tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.sky', same_image=True)
# This adds a different virtual reference to ensure that that also works.
tests.add('virtual/passes/reftest-expected.html', actual_checksum='xxx', actual_image='XXX', is_reftest=True)
tests.add('virtual/passes/reftest-expected.sky', actual_checksum='xxx', actual_image='XXX', is_reftest=True)
tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.sky', same_image=False)
tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.sky', same_image=True)
tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpected/crash-reftest-expected.html', same_image=True, crash=True)
tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.sky', same_image=False)
tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.sky', same_image=True)
tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpected/crash-reftest-expected.sky', same_image=True, crash=True)
tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.sky', same_image=False)
tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.sky', same_image=True)
tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
tests.add('failures/unexpected/reftest-nopixel-expected.sky', actual_checksum=None, actual_image=None, is_reftest=True)
tests.add('reftests/foo/test.html')
tests.add('reftests/foo/test-ref.html')
@ -237,7 +237,7 @@ layer at (0,0) size 800x34
tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
# The following files shouldn't be treated as reftests
tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.sky', same_image=True)
tests.add('reftests/foo/reference/bar/common.html')
tests.add('reftests/foo/reftest/bar/shared.html')

View File

@ -750,16 +750,16 @@ class RunTest(unittest.TestCase, StreamTestingMixin):
self.assertEqual(['passes/reftest.html'], tests_run)
def test_reftest_expected_html_should_be_ignored(self):
tests_run = get_tests_run(['passes/reftest-expected.html'])
tests_run = get_tests_run(['passes/reftest-expected.sky'])
self.assertEqual([], tests_run)
def test_reftest_driver_should_run_expected_html(self):
tests_run = get_test_results(['passes/reftest.html'])
self.assertEqual(tests_run[0].references, ['passes/reftest-expected.html'])
self.assertEqual(tests_run[0].references, ['passes/reftest-expected.sky'])
def test_reftest_driver_should_run_expected_mismatch_html(self):
tests_run = get_test_results(['passes/mismatch.html'])
self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html'])
self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.sky'])
def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
host = MockHost()
@ -777,7 +777,7 @@ class RunTest(unittest.TestCase, StreamTestingMixin):
def test_reftest_with_virtual_reference(self):
_, err, _ = logging_run(['--details', 'virtual/passes/reftest.html'], tests_included=True)
self.assertTrue('ref: virtual/passes/reftest-expected.html' in err.getvalue())
self.assertTrue('ref: virtual/passes/reftest-expected.sky' in err.getvalue())
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))

View File

@ -263,7 +263,7 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]
def test_rebaseline_reftest(self):
self._write("userscripts/another-test.html", "test data")
self._write("userscripts/another-test-expected.html", "generic result")
self._write("userscripts/another-test-expected.sky", "generic result")
OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options],
expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n")
self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': [], "delete": []})