[Utils] add update-verify-tests.py (llvm#97369)

Adds a python script to automatically take output from a failed clang
-verify test and update the test case(s) to expect the new behaviour.

Author: hnrklssn

clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c

@@ -0,0 +1,8 @@
+void foo() {
+    // expected-error@+1{{use of undeclared identifier 'a'}}
+    a = 2; a = 2;
+    b = 2; b = 2;
+    // expected-error@+1 3{{use of undeclared identifier 'c'}}
+    c = 2; c = 2;
+    // expected-error 2{{asdf}}
+}

clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected

@@ -0,0 +1,8 @@
+void foo() {
+    // expected-error@+1 2{{use of undeclared identifier 'a'}}
+    a = 2; a = 2;
+    // expected-error@+1 2{{use of undeclared identifier 'b'}}
+    b = 2; b = 2;
+    // expected-error@+1 2{{use of undeclared identifier 'c'}}
+    c = 2; c = 2;
+}

clang/test/utils/update-verify-tests/Inputs/infer-indentation.c

@@ -0,0 +1,8 @@
+void foo() {
+         //     expected-error@+1    2      {{use of undeclared identifier 'a'}}
+    a = 2; a = 2; b = 2; b = 2; c = 2;
+         //     expected-error@+1    2      {{asdf}}
+    d = 2;
+    e = 2; f = 2;                 //     expected-error    2      {{use of undeclared identifier 'e'}}
+}
+

clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected

@@ -0,0 +1,11 @@
+void foo() {
+         //     expected-error@+3          {{use of undeclared identifier 'c'}}
+         //     expected-error@+2    2      {{use of undeclared identifier 'b'}}
+         //     expected-error@+1    2      {{use of undeclared identifier 'a'}}
+    a = 2; a = 2; b = 2; b = 2; c = 2;
+         //     expected-error@+1          {{use of undeclared identifier 'd'}}
+    d = 2;
+    //     expected-error@+1          {{use of undeclared identifier 'f'}}
+    e = 2; f = 2;                 //     expected-error          {{use of undeclared identifier 'e'}}
+}
+

clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c

@@ -0,0 +1,11 @@
+void foo() {
+    a = 2;
+    // expected-error@-1{{use of undeclared identifier 'a'}}
+    b = 2;// expected-error{{use of undeclared identifier 'b'}}
+    c = 2;
+    // expected-error@5{{use of undeclared identifier 'c'}}
+    d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
+
+    e = 2; // error to trigger mismatch
+}
+

clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected

@@ -0,0 +1,12 @@
+void foo() {
+    a = 2;
+    // expected-error@-1{{use of undeclared identifier 'a'}}
+    b = 2;// expected-error{{use of undeclared identifier 'b'}}
+    c = 2;
+    // expected-error@5{{use of undeclared identifier 'c'}}
+    d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
+
+    // expected-error@+1{{use of undeclared identifier 'e'}}
+    e = 2; // error to trigger mismatch
+}
+

clang/test/utils/update-verify-tests/Inputs/multiple-errors.c

@@ -0,0 +1,6 @@
+void foo() {
+    a = 2;
+    b = 2;
+
+    c = 2;
+}

clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected

@@ -0,0 +1,9 @@
+void foo() {
+    // expected-error@+1{{use of undeclared identifier 'a'}}
+    a = 2;
+    // expected-error@+1{{use of undeclared identifier 'b'}}
+    b = 2;
+
+    // expected-error@+1{{use of undeclared identifier 'c'}}
+    c = 2;
+}

clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c

@@ -0,0 +1,8 @@
+void foo() {
+    a = 2; b = 2; c = 2;
+}
+
+void bar() {
+    x = 2; y = 2; z = 2;
+    // expected-error@-1{{use of undeclared identifier 'x'}}
+}

clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected

@@ -0,0 +1,13 @@
+void foo() {
+    // expected-error@+3{{use of undeclared identifier 'c'}}
+    // expected-error@+2{{use of undeclared identifier 'b'}}
+    // expected-error@+1{{use of undeclared identifier 'a'}}
+    a = 2; b = 2; c = 2;
+}
+
+void bar() {
+    x = 2; y = 2; z = 2;
+    // expected-error@-1{{use of undeclared identifier 'x'}}
+    // expected-error@-2{{use of undeclared identifier 'y'}}
+    // expected-error@-3{{use of undeclared identifier 'z'}}
+}

clang/test/utils/update-verify-tests/Inputs/no-checks.c

@@ -0,0 +1,3 @@
+void foo() {
+    bar = 2;
+}

clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected

@@ -0,0 +1,4 @@
+void foo() {
+    // expected-error@+1{{use of undeclared identifier 'bar'}}
+    bar = 2;
+}

clang/test/utils/update-verify-tests/Inputs/no-diags.c

@@ -0,0 +1,5 @@
+void foo() {
+    // expected-error@+1{{asdf}}
+    int a = 2;
+}
+

clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected

@@ -0,0 +1,5 @@
+// expected-no-diagnostics
+void foo() {
+    int a = 2;
+}
+

clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c

@@ -0,0 +1,4 @@
+// expected-no-diagnostics
+void foo() {
+    a = 2;
+}

clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected

@@ -0,0 +1,4 @@
+void foo() {
+    // expected-error@+1{{use of undeclared identifier 'a'}}
+    a = 2;
+}

clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c

@@ -0,0 +1,5 @@
+void foo() {
+    a = 2; // check-error{{asdf}}
+           // expected-error@-1{ignored}}
+}
+

clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected

@@ -0,0 +1,5 @@
+void foo() {
+    a = 2; // check-error{{use of undeclared identifier 'a'}}
+           // expected-error@-1{ignored}}
+}
+

clang/test/utils/update-verify-tests/Inputs/update-same-line.c

@@ -0,0 +1,4 @@
+void foo() {
+    bar = 2;     //   expected-error       {{asdf}}
+}
+

clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected

@@ -0,0 +1,4 @@
+void foo() {
+    bar = 2;     //   expected-error       {{use of undeclared identifier 'bar'}}
+}
+

clang/test/utils/update-verify-tests/Inputs/update-single-check.c

@@ -0,0 +1,4 @@
+void foo() {
+    // expected-error@+1{{asdf}}
+    bar = 2;
+}

clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected

@@ -0,0 +1,4 @@
+void foo() {
+    // expected-error@+1{{use of undeclared identifier 'bar'}}
+    bar = 2;
+}

clang/test/utils/update-verify-tests/duplicate-diag.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/duplicate-diag.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/duplicate-diag.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+

clang/test/utils/update-verify-tests/infer-indentation.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/infer-indentation.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/infer-indentation.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c

clang/test/utils/update-verify-tests/leave-existing-diags.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/leave-existing-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/leave-existing-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+

clang/test/utils/update-verify-tests/lit.local.cfg

@@ -0,0 +1,25 @@
+import lit.util
+
+# python 2.7 backwards compatibility
+try:
+    from shlex import quote as shell_quote
+except ImportError:
+    from pipes import quote as shell_quote
+
+if config.standalone_build:
+    # These tests require the update-verify-tests.py script from the clang
+    # source tree, so skip these tests if we are doing standalone builds.
+    config.unsupported = True
+else:
+    config.suffixes = [".test"]
+
+    script_path = os.path.join(
+        config.clang_src_dir, "utils", "update-verify-tests.py"
+    )
+    python = shell_quote(config.python_executable)
+    config.substitutions.append(
+        (
+            "%update-verify-tests",
+            "%s %s" % (python, shell_quote(script_path)),
+        )
+    )

clang/test/utils/update-verify-tests/multiple-errors.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/multiple-errors.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/multiple-errors.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c

clang/test/utils/update-verify-tests/multiple-missing-errors-same-line.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/multiple-missing-errors-same-line.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/multiple-missing-errors-same-line.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c

clang/test/utils/update-verify-tests/no-checks.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/no-checks.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-checks.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c

clang/test/utils/update-verify-tests/no-diags.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/no-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+

clang/test/utils/update-verify-tests/no-expected-diags.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/no-expected-diags.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/no-expected-diags.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+

clang/test/utils/update-verify-tests/non-default-prefix.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/non-default-prefix.c %t.c && not %clang_cc1 -verify=check %t.c 2>&1 | %update-verify-tests --prefix check
+# RUN: diff -u %S/Inputs/non-default-prefix.c.expected %t.c
+# RUN: %clang_cc1 -verify=check %t.c
+

clang/test/utils/update-verify-tests/update-same-line.test

@@ -0,0 +1,4 @@
+# RUN: cp %S/Inputs/update-same-line.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/update-same-line.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c
+

clang/test/utils/update-verify-tests/update-single-check.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/Inputs/update-single-check.c %t.c && not %clang_cc1 -verify %t.c 2>&1 | %update-verify-tests
+# RUN: diff -u %S/Inputs/update-single-check.c.expected %t.c
+# RUN: %clang_cc1 -verify %t.c