Merge pull request #6986 from ZeaLoVe/improve-logictest-output

ci: improve logictest output

Author: mergify[bot]

tests/logictest/README.md

@@ -39,6 +39,12 @@ Get help with:
 python main.py -h
 ```
 
+Useful arguments:
+1. --run-dir ydb  will only run the suites in dir ./suites/ydb/
+2. --skip-dir ydb  will skip the suites in dir ./suites/ydb
+3. --suites other_dir  wiil use suites file in dir ./other_dir
+4. Run files by pattern string like: python main.py "03_0001"
+
 ## Docker
 
 ### Build image
@@ -59,6 +65,56 @@ docker build -t sqllogic/test:latest .
 - ADDITIONAL_HEADERS (for security scenario)
 3. docker run --name logictest --rm --network host datafuselabs/sqllogictest:latest
 
+## How to write logic test
+
+Fast start, you can follow this demo: https://github.com/datafuselabs/databend/blob/main/tests/logictest/suites/select_0
+
+Runner supported: mysql handler, http handler, clickhouse handler.
+
+- ok
+  - Returns no error, don't care about the result
+- error
+  - Returns with error and expected error message, usually with an error code, but also with a message string; the way to determine whether the specified string is in the returned message
+- query
+  - Return result and check the result with expected, follow by query_type and query_label
+  - query_type is a char represent a column in result, multi char means multi column
+    - B Boolean
+    - T text   
+    - F floating point
+    - I integer
+  - query_label If different runner return inconsistency, you can write like this(suppose that mysql handler is get different result)
+
+This is a query demo(query_label is optional):
+
+```
+statement query III label(mysql)
+select number, number + 1, number + 999 from numbers(10);
+
+----
+     0     1   999
+     1     2  1000
+     2     3  1001
+     3     4  1002
+     4     5  1003
+     5     6  1004
+     6     7  1005
+     7     8  1006
+     8     9  1007
+     9    10  1008.0
+
+----  mysql
+     0     1   999
+     1     2  1000
+     2     3  1001
+     3     4  1002
+     4     5  1003
+     5     6  1004
+     6     7  1005
+     7     8  1006
+     8     9  1007
+     9    10  1008
+```
+
 ## Write logic test tips
 
 1. skipif  help you skip test of given handler
@@ -91,8 +147,10 @@ select 1;
 ```
 
 **tips** If you do not care about result, use statement ok instead of statement query
+**tips** Add ORDER BY to ensure that the order of returned results is always consistent
 **warning** A statement query need result, and even if you want to skip a case, you still need to keep the results in the test content
 
 # Learn More
 
-Ref pr: https://github.com/datafuselabs/databend/pull/5048
+RFC: https://github.com/datafuselabs/databend/blob/main/docs/doc/60-contributing/03-rfcs/20220425-new_sql_logic_test_framework.md
+Migration discussion: https://github.com/datafuselabs/databend/discussions/5838

tests/logictest/logictest.py

@@ -94,12 +94,13 @@ def parse_token_args(tokens, arg):
 
 class LogicError(Exception):
 
-    def __init__(self, message, expected):
+    def __init__(self, message, errorType, runner):
         self.message = message
-        self.expected = expected
+        self.errorType = errorType
+        self.runner = runner
 
     def __str__(self):
-        return f"Expected: {self.expected}\nMessage: {self.message}"
+        return f"Ruuner: {self.runner}\nErrorType: {self.errorType}\nMessage: {self.message}"
 
 
 class Statement:
@@ -225,15 +226,12 @@ def get_statements(suite_path, suite_name):
 
 def format_value(vals, val_num):
     row = len(vals) // val_num
-    width = len(str(vals[0])) + 2
-    for i in range(len(vals)):
-        width = max(width, len(vals[i]) + 2)
     table = ""
     for i in range(row):
         ans = []
         for j in range(val_num):
-            ans.append('{: >{w}}'.format(str(vals[i * val_num + j]), w=width))
-        table += "".join(ans)
+            ans.append(str(vals[i * val_num + j]))
+        table += "  ".join(ans)
         table += "\n"
     return table
 
@@ -256,7 +254,7 @@ def __init__(self, kind, args):
         self.driver = None     
         self.statement_files = []
         self.kind = kind
-        self.show_query_on_execution = True
+        self.show_query_on_execution = False
         self.on_error_return = False
         self.dir = dir
         self.args = args
@@ -279,7 +277,7 @@ def fetch_files(self):
 
                 if self.args.skip_dir and any(
                         s in dirs for s in self.args.skip_dir):
-                    log.info(f"Skip test file {filename}, skip test in dirs {self.args.skip_dir}")
+                    log.info(f"Skip test file {filename}, in dirs {self.args.skip_dir}")
                     continue
 
                 if self.args.skip and any(
@@ -305,9 +303,6 @@ def execute(self):
                 # case batch
                 for (file_path, suite_name) in self.statement_files:
                     self.suite_now = suite_name
-                    log.info(
-                        f"Run query with the same session every suite file, suite file path:{file_path}"
-                    )
                     statement_list = list()
                     for state in get_statements(file_path, suite_name):
                         statement_list.append(state)
@@ -320,6 +315,10 @@ def execute(self):
                         global_statistics.add_failed(self.kind, self.suite_now,
                                                      e)
                         continue
+
+                    log.info(
+                        f"Suite file:{file_path} pass!"
+                    )
             else:
                 raise RuntimeError(
                     f"batch_execute is not implement in runner {self.kind}")
@@ -331,7 +330,7 @@ def execute_statement(self, statement):
             )
             return
         if self.show_query_on_execution:
-            log.debug(
+            log.Info(
                 f"executing statement, type {statement.s_type.type}\n{statement.text}\n"
             )
         start = time.perf_counter()
@@ -354,22 +353,24 @@ def assert_execute_ok(self, statement):
                               statement)
         if actual is not None:
             raise LogicError(
+                runner=self.kind,
                 message=str(statement),
-                expected=
-                "statement ok must get success response, not error code in response"
+                errorType="statement ok get error in response"
             )
 
     def assert_query_equal(self, f, resultset, statement):
         # use join after split instead of strip
         compare_f = "".join(f.split())
         compare_result = "".join(resultset[2].split())
         if compare_f != compare_result:
-            raise LogicError(message="\n Expected:\n{}\n Actual:\n{}\n Statement:{}\n Start " \
+            raise LogicError(message="\n Expected:\n{:<80}\n Actual:\n{:<80}\n Statement:{}\n Start " \
                                             "Line: {}, Result Label: {}".format(resultset[2].rstrip(),
                                                                                 f.rstrip(),
                                                                                 str(statement), resultset[1],
                                                                                 resultset[0].group("label")),
-                            expected="statement query must get result equal to expected")
+                            errorType="statement query get result not equal to expected",
+                            runner=self.kind,
+                            )
 
     def assert_execute_query(self, statement):
         if statement.s_type.query_type == "skipped":
@@ -381,11 +382,13 @@ def assert_execute_query(self, statement):
         except Exception:
             raise LogicError(
                 message=f"{statement} statement type is query but get no result",
-                expected="query get result")
+                errorType="statement query get no result",
+                runner=self.kind)
 
         if statement.results is None or len(statement.results) == 0:
-            raise LogicError(message=f"No result found {statement}",
-                             expected="query get result")
+            raise LogicError(message=f"{statement} no result found by query",
+                             errorType="statement query get empty result",
+                             runner=self.kind)
         hasResult = False
         for resultset in statement.results:
             if resultset[0].group("label") is not None and resultset[0].group(
@@ -399,26 +402,29 @@ def assert_execute_query(self, statement):
                     self.assert_query_equal(f, resultset, statement)
                     hasResult = True
         if not hasResult:
-            raise LogicError(message=f"No result found {statement}",
-                             expected="query get result")
+            raise LogicError(message=f"{statement} no result found in test file",
+                             errorType="statement query has no result in test file",
+                             runner=self.kind)
 
     # expect the query just return error
     def assert_execute_error(self, statement):
         actual = safe_execute(lambda: self.execute_error(statement.text),
                               statement)
         if actual is None:
-            raise Exception("Expected error but got none")
+            raise LogicError(message=f"{str(statement)}",
+            errorType="statement error get no error message",
+            runner=self.kind)
         match = re.search(statement.s_type.expect_error, actual.msg)
         if match is None:
             raise LogicError(
                 message=
-                f"statement {str(statement)}, expect error regex {statement.s_type.expect_error}, found {actual}",
-                expected=
-                f"statement error must get error message in expected string")
+                f"\n expected error regex is {statement.s_type.expect_error}\n actual found {actual}{str(statement)}",
+                errorType=f"statement error get error message not equal to expected",
+                runner=self.kind)
 
     def run_sql_suite(self):
         log.info(
-            f"run_sql_suite for {self.kind} on base {os.path.abspath(self.args.suites)}"
+            f"run_sql_suite for {self.kind} with suites dir {os.path.abspath(self.args.suites)}"
         )
         self.fetch_files()
         self.execute()