Feat!: support Unicode in sqlite, mysql, tsql, postgres, oracle (#4554)

* add support for UNICODE for sqlite, mysql, tsql, postgres, and oracle

* UNICODE tests

* linter fix

* oracle unicode fix

* clean-up

* ruff

Author: pruzko

sqlglot/dialects/mysql.py

@@ -775,6 +775,7 @@ class Generator(generator.Generator):
             exp.TsOrDsAdd: date_add_sql("ADD"),
             exp.TsOrDsDiff: lambda self, e: self.func("DATEDIFF", e.this, e.expression),
             exp.TsOrDsToDate: _ts_or_ds_to_date_sql,
+            exp.Unicode: lambda self, e: f"ORD(CONVERT({self.sql(e.this)} USING utf32))",
             exp.UnixToTime: _unix_to_time_sql,
             exp.Week: _remove_ts_or_ds_to_date(),
             exp.WeekOfYear: _remove_ts_or_ds_to_date(rename_func("WEEKOFYEAR")),

sqlglot/dialects/oracle.py

@@ -333,6 +333,7 @@ class Generator(generator.Generator):
             exp.ToChar: lambda self, e: self.function_fallback_sql(e),
             exp.ToNumber: to_number_with_nls_param,
             exp.Trim: _trim_sql,
+            exp.Unicode: lambda self, e: f"ASCII(UNISTR({self.sql(e.this)}))",
             exp.UnixToTime: lambda self,
             e: f"TO_DATE('1970-01-01', 'YYYY-MM-DD') + ({self.sql(e, 'this')} / 86400)",
         }

sqlglot/dialects/postgres.py

@@ -370,6 +370,7 @@ class Parser(parser.Parser):
 
         FUNCTIONS = {
             **parser.Parser.FUNCTIONS,
+            "ASCII": exp.Unicode.from_arg_list,
             "DATE_TRUNC": build_timestamp_trunc,
             "DIV": lambda args: exp.cast(
                 binary_from_function(exp.IntDiv)(args), exp.DataType.Type.DECIMAL
@@ -605,6 +606,7 @@ class Generator(generator.Generator):
             exp.VariancePop: rename_func("VAR_POP"),
             exp.Variance: rename_func("VAR_SAMP"),
             exp.Xor: bool_xor_sql,
+            exp.Unicode: rename_func("ASCII"),
             exp.UnixToTime: _unix_to_time_sql,
             exp.Levenshtein: _levenshtein_sql,
         }

sqlglot/expressions.py

@@ -6603,6 +6603,10 @@ class Unhex(Func):
     pass
 
 
+class Unicode(Func):
+    pass
+
+
 # https://cloud.google.com/bigquery/docs/reference/standard-sql/date_functions#unix_date
 class UnixDate(Func):
     pass

tests/dialects/test_sqlite.py

@@ -92,6 +92,14 @@ def test_sqlite(self):
             read={"snowflake": "LEAST(x, y, z)"},
             write={"snowflake": "LEAST(x, y, z)"},
         )
+        self.validate_all(
+            "UNICODE(x)",
+            write={
+                "mysql": "ORD(CONVERT(x USING utf32))",
+                "postgres": "ASCII(x)",
+                "oracle": "ASCII(UNISTR(x))",
+            },
+        )
         self.validate_identity(
             "SELECT * FROM station WHERE city IS NOT ''",
             "SELECT * FROM station WHERE NOT city IS ''",