Revise tools scripts to be python3 compatible on win32

We introduce setup_stdio function to setup stdout/stderr properly.
For python <-> python pipe, we always use 'utf8'/'ignore' encoding for not lost
characters.
For tty <-> python, we using native encoding with xmlcharrefreplace to encode, to
preserve maximal information.
For python <-> native program, we use naive encoding with 'ignore' to not cause error

update_exclude_list with binary mode so that on win32 would not generate \r\n

run-test-suite.py: Handling skiplist properly on win32

Fixes jerryscript-project#4854

Fixes test262-harness.py complain cannot use a string pattern on a bytes-like object with running test262 with python3

For reading/writing to file, we use 'utf8' /'ignore' encoding for not lost characters.
For decoding from process stdout, using native encoding with decoding error ignored for not lost data.
Execute commands also ignore errors
Fixes jerryscript-project#4853

Fixes running test262-esnext failed with installed python3.9 on win32 with space in path
Fixes jerryscript-project#4852

```
support both / \ in --test262-test-list arg

On win32.
python tools/run-tests.py  --test262-es2015=update --test262-test-list=built-ins/decodeURI/
python tools/run-tests.py  --test262-es2015=update --test262-test-list=built-ins\decodeURI\
should be both valid,
currently only --test262-test-list=built-ins\decodeURI\ are valid.
```

```
Support snapshot-tests-skiplist.txt on win32 by use os.path.normpath
```

Guard run-tests.py with timer.

All run-tests.py are finished in 30 minutes in normal situation.
May increase the timeout future.

wait JERRY_CHECK_TIMEOUT

```

Move Windows CI to github actions

Convert run-debugger-test.sh to run-debugger-test.py

After this change, run-debugger-test.py could running on Win32 and OSX

Define TERM colors for win32 properly

```

```

flush stderr.write stdout.write

On CI, the stderr are redirect to stdout, and if we don't flush stderr and stdout,
The output from stderr/stdout would out of sync.

```

`Testing new Date(-8640000000000000) and fixes date for win32`
So that the CI can passed

JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo [email protected]

Author: lygstate

.github/workflows/gh-actions.yml

@@ -56,6 +56,34 @@ jobs:
       - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on
       - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on --build-debug
 
+  Win_x86-64_Conformance_Tests_ESNext:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER --test262 update
+
+  Win_x86-64_Conformance_Tests_ESNext_Debug:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER --test262 update --build-debug
+
+  Win_x86-64_Tests:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER -q --jerry-tests
+      - run: python $env:RUNNER -q --unittests
+      - run: python $env:RUNNER -q --buildoption-test
+
+  Win_x86-64_Tests_Debug:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER -q --jerry-tests --build-debug
+      - run: python $env:RUNNER -q --unittests --build-debug
+      - run: python $env:RUNNER -q --buildoption-test --build-debug
+
   OSX_x86-64_Build_Correctness_Unit_Tests:
     runs-on: macos-13
     steps:
@@ -65,6 +93,7 @@ jobs:
           python-version: '>=3.6'
       - run: $RUNNER -q --jerry-tests
       - run: $RUNNER -q --unittests
+      - run: $RUNNER -q --jerry-debugger
 
   OSX_x86-64_Build_Correctness_Unit_Tests_Debug:
     runs-on: macos-13
@@ -75,6 +104,7 @@ jobs:
           python-version: '>=3.6'
       - run: $RUNNER -q --jerry-tests --build-debug
       - run: $RUNNER -q --unittests --build-debug
+      - run: $RUNNER -q --jerry-debugger --build-debug
 
   Linux_x86-64_Build_Option_Tests:
     runs-on: ubuntu-latest

README.md

@@ -2,7 +2,6 @@
 # JerryScript: JavaScript engine for the Internet of Things
 [![License](https://img.shields.io/badge/licence-Apache%202.0-brightgreen.svg?style=flat)](LICENSE)
 [![GitHub Actions Status](https://github.com/jerryscript-project/jerryscript/workflows/JerryScript%20CI/badge.svg)](https://github.com/jerryscript-project/jerryscript/actions)
-[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/ct8reap35u2vooa5/branch/master?svg=true)](https://ci.appveyor.com/project/jerryscript-project/jerryscript/branch/master)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fjerryscript-project%2Fjerryscript.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Fjerryscript-project%2Fjerryscript?ref=badge_shield)
 [![IRC Channel](https://img.shields.io/badge/chat-on%20freenode-brightgreen.svg)](https://kiwiirc.com/client/irc.freenode.net/#jerryscript)
 

appveyor.yml

@@ -326,6 +326,8 @@ def main():
             break
         if res_type == result.PROMPT:
             prompt.cmdloop()
+            sys.stdout.flush()
+            sys.stderr.flush()
         elif res_type == result.TEXT:
             write(result.get_text())
         continue

jerry-debugger/jerry_client.py

@@ -23,8 +23,8 @@
 #include <winbase.h>
 #include <winnt.h>
 
-#define UNIX_EPOCH_IN_TICKS 116444736000000000ull /* difference between 1970 and 1601 */
-#define TICKS_PER_MS        10000ull /* 1 tick is 100 nanoseconds */
+#define UNIX_EPOCH_IN_TICKS 116444736000000000LL /* difference between 1970 and 1601 */
+#define TICKS_PER_MS        10000LL /* 1 tick is 100 nanoseconds */
 
 /*
  * If you take the limit of SYSTEMTIME (last millisecond in 30827) then you end up with
@@ -44,15 +44,9 @@
  * https://support.microsoft.com/en-us/help/167296/how-to-convert-a-unix-time-t-to-a-win32-filetime-or-systemtime
  */
 static void
-unix_time_to_filetime (double t, LPFILETIME ft_p)
+unix_time_to_filetime (LONGLONG t, LPFILETIME ft_p)
 {
-  LONGLONG ll = (LONGLONG) t * TICKS_PER_MS + UNIX_EPOCH_IN_TICKS;
-
-  /* FILETIME values before the epoch are invalid. */
-  if (ll < 0)
-  {
-    ll = 0;
-  }
+  LONGLONG ll = t * TICKS_PER_MS + UNIX_EPOCH_IN_TICKS;
 
   ft_p->dwLowDateTime = (DWORD) ll;
   ft_p->dwHighDateTime = (DWORD) (ll >> 32);
@@ -63,13 +57,15 @@ unix_time_to_filetime (double t, LPFILETIME ft_p)
  *
  * @return unix time
  */
-static double
+static LONGLONG
 filetime_to_unix_time (LPFILETIME ft_p)
 {
   ULARGE_INTEGER date;
+  LONGLONG ll;
   date.HighPart = ft_p->dwHighDateTime;
   date.LowPart = ft_p->dwLowDateTime;
-  return (double) (((LONGLONG) date.QuadPart - UNIX_EPOCH_IN_TICKS) / TICKS_PER_MS);
+  ll = date.QuadPart - UNIX_EPOCH_IN_TICKS;
+  return ll / TICKS_PER_MS;
 } /* filetime_to_unix_time */
 
 /**
@@ -85,6 +81,7 @@ jerry_port_local_tza (double unix_ms)
   FILETIME local;
   SYSTEMTIME utc_sys;
   SYSTEMTIME local_sys;
+  LONGLONG t = (LONGLONG) (unix_ms);
 
   /*
    * If the time is earlier than the date 1601-01-02, then always using date 1601-01-02 to
@@ -93,23 +90,23 @@ jerry_port_local_tza (double unix_ms)
    * after converting between local time and utc time, the time may be earlier than 1601-01-01
    * in UTC time, that exceeds the FILETIME representation range.
    */
-  if (unix_ms < (double) UNIX_EPOCH_DATE_1601_01_02)
+  if (t < UNIX_EPOCH_DATE_1601_01_02)
   {
-    unix_ms = (double) UNIX_EPOCH_DATE_1601_01_02;
+    t = UNIX_EPOCH_DATE_1601_01_02;
   }
 
   /* Like above, do not use the last supported day */
-  if (unix_ms > (double) UNIX_EPOCH_DATE_30827_12_29)
+  if (t > UNIX_EPOCH_DATE_30827_12_29)
   {
-    unix_ms = (double) UNIX_EPOCH_DATE_30827_12_29;
+    t = UNIX_EPOCH_DATE_30827_12_29;
   }
-  unix_time_to_filetime (unix_ms, &utc);
+  unix_time_to_filetime (t, &utc);
 
   if (FileTimeToSystemTime (&utc, &utc_sys) && SystemTimeToTzSpecificLocalTime (NULL, &utc_sys, &local_sys)
       && SystemTimeToFileTime (&local_sys, &local))
   {
-    double unix_local = filetime_to_unix_time (&local);
-    return (int32_t) (unix_local - unix_ms);
+    LONGLONG unix_local = filetime_to_unix_time (&local);
+    return (int32_t) (unix_local - t);
   }
 
   return 0;
@@ -125,7 +122,7 @@ jerry_port_current_time (void)
 {
   FILETIME ft;
   GetSystemTimeAsFileTime (&ft);
-  return filetime_to_unix_time (&ft);
+  return (double) filetime_to_unix_time (&ft);
 } /* jerry_port_current_time */
 
 #endif /* defined(_WIN32) */

jerry-port/win/jerry-port-win-date.c

@@ -110,8 +110,16 @@ assert (new Date(-1, -1, -1, -1, -1, -1, -1, -1).getMilliseconds() === 999);
 assert (isNaN(new Date(20000000, 0).getFullYear()));
 assert (new Date(0, 0).getFullYear() === 1900);
 assert (new Date(1.2, 0).getFullYear() === 1901);
+
+/* 7. test case */
+/* A Number can exactly represent all integers from -9,007,199,254,740,992 to 9,007,199,254,740,992 (21.1.2.8 and 21.1.2.6).
+   A time value supports a slightly smaller range of -8,640,000,000,000,000 to 8,640,000,000,000,000 milliseconds. */
 assert((new Date(8640000000000000).getFullYear()) == 275760);
 assert(isNaN(new Date(8640000000000001).getFullYear()));
+assert((new Date(-8640000000000000).getFullYear()) == -271821);
+assert(isNaN(new Date(-8640000000000001).getFullYear()));
+
+/* 8. test case */
 assert((new Date(-271821, 3, 21).getFullYear()) == -271821);
 assert(isNaN(new Date(1970, 0, -100000000).getFullYear())); 
 assert(new Date(1970, 0, -100000000 + 1).getFullYear() == -271821);

tests/jerry/date-getters.js

@@ -31,6 +31,9 @@
 
 OUTPUT_DIR = os.path.join(settings.PROJECT_DIR, 'build', 'tests')
 
+# All run_check proc must finished in 30 minutes, may increase in future
+JERRY_CHECK_TIMEOUT = 30 * 60
+
 Options = collections.namedtuple('Options', ['name', 'build_args', 'test_args', 'skip'])
 Options.__new__.__defaults__ = ([], [], False)
 
@@ -197,25 +200,22 @@ def get_arguments():
 
 BINARY_CACHE = {}
 
-TERM_NORMAL = '\033[0m'
-TERM_YELLOW = '\033[1;33m'
-TERM_BLUE = '\033[1;34m'
-TERM_RED = '\033[1;31m'
-
 def report_command(cmd_type, cmd, env=None):
-    sys.stderr.write(f'{TERM_BLUE}{cmd_type}{TERM_NORMAL}\n')
+    sys.stderr.write(f'{util.TERM_BLUE}{cmd_type}{util.TERM_NORMAL}\n')
     if env is not None:
-        sys.stderr.write(''.join(f'{TERM_BLUE}{var}={val!r} \\{TERM_NORMAL}\n'
+        sys.stderr.write(''.join(f'{util.TERM_BLUE}{var}={val!r} \\{util.TERM_NORMAL}\n'
                                  for var, val in sorted(env.items())))
-    sys.stderr.write(f"{TERM_BLUE}" +
-                     f" \\{TERM_NORMAL}\n\t{TERM_BLUE}".join(cmd) +
-                     f"{TERM_NORMAL}\n")
+    sys.stderr.write(f"{util.TERM_BLUE}" +
+                     f" \\{util.TERM_NORMAL}\n\t{util.TERM_BLUE}".join(cmd) +
+                     f"{util.TERM_NORMAL}\n")
+    sys.stderr.flush()
 
 def report_skip(job):
-    sys.stderr.write(f'{TERM_YELLOW}Skipping: {job.name}')
+    sys.stderr.write(f'{util.TERM_YELLOW}Skipping: {job.name}')
     if job.skip:
         sys.stderr.write(f' ({job.skip})')
-    sys.stderr.write(f'{TERM_NORMAL}\n')
+    sys.stderr.write(f'{util.TERM_NORMAL}\n')
+    sys.stderr.flush()
 
 def create_binary(job, options):
     build_args = job.build_args[:]
@@ -246,13 +246,15 @@ def create_binary(job, options):
     if binary_key in BINARY_CACHE:
         ret, build_dir_path = BINARY_CACHE[binary_key]
         sys.stderr.write(f'(skipping: already built at {build_dir_path} with returncode {ret})\n')
+        sys.stderr.flush()
         return ret, build_dir_path
 
     try:
         subprocess.check_output(build_cmd)
         ret = 0
     except subprocess.CalledProcessError as err:
-        print(err.output.decode("utf8"))
+        # For python <-> native program, we use default encoding with error='ignore' to not lost data
+        print(err.output.decode(errors="ignore"))
         ret = err.returncode
 
     BINARY_CACHE[binary_key] = (ret, build_dir_path)
@@ -283,6 +285,7 @@ def iterate_test_runner_jobs(jobs, options):
 
         if build_dir_path in tested_paths:
             sys.stderr.write(f'(skipping: already tested with {build_dir_path})\n')
+            sys.stderr.flush()
             continue
         tested_paths.add(build_dir_path)
 
@@ -291,6 +294,7 @@ def iterate_test_runner_jobs(jobs, options):
 
         if bin_hash in tested_hashes:
             sys.stderr.write(f'(skipping: already tested with equivalent {tested_hashes[bin_hash]})\n')
+            sys.stderr.flush()
             continue
         tested_hashes[bin_hash] = build_dir_path
 
@@ -308,23 +312,23 @@ def run_check(runnable, env=None):
         env = full_env
 
     with subprocess.Popen(runnable, env=env) as proc:
-        proc.wait()
+        proc.wait(timeout=JERRY_CHECK_TIMEOUT)
         return proc.returncode
 
 def run_jerry_debugger_tests(options):
     ret_build = ret_test = 0
     for job in DEBUGGER_TEST_OPTIONS:
         ret_build, build_dir_path = create_binary(job, options)
         if ret_build:
-            print(f"\n{TERM_RED}Build failed{TERM_NORMAL}\n")
+            print(f"\n{util.TERM_RED}Build failed{util.TERM_NORMAL}\n")
             break
 
         for channel in ["websocket", "rawpacket"]:
             for test_file in os.listdir(settings.DEBUGGER_TESTS_DIR):
                 if test_file.endswith(".cmd"):
                     test_case, _ = os.path.splitext(test_file)
                     test_case_path = os.path.join(settings.DEBUGGER_TESTS_DIR, test_case)
-                    test_cmd = [
+                    test_cmd = util.get_python_cmd_prefix() + [
                         settings.DEBUGGER_TEST_RUNNER_SCRIPT,
                         get_binary_path(build_dir_path),
                         channel,
@@ -379,7 +383,7 @@ def run_test262_test_suite(options):
     for job in jobs:
         ret_build, build_dir_path = create_binary(job, options)
         if ret_build:
-            print(f"\n{TERM_RED}Build failed{TERM_NORMAL}\n")
+            print(f"\n{util.TERM_RED}Build failed{util.TERM_NORMAL}\n")
             break
 
         test_cmd = util.get_python_cmd_prefix() + [
@@ -409,7 +413,7 @@ def run_unittests(options):
             continue
         ret_build, build_dir_path = create_binary(job, options)
         if ret_build:
-            print(f"\n{TERM_RED}Build failed{TERM_NORMAL}\n")
+            print(f"\n{util.TERM_RED}Build failed{util.TERM_NORMAL}\n")
             break
 
         if sys.platform == 'win32':
@@ -438,14 +442,15 @@ def run_buildoption_test(options):
 
         ret, _ = create_binary(job, options)
         if ret:
-            print(f"\n{TERM_RED}Build failed{TERM_NORMAL}\n")
+            print(f"\n{util.TERM_RED}Build failed{util.TERM_NORMAL}\n")
             break
 
     return ret
 
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
 def main(options):
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),