[TASK] Migrate from yarn to npm

Yarn has been added to TYPO3 back in 2017 because
npm shrinkwrap files were not stable and package-lock.json
and `npm ci` not yet established.

By now the yarn version in TYPO3 (v1) got outdated
and an update to yarn v3 would be required sooner
or later. The update to yarn v3 brings quite some
problems for the TYPO3 codebase:

 - Yarn v3 is executed/proxied through yarn v1
   (installed on the developer machine) – therefore the
   yarn v3 executable needs to be tracked within git.
   See https://yarnpkg.com/cli/set/version#details

 - Yarn v3 pulls in @types/node (through the "got" package) for
   unknown reasons, causing clashes with TypeScript declarations
   for requirejs (window.require vs node's require())

As package-lock.json is solid thesedays, there is no real
need to depend on yarn anymore. We basically switch
to npm because that switch is way more straightforward than
migrating to yarn v3 and it removes one dependency.

Note that `npm install` properly honors `package-lock.json`
these days (no implicit version upgrades due to the lock file),
but is faster than `npm ci`, therefore it is used in `grunt build`
and runTests.sh for all developer oriented build-targets.
All CI checks use `npm ci` to enfore a clean install.

We set the required npm verson to v8 in order to use the
newest package-lock.json v3 format and .nvmrc is updated to
suggest the recent nodejs v16 LTS release, as npm v8 is
shipped with nodejs v16 by default.

Note that node v14 may still be used to compile packages, but
npm v8 is needed for dependency installation (installable
via `npm install -g npm` in node v14 installations). For that
reason the CI docker images are updated to use nodejs v16
instead of nodejs v14, see following ticket for details:
https://git.typo3.org/typo3/CI/testing-infrastructure/-/issues/3

Executed Commands:

  cd Build/
  # be sure the node_modules folder dir and is updated
  yarn install
  sed -i 's/"yarn": "^1.22.0"/"npm": ">=8.0.0 <9.0.0"/' package.json
  rm -f package-lock.json
  # npm install >= v7 can read yarn.lock and will create
  # package-lock.json as a result
  npm install
  git add package-lock.json
  git rm -f yarn.lock
  npm remove "jquery-ui"
  # The use of `git@` in the HTTPS URLs is a workaround for NPM's
  # auto-rewrite that results in an SSH URL being used instead,
  # which fails to install in CI.
  # See npm/cli#2610 for more information.
  npm install "git+https://[email protected]/jquery/jquery-ui#1.11.4"
  # Fix fsevents not being installed
  npm install rollup@~2.32.0
  # Explicitly pull in popperjs which would be included by npm's
  # resolver in npm >8.5
  npm install @popperjs/core@^2.11.6
  # Let npm build a new, fully clean tree, without old yarn cruft
  rm -rf node_modules/
  npm install
  grunt default

Resolves: #96355
Related: #83038
Releases: main
Change-Id: I88c1ecb1b12dcd117ac4200307af72d330c76d27
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/72642
Tested-by: Benni Mack <[email protected]>
Tested-by: core-ci <[email protected]>
Tested-by: Stefan Bürk <[email protected]>
Tested-by: Benjamin Franzke <[email protected]>
Reviewed-by: Benni Mack <[email protected]>
Reviewed-by: Stefan Bürk <[email protected]>
Reviewed-by: Benjamin Franzke <[email protected]>

Author: bnf

.cache/.npmrc

@@ -0,0 +1,3 @@
+# Required for Docker builds on CI (Gitlab)
+# https://github.com/ds300/patch-package/issues/185
+unsafe-perm = true

.gitignore

@@ -25,7 +25,8 @@ nbproject
 .webprj
 #
 # Temporary files and folders
-.cache
+/.cache/*
+!/.cache/.npmrc
 .php_cs.cache
 .php-cs-fixer.cache
 .sass-cache
@@ -34,6 +35,7 @@ nbproject
 #
 # Ignore build stuff
 /.ddev/*
+/Build/.cache
 /Build/phpunit/FunctionalTests-Job-*
 /Build/bower_components/*
 /Build/node_modules/*

Build/.npmrc

@@ -0,0 +1,2 @@
+lockfile-version=3
+engine-strict=true

Build/.nvmrc

@@ -1 +1 @@
-v16.10
+v16.15

Build/Gruntfile.js

@@ -185,7 +185,7 @@ module.exports = function (grunt) {
     },
     exec: {
       ts: ((process.platform === 'win32') ? 'node_modules\\.bin\\tsc.cmd' : './node_modules/.bin/tsc') + ' --project tsconfig.json',
-      'yarn-install': 'yarn install'
+      'npm-install': 'npm install'
     },
     eslint: {
       options: {
@@ -357,7 +357,7 @@ module.exports = function (grunt) {
             expand: true,
             cwd: '<%= paths.node_modules %>codemirror',
             dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/codemirror',
-            src: ['**/*', '!**/src/**', '!rollup.config.js']
+            src: ['**/*', '!**/src/**', '!rollup.config.js', '!package.json']
           }
         ]
       }
@@ -777,15 +777,6 @@ module.exports = function (grunt) {
   grunt.loadNpmTasks('grunt-newer');
   grunt.loadNpmTasks('grunt-concurrent');
 
-  /**
-   * grunt default task
-   *
-   * call "$ grunt"
-   *
-   * this will trigger the CSS build
-   */
-  grunt.registerTask('default', ['css']);
-
   /**
    * grunt lint
    *
@@ -816,10 +807,9 @@ module.exports = function (grunt) {
    * call "$ grunt update"
    *
    * this task does the following things:
-   * - yarn install
    * - copy some components to a specific destinations because they need to be included via PHP
    */
-  grunt.registerTask('update', ['exec:yarn-install', 'rollup', 'concurrent:npmcopy']);
+  grunt.registerTask('update', ['rollup', 'concurrent:npmcopy']);
 
   /**
    * grunt compile-typescript task
@@ -877,9 +867,9 @@ module.exports = function (grunt) {
   });
 
   /**
-   * grunt build task
+   * grunt default task
    *
-   * call "$ grunt build"
+   * call "$ grunt default"
    *
    * this task does the following things:
    * - execute update task
@@ -889,5 +879,16 @@ module.exports = function (grunt) {
    * - minifies svg files
    * - compiles TypeScript files
    */
-  grunt.registerTask('build', ['clear-build', 'update', 'concurrent:copy_static', 'concurrent:compile_assets', 'concurrent:minify_assets', 'imagemin']);
+  grunt.registerTask('default', ['clear-build', 'update', 'concurrent:copy_static', 'concurrent:compile_assets', 'concurrent:minify_assets', 'imagemin']);
+
+  /**
+   * grunt build task (legacy, for those used to it). Use `grunt default` instead.
+   *
+   * call "$ grunt build"
+   *
+   * this task does the following things:
+   * - execute exec:npm-install task
+   * - execute all task
+   */
+  grunt.registerTask('build', ['exec:npm-install', 'default']);
 };

Build/gitlab-ci.yml

@@ -31,7 +31,7 @@ cache:
   # it is more important that single jobs don't unpack too much every time.
   # The default key is: "Cache everything created by a 'composer install' for main branch.
   # This means jobs using this default key should not create additional stuff in .cache
-  # directory, for instance by calling a 'yarn install' or 'composer min' or similar.
+  # directory, for instance by calling a 'npm ci' or 'composer min' or similar.
   key: main-composer
   paths:
     - .cache