start marking Windows-specific build settings [ci skip]

Also add future targets: linux and mac

Author: vszakats

.github/workflows/daily.yml

@@ -11,15 +11,15 @@ on:
 permissions: {}
 
 jobs:
-  build:
+  win:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 8
       - name: 'build'
         env:
-          CW_CONFIG: 'dev'
+          CW_CONFIG: 'dev-win'
           CW_LLVM_MINGW_DL: '1'
           CW_LLVM_MINGW_ONLY: '0'
           CW_MAP: '1'
@@ -43,7 +43,7 @@ jobs:
       # https://github.com/actions/upload-artifact
       - uses: actions/upload-artifact@v3
         with:
-          name: 'binaries'
+          name: 'Windows binaries'
           retention-days: 42
           path: |
             *-*-mingw*.*

README.md

@@ -77,7 +77,7 @@ Features: alt-svc AsynchDNS brotli       HSTS HTTP2 HTTP3 HTTPS-proxy IDN IPv6 K
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
 Features: alt-svc AsynchDNS brotli       HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz MultiSSL NTLM     SPNEGO SSL SSPI threadsafe UnixSockets zstd
 
-"mini", Schannel, without brotli and zstd:
+"mini", without brotli and zstd, with OS TLS backend (Schannel, SecureTransport) if available:
 Protocols: dict file ftp ftps gopher gophers http https imap imaps ldap ldaps mqtt pop3 pop3s rtsp scp sftp smb smbs smtp smtps telnet tftp ws wss
 Features: alt-svc AsynchDNS              HSTS HTTP2       HTTPS-proxy IDN IPv6 Kerberos Largefile libz          NTLM     SPNEGO SSL SSPI threadsafe UnixSockets
 

_build.sh

@@ -51,6 +51,8 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 #        unicode    build curl in UNICODE mode [EXPERIMENTAL]
 #        werror     turn compiler warnings into errors
 #        debug      debug build
+#        mac        build macOS target (requires macOS host)
+#        linux      build Linux target (requires Linux host)
 #
 # CW_JOBS
 #      Number of parallel make jobs. Default: 2
@@ -70,6 +72,9 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 
 # TODO:
 #   - Change default TLS to BoringSSL (with OPENSSL_SMALL?) or LibreSSL?
+#   - Linux: use musl.
+#   - Rename _BRANCH to CW_CONFIG internally.
+#   - Replace .zip with .tar.gz for all-packages artifact (in _ul.sh)?
 #   - Drop x86 builds.
 #       https://data.firefox.com/dashboard/hardware
 #       https://gs.statcounter.com/windows-version-market-share
@@ -186,6 +191,20 @@ else
   _TAR="${_URL_BASE}/archive/refs/heads/${_BRANCH}.tar.gz"
 fi
 
+# Detect host OS
+export _HOSTOS
+case "$(uname)" in
+  *_NT*)   _HOSTOS='win';;
+  Linux*)  _HOSTOS='linux';;
+  Darwin*) _HOSTOS='mac';;
+  *BSD)    _HOSTOS='bsd';;
+  *)       _HOSTOS='unrecognized';;
+esac
+
+export _OS='win'
+[ ! "${_BRANCH#*mac*}" = "${_BRANCH}" ] && _OS='mac'
+[ ! "${_BRANCH#*linux*}" = "${_BRANCH}" ] && _OS='linux'
+
 export _CACERT='cacert.pem'
 
 [ -n "${CW_CCSUFFIX:-}" ] || CW_CCSUFFIX=''
@@ -207,16 +226,6 @@ export _JOBS=2
 my_time='time'
 [ -n "${CW_NOTIME:-}" ] && my_time=
 
-# Detect host OS
-export _HOSTOS
-case "$(uname)" in
-  *_NT*)   _HOSTOS='win';;
-  Linux*)  _HOSTOS='linux';;
-  Darwin*) _HOSTOS='mac';;
-  *BSD)    _HOSTOS='bsd';;
-  *)       _HOSTOS='unrecognized';;
-esac
-
 # Form suffix for alternate builds
 export _FLAV=''
 if [ "${_BRANCH#*pico*}" != "${_BRANCH}" ]; then
@@ -271,7 +280,7 @@ export _REVSUFFIX="${_REV}"; [ -z "${_REVSUFFIX}" ] || _REVSUFFIX="_${_REVSUFFIX
 . ./_dl.sh
 
 # Install required component
-if [ "${_HOSTOS}" = 'mac' ]; then
+if [ "${_OS}" = 'win' ] && [ "${_HOSTOS}" = 'mac' ]; then
   if [ ! -d .venv ]; then
     python3 -m venv .venv
     PIP_PROGRESS_BAR=off .venv/bin/python3 -m pip --disable-pip-version-check --no-cache-dir --require-virtualenv install pefile
@@ -585,7 +594,7 @@ build_single_target() {
       _CC_GLOBAL="${_CC_GLOBAL} --sysroot=${_SYSROOT}"
       _CONFIGURE_GLOBAL="${_CONFIGURE_GLOBAL} --with-sysroot=${_SYSROOT}"
     fi
-    if [ "${_HOSTOS}" = 'linux' ]; then
+    if [ "${_HOSTOS}" = 'linux' ] && [ "${_OS}" = 'win' ]; then
       # We used to pass this via CFLAGS for CMake to make it detect llvm/clang,
       # so we need to pass this via CMAKE_C_FLAGS, though meant for the linker.
       if [ "${_TOOLCHAIN}" = 'llvm-mingw' ]; then
@@ -640,7 +649,7 @@ build_single_target() {
       _BINUTILS_PREFIX='llvm-'
       _BINUTILS_SUFFIX="${_CCSUFFIX}"
       _LDFLAGS_GLOBAL="${_LDFLAGS_GLOBAL} -fuse-ld=lld${_CCSUFFIX}"
-      if [ "${_HOSTOS}" = 'mac' ]; then
+      if [ "${_HOSTOS}" = 'mac' ] && [ "${_OS}" = 'win' ]; then
         _RCFLAGS_GLOBAL="${_RCFLAGS_GLOBAL} -I${_SYSROOT}/${_TRIPLET}/include"
       fi
     fi

_ci-linux-debian.sh

@@ -12,14 +12,14 @@ export CW_CCSUFFIX='-16'
 
 extra=''
 [[ "${CW_CONFIG:-}" = *'boringssl'* ]] && extra="${extra} golang nasm"
+[[ "${CW_CONFIG:-}" = *'win'* ]] && extra="${extra} mingw-w64 osslsigncode wine64"
 
 apt-get --quiet 2 --option Dpkg::Use-Pty=0 update
 # shellcheck disable=SC2086
 apt-get --quiet 2 --option Dpkg::Use-Pty=0 install \
   curl git gpg rsync python3-pefile make cmake \
-  mingw-w64 \
   "llvm${CW_CCSUFFIX}" "clang${CW_CCSUFFIX}" "lld${CW_CCSUFFIX}" \
-  autoconf automake autopoint libtool osslsigncode \
-  zip time jq dos2unix secure-delete wine64 ${extra}
+  autoconf automake autopoint libtool \
+  zip time jq dos2unix secure-delete ${extra}
 
 ./_build.sh

_ci-mac-homebrew.sh

@@ -8,13 +8,14 @@ set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o p
 
 extra=''
 [[ "${CW_CONFIG:-}" = *'boringssl'* ]] && extra="${extra} go nasm"
+[[ "${CW_CONFIG:-}" = *'win'* ]] && extra="${extra} mingw-w64 osslsigncode wine-stable openssh"
 
 export HOMEBREW_NO_AUTO_UPDATE=1
 export HOMEBREW_NO_ANALYTICS=1
 brew update >/dev/null
 # shellcheck disable=SC2086
-brew install coreutils mingw-w64 llvm \
-             dos2unix osslsigncode openssh wine-stable ${extra}
-wineboot --init
+brew install coreutils llvm dos2unix ${extra}
+
+[[ "${CW_CONFIG:-}" = *'win'* ]] && wineboot --init
 
 ./_build.sh

_clean-bin.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Copyright (C) Viktor Szakats. See LICENSE.md
+# SPDX-License-Identifier: MIT
+
+# shellcheck disable=SC3040,SC2039
+set -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o pipefail
+
+if [ "${_OS:-}" = 'win' ]; then
+  "$(dirname "$0")"/_peclean.py "$@"
+fi

_dl.sh

@@ -638,7 +638,8 @@ EOF
 }
 
 # Download llvm-mingw
-if [ "${CW_LLVM_MINGW_DL:-}" = '1' ] && \
+if [ "${_OS}" = 'win' ] && \
+   [ "${CW_LLVM_MINGW_DL:-}" = '1' ] && \
    [ ! -d 'llvm-mingw' ]; then
   name=''; vers=''; hash=''; arch="$(uname -m)"
   if   [ "${_HOSTOS}-${arch}" = 'linux-x86_64' ]; then
@@ -725,13 +726,19 @@ if [ "${_BRANCH#*mbedtls*}" != "${_BRANCH}" ]; then
   live_xt mbedtls "${MBEDTLS_HASH}"
 fi
 
-need_cacert=0
+need_openssl=0
+if [ "${_OS}" != 'win' ]; then
+  need_openssl=1
+elif [ "${_BRANCH#*pico*}" = "${_BRANCH}" ] && \
+     [ "${_BRANCH#*nano*}" = "${_BRANCH}" ] && \
+     [ "${_BRANCH#*micro*}" = "${_BRANCH}" ] && \
+     [ "${_BRANCH#*mini*}" = "${_BRANCH}" ] && \
+     [ "${_BRANCH#*schannel*}" = "${_BRANCH}" ]; then
+  need_openssl=1
+fi
 
-if [ "${_BRANCH#*pico*}" = "${_BRANCH}" ] && \
-   [ "${_BRANCH#*nano*}" = "${_BRANCH}" ] && \
-   [ "${_BRANCH#*micro*}" = "${_BRANCH}" ] && \
-   [ "${_BRANCH#*mini*}" = "${_BRANCH}" ] && \
-   [ "${_BRANCH#*schannel*}" = "${_BRANCH}" ]; then
+need_cacert=0
+if [ "${need_openssl}" = '1' ]; then
   if [ "${_BRANCH#*libressl*}" != "${_BRANCH}" ]; then
     live_dl libressl "${LIBRESSL_VER_}"
     live_xt libressl "${LIBRESSL_HASH}"

_pkg.sh

@@ -20,7 +20,9 @@ if [ "${_NAM}" != "${_UNIPKG}" ]; then
   chmod -R a+rw-s,go-w "${_DST}"
   # NOTE: Not effective on MSYS2:
   find "${_DST}" -name '*.a' -exec chmod a-x '{}' +
-  find "${_DST}" \( -name '*.exe' -o -name '*.dll' \) -exec chmod a+x '{}' +
+  if [ "${_OS}" = 'win' ]; then
+    find "${_DST}" \( -name '*.exe' -o -name '*.dll' \) -exec chmod a+x '{}' +
+  fi
 
   # First, merge this package into the unified package
   unipkg="${_UNIPKG}"

_sign-code.sh

@@ -6,7 +6,11 @@
 # shellcheck disable=SC3040,SC2039
 set -o xtrace -o errexit -o nounset; [ -n "${BASH:-}${ZSH_NAME:-}" ] && set -o pipefail
 
-if [ -s "${SIGN_CODE_KEY}" ] && \
+# TODO: add support for code signing Unixy binaries
+#       E.g. 'codesign' for mac.
+#       Linux: https://stackoverflow.com/questions/1732927/signed-executables-under-linux
+if [ "${_OS}" = 'win' ] && \
+   [ -s "${SIGN_CODE_KEY}" ] && \
    [ -n "${SIGN_CODE_KEY_PASS:+1}" ]; then
 
   _ref="$1"

_ul.sh

@@ -36,6 +36,10 @@ sed 's/-built-on-[^.]*//g' hashes.txt | sort > hashes.txt.all
 touch -r hashes.txt hashes.txt.all
 mv -f hashes.txt.all hashes.txt
 
+if [ "${_OS}" != 'win' ]; then
+  exit 0
+fi
+
 # Create an artifact that includes all packages
 _ALL="all-mingw-${CURL_VER_}${_REVSUFFIX}${_FLAV}.zip"
 {

appveyor.yml

@@ -46,7 +46,7 @@ environment:
   DOCKER_CONTENT_TRUST: '1'
 build_script:
   - sh: |
-      export CW_CONFIG="${APPVEYOR_REPO_BRANCH}"
+      export CW_CONFIG="${APPVEYOR_REPO_BRANCH}-win"
       . ./_versions.sh
       docker trust inspect --pretty "${DOCKER_IMAGE}"
       time docker pull "${DOCKER_IMAGE}"

curl-autotools.sh

@@ -51,17 +51,17 @@ _VER="$1"
 
     if [ "${CW_DEV_LLD_REPRODUCE:-}" = '1' ] && [ "${_LD}" = 'lld' ]; then
       if [ "${pass}" = 'shared' ]; then
-        LDFLAGS="${LDFLAGS} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-dll.tar"
+        LDFLAGS="${LDFLAGS} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-dyn.tar"
       else
-        LDFLAGS="${LDFLAGS} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-exe.tar"
+        LDFLAGS="${LDFLAGS} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-bin.tar"
       fi
     fi
 
     if [ "${pass}" = 'static' ]; then
       LDFLAGS="${LDFLAGS} ${_LDFLAGS_BIN_GLOBAL}"
     fi
 
-    if [ ! "${_BRANCH#*unicode*}" = "${_BRANCH}" ]; then
+    if [ "${_OS}" = 'win' ] && [ "${_BRANCH#*unicode*}" != "${_BRANCH}" ]; then
       CPPFLAGS="${CPPFLAGS} -Dmain=wmain"  # FIXME: upstream. https://github.com/curl/curl/issues/7229
       CPPFLAGS="${CPPFLAGS} -DUNICODE -D_UNICODE"
       LDFLAGS="${LDFLAGS} -municode"
@@ -125,7 +125,9 @@ _VER="$1"
       options="${options} --without-zstd"
     fi
 
-    options="${options} --with-schannel"
+    if [ "${_OS}" = 'win' ]; then
+      options="${options} --with-schannel"
+    fi
     CPPFLAGS="${CPPFLAGS} -DHAS_ALPN"
 
     h3=0
@@ -142,7 +144,9 @@ _VER="$1"
         fi
         h3=1
       elif [ "${_OPENSSL}" = 'quictls' ] || [ "${_OPENSSL}" = 'libressl' ] || [ "${_OPENSSL}" = 'openssl' ]; then
-        LIBS="${LIBS} -lbcrypt"  # for auto-detection
+        if [ "${_OS}" = 'win' ]; then
+          LIBS="${LIBS} -lbcrypt"  # for auto-detection
+        fi
         [ "${_OPENSSL}" = 'openssl' ] || h3=1
       fi
     fi
@@ -180,11 +184,13 @@ _VER="$1"
       options="${options} --with-libssh2=${_TOP}/libssh2/${_PP}"
       options="${options} --without-wolfssh"
       options="${options} --without-libssh"
-      LIBS="${LIBS} -lbcrypt"  # for auto-detection
+      if [ "${_OS}" = 'win' ]; then
+        LIBS="${LIBS} -lbcrypt"  # for auto-detection
 
-      # Workaround for libssh2 1.11.0 regression:
-      # Omit __declspec(dllimport) with libssh2 1.11.0 to link statically
-      [ "${LIBSSH2_VER_}" = '1.11.0' ] && CPPFLAGS="${CPPFLAGS} -DLIBSSH2_API="
+        # Workaround for libssh2 1.11.0 regression:
+        # Omit __declspec(dllimport) with libssh2 1.11.0 to link statically
+        [ "${LIBSSH2_VER_}" = '1.11.0' ] && CPPFLAGS="${CPPFLAGS} -DLIBSSH2_API="
+      fi
     else
       options="${options} --without-wolfssh"
       options="${options} --without-libssh"
@@ -217,7 +223,9 @@ _VER="$1"
       fi
     elif [ "${_BRANCH#*pico*}" = "${_BRANCH}" ]; then
       options="${options} --without-libidn2"
-      options="${options} --with-winidn"
+      if [ "${_OS}" = 'win' ]; then
+        options="${options} --with-winidn"
+      fi
     fi
 
     if [ -d ../cares ]; then
@@ -273,6 +281,10 @@ _VER="$1"
       options="${options} --without-ngtcp2"
     fi
 
+    if [ "${_OS}" = 'win' ]; then
+      options="${options} --enable-sspi"
+    fi
+
     options="${options} --without-quiche --without-msh3"
 
     options="${options} --enable-websockets"
@@ -310,7 +322,6 @@ _VER="$1"
         --enable-libcurl-option \
         --enable-ipv6 \
         --enable-verbose \
-        --enable-sspi \
         --enable-ntlm \
         --enable-cookies \
         --enable-http-auth \

curl-cmake.sh

@@ -31,7 +31,7 @@ _VER="$1"
   LDFLAGS_BIN="${_LDFLAGS_BIN_GLOBAL}"
   LDFLAGS_LIB=''
 
-  if [ ! "${_BRANCH#*unicode*}" = "${_BRANCH}" ]; then
+  if [ "${_OS}" = 'win' ] && [ "${_BRANCH#*unicode*}" != "${_BRANCH}" ]; then
     options="${options} -DENABLE_UNICODE=ON"
   fi
 
@@ -258,7 +258,9 @@ _VER="$1"
     fi
   elif [ "${_BRANCH#*pico*}" = "${_BRANCH}" ]; then
     options="${options} -DUSE_LIBIDN2=OFF"
-    options="${options} -DUSE_WIN32_IDN=ON"
+    if [ "${_OS}" = 'win' ]; then
+      options="${options} -DUSE_WIN32_IDN=ON"
+    fi
   fi
 
   # Official method correctly enables the manual, but with the side-effect
@@ -268,8 +270,8 @@ _VER="$1"
   CPPFLAGS="${CPPFLAGS} -DUSE_MANUAL=1"
 
   if [ "${CW_DEV_LLD_REPRODUCE:-}" = '1' ] && [ "${_LD}" = 'lld' ]; then
-    LDFLAGS_BIN="${LDFLAGS_BIN} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-exe.tar"
-    LDFLAGS_LIB="${LDFLAGS_LIB} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-dll.tar"
+    LDFLAGS_BIN="${LDFLAGS_BIN} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-bin.tar"
+    LDFLAGS_LIB="${LDFLAGS_LIB} -Wl,--reproduce=$(pwd)/$(basename "$0" .sh)-dyn.tar"
   fi
 
   # shellcheck disable=SC2086