Refactor cpu_model support to reduce #if nesting. NFCI

Reviewers: petrhosek, DavidSpickett

Pull Request: llvm#75635

Author: jroelofs

compiler-rt/lib/builtins/CMakeLists.txt

@@ -266,7 +266,7 @@ endif()
 
 # These files are used on 32-bit and 64-bit x86.
 set(x86_ARCH_SOURCES
-  cpu_model.c
+  cpu_model/x86.c
   )
 
 if (NOT MSVC)
@@ -551,7 +551,7 @@ endif()
 set(aarch64_SOURCES
   ${GENERIC_TF_SOURCES}
   ${GENERIC_SOURCES}
-  cpu_model.c
+  cpu_model/aarch64.c
   aarch64/fp_mode.c
 )
 

compiler-rt/lib/builtins/cpu_model/aarch64.c

@@ -0,0 +1,149 @@
+//===-- cpu_model/aarch64.c - Support for __cpu_model builtin  ----*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file is based on LLVM's lib/Support/Host.cpp.
+//  It implements __aarch64_have_lse_atomics, __aarch64_cpu_features for
+//  AArch64.
+//
+//===----------------------------------------------------------------------===//
+
+#include "cpu_model.h"
+
+#if !defined(__aarch64__)
+#error This file is intended only for aarch64-based targets
+#endif
+
+#if __has_include(<sys/ifunc.h>)
+#include <sys/ifunc.h>
+#else
+typedef struct __ifunc_arg_t {
+  unsigned long _size;
+  unsigned long _hwcap;
+  unsigned long _hwcap2;
+} __ifunc_arg_t;
+#endif // __has_include(<sys/ifunc.h>)
+
+// LSE support detection for out-of-line atomics
+// using HWCAP and Auxiliary vector
+_Bool __aarch64_have_lse_atomics
+    __attribute__((visibility("hidden"), nocommon)) = false;
+
+#if defined(__FreeBSD__)
+// clang-format off: should not reorder sys/auxv.h alphabetically
+#include <sys/auxv.h>
+// clang-format on
+#include "aarch64/hwcap.inc"
+#include "aarch64/lse_atomics/freebsd.inc"
+#elif defined(__Fuchsia__)
+#include "aarch64/hwcap.inc"
+#include "aarch64/lse_atomics/fuchsia.inc"
+#elif defined(__ANDROID__)
+#include "aarch64/hwcap.inc"
+#include "aarch64/lse_atomics/android.inc"
+#elif __has_include(<sys/auxv.h>)
+#include "aarch64/hwcap.inc"
+#include "aarch64/lse_atomics/sysauxv.inc"
+#else
+// When unimplemented, we leave __aarch64_have_lse_atomics initialized to false.
+#endif
+
+#if !defined(DISABLE_AARCH64_FMV)
+// CPUFeatures must correspond to the same AArch64 features in
+// AArch64TargetParser.h
+enum CPUFeatures {
+  FEAT_RNG,
+  FEAT_FLAGM,
+  FEAT_FLAGM2,
+  FEAT_FP16FML,
+  FEAT_DOTPROD,
+  FEAT_SM4,
+  FEAT_RDM,
+  FEAT_LSE,
+  FEAT_FP,
+  FEAT_SIMD,
+  FEAT_CRC,
+  FEAT_SHA1,
+  FEAT_SHA2,
+  FEAT_SHA3,
+  FEAT_AES,
+  FEAT_PMULL,
+  FEAT_FP16,
+  FEAT_DIT,
+  FEAT_DPB,
+  FEAT_DPB2,
+  FEAT_JSCVT,
+  FEAT_FCMA,
+  FEAT_RCPC,
+  FEAT_RCPC2,
+  FEAT_FRINTTS,
+  FEAT_DGH,
+  FEAT_I8MM,
+  FEAT_BF16,
+  FEAT_EBF16,
+  FEAT_RPRES,
+  FEAT_SVE,
+  FEAT_SVE_BF16,
+  FEAT_SVE_EBF16,
+  FEAT_SVE_I8MM,
+  FEAT_SVE_F32MM,
+  FEAT_SVE_F64MM,
+  FEAT_SVE2,
+  FEAT_SVE_AES,
+  FEAT_SVE_PMULL128,
+  FEAT_SVE_BITPERM,
+  FEAT_SVE_SHA3,
+  FEAT_SVE_SM4,
+  FEAT_SME,
+  FEAT_MEMTAG,
+  FEAT_MEMTAG2,
+  FEAT_MEMTAG3,
+  FEAT_SB,
+  FEAT_PREDRES,
+  FEAT_SSBS,
+  FEAT_SSBS2,
+  FEAT_BTI,
+  FEAT_LS64,
+  FEAT_LS64_V,
+  FEAT_LS64_ACCDATA,
+  FEAT_WFXT,
+  FEAT_SME_F64,
+  FEAT_SME_I64,
+  FEAT_SME2,
+  FEAT_RCPC3,
+  FEAT_MAX,
+  FEAT_EXT = 62, // Reserved to indicate presence of additional features field
+                 // in __aarch64_cpu_features
+  FEAT_INIT      // Used as flag of features initialization completion
+};
+
+// Architecture features used
+// in Function Multi Versioning
+struct {
+  unsigned long long features;
+  // As features grows new fields could be added
+} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
+
+// The formatter wants to re-order these includes, but doing so is incorrect:
+// clang-format off
+#if defined(__FreeBSD__)
+#include "aarch64/fmv/mrs.inc"
+#include "aarch64/fmv/freebsd.inc"
+#elif defined(__Fuchsia__)
+#include "aarch64/fmv/fuchsia.inc"
+#elif defined(__ANDROID__)
+#include "aarch64/fmv/mrs.inc"
+#include "aarch64/fmv/android.inc"
+#elif __has_include(<sys/auxv.h>)
+#include "aarch64/fmv/mrs.inc"
+#include "aarch64/fmv/sysauxv.inc"
+#else
+#include "aarch64/fmv/unimplemented.inc"
+#endif
+// clang-format on
+
+#endif // !defined(DISABLE_AARCH64_FMV)

compiler-rt/lib/builtins/cpu_model/aarch64/fmv/android.inc

@@ -0,0 +1,36 @@
+void __init_cpu_features_resolver(unsigned long hwcap,
+                                  const __ifunc_arg_t *arg) {
+  if (__aarch64_cpu_features.features)
+    return;
+
+  // ifunc resolvers don't have hwcaps in arguments on Android API lower
+  // than 30. If so, set feature detection done and keep all CPU features
+  // unsupported (zeros). To detect this case in runtime we check existence
+  // of memfd_create function from Standard C library which was introduced in
+  // Android API 30.
+  int memfd_create(const char *, unsigned int) __attribute__((weak));
+  if (!memfd_create)
+    return;
+
+  __init_cpu_features_constructor(hwcap, arg);
+}
+
+void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
+  // CPU features already initialized.
+  if (__aarch64_cpu_features.features)
+    return;
+
+  // Don't set any CPU features,
+  // detection could be wrong on Exynos 9810.
+  if (__isExynos9810())
+    return;
+
+  unsigned long hwcap = getauxval(AT_HWCAP);
+  unsigned long hwcap2 = getauxval(AT_HWCAP2);
+
+  __ifunc_arg_t arg;
+  arg._size = sizeof(__ifunc_arg_t);
+  arg._hwcap = hwcap;
+  arg._hwcap2 = hwcap2;
+  __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg);
+}

compiler-rt/lib/builtins/cpu_model/aarch64/fmv/freebsd.inc

@@ -0,0 +1,27 @@
+void __init_cpu_features_resolver(unsigned long hwcap,
+                                  const __ifunc_arg_t *arg) {
+  if (__aarch64_cpu_features.features)
+    return;
+
+  __init_cpu_features_constructor(hwcap, arg);
+}
+
+void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
+  unsigned long hwcap = 0;
+  unsigned long hwcap2 = 0;
+  // CPU features already initialized.
+  if (__aarch64_cpu_features.features)
+    return;
+
+  int res = 0;
+  res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
+  res |= elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2);
+  if (res)
+    return;
+
+  __ifunc_arg_t arg;
+  arg._size = sizeof(__ifunc_arg_t);
+  arg._hwcap = hwcap;
+  arg._hwcap2 = hwcap2;
+  __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg);
+}

compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc

@@ -0,0 +1,51 @@
+#include <zircon/features.h>
+#include <zircon/syscalls.h>
+
+void __init_cpu_features_resolver() {
+  if (__aarch64_cpu_features.features)
+    return;
+
+    // This ensures the vDSO is a direct link-time dependency of anything that
+    // needs this initializer code.
+#pragma comment(lib, "zircon")
+  uint32_t features;
+  zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
+  if (status != ZX_OK)
+    return;
+
+#define setCPUFeature(cpu_feature)                                             \
+  __aarch64_cpu_features.features |= 1ULL << cpu_feature
+
+  if (features & ZX_ARM64_FEATURE_ISA_FP)
+    setCPUFeature(FEAT_FP);
+  if (features & ZX_ARM64_FEATURE_ISA_ASIMD)
+    setCPUFeature(FEAT_SIMD);
+  if (features & ZX_ARM64_FEATURE_ISA_AES)
+    setCPUFeature(FEAT_AES);
+  if (features & ZX_ARM64_FEATURE_ISA_PMULL)
+    setCPUFeature(FEAT_PMULL);
+  if (features & ZX_ARM64_FEATURE_ISA_SHA1)
+    setCPUFeature(FEAT_SHA1);
+  if (features & ZX_ARM64_FEATURE_ISA_SHA256)
+    setCPUFeature(FEAT_SHA2);
+  if (features & ZX_ARM64_FEATURE_ISA_CRC32)
+    setCPUFeature(FEAT_CRC);
+  if (features & ZX_ARM64_FEATURE_ISA_RDM)
+    setCPUFeature(FEAT_RDM);
+  if (features & ZX_ARM64_FEATURE_ISA_SHA3)
+    setCPUFeature(FEAT_SHA3);
+  if (features & ZX_ARM64_FEATURE_ISA_SM4)
+    setCPUFeature(FEAT_SM4);
+  if (features & ZX_ARM64_FEATURE_ISA_DP)
+    setCPUFeature(FEAT_DOTPROD);
+  if (features & ZX_ARM64_FEATURE_ISA_FHM)
+    setCPUFeature(FEAT_FP16FML);
+  if (features & ZX_ARM64_FEATURE_ISA_SHA512)
+    setCPUFeature(FEAT_SHA3);
+  if (features & ZX_ARM64_FEATURE_ISA_I8MM)
+    setCPUFeature(FEAT_I8MM);
+  if (features & ZX_ARM64_FEATURE_ISA_SVE)
+    setCPUFeature(FEAT_SVE);
+
+  setCPUFeature(FEAT_INIT);
+}