ggml_add_backend_library(ggml-cpu) list (APPEND GGML_CPU_SOURCES ggml-cpu.c ggml-cpu.cpp ggml-cpu-aarch64.c ggml-cpu-aarch64.h ggml-cpu-quants.c ggml-cpu-quants.h amx/amx.cpp amx/amx.h amx/mmq.cpp amx/mmq.h ggml-cpu-impl.h ) target_compile_features(ggml-cpu PRIVATE c_std_11 cxx_std_17) target_include_directories(ggml-cpu PRIVATE .) if (APPLE AND GGML_ACCELERATE) find_library(ACCELERATE_FRAMEWORK Accelerate) if (ACCELERATE_FRAMEWORK) message(STATUS "Accelerate framework found") target_compile_definitions(ggml-cpu PRIVATE GGML_USE_ACCELERATE) target_compile_definitions(ggml-cpu PRIVATE ACCELERATE_NEW_LAPACK) target_compile_definitions(ggml-cpu PRIVATE ACCELERATE_LAPACK_ILP64) target_link_libraries(ggml-cpu PRIVATE ${ACCELERATE_FRAMEWORK}) else() message(WARNING "Accelerate framework not found") endif() endif() if (GGML_OPENMP) find_package(OpenMP) if (OpenMP_FOUND) message(STATUS "OpenMP found") target_compile_definitions(ggml-cpu PRIVATE GGML_USE_OPENMP) target_link_libraries(ggml-cpu PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) else() message(WARNING "OpenMP not found") endif() endif() if (GGML_LLAMAFILE) message(STATUS "Using llamafile") target_compile_definitions(ggml-cpu PRIVATE GGML_USE_LLAMAFILE) list(APPEND GGML_CPU_SOURCES llamafile/sgemm.cpp llamafile/sgemm.h) endif() if (GGML_CPU_HBM) find_library(memkind memkind REQUIRED) message(STATUS "Using memkind for CPU HBM") target_compile_definitions(ggml-cpu PRIVATE GGML_USE_CPU_HBM) target_link_libraries(ggml-cpu PUBLIC memkind) endif() if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) message(STATUS "ARM detected") if (MSVC) list(APPEND ARCH_DEFINITIONS __aarch64__) # MSVC defines _M_ARM64 instead list(APPEND ARCH_DEFINITIONS __ARM_NEON) list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_FMA) set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS}) string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2") check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD) if (GGML_COMPILER_SUPPORT_DOTPROD) list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_DOTPROD) message(STATUS "ARM feature DOTPROD enabled") endif () check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vmmlaq_f32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8) if (GGML_COMPILER_SUPPORT_MATMUL_INT8) list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_MATMUL_INT8) message(STATUS "ARM feature MATMUL_INT8 enabled") endif () check_cxx_source_compiles("#include \nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_FP16_VECTOR_ARITHMETIC) message(STATUS "ARM feature FP16_VECTOR_ARITHMETIC enabled") endif () set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV}) elseif (APPLE) if (GGML_NATIVE) set(USER_PROVIDED_MARCH FALSE) foreach(flag_var IN ITEMS CMAKE_C_FLAGS CMAKE_CXX_FLAGS CMAKE_REQUIRED_FLAGS) if ("${${flag_var}}" MATCHES "-march=[a-zA-Z0-9+._-]+") set(USER_PROVIDED_MARCH TRUE) break() endif() endforeach() if (NOT USER_PROVIDED_MARCH) set(MARCH_FLAGS "-march=armv8.2a") check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD) if (GGML_COMPILER_SUPPORT_DOTPROD) set(MARCH_FLAGS "${MARCH_FLAGS}+dotprod") list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_DOTPROD) message(STATUS "ARM feature DOTPROD enabled") endif () set(TEST_I8MM_FLAGS "-march=armv8.2a+i8mm") set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${TEST_I8MM_FLAGS}") check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8) if (GGML_COMPILER_SUPPORT_MATMUL_INT8) set(MARCH_FLAGS "${MARCH_FLAGS}+i8mm") list(APPEND ARCH_DEFINITIONS __ARM_FEATURE_MATMUL_INT8) message(STATUS "ARM feature MATMUL_INT8 enabled") endif () set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) list(APPEND ARCH_FLAGS "${MARCH_FLAGS}") endif () endif () else() check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E) if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") list(APPEND ARCH_FLAGS -mfp16-format=ieee) endif() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6") # Raspberry Pi 1, Zero list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access) endif() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7") if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") # Android armeabi-v7a list(APPEND ARCH_FLAGS -mfpu=neon-vfpv4 -mno-unaligned-access -funsafe-math-optimizations) else() # Raspberry Pi 2 list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations) endif() endif() if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8") # Android arm64-v8a # Raspberry Pi 3, 4, Zero 2 (32-bit) list(APPEND ARCH_FLAGS -mno-unaligned-access) endif() if (GGML_SVE) list(APPEND ARCH_FLAGS -march=armv8.6-a+sve) endif() endif() elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$")) message(STATUS "x86 detected") if (MSVC) # instruction set detection for MSVC only if (GGML_NATIVE) include(cmake/FindSIMD.cmake) endif () if (GGML_AVX512) list(APPEND ARCH_FLAGS /arch:AVX512) # MSVC has no compile-time flags enabling specific # AVX512 extensions, neither it defines the # macros corresponding to the extensions. # Do it manually. if (GGML_AVX512_VBMI) list(APPEND ARCH_DEFINITIONS __AVX512VBMI__) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") list(APPEND ARCH_FLAGS -mavx512vbmi) endif() endif() if (GGML_AVX512_VNNI) list(APPEND ARCH_DEFINITIONS __AVX512VNNI__) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") list(APPEND ARCH_FLAGS -mavx512vnni) endif() endif() if (GGML_AVX512_BF16) list(APPEND ARCH_DEFINITIONS __AVX512BF16__) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") list(APPEND ARCH_FLAGS -mavx512bf16) endif() endif() if (GGML_AMX_TILE) list(APPEND ARCH_DEFINITIONS __AMX_TILE__) endif() if (GGML_AMX_INT8) list(APPEND ARCH_DEFINITIONS __AMX_INT8__) endif() if (GGML_AMX_BF16) list(APPEND ARCH_DEFINITIONS __AMX_BF16__) endif() elseif (GGML_AVX2) list(APPEND ARCH_FLAGS /arch:AVX2) elseif (GGML_AVX) list(APPEND ARCH_FLAGS /arch:AVX) endif() if (GGML_AVX_VNNI) list(APPEND ARCH_DEFINITIONS __AVXVNNI__) if (CMAKE_C_COMPILER_ID STREQUAL "Clang") list(APPEND ARCH_FLAGS -mavxvnni) endif() endif() else() if (GGML_NATIVE) list(APPEND ARCH_FLAGS -march=native) endif() if (GGML_F16C) list(APPEND ARCH_FLAGS -mf16c) endif() if (GGML_FMA) list(APPEND ARCH_FLAGS -mfma) endif() if (GGML_AVX) list(APPEND ARCH_FLAGS -mavx) endif() if (GGML_AVX2) list(APPEND ARCH_FLAGS -mavx2) endif() if (GGML_AVX_VNNI) list(APPEND ARCH_FLAGS -mavxvnni) endif() if (GGML_AVX512) list(APPEND ARCH_FLAGS -mavx512f) list(APPEND ARCH_FLAGS -mavx512dq) list(APPEND ARCH_FLAGS -mavx512bw) endif() if (GGML_AVX512_VBMI) list(APPEND ARCH_FLAGS -mavx512vbmi) endif() if (GGML_AVX512_VNNI) list(APPEND ARCH_FLAGS -mavx512vnni) endif() if (GGML_AVX512_BF16) list(APPEND ARCH_FLAGS -mavx512bf16) endif() if (GGML_AMX_TILE) list(APPEND ARCH_FLAGS -mamx-tile) endif() if (GGML_AMX_INT8) list(APPEND ARCH_FLAGS -mamx-int8) endif() if (GGML_AMX_BF16) list(APPEND ARCH_FLAGS -mamx-bf16) endif() endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") message(STATUS "PowerPC detected") execute_process(COMMAND bash -c "grep POWER10 /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER10_M) string(FIND "${POWER10_M}" "POWER10" substring_index) if (NOT DEFINED substring_index OR "${substring_index}" STREQUAL "") set(substring_index -1) endif() if (${substring_index} GREATER_EQUAL 0) list(APPEND ARCH_FLAGS -mcpu=power10) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le") list(APPEND ARCH_FLAGS -mcpu=powerpc64le) else() list(APPEND ARCH_FLAGS -mcpu=native -mtune=native) # TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") message(STATUS "loongarch64 detected") list(APPEND ARCH_FLAGS -march=loongarch64) if (GGML_LASX) list(APPEND ARCH_FLAGS -mlasx) endif() if (GGML_LSX) list(APPEND ARCH_FLAGS -mlsx) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") message(STATUS "RISC-V detected") if (GGML_RVV) list(APPEND ARCH_FLAGS -march=rv64gcv -mabi=lp64d) endif() else() message(STATUS "Unknown architecture") endif() if (GGML_CPU_AARCH64) message(STATUS "Using runtime weight conversion of Q4_0 to Q4_0_x_x to enable optimized GEMM/GEMV kernels") target_compile_definitions(ggml-cpu PRIVATE GGML_USE_CPU_AARCH64) endif() target_sources(ggml-cpu PRIVATE ${GGML_CPU_SOURCES}) set_source_files_properties(${GGML_CPU_SOURCES} PROPERTIES COMPILE_OPTIONS "${ARCH_FLAGS}") set_source_files_properties(${GGML_CPU_SOURCES} PROPERTIES COMPILE_DEFINITIONS "${ARCH_DEFINITIONS}") # the feature detection code must be compiled without any architecture flags target_sources(ggml-cpu PRIVATE cpu-feats-x86.cpp) # target_sources(ggml-cpu PRIVATE cpu-feats-arm.cpp) # TODO: ARM feature detection if (EMSCRIPTEN) set_target_properties(ggml-cpu PROPERTIES COMPILE_FLAGS "-msimd128") endif()