Skip to content

Commit

Permalink
Eti extern marking (kokkos#2292)
Browse files Browse the repository at this point in the history
* ETI: adding headers for template instantiation declarations

* ETI: adding more decl files to eti

Now all the sparse kernels have an ETI decl file to prevent
multiple template instantiation.

* ETI: adding BLAS kernels that are called from Sparse kernels.

* Fixing CMAKE logic

* clang-format

* ETI work: finish the declarations for BLAS and adding GRAPH, LAPACK and BATCHED

* clang-format

* Fixing two headers and rename Nrm2w<2-D> as Nrm2w_mv
  • Loading branch information
lucbv authored Aug 14, 2024
1 parent cf1534e commit a9721ba
Show file tree
Hide file tree
Showing 135 changed files with 2,038 additions and 29 deletions.
77 changes: 77 additions & 0 deletions batched/dense/impl/KokkosBatched_HostLevel_Gemm_Spec.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,72 @@ struct BatchedGemmSpec {
} // namespace Impl
} // namespace KokkosBatched

// ETI instantiation macros, consumed by *.cpp.in files
#define KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, EXEC_SPACE, \
MEM_SPACE) \
extern template struct BatchedGemmSpec<ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, BatchedGemmHandle, SCALAR, \
Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
Kokkos::View<SCALAR ***, LAYOUT, Kokkos::Device<EXEC_SPACE, MEM_SPACE>, \
Kokkos::MemoryTraits<Kokkos::Unmanaged>>, \
false, true>;

#if defined(KOKKOSKERNELS_DECL_LAYOUTRIGHT)
#define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutRight, \
EXEC_SPACE, MEM_SPACE)
#else
#define KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)
#endif

#if defined(KOKKOSKERNELS_DECL_LAYOUTLEFT)
#define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, Kokkos::LayoutLeft, \
EXEC_SPACE, MEM_SPACE)
#else
#define KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)
#endif

///////////////// BatchLayout::Left Permutations /////////////////
#define KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Left, SCALAR, \
LAYOUT, EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Left, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLL_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Left, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

///////////////// BatchLayout::Right Permutations /////////////////
#define KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::NoTranspose, BatchLayout::Right, SCALAR, \
LAYOUT, EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::NoTranspose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::NoTranspose, BatchLayout::Right, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

#define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_DECL(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_DECL_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

// ETI instantiation macros, consumed by *.cpp.in files
#define KOKKOSBATCHED_GEMM_ETI_SPEC_INST_INNER(ARG_TRANS_A, ARG_TRANS_B, ARG_BATCH_LAYOUT, SCALAR, LAYOUT, EXEC_SPACE, \
MEM_SPACE) \
Expand Down Expand Up @@ -230,4 +296,15 @@ struct BatchedGemmSpec {
#define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_INST(SCALAR, LAYOUT, EXEC_SPACE, MEM_SPACE) \
KOKKOSBATCHED_GEMM_BLR_ETI_SPEC_INST_INNER(Trans::Transpose, Trans::Transpose, BatchLayout::Right, SCALAR, LAYOUT, \
EXEC_SPACE, MEM_SPACE)

#include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_bll_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_bll_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_bll_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_bll_eti_spec_avail.hpp>

#include <generated_specializations_hpp/KokkosBatched_Gemm_nt_nt_blr_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_nt_t_blr_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_t_nt_blr_eti_spec_avail.hpp>
#include <generated_specializations_hpp/KokkosBatched_Gemm_t_t_blr_eti_spec_avail.hpp>

#endif // __KOKKOSBATCHED_HOSTLEVEL_GEMM_SPEC_HPP__
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_NT_NT_BLL_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_NT_NT_BLL_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_NT_NT_BLR_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_NT_NT_BLR_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_NT_T_BLL_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_NT_T_BLL_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_NT_T_BLR_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_NT_T_BLR_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_T_NT_BLL_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_T_NT_BLL_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_T_NT_BLR_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_T_NT_BLR_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_T_T_BLL_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_T_T_BLL_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_DECL_HPP_
#define KOKKOSBATCHED_GEMM_T_T_BLR_ETI_SPEC_DECL_HPP_
namespace KokkosBatched {
namespace Impl {
using KokkosBlas::Trans;
@BATCHED_GEMM_T_T_BLR_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBLAS1_ABS_ETI_SPEC_DECL_HPP_
#define KOKKOSBLAS1_ABS_ETI_SPEC_DECL_HPP_
namespace KokkosBlas {
namespace Impl {

@BLAS1_ABS_ETI_DECL_BLOCK@

} // Impl
} // KokkosBlas
#endif // KOKKOSBLAS1_ABS_ETI_SPEC_DECL_HPP_
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBLAS1_ABS_MV_ETI_SPEC_DECL_HPP_
#define KOKKOSBLAS1_ABS_MV_ETI_SPEC_DECL_HPP_
namespace KokkosBlas {
namespace Impl {

@BLAS1_ABS_MV_ETI_DECL_BLOCK@

} // Impl
} // KokkosBlas
#endif // KOKKOSBLAS1_ABS_MV_ETI_SPEC_DECL_HPP_
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@
namespace KokkosBlas {
namespace Impl {
@BLAS1_AXPBY_ETI_AVAIL_BLOCK@
} //IMPL
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//@HEADER
// ************************************************************************
//
// Kokkos v. 4.0
// Copyright (2022) National Technology & Engineering
// Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
// See https://kokkos.org/LICENSE for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//@HEADER

#ifndef KOKKOSBLAS1_AXPBY_ETI_SPEC_DECL_HPP_
#define KOKKOSBLAS1_AXPBY_ETI_SPEC_DECL_HPP_
namespace KokkosBlas {
namespace Impl {
@BLAS1_AXPBY_ETI_DECL_BLOCK@
} //IMPL
} //Kokkos
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@
namespace KokkosBlas {
namespace Impl {
@BLAS1_AXPBY_MV_ETI_AVAIL_BLOCK@
} //IMPL
} //IMPL
} //Kokkos
#endif
Loading

0 comments on commit a9721ba

Please sign in to comment.